///|
/// Query filters over audit findings are used by review screens and export
/// jobs without exposing original document text.
pub(all) struct FindingQuery {
kind : PhiKind?
risk : RiskLevel?
min_confidence : Int
max_confidence : Int
rule_prefix : String
document_span : Span?
source_label : String
} derive(Debug, Eq)
///|
pub(all) struct QueryPage {
items : Array[Finding]
total : Int
offset : Int
limit : Int
has_more : Bool
checksum : String
} derive(Debug)
///|
pub(all) struct AuditSelection {
findings : Array[Finding]
counts : Map[String, Int]
risk_counts : Map[String, Int]
checksum : String
} derive(Debug)
///|
pub fn FindingQuery::default() -> FindingQuery {
{
kind: None,
risk: None,
min_confidence: 0,
max_confidence: 100,
rule_prefix: "",
document_span: None,
source_label: "",
}
}
///|
pub fn query_kind(query : FindingQuery, kind : PhiKind) -> FindingQuery {
{ ..query, kind: Some(kind) }
}
///|
pub fn query_risk(query : FindingQuery, risk : RiskLevel) -> FindingQuery {
{ ..query, risk: Some(risk) }
}
///|
pub fn query_confidence(
query : FindingQuery,
minimum : Int,
maximum : Int,
) -> FindingQuery {
{
..query,
min_confidence: if minimum < 0 {
0
} else {
minimum
},
max_confidence: if maximum > 100 {
100
} else {
maximum
},
}
}
///|
pub fn query_rule_prefix(query : FindingQuery, prefix : String) -> FindingQuery {
{ ..query, rule_prefix: prefix }
}
///|
pub fn query_span(query : FindingQuery, span : Span) -> FindingQuery {
{ ..query, document_span: Some(span) }
}
///|
pub fn query_label(query : FindingQuery, label : String) -> FindingQuery {
{ ..query, source_label: label }
}
///|
pub fn query_matches(query : FindingQuery, finding : Finding) -> Bool {
let kind_ok = match query.kind {
Some(kind) => finding.kind == kind
None => true
}
let risk_ok = match query.risk {
Some(risk) => risk_level(finding) == risk
None => true
}
let span_ok = match query.document_span {
Some(span) => span.overlaps({ start: finding.start, end: finding.end })
None => true
}
kind_ok &&
risk_ok &&
finding.confidence >= query.min_confidence &&
finding.confidence <= query.max_confidence &&
(
query.rule_prefix.is_empty() ||
finding.rule_id.has_prefix(query.rule_prefix)
) &&
(query.source_label.is_empty() || finding.label == query.source_label) &&
span_ok
}
///|
pub fn query_findings(
findings : Array[Finding],
query : FindingQuery,
) -> Array[Finding] {
findings.filter(fn(finding) { query_matches(query, finding) })
}
///|
pub fn query_page(
findings : Array[Finding],
query : FindingQuery,
offset : Int,
limit : Int,
) -> QueryPage {
let selected = query_findings(findings, query)
let start = if offset < 0 {
0
} else if offset > selected.length() {
selected.length()
} else {
offset
}
let safe_limit = if limit <= 0 { selected.length() } else { limit }
let end = if start + safe_limit > selected.length() {
selected.length()
} else {
start + safe_limit
}
let items = selected[start:end].to_owned()
{
items,
total: selected.length(),
offset: start,
limit: safe_limit,
has_more: end < selected.length(),
checksum: stable_hash(selected.map(fn(item) { item.id }).join("\n")),
}
}
///|
pub fn QueryPage::is_empty(self : QueryPage) -> Bool {
self.items.is_empty()
}
///|
pub fn QueryPage::next_offset(self : QueryPage) -> Int {
self.offset + self.items.length()
}
///|
pub fn QueryPage::summary(self : QueryPage) -> String {
[
"items=\{self.items.length()}",
"total=\{self.total}",
"offset=\{self.offset}",
"limit=\{self.limit}",
"has_more=\{self.has_more}",
"checksum=\{self.checksum}",
].join("\n")
}
///|
pub fn query_count_by_kind(findings : Array[Finding]) -> Map[String, Int] {
let counts : Map[String, Int] = Map([])
for finding in findings {
let key = phi_kind_name(finding.kind)
counts[key] = counts.get_or_default(key, 0) + 1
}
counts
}
///|
pub fn query_count_by_rule(findings : Array[Finding]) -> Map[String, Int] {
let counts : Map[String, Int] = Map([])
for finding in findings {
counts[finding.rule_id] = counts.get_or_default(finding.rule_id, 0) + 1
}
counts
}
///|
pub fn query_count_by_label(findings : Array[Finding]) -> Map[String, Int] {
let counts : Map[String, Int] = Map([])
for finding in findings {
counts[finding.label] = counts.get_or_default(finding.label, 0) + 1
}
counts
}
///|
pub fn query_average_confidence(findings : Array[Finding]) -> Float {
if findings.is_empty() {
0.0
} else {
Float::from_int(
findings.fold(init=0, (sum, finding) => sum + finding.confidence),
) /
Float::from_int(findings.length())
}
}
///|
pub fn query_average_span_length(findings : Array[Finding]) -> Float {
if findings.is_empty() {
0.0
} else {
Float::from_int(
findings.fold(init=0, (sum, finding) => sum + finding.end - finding.start),
) /
Float::from_int(findings.length())
}
}
///|
pub fn query_min_confidence(findings : Array[Finding]) -> Int {
match findings.get(0) {
None => 0
Some(first) =>
findings.fold(init=first.confidence, (best, finding) => {
if finding.confidence < best {
finding.confidence
} else {
best
}
})
}
}
///|
pub fn query_max_confidence(findings : Array[Finding]) -> Int {
match findings.get(0) {
None => 0
Some(first) =>
findings.fold(init=first.confidence, (best, finding) => {
if finding.confidence > best {
finding.confidence
} else {
best
}
})
}
}
///|
pub fn select_audit(findings : Array[Finding]) -> AuditSelection {
{
findings,
counts: query_count_by_kind(findings),
risk_counts: risk_summary(findings),
checksum: stable_hash(
findings
.map(fn(item) {
item.id + ":" + item.rule_id + ":\{item.start}:\{item.end}"
})
.join("\n"),
),
}
}
///|
pub fn AuditSelection::size(self : AuditSelection) -> Int {
self.findings.length()
}
///|
pub fn AuditSelection::has_critical(self : AuditSelection) -> Bool {
self.risk_counts.get_or_default("critical", 0) > 0
}
///|
pub fn AuditSelection::summary(self : AuditSelection) -> String {
[
"findings=\{self.size()}",
"average_confidence=\{query_average_confidence(self.findings)}",
"average_span=\{query_average_span_length(self.findings)}",
"critical=\{self.has_critical()}",
"checksum=\{self.checksum}",
].join("\n")
}
///|
pub fn AuditSelection::to_json(self : AuditSelection) -> String {
"{" +
"\"findings\":\{self.findings.length()}," +
"\"counts\":\{map_to_json(self.counts)}," +
"\"risk_counts\":\{map_to_json(self.risk_counts)}," +
"\"average_confidence\":\{query_average_confidence(self.findings)}," +
"\"checksum\":\{json_escape(self.checksum)}" +
"}"
}
///|
pub fn query_findings_in_line(
text : String,
findings : Array[Finding],
line_number : Int,
) -> Array[Finding] {
let lines = split_lines_with_offsets(text)
match lines.get(line_number - 1) {
Some(line) =>
query_findings(
findings,
query_span(FindingQuery::default(), { start: line.start, end: line.end }),
)
None => []
}
}
///|
pub fn query_findings_near(
findings : Array[Finding],
position : Int,
radius : Int,
) -> Array[Finding] {
query_findings(
findings,
query_span(FindingQuery::default(), {
start: position - radius,
end: position + radius,
}),
)
}
///|
pub fn query_findings_for_kind_names(
findings : Array[Finding],
names : Array[String],
) -> Array[Finding] {
findings.filter(fn(item) { names.contains(phi_kind_name(item.kind)) })
}
///|
pub fn query_findings_for_rules(
findings : Array[Finding],
rule_ids : Array[String],
) -> Array[Finding] {
findings.filter(fn(item) { rule_ids.contains(item.rule_id) })
}
///|
pub fn query_findings_sorted_by_confidence(
findings : Array[Finding],
descending : Bool,
) -> Array[Finding] {
let result = findings.copy()
if descending {
result.sort_by(fn(left, right) { right.confidence - left.confidence })
} else {
result.sort_by(fn(left, right) { left.confidence - right.confidence })
}
result
}
///|
pub fn query_findings_sorted_by_risk(
findings : Array[Finding],
) -> Array[Finding] {
let result = findings.copy()
result.sort_by(fn(left, right) {
let l = match risk_level(left) {
Low => 0
Medium => 1
High => 2
Critical => 3
}
let r = match risk_level(right) {
Low => 0
Medium => 1
High => 2
Critical => 3
}
r - l
})
result
}
///|
pub fn query_findings_with_replacements(
findings : Array[Finding],
replacement : String,
) -> Array[Finding] {
findings.filter(fn(item) { item.replacement == replacement })
}
///|
pub fn query_findings_unapplied(
findings : Array[Finding],
offsets : Array[OffsetMap],
) -> Array[Finding] {
findings.filter(fn(item) {
!offsets.any(fn(offset) { offset.finding_id == item.id })
})
}
///|
pub fn query_findings_applied(
findings : Array[Finding],
offsets : Array[OffsetMap],
) -> Array[Finding] {
findings.filter(fn(item) {
offsets.any(fn(offset) { offset.finding_id == item.id })
})
}
///|
pub fn query_text_windows(
text : String,
findings : Array[Finding],
radius : Int,
) -> Array[TextWindow] {
findings.map(fn(item) { text_window(text, item.start, radius) })
}
///|
pub fn query_export_csv(findings : Array[Finding]) -> String {
let lines = ["id,kind,label,start,end,confidence,rule_id"]
for finding in findings {
lines.push(
[
csv_cell(finding.id),
csv_cell(phi_kind_name(finding.kind)),
csv_cell(finding.label),
"\{finding.start}",
"\{finding.end}",
"\{finding.confidence}",
csv_cell(finding.rule_id),
].join(","),
)
}
lines.join("\n")
}
///|
pub fn query_checksum(findings : Array[Finding]) -> String {
select_audit(findings).checksum
}
///|
pub fn query_is_repeatable(
findings : Array[Finding],
query : FindingQuery,
) -> Bool {
query_findings(findings, query) == query_findings(findings, query)
}