///|
pub fn scan_statistics(
input : String,
rules : Array[Rule],
findings : Array[Finding],
) -> ScanStatistics {
let high_risk = findings
.filter(fn(item) { risk_level(item) == High })
.length()
let critical = findings
.filter(fn(item) { risk_level(item) == Critical })
.length()
{
input_length: input.length(),
rule_count: rules.length(),
candidate_count: findings.length(),
accepted_count: findings.length(),
rejected_count: 0,
character_count: input.char_length(),
line_count: line_count(input),
high_risk_count: high_risk,
critical_count: critical,
}
}
///|
pub fn scan_statistics_for(
input : String,
rules? : Array[Rule] = comprehensive_rules(),
) -> ScanStatistics raise DeidError {
let findings = scan(input, rules~)
scan_statistics(input, rules, findings)
}
///|
pub fn redaction_ratio(result : DeidResult) -> Float {
if result.audit.input_length == 0 {
0.0
} else {
Float::from_int(audit_offset_coverage(result.audit)) /
Float::from_int(result.audit.input_length)
}
}
///|
pub fn output_change_ratio(result : DeidResult) -> Float {
if result.audit.input_length == 0 {
0.0
} else {
let delta = result.audit.output_length - result.audit.input_length
Float::from_int(delta) / Float::from_int(result.audit.input_length)
}
}
///|
pub fn kind_coverage(
findings : Array[Finding],
expected : Array[PhiKind],
) -> Float {
if expected.is_empty() {
1.0
} else {
let found = expected
.filter(fn(kind) { findings.any(fn(item) { item.kind == kind }) })
.length()
Float::from_int(found) / Float::from_int(expected.length())
}
}
///|
pub fn average_confidence(findings : Array[Finding]) -> Float {
if findings.is_empty() {
0.0
} else {
Float::from_int(findings.fold(init=0, (sum, item) => sum + item.confidence)) /
Float::from_int(findings.length())
}
}
///|
pub fn minimum_confidence(findings : Array[Finding]) -> Int {
match findings.get(0) {
None => 0
Some(first) =>
findings.fold(init=first.confidence, (current, item) => {
if item.confidence < current {
item.confidence
} else {
current
}
})
}
}
///|
pub fn maximum_confidence(findings : Array[Finding]) -> Int {
findings.fold(init=0, (current, item) => {
if item.confidence > current {
item.confidence
} else {
current
}
})
}
///|
pub fn risk_score(findings : Array[Finding]) -> Int {
findings.fold(init=0, (score, item) => {
let value = match risk_level(item) {
Low => 1
Medium => 3
High => 6
Critical => 10
}
score + value
})
}
///|
pub fn risk_score_per_character(
findings : Array[Finding],
input_length : Int,
) -> Float {
if input_length <= 0 {
0.0
} else {
Float::from_int(risk_score(findings)) / Float::from_int(input_length)
}
}
///|
pub fn confidence_histogram(findings : Array[Finding]) -> Map[String, Int] {
let result : Map[String, Int] = Map([])
for item in findings {
let key = confidence_band_name(confidence_band(item.confidence))
result[key] = result.get_or_default(key, 0) + 1
}
result
}
///|
pub fn length_histogram(findings : Array[Finding]) -> Map[String, Int] {
let result : Map[String, Int] = Map([])
for item in findings {
let bucket = if finding_length(item) <= 4 {
"0-4"
} else if finding_length(item) <= 12 {
"5-12"
} else if finding_length(item) <= 32 {
"13-32"
} else {
"33+"
}
result[bucket] = result.get_or_default(bucket, 0) + 1
}
result
}
///|
pub fn unique_kind_count(findings : Array[Finding]) -> Int {
finding_kind_counts(findings).length()
}
///|
pub fn unique_rule_count(findings : Array[Finding]) -> Int {
finding_rule_counts(findings).length()
}
///|
pub fn average_finding_length(findings : Array[Finding]) -> Float {
if findings.is_empty() {
0.0
} else {
Float::from_int(total_finding_length(findings)) /
Float::from_int(findings.length())
}
}
///|
pub fn unchanged_text(input : String, result : DeidResult) -> String {
let output = StringBuilder()
let mut cursor = 0
for offset in result.offsets {
if offset.original_start > cursor {
output.write_string(input[cursor:offset.original_start].to_owned())
}
cursor = offset.original_end
}
if cursor < input.length() {
output.write_string(input[cursor:].to_owned())
}
output.to_string()
}
///|
pub fn result_integrity_score(input : String, result : DeidResult) -> Float {
let issues = validate_findings(input, result.findings)
let mapping = mapping_from_result(result)
if issues.is_empty() && mapping.is_monotonic() {
1.0
} else {
0.0
}
}
///|
pub fn quality_report(input : String, result : DeidResult) -> String {
[
"finding_count=\{result.findings.length()}",
"risk_score=\{risk_score(result.findings)}",
"redaction_ratio=\{redaction_ratio(result)}",
"output_change_ratio=\{output_change_ratio(result)}",
"average_confidence=\{average_confidence(result.findings)}",
"average_finding_length=\{average_finding_length(result.findings)}",
"integrity=\{result_integrity_score(input, result)}",
].join("\n")
}
///|
pub fn workload_summary(documents : Array[String]) -> String {
let characters = documents.fold(init=0, (total, item) => {
total + item.char_length()
})
let code_units = documents.fold(init=0, (total, item) => total + item.length())
let average_characters : Float = if documents.is_empty() {
0.0
} else {
Float::from_int(characters) / Float::from_int(documents.length())
}
[
"documents=\{documents.length()}",
"characters=\{characters}",
"code_units=\{code_units}",
"average_characters=\{average_characters}",
].join("\n")
}
///|
pub fn compare_results(
left : DeidResult,
right : DeidResult,
) -> Map[String, Bool] {
{
"text": left.text == right.text,
"findings": left.findings == right.findings,
"offsets": left.offsets == right.offsets,
"audit": audit_to_json(left.audit) == audit_to_json(right.audit),
}
}
///|
pub fn stable_result_checksum(result : DeidResult) -> String {
stable_hash(result_to_json(result))
}