///|
/// A single quality-control probe run against a de-identified output.
pub(all) struct QualityProbe {
id : String
description : String
passed : Bool
severity : DiagnosticSeverity
evidence : String
} derive(Debug, Eq)
///|
pub(all) struct QualityThresholds {
max_findings : Int
max_critical : Int
min_recall_percent : Int
min_output_reduction_percent : Int
require_determinism : Bool
require_non_sensitive_stability : Bool
} derive(Debug, Eq)
///|
pub(all) struct QualityReport {
input_length : Int
output_length : Int
probes : Array[QualityProbe]
passed : Bool
score : Int
checksum : String
} derive(Debug)
///|
pub fn QualityThresholds::default() -> QualityThresholds {
{
max_findings: 100000,
max_critical: 0,
min_recall_percent: 0,
min_output_reduction_percent: 0,
require_determinism: true,
require_non_sensitive_stability: true,
}
}
///|
pub fn QualityThresholds::strict() -> QualityThresholds {
{
max_findings: 10000,
max_critical: 0,
min_recall_percent: 80,
min_output_reduction_percent: 1,
require_determinism: true,
require_non_sensitive_stability: true,
}
}
///|
pub fn quality_probe(
id : String,
description : String,
passed : Bool,
severity : DiagnosticSeverity,
evidence : String,
) -> QualityProbe {
{ id, description, passed, severity, evidence }
}
///|
pub fn quality_probe_status(probe : QualityProbe) -> String {
if probe.passed {
"pass"
} else {
"fail"
}
}
///|
pub fn quality_probe_json(probe : QualityProbe) -> String {
"{" +
"\"id\":\{json_escape(probe.id)}," +
"\"description\":\{json_escape(probe.description)}," +
"\"passed\":\{probe.passed}," +
"\"severity\":\{json_escape(severity_name(probe.severity))}," +
"\"evidence\":\{json_escape(probe.evidence)}" +
"}"
}
///|
pub fn QualityReport::failed_probes(
self : QualityReport,
) -> Array[QualityProbe] {
self.probes.filter(fn(probe) { !probe.passed })
}
///|
pub fn QualityReport::critical_failures(
self : QualityReport,
) -> Array[QualityProbe] {
self.failed_probes().filter(fn(probe) { probe.severity == Error })
}
///|
pub fn QualityReport::is_empty(self : QualityReport) -> Bool {
self.probes.is_empty()
}
///|
pub fn QualityReport::summary(self : QualityReport) -> String {
[
"input_length=\{self.input_length}",
"output_length=\{self.output_length}",
"probes=\{self.probes.length()}",
"failed=\{self.failed_probes().length()}",
"score=\{self.score}",
"passed=\{self.passed}",
"checksum=\{self.checksum}",
].join("\n")
}
///|
pub fn quality_output_reduction_percent(
input_length : Int,
output : String,
) -> Int {
if input_length == 0 {
0
} else {
(input_length - output.length()) * 100 / input_length
}
}
///|
pub fn quality_findings_by_risk(
findings : Array[Finding],
risk : RiskLevel,
) -> Int {
findings.filter(fn(item) { risk_level(item) == risk }).length()
}
///|
pub fn quality_findings_by_kind(
findings : Array[Finding],
kind : PhiKind,
) -> Int {
findings.filter(fn(item) { item.kind == kind }).length()
}
///|
pub fn quality_distinct_rule_count(findings : Array[Finding]) -> Int {
let seen : Map[String, Unit] = Map([])
for finding in findings {
seen[finding.rule_id] = ()
}
seen.length()
}
///|
pub fn quality_findings_are_sorted(findings : Array[Finding]) -> Bool {
let mut sorted = true
for i in 1.. findings[i].start {
sorted = false
}
}
sorted
}
///|
pub fn quality_findings_are_non_overlapping(findings : Array[Finding]) -> Bool {
let mut valid = true
for i in 1.. findings[i].start {
valid = false
}
}
valid
}
///|
pub fn quality_finding_spans_valid(
input : String,
findings : Array[Finding],
) -> Bool {
findings.all(fn(item) {
item.start >= 0 &&
item.end > item.start &&
item.end <= input.length() &&
input[item.start:item.end] == item.text
})
}
///|
pub fn quality_offsets_valid(
input : String,
output : String,
offsets : Array[OffsetMap],
) -> Bool {
offsets.all(fn(item) {
item.original_start >= 0 &&
item.original_end > item.original_start &&
item.original_end <= input.length() &&
item.replacement_start >= 0 &&
item.replacement_end >= item.replacement_start &&
item.replacement_end <= output.length()
})
}
///|
pub fn quality_no_original_phi(
output : String,
findings : Array[Finding],
) -> Bool {
!findings.any(fn(item) {
item.text.length() > 0 && output.contains(item.text)
})
}
///|
pub fn quality_replacement_nonempty(findings : Array[Finding]) -> Bool {
findings.all(fn(item) { item.replacement.length() > 0 })
}
///|
pub fn quality_replacement_changes(findings : Array[Finding]) -> Bool {
findings.all(fn(item) { item.text != item.replacement })
}
///|
pub fn quality_id_unique(findings : Array[Finding]) -> Bool {
let seen : Map[String, Unit] = Map([])
let mut unique = true
for item in findings {
if seen.contains(item.id) {
unique = false
} else {
seen[item.id] = ()
}
}
unique
}
///|
pub fn quality_kind_coverage(
findings : Array[Finding],
kinds : Array[PhiKind],
) -> Int {
let mut covered = 0
for kind in kinds {
if findings.any(fn(item) { item.kind == kind }) {
covered += 1
}
}
covered
}
///|
pub fn quality_check_core_contract(
input : String,
result : DeidResult,
) -> Array[QualityProbe] {
[
quality_probe(
"spans",
"finding spans refer to the original input",
quality_finding_spans_valid(input, result.findings),
Error,
"\{result.findings.length()} findings",
),
quality_probe(
"sorted",
"findings are sorted and non-overlapping",
quality_findings_are_sorted(result.findings) &&
quality_findings_are_non_overlapping(result.findings),
Error,
"ordered=\{quality_findings_are_sorted(result.findings)}",
),
quality_probe(
"offsets",
"offset maps remain within both documents",
quality_offsets_valid(input, result.text, result.offsets),
Error,
"\{result.offsets.length()} offsets",
),
quality_probe(
"replacements",
"all selected findings have non-empty replacements",
quality_replacement_nonempty(result.findings),
Error,
"findings=\{result.findings.length()}",
),
quality_probe(
"identifiers",
"finding identifiers are unique",
quality_id_unique(result.findings),
Error,
"ids=\{result.findings.length()}",
),
quality_probe(
"changed",
"each selected value is transformed",
quality_replacement_changes(result.findings),
Warning,
"replacement comparison",
),
]
}
///|
pub fn quality_check_determinism(
input : String,
config : RedactionConfig,
) -> QualityProbe raise DeidError {
let first = redact_with_config(input, config)
let second = redact_with_config(input, config)
quality_probe(
"determinism",
"same input and policy produce the same output",
first.text == second.text && first.audit.counts == second.audit.counts,
Error,
"first=\{stable_hash(first.text)} second=\{stable_hash(second.text)}",
)
}
///|
pub fn quality_check_non_phi_stability(
input : String,
config : RedactionConfig,
) -> QualityProbe raise DeidError {
let result = redact_with_config(input, config)
let mut stable = true
let mut cursor = 0
for finding in result.findings {
if input[cursor:finding.start] != result.text[cursor:finding.start] {
stable = false
}
cursor = finding.end
}
if input[cursor:] != result.text[cursor:] {
stable = false
}
quality_probe(
"non_phi_stability",
"text outside selected spans remains stable",
stable,
Warning,
"findings=\{result.findings.length()}",
)
}
///|
pub fn quality_check_leakage(result : DeidResult) -> QualityProbe {
quality_probe(
"leakage",
"original detected values do not remain in output",
quality_no_original_phi(result.text, result.findings),
Error,
"output_length=\{result.text.length()}",
)
}
///|
pub fn quality_check_limits(
result : DeidResult,
thresholds : QualityThresholds,
) -> Array[QualityProbe] {
[
quality_probe(
"finding_limit",
"finding volume remains within configured limit",
result.findings.length() <= thresholds.max_findings,
Error,
"findings=\{result.findings.length()} limit=\{thresholds.max_findings}",
),
quality_probe(
"critical_limit",
"critical finding volume remains within configured limit",
quality_findings_by_risk(result.findings, Critical) <=
thresholds.max_critical,
Error,
"critical=\{quality_findings_by_risk(result.findings, Critical)}",
),
quality_probe(
"reduction",
"output reduction meets the configured minimum",
quality_output_reduction_percent(result.audit.input_length, result.text) >=
thresholds.min_output_reduction_percent,
Warning,
"reduction=\{quality_output_reduction_percent(result.audit.input_length, result.text)}%",
),
]
}
///|
pub fn quality_score(probes : Array[QualityProbe]) -> Int {
if probes.is_empty() {
100
} else {
let passed = probes.filter(fn(probe) { probe.passed }).length()
passed * 100 / probes.length()
}
}
///|
pub fn quality_passes(
probes : Array[QualityProbe],
thresholds : QualityThresholds,
) -> Bool {
!probes.any(fn(probe) { !probe.passed && probe.severity == Error }) &&
(
thresholds.require_determinism ||
!probes.any(fn(probe) { probe.id == "determinism" && !probe.passed })
) &&
(
thresholds.require_non_sensitive_stability ||
!probes.any(fn(probe) { probe.id == "non_phi_stability" && !probe.passed })
)
}
///|
pub fn run_quality_gate(
input : String,
result : DeidResult,
config : RedactionConfig,
thresholds : QualityThresholds,
) -> QualityReport raise DeidError {
let probes = quality_check_core_contract(input, result)
probes.push(quality_check_leakage(result))
if thresholds.require_determinism {
probes.push(quality_check_determinism(input, config))
}
if thresholds.require_non_sensitive_stability {
probes.push(quality_check_non_phi_stability(input, config))
}
probes.append(quality_check_limits(result, thresholds))
let score = quality_score(probes)
{
input_length: input.length(),
output_length: result.text.length(),
probes,
passed: quality_passes(probes, thresholds),
score,
checksum: stable_hash(
probes
.map(fn(probe) {
probe.id + ":" + "\{probe.passed}" + ":" + probe.evidence
})
.join("\n"),
),
}
}
///|
pub fn quality_report_json(report : QualityReport) -> String {
"{" +
"\"input_length\":\{report.input_length}," +
"\"output_length\":\{report.output_length}," +
"\"passed\":\{report.passed}," +
"\"score\":\{report.score}," +
"\"checksum\":\{json_escape(report.checksum)}," +
"\"probes\":[" +
report.probes.map(quality_probe_json).join(",") +
"]}"
}
///|
pub fn quality_report_markdown(report : QualityReport) -> String {
let lines = [
"| Probe | Status | Severity | Evidence |", "| --- | --- | --- | --- |",
]
for probe in report.probes {
lines.push(
"| \{probe.id} | \{quality_probe_status(probe)} | \{severity_name(probe.severity)} | \{probe.evidence} |",
)
}
lines.push("")
lines.push("score=\{report.score}; passed=\{report.passed}")
lines.join("\n")
}
///|
pub fn quality_failed_ids(report : QualityReport) -> Array[String] {
report.failed_probes().map(fn(probe) { probe.id })
}
///|
pub fn quality_error_count(report : QualityReport) -> Int {
report.failed_probes().filter(fn(probe) { probe.severity == Error }).length()
}
///|
pub fn quality_warning_count(report : QualityReport) -> Int {
report
.failed_probes()
.filter(fn(probe) { probe.severity == Warning })
.length()
}
///|
pub fn quality_report_checksum(report : QualityReport) -> String {
stable_hash(quality_report_json(report))
}
///|
pub fn quality_gate_ready(report : QualityReport) -> Bool {
report.passed && quality_error_count(report) == 0
}
///|
pub fn quality_gate_message(report : QualityReport) -> String {
if report.passed {
"quality gate passed with score \{report.score}"
} else {
let ids = quality_failed_ids(report).join(",")
"quality gate failed: \{ids}"
}
}