///|
/// Offline policy simulation compares redaction configurations on the same
/// fixture corpus before a policy is activated. It reports observed output and
/// finding metrics, allowing operators to review trade-offs without mutating
/// production data.
pub(all) struct PolicyScenario {
scenario_id : String
name : String
description : String
config : RedactionConfig
corpus_label : String
expected_min_findings : Int
expected_max_unresolved : Int
} derive(Debug)
///|
pub(all) struct PolicyObservation {
scenario_id : String
input_length : Int
output_length : Int
finding_count : Int
redaction_count : Int
review_count : Int
critical_count : Int
unresolved_count : Int
changed : Bool
output_checksum : String
warnings : Array[String]
} derive(Debug)
///|
pub(all) struct PolicyComparison {
corpus_label : String
observations : Array[PolicyObservation]
best_scenario : String
stable : Bool
checksum : String
} derive(Debug)
///|
pub fn policy_scenario(
scenario_id : String,
name : String,
description : String,
config : RedactionConfig,
corpus_label : String,
) -> PolicyScenario {
{
scenario_id,
name,
description,
config,
corpus_label,
expected_min_findings: 0,
expected_max_unresolved: 0,
}
}
///|
pub fn PolicyScenario::with_expectations(
scenario : PolicyScenario,
minimum_findings : Int,
maximum_unresolved : Int,
) -> PolicyScenario {
{
..scenario,
expected_min_findings: if minimum_findings < 0 {
0
} else {
minimum_findings
},
expected_max_unresolved: if maximum_unresolved < 0 {
0
} else {
maximum_unresolved
},
}
}
///|
pub fn PolicyScenario::is_valid(self : PolicyScenario) -> Bool {
self.scenario_id.length() > 0 &&
self.name.length() > 0 &&
self.description.length() > 0 &&
self.corpus_label.length() > 0
}
///|
pub fn PolicyScenario::checksum(self : PolicyScenario) -> String {
stable_hash(
self.scenario_id +
":" +
self.name +
":" +
mode_name(self.config.policy.mode) +
":" +
self.corpus_label,
)
}
///|
fn observation_warning(
scenario : PolicyScenario,
findings : Array[Finding],
unresolved : Int,
) -> Array[String] {
let warnings : Array[String] = []
if findings.length() < scenario.expected_min_findings {
warnings.push("finding count below scenario expectation")
}
if unresolved > scenario.expected_max_unresolved {
warnings.push("unresolved findings exceed scenario expectation")
}
if findings.any(fn(finding) { risk_level(finding) == Critical }) {
warnings.push("critical-risk finding observed")
}
warnings
}
///|
pub fn observe_policy(
scenario : PolicyScenario,
input : String,
) -> PolicyObservation raise DeidError {
let result = redact_with_config(input, scenario.config)
let unresolved = result.findings
.filter(fn(finding) {
finding.replacement.is_empty() || finding.replacement == finding.text
})
.length()
let review_count = if scenario.config.policy.action == ReviewOnly {
result.findings.length()
} else {
0
}
{
scenario_id: scenario.scenario_id,
input_length: input.length(),
output_length: result.text.length(),
finding_count: result.findings.length(),
redaction_count: result.findings
.filter(fn(finding) { !finding.replacement.is_empty() })
.length(),
review_count,
critical_count: result.findings
.filter(fn(finding) { risk_level(finding) == Critical })
.length(),
unresolved_count: unresolved,
changed: result.text != input,
output_checksum: stable_hash(result.text),
warnings: observation_warning(scenario, result.findings, unresolved),
}
}
///|
pub fn PolicyObservation::passes(self : PolicyObservation) -> Bool {
self.unresolved_count == 0 &&
self.output_length > 0 &&
self.warnings.is_empty()
}
///|
pub fn PolicyObservation::reduction_percent(self : PolicyObservation) -> Int {
if self.input_length <= 0 || self.output_length >= self.input_length {
0
} else {
(self.input_length - self.output_length) * 100 / self.input_length
}
}
///|
pub fn PolicyObservation::risk_score(self : PolicyObservation) -> Int {
let weighted = self.critical_count * 10 +
self.unresolved_count * 20 +
self.review_count
if weighted > 100 {
100
} else {
weighted
}
}
///|
pub fn PolicyObservation::summary(self : PolicyObservation) -> String {
[
"scenario_id=" + self.scenario_id,
"input_length=" + self.input_length.to_string(),
"output_length=" + self.output_length.to_string(),
"findings=" + self.finding_count.to_string(),
"redactions=" + self.redaction_count.to_string(),
"reviews=" + self.review_count.to_string(),
"critical=" + self.critical_count.to_string(),
"unresolved=" + self.unresolved_count.to_string(),
"reduction_percent=" + self.reduction_percent().to_string(),
"risk_score=" + self.risk_score().to_string(),
"passes=" + self.passes().to_string(),
"output_checksum=" + self.output_checksum,
].join("\n")
}
///|
pub fn PolicyObservation::to_json(self : PolicyObservation) -> String {
"{" +
"\"scenario_id\":\"" +
json_escape(self.scenario_id) +
"\"," +
"\"input_length\":" +
self.input_length.to_string() +
"," +
"\"output_length\":" +
self.output_length.to_string() +
"," +
"\"finding_count\":" +
self.finding_count.to_string() +
"," +
"\"redaction_count\":" +
self.redaction_count.to_string() +
"," +
"\"review_count\":" +
self.review_count.to_string() +
"," +
"\"critical_count\":" +
self.critical_count.to_string() +
"," +
"\"unresolved_count\":" +
self.unresolved_count.to_string() +
"," +
"\"changed\":" +
self.changed.to_string() +
"," +
"\"warnings\":[" +
self.warnings.map(json_escape).join(",") +
"]," +
"\"output_checksum\":\"" +
json_escape(self.output_checksum) +
"\"}"
}
///|
pub fn compare_policies(
scenarios : Array[PolicyScenario],
input : String,
) -> PolicyComparison raise DeidError {
let observations = scenarios.map(scenario => observe_policy(scenario, input))
let mut best = ""
let mut best_score = 1000000
for observation in observations {
let score = observation.risk_score() * 1000 + observation.output_length
if score < best_score {
best_score = score
best = observation.scenario_id
}
}
let stable = observations.all(fn(observation) {
observation.output_checksum.length() > 0 && observation.output_length > 0
})
{
corpus_label: scenarios
.get(0)
.map(fn(item) { item.corpus_label })
.unwrap_or("adhoc"),
observations,
best_scenario: best,
stable,
checksum: stable_hash(
observations.map(fn(item) { item.output_checksum }).join("\n"),
),
}
}
///|
pub fn PolicyComparison::passed_count(self : PolicyComparison) -> Int {
self.observations.filter(PolicyObservation::passes).length()
}
///|
pub fn PolicyComparison::failed_count(self : PolicyComparison) -> Int {
self.observations.length() - self.passed_count()
}
///|
pub fn PolicyComparison::is_actionable(self : PolicyComparison) -> Bool {
!self.observations.is_empty() &&
self.stable &&
self.best_scenario.length() > 0
}
///|
pub fn PolicyComparison::summary(self : PolicyComparison) -> String {
[
"corpus_label=" + self.corpus_label,
"scenarios=" + self.observations.length().to_string(),
"passed=" + self.passed_count().to_string(),
"failed=" + self.failed_count().to_string(),
"best_scenario=" + self.best_scenario,
"stable=" + self.stable.to_string(),
"actionable=" + self.is_actionable().to_string(),
"checksum=" + self.checksum,
].join("\n")
}
///|
pub fn PolicyComparison::to_json(self : PolicyComparison) -> String {
"{" +
"\"corpus_label\":\"" +
json_escape(self.corpus_label) +
"\"," +
"\"observations\":[" +
self.observations.map(PolicyObservation::to_json).join(",") +
"]," +
"\"best_scenario\":\"" +
json_escape(self.best_scenario) +
"\"," +
"\"stable\":" +
self.stable.to_string() +
"," +
"\"checksum\":\"" +
json_escape(self.checksum) +
"\"}"
}
///|
pub fn policy_comparison_is_reproducible(
left : PolicyComparison,
right : PolicyComparison,
) -> Bool {
left.corpus_label == right.corpus_label && left.checksum == right.checksum
}
///|
pub fn policy_scenario_modes(
scenarios : Array[PolicyScenario],
) -> Array[String] {
scenarios.map(fn(scenario) { mode_name(scenario.config.policy.mode) })
}
///|
pub fn policy_simulation_requires_review(comparison : PolicyComparison) -> Bool {
comparison.observations.any(fn(observation) {
observation.critical_count > 0 || !observation.passes()
})
}