///|
/// A plan makes selection and execution separate, so policy changes can be
/// reviewed before any text is transformed.
pub(all) enum PlanAction {
PlanApply
PlanKeep
PlanReview
PlanDrop
} derive(Debug, Eq)
///|
pub(all) struct PlanStep {
finding_id : String
action : PlanAction
replacement : String
reason : String
order : Int
} derive(Debug, Eq)
///|
pub(all) struct RedactionPlan {
input_checksum : String
policy_name : String
steps : Array[PlanStep]
warnings : Array[String]
checksum : String
} derive(Debug, Eq)
///|
pub(all) struct PlanExecution {
text : String
applied : Array[Finding]
kept : Array[Finding]
reviewed : Array[Finding]
dropped : Array[Finding]
offsets : Array[OffsetMap]
checksum : String
} derive(Debug)
///|
pub fn plan_action_name(action : PlanAction) -> String {
match action {
PlanApply => "apply"
PlanKeep => "keep"
PlanReview => "review"
PlanDrop => "drop"
}
}
///|
pub fn plan_action_for(
finding : Finding,
policy : RedactionPolicy,
) -> PlanAction {
if policy_protects(policy, finding) ||
policy_blocks_rule(policy, finding.rule_id) {
PlanKeep
} else if !policy_allows_kind(policy, finding.kind) {
PlanKeep
} else if policy.action == ReviewOnly {
PlanReview
} else if policy.action == Keep {
PlanKeep
} else {
PlanApply
}
}
///|
pub fn plan_step(
finding : Finding,
policy : RedactionPolicy,
order : Int,
) -> PlanStep {
let action = plan_action_for(finding, policy)
let reason = match action {
PlanApply => "policy selected"
PlanKeep => "protected or denied"
PlanReview => "review-only policy"
PlanDrop => "explicitly dropped"
}
{
finding_id: finding.id,
action,
replacement: finding.replacement,
reason,
order,
}
}
///|
pub fn build_redaction_plan(
input : String,
findings : Array[Finding],
policy : RedactionPolicy,
) -> RedactionPlan {
let steps = []
for i in 0.. PlanStep? {
let mut result : PlanStep? = None
for step in self.steps {
if step.finding_id == finding_id {
result = Some(step)
}
}
result
}
///|
pub fn RedactionPlan::action_counts(self : RedactionPlan) -> Map[String, Int] {
let counts : Map[String, Int] = Map([])
for step in self.steps {
let key = plan_action_name(step.action)
counts[key] = counts.get_or_default(key, 0) + 1
}
counts
}
///|
pub fn RedactionPlan::applied_count(self : RedactionPlan) -> Int {
self.steps.filter(fn(step) { step.action == PlanApply }).length()
}
///|
pub fn RedactionPlan::review_count(self : RedactionPlan) -> Int {
self.steps.filter(fn(step) { step.action == PlanReview }).length()
}
///|
pub fn RedactionPlan::kept_count(self : RedactionPlan) -> Int {
self.steps.filter(fn(step) { step.action == PlanKeep }).length()
}
///|
pub fn RedactionPlan::is_ordered(self : RedactionPlan) -> Bool {
let mut ordered = true
for i in 1..= self.steps[i].order {
ordered = false
}
}
ordered
}
///|
pub fn RedactionPlan::is_safe(self : RedactionPlan) -> Bool {
self.input_checksum.length() > 0 &&
self.checksum.length() > 0 &&
self.is_ordered() &&
self.warnings.is_empty()
}
///|
pub fn RedactionPlan::summary(self : RedactionPlan) -> String {
[
"policy=\{self.policy_name}",
"steps=\{self.steps.length()}",
"apply=\{self.applied_count()}",
"review=\{self.review_count()}",
"keep=\{self.kept_count()}",
"ordered=\{self.is_ordered()}",
"safe=\{self.is_safe()}",
"checksum=\{self.checksum}",
].join("\n")
}
///|
pub fn plan_execution(
input : String,
findings : Array[Finding],
plan : RedactionPlan,
) -> PlanExecution {
let applied = []
let kept = []
let reviewed = []
let dropped = []
let executable = []
for finding in findings {
match plan.step_for(finding.id) {
Some(step) =>
match step.action {
PlanApply => {
applied.push({ ..finding, replacement: step.replacement })
executable.push({ ..finding, replacement: step.replacement })
}
PlanKeep => kept.push(finding)
PlanReview => reviewed.push(finding)
PlanDrop => dropped.push(finding)
}
None => kept.push(finding)
}
}
let (text, offsets) = apply_findings(input, executable)
{
text,
applied,
kept,
reviewed,
dropped,
offsets,
checksum: stable_hash(text),
}
}
///|
pub fn PlanExecution::applied_count(self : PlanExecution) -> Int {
self.applied.length()
}
///|
pub fn PlanExecution::review_count(self : PlanExecution) -> Int {
self.reviewed.length()
}
///|
pub fn PlanExecution::kept_count(self : PlanExecution) -> Int {
self.kept.length()
}
///|
pub fn PlanExecution::has_review(self : PlanExecution) -> Bool {
!self.reviewed.is_empty()
}
///|
pub fn PlanExecution::release_ready(self : PlanExecution) -> Bool {
self.reviewed.is_empty() && self.dropped.is_empty()
}
///|
pub fn PlanExecution::summary(self : PlanExecution) -> String {
[
"output_length=\{self.text.length()}",
"applied=\{self.applied_count()}",
"kept=\{self.kept_count()}",
"review=\{self.review_count()}",
"dropped=\{self.dropped.length()}",
"release_ready=\{self.release_ready()}",
"checksum=\{self.checksum}",
].join("\n")
}
///|
pub fn PlanExecution::to_json(self : PlanExecution) -> String {
"{" +
"\"text_checksum\":\{json_escape(self.checksum)}," +
"\"text_length\":\{self.text.length()}," +
"\"applied\":\{self.applied.length()}," +
"\"kept\":\{self.kept.length()}," +
"\"reviewed\":\{self.reviewed.length()}," +
"\"dropped\":\{self.dropped.length()}," +
"\"release_ready\":\{self.release_ready()}" +
"}"
}
///|
pub fn plan_findings_by_action(
findings : Array[Finding],
plan : RedactionPlan,
action : PlanAction,
) -> Array[Finding] {
findings.filter(fn(finding) {
match plan.step_for(finding.id) {
Some(step) => step.action == action
None => false
}
})
}
///|
pub fn plan_replacement_map(
findings : Array[Finding],
plan : RedactionPlan,
) -> Map[String, String] {
let result : Map[String, String] = Map([])
for finding in findings {
match plan.step_for(finding.id) {
Some(step) if step.action == PlanApply =>
result[finding.id] = step.replacement
_ => ()
}
}
result
}
///|
pub fn plan_has_conflicting_steps(plan : RedactionPlan) -> Bool {
let seen : Map[String, PlanAction] = Map([])
let mut conflict = false
for step in plan.steps {
match seen.get(step.finding_id) {
Some(action) if action != step.action => conflict = true
_ => seen[step.finding_id] = step.action
}
}
conflict
}
///|
pub fn plan_validate(
input : String,
findings : Array[Finding],
plan : RedactionPlan,
) -> Array[String] {
let issues = []
if plan.input_checksum != stable_hash(input) {
issues.push("input checksum mismatch")
}
if plan.steps.length() != findings.length() {
issues.push("step count does not match finding count")
}
if plan_has_conflicting_steps(plan) {
issues.push("finding has conflicting actions")
}
if !quality_finding_spans_valid(input, findings) {
issues.push("finding span is invalid")
}
issues.append(plan.warnings)
issues
}
///|
pub fn plan_round_trip(
input : String,
findings : Array[Finding],
policy : RedactionPolicy,
) -> PlanExecution {
let plan = build_redaction_plan(input, findings, policy)
plan_execution(input, findings, plan)
}
///|
pub fn plan_for_text(
input : String,
policy : RedactionPolicy,
) -> (RedactionPlan, PlanExecution) raise DeidError {
let findings = scan(input, rules=comprehensive_rules(), options={
mode: policy.mode,
min_confidence: policy.min_confidence,
include_disabled_rules: false,
})
let plan = build_redaction_plan(input, findings, policy)
(plan, plan_execution(input, findings, plan))
}
///|
pub fn plan_markdown(plan : RedactionPlan) -> String {
let lines = [
"| Finding | Action | Order | Reason |", "| --- | --- | ---: | --- |",
]
for step in plan.steps {
lines.push(
"| \{step.finding_id} | \{plan_action_name(step.action)} | \{step.order} | \{step.reason} |",
)
}
lines.join("\n")
}
///|
pub fn plan_json(plan : RedactionPlan) -> String {
let steps = plan.steps
.map(fn(step) {
"{" +
"\"finding_id\":\{json_escape(step.finding_id)}," +
"\"action\":\{json_escape(plan_action_name(step.action))}," +
"\"replacement\":\{json_escape(step.replacement)}," +
"\"reason\":\{json_escape(step.reason)}," +
"\"order\":\{step.order}" +
"}"
})
.join(",")
"{" +
"\"input_checksum\":\{json_escape(plan.input_checksum)}," +
"\"policy\":\{json_escape(plan.policy_name)}," +
"\"checksum\":\{json_escape(plan.checksum)}," +
"\"warnings\":[" +
plan.warnings.map(json_escape).join(",") +
"]," +
"\"steps\":[" +
steps +
"]}"
}
///|
pub fn plan_checksum_matches(
input : String,
findings : Array[Finding],
policy : RedactionPolicy,
plan : RedactionPlan,
) -> Bool {
let expected = build_redaction_plan(input, findings, policy)
expected.checksum == plan.checksum
}
///|
pub fn plan_apply_count(
input : String,
policy : RedactionPolicy,
) -> Int raise DeidError {
let (plan, _) = plan_for_text(input, policy)
plan.applied_count()
}