///|
/// Locale hints used when selecting a rule pack. A hint never changes the
/// input text; it only narrows the deterministic detector set.
pub(all) enum LocaleHint {
Chinese
English
Mixed
Neutral
} derive(Debug, Eq)
///|
pub impl Show for LocaleHint with fn output(self, logger) {
match self {
Chinese => logger.write_string("zh")
English => logger.write_string("en")
Mixed => logger.write_string("mixed")
Neutral => logger.write_string("neutral")
}
}
///|
/// A bounded half-open interval in the original document.
pub(all) struct Span {
start : Int
end : Int
} derive(Debug, Eq)
///|
pub fn span(start~ : Int, end~ : Int) -> Span {
{ start, end }
}
///|
pub fn Span::length(self : Span) -> Int {
if self.end > self.start {
self.end - self.start
} else {
0
}
}
///|
pub fn Span::is_empty(self : Span) -> Bool {
self.end <= self.start
}
///|
pub fn Span::is_valid(self : Span, text_length : Int) -> Bool {
self.start >= 0 && self.start <= self.end && self.end <= text_length
}
///|
pub fn Span::contains(self : Span, position : Int) -> Bool {
position >= self.start && position < self.end
}
///|
pub fn Span::contains_span(self : Span, other : Span) -> Bool {
other.start >= self.start && other.end <= self.end
}
///|
pub fn Span::overlaps(self : Span, other : Span) -> Bool {
self.start < other.end && other.start < self.end
}
///|
pub fn Span::touches(self : Span, other : Span) -> Bool {
self.end == other.start || other.end == self.start
}
///|
pub fn Span::intersection(self : Span, other : Span) -> Span? {
let start = if self.start > other.start { self.start } else { other.start }
let end = if self.end < other.end { self.end } else { other.end }
if start < end {
Some({ start, end })
} else {
None
}
}
///|
pub fn Span::clamp(self : Span, text_length : Int) -> Span {
let start = if self.start < 0 {
0
} else if self.start > text_length {
text_length
} else {
self.start
}
let end = if self.end < start {
start
} else if self.end > text_length {
text_length
} else {
self.end
}
{ start, end }
}
///|
pub fn Span::shift(self : Span, amount : Int) -> Span {
{ start: self.start + amount, end: self.end + amount }
}
///|
pub enum SpanRelation {
Disjoint
Touching
LeftOverlap
RightOverlap
Contains
ContainedBy
Equal
} derive(Debug, Eq)
///|
pub fn span_relation(left : Span, right : Span) -> SpanRelation {
if left.start == right.start && left.end == right.end {
Equal
} else if left.end == right.start || right.end == left.start {
Touching
} else if left.end <= right.start || right.end <= left.start {
Disjoint
} else if left.contains_span(right) {
Contains
} else if right.contains_span(left) {
ContainedBy
} else if left.start < right.start {
LeftOverlap
} else {
RightOverlap
}
}
///|
pub enum ConfidenceBand {
Low
Medium
High
Certain
} derive(Debug, Eq, Compare)
///|
pub impl Show for ConfidenceBand with fn output(self, logger) {
match self {
Low => logger.write_string("low")
Medium => logger.write_string("medium")
High => logger.write_string("high")
Certain => logger.write_string("certain")
}
}
///|
pub fn confidence_band(value : Int) -> ConfidenceBand {
if value >= 95 {
Certain
} else if value >= 80 {
High
} else if value >= 60 {
Medium
} else {
Low
}
}
///|
pub fn confidence_band_name(band : ConfidenceBand) -> String {
match band {
Low => "low"
Medium => "medium"
High => "high"
Certain => "certain"
}
}
///|
pub enum PolicyAction {
Redact
ReviewOnly
Keep
} derive(Debug, Eq)
///|
pub enum RuleLifecycle {
Stable
Experimental
Deprecated
} derive(Debug, Eq)
///|
pub fn rule_lifecycle_values() -> Array[RuleLifecycle] {
[Stable, Experimental, Deprecated]
}
///|
/// A range that is explicitly protected from automatic replacement.
pub(all) struct ProtectedRange {
start : Int
end : Int
reason : String
} derive(Debug, Eq)
///|
pub fn protected_range(
start~ : Int,
end~ : Int,
reason~ : String,
) -> ProtectedRange {
{ start, end, reason }
}
///|
pub fn ProtectedRange::as_span(self : ProtectedRange) -> Span {
{ start: self.start, end: self.end }
}
///|
/// A reusable selection and masking policy for one application.
pub(all) struct RedactionPolicy {
name : String
locale : LocaleHint
action : PolicyAction
mode : ReplacementMode
min_confidence : Int
protected_ranges : Array[ProtectedRange]
allowed_kinds : Array[PhiKind]
denied_rule_ids : Array[String]
context_window : Int
} derive(Debug, Eq)
///|
pub fn RedactionPolicy::default() -> RedactionPolicy {
{
name: "default-clinical",
locale: Mixed,
action: Redact,
mode: Token,
min_confidence: 60,
protected_ranges: [],
allowed_kinds: [],
denied_rule_ids: [],
context_window: 24,
}
}
///|
pub fn RedactionPolicy::strict() -> RedactionPolicy {
{
name: "strict-clinical",
locale: Mixed,
action: Redact,
mode: PreserveLength,
min_confidence: 80,
protected_ranges: [],
allowed_kinds: [],
denied_rule_ids: [],
context_window: 32,
}
}
///|
pub fn RedactionPolicy::review() -> RedactionPolicy {
{
name: "review-only",
locale: Mixed,
action: ReviewOnly,
mode: Token,
min_confidence: 50,
protected_ranges: [],
allowed_kinds: [],
denied_rule_ids: [],
context_window: 32,
}
}
///|
pub(all) struct RulePack {
name : String
version : String
locale : LocaleHint
description : String
rules : Array[Rule]
lifecycle : RuleLifecycle
} derive(Debug, Eq)
///|
pub fn rule_pack(
name~ : String,
version~ : String,
locale~ : LocaleHint,
description~ : String,
rules~ : Array[Rule],
) -> RulePack {
{ name, version, locale, description, rules, lifecycle: Stable }
}
///|
pub(all) struct PolicyDecision {
finding_id : String
action : PolicyAction
reason : String
} derive(Debug, Eq)
///|
pub(all) struct MappingSegment {
original : Span
redacted : Span
finding_id : String
kind : PhiKind
} derive(Debug, Eq)
///|
pub(all) struct MappingIndex {
original_length : Int
redacted_length : Int
segments : Array[MappingSegment]
} derive(Debug)
///|
pub(all) struct LineColumn {
line : Int
column : Int
} derive(Debug, Eq)
///|
pub(all) struct TextWindow {
start : Int
end : Int
text : String
} derive(Debug, Eq)
///|
pub(all) struct ScanStatistics {
input_length : Int
rule_count : Int
candidate_count : Int
accepted_count : Int
rejected_count : Int
character_count : Int
line_count : Int
high_risk_count : Int
critical_count : Int
} derive(Debug, Eq)
///|
pub(all) struct AuditTotals {
documents : Int
findings : Int
applied : Int
protected_count : Int
by_kind : Map[String, Int]
by_risk : Map[String, Int]
} derive(Debug)
///|
pub(all) struct FieldValue {
key : String
value : String
start : Int
end : Int
separator : String
} derive(Debug, Eq)
///|
pub(all) struct TextToken {
kind : String
text : String
start : Int
end : Int
} derive(Debug, Eq)
///|
pub(all) struct EvaluationCase {
id : String
text : String
expected : Array[Span]
} derive(Debug, Eq)
///|
pub(all) struct EvaluationResult {
cases : Int
expected : Int
predicted : Int
true_positive : Int
false_positive : Int
false_negative : Int
} derive(Debug, Eq)
///|
pub fn EvaluationResult::precision(self : EvaluationResult) -> Float {
if self.true_positive + self.false_positive == 0 {
0.0
} else {
Float::from_int(self.true_positive) /
Float::from_int(self.true_positive + self.false_positive)
}
}
///|
pub fn EvaluationResult::recall(self : EvaluationResult) -> Float {
if self.true_positive + self.false_negative == 0 {
0.0
} else {
Float::from_int(self.true_positive) /
Float::from_int(self.true_positive + self.false_negative)
}
}
///|
pub fn EvaluationResult::f1(self : EvaluationResult) -> Float {
let p = self.precision()
let r = self.recall()
if p + r == 0.0 {
0.0
} else {
2.0 * p * r / (p + r)
}
}
///|
pub fn phi_kind_name(kind : PhiKind) -> String {
"\{kind}"
}
///|
pub fn locale_name(locale : LocaleHint) -> String {
"\{locale}"
}
///|
pub fn action_name(action : PolicyAction) -> String {
match action {
Redact => "redact"
ReviewOnly => "review"
Keep => "keep"
}
}
///|
pub fn rule_lifecycle_name(lifecycle : RuleLifecycle) -> String {
match lifecycle {
Stable => "stable"
Experimental => "experimental"
Deprecated => "deprecated"
}
}