///|
pub(all) suberror DeidError {
DeidError(String)
}
///|
pub impl Show for DeidError with fn output(self, logger) {
match self {
DeidError(msg) => logger.write_string("DeidError: \{msg}")
}
}
///|
fn contextual_value_start(text : String) -> Int {
let mut marker = -1
for i in 0..= 0 {
let mut pos = marker + 1
while pos < text.length() && text[pos] == ' ' {
pos += 1
}
pos
} else {
0
}
}
///|
fn make_replacement(
rule : Rule,
value : String,
mode : ReplacementMode,
) -> String {
match mode {
Token => rule.placeholder
PreserveLength => "*".repeat(value.char_length())
Partial => partial_mask(value, rule.placeholder)
StableHash => rule.placeholder + "#" + stable_hash(value)
}
}
///|
fn partial_mask(value : String, fallback : String) -> String {
let chars = value.to_array()
let n = chars.length()
if n <= 2 {
fallback
} else {
let first = chars[0]
let last = chars[n - 1]
String::make(1, first) + "*".repeat(n - 2) + String::make(1, last)
}
}
///|
fn stable_hash(value : String) -> String {
let mut hash = 2166136261U
for c in value {
hash = hash ^ c.to_int().reinterpret_as_uint()
hash = hash * 16777619U
}
uint_to_hex(hash)
}
///|
fn uint_to_hex(val : UInt) -> String {
let hex_chars = "0123456789abcdef"
let mut res = ""
for i in 0..<8 {
let shift = (7 - i) * 4
let digit = (val >> shift) & 0xFU
res += String::make(
1,
hex_chars[digit.reinterpret_as_int()].to_int().to_char().unwrap(),
)
}
res
}
///|
fn scan_rule(
input : String,
rule : Rule,
mode : ReplacementMode,
) -> Array[Finding] raise DeidError {
let regexp = @regexp.compile(rule.pattern) catch {
err => raise DeidError("Failed to compile rule \{rule.id}: \{err}")
}
let findings = []
let mut suffix = input[:]
let mut serial = 1
while !suffix.is_empty() {
let result = regexp.execute(suffix)
if !result.matched() {
break
}
let matched = match result.get(0) {
Some(v) => v
None => break
}
let raw = matched.to_owned()
let adjust = if rule.contextual { contextual_value_start(raw) } else { 0 }
let start = matched.start_offset() + adjust
let end = matched.start_offset() + matched.length()
if end > start {
let value = input[start:end].to_owned()
findings.push({
id: "\{rule.id}-\{serial}",
kind: rule.kind,
label: rule.label,
start,
end,
text: value,
replacement: make_replacement(rule, value, mode),
rule_id: rule.id,
confidence: rule.confidence,
})
serial += 1
}
let after = result.after()
if after.start_offset() <= suffix.start_offset() {
break
}
suffix = after
}
findings
}
///|
fn overlaps(a : Finding, b : Finding) -> Bool {
a.start < b.end && b.start < a.end
}
///|
fn better_finding(a : Finding, b : Finding) -> Bool {
if a.confidence != b.confidence {
a.confidence > b.confidence
} else {
a.end - a.start >= b.end - b.start
}
}
///|
fn resolve_overlaps(findings : Array[Finding]) -> Array[Finding] {
let sorted = findings.copy()
sorted.sort_by(fn(a, b) {
if a.start != b.start {
a.start - b.start
} else {
b.confidence - a.confidence
}
})
let result = []
for f in sorted {
let mut blocked = false
for i in 0.. Array[Finding] raise DeidError {
let raw = []
for rule in rules {
if rule.enabled || options.include_disabled_rules {
let found = scan_rule(input, rule, options.mode)
for item in found {
if item.confidence >= options.min_confidence {
raw.push(item)
}
}
}
}
resolve_overlaps(raw)
}
///|
pub fn merge_external_entities(
input : String,
findings : Array[Finding],
entities : Array[ExternalEntity],
mode? : ReplacementMode = Token,
) -> Array[Finding] {
let merged = findings.copy()
let mut serial = 1
for entity in entities {
if entity.start >= 0 &&
entity.end <= input.length() &&
entity.end > entity.start {
let text = input[entity.start:entity.end].to_owned()
let placeholder = "[\{entity.source.to_upper()}]"
merged.push({
id: "external-\{serial}",
kind: entity.kind,
label: entity.label,
start: entity.start,
end: entity.end,
text,
replacement: make_replacement(
{
id: entity.source,
label: entity.label,
kind: entity.kind,
pattern: "",
placeholder,
confidence: entity.confidence,
enabled: true,
contextual: false,
},
text,
mode,
),
rule_id: entity.source,
confidence: entity.confidence,
})
serial += 1
}
}
resolve_overlaps(merged)
}