///|
/// A composable query for extracting matches from an automaton.
pub(all) struct MatchQuery {
min_weight : Int
require_label : String
prefer_longest : Bool
include_overlaps : Bool
max_matches : Int
} derive(Debug, Eq, ToJson)
///|
/// Return a query with permissive defaults.
pub fn MatchQuery::default() -> MatchQuery {
{
min_weight: -1000000,
require_label: "",
prefer_longest: true,
include_overlaps: true,
max_matches: 1000000,
}
}
///|
/// Return a query specialized for one label.
pub fn MatchQuery::for_label(label : String) -> MatchQuery {
let query = MatchQuery::default()
{ ..query, require_label: label }
}
///|
/// Return a query that excludes low-confidence dictionary entries.
pub fn MatchQuery::with_min_weight(
self : MatchQuery,
minimum : Int,
) -> MatchQuery {
{ ..self, min_weight: minimum }
}
///|
/// Return a query with an explicit output bound.
pub fn MatchQuery::with_limit(self : MatchQuery, limit : Int) -> MatchQuery {
{ ..self, max_matches: if limit < 0 { 0 } else { limit } }
}
///|
/// Decide whether a match satisfies a query's scalar filters.
pub fn MatchQuery::accepts(self : MatchQuery, hit : LexiconMatch) -> Bool {
hit.weight >= self.min_weight &&
(self.require_label.length() == 0 || hit.label == self.require_label)
}
///|
/// Extract matches according to query filters and output policy.
pub fn query_matches(
automaton : Automaton,
text : String,
query : MatchQuery,
) -> Array[LexiconMatch] {
let filtered : Array[LexiconMatch] = []
for hit in automaton.find_all(text) {
if query.accepts(hit) {
filtered.push(hit)
}
}
let selected = if query.include_overlaps {
filtered
} else if query.prefer_longest {
select_leftmost_longest(filtered)
} else {
select_leftmost_shortest(filtered)
}
limit_matches(selected, query.max_matches)
}
///|
/// Query a lexicon by compiling its current entries into an automaton.
pub fn query_lexicon(
lexicon : Lexicon,
text : String,
query : MatchQuery,
) -> Array[LexiconMatch] {
query_matches(compile_automaton(lexicon.entries()), text, query)
}
///|
/// Limit output while preserving the order produced by the query.
pub fn limit_matches(
matches : ArrayView[LexiconMatch],
limit : Int,
) -> Array[LexiconMatch] {
let output : Array[LexiconMatch] = []
if limit <= 0 {
return output
}
for hit in matches {
if output.length() >= limit {
break
}
output.push(hit)
}
output
}
///|
/// Sort matches by start, then by span length and weight.
pub fn sort_matches_by_position(
matches : ArrayView[LexiconMatch],
) -> Array[LexiconMatch] {
let output = matches.to_owned()
output.sort_by(fn(left, right) {
if left.start != right.start {
left.start - right.start
} else {
let left_length = left.end - left.start
let right_length = right.end - right.start
if left_length != right_length {
right_length - left_length
} else {
right.weight - left.weight
}
}
})
output
}
///|
/// Keep matches fully contained in a source interval.
pub fn matches_in_range(
matches : ArrayView[LexiconMatch],
start : Int,
end : Int,
) -> Array[LexiconMatch] {
[
for hit in matches if hit.start >= start && hit.end <= end => hit
]
}
///|
/// Keep matches for one label.
pub fn matches_for_label(
matches : ArrayView[LexiconMatch],
label : String,
) -> Array[LexiconMatch] {
[
for hit in matches if hit.label == label => hit
]
}
///|
/// Return the length of a match span.
pub fn match_span_length(hit : LexiconMatch) -> Int {
hit.end - hit.start
}
///|
/// Return true when two matches share at least one source character.
pub fn matches_overlap(left : LexiconMatch, right : LexiconMatch) -> Bool {
left.start < right.end && right.start < left.end
}
///|
/// Return true when one match fully contains another.
pub fn match_contains(outer : LexiconMatch, inner : LexiconMatch) -> Bool {
outer.start <= inner.start && outer.end >= inner.end
}
///|
/// Return all matches that overlap a target span.
pub fn overlapping_matches(
matches : ArrayView[LexiconMatch],
target : LexiconMatch,
) -> Array[LexiconMatch] {
[
for hit in matches if matches_overlap(hit, target) => hit
]
}
///|
/// Collapse overlaps using leftmost-longest selection.
pub fn merge_overlapping(
matches : ArrayView[LexiconMatch],
) -> Array[LexiconMatch] {
select_leftmost_longest(sort_matches_by_position(matches))
}
///|
/// A match together with bounded source context.
pub(all) struct ContextWindow {
source : String
before : String
matched : String
after : String
label : String
start : Int
end : Int
weight : Int
} derive(Debug, Eq, ToJson)
///|
/// Extract context windows around each match.
pub fn context_windows(
text : String,
matches : ArrayView[LexiconMatch],
radius : Int,
) -> Array[ContextWindow] {
let output : Array[ContextWindow] = []
let safe_radius = if radius < 0 { 0 } else { radius }
let text_length = text.length()
for hit in matches {
let start = clamp_int(hit.start, 0, text_length)
let end = clamp_int(hit.end, start, text_length)
let before_start = clamp_int(start - safe_radius, 0, start)
let after_end = clamp_int(end + safe_radius, end, text_length)
output.push({
source: text,
before: substring_range(text, before_start, start),
matched: substring_range(text, start, end),
after: substring_range(text, end, after_end),
label: hit.label,
start,
end,
weight: hit.weight,
})
}
output
}
///|
/// Replace every non-overlapping match with `replacement`.
pub fn redact_matches(
text : String,
matches : ArrayView[LexiconMatch],
replacement : String,
) -> String {
render_match_replacements(text, merge_overlapping(matches), replacement)
}
///|
/// Add a prefix and suffix around every non-overlapping match.
pub fn highlight_matches(
text : String,
matches : ArrayView[LexiconMatch],
prefix : String,
suffix : String,
) -> String {
let output = StringBuilder()
let mut cursor = 0
for hit in merge_overlapping(matches) {
let start = clamp_int(hit.start, cursor, text.length())
let end = clamp_int(hit.end, start, text.length())
output.write_string(substring_range(text, cursor, start))
output.write_string(prefix)
output.write_string(substring_range(text, start, end))
output.write_string(suffix)
cursor = end
}
output.write_string(substring_range(text, cursor, text.length()))
output.to_string()
}
///|
/// Render match text as HTML-safe highlighted output.
pub fn highlight_html(
text : String,
matches : ArrayView[LexiconMatch],
) -> String {
let output = StringBuilder()
let mut cursor = 0
for hit in merge_overlapping(matches) {
let start = clamp_int(hit.start, cursor, text.length())
let end = clamp_int(hit.end, start, text.length())
output.write_string(escape_html(substring_range(text, cursor, start)))
output.write_string("")
output.write_string(escape_html(substring_range(text, start, end)))
output.write_string("")
cursor = end
}
output.write_string(escape_html(substring_range(text, cursor, text.length())))
output.to_string()
}
///|
/// Replace matches only when their label belongs to a supplied set.
pub fn redact_labels(
text : String,
matches : ArrayView[LexiconMatch],
labels : ArrayView[String],
replacement : String,
) -> String {
let allowed : Map[String, Bool] = Map([])
for label in labels {
allowed[label] = true
}
let selected : Array[LexiconMatch] = []
for hit in matches {
if allowed.contains(hit.label) {
selected.push(hit)
}
}
redact_matches(text, selected, replacement)
}
///|
/// Return a compact, deterministic signature for a match.
pub fn match_signature(hit : LexiconMatch) -> String {
"\{hit.start}:\{hit.end}:\{hit.label}:\{hit.keyword}"
}
///|
/// Check that all match spans are valid for a source string.
pub fn spans_valid(text : String, matches : ArrayView[LexiconMatch]) -> Bool {
for hit in matches {
if hit.start < 0 || hit.end < hit.start || hit.end > text.length() {
return false
}
}
true
}
///|
/// Clamp a match to a source string, preserving its metadata.
pub fn clamp_match(hit : LexiconMatch, text_length : Int) -> LexiconMatch {
let start = clamp_int(hit.start, 0, text_length)
let end = clamp_int(hit.end, start, text_length)
{ ..hit, start, end }
}
///|
fn select_leftmost_shortest(
matches : ArrayView[LexiconMatch],
) -> Array[LexiconMatch] {
let sorted = matches.to_owned()
sorted.sort_by(fn(left, right) {
if left.start == right.start {
match_span_length(left) - match_span_length(right)
} else {
left.start - right.start
}
})
let output : Array[LexiconMatch] = []
let mut cursor = 0
for hit in sorted {
if hit.start >= cursor {
output.push(hit)
cursor = hit.end
}
}
output
}
///|
fn render_match_replacements(
text : String,
matches : ArrayView[LexiconMatch],
replacement : String,
) -> String {
let output = StringBuilder()
let mut cursor = 0
for hit in matches {
let start = clamp_int(hit.start, cursor, text.length())
let end = clamp_int(hit.end, start, text.length())
output.write_string(substring_range(text, cursor, start))
output.write_string(replacement)
cursor = end
}
output.write_string(substring_range(text, cursor, text.length()))
output.to_string()
}
///|
fn substring_range(text : String, start : Int, end : Int) -> String {
if end <= start {
""
} else {
let characters = text.to_array()
let output = StringBuilder()
for index in start.. Int {
if value < minimum {
minimum
} else if value > maximum {
maximum
} else {
value
}
}
///|
fn escape_html(value : String) -> String {
let output = StringBuilder()
for ch in value {
match ch {
'&' => output.write_string("&")
'<' => output.write_string("<")
'>' => output.write_string(">")
'"' => output.write_string(""")
'\'' => output.write_string("'")
_ => output.write_char(ch)
}
}
output.to_string()
}