///|
/// Deterministic Lehmer generator; Int64 keeps the multiplication from
/// overflowing 32-bit workspaces.
fn lcg_next(state : Int64) -> Int64 {
state * 48271L % 2147483647L
}
///|
fn make_word(state : Int64, length : Int) -> (String, Int64) {
let letters = "abcdefghijklmnopqrstuvwxyz"
let buf = StringBuilder()
let mut s = state
for _ in 0.. (Array[String], Int) {
let mut state = 20260929L
let corpus : Array[String] = []
let mut bytes = 0
for i = 0; i < lines; i = i + 1 {
let line = if i % 10 == 0 {
let (word, next) = make_word(state, 6)
state = next
match i % 50 {
0 => "login user=" + word + "@example.com ok"
10 => "password=" + word + "value"
20 => "client " + word + " 10.0.0." + (i % 200 + 1).to_string()
30 => "authorization: Bearer " + word + word + "tail"
_ => "token=ghp_" + word + word + "12345678"
}
} else {
let (w1, s1) = make_word(state, 7)
let (w2, s2) = make_word(s1, 5)
state = s2
"request " + w1 + " handler=" + w2 + " status=200"
}
bytes += line.length() + 1
corpus.push(line)
}
(corpus, bytes)
}
///|
/// Mill a fill corpus for the linearity gate: every seventh line plants a
/// credential, the rest is plain padding.
fn make_fill_corpus(lines : Int) -> Array[String] {
let corpus : Array[String] = []
for i = 0; i < lines; i = i + 1 {
let (word, _) = make_word(101L + i.to_int64(), 40)
if i % 7 == 0 {
corpus.push("password=" + word + " tail")
} else {
corpus.push("plain " + word + " done")
}
}
corpus
}
///|
fn time_pass_micros(corpus : Array[String]) -> Double {
let start = @mbench.monotonic_clock_start()
let result = @scrub.redact_batch(corpus)
let micros = @mbench.monotonic_clock_end(start)
if result.lines.length() != corpus.length() {
abort("corpus line count changed during redaction")
}
micros
}
///|
/// Build a deterministic JSON document with planted findings: several
/// hundred string leaves, a third of them carrying an email or credential.
fn make_json_document(records : Int) -> String {
let buf = StringBuilder()
buf.write_string("{\"records\":[")
for i = 0; i < records; i = i + 1 {
if i > 0 {
buf.write_char(Int::unsafe_to_char(44))
}
let (word, _) = make_word(77L + i.to_int64(), 8)
let note = if i % 3 == 0 {
"mail " + word + "@example.com"
} else if i % 3 == 1 {
"password=" + word + "value"
} else {
"ordinary note " + word
}
buf.write_string("{\"id\":" + i.to_string() + ",\"note\":\"" + note + "\"}")
}
buf.write_string("]}")
buf.to_string()
}
///|
/// One pass per detector family over a mixed sample, reporting the
/// relative cost of each family in isolation.
fn family_breakdown() -> Unit {
let sample = make_corpus(4000)
let corpus = sample.0
let families : Array[(String, @scrub.ScanConfig)] = []
families.push(
(
"access-token",
{
..@scrub.ScanConfig::standard(),
detect_bearer: false,
detect_jwt: false,
detect_private_key: false,
detect_url_credential: false,
detect_assignment: false,
detect_email: false,
detect_ipv4: false,
detect_ipv6: false,
detect_resident_id: false,
detect_mac: false,
detect_uuid: false,
detect_public_key: false,
detect_wallet: false,
detect_webhook: false,
detect_payment_card: false,
},
),
)
families.push(
(
"assignment",
{
..@scrub.ScanConfig::standard(),
detect_access_token: false,
detect_bearer: false,
detect_jwt: false,
detect_private_key: false,
detect_url_credential: false,
detect_email: false,
detect_ipv4: false,
detect_ipv6: false,
detect_resident_id: false,
detect_mac: false,
detect_uuid: false,
detect_public_key: false,
detect_wallet: false,
detect_webhook: false,
detect_payment_card: false,
},
),
)
families.push(
(
"email",
{
..@scrub.ScanConfig::standard(),
detect_access_token: false,
detect_bearer: false,
detect_jwt: false,
detect_private_key: false,
detect_url_credential: false,
detect_assignment: false,
detect_ipv4: false,
detect_ipv6: false,
detect_resident_id: false,
detect_mac: false,
detect_uuid: false,
detect_public_key: false,
detect_wallet: false,
detect_webhook: false,
detect_payment_card: false,
},
),
)
families.push(
(
"ipv4",
{
..@scrub.ScanConfig::standard(),
detect_access_token: false,
detect_bearer: false,
detect_jwt: false,
detect_private_key: false,
detect_url_credential: false,
detect_assignment: false,
detect_email: false,
detect_ipv6: false,
detect_resident_id: false,
detect_mac: false,
detect_uuid: false,
detect_public_key: false,
detect_wallet: false,
detect_webhook: false,
detect_payment_card: false,
},
),
)
families.push(
(
"cards",
{
..@scrub.ScanConfig::standard(),
detect_access_token: false,
detect_bearer: false,
detect_jwt: false,
detect_private_key: false,
detect_url_credential: false,
detect_assignment: false,
detect_email: false,
detect_ipv4: false,
detect_ipv6: false,
detect_resident_id: false,
detect_mac: false,
detect_uuid: false,
detect_public_key: false,
detect_wallet: false,
detect_webhook: false,
},
),
)
families.push(("everything", @scrub.ScanConfig::standard()))
for pair in families {
let start = @mbench.monotonic_clock_start()
let mut total = 0
for _ in 0..<3 {
let result = @scrub.redact_batch_with_config(corpus, pair.1)
total += result.summary.findings
}
let micros = @mbench.monotonic_clock_end(start)
println(
"family-" +
pair.0 +
"-micros=" +
micros.to_string() +
" findings=" +
(total / 3).to_string(),
)
}
}
///|
fn main {
let lines = 20000
let (corpus, bytes) = make_corpus(lines)
let iterations = 20
// Warm up caches and code paths before timing.
ignore(@scrub.redact_batch(corpus))
let start = @mbench.monotonic_clock_start()
let mut total_findings = 0
for _ in 0..