// Golden-file driver for the twelve Snowball stemmers (Task 7 / FR-12..FR-14).
//
// Twelve rows = the eleven registry entries (`lang_registry.mbt`, contract §3)
// plus the porter311 control row (contract §3.4: same corpus as porter, never
// in the registry). Each row is compared line by line against
// `//output.txt` and reported as
//
// lang= checked= mismatch= ms=
//
// with the `lang=` prefix the prototype lacked (SCD-08). After the twelve
// comparison rows the driver prints twelve
//
// lang= lines=
//
// rows carrying the line count of the generated `/_gen.mbt`.
// Task 12 (FR-20) judges each of those rows against the committed baseline
// below and appends ` WARN line-count drift` when the count moves more than
// 20% away from it, then prints the summary row
//
// line-count drift: /12
//
// with k the number of warned rows. A WARN is a warning, never a failure:
// `scripts/gate.sh` judges only the comparison rows. The driver is judged
// by that gate; it never prints `GATE OK` itself.
//
// Data root precedence (R-2 / contract §4): `--data-root
` >
// `SNOWBALL_DATA_ROOT` > module-relative `.planning/data/sd` (resolved against
// the current directory, which is the module root under `moon run`). The
// resolved root is printed as an absolute path at startup; no absolute path is
// ever hardcoded here.
//
// arabic ships gzipped upstream and MoonBit has no gzip: the driver never
// opens a `.gz`. Its plain text is the unpacked artifact the pipeline's
// unpack stage writes to `/build/data/arabic`, so a compressed
// registry entry resolves to
//
// `--arabic-dir
` > `/arabic` (when it already holds plain
// `voc.txt` + `output.txt`) > `/build/data/arabic`
//
// and the chosen directory plus its source is printed at startup.
//
// Three further observable lines come from the non-latin spec §2.2 (X-1/X-2),
// on top of the `source=flag|data-root|build` precedence token above (which
// keeps meaning "which directory won the arabic race" and is NOT overloaded):
//
// source=voc.txt.gz — printed inside the arabic row's segment,
// followed by one `unpacked_preferred ...`
// line naming the directory actually read
// progress= elapsed= — every 1,000,000 compared lines (interval
// fixed by ruling R-B), `` in SECONDS
// total= — the run's wall time in seconds, printed
// exactly once, after the boundary segment's
// tally (the last ordinary line; a failure
// abort may follow it)
//
// No line ever claims a sub-sample: every row compares all of its lines
// (U4-D6).
//
// After the twelve comparison rows and the `lines=` block the driver prints the
// boundary segment of the non-latin spec §4.4: the eleven hand-picked non-ASCII
// cases B-01..B-11 over russian / arabic / turkish. A fixture that feeds this
// segment must hold the full corpus, not a head: russian/voc.txt and
// russian/output.txt need at least 49,675 lines (B-02's anchor) and each of the
// russian, arabic and turkish pairs needs at least 1,000 lines (B-11's head).
// A shorter file aborts loudly instead of turning an anchored case into a pass.
// Each case contributes at least two lines at column 0 — a
// `boundary ...` provenance line (its corpus anchor, or its synthetic
// origin) and then the observation line of spec §3.2:
//
// case=<输入> got=<输出> ok=
//
// UNQUOTED for B-01..B-08 and B-10, exactly as §4.4 tabulates them. B-09 is
// the one case whose declared form quotes the value (`case="" got="" ok=true`),
// because an empty input is unreadable unquoted; that quoting is not a style
// choice and is not extended to any other case. B-11 keeps §4.4's single
// aggregate `same=true` token: the per-language detail lines report
// `differing=` so the token appears exactly once. Nothing here preprocesses
// the input (U4-D4: apostrophes reach `stem_word` verbatim) and nothing maps
// U+0131 (U4-D3).
///|
/// One gate row: registry name, corpus directory, whether the corpus is
/// stored compressed upstream, and the stem callback. The declared
/// `checked` value is not duplicated here: the gate reads it from
/// `lang_registry.mbt`, so there is exactly one source for it.
struct Row {
name : String
dir : String
compressed : Bool
stem : (String) -> String raise @rt.Lab
}
///|
/// Target-scoped exit primitive from `moonbitlang/x/sys` (native C `exit`,
/// wasm/wasm-gc/js backend implementations). The driver decides its own exit
/// code (0 clean, 1 corpus/comparison/usage failure) instead of relying on an
/// uncaught error's status.
fn native_exit(code : Int32) -> Unit {
@sys.exit(code)
}
///|
/// Abort with one stderr line. The caller (`main`) turns the raised error
/// into exit code 1; the message stays the first stderr line.
fn[T] fail(msg : String) -> T raise @fs.IOError {
eprintln(msg)
raise @fs.IOError(msg)
}
///|
/// The twelve rows, in registry order plus the porter311 control row last.
/// Every callback is built once around a single stemmer instance, so the
/// per-word call allocates nothing beyond the word itself.
fn wired_rows() -> Array[Row] raise @fs.IOError {
let rows : Array[Row] = []
for e in @registry.langs {
let stem : (String) -> String raise @rt.Lab = match e.name {
"arabic" => {
let p = @arabic.Arabic::new()
word => @arabic.stem_word(p, word)
}
"dutch" => {
let p = @dutch.Dutch::new()
word => @dutch.stem_word(p, word)
}
"english" => {
let p = @english.English::new()
word => @english.stem_word(p, word)
}
"french" => {
let p = @french.French::new()
word => @french.stem_word(p, word)
}
"german" => {
let p = @german.German::new()
word => @german.stem_word(p, word)
}
"italian" => {
let p = @italian.Italian::new()
word => @italian.stem_word(p, word)
}
"porter" => {
let p = @porter.Porter::new()
word => @porter.stem_word(p, word)
}
"portuguese" => {
let p = @portuguese.Portuguese::new()
word => @portuguese.stem_word(p, word)
}
"russian" => {
let p = @russian.Russian::new()
word => @russian.stem_word(p, word)
}
"spanish" => {
let p = @spanish.Spanish::new()
word => @spanish.stem_word(p, word)
}
"turkish" => {
let p = @turkish.Turkish::new()
word => @turkish.stem_word(p, word)
}
_ => fail("golden: unknown registry entry: \{e.name}")
}
rows.push({
name: e.name,
dir: e.golden_dir,
compressed: e.compressed,
stem,
})
}
// The control row reuses porter's corpus (contract §3.4) and is reported
// under its own name; its declared `checked` value is porter's, which the
// gate derives from the registry.
let p311 = @porter311.Porter311::new()
rows.push({
name: "porter311",
dir: "porter",
compressed: false,
stem: word => @porter311.stem_word(p311, word),
})
rows
}
///|
/// True for POSIX-rooted paths and Windows drive paths (`C:...`).
fn is_absolute_path(p : String) -> Bool {
if p.length() == 0 {
false
} else if p[0] == '/' || p[0] == '\\' {
true
} else {
p.length() >= 2 && p[1] == ':'
}
}
///|
/// Join a base directory with a relative fragment using `/` (accepted by the
/// native IO layer on every platform) and collapse trailing separators.
fn join_path(base : String, rel : String) -> String {
let mut end = base.length()
while end > 0 && (base[end - 1] == '/' || base[end - 1] == '\\') {
end -= 1
}
let trimmed = if end == base.length() {
base
} else {
base.view(start_offset=0, end_offset=end).to_owned()
}
if trimmed.length() == 0 {
rel
} else {
trimmed + "/" + rel
}
}
///|
/// The current directory as an absolute path, or a hard error: every path the
/// driver prints must be absolute.
fn cwd() -> String raise @fs.IOError {
match @env.current_dir() {
Some(d) =>
if is_absolute_path(d) {
d
} else {
fail("golden: cannot resolve the absolute current directory: \{d}")
}
None =>
fail("golden: cannot resolve data root: current directory unavailable")
}
}
///|
/// Make `p` absolute: an already-absolute value is kept, anything else is
/// resolved against the current directory.
fn absolute(p : String) -> String raise @fs.IOError {
if is_absolute_path(p) {
p
} else {
join_path(cwd(), p)
}
}
///|
/// Data root precedence: `--data-root` > `SNOWBALL_DATA_ROOT` > the
/// module-relative default `.planning/data/sd`. Returns the absolute path and
/// the winning source (`flag` / `env` / `default`).
fn resolve_data_root(flag : String?) -> (String, String) raise @fs.IOError {
match flag {
Some(p) => (absolute(p), "flag")
None =>
match @env.get_env_var("SNOWBALL_DATA_ROOT") {
Some(v) =>
if v.length() == 0 {
(absolute(".planning/data/sd"), "default")
} else {
(absolute(v), "env")
}
None => (absolute(".planning/data/sd"), "default")
}
}
}
///|
/// True when a directory already holds the arabic plain-text pair.
fn has_plain_arabic(dir : String) -> Bool {
@fs.path_exists(join_path(dir, "voc.txt")) &&
@fs.path_exists(join_path(dir, "output.txt"))
}
///|
/// Where arabic's plain text lives (see the file header): `--arabic-dir` >
/// `/arabic` when already unpacked > `/build/data/arabic`
/// (the artifact the pipeline's unpack stage writes). The driver never opens
/// the `.gz` files.
fn resolve_arabic_dir(
flag : String?,
data_root : String,
) -> (String, String) raise @fs.IOError {
match flag {
Some(p) => (absolute(p), "flag")
None => {
let in_root = join_path(data_root, "arabic")
if has_plain_arabic(in_root) {
(in_root, "data-root")
} else {
(absolute("build/data/arabic"), "build")
}
}
}
}
///|
/// The corpus directory of one row: compressed entries (arabic) use the
/// unpacked directory, everything else `/`.
fn row_dir(row : Row, data_root : String, arabic_dir : String) -> String {
if row.compressed {
arabic_dir
} else {
join_path(data_root, row.dir)
}
}
///|
/// A missing corpus file must abort with the ABSOLUTE path and the literal
/// `missing` as the first stderr line, before any summary is printed.
fn require_file(path : String, lang : String) -> Unit raise @fs.IOError {
if !@fs.path_exists(path) {
fail("golden: missing corpus file for \{lang}: \{path}")
}
if !@fs.is_file(path) {
fail("golden: missing regular corpus file for \{lang}: \{path}")
}
}
///|
/// Number of lines under the driver's own reading rule: every LF starts a new
/// line, and a final line without LF still counts. Identical to the
/// `split_lines` rule the earlier version used.
fn count_lines(s : String) -> Int32 {
let n = s.length()
let mut count = 0
let mut i = 0
while i < n {
if s[i] == '\n' {
count += 1
}
i += 1
}
if n > 0 && s[n - 1] != '\n' {
count += 1
}
count
}
///|
/// One row: read both files, assert equal line counts (before the per-line
/// loop), compare every stemmed word with the golden output, show at most five
/// `MISMATCH` detail lines, and print the `lang=... checked=... mismatch=...`
/// row. Returns the mismatch count.
fn run_one(
row : Row,
voc_path : String,
out_path : String,
) -> Int32 raise @fs.IOError {
let voc = @fs.read_file_to_string(voc_path, encoding="utf8")
let exp = @fs.read_file_to_string(out_path, encoding="utf8")
let voc_n = count_lines(voc)
let out_n = count_lines(exp)
if voc_n != out_n {
println("voc=\{voc_n} output=\{out_n}")
fail(
"golden: line count mismatch for \{row.name}: voc=\{voc_n} output=\{out_n} (voc=\{voc_path} output=\{out_path})",
)
}
let t0 = @env.now()
let mut checked = 0
let mut mismatch = 0
let mut shown = 0
let vn = voc.length()
let en = exp.length()
let mut vs = 0
let mut es = 0
while vs < vn && es < en {
let mut ve = vs
while ve < vn && voc[ve] != '\n' {
ve += 1
}
let mut ee = es
while ee < en && exp[ee] != '\n' {
ee += 1
}
let word = voc.view(start_offset=vs, end_offset=ve).to_owned()
let want = exp.view(start_offset=es, end_offset=ee)
let got = (row.stem)(word) catch { _ => "" }
if !(got.view() == want) {
mismatch += 1
if shown < 5 {
shown += 1
println(
"MISMATCH lang=\{row.name} line=\{checked + 1} word=\{word} want=\{want.to_owned()} got=\{got}",
)
}
}
checked += 1
// Spec §2.2 X-2 / ruling R-B: one progress line every 1,000,000 compared
// lines, elapsed in SECONDS. The interval is fixed: never derived from
// the corpus size, never emitted as a per-row trailing summary.
if checked % 1000000 == 0 {
println("progress=\{checked} elapsed=\{(@env.now() - t0) / 1000}")
}
vs = ve + 1
es = ee + 1
}
let elapsed = @env.now() - t0
println(
"lang=\{row.name} checked=\{checked} mismatch=\{mismatch} ms=\{elapsed}",
)
mismatch
}
///|
/// Line count of the generated package source `/_gen.mbt`, as the
/// `lang= lines=` row (Task 12's drift baseline).
fn gen_lines(module_root : String, name : String) -> Int32 raise @fs.IOError {
let path = join_path(module_root, name + "/" + name + "_gen.mbt")
require_file(path, name)
count_lines(@fs.read_file_to_string(path, encoding="utf8"))
}
///|
/// Line-count baselines of the twelve generated packages (Task 12 / FR-20).
///
/// Source: spec F-U2-21, the T5 post-`moon fmt` committed form of each
/// `/_gen.mbt` (10,837 lines for the eleven deliverable packages,
/// 11,407 including the porter311 control row). This is the ONE baseline
/// table; it is in row order (registry order, porter311 last). The registry's
/// own `lines` field is the CORPUS line count, so it cannot serve here.
fn baseline_rows() -> Array[(String, Int32)] {
[
("arabic", 1063),
("dutch", 1572),
("english", 885),
("french", 1171),
("german", 553),
("italian", 837),
("porter", 538),
("portuguese", 730),
("russian", 511),
("spanish", 768),
("turkish", 2209),
("porter311", 570),
]
}
///|
/// The baseline for one row name. An unknown name is a driver bug, not a
/// silently accepted row: fail instead of guessing a baseline.
fn baseline_of(name : String) -> Int32 raise @fs.IOError {
for entry in baseline_rows() {
if entry.0 == name {
return entry.1
}
}
fail("golden: no line-count baseline for \{name}")
}
///|
/// True when `measured` drifted more than 20% from `baseline`, i.e.
/// `|M - B| / B > 0.20`. Evaluated as `|M - B| * 5 > B` so the comparison is
/// exact and a move of exactly 20% does not warn.
fn line_count_drifted(measured : Int32, baseline : Int32) -> Bool {
let diff = if measured >= baseline {
measured - baseline
} else {
baseline - measured
}
diff * 5 > baseline
}
///|
/// Parse `--data-root` / `--arabic-dir` (space or `=` form). Anything else is
/// a usage error: the parse raises and `main` maps it to exit 1 (driver exit
/// contract is 0/1; the gate's usage exit 2 belongs to `scripts/gate.sh`).
/// Never a silent fallback.
fn parse_args(args : Array[String]) -> (String?, String?) raise @fs.IOError {
let mut data_root : String? = None
let mut arabic_dir : String? = None
let mut i = 0
let n = args.length()
while i < n {
let a = args[i]
if a == "--data-root" {
if i + 1 >= n {
fail("golden: --data-root needs a value")
}
data_root = Some(args[i + 1])
i += 2
} else if a.length() > 12 &&
a.view(start_offset=0, end_offset=12).to_owned() == "--data-root=" {
data_root = Some(a.view(start_offset=12).to_owned())
i += 1
} else if a == "--arabic-dir" {
if i + 1 >= n {
fail("golden: --arabic-dir needs a value")
}
arabic_dir = Some(args[i + 1])
i += 2
} else if a.length() > 13 &&
a.view(start_offset=0, end_offset=13).to_owned() == "--arabic-dir=" {
arabic_dir = Some(a.view(start_offset=13).to_owned())
i += 1
} else {
fail("golden: unknown argument: \{a}")
}
}
(data_root, arabic_dir)
}
///|
/// Print one boundary line (`case="" got="" ok=`) and stop
/// the run when the case fails: exactly one `ok=` token per line (FR-09); the
/// golden gate counts `ok=true`/`ok=false` lines.
fn report_word_case(
word : String,
got : String,
ok : Bool,
) -> Unit raise @fs.IOError {
println("case=\"\{word}\" got=\"\{got}\" ok=\{ok}")
if !ok {
fail("golden: boundary case failed: case=\"\{word}\" got=\"\{got}\"")
}
}
///|
/// Boundary segment (FR-09/FR-19): the primitive `go_out_grouping` case, then
/// word-level cases for porter and english. The four empty/single-character
/// inputs are the brief's assertion set; porter also pins the two non-ASCII
/// prototype values and runs an `Aargauer` determinism probe plus a non-BMP
/// emoji probe ([LOCAL_DEAD_LINK] DL-3). A failed case aborts the run: the
/// primitive case uses `abort`, the word cases raise, and `main` maps either
/// to exit 1.
fn check_boundary_case() -> Unit raise @fs.IOError {
// Primitive case (unchanged): go_out_grouping(Env("bcda"), "aeiou") walks
// b, c, d and stops on the member `a` at cursor 3.
let env = @rt.Env::new("bcda")
let result = @rt.go_out_grouping(env, "aeiou")
let cursor = env.cursor
let ok = result && cursor == 3
println(
"case=go_out_grouping(Env(\"bcda\"),\"aeiou\") got=\{result} ok=\{ok} cursor=\{cursor}",
)
if !ok {
abort(
"boundary case failed: go_out_grouping(Env(\"bcda\"),\"aeiou\") expected true with cursor=3",
)
}
let porter = @porter.Porter::new()
let english = @english.English::new()
// Porter word cases. `s` -> "" and `'` -> `'` are also the corpus lines
// `sed -n '32220p'` and `sed -n '1p'` of
// .planning/data/sd/porter/{voc,output}.txt.
let porter_cases : Array[(String, String)] = [
("", ""),
("s", ""),
("'", "'"),
("y", "y"),
// Non-ASCII (BMP) words from the prototype set: no among-table membership,
// so they pass through untouched (.probe/slice/golden.mbt:104-105).
("café", "café"),
("北京", "北京"),
]
for c in porter_cases {
let (word, want) = c
let got = @porter.stem_word(porter, word) catch {
_ => fail("golden: porter boundary case raised: case=\"\{word}\"")
}
report_word_case(word, got, got == want)
}
// English word cases. The brief's generic set says `s` -> ""; the english
// (Porter2) stemmer keeps a lone `s` -- corpus line 32260 has voc `s` and
// output `s`, and the english golden row compares 42649 lines at
// mismatch=0 -- so the observed value `s` is pinned (reported as a concern).
let english_cases : Array[(String, String)] = [
("", ""),
("s", "s"),
("'", "'"),
("y", "y"),
]
for c in english_cases {
let (word, want) = c
let got = @english.stem_word(english, word) catch {
_ => fail("golden: english boundary case raised: case=\"\{word}\"")
}
report_word_case(word, got, got == want)
}
// `Aargauer` has no exact prototype expectation: the assertion is only
// "returns without raising, and deterministically".
let aargauer_first = @porter.stem_word(porter, "Aargauer") catch {
_ => fail("golden: porter boundary case raised: case=\"Aargauer\"")
}
let aargauer_second = @porter.stem_word(porter, "Aargauer") catch {
_ => fail("golden: porter boundary case raised: case=\"Aargauer\"")
}
report_word_case(
"Aargauer",
aargauer_first,
aargauer_first == aargauer_second,
)
// [LOCAL_DEAD_LINK] DL-3: the upstream non-BMP fixture is not available in
// this tree, so this local probe judges only "returns without raising and
// deterministically". A returned MoonBit String is well-formed by
// construction.
let emoji = "😀"
let emoji_first = @porter.stem_word(porter, emoji) catch {
_ => fail("golden: porter boundary case raised: non-BMP emoji probe")
}
let emoji_second = @porter.stem_word(porter, emoji) catch {
_ => fail("golden: porter boundary case raised: non-BMP emoji probe")
}
report_word_case(emoji, emoji_first, emoji_first == emoji_second)
}
// ---------------------------------------------------------------------------
// U3 Latin boundary segment (spec §4.4, B-01..B-21) — appended by Task 5.
//
// Append-only: the twelve comparison rows, `check_boundary_case`, the row
// format and the gate summary above are untouched. Every line this segment
// prints has the spec §3.2 shape
//
// case= got=