///|
fn help_text() -> String {
let text =
#|MoonRAGBench
#|
#|MoonBit-native retrieval and RAG benchmark toolkit.
#|
#|Commands:
#| demo
#| Run the bundled demo benchmark and print a compact report.
#|
#| eval --qrels --run [--format text|markdown|json]
#| [--cutoffs 1,3,5,10] [--threshold 1] [--gain linear|exp2]
#|
#| sample-negatives --qrels --run [--pool ]
#| [--count 5] [--threshold 1]
#| [--strategy hard|tail|stride] [--window 50]
#|
#| validate --qrels --run [--format text|markdown|json]
#| Validate identifiers, scores, duplicates, and query coverage.
#|
#| inspect --qrels --run
#| Print dataset size, score, and judgment-coverage diagnostics.
#|
#| compare --qrels --baseline --candidate [--cutoff 10]
#| Compare two retrieval runs by per-query nDCG and overlap.
#|
#|Examples:
#| moon run cmd/main -- demo
#| moon run cmd/main -- eval --qrels examples/demo/qrels.tsv --run examples/demo/run.tsv --format markdown
#| moon run cmd/main -- sample-negatives --qrels examples/demo/qrels.tsv --run examples/demo/run.tsv --count 2 --strategy hard --window 3
#|
text
}
///|
fn has_flag(args : Array[String], flag : String) -> Bool {
args.contains(flag)
}
///|
fn flag_value(args : Array[String], flag : String) -> String? {
if args.length() < 2 {
return None
}
for index in 1.. Result[Int, String] {
let trimmed = value.trim(chars=" \t\r")
Ok(@strconv.from_str(trimmed)) catch {
_ => Err("invalid integer for \{label}: \{trimmed}")
}
}
///|
fn parse_cutoffs_arg(value : String) -> Result[Array[Int], String] {
let cutoffs : Array[Int] = []
for piece in value.split(",") {
let trimmed = piece.trim(chars=" \t\r")
if trimmed.is_empty() {
continue
}
let parsed = match parse_int_arg(trimmed.to_owned(), "--cutoffs") {
Ok(number) => number
Err(err) => return Err(err)
}
cutoffs.push(parsed)
}
if cutoffs.is_empty() {
Err("no cutoffs were parsed from --cutoffs")
} else {
Ok(cutoffs)
}
}
///|
fn parse_gain(value : String) -> Result[@bench.GainScheme, String] {
match value {
"linear" => Ok(@bench.GainScheme::linear())
"exp2" => Ok(@bench.GainScheme::exp2())
_ => Err("unsupported gain: \{value} (expected linear or exp2)")
}
}
///|
fn parse_strategy(
args : Array[String],
) -> Result[@bench.NegativeStrategy, String] {
let name = flag_value(args, "--strategy").unwrap_or("hard")
let window_value = flag_value(args, "--window").unwrap_or("50")
let window = match parse_int_arg(window_value, "--window") {
Ok(value) => value
Err(err) => return Err(err)
}
match name {
"hard" => Ok(@bench.NegativeStrategy::hard_window(window))
"tail" => Ok(@bench.NegativeStrategy::tail(window))
"stride" => Ok(@bench.NegativeStrategy::stride(window))
_ => Err("unsupported strategy: \{name} (expected hard, tail, or stride)")
}
}
///|
fn read_text(path : String) -> Result[String, String] {
Ok(@fs.read_file_to_string(path)) catch {
_ => Err("failed to read file: \{path}")
}
}
///|
fn load_qrels(path : String) -> Result[Array[@bench.JudgedDoc], String] {
let content = match read_text(path) {
Ok(value) => value
Err(err) => return Err(err)
}
@bench.parse_qrels_tsv(content)
}
///|
fn load_run(path : String) -> Result[Array[@bench.RetrievedDoc], String] {
let content = match read_text(path) {
Ok(value) => value
Err(err) => return Err(err)
}
@bench.parse_run_tsv(content)
}
///|
fn load_pools(path : String) -> Result[Array[@bench.CandidatePool], String] {
let content = match read_text(path) {
Ok(value) => value
Err(err) => return Err(err)
}
@bench.parse_candidate_pool_tsv(content)
}
///|
fn load_eval_inputs(
qrels_path : String,
run_path : String,
) -> Result[(Array[@bench.JudgedDoc], Array[@bench.RetrievedDoc]), String] {
let qrels = match load_qrels(qrels_path) {
Ok(value) => value
Err(err) => return Err(err)
}
let run = match load_run(run_path) {
Ok(value) => value
Err(err) => return Err(err)
}
Ok((qrels, run))
}
///|
fn render_eval_command(args : Array[String]) -> Result[String, String] {
let qrels_path = match flag_value(args, "--qrels") {
Some(path) => path
None => return Err("missing required flag --qrels")
}
let run_path = match flag_value(args, "--run") {
Some(path) => path
None => return Err("missing required flag --run")
}
let cutoffs = match flag_value(args, "--cutoffs") {
Some(text) =>
match parse_cutoffs_arg(text) {
Ok(values) => values
Err(err) => return Err(err)
}
None => @bench.default_cutoffs()
}
let threshold = match flag_value(args, "--threshold") {
Some(text) =>
match parse_int_arg(text, "--threshold") {
Ok(value) => value
Err(err) => return Err(err)
}
None => 1
}
let gain = match flag_value(args, "--gain") {
Some(text) =>
match parse_gain(text) {
Ok(value) => value
Err(err) => return Err(err)
}
None => @bench.GainScheme::linear()
}
let (qrels, run) = match load_eval_inputs(qrels_path, run_path) {
Ok(value) => value
Err(err) => return Err(err)
}
let report = @bench.evaluate_benchmark(
qrels,
run,
config=@bench.EvalConfig::new(
cutoffs~,
relevant_threshold=threshold,
gain_scheme=gain,
),
)
let format = flag_value(args, "--format").unwrap_or("text")
match format {
"text" => Ok(@bench.render_compact_report(report))
"markdown" => Ok(@bench.render_markdown_report(report))
"json" => Ok(@bench.render_json_report(report))
"csv" => Ok(@bench.render_csv_report(report))
"jsonl" => Ok(@bench.render_json_lines_report(report))
_ =>
Err(
"unsupported format: \{format} (expected text, markdown, json, csv, or jsonl)",
)
}
}
///|
fn render_validate_command(args : Array[String]) -> Result[String, String] {
let qrels_path = match flag_value(args, "--qrels") {
Some(path) => path
None => return Err("missing required flag --qrels")
}
let run_path = match flag_value(args, "--run") {
Some(path) => path
None => return Err("missing required flag --run")
}
let (qrels, run) = match load_eval_inputs(qrels_path, run_path) {
Ok(value) => value
Err(err) => return Err(err)
}
let summary = @bench.validate_dataset(qrels, run)
let format = flag_value(args, "--format").unwrap_or("markdown")
match format {
"text" => Ok(@bench.render_validation_report(summary))
"markdown" => Ok(@bench.render_validation_report(summary))
"json" => Ok(ToJson::to_json(summary).stringify(indent=2))
_ => Err("unsupported format: \{format} (expected text, markdown, or json)")
}
}
///|
fn render_inspect_command(args : Array[String]) -> Result[String, String] {
let qrels_path = match flag_value(args, "--qrels") {
Some(path) => path
None => return Err("missing required flag --qrels")
}
let run_path = match flag_value(args, "--run") {
Some(path) => path
None => return Err("missing required flag --run")
}
let (qrels, run) = match load_eval_inputs(qrels_path, run_path) {
Ok(value) => value
Err(err) => return Err(err)
}
Ok(@bench.render_profile_text(@bench.profile_dataset(qrels, run)))
}
///|
fn render_compare_command(args : Array[String]) -> Result[String, String] {
let qrels_path = match flag_value(args, "--qrels") {
Some(path) => path
None => return Err("missing required flag --qrels")
}
let baseline_path = match flag_value(args, "--baseline") {
Some(path) => path
None => return Err("missing required flag --baseline")
}
let candidate_path = match flag_value(args, "--candidate") {
Some(path) => path
None => return Err("missing required flag --candidate")
}
let qrels = match load_qrels(qrels_path) {
Ok(value) => value
Err(err) => return Err(err)
}
let baseline = match load_run(baseline_path) {
Ok(value) => value
Err(err) => return Err(err)
}
let candidate = match load_run(candidate_path) {
Ok(value) => value
Err(err) => return Err(err)
}
let cutoff = match flag_value(args, "--cutoff") {
Some(value) =>
match parse_int_arg(value, "--cutoff") {
Ok(parsed) => Int::max(parsed, 1)
Err(err) => return Err(err)
}
None => 10
}
Ok(
@bench.render_comparison_text(
@bench.compare_runs(qrels, baseline, candidate, cutoff~),
),
)
}
///|
fn render_negative_sampling_command(
args : Array[String],
) -> Result[String, String] {
let qrels_path = match flag_value(args, "--qrels") {
Some(path) => path
None => return Err("missing required flag --qrels")
}
let run_path = match flag_value(args, "--run") {
Some(path) => path
None => return Err("missing required flag --run")
}
let (qrels, run) = match load_eval_inputs(qrels_path, run_path) {
Ok(value) => value
Err(err) => return Err(err)
}
let pools = match flag_value(args, "--pool") {
Some(path) =>
match load_pools(path) {
Ok(value) => value
Err(err) => return Err(err)
}
None => []
}
let count = match flag_value(args, "--count") {
Some(text) =>
match parse_int_arg(text, "--count") {
Ok(value) => value
Err(err) => return Err(err)
}
None => 5
}
let threshold = match flag_value(args, "--threshold") {
Some(text) =>
match parse_int_arg(text, "--threshold") {
Ok(value) => value
Err(err) => return Err(err)
}
None => 1
}
let strategy = match parse_strategy(args) {
Ok(value) => value
Err(err) => return Err(err)
}
let samples = @bench.sample_negatives(
qrels,
run,
pools~,
config=@bench.NegativeSampleConfig::new(
per_query=count,
relevant_threshold=threshold,
strategy~,
),
)
Ok(@bench.render_negative_samples_tsv(samples))
}
///|
fn render_demo() -> Result[String, String] {
render_eval_command([
"moonrag-bench", "eval", "--qrels", "examples/demo/qrels.tsv", "--run", "examples/demo/run.tsv",
"--format", "text", "--cutoffs", "1,3,5", "--gain", "exp2",
])
}
///|
fn print_result(result : Result[String, String]) -> Unit {
match result {
Ok(text) => println(text)
Err(err) => {
println("error: \{err}\n")
println(help_text())
}
}
}
///|
fn main {
let args = @env.args()
if args.length() <= 1 || has_flag(args, "--help") || has_flag(args, "-h") {
println(help_text())
return
}
let command = args[1]
match command {
"demo" => print_result(render_demo())
"eval" => print_result(render_eval_command(args))
"sample-negatives" => print_result(render_negative_sampling_command(args))
"validate" => print_result(render_validate_command(args))
"inspect" => print_result(render_inspect_command(args))
"compare" => print_result(render_compare_command(args))
_ => {
println("error: unknown command '\{command}'\n")
println(help_text())
}
}
}