///|
pub const API_VERSION : String = "0.2"
///|
pub const DATA_SCHEMA_VERSION : String = "1"
///|
pub fn safe_evaluate(
qrels : Array[JudgedDoc],
run : Array[RetrievedDoc],
config? : EvalConfig = EvalConfig::default(),
) -> Result[BenchmarkReport, String] {
match validate_eval_config(config) {
Ok(_) => ()
Err(error) => return Err(error)
}
let validation = validate_dataset(qrels, run)
if validation.error_count > 0 {
return Err(render_validation_report(validation))
}
Ok(evaluate_benchmark(qrels, run, config~))
}
///|
pub fn safe_parse_and_evaluate(
qrels_source : String,
run_source : String,
config : EvalConfig,
) -> Result[BenchmarkReport, String] {
let qrels = match parse_qrels_tsv_strict(qrels_source) {
Ok(value) => value
Err(error) => return Err(error)
}
let run = match parse_run_tsv_strict(run_source) {
Ok(value) => value
Err(error) => return Err(error)
}
safe_evaluate(qrels, run, config~)
}
///|
pub fn safe_sample(
qrels : Array[JudgedDoc],
run : Array[RetrievedDoc],
config : NegativeSampleConfig,
) -> Result[Array[NegativeSample], String] {
if config.per_query < 0 {
return Err("per_query must be non-negative")
}
if config.relevant_threshold < 0 {
return Err("relevant_threshold must be non-negative")
}
Ok(sample_negatives(qrels, run, config~))
}
///|
pub fn safe_compare(
qrels : Array[JudgedDoc],
baseline : Array[RetrievedDoc],
candidate : Array[RetrievedDoc],
) -> Result[RunComparison, String] {
if qrels.is_empty() {
return Err("qrels must contain at least one row")
}
let validation = validate_dataset(qrels, baseline)
if validation.error_count > 0 {
return Err(render_validation_report(validation))
}
Ok(compare_runs(qrels, baseline, candidate, cutoff=10))
}
///|
pub fn safe_report_json(
qrels : Array[JudgedDoc],
run : Array[RetrievedDoc],
config? : EvalConfig = EvalConfig::default(),
) -> Result[String, String] {
match safe_evaluate(qrels, run, config~) {
Ok(report) => Ok(render_json_report(report))
Err(error) => Err(error)
}
}
///|
pub fn normalize_inputs(
qrels : Array[JudgedDoc],
run : Array[RetrievedDoc],
) -> (Array[JudgedDoc], Array[RetrievedDoc]) {
(deduplicate_qrels(qrels), normalize_run(run))
}
///|
pub fn api_healthcheck() -> String {
"moonrag-bench/\{API_VERSION} schema/\{DATA_SCHEMA_VERSION} ready"
}