///|
pub fn available_presets() -> Array[String] {
["balanced", "strict", "graded", "fast"]
}
///|
pub fn preset_config(name : String) -> Result[EvalConfig, String] {
match name {
"balanced" =>
Ok(
EvalConfig::new(
cutoffs=[1, 3, 5, 10],
relevant_threshold=1,
gain_scheme=GainScheme::linear(),
),
)
"strict" =>
Ok(
EvalConfig::new(
cutoffs=[1, 3, 10],
relevant_threshold=2,
gain_scheme=GainScheme::linear(),
),
)
"graded" =>
Ok(
EvalConfig::new(
cutoffs=[1, 3, 5, 10],
relevant_threshold=1,
gain_scheme=GainScheme::exp2(),
),
)
"fast" =>
Ok(
EvalConfig::new(
cutoffs=[5],
relevant_threshold=1,
gain_scheme=GainScheme::linear(),
),
)
_ =>
Err(
"unknown preset '\{name}'; expected \{available_presets().join(", ")}",
)
}
}
///|
pub fn preset_description(name : String) -> Result[String, String] {
match name {
"balanced" => Ok("common binary relevance metrics at four cutoffs")
"strict" =>
Ok("high-relevance-only scoring for precision-sensitive retrieval")
"graded" => Ok("graded relevance with exponential gain for RAG reranking")
"fast" => Ok("single-cutoff smoke evaluation for local iteration")
_ => Err("unknown preset '\{name}'")
}
}
///|
pub fn evaluate_with_preset(
name : String,
qrels : Array[JudgedDoc],
run : Array[RetrievedDoc],
) -> Result[BenchmarkReport, String] {
let config = match preset_config(name) {
Ok(value) => value
Err(error) => return Err(error)
}
safe_evaluate(qrels, run, config~)
}
///|
pub fn render_preset_catalog() -> String {
let rows : Array[String] = ["| Preset | Description |", "| --- | --- |"]
for name in available_presets() {
let description = preset_description(name).unwrap_or("")
rows.push("| \{name} | \{description} |")
}
rows.join("\n")
}
///|
pub fn preset_cutoffs(name : String) -> Result[Array[Int], String] {
match preset_config(name) {
Ok(config) => Ok(config.cutoffs)
Err(error) => Err(error)
}
}
///|
pub fn is_known_preset(name : String) -> Bool {
available_presets().contains(name)
}
///|
pub fn validate_eval_config(config : EvalConfig) -> Result[Unit, String] {
if config.relevant_threshold < 0 {
return Err("relevant_threshold must be non-negative")
}
if normalize_cutoffs(config.cutoffs).is_empty() {
return Err("cutoffs must contain a positive integer")
}
Ok(())
}
///|
pub fn config_signature(config : EvalConfig) -> String {
let cutoffs = normalize_cutoffs(config.cutoffs)
.map(fn(item) { item.to_string() })
.join(",")
let gain = match config.gain_scheme {
Linear => "linear"
Exp2 => "exp2"
}
"cutoffs=\{cutoffs};threshold=\{config.relevant_threshold};gain=\{gain};missing=\{config.missing_relevance}"
}