///|
pub fn rank_movement(
baseline : Array[RetrievedDoc],
candidate : Array[RetrievedDoc],
query_id : String,
cutoff : Int,
) -> Int {
let left = unique_documents_at(baseline, query_id, cutoff)
let right = unique_documents_at(candidate, query_id, cutoff)
let right_positions : Map[String, Int] = Map([])
for index, doc_id in right {
right_positions[doc_id] = index + 1
}
let mut maximum = 0
for index, doc_id in left {
match right_positions.get(doc_id) {
Some(position) =>
maximum = Int::max(maximum, Int::abs(index + 1 - position))
None => maximum = Int::max(maximum, cutoff)
}
}
maximum
}
///|
pub fn run_signature(run : Array[RetrievedDoc]) -> String {
if run.is_empty() {
return "empty"
}
let normalized = normalize_run(run)
let rows : Array[String] = []
for item in normalized {
rows.push("\{item.query_id}:\{item.doc_id}:\{item.score}")
}
rows.join("|")
}
///|
pub fn score_order_correlation(
left : Array[RetrievedDoc],
right : Array[RetrievedDoc],
query_id : String,
cutoff : Int,
) -> Double {
let left_docs = unique_documents_at(left, query_id, cutoff)
let right_docs = unique_documents_at(right, query_id, cutoff)
if left_docs.is_empty() || right_docs.is_empty() {
return 0.0
}
let right_positions : Map[String, Int] = Map([])
for index, doc_id in right_docs {
right_positions[doc_id] = index
}
let paired : Array[(Int, Int)] = []
for index, doc_id in left_docs {
match right_positions.get(doc_id) {
Some(position) => paired.push((index, position))
None => ()
}
}
if paired.length() < 2 {
return if paired.length() == 1 { 1.0 } else { 0.0 }
}
let mut concordant = 0
let mut discordant = 0
for i in 0.. 0 {
concordant += 1
} else if left_delta * right_delta < 0 {
discordant += 1
}
}
}
let total = concordant + discordant
if total == 0 {
0.0
} else {
Double::from_int(concordant - discordant) / Double::from_int(total)
}
}
///|
pub fn render_comparison_markdown(comparison : RunComparison) -> String {
let builder = StringBuilder()
builder.write_string("## Run comparison\n\n")
builder.write_string(
"- Cutoff: \{comparison.cutoff}\n- Queries: \{comparison.query_count}\n- Wins: \{comparison.wins}\n- Losses: \{comparison.losses}\n- Ties: \{comparison.ties}\n- Mean delta: \{format_metric(comparison.mean_delta)}\n\n",
)
builder.write_string(
"| Query | Baseline | Candidate | Delta | Outcome | Overlap |\n",
)
builder.write_string("| --- | ---: | ---: | ---: | --- | ---: |\n")
for item in comparison.queries {
builder.write_string(
"| \{item.query_id} | \{format_metric(item.baseline)} | \{format_metric(item.candidate)} | \{format_metric(item.delta)} | \{item.outcome} | \{item.overlap} |\n",
)
}
builder.to_string()
}