///|
/// A reproducible evaluation summary for an approximate index.
pub(all) struct BenchmarkReport {
queries : Int
corpus : Int
top_k : Int
recall : Double
index_name : String
}
///|
pub impl Show for BenchmarkReport with fn output(self, logger) {
logger.write_string(
"BenchmarkReport{index: " +
self.index_name +
", corpus: " +
self.corpus.to_string() +
", queries: " +
self.queries.to_string() +
", top_k: " +
self.top_k.to_string() +
", recall: " +
self.recall.to_string() +
"}",
)
}
///|
/// Build a deterministic corpus shaped like a small semantic embedding set.
/// The four topics make metadata filtering and cluster locality observable.
pub fn benchmark_corpus() -> Array[Document] {
let topics = ["science", "systems", "language", "retrieval"]
let docs = []
for i = 0; i < 48; i = i + 1 {
let topic = i % topics.length()
let band = (i / topics.length()).to_double()
let topic_bias = topic.to_double() * 4.0
let vector = [
1.0 + topic_bias + band * 0.03,
0.5 + topic_bias * 0.2 + band * 0.07,
2.0 + topic_bias * 0.1 + (i % 5).to_double() * 0.05,
0.25 + topic.to_double() * 0.4 + band * 0.02,
1.5 + (i % 7).to_double() * 0.08,
0.75 + topic_bias * 0.15 + band * 0.04,
0.2 + (i % 3).to_double() * 0.3,
0.9 + topic.to_double() * 0.25 + band * 0.01,
]
docs.push(
Document::new("bench-" + i.to_string(), vector, [
("topic", topics[topic]),
("split", if i % 5 == 0 { "query" } else { "train" }),
]),
)
}
docs
}
///|
/// Return queries selected from different semantic regions of the corpus.
pub fn benchmark_queries() -> Array[Array[Double]] {
[
[1.02, 0.53, 2.04, 0.26, 1.58, 0.77, 0.21, 0.91],
[5.03, 1.34, 2.44, 0.68, 1.66, 1.38, 0.52, 1.18],
[9.01, 2.15, 2.84, 1.12, 1.73, 1.99, 0.81, 1.45],
[13.08, 2.94, 3.25, 1.53, 1.88, 2.61, 0.23, 1.76],
[1.22, 0.91, 2.16, 0.31, 1.91, 0.96, 0.51, 0.95],
[5.21, 1.70, 2.56, 0.76, 1.84, 1.55, 0.22, 1.31],
]
}
///|
/// Evaluate IVF recall against the exact Flat baseline.
pub fn evaluate_ivf(
docs : Array[Document],
queries : Array[Array[Double]],
k : Int,
nprobe : Int,
) -> BenchmarkReport raise VectorError {
validate_documents(docs)
let flat = FlatIndex::new()
for doc in docs {
flat.add(doc)
}
let ivf = IvfIndex::new(4, Cosine)
ivf.build(docs)
let exact = flat.search_batch(queries, k, Cosine, [])
let approximate = ivf.search_batch(queries, k, nprobe, [])
{
queries: queries.length(),
corpus: docs.length(),
top_k: k,
recall: mean_recall(approximate, exact, k),
index_name: "IVF-Flat",
}
}
///|
/// Evaluate deterministic LSH recall against the exact Flat baseline.
pub fn evaluate_lsh(
docs : Array[Document],
queries : Array[Array[Double]],
k : Int,
) -> BenchmarkReport raise VectorError {
validate_documents(docs)
let flat = FlatIndex::new()
let lsh = LshIndex::new(6, docs[0].vector.length())
for doc in docs {
flat.add(doc)
lsh.add(doc)
}
let exact = flat.search_batch(queries, k, Cosine, [])
let approximate = lsh.search_batch(queries, k, [])
{
queries: queries.length(),
corpus: docs.length(),
top_k: k,
recall: mean_recall(approximate, exact, k),
index_name: "LSH",
}
}