///|
/// A machine-readable health snapshot for monitoring an index in production.
pub(all) struct HealthReport {
healthy : Bool
document_count : Int
dimension : Int
duplicate_ids : Int
empty_ids : Int
dimension_errors : Int
message : String
}
///|
pub impl Show for HealthReport with fn output(self, logger) {
logger.write_string(
"HealthReport{healthy: " +
self.healthy.to_string() +
", documents: " +
self.document_count.to_string() +
", dimension: " +
self.dimension.to_string() +
", duplicate_ids: " +
self.duplicate_ids.to_string() +
", empty_ids: " +
self.empty_ids.to_string() +
", dimension_errors: " +
self.dimension_errors.to_string() +
", message: " +
self.message +
"}",
)
}
///|
/// Inspect a corpus without raising, so it can be used by readiness probes.
pub fn inspect_corpus(docs : Array[Document]) -> HealthReport {
if docs.length() == 0 {
return {
healthy: false,
document_count: 0,
dimension: 0,
duplicate_ids: 0,
empty_ids: 0,
dimension_errors: 0,
message: "empty corpus",
}
}
let dimension = docs[0].vector.length()
let ids = Map([])
let mut duplicate_ids = 0
let mut empty_ids = 0
let mut dimension_errors = 0
for doc in docs {
if doc.id.trim().length() == 0 {
empty_ids = empty_ids + 1
}
if ids.contains(doc.id) {
duplicate_ids = duplicate_ids + 1
}
ids.set(doc.id, true)
if doc.vector.length() != dimension || doc.vector.length() == 0 {
dimension_errors = dimension_errors + 1
}
}
let healthy = duplicate_ids == 0 && empty_ids == 0 && dimension_errors == 0
let message = if healthy { "ready" } else { "corpus requires repair" }
{
healthy,
document_count: docs.length(),
dimension,
duplicate_ids,
empty_ids,
dimension_errors,
message,
}
}
///|
/// Check that a query can be served by the corpus dimension.
pub fn query_is_compatible(
query : Array[Double],
docs : Array[Document],
) -> Bool {
if docs.length() == 0 || query.length() == 0 {
return false
}
query.length() == docs[0].vector.length()
}
///|
/// Return the distribution of vector dimensions in a possibly malformed corpus.
pub fn dimension_counts(docs : Array[Document]) -> Array[(Int, Int)] {
let counts = Map([])
for doc in docs {
let dimension = doc.vector.length()
let count = match counts.get(dimension) {
Some(value) => value
None => 0
}
counts.set(dimension, count + 1)
}
let result = []
for entry in counts {
result.push((entry.0, entry.1))
}
result
}
///|
/// Return the fraction of documents carrying a metadata key.
pub fn metadata_coverage(docs : Array[Document], key : String) -> Double {
if docs.length() == 0 {
return 0.0
}
let mut covered = 0
for doc in docs {
let mut found = false
for pair in doc.metadata {
if pair.0 == key {
found = true
break
}
}
if found {
covered = covered + 1
}
}
covered.to_double() / docs.length().to_double()
}
///|
/// Return a stable list of all metadata keys in first-seen order.
pub fn metadata_keys(docs : Array[Document]) -> Array[String] {
let keys = []
let seen = Map([])
for doc in docs {
for pair in doc.metadata {
if !seen.contains(pair.0) {
seen.set(pair.0, true)
keys.push(pair.0)
}
}
}
keys
}
///|
/// Return a copy of a corpus with one metadata pair appended to every document.
pub fn annotate_documents(
docs : Array[Document],
key : String,
value : String,
) -> Array[Document] {
let annotated = []
for doc in docs {
let metadata = []
for pair in doc.metadata {
metadata.push(pair)
}
metadata.push((key, value))
annotated.push(Document::new(doc.id, doc.vector, metadata))
}
annotated
}