///|
/// Descriptive statistics for a vector corpus.
pub(all) struct VectorSummary {
count : Int
dimension : Int
min_norm : Double
max_norm : Double
mean_norm : Double
mean_vector : Array[Double]
}
///|
pub impl Show for VectorSummary with fn output(self, logger) {
logger.write_string(
"VectorSummary{count: " +
self.count.to_string() +
", dimension: " +
self.dimension.to_string() +
", min_norm: " +
self.min_norm.to_string() +
", max_norm: " +
self.max_norm.to_string() +
", mean_norm: " +
self.mean_norm.to_string() +
"}",
)
}
///|
/// Calculate corpus-level norms and coordinate means.
pub fn summarize_vectors(
vectors : Array[Array[Double]],
) -> VectorSummary raise VectorError {
if vectors.length() == 0 {
raise EmptyVector
}
let dimension = vectors[0].length()
if dimension == 0 {
raise EmptyVector
}
let mean = Array::make(dimension, 0.0)
let mut min_norm = -1.0
let mut max_norm = 0.0
let mut norm_total = 0.0
for vector in vectors {
if vector.length() != dimension {
raise DimensionMismatch("Vector dimensions differ in summary")
}
let mut squared = 0.0
for i = 0; i < dimension; i = i + 1 {
mean[i] = mean[i] + vector[i]
squared = squared + vector[i] * vector[i]
}
let norm = squared.sqrt()
if min_norm < 0.0 || norm < min_norm {
min_norm = norm
}
if norm > max_norm {
max_norm = norm
}
norm_total = norm_total + norm
}
for i = 0; i < dimension; i = i + 1 {
mean[i] = mean[i] / vectors.length().to_double()
}
{
count: vectors.length(),
dimension,
min_norm,
max_norm,
mean_norm: norm_total / vectors.length().to_double(),
mean_vector: mean,
}
}
///|
/// Calculate the population variance for every coordinate.
pub fn coordinate_variance(
vectors : Array[Array[Double]],
) -> Array[Double] raise VectorError {
let summary = summarize_vectors(vectors)
let variance = Array::make(summary.dimension, 0.0)
for vector in vectors {
for i = 0; i < summary.dimension; i = i + 1 {
let delta = vector[i] - summary.mean_vector[i]
variance[i] = variance[i] + delta * delta
}
}
for i = 0; i < variance.length(); i = i + 1 {
variance[i] = variance[i] / summary.count.to_double()
}
variance
}
///|
/// Calculate the average cosine similarity among all distinct pairs.
pub fn average_pair_similarity(
vectors : Array[Array[Double]],
) -> Double raise VectorError {
if vectors.length() < 2 {
return 1.0
}
let mut total = 0.0
let mut pairs = 0
for i = 0; i < vectors.length(); i = i + 1 {
for j = i + 1; j < vectors.length(); j = j + 1 {
total = total + cosine_similarity(vectors[i], vectors[j])
pairs = pairs + 1
}
}
total / pairs.to_double()
}
///|
/// Return the nearest document distance for a query, or no value for empty data.
pub fn nearest_distance(
query : Array[Double],
docs : Array[Document],
metric : DistanceMetric,
) -> Double? raise VectorError {
if docs.length() == 0 {
return None
}
let mut best = -1.0
for doc in docs {
let score = calculate_distance(query, doc.vector, metric)
let lower_is_better = match metric {
Euclidean | Manhattan => true
_ => false
}
if best < 0.0 ||
(lower_is_better && score < best) ||
(!lower_is_better && score > best) {
best = score
}
}
Some(best)
}
///|
/// Keep documents within a distance/ranking threshold.
pub fn search_with_threshold(
docs : Array[Document],
query : Array[Double],
metric : DistanceMetric,
threshold : Double,
) -> Array[SearchResult] raise VectorError {
let result = []
let higher_is_better = match metric {
Cosine | DotProduct => true
_ => false
}
for doc in docs {
let score = calculate_distance(query, doc.vector, metric)
let accepted = if higher_is_better {
score >= threshold
} else {
score <= threshold
}
if accepted {
result.push({ id: doc.id, score, metadata: doc.metadata })
}
}
let ascending = match metric {
Euclidean | Manhattan => true
_ => false
}
sort_results(result, ascending)
result
}
///|
/// Return a compact deterministic fingerprint of a vector for cache keys.
pub fn vector_fingerprint(vector : Array[Double]) -> String {
let sb = StringBuilder::new()
for i = 0; i < vector.length(); i = i + 1 {
if i > 0 {
sb.write_string(",")
}
sb.write_string(vector[i].to_string())
}
sb.to_string()
}