///|
/// Read-only collection statistics for one Searcher snapshot.
///
/// Query implementations use these global values while constructing Weight.
/// A Weight can then create comparable Scorers for every immutable Segment.
pub struct SearchStatistics {
segments : ReadOnlyArray[SnapshotSegment]
live_document_count : Int
field_cache_ids : Array[FieldId]
field_cache_doc_counts : Array[Int]
field_cache_total_lengths : Array[Int]
term_cache_terms : Array[Term]
term_cache_frequencies : Array[Int]
}
///|
fn SearchStatistics::from_segments(
segments : ReadOnlyArray[SnapshotSegment],
) -> SearchStatistics {
let mut live_document_count = 0
for snapshot in segments {
live_document_count += snapshot.live_doc_count()
}
{
segments,
live_document_count,
field_cache_ids: [],
field_cache_doc_counts: [],
field_cache_total_lengths: [],
term_cache_terms: [],
term_cache_frequencies: [],
}
}
///|
pub fn SearchStatistics::segment_count(self : SearchStatistics) -> Int {
self.segments.length()
}
///|
pub fn SearchStatistics::doc_count(self : SearchStatistics) -> Int {
self.live_document_count
}
///|
fn SearchStatistics::field_statistics(
self : SearchStatistics,
field_id : FieldId,
) -> (Int, Int) {
match self.field_cache_ids.search_by(cached => cached == field_id) {
Some(index) =>
return (
self.field_cache_doc_counts[index],
self.field_cache_total_lengths[index],
)
None => ()
}
let mut document_count = 0
let mut total_length = 0
for snapshot in self.segments {
for doc_index in 0.. Int {
self.field_statistics(field_id).0
}
///|
pub fn SearchStatistics::document_frequency(
self : SearchStatistics,
term : Term,
) -> Int {
match self.term_cache_terms.search_by(cached => cached == term) {
Some(index) => return self.term_cache_frequencies[index]
None => ()
}
let mut count = 0
for snapshot in self.segments {
for posting in snapshot.segment.postings_for(term) {
if !snapshot.is_deleted(posting.doc_id) {
count += 1
}
}
}
self.term_cache_terms.push(term)
self.term_cache_frequencies.push(count)
count
}
///|
pub fn SearchStatistics::average_field_length(
self : SearchStatistics,
field_id : FieldId,
) -> Double {
let (document_count, total_length) = self.field_statistics(field_id)
if document_count == 0 {
0.0
} else {
total_length.to_double() / document_count.to_double()
}
}
///|
/// BM25 scorer using the common defaults k1=1.2 and b=0.75.
///
/// Query Weight implementations construct it from snapshot-wide statistics
/// and use it while producing comparable segment-local Scorer results.
pub struct Bm25Scorer {
field_id : FieldId
idf : Double
average_length : Double
k1 : Double
b : Double
}
///|
/// Tunable BM25 parameters. Defaults match Tantivy/Lucene conventions.
pub struct Bm25Config {
k1 : Double
b : Double
}
///|
pub fn Bm25Config::new(k1 : Double, b : Double) -> Bm25Config {
guard k1 >= 0.0 else { abort("BM25 k1 must be non-negative") }
guard b >= 0.0 && b <= 1.0 else {
abort("BM25 b must be between zero and one")
}
{ k1, b }
}
///|
pub fn Bm25Config::default() -> Bm25Config {
{ k1: 1.2, b: 0.75 }
}
///|
pub fn Bm25Config::k1(self : Bm25Config) -> Double {
self.k1
}
///|
pub fn Bm25Config::b(self : Bm25Config) -> Double {
self.b
}
///|
priv struct EmptyScorer {}
///|
impl Scorer for EmptyScorer with fn advance(_self) {
false
}
///|
impl Scorer for EmptyScorer with fn advance_to(_self, _target) {
false
}
///|
impl Scorer for EmptyScorer with fn doc(_self) {
DocId::new(-1)
}
///|
impl Scorer for EmptyScorer with fn score(_self) {
0.0
}
///|
/// Builds single-Segment BM25 state. Retained for M1-M3 API compatibility.
pub fn Bm25Scorer::new(segment : Segment, term : Term) -> Bm25Scorer {
Bm25Scorer::from_statistics(
SearchStatistics::from_segments(
ReadOnlyArray::from_array([live_snapshot(segment)]),
),
term,
)
}
///|
/// Builds query state from the complete Searcher snapshot.
pub fn Bm25Scorer::from_statistics(
statistics : SearchStatistics,
term : Term,
) -> Bm25Scorer {
Bm25Scorer::from_statistics_with_config(
statistics,
term,
Bm25Config::default(),
)
}
///|
pub fn Bm25Scorer::from_statistics_with_config(
statistics : SearchStatistics,
term : Term,
config : Bm25Config,
) -> Bm25Scorer {
let document_count = statistics.field_doc_count(term.field_id)
let document_frequency = statistics.document_frequency(term)
let idf = if document_count <= 0 || document_frequency <= 0 {
0.0
} else {
let n = document_count.to_double()
let df = document_frequency.to_double()
@math.ln(1.0 + (n - df + 0.5) / (df + 0.5))
}
{
field_id: term.field_id,
idf,
average_length: statistics.average_field_length(term.field_id),
k1: config.k1,
b: config.b,
}
}
///|
pub fn Bm25Scorer::idf(self : Bm25Scorer) -> Double {
self.idf
}
///|
pub fn Bm25Scorer::average_field_length(self : Bm25Scorer) -> Double {
self.average_length
}
///|
pub fn Bm25Scorer::score(
self : Bm25Scorer,
posting : Posting,
segment : Segment,
) -> Double {
let field_length = segment.field_length(posting.doc_id, self.field_id)
if self.idf <= 0.0 || self.average_length <= 0.0 || field_length <= 0 {
return 0.0
}
let term_frequency = posting.term_freq.to_double()
let length_normalization = 1.0 -
self.b +
self.b * field_length.to_double() / self.average_length
let denominator = term_frequency + self.k1 * length_normalization
self.idf * term_frequency * (self.k1 + 1.0) / denominator
}