///|
fn update_merged_field_stats(
stats : Array[FieldStats],
lengths : ReadOnlyArray[FieldLength],
) -> Unit {
for entry in lengths {
match stats.search_by(current => current.field_id == entry.field_id) {
Some(index) => {
stats[index].document_count += 1
stats[index].total_length += entry.length
}
None =>
stats.push({
field_id: entry.field_id,
document_count: 1,
total_length: entry.length,
})
}
}
}
///|
/// Combines live documents by copying immutable index structures and assigning
/// consecutive DocIds in Segment order. No field is re-analyzed.
pub fn merge_snapshot_segments(
snapshots : ReadOnlyArray[SnapshotSegment],
) -> Segment {
guard snapshots.length() > 0 else {
abort("cannot merge an empty segment set")
}
let mappings : Array[Array[Int]] = []
let stored_documents : Array[StoredDocument] = []
let document_values : Array[ReadOnlyArray[FieldValueEntry]] = []
let document_field_lengths : Array[ReadOnlyArray[FieldLength]] = []
let field_stats : Array[FieldStats] = []
let mut next_doc_id = 0
for snapshot in snapshots {
let mapping = Array::make(snapshot.segment.doc_count(), -1)
for old_doc_index in 0.. document
None => abort("live document is missing stored fields")
},
)
document_values.push(snapshot.segment.document_values[old_doc_index])
let lengths = snapshot.segment.document_field_lengths[old_doc_index]
document_field_lengths.push(lengths)
update_merged_field_stats(field_stats, lengths)
next_doc_id += 1
}
mappings.push(mapping)
}
let indexed_terms : Array[Term] = []
let posting_lists : Array[Array[Posting]] = []
for segment_index in 0.. indexed == term) {
Some(index) => index
None => {
indexed_terms.push(term)
posting_lists.push([])
indexed_terms.length() - 1
}
}
for posting in segment.posting_lists[term_index] {
let new_doc_index = mapping[posting.doc_id.value]
if new_doc_index >= 0 {
posting_lists[output_index].push({
doc_id: DocId::new(new_doc_index),
term_freq: posting.term_freq,
positions: posting.positions,
position_lengths: posting.position_lengths,
start_offsets: posting.start_offsets,
end_offsets: posting.end_offsets,
value_indexes: posting.value_indexes,
})
}
}
}
}
let live_terms : Array[Term] = []
let live_posting_lists : Array[ReadOnlyArray[Posting]] = []
for term_index in 0.. 0 {
live_terms.push(indexed_terms[term_index])
live_posting_lists.push(
ReadOnlyArray::from_array(posting_lists[term_index]),
)
}
}
let (sorted_terms, sorted_posting_lists) = sort_term_postings(
live_terms, live_posting_lists,
)
let frozen_document_values = ReadOnlyArray::from_array(document_values)
{
schema: snapshots[0].segment.schema,
indexed_terms: sorted_terms,
posting_lists: sorted_posting_lists,
stored_field_blocks: build_stored_field_blocks(stored_documents),
document_values: frozen_document_values,
fast_fields: build_fast_fields(
snapshots[0].segment.schema,
frozen_document_values,
),
document_field_lengths: ReadOnlyArray::from_array(document_field_lengths),
field_stats: ReadOnlyArray::from_array(field_stats),
document_count: next_doc_id,
}
}