///|
let manifest_file_name : String = "meta.msi"
///|
let manifest_magic : Bytes = b"MOONMETA"
///|
let manifest_format_version : UInt = 2U
///|
let legacy_manifest_format_version : UInt = 1U
///|
priv struct SegmentMeta {
id : Int
file_name : String
deleted_docs : ReadOnlyArray[DocId]
}
///|
priv struct IndexManifest {
generation : Int
next_segment_id : Int
segments : ReadOnlyArray[SegmentMeta]
}
///|
fn empty_manifest() -> IndexManifest {
{ generation: 0, next_segment_id: 0, segments: ReadOnlyArray::from_array([]) }
}
///|
fn segment_file_name(id : Int) -> String {
"segment-\{id}.msi"
}
///|
fn encode_manifest_payload(manifest : IndexManifest) -> Bytes {
let buffer = @buffer.Buffer()
buffer.write_int_be(manifest.generation)
buffer.write_int_be(manifest.next_segment_id)
buffer.write_int_be(manifest.segments.length())
for segment in manifest.segments {
buffer.write_int_be(segment.id)
write_string(buffer, segment.file_name)
buffer.write_int_be(segment.deleted_docs.length())
for doc_id in segment.deleted_docs {
buffer.write_int_be(doc_id.value)
}
}
buffer.to_bytes()
}
///|
fn encode_manifest(manifest : IndexManifest) -> Bytes {
let payload = encode_manifest_payload(manifest)
let buffer = @buffer.Buffer(
size_hint=segment_header_length + payload.length(),
)
buffer.write_bytes(manifest_magic[:])
buffer.write_uint_be(manifest_format_version)
buffer.write_int_be(payload.length())
buffer.write_uint_be(segment_checksum(payload))
buffer.write_bytes(payload[:])
buffer.to_bytes()
}
///|
fn validate_manifest(manifest : IndexManifest) -> Unit raise PersistenceError {
guard manifest.generation > 0 else {
raise PersistenceError::InvalidFormat("invalid manifest generation")
}
guard manifest.segments.length() > 0 else {
raise PersistenceError::InvalidFormat("manifest contains no segments")
}
let mut previous_id = -1
let names : Array[String] = []
for segment in manifest.segments {
guard segment.id > previous_id &&
segment.file_name == segment_file_name(segment.id) &&
names.search_by(name => name == segment.file_name) is None else {
raise PersistenceError::InvalidFormat("invalid manifest segment entry")
}
names.push(segment.file_name)
previous_id = segment.id
let mut previous_doc_id = -1
for doc_id in segment.deleted_docs {
guard doc_id.value > previous_doc_id else {
raise PersistenceError::InvalidFormat(
"manifest tombstones must be non-negative and strictly increasing",
)
}
previous_doc_id = doc_id.value
}
}
guard manifest.next_segment_id == previous_id + 1 else {
raise PersistenceError::InvalidFormat("invalid next segment id")
}
}
///|
fn decode_manifest_payload(
payload : Bytes,
version : UInt,
) -> IndexManifest raise PersistenceError {
let decoder = SegmentDecoder::new(payload)
let generation = decoder.read_count("manifest generation")
let next_segment_id = decoder.read_count("next segment id")
let segment_count = decoder.read_count("manifest segment count")
let segments : Array[SegmentMeta] = []
for _ in 0.. IndexManifest raise PersistenceError {
guard bytes.length() >= segment_header_length else {
raise PersistenceError::InvalidFormat("truncated manifest header")
}
for index in 0.. IndexManifest {
let id = manifest.next_segment_id
let segments : Array[SegmentMeta] = []
for segment in manifest.segments {
segments.push(segment)
}
segments.push({
id,
file_name: segment_file_name(id),
deleted_docs: ReadOnlyArray::from_array([]),
})
{
generation: manifest.generation + 1,
next_segment_id: id + 1,
segments: ReadOnlyArray::from_array(segments),
}
}