///|
/// Growable little-endian byte sink.
priv struct ByteBuf {
  mut buf : FixedArray[Byte]
  mut len : Int32
}

///|
fn ByteBuf::new() -> ByteBuf {
  { buf: FixedArray::make(256, b'\x00'), len: 0, }
}

///|
fn ByteBuf::ensure(self : ByteBuf, extra : Int32) -> Unit {
  if self.len + extra <= self.buf.length() {
    return
  }
  let mut cap = self.buf.length() * 2
  while cap < self.len + extra {
    cap = cap * 2
  }
  let grown : FixedArray[Byte] = FixedArray::make(cap, b'\x00')
  FixedArray::unsafe_blit(grown, 0, self.buf, 0, self.len)
  self.buf = grown
}

///|
fn ByteBuf::put_byte(self : ByteBuf, v : Byte) -> Unit {
  self.ensure(1)
  self.buf[self.len] = v
  self.len += 1
}

///|
fn ByteBuf::put_u32(self : ByteBuf, v : Int32) -> Unit {
  self.ensure(4)
  self.buf[self.len] = (v & 0xFF).to_byte()
  self.buf[self.len + 1] = ((v >> 8) & 0xFF).to_byte()
  self.buf[self.len + 2] = ((v >> 16) & 0xFF).to_byte()
  self.buf[self.len + 3] = ((v >> 24) & 0xFF).to_byte()
  self.len += 4
}

///|
fn ByteBuf::put_u64(self : ByteBuf, v : UInt64) -> Unit {
  self.put_byte(v.to_byte())
  self.put_byte((v >> 8).to_byte())
  self.put_byte((v >> 16).to_byte())
  self.put_byte((v >> 24).to_byte())
  self.put_byte((v >> 32).to_byte())
  self.put_byte((v >> 40).to_byte())
  self.put_byte((v >> 48).to_byte())
  self.put_byte((v >> 56).to_byte())
}

///|
/// Appends `count` f32 values in one bounds-checked pass.
fn ByteBuf::put_f32_bulk(
  self : ByteBuf,
  src : FixedArray[Float],
  count : Int32,
) -> Unit {
  self.ensure(count * 4)
  let mut i = 0
  while i < count {
    let bits = src.unsafe_get(i).reinterpret_as_uint()
    self.buf[self.len] = bits.to_byte()
    self.buf[self.len + 1] = (bits >> 8).to_byte()
    self.buf[self.len + 2] = (bits >> 16).to_byte()
    self.buf[self.len + 3] = (bits >> 24).to_byte()
    self.len += 4
    i += 1
  }
}

///|
/// Appends `count` bytes in one bounds-checked pass.
fn ByteBuf::put_bytes_bulk(
  self : ByteBuf,
  src : FixedArray[Byte],
  count : Int32,
) -> Unit {
  self.ensure(count)
  let mut i = 0
  while i < count {
    self.buf[self.len + i] = src.unsafe_get(i)
    i += 1
  }
  self.len += count
}

///|
/// Appends `count` little-endian u32 values from `src` in one
/// bounds-checked pass.
fn ByteBuf::put_i32_bulk(
  self : ByteBuf,
  src : Array[Int32],
  count : Int32,
) -> Unit {
  self.ensure(count * 4)
  let mut i = 0
  while i < count {
    let v = src.unsafe_get(i)
    self.buf[self.len] = (v & 0xFF).to_byte()
    self.buf[self.len + 1] = ((v >> 8) & 0xFF).to_byte()
    self.buf[self.len + 2] = ((v >> 16) & 0xFF).to_byte()
    self.buf[self.len + 3] = ((v >> 24) & 0xFF).to_byte()
    self.len += 4
    i += 1
  }
}

///|
/// Appends `count` little-endian u64 values from `src` in one
/// bounds-checked pass.
fn ByteBuf::put_u64_bulk(
  self : ByteBuf,
  src : Array[UInt64],
  count : Int32,
) -> Unit {
  self.ensure(count * 8)
  let mut i = 0
  while i < count {
    let v = src.unsafe_get(i)
    let mut b = 0
    while b < 8 {
      self.buf[self.len + i * 8 + b] = (v >> (b * 8)).to_byte()
      b += 1
    }
    i += 1
  }
  self.len += count * 8
}

///|
fn ByteBuf::to_fixed(self : ByteBuf) -> FixedArray[Byte] {
  let out : FixedArray[Byte] = FixedArray::make(self.len, b'\x00')
  FixedArray::unsafe_blit(out, 0, self.buf, 0, self.len)
  out
}

///|
/// Little-endian cursor over a byte buffer.
priv struct ByteReader {
  data : FixedArray[Byte]
  mut pos : Int32
}

///|
fn ByteReader::new(data : FixedArray[Byte]) -> ByteReader {
  { data, pos: 0, }
}

///|
fn ByteReader::byte(self : ByteReader) -> Int32 raise HnswError {
  if self.pos >= self.data.length() {
    raise CorruptData("unexpected end of data")
  }
  let b = self.data[self.pos]
  self.pos += 1
  b.to_int() & 0xFF
}

///|
fn ByteReader::u32(self : ByteReader) -> Int32 raise HnswError {
  let b0 = self.byte()
  let b1 = self.byte()
  let b2 = self.byte()
  let b3 = self.byte()
  b0 | (b1 << 8) | (b2 << 16) | (b3 << 24)
}

///|
fn ByteReader::u64(self : ByteReader) -> UInt64 raise HnswError {
  let mut v : UInt64 = 0UL
  let mut i = 0
  while i < 8 {
    v = v | (self.byte().to_uint64() << (i * 8))
    i += 1
  }
  v
}

///|
/// Reads labels with one length check instead of eight checked byte reads
/// per label. The wire representation remains little-endian u64.
fn ByteReader::u64_bulk(
  self : ByteReader,
  dst : Array[UInt64],
  count : Int32,
) -> Unit raise HnswError {
  if count < 0 ||
    count.to_int64() * 8L > (self.data.length() - self.pos).to_int64() {
    raise CorruptData("unexpected end of data")
  }
  for _ in 0.. Unit raise HnswError {
  if count < 0 ||
    count.to_int64() * 4L > (self.data.length() - self.pos).to_int64() {
    raise CorruptData("unexpected end of data")
  }
  let mut i = 0
  while i < count {
    let b0 = self.data.unsafe_get(self.pos).to_int() & 0xFF
    let b1 = self.data.unsafe_get(self.pos + 1).to_int() & 0xFF
    let b2 = self.data.unsafe_get(self.pos + 2).to_int() & 0xFF
    let b3 = self.data.unsafe_get(self.pos + 3).to_int() & 0xFF
    dst[i] = Float::reinterpret_from_int(
      b0 | (b1 << 8) | (b2 << 16) | (b3 << 24),
    )
    self.pos += 4
    i += 1
  }
}

///|
/// Reads `count` bytes in one bounds-checked pass into `dst`.
fn ByteReader::bytes_bulk(
  self : ByteReader,
  dst : FixedArray[Byte],
  count : Int32,
) -> Unit raise HnswError {
  if count < 0 || count > self.data.length() - self.pos {
    raise CorruptData("unexpected end of data")
  }
  let mut i = 0
  while i < count {
    dst[i] = self.data.unsafe_get(self.pos + i)
    i += 1
  }
  self.pos += count
}

///|
/// Reads `count` little-endian u32 values in one bounds-checked pass,
/// appending them to `dst`.
fn ByteReader::i32_bulk(
  self : ByteReader,
  dst : Array[Int32],
  count : Int32,
) -> Unit raise HnswError {
  if count < 0 ||
    count.to_int64() * 4L > (self.data.length() - self.pos).to_int64() {
    raise CorruptData("unexpected end of data")
  }
  let mut i = 0
  while i < count {
    let b0 = self.data.unsafe_get(self.pos).to_int() & 0xFF
    let b1 = self.data.unsafe_get(self.pos + 1).to_int() & 0xFF
    let b2 = self.data.unsafe_get(self.pos + 2).to_int() & 0xFF
    let b3 = self.data.unsafe_get(self.pos + 3).to_int() & 0xFF
    dst.push(b0 | (b1 << 8) | (b2 << 16) | (b3 << 24))
    self.pos += 4
    i += 1
  }
}

///|
fn read_magic(r : ByteReader) -> Unit raise HnswError {
  if r.byte() != b'M'.to_int() ||
    r.byte() != b'H'.to_int() ||
    r.byte() != b'N'.to_int() ||
    r.byte() != b'S'.to_int() ||
    r.byte() != b'W'.to_int() {
    raise CorruptData("bad magic, not a moonhnsw payload")
  }
}

///|
/// Serializes the whole index (parameters, vectors and graph) into a
/// self-contained byte buffer. The payload can be written to a file or sent
/// over the network and restored with `Hnsw::load`; searching a loaded index
/// yields exactly the same results as the original.
pub fn Hnsw::save(self : Hnsw) -> FixedArray[Byte] {
  let b = ByteBuf::new()
  b.put_byte(b'M')
  b.put_byte(b'H')
  b.put_byte(b'N')
  b.put_byte(b'S')
  b.put_byte(b'W')
  b.put_byte(b'\x05')
  b.put_u32(self.dim)
  b.put_byte(
    match self.distance {
      L2 => b'\x00'
      InnerProduct => b'\x01'
      Cosine => b'\x02'
    },
  )
  b.put_u32(self.params.m)
  b.put_u32(self.params.ef_construction)
  b.put_u64(self.params.seed)
  b.put_u32(self.entry)
  b.put_u32(self.max_level)
  b.put_u32(self.count)
  b.put_u64(self.rng_state)
  let mut i = 0
  while i < self.count {
    b.put_byte(if self.alive[i] { b'\x01' } else { b'\x00' })
    i += 1
  }
  b.put_u64_bulk(self.labels, self.labels.length())
  // v5: storage precision. SQ8 stores its bounds and, once frozen, one byte
  // per dimension instead of full floats.
  b.put_byte(
    match self.storage {
      Float32 => b'\x00'
      SQ8 => b'\x01'
    },
  )
  if self.storage is SQ8 {
    b.put_byte(if self.q_frozen { b'\x01' } else { b'\x00' })
    b.put_f32_bulk(self.q_mins, self.dim)
    b.put_f32_bulk(self.q_maxs, self.dim)
    b.put_f32_bulk(self.q_scales, self.dim)
  }
  let total = self.count * self.dim
  if self.q_active() {
    b.put_bytes_bulk(self.quants, total)
  } else {
    b.put_f32_bulk(self.vectors, total)
  }
  // Flatten the link section into one u32 stream for a single bulk write.
  let stream : Array[Int32] = []
  for node in self.links {
    stream.push(node.length())
    for lst in node {
      stream.push(lst.length())
      for e in lst {
        stream.push(e)
      }
    }
  }
  b.put_i32_bulk(stream, stream.length())
  // v4 trailer: CRC-32 over everything above, so flipped bits and silent
  // truncation are detected on load instead of surfacing as bad searches.
  b.put_u32(crc32_of(b.buf, b.len).reinterpret_as_int())
  b.to_fixed()
}

///|
/// Restores an index produced by `Hnsw::save`. Raises `CorruptData` when the
/// payload is truncated or inconsistent. The returned index keeps accepting
/// new insertions.
pub fn Hnsw::load(data : FixedArray[Byte]) -> Hnsw raise HnswError {
  let r = ByteReader::new(data)
  read_magic(r)
  let version = r.byte()
  if version != 1 &&
    version != 2 &&
    version != 3 &&
    version != 4 &&
    version != 5 {
    raise CorruptData("unsupported format version \{version}")
  }
  // Validate the trailer before trusting payload lengths for allocations.
  if version >= 4 {
    if data.length() < 10 {
      raise CorruptData("payload too short for checksum")
    }
    let trailer_pos = data.length() - 4
    let trailer = ByteReader::{ data, pos: trailer_pos, }
    if trailer.u32() != crc32_of(data, trailer_pos).reinterpret_as_int() {
      raise CorruptData("checksum mismatch")
    }
  }
  let dim = r.u32()
  if dim < 1 || dim > 1000000 {
    raise CorruptData("dim out of range")
  }
  let metric_code = r.byte()
  let distance = match metric_code {
    0 => L2
    1 => InnerProduct
    2 => Cosine
    _ => raise CorruptData("unknown distance metric code \{metric_code}")
  }
  let m = r.u32()
  let ef_construction = r.u32()
  let seed = r.u64()
  let entry = r.u32()
  let max_level = r.u32()
  let count = r.u32()
  let rng_state = r.u64()
  if count < 0 || count.to_int64() * dim.to_int64() > 2147483647L {
    raise CorruptData("vector count or dimensions out of range")
  }
  if count.to_int64() * dim.to_int64() > data.length().to_int64() {
    raise CorruptData("vector data exceeds payload length")
  }
  // Version 2 carries one alive byte per vector (tombstone deletion);
  // version 1 payloads predate deletion and are fully alive.
  let alive : FixedArray[Bool] = FixedArray::make(count, true)
  let mut alive_count = count
  if version >= 2 {
    alive_count = 0
    let mut i = 0
    while i < count {
      let flag = r.byte()
      if flag > 1 {
        raise CorruptData("invalid alive flag")
      }
      alive[i] = flag == 1
      if flag == 1 {
        alive_count += 1
      }
      i += 1
    }
  }
  // Version 3 carries one label per vector; earlier versions used the id.
  let labels : Array[UInt64] = []
  if version >= 3 {
    r.u64_bulk(labels, count)
  } else {
    let mut i = 0
    while i < count {
      labels.push(i.to_uint64())
      i += 1
    }
  }
  if m < 2 || m > 1073741823 || ef_construction < 1 {
    raise CorruptData("invalid graph parameters")
  }
  if count == 0 {
    if entry != -1 || max_level != -1 {
      raise CorruptData("empty index must have no entry point")
    }
  } else {
    if entry < 0 || entry >= count {
      raise CorruptData("entry point out of range")
    }
    if max_level < 0 || max_level > 1024 {
      raise CorruptData("non-empty index must have a level")
    }
  }
  // Version 5 carries the storage precision and, for SQ8, the quantization
  // bounds and byte-per-dimension values.
  let storage : Storage = if version >= 5 {
    match r.byte() {
      0 => Float32
      1 => SQ8
      code => raise CorruptData("unknown storage code \{code}")
    }
  } else {
    Float32
  }
  let mut q_frozen = false
  let q_mins : FixedArray[Float] = FixedArray::make(
    dim,
    Float::reinterpret_from_int(0x7F7FFFFF),
  )
  let q_maxs : FixedArray[Float] = FixedArray::make(
    dim,
    -Float::reinterpret_from_int(0x7F7FFFFF),
  )
  let q_scales : FixedArray[Float] = FixedArray::make(dim, 1.0)
  if version >= 5 && storage is SQ8 {
    let frozen_flag = r.byte()
    if frozen_flag > 1 {
      raise CorruptData("invalid SQ8 frozen flag")
    }
    q_frozen = frozen_flag == 1
    r.f32_bulk(q_mins, dim)
    r.f32_bulk(q_maxs, dim)
    r.f32_bulk(q_scales, dim)
  }
  let total = count * dim
  let quantized = storage is SQ8 && q_frozen
  let value_bytes = if quantized { 1L } else { 4L }
  if total.to_int64() * value_bytes > (data.length() - r.pos).to_int64() {
    raise CorruptData("vector data exceeds payload length")
  }
  let vectors : FixedArray[Float] = if quantized {
    FixedArray::make(0, 0.0)
  } else {
    FixedArray::make(total, 0.0)
  }
  let quants : FixedArray[Byte] = if quantized {
    FixedArray::make(total, 0)
  } else {
    FixedArray::make(0, 0)
  }
  if quantized {
    r.bytes_bulk(quants, total)
  } else {
    r.f32_bulk(vectors, total)
    for value in vectors {
      let v = value.to_double()
      if !(v > -@double.infinity && v < @double.infinity) {
        raise CorruptData("non-finite vector value")
      }
    }
  }
  let links : Array[Array[Array[Int32]]] = []
  for _ in 0.. max_level + 1 {
      raise CorruptData("node level out of range")
    }
    let node : Array[Array[Int32]] = []
    for _ in 0.. count {
        raise CorruptData("link list length out of range")
      }
      if n_links.to_int64() * 4L > (data.length() - r.pos).to_int64() {
        raise CorruptData("link list exceeds payload length")
      }
      let lst : Array[Int32] = []
      r.i32_bulk(lst, n_links)
      for e in lst {
        if e < 0 || e >= count {
          raise CorruptData("link id out of range")
        }
      }
      node.push(lst)
    }
    links.push(node)
  }
  // The checksum was checked before allocation. Consume exactly its trailer
  // and reject extra bytes even when the checksum is valid.
  if version >= 4 {
    if r.pos != data.length() - 4 {
      raise CorruptData("trailing data before checksum")
    }
    let _ = r.u32()
  } else if r.pos != data.length() {
    raise CorruptData("trailing data")
  }
  let visited_len = if count < 64 { 64 } else { count }
  let idx = Hnsw::{
    dim,
    distance,
    params: HnswParams::new(m~, ef_construction~, seed~, storage~),
    normalize: match distance {
      Cosine => true
      _ => false
    },
    level_mult: 1.0 / @math.ln(m.to_double()),
    entry,
    max_level,
    count,
    alive_count,
    alive,
    labels,
    storage,
    vectors,
    f_capacity: if quantized {
      0
    } else {
      count
    },
    quants,
    q_capacity: if quantized {
      count
    } else {
      0
    },
    q_mins,
    q_maxs,
    q_scales,
    q_frozen,
    f_scratch: FixedArray::make(dim, 0.0),
    cand_items: [],
    result_items: [],
    links,
    visited: FixedArray::make(visited_len, 0),
    stamp: 0,
    rng_state,
    dist_calls: 0L,
  }
  if storage is SQ8 && !q_frozen {
    // Keep rejecting malformed saved bounds even though they are replaced.
    idx.verify()
    // Earlier payloads could persist ±3e38 sentinels even when finite f32
    // components lay beyond them. Rebuild the running bounds from the stored
    // pre-freeze floats so continued insertion learns the true range.
    let largest = Float::reinterpret_from_int(0x7F7FFFFF)
    for j in 0..