///|
/// Byte, structural syntax overhead, and BPE-aware heuristic token metrics for
/// a serialized text payload.
#derive(Eq, @debug.Debug)
pub(all) struct TextMetrics {
  utf8_bytes : Int32
  chars : Int32
  structural_bytes : Int32
  estimated_tokens : Int32
}

///|
fn is_ascii_letter(code : Int32) -> Bool {
  (code >= 0x41 && code <= 0x5A) || (code >= 0x61 && code <= 0x7A)
}

///|
fn is_ascii_digit(code : Int32) -> Bool {
  code >= 0x30 && code <= 0x39
}

///|
fn is_ident_char(code : Int32) -> Bool {
  is_ascii_letter(code) || is_ascii_digit(code) || code == 0x5F
}

///|
fn is_structural_ascii(code : Int32) -> Bool {
  code == 0x22 || // '"'
  code == 0x7B || // '{'
  code == 0x7D || // '}'
  code == 0x5B || // '['
  code == 0x5D || // ']'
  code == 0x3A || // ':'
  code == 0x2C || // ','
  code == 0x7C || // '|'
  code == 0x0A || // '\n'
  code == 0x09 // '\t'
}

///|
fn is_json_punct_cluster(code : Int32) -> Bool {
  code == 0x22 || // '"'
  code == 0x7B || // '{'
  code == 0x7D || // '}'
  code == 0x5B || // '['
  code == 0x5D || // ']'
  code == 0x3A || // ':'
  code == 0x2C // ','
}

///|
/// Computes exact UTF-8 byte count, Unicode scalar count, syntax overhead
/// bytes, and a deterministic BPE-aware token estimate (modeled on o200k/cl100k
/// chunking: <=8-char words with optional leading space, <=3-digit number
/// chunks, merged JSON punctuation clusters like `{"`, `":"`, `","`, `"},{"`,
/// newline+indent runs, and per-scalar CJK/emoji).
pub fn measure_text(text : String) -> TextMetrics {
  let utf8_bytes = @utf8.encode(text).length()
  let scalars : Array[Int32] = []
  for ch in text {
    scalars.push(ch.to_int())
  }
  let n = scalars.length()
  let mut structural = 0
  for i in 0.. 0 &&
        (scalars[i - 1] == 0x3A || scalars[i - 1] == 0x2C)
      if at_line_start || after_colon_or_comma {
        structural += 1
      }
    }
  }
  let mut tokens = 0
  let mut i = 0
  while i < n {
    let c = scalars[i]
    // Optional leading single space merged into word/number token (standard BPE)
    if c == 0x20 && i + 1 < n {
      let next = scalars[i + 1]
      if is_ascii_letter(next) {
        i += 1
        let start = i
        while i < n && is_ident_char(scalars[i]) && i - start < 8 {
          i += 1
        }
        tokens += 1
        continue
      }
      if is_ascii_digit(next) {
        i += 1
        let start = i
        while i < n && is_ascii_digit(scalars[i]) && i - start < 3 {
          i += 1
        }
        tokens += 1
        continue
      }
    }
    if is_ascii_letter(c) || c == 0x5F {
      let start = i
      while i < n && is_ident_char(scalars[i]) && i - start < 8 {
        i += 1
      }
      tokens += 1
    } else if is_ascii_digit(c) {
      let start = i
      while i < n && is_ascii_digit(scalars[i]) && i - start < 3 {
        i += 1
      }
      tokens += 1
    } else if c == 0x20 || c == 0x09 {
      let start = i
      while i < n && (scalars[i] == 0x20 || scalars[i] == 0x09) && i - start < 4 {
        i += 1
      }
      tokens += 1
    } else if c == 0x0A {
      // '\n' + up to 2 spaces merges in typical BPE; deeper indentation costs extra tokens
      i += 1
      let sp_start = i
      while i < n && scalars[i] == 0x20 && i - sp_start < 2 {
        i += 1
      }
      tokens += 1
    } else if is_json_punct_cluster(c) {
      // Modern tokenizers (o200k/cl100k) merge common JSON delimiter sequences
      // such as `{"`, `":"`, `","`, `"},{"`, `":[`, `"]` into single tokens (up to 3-4 chars).
      let start = i
      while i < n && is_json_punct_cluster(scalars[i]) && i - start < 4 {
        i += 1
      }
      if i < n && scalars[i] == 0x20 && i - start <= 2 {
        i += 1
      }
      tokens += 1
    } else if c < 0x80 {
      if (c == 0x7C || c == 0x2D) && i + 1 < n && scalars[i + 1] == 0x20 {
        i += 2
      } else {
        i += 1
      }
      tokens += 1
    } else if c >= 0x4E00 && c <= 0x9FFF {
      let mut cjk_run = 0
      while i < n && scalars[i] >= 0x4E00 && scalars[i] <= 0x9FFF {
        cjk_run += 1
        i += 1
      }
      tokens += cjk_run + cjk_run / 4
    } else {
      i += 1
      tokens += if c > 0xFFFF { 2 } else { 1 }
    }
  }
  {
    utf8_bytes,
    chars: n,
    structural_bytes: structural,
    estimated_tokens: tokens,
  }
}