///|
/// A token emitted by the lightweight dictionary-aware tokenizer.
pub(all) struct Token {
  text : String
  start : Int
  end : Int
  matched : Bool
  label : String
} derive(Debug, Eq, ToJson)

///|
/// Tokenize text into dictionary hits and single-character gaps.
pub fn Lexicon::tokenize(self : Lexicon, text : String) -> Array[Token] {
  let out : Array[Token] = []
  let mut cursor = 0
  for hit in self.segment_longest(text) {
    while cursor < hit.start {
      out.push({
        text: text.to_array()[cursor].to_string(),
        start: cursor,
        end: cursor + 1,
        matched: false,
        label: "",
      })
      cursor = cursor + 1
    }
    out.push({
      text: hit.keyword,
      start: hit.start,
      end: hit.end,
      matched: true,
      label: hit.label,
    })
    cursor = hit.end
  }
  while cursor < text.length() {
    out.push({
      text: text.to_array()[cursor].to_string(),
      start: cursor,
      end: cursor + 1,
      matched: false,
      label: "",
    })
    cursor = cursor + 1
  }
  out
}

///|
/// Return only the matched tokens from a token stream.
pub fn matched_tokens(tokens : ArrayView[Token]) -> Array[Token] {
  [
    for token in tokens if token.matched => token
  ]
}

///|
/// Return the unmatched text while preserving source order.
pub fn unmatched_text(tokens : ArrayView[Token]) -> String {
  let builder = StringBuilder()
  for token in tokens {
    if !token.matched {
      builder.write_string(token.text)
    }
  }
  builder.to_string()
}

///|
/// A streaming scanner keeps the automaton state between input chunks.
pub struct Scanner {
  automaton : Automaton
  mut state : Int
  mut position : Int
} derive(Debug)

///|
pub fn Scanner::new(automaton : Automaton) -> Scanner {
  { automaton, state: 0, position: 0 }
}

///|
/// Append a chunk and emit matches without retaining the source text.
pub fn Scanner::push(self : Scanner, chunk : String) -> Array[LexiconMatch] {
  let out : Array[LexiconMatch] = []
  for ch in chunk {
    self.state = self.automaton.step(self.state, ch)
    for id in self.automaton.states[self.state].outputs {
      let entry = self.automaton.entries[id]
      out.push({
        keyword: entry.keyword,
        label: entry.label,
        start: self.position + 1 - entry.keyword.length(),
        end: self.position + 1,
        weight: entry.weight,
      })
    }
    self.position = self.position + 1
  }
  out
}

///|
/// Finish a scanner. Complete matches are emitted by push, so this returns an
/// empty tail and makes the end-of-stream step explicit.
pub fn Scanner::finish(self : Scanner) -> Array[LexiconMatch] {
  self.state = 0
  self.position = 0
  []
}