///|
/// Errors raised while constructing the built-in immutable Chinese lexicon.
pub(all) suberror ChineseLexiconError {
  EmptyWord
  NonHanWord(String)
  InvalidFrequency(String, Int)
  DuplicateWord(String)
  FrequencyOverflow
} derive(Eq, @debug.Debug)

///|
pub impl Show for ChineseLexiconError with fn output(self, logger) {
  match self {
    EmptyWord => logger.write_string("Chinese lexicon words must not be empty")
    NonHanWord(word) =>
      logger.write_string(
        "Chinese lexicon entries must contain only Han characters: \{word}",
      )
    InvalidFrequency(word, frequency) =>
      logger.write_string(
        "Chinese lexicon frequency must be positive: \{word}=\{frequency}",
      )
    DuplicateWord(word) =>
      logger.write_string("Duplicate Chinese lexicon entry: \{word}")
    FrequencyOverflow =>
      logger.write_string("Chinese lexicon total frequency exceeds Int range")
  }
}

///|
/// One immutable Chinese lexicon entry and its corpus frequency.
pub(all) struct ChineseLexiconEntry {
  word : String
  frequency : Int
} derive(Eq, @debug.Debug)

///|
pub fn ChineseLexiconEntry::new(
  word : String,
  frequency : Int,
) -> ChineseLexiconEntry {
  { word, frequency }
}

///|
/// One dictionary word beginning at a requested character position.
pub(all) struct ChineseWordMatch {
  length : Int
  frequency : Int
} derive(Eq, @debug.Debug)

///|
/// Dictionary extension point consumed by ChineseTokenizer.
///
/// Implementations return every word beginning at `start`, bounded by `end`.
/// Returning candidates instead of choosing one keeps segmentation policy out
/// of the dictionary and leaves the boundary usable by later DAG routing.
pub(open) trait ChineseDictionary {
  fn matches_at(Self, ReadOnlyArray[Char], Int, Int) -> Array[ChineseWordMatch]
  fn total_frequency(Self) -> Int = _
}

///|
/// Backward-compatible normalization for M5b-4a dictionaries without weights.
impl ChineseDictionary with fn total_frequency(_self) {
  1
}

///|
priv struct TrieNode {
  labels : Array[Char]
  children : Array[Int]
  mut terminal_frequency : Int
}

///|
fn TrieNode::new() -> TrieNode {
  { labels: [], children: [], terminal_frequency: 0 }
}

///|
/// Immutable trie-backed snapshot of application-provided Chinese words.
pub struct ChineseLexicon {
  labels : ReadOnlyArray[ReadOnlyArray[Char]]
  children : ReadOnlyArray[ReadOnlyArray[Int]]
  terminal_frequency : ReadOnlyArray[Int]
  total_frequency : Int
}

///|
fn is_han_character(ch : Char) -> Bool {
  let code = ch.to_int()
  code == 0x3007 ||
  (code >= 0x3400 && code <= 0x4DBF) ||
  (code >= 0x4E00 && code <= 0x9FFF) ||
  (code >= 0xF900 && code <= 0xFAFF) ||
  (code >= 0x20000 && code <= 0x2A6DF) ||
  (code >= 0x2A700 && code <= 0x2B73F) ||
  (code >= 0x2B740 && code <= 0x2B81F) ||
  (code >= 0x2B820 && code <= 0x2CEAF) ||
  (code >= 0x2CEB0 && code <= 0x2EBEF) ||
  (code >= 0x30000 && code <= 0x3134F) ||
  (code >= 0x31350 && code <= 0x323AF)
}

///|
fn trie_child(nodes : Array[TrieNode], node : Int, label : Char) -> Int? {
  match nodes[node].labels.search_by(existing => existing == label) {
    Some(index) => Some(nodes[node].children[index])
    None => None
  }
}

///|
fn insert_trie_word(
  nodes : Array[TrieNode],
  word : Array[Char],
  frequency : Int,
) -> Bool {
  let mut node = 0
  for label in word {
    match trie_child(nodes, node, label) {
      Some(child) => node = child
      None => {
        let child = nodes.length()
        nodes.push(TrieNode::new())
        nodes[node].labels.push(label)
        nodes[node].children.push(child)
        node = child
      }
    }
  }
  if nodes[node].terminal_frequency > 0 {
    false
  } else {
    nodes[node].terminal_frequency = frequency
    true
  }
}

///|
/// Package-internal incremental builder shared by array and text resources.
priv struct ChineseLexiconBuilder {
  nodes : Array[TrieNode]
  mut total_frequency : Int
}

///|
fn ChineseLexiconBuilder::new() -> ChineseLexiconBuilder {
  { nodes: [TrieNode::new()], total_frequency: 0 }
}

///|
fn ChineseLexiconBuilder::add(
  self : ChineseLexiconBuilder,
  word : String,
  frequency : Int,
  reject_duplicates : Bool,
) -> Unit raise ChineseLexiconError {
  let characters = word.to_array()
  guard characters.length() > 0 else { raise ChineseLexiconError::EmptyWord }
  for character in characters {
    guard is_han_character(character) else {
      raise ChineseLexiconError::NonHanWord(word)
    }
  }
  guard frequency > 0 else {
    raise ChineseLexiconError::InvalidFrequency(word, frequency)
  }
  if !insert_trie_word(self.nodes, characters, frequency) {
    if reject_duplicates {
      raise ChineseLexiconError::DuplicateWord(word)
    }
    return
  }
  guard self.total_frequency <= 0x7FFFFFFF - frequency else {
    raise ChineseLexiconError::FrequencyOverflow
  }
  self.total_frequency += frequency
}

///|
fn ChineseLexiconBuilder::finish(
  self : ChineseLexiconBuilder,
) -> ChineseLexicon {
  let labels : Array[ReadOnlyArray[Char]] = []
  let children : Array[ReadOnlyArray[Int]] = []
  let terminal_frequency : Array[Int] = []
  for node in self.nodes {
    labels.push(ReadOnlyArray::from_array(node.labels))
    children.push(ReadOnlyArray::from_array(node.children))
    terminal_frequency.push(node.terminal_frequency)
  }
  {
    labels: ReadOnlyArray::from_array(labels),
    children: ReadOnlyArray::from_array(children),
    terminal_frequency: ReadOnlyArray::from_array(terminal_frequency),
    total_frequency: self.total_frequency,
  }
}

///|
pub fn ChineseLexicon::new(
  words : Array[String],
) -> ChineseLexicon raise ChineseLexiconError {
  let entries : Array[ChineseLexiconEntry] = words.map(word => ChineseLexiconEntry::{
    word,
    frequency: 1,
  })
  build_chinese_lexicon(entries, false)
}

///|
/// Builds an immutable weighted Trie snapshot for frequency-DAG routing.
pub fn ChineseLexicon::from_entries(
  entries : Array[ChineseLexiconEntry],
) -> ChineseLexicon raise ChineseLexiconError {
  build_chinese_lexicon(entries, true)
}

///|
fn build_chinese_lexicon(
  entries : Array[ChineseLexiconEntry],
  reject_duplicates : Bool,
) -> ChineseLexicon raise ChineseLexiconError {
  let builder = ChineseLexiconBuilder::new()
  for entry in entries {
    builder.add(entry.word, entry.frequency, reject_duplicates)
  }
  builder.finish()
}

///|
pub impl ChineseDictionary for ChineseLexicon with fn matches_at(
  self,
  characters,
  start,
  end,
) {
  let matches : Array[ChineseWordMatch] = []
  if start < 0 || start >= end || end > characters.length() {
    return matches
  }
  let mut node = 0
  let mut cursor = start
  while cursor < end {
    match
      self.labels[node].search_by(existing => existing == characters[cursor]) {
      Some(edge) => {
        node = self.children[node][edge]
        cursor += 1
        if self.terminal_frequency[node] > 0 {
          matches.push({
            length: cursor - start,
            frequency: self.terminal_frequency[node],
          })
        }
      }
      None => return matches
    }
  }
  matches
}

///|
pub impl ChineseDictionary for ChineseLexicon with fn total_frequency(self) {
  self.total_frequency
}

///|
priv struct CharacterSpan {
  character : Char
  start_bytes : Int
  end_bytes : Int
}

///|
fn utf8_width(ch : Char) -> Int {
  let code = ch.to_int()
  if code <= 0x7F {
    1
  } else if code <= 0x7FF {
    2
  } else if code <= 0xFFFF {
    3
  } else {
    4
  }
}

///|
fn character_spans(text : String) -> Array[CharacterSpan] {
  let spans : Array[CharacterSpan] = []
  let mut byte_offset = 0
  for ch in text {
    let next_offset = byte_offset + utf8_width(ch)
    spans.push({
      character: ch,
      start_bytes: byte_offset,
      end_bytes: next_offset,
    })
    byte_offset = next_offset
  }
  spans
}

///|
fn common_unicode_punctuation(ch : Char) -> Bool {
  let code = ch.to_int()
  code == 0x3000 ||
  (code >= 0x2000 && code <= 0x206F) ||
  (code >= 0x2E00 && code <= 0x2E7F) ||
  (code >= 0x3001 && code <= 0x303F) ||
  (code >= 0xFE10 && code <= 0xFE1F) ||
  (code >= 0xFE30 && code <= 0xFE4F) ||
  (code >= 0xFF01 && code <= 0xFF0F) ||
  (code >= 0xFF1A && code <= 0xFF20) ||
  (code >= 0xFF3B && code <= 0xFF40) ||
  (code >= 0xFF5B && code <= 0xFF65)
}

///|
fn is_separator(ch : Char) -> Bool {
  ch.is_whitespace() ||
  ch.is_ascii_punctuation() ||
  common_unicode_punctuation(ch)
}

///|
fn token_text(spans : Array[CharacterSpan], start : Int, end : Int) -> String {
  let characters : Array[Char] = []
  for index in start.. Unit {
  tokens.push({
    text: token_text(spans, start, end),
    position,
    position_length: 1,
    start_offset: spans[start].start_bytes,
    end_offset: spans[end - 1].end_bytes,
  })
}

///|
/// Dictionary-backed tokenizer for mixed Chinese and non-Chinese text.
///
/// `new` uses deterministic longest matching; the other constructors opt into
/// frequency-DAG routing and optional HMM recognition. Consecutive non-Han,
/// non-punctuation text is emitted as one token; normalization belongs to the
/// surrounding TextAnalyzer pipeline.
pub struct ChineseTokenizer {
  dictionary : &ChineseDictionary
  frequency_dag : Bool
  hmm_model : &ChineseHmmModel?
}

///|
pub fn ChineseTokenizer::new(
  dictionary : &ChineseDictionary,
) -> ChineseTokenizer {
  { dictionary, frequency_dag: false, hmm_model: None }
}

///|
/// Creates a tokenizer that selects a globally optimal frequency-DAG route.
pub fn ChineseTokenizer::with_frequency_dag(
  dictionary : &ChineseDictionary,
) -> ChineseTokenizer {
  { dictionary, frequency_dag: true, hmm_model: None }
}

///|
/// Creates a frequency-DAG tokenizer with HMM unknown-word recognition.
pub fn ChineseTokenizer::with_frequency_dag_and_hmm(
  dictionary : &ChineseDictionary,
  model : &ChineseHmmModel,
) -> ChineseTokenizer {
  { dictionary, frequency_dag: true, hmm_model: Some(model) }
}

///|
priv struct ChineseTokenStream {
  tokens : ReadOnlyArray[@analysis.Token]
  mut cursor : Int
}

///|
impl @analysis.TokenStream for ChineseTokenStream with fn advance(self) {
  if self.cursor + 1 < self.tokens.length() {
    self.cursor += 1
    true
  } else {
    false
  }
}

///|
impl @analysis.TokenStream for ChineseTokenStream with fn token(self) {
  if self.cursor < 0 || self.cursor >= self.tokens.length() {
    None
  } else {
    Some(self.tokens[self.cursor])
  }
}

///|
pub impl @analysis.Tokenizer for ChineseTokenizer with fn token_stream(
  self,
  text,
) {
  let spans = character_spans(text)
  let characters : Array[Char] = spans.map(span => span.character)
  let read_only_characters = ReadOnlyArray::from_array(characters)
  let tokens : Array[@analysis.Token] = []
  let mut cursor = 0
  let mut position = 0
  while cursor < spans.length() {
    let current = spans[cursor].character
    if is_separator(current) {
      cursor += 1
      continue
    }
    if is_han_character(current) {
      let han_start = cursor
      let mut han_end = cursor + 1
      while han_end < spans.length() &&
            is_han_character(spans[han_end].character) {
        han_end += 1
      }
      if self.frequency_dag {
        let route = frequency_dag_route(
          self.dictionary,
          read_only_characters,
          han_start,
          han_end,
        )
        match self.hmm_model {
          None =>
            while cursor < han_end {
              let length = route[cursor - han_start]
              push_token(tokens, spans, cursor, cursor + length, position)
              cursor += length
              position += 1
            }
          Some(model) =>
            while cursor < han_end {
              let length = route[cursor - han_start]
              if length > 1 {
                push_token(tokens, spans, cursor, cursor + length, position)
                cursor += length
                position += 1
                continue
              }
              let singleton_start = cursor
              cursor += 1
              while cursor < han_end && route[cursor - han_start] == 1 {
                cursor += 1
              }
              let singleton_end = cursor
              if singleton_end - singleton_start > 1 &&
                !chinese_dictionary_has_exact_match(
                  self.dictionary,
                  read_only_characters,
                  singleton_start,
                  singleton_end,
                ) {
                let hmm_lengths = chinese_hmm_segment_lengths(
                  model, read_only_characters, singleton_start, singleton_end,
                )
                let mut hmm_cursor = singleton_start
                for hmm_length in hmm_lengths {
                  push_token(
                    tokens,
                    spans,
                    hmm_cursor,
                    hmm_cursor + hmm_length,
                    position,
                  )
                  hmm_cursor += hmm_length
                  position += 1
                }
              } else {
                for singleton_cursor in singleton_start.. length && cursor + candidate.length <= han_end {
          length = candidate.length
        }
      }
      push_token(tokens, spans, cursor, cursor + length, position)
      cursor += length
      position += 1
      continue
    }
    let start = cursor
    cursor += 1
    while cursor < spans.length() &&
          !is_separator(spans[cursor].character) &&
          !is_han_character(spans[cursor].character) {
      cursor += 1
    }
    push_token(tokens, spans, start, cursor, position)
    position += 1
  }
  ChineseTokenStream::{ tokens: ReadOnlyArray::from_array(tokens), cursor: -1 }
}

///|
pub fn ChineseTokenizer::analyze(
  self : ChineseTokenizer,
  text : String,
) -> Array[@analysis.Token] {
  let stream = self.token_stream(text)
  let tokens : Array[@analysis.Token] = []
  while stream.advance() {
    match stream.token() {
      Some(token) => tokens.push(token)
      None => ()
    }
  }
  tokens
}

///|
/// Creates a lowercase, maximum-length-limited analyzer around one dictionary.
pub fn chinese_analyzer(
  dictionary : &ChineseDictionary,
) -> @analysis.TextAnalyzer {
  let analyzer = @analysis.TextAnalyzer::new(ChineseTokenizer::new(dictionary))
  analyzer.add_filter(@analysis.RemoveLongFilter::new(40))
  analyzer.add_filter(@analysis.LowerCaseFilter::new())
  analyzer
}

///|
/// Creates a longest-match analyzer with dictionary-backed search subwords.
pub fn chinese_search_analyzer(
  dictionary : &ChineseDictionary,
) -> @analysis.TextAnalyzer {
  let analyzer = @analysis.TextAnalyzer::new(ChineseTokenizer::new(dictionary))
  analyzer.add_filter(ChineseSearchModeFilter::new(dictionary))
  analyzer.add_filter(@analysis.RemoveLongFilter::new(40))
  analyzer.add_filter(@analysis.LowerCaseFilter::new())
  analyzer
}

///|
/// Creates the longest-match search expansion intended for indexing.
pub fn chinese_search_index_analyzer(
  dictionary : &ChineseDictionary,
) -> @analysis.TextAnalyzer {
  let analyzer = @analysis.TextAnalyzer::new(ChineseTokenizer::new(dictionary))
  analyzer.add_filter(ChineseSearchModeFilter::for_index(dictionary))
  analyzer.add_filter(@analysis.RemoveLongFilter::new(40))
  analyzer.add_filter(@analysis.LowerCaseFilter::new())
  analyzer
}

///|
/// Creates a lowercase analyzer using globally optimal frequency-DAG routing.
pub fn chinese_dag_analyzer(
  dictionary : &ChineseDictionary,
) -> @analysis.TextAnalyzer {
  let analyzer = @analysis.TextAnalyzer::new(
    ChineseTokenizer::with_frequency_dag(dictionary),
  )
  analyzer.add_filter(@analysis.RemoveLongFilter::new(40))
  analyzer.add_filter(@analysis.LowerCaseFilter::new())
  analyzer
}

///|
/// Creates a frequency-DAG analyzer with dictionary-backed search subwords.
pub fn chinese_dag_search_analyzer(
  dictionary : &ChineseDictionary,
) -> @analysis.TextAnalyzer {
  let analyzer = @analysis.TextAnalyzer::new(
    ChineseTokenizer::with_frequency_dag(dictionary),
  )
  analyzer.add_filter(ChineseSearchModeFilter::new(dictionary))
  analyzer.add_filter(@analysis.RemoveLongFilter::new(40))
  analyzer.add_filter(@analysis.LowerCaseFilter::new())
  analyzer
}

///|
/// Creates the frequency-DAG search expansion intended for indexing.
pub fn chinese_dag_search_index_analyzer(
  dictionary : &ChineseDictionary,
) -> @analysis.TextAnalyzer {
  let analyzer = @analysis.TextAnalyzer::new(
    ChineseTokenizer::with_frequency_dag(dictionary),
  )
  analyzer.add_filter(ChineseSearchModeFilter::for_index(dictionary))
  analyzer.add_filter(@analysis.RemoveLongFilter::new(40))
  analyzer.add_filter(@analysis.LowerCaseFilter::new())
  analyzer
}

///|
/// Creates a lowercase frequency-DAG analyzer with injected HMM recognition.
pub fn chinese_dag_hmm_analyzer(
  dictionary : &ChineseDictionary,
  model : &ChineseHmmModel,
) -> @analysis.TextAnalyzer {
  let analyzer = @analysis.TextAnalyzer::new(
    ChineseTokenizer::with_frequency_dag_and_hmm(dictionary, model),
  )
  analyzer.add_filter(@analysis.RemoveLongFilter::new(40))
  analyzer.add_filter(@analysis.LowerCaseFilter::new())
  analyzer
}

///|
/// Creates a frequency-DAG/HMM analyzer with dictionary-backed search subwords.
pub fn chinese_dag_hmm_search_analyzer(
  dictionary : &ChineseDictionary,
  model : &ChineseHmmModel,
) -> @analysis.TextAnalyzer {
  let analyzer = @analysis.TextAnalyzer::new(
    ChineseTokenizer::with_frequency_dag_and_hmm(dictionary, model),
  )
  analyzer.add_filter(ChineseSearchModeFilter::new(dictionary))
  analyzer.add_filter(@analysis.RemoveLongFilter::new(40))
  analyzer.add_filter(@analysis.LowerCaseFilter::new())
  analyzer
}

///|
/// Creates the DAG/HMM search expansion intended for indexing.
pub fn chinese_dag_hmm_search_index_analyzer(
  dictionary : &ChineseDictionary,
  model : &ChineseHmmModel,
) -> @analysis.TextAnalyzer {
  let analyzer = @analysis.TextAnalyzer::new(
    ChineseTokenizer::with_frequency_dag_and_hmm(dictionary, model),
  )
  analyzer.add_filter(ChineseSearchModeFilter::for_index(dictionary))
  analyzer.add_filter(@analysis.RemoveLongFilter::new(40))
  analyzer.add_filter(@analysis.LowerCaseFilter::new())
  analyzer
}