///|
/// Adds dictionary-backed two-character and three-character subwords to a
/// linear Chinese token stream.
///
/// The filter follows Jieba search-mode candidate selection while expressing
/// overlaps as Token Graph edges. It expects strictly increasing input
/// positions with `position_length == 1`, as emitted by ChineseTokenizer.
pub struct ChineseSearchModeFilter {
  dictionary : &ChineseDictionary
  flatten_for_index : Bool
}

///|
pub fn ChineseSearchModeFilter::new(
  dictionary : &ChineseDictionary,
) -> ChineseSearchModeFilter {
  { dictionary, flatten_for_index: false }
}

///|
/// Creates the lossy index-time position policy.
///
/// The original word and all of its subwords are stacked at one ordinal
/// position because Segment postings do not persist graph edge lengths.
pub fn ChineseSearchModeFilter::for_index(
  dictionary : &ChineseDictionary,
) -> ChineseSearchModeFilter {
  { dictionary, flatten_for_index: true }
}

///|
priv struct ChineseSearchModeTokenStream {
  input : &@analysis.TokenStream
  dictionary : &ChineseDictionary
  flatten_for_index : Bool
  mut pending : Array[@analysis.Token]
  mut pending_cursor : Int
  mut position_shift : Int
  mut previous_input_position : Int
  mut current : @analysis.Token?
}

///|
fn search_mode_has_word(
  dictionary : &ChineseDictionary,
  characters : ReadOnlyArray[Char],
  start : Int,
  length : Int,
) -> Bool {
  for candidate in dictionary.matches_at(characters, start, characters.length()) {
    if candidate.length == length &&
      candidate.frequency > 0 &&
      start + candidate.length <= characters.length() {
      return true
    }
  }
  false
}

///|
fn all_han_characters(characters : Array[Char]) -> Bool {
  if characters.length() == 0 {
    return false
  }
  for character in characters {
    if !is_han_character(character) {
      return false
    }
  }
  true
}

///|
fn push_search_subword(
  tokens : Array[@analysis.Token],
  spans : Array[CharacterSpan],
  parent : @analysis.Token,
  output_start : Int,
  start : Int,
  length : Int,
) -> Unit {
  let end = start + length
  tokens.push({
    text: token_text(spans, start, end),
    position: output_start + start,
    position_length: length,
    start_offset: parent.start_offset + spans[start].start_bytes,
    end_offset: parent.start_offset + spans[end - 1].end_bytes,
  })
}

///|
fn expand_search_mode_token(
  dictionary : &ChineseDictionary,
  token : @analysis.Token,
  output_start : Int,
  flatten_for_index : Bool,
) -> (Array[@analysis.Token], Int) {
  let spans = character_spans(token.text)
  let characters : Array[Char] = spans.map(span => span.character)
  if !all_han_characters(characters) {
    return (
      [
        {
          text: token.text,
          position: output_start,
          position_length: if flatten_for_index {
            1
          } else {
            token.position_length
          },
          start_offset: token.start_offset,
          end_offset: token.end_offset,
        },
      ],
      if flatten_for_index {
        1
      } else {
        token.position_length
      },
    )
  }
  let character_count = characters.length()
  let read_only_characters = ReadOnlyArray::from_array(characters)
  let expanded : Array[@analysis.Token] = [
    {
      text: token.text,
      position: output_start,
      position_length: character_count,
      start_offset: token.start_offset,
      end_offset: token.end_offset,
    },
  ]
  for start in 0.. 2 &&
      start + 2 <= character_count &&
      search_mode_has_word(dictionary, read_only_characters, start, 2) {
      push_search_subword(expanded, spans, token, output_start, start, 2)
    }
    if character_count > 3 &&
      start + 3 <= character_count &&
      search_mode_has_word(dictionary, read_only_characters, start, 3) {
      push_search_subword(expanded, spans, token, output_start, start, 3)
    }
  }
  expanded.sort_by((left, right) => {
    if left.position != right.position {
      left.position.compare(right.position)
    } else if left.text == token.text && right.text != token.text {
      -1
    } else if right.text == token.text && left.text != token.text {
      1
    } else {
      left.position_length.compare(right.position_length)
    }
  })
  if flatten_for_index {
    for index in 0.. ()
      Some(token) => {
        guard token.position > self.previous_input_position &&
          token.position_length == 1 else {
          abort(
            "ChineseSearchModeFilter requires a linear token stream with strictly increasing positions",
          )
        }
        let output_start = token.position + self.position_shift
        let (expanded, output_span) = expand_search_mode_token(
          self.dictionary,
          token,
          output_start,
          self.flatten_for_index,
        )
        self.position_shift += output_span - token.position_length
        self.previous_input_position = token.position
        self.pending = expanded
        self.pending_cursor = -1
      }
    }
  }
  false
}

///|
impl @analysis.TokenStream for ChineseSearchModeTokenStream with fn token(self) {
  self.current
}

///|
pub impl @analysis.TokenFilter for ChineseSearchModeFilter with fn transform(
  self,
  input,
) {
  ChineseSearchModeTokenStream::{
    input,
    dictionary: self.dictionary,
    flatten_for_index: self.flatten_for_index,
    pending: [],
    pending_cursor: -1,
    position_shift: 0,
    previous_input_position: -1,
    current: None,
  }
}