///|
/// Options controlling dictionary-aware segmentation.
pub(all) struct SegmentationOptions {
  combine_unmatched : Bool
  prefer_longest : Bool
  include_empty : Bool
} derive(Debug, Eq, ToJson)

///|
/// Conservative segmentation defaults suitable for user-facing text tools.
pub fn SegmentationOptions::default() -> SegmentationOptions {
  { combine_unmatched: true, prefer_longest: true, include_empty: false }
}

///|
/// A token sequence retains source order and original character offsets.
pub(all) struct TokenSequence {
  tokens : Array[Token]
  source_length : Int
} derive(Debug, Eq, ToJson)

///|
/// Segment text with explicit matching and gap policies.
pub fn segment_with_options(
  lexicon : Lexicon,
  text : String,
  options : SegmentationOptions,
) -> TokenSequence {
  let matches = if options.prefer_longest {
    lexicon.segment_longest(text)
  } else {
    query_lexicon(lexicon, text, {
      min_weight: -1000000,
      require_label: "",
      prefer_longest: false,
      include_overlaps: false,
      max_matches: 1000000,
    })
  }
  let characters = text.to_array()
  let tokens : Array[Token] = []
  let mut cursor = 0
  for hit in sort_matches_by_position(matches) {
    let start = segment_clamp(hit.start, cursor, characters.length())
    let end = segment_clamp(hit.end, start, characters.length())
    append_gap_tokens(
      tokens,
      characters,
      cursor,
      start,
      options.combine_unmatched,
    )
    if end > start {
      tokens.push({
        text: segment_slice(text, start, end),
        start,
        end,
        matched: true,
        label: hit.label,
      })
      cursor = end
    }
  }
  append_gap_tokens(
    tokens,
    characters,
    cursor,
    characters.length(),
    options.combine_unmatched,
  )
  if options.include_empty && text.length() == 0 {
    tokens.push({ text: "", start: 0, end: 0, matched: false, label: "" })
  }
  { tokens, source_length: text.length() }
}

///|
/// Segment with the standard longest-match policy.
pub fn segment_text(lexicon : Lexicon, text : String) -> TokenSequence {
  segment_with_options(lexicon, text, SegmentationOptions::default())
}

///|
pub fn TokenSequence::matched_count(self : TokenSequence) -> Int {
  let mut count = 0
  for token in self.tokens {
    if token.matched {
      count += 1
    }
  }
  count
}

///|
pub fn TokenSequence::unmatched_count(self : TokenSequence) -> Int {
  self.tokens.length() - self.matched_count()
}

///|
pub fn TokenSequence::to_text(self : TokenSequence) -> String {
  let output = StringBuilder()
  for token in self.tokens {
    output.write_string(token.text)
  }
  output.to_string()
}

///|
pub fn TokenSequence::for_label(
  self : TokenSequence,
  label : String,
) -> Array[Token] {
  [
    for token in self.tokens if token.matched && token.label == label => token
  ]
}

///|
pub fn TokenSequence::unmatched_text(self : TokenSequence) -> String {
  unmatched_text(self.tokens)
}

///|
pub fn TokenSequence::has_contiguous_offsets(self : TokenSequence) -> Bool {
  let mut cursor = 0
  for token in self.tokens {
    if token.start != cursor || token.end < token.start {
      return false
    }
    cursor = token.end
  }
  cursor == self.source_length
}

///|
pub(all) struct TokenStats {
  token_count : Int
  matched_count : Int
  unmatched_count : Int
  total_length : Int
  matched_length : Int
  unmatched_length : Int
  distinct_label_count : Int
} derive(Debug, Eq, ToJson)

///|
pub fn token_stats(tokens : ArrayView[Token]) -> TokenStats {
  let labels : Map[String, Bool] = Map([])
  let mut matched_count = 0
  let mut matched_length = 0
  let mut total_length = 0
  for token in tokens {
    let length = token.end - token.start
    total_length += length
    if token.matched {
      matched_count += 1
      matched_length += length
      labels[token.label] = true
    }
  }
  {
    token_count: tokens.length(),
    matched_count,
    unmatched_count: tokens.length() - matched_count,
    total_length,
    matched_length,
    unmatched_length: total_length - matched_length,
    distinct_label_count: labels.length(),
  }
}

///|
pub fn tokens_have_valid_offsets(tokens : ArrayView[Token]) -> Bool {
  let mut cursor = 0
  for token in tokens {
    if token.start != cursor || token.end < token.start {
      return false
    }
    cursor = token.end
  }
  true
}

///|
pub fn combine_unmatched_tokens(tokens : ArrayView[Token]) -> Array[Token] {
  let output : Array[Token] = []
  for token in tokens {
    if !token.matched &&
      output.length() > 0 &&
      !output[output.length() - 1].matched {
      let previous = output[output.length() - 1]
      output[output.length() - 1] = {
        text: previous.text + token.text,
        start: previous.start,
        end: token.end,
        matched: false,
        label: "",
      }
    } else {
      output.push(token)
    }
  }
  output
}

///|
pub fn tokens_to_tsv(tokens : ArrayView[Token]) -> String {
  let output = StringBuilder()
  output.write_string("text\tstart\tend\tmatched\tlabel\n")
  for token in tokens {
    let matched = if token.matched { "1" } else { "0" }
    output.write_string(
      "\{escape_dictionary_field(token.text)}\t\{token.start}\t\{token.end}\t\{matched}\t\{escape_dictionary_field(token.label)}\n",
    )
  }
  output.to_string()
}

///|
pub fn tokens_containing(
  tokens : ArrayView[Token],
  needle : String,
) -> Array[Token] {
  [
    for token in tokens if token.text.contains(needle) => token
  ]
}

///|
pub fn first_matched_token(tokens : ArrayView[Token]) -> Token? {
  for token in tokens {
    if token.matched {
      return Some(token)
    }
  }
  None
}

///|
pub fn last_matched_token(tokens : ArrayView[Token]) -> Token? {
  let mut result : Token? = None
  for token in tokens {
    if token.matched {
      result = Some(token)
    }
  }
  result
}

///|
pub fn matched_tokens_at_least(
  tokens : ArrayView[Token],
  minimum_length : Int,
) -> Array[Token] {
  [
    for token in tokens if token.matched &&
    token.end - token.start >= minimum_length => token
  ]
}

///|
pub fn token_span(tokens : ArrayView[Token]) -> (Int, Int) {
  match tokens.get(0) {
    None => (0, 0)
    Some(first) =>
      match tokens.get(tokens.length() - 1) {
        Some(last) => (first.start, last.end)
        None => (first.start, first.end)
      }
  }
}

///|
fn append_gap_tokens(
  tokens : Array[Token],
  characters : Array[Char],
  start : Int,
  end : Int,
  combine : Bool,
) -> Unit {
  if end <= start {
    return
  }
  if combine {
    tokens.push({
      text: chars_to_string(characters, start, end),
      start,
      end,
      matched: false,
      label: "",
    })
  } else {
    for index in start.. String {
  let output = StringBuilder()
  for index in start.. String {
  if end <= start {
    ""
  } else {
    let characters = text.to_array()
    let output = StringBuilder()
    for index in start.. Int {
  if value < minimum {
    minimum
  } else if value > maximum {
    maximum
  } else {
    value
  }
}