///|
fn token_kind_for_char(c : Char) -> String {
  if is_blank_char(c) {
    "space"
  } else if c.is_ascii_digit() {
    "number"
  } else if c.is_ascii_alphabetic() {
    "word"
  } else if is_cjk_char(c) {
    "cjk"
  } else if c.is_ascii_punctuation() || c == '，' || c == '。' || c == '：' {
    "punctuation"
  } else {
    "symbol"
  }
}

///|
pub fn tokenize(text : String) -> Array[TextToken] {
  let chars = text.to_array()
  let tokens = []
  if chars.is_empty() {
    tokens
  } else {
    let mut start = 0
    let mut kind = token_kind_for_char(chars[0])
    let mut position = chars[0].to_string().length()
    for i in 1.. Array[TextToken] {
  tokens.filter(fn(item) { item.kind == kind })
}

///|
pub fn non_space_tokens(tokens : Array[TextToken]) -> Array[TextToken] {
  tokens.filter(fn(item) { item.kind != "space" })
}

///|
pub fn token_spans(tokens : Array[TextToken]) -> Array[Span] {
  tokens.map(fn(item) { { start: item.start, end: item.end } })
}

///|
pub fn token_texts(tokens : Array[TextToken]) -> Array[String] {
  tokens.map(fn(item) { item.text })
}

///|
pub fn token_kind_counts(tokens : Array[TextToken]) -> Map[String, Int] {
  let result : Map[String, Int] = Map([])
  for token in tokens {
    result[token.kind] = result.get_or_default(token.kind, 0) + 1
  }
  result
}

///|
pub fn token_char_counts(tokens : Array[TextToken]) -> Map[String, Int] {
  let result : Map[String, Int] = Map([])
  for token in tokens {
    result[token.kind] = result.get_or_default(token.kind, 0) +
      token.text.char_length()
  }
  result
}

///|
pub fn token_count(text : String) -> Int {
  tokenize(text).length()
}

///|
pub fn word_tokens(text : String) -> Array[TextToken] {
  tokens_of_kind(tokenize(text), "word")
}

///|
pub fn number_tokens(text : String) -> Array[TextToken] {
  tokens_of_kind(tokenize(text), "number")
}

///|
pub fn cjk_tokens(text : String) -> Array[TextToken] {
  tokens_of_kind(tokenize(text), "cjk")
}

///|
pub fn token_before(tokens : Array[TextToken], position : Int) -> TextToken? {
  let mut result = None
  for token in tokens {
    if token.end <= position {
      result = Some(token)
    }
  }
  result
}

///|
pub fn token_after(tokens : Array[TextToken], position : Int) -> TextToken? {
  for item in tokens {
    if item.start >= position {
      return Some(item)
    }
  }
  None
}

///|
pub fn tokens_in_span(
  tokens : Array[TextToken],
  span : Span,
) -> Array[TextToken] {
  tokens.filter(fn(item) {
    span.contains_span({ start: item.start, end: item.end })
  })
}

///|
pub fn neighboring_tokens(
  tokens : Array[TextToken],
  index : Int,
  radius : Int,
) -> Array[TextToken] {
  let safe_radius = if radius < 0 { 0 } else { radius }
  let start = if index - safe_radius < 0 { 0 } else { index - safe_radius }
  let end = if index + safe_radius + 1 > tokens.length() {
    tokens.length()
  } else {
    index + safe_radius + 1
  }
  tokens[start:end].to_owned()
}

///|
pub fn token_context(
  text : String,
  token : TextToken,
  radius : Int,
) -> TextWindow {
  text_window(text, (token.start + token.end) / 2, radius)
}

///|
pub fn merge_adjacent_tokens(tokens : Array[TextToken]) -> Array[TextToken] {
  let result : Array[TextToken] = []
  for token in tokens {
    match result.get(result.length() - 1) {
      Some(previous) =>
        if previous.kind == token.kind && previous.end == token.start {
          result[result.length() - 1] = {
            ..previous,
            text: previous.text + token.text,
            end: token.end,
          }
        } else {
          result.push(token)
        }
      None => result.push(token)
    }
  }
  result
}

///|
pub fn token_reconstruct(tokens : Array[TextToken]) -> String {
  tokens.map(fn(item) { item.text }).join("")
}

///|
pub fn tokenization_is_lossless(text : String) -> Bool {
  token_reconstruct(tokenize(text)) == text
}

///|
pub fn tokenization_report(text : String) -> String {
  let tokens = tokenize(text)
  let counts = token_kind_counts(tokens)
  [
    "input_length=\{text.length()}",
    "characters=\{text.char_length()}",
    "tokens=\{tokens.length()}",
    "kinds=\{counts.length()}",
    "lossless=\{tokenization_is_lossless(text)}",
  ].join("\n")
}

///|
pub fn redaction_candidates_from_tokens(
  text : String,
  tokens : Array[TextToken],
  rule_id : String,
  kind : PhiKind,
  confidence : Int,
) -> Array[Finding] {
  tokens
  .filter(fn(item) { item.kind == "word" || item.kind == "cjk" })
  .map(fn(item) {
    let finding : Finding = {
      id: "token-\{item.start}",
      kind,
      label: "token candidate",
      start: item.start,
      end: item.end,
      text: item.text,
      replacement: "[\{phi_kind_name(kind)}]",
      rule_id,
      confidence,
    }
    finding
  })
  .filter(fn(item) { validate_finding_text(text, item) })
}

///|
pub fn redact_token_candidates(
  text : String,
  kind : PhiKind,
  config : RedactionConfig,
) -> String {
  let tokens = non_space_tokens(tokenize(text))
  let candidates = redaction_candidates_from_tokens(
    text, tokens, "token-fallback", kind, 50,
  )
  let filtered = policy_filter_findings(candidates, config.policy)
  let replacement = filtered.map(fn(item) {
    { ..item, replacement: replacement_preview(item, config.policy.mode) }
  })
  let (output, _) = apply_findings(text, replacement)
  output
}

///|
pub fn token_lengths(tokens : Array[TextToken]) -> Array[Int] {
  tokens.map(fn(item) { item.text.length() })
}

///|
pub fn longest_token(tokens : Array[TextToken]) -> TextToken? {
  let mut result = None
  for token in tokens {
    match result {
      None => result = Some(token)
      Some(previous) =>
        if token.text.length() > previous.text.length() {
          result = Some(token)
        }
    }
  }
  result
}

///|
pub fn punctuation_tokens(text : String) -> Array[TextToken] {
  tokens_of_kind(tokenize(text), "punctuation")
}

///|
pub fn whitespace_spans(text : String) -> Array[Span] {
  tokenize(text)
  .filter(fn(item) { item.kind == "space" })
  .map(fn(item) { { start: item.start, end: item.end } })
}

///|
pub fn token_positions(tokens : Array[TextToken]) -> Array[LineColumn] {
  tokens.map(fn(item) { line_column(token_reconstruct(tokens), item.start) })
}

///|
pub fn token_kind_at(tokens : Array[TextToken], position : Int) -> String? {
  for item in tokens {
    if item.start <= position && position < item.end {
      return Some(item.kind)
    }
  }
  None
}

///|
pub fn replace_token(
  tokens : Array[TextToken],
  index : Int,
  value : String,
) -> Array[TextToken] {
  let result = tokens.copy()
  match result.get(index) {
    Some(token) =>
      result[index] = {
        ..token,
        text: value,
        end: token.start + value.length(),
      }
    None => ()
  }
  result
}