///|
priv struct CharacterSpan {
  start_units : Int
  end_units : Int
  start_bytes : Int
  end_bytes : Int
}

///|
fn character_spans(text : String) -> Array[CharacterSpan] {
  let spans : Array[CharacterSpan] = []
  let mut utf16_offset = 0
  let mut utf8_offset = 0
  for ch in text {
    let next_utf16 = utf16_offset + utf16_width(ch)
    let next_utf8 = utf8_offset + utf8_width(ch)
    spans.push({
      start_units: utf16_offset,
      end_units: next_utf16,
      start_bytes: utf8_offset,
      end_bytes: next_utf8,
    })
    utf16_offset = next_utf16
    utf8_offset = next_utf8
  }
  spans
}

///|
fn common_unicode_punctuation(ch : Char) -> Bool {
  let code = ch.to_int()
  code == 0x3000 ||
  (code >= 0x2000 && code <= 0x206F) ||
  (code >= 0x2E00 && code <= 0x2E7F) ||
  (code >= 0x3001 && code <= 0x303F) ||
  (code >= 0xFE10 && code <= 0xFE1F) ||
  (code >= 0xFE30 && code <= 0xFE4F) ||
  (code >= 0xFF01 && code <= 0xFF0F) ||
  (code >= 0xFF1A && code <= 0xFF20) ||
  (code >= 0xFF3B && code <= 0xFF40) ||
  (code >= 0xFF5B && code <= 0xFF65)
}

///|
fn split_text(text : String, split_punctuation : Bool) -> Array[Token] {
  let tokens : Array[Token] = []
  let mut token_start_units = -1
  let mut token_start_bytes = -1
  let mut utf16_offset = 0
  let mut utf8_offset = 0
  let mut position = 0
  for ch in text {
    let separator = ch.is_whitespace() ||
      (
        split_punctuation &&
        (ch.is_ascii_punctuation() || common_unicode_punctuation(ch))
      )
    if separator {
      if token_start_units >= 0 {
        tokens.push({
          text: text[token_start_units:utf16_offset].to_owned(),
          position,
          position_length: 1,
          start_offset: token_start_bytes,
          end_offset: utf8_offset,
        })
        position += 1
        token_start_units = -1
        token_start_bytes = -1
      }
    } else if token_start_units < 0 {
      token_start_units = utf16_offset
      token_start_bytes = utf8_offset
    }
    utf16_offset += utf16_width(ch)
    utf8_offset += utf8_width(ch)
  }
  if token_start_units >= 0 {
    tokens.push({
      text: text[token_start_units:utf16_offset].to_owned(),
      position,
      position_length: 1,
      start_offset: token_start_bytes,
      end_offset: utf8_offset,
    })
  }
  tokens
}

///|
/// Emits the complete non-empty input as one token.
pub struct RawTokenizer {}

///|
pub fn RawTokenizer::new() -> RawTokenizer {
  RawTokenizer::{  }
}

///|
pub impl Tokenizer for RawTokenizer with fn token_stream(_self, text) {
  let tokens : Array[Token] = []
  if text.length() > 0 {
    tokens.push({
      text,
      position: 0,
      position_length: 1,
      start_offset: 0,
      end_offset: @utf8.encode(text).length(),
    })
  }
  ArrayTokenStream::new(tokens)
}

///|
pub fn RawTokenizer::analyze(
  self : RawTokenizer,
  text : String,
) -> Array[Token] {
  collect_token_stream(self.token_stream(text))
}

///|
/// Splits only on Unicode whitespace and performs no normalization.
pub struct WhitespaceTokenizer {}

///|
pub fn WhitespaceTokenizer::new() -> WhitespaceTokenizer {
  WhitespaceTokenizer::{  }
}

///|
pub impl Tokenizer for WhitespaceTokenizer with fn token_stream(_self, text) {
  ArrayTokenStream::new(split_text(text, false))
}

///|
pub fn WhitespaceTokenizer::analyze(
  self : WhitespaceTokenizer,
  text : String,
) -> Array[Token] {
  collect_token_stream(self.token_stream(text))
}

///|
/// Splits on Unicode whitespace, ASCII punctuation, and common CJK punctuation.
pub struct SimpleTokenizer {}

///|
pub fn SimpleTokenizer::new() -> SimpleTokenizer {
  SimpleTokenizer::{  }
}

///|
pub impl Tokenizer for SimpleTokenizer with fn token_stream(_self, text) {
  ArrayTokenStream::new(split_text(text, true))
}

///|
pub fn SimpleTokenizer::analyze(
  self : SimpleTokenizer,
  text : String,
) -> Array[Token] {
  collect_token_stream(self.token_stream(text))
}

///|
/// Generic character N-gram tokenizer. It performs no linguistic segmentation.
///
/// Every token has position zero, matching Tantivy's NgramTokenizer semantics.
/// UTF-8 byte offsets still identify the source span of each emitted gram.
pub struct NgramTokenizer {
  min_gram : Int
  max_gram : Int
  prefix_only : Bool
}

///|
pub fn NgramTokenizer::new(
  min_gram : Int,
  max_gram : Int,
  prefix_only : Bool,
) -> NgramTokenizer raise AnalysisError {
  guard min_gram > 0 else {
    raise AnalysisError::InvalidNgram("min_gram must be positive")
  }
  guard max_gram >= min_gram else {
    raise AnalysisError::InvalidNgram(
      "max_gram must be greater than or equal to min_gram",
    )
  }
  { min_gram, max_gram, prefix_only }
}

///|
pub impl Tokenizer for NgramTokenizer with fn token_stream(self, text) {
  let spans = character_spans(text)
  let tokens : Array[Token] = []
  let start_limit = if self.prefix_only {
    if spans.length() == 0 {
      0
    } else {
      1
    }
  } else {
    spans.length()
  }
  for start in 0.. spans.length() {
        continue
      }
      let first = spans[start]
      let last = spans[end - 1]
      tokens.push({
        text: text[first.start_units:last.end_units].to_owned(),
        position: 0,
        position_length: 1,
        start_offset: first.start_bytes,
        end_offset: last.end_bytes,
      })
    }
  }
  ArrayTokenStream::new(tokens)
}

///|
pub fn NgramTokenizer::analyze(
  self : NgramTokenizer,
  text : String,
) -> Array[Token] {
  collect_token_stream(self.token_stream(text))
}