///|
/// One analyzed token.
///
/// Offsets are UTF-8 byte offsets. `position` is a zero-based graph node and
/// `position_length` is the number of original positions spanned by the token.
pub(all) struct Token {
  text : String
  position : Int
  position_length : Int
  start_offset : Int
  end_offset : Int
} derive(Eq, @debug.Debug)

///|
/// Errors raised while configuring or resolving text analysis.
pub(all) suberror AnalysisError {
  InvalidNgram(String)
  InvalidTokenGraph(String)
  TokenGraphTooComplex(Int)
  InvalidTokenizerName(String)
  UnknownTokenizer(String)
  FieldNotIndexed(FieldId)
  EmptyQuery
} derive(Eq, @debug.Debug)

///|
pub impl Show for AnalysisError with fn output(self, logger) {
  match self {
    InvalidNgram(message) =>
      logger.write_string("invalid n-gram configuration: \{message}")
    InvalidTokenGraph(message) =>
      logger.write_string("invalid token graph: \{message}")
    TokenGraphTooComplex(limit) =>
      logger.write_string("token graph exceeds path limit: \{limit}")
    InvalidTokenizerName(name) =>
      logger.write_string("invalid tokenizer name: \{name}")
    UnknownTokenizer(name) => logger.write_string("unknown tokenizer: \{name}")
    FieldNotIndexed(field_id) =>
      logger.write_string("field is not indexed: \{field_id.value}")
    EmptyQuery => logger.write_string("query analysis produced no tokens")
  }
}

///|
/// Stateful stream of analyzed tokens.
///
/// Tokenizers and token filters compose through this interface. As in Tantivy,
/// `token()` exposes the token produced by the latest successful `advance()`
/// call.
pub(open) trait TokenStream {
  fn advance(Self) -> Bool
  fn token(Self) -> Token?
}

///|
priv struct ArrayTokenStream {
  tokens : ReadOnlyArray[Token]
  mut cursor : Int
}

///|
fn ArrayTokenStream::new(tokens : Array[Token]) -> ArrayTokenStream {
  { tokens: ReadOnlyArray::from_array(tokens), cursor: -1 }
}

///|
impl TokenStream for ArrayTokenStream with fn advance(self) {
  if self.cursor + 1 < self.tokens.length() {
    self.cursor += 1
    true
  } else {
    false
  }
}

///|
impl TokenStream for ArrayTokenStream with fn token(self) {
  if self.cursor < 0 || self.cursor >= self.tokens.length() {
    None
  } else {
    Some(self.tokens[self.cursor])
  }
}

///|
fn collect_token_stream(stream : &TokenStream) -> Array[Token] {
  let tokens : Array[Token] = []
  while stream.advance() {
    match stream.token() {
      Some(token) => tokens.push(token)
      None => ()
    }
  }
  tokens
}

///|
/// Splits source text into a TokenStream before filters are applied.
pub(open) trait Tokenizer {
  fn token_stream(Self, String) -> &TokenStream
}

///|
/// Analyzer that separates tokens at Unicode whitespace and performs no
/// normalization. Retained as a compatibility wrapper around
/// WhitespaceTokenizer.
pub struct WhitespaceAnalyzer {}

///|
pub fn WhitespaceAnalyzer::new() -> WhitespaceAnalyzer {
  WhitespaceAnalyzer::{  }
}

///|
fn utf8_width(ch : Char) -> Int {
  let code_point = ch.to_int()
  if code_point <= 0x7F {
    1
  } else if code_point <= 0x7FF {
    2
  } else if code_point <= 0xFFFF {
    3
  } else {
    4
  }
}

///|
fn utf16_width(ch : Char) -> Int {
  if ch.to_int() <= 0xFFFF {
    1
  } else {
    2
  }
}

///|
pub impl Tokenizer for WhitespaceAnalyzer with fn token_stream(_self, text) {
  WhitespaceTokenizer::new().token_stream(text)
}

///|
/// Eager convenience API for callers that need a token collection.
pub fn WhitespaceAnalyzer::analyze(
  self : WhitespaceAnalyzer,
  text : String,
) -> Array[Token] {
  collect_token_stream(self.token_stream(text))
}