///|
/// Wraps one TokenStream with a lazy token transformation.
pub(open) trait TokenFilter {
  fn transform(Self, &TokenStream) -> &TokenStream
}

///|
/// Unicode lowercase normalization.
pub struct LowerCaseFilter {}

///|
priv struct LowerCaseTokenStream {
  input : &TokenStream
  mut current : Token?
}

///|
pub fn LowerCaseFilter::new() -> LowerCaseFilter {
  LowerCaseFilter::{  }
}

///|
impl TokenStream for LowerCaseTokenStream with fn advance(self) {
  if !self.input.advance() {
    self.current = None
    return false
  }
  match self.input.token() {
    Some(token) => {
      self.current = Some({
        text: token.text.to_lower(),
        position: token.position,
        position_length: token.position_length,
        start_offset: token.start_offset,
        end_offset: token.end_offset,
      })
      true
    }
    None => {
      self.current = None
      false
    }
  }
}

///|
impl TokenStream for LowerCaseTokenStream with fn token(self) {
  self.current
}

///|
pub impl TokenFilter for LowerCaseFilter with fn transform(_self, input) {
  LowerCaseTokenStream::{ input, current: None }
}

///|
/// Removes tokens whose UTF-8 byte length exceeds the configured limit.
pub struct RemoveLongFilter {
  limit : Int
}

///|
priv struct RemoveLongTokenStream {
  input : &TokenStream
  limit : Int
  mut current : Token?
}

///|
pub fn RemoveLongFilter::new(limit : Int) -> RemoveLongFilter {
  guard limit >= 0 else { abort("token length limit must be non-negative") }
  { limit, }
}

///|
impl TokenStream for RemoveLongTokenStream with fn advance(self) {
  while self.input.advance() {
    match self.input.token() {
      Some(token) =>
        if @utf8.encode(token.text).length() <= self.limit {
          self.current = Some(token)
          return true
        }
      None => ()
    }
  }
  self.current = None
  false
}

///|
impl TokenStream for RemoveLongTokenStream with fn token(self) {
  self.current
}

///|
pub impl TokenFilter for RemoveLongFilter with fn transform(self, input) {
  RemoveLongTokenStream::{ input, limit: self.limit, current: None }
}

///|
/// Removes exact token texts contained in an immutable stop-word snapshot.
///
/// Place this filter after LowerCaseFilter when stop words are normalized to
/// lowercase. Removed tokens retain their position gaps in the surrounding
/// stream.
pub struct StopWordFilter {
  stop_words : @hashset.HashSet[String]
}

///|
priv struct StopWordTokenStream {
  input : &TokenStream
  stop_words : @hashset.HashSet[String]
  mut current : Token?
}

///|
pub fn StopWordFilter::new(stop_words : Array[String]) -> StopWordFilter {
  { stop_words: @hashset.HashSet(stop_words[:]) }
}

///|
impl TokenStream for StopWordTokenStream with fn advance(self) {
  while self.input.advance() {
    match self.input.token() {
      Some(token) =>
        if !self.stop_words.contains(token.text) {
          self.current = Some(token)
          return true
        }
      None => ()
    }
  }
  self.current = None
  false
}

///|
impl TokenStream for StopWordTokenStream with fn token(self) {
  self.current
}

///|
pub impl TokenFilter for StopWordFilter with fn transform(self, input) {
  StopWordTokenStream::{ input, stop_words: self.stop_words, current: None }
}

///|
/// Composes one Tokenizer with an ordered filter chain.
pub struct TextAnalyzer {
  tokenizer : &Tokenizer
  filters : Array[&TokenFilter]
}

///|
pub fn TextAnalyzer::new(tokenizer : &Tokenizer) -> TextAnalyzer {
  { tokenizer, filters: [] }
}

///|
pub fn TextAnalyzer::add_filter(
  self : TextAnalyzer,
  filter : &TokenFilter,
) -> Unit {
  self.filters.push(filter)
}

///|
pub impl Tokenizer for TextAnalyzer with fn token_stream(self, text) {
  let mut stream = self.tokenizer.token_stream(text)
  for filter in self.filters {
    stream = filter.transform(stream)
  }
  stream
}

///|
/// Eager convenience API built on top of the stream consumed by indexing and
/// query parsing.
pub fn TextAnalyzer::analyze(
  self : TextAnalyzer,
  text : String,
) -> Array[Token] {
  collect_token_stream(self.token_stream(text))
}