///|
pub struct Tokenizer {
  dictionary : @core.Dictionary
  hmm_model : @core.HmmModel?
}

///|
pub fn Tokenizer::Tokenizer() -> Tokenizer raise JiebaError {
  let dictionary = @adaptor.default_dictionary() catch {
    error => raise InvalidDictionary(message=error.message())
  }
  let hmm_model = @adaptor.default_hmm() catch {
    error => raise InvalidDictionary(message=error.message())
  }
  { dictionary, hmm_model: Some(hmm_model) }
}

///|
pub fn Tokenizer::from_dictionary(
  dictionary : String,
) -> Tokenizer raise JiebaError {
  let parsed = @core.Dictionary::from_text(dictionary) catch {
    error => raise InvalidDictionary(message=error.message())
  }
  let hmm_model = @adaptor.default_hmm() catch {
    error => raise InvalidDictionary(message=error.message())
  }
  { dictionary: parsed, hmm_model: Some(hmm_model) }
}

///|
/// Creates a tokenizer from backend-independent resource text.
///
/// Platform-specific file, network, or database access should happen outside
/// this package and pass the resulting text through this constructor.
pub fn Tokenizer::from_resources(
  dictionary : String,
  hmm_model? : String,
) -> Tokenizer raise JiebaError {
  let parsed_dictionary = @core.Dictionary::from_text(dictionary) catch {
    error => raise InvalidDictionary(message=error.message())
  }
  let parsed_hmm = match hmm_model {
    Some(text) =>
      Some(@core.HmmModel::from_text(text)) catch {
        error => raise InvalidDictionary(message=error.message())
      }
    None => None
  }
  { dictionary: parsed_dictionary, hmm_model: parsed_hmm }
}

///|
pub fn Tokenizer::cut(
  self : Tokenizer,
  sentence : String,
  mode? : CutMode = Accurate,
  hmm? : Bool = true,
) -> Array[String] {
  let model = if hmm { self.hmm_model } else { None }
  self.dictionary.cut(sentence, mode=mode.to_core(), hmm_model=model)
}

///|
pub fn Tokenizer::tokenize(
  self : Tokenizer,
  sentence : String,
  mode? : CutMode = Accurate,
  hmm? : Bool = true,
) -> Array[Token] {
  let model = if hmm { self.hmm_model } else { None }
  self.dictionary
  .cut_spans(sentence, mode=mode.to_core(), hmm_model=model)
  .map(span => { word: span.word, start: span.start, end: span.end })
}

///|
pub fn Tokenizer::add_word(
  self : Tokenizer,
  word : String,
  frequency? : Int,
  tag? : String = "",
) -> Int {
  let actual = frequency.unwrap_or_else(() => {
    self.dictionary.suggest_frequency(word)
  })
  self.dictionary.add(word, frequency=actual, tag~)
  actual
}

///|
pub fn Tokenizer::delete_word(self : Tokenizer, word : String) -> Unit {
  self.dictionary.delete(word)
}

///|
pub fn Tokenizer::contains(self : Tokenizer, word : String) -> Bool {
  self.dictionary.contains(word)
}

///|
pub fn Tokenizer::load_user_dictionary(
  self : Tokenizer,
  dictionary : String,
) -> Unit raise JiebaError {
  self.dictionary.load_user_text(dictionary) catch {
    error => raise InvalidDictionary(message=error.message())
  }
}

///|
pub fn Tokenizer::suggest_frequency(self : Tokenizer, word : String) -> Int {
  self.dictionary.suggest_frequency(word)
}

///|
pub fn Tokenizer::suggest_split_frequency(
  self : Tokenizer,
  segments : Array[String],
) -> Int {
  self.dictionary.suggest_split_frequency(segments)
}