///|
/// Text normalization controls for dictionaries and search keys.
pub(all) struct NormalizationOptions {
  lowercase : Bool
  trim : Bool
  collapse_whitespace : Bool
  map_separators : Bool
  drop_controls : Bool
  max_length : Int
} derive(Debug, Eq, ToJson)

///|
/// Defaults for case-insensitive human-language lookup.
pub fn NormalizationOptions::default() -> NormalizationOptions {
  {
    lowercase: true,
    trim: true,
    collapse_whitespace: true,
    map_separators: false,
    drop_controls: true,
    max_length: 0,
  }
}

///|
/// A reusable normalizer keeps policy configuration separate from data.
pub struct Normalizer {
  options : NormalizationOptions
} derive(Debug)

///|
pub fn Normalizer::new(options : NormalizationOptions) -> Normalizer {
  { options, }
}

///|
pub fn Normalizer::apply(self : Normalizer, text : String) -> String {
  normalize_with_report(text, self.options).text
}

///|
pub fn Normalizer::apply_entry(self : Normalizer, item : Entry) -> Entry {
  normalize_entry(item, self.options)
}

///|
pub fn Normalizer::apply_entries(
  self : Normalizer,
  entries : ArrayView[Entry],
) -> Array[Entry] {
  normalize_entries(entries, self.options)
}

///|
/// Detailed information about one normalization pass.
pub(all) struct NormalizationReport {
  text : String
  input_length : Int
  output_length : Int
  changed_count : Int
  whitespace_count : Int
  dropped_count : Int
  truncated : Bool
} derive(Debug, Eq, ToJson)

///|
/// Normalize text while retaining counters useful for data-quality reports.
pub fn normalize_with_report(
  text : String,
  options : NormalizationOptions,
) -> NormalizationReport {
  let input = text.to_array()
  let output : Array[Char] = []
  let mut whitespace_count = 0
  let mut dropped_count = 0
  let mut pending_space = false
  for ch in input {
    if is_normalization_whitespace(ch) {
      whitespace_count += 1
      if options.collapse_whitespace {
        pending_space = output.length() > 0
      } else {
        output.push(ch)
      }
    } else if options.drop_controls && ch.to_int() < 32 {
      dropped_count += 1
    } else {
      if pending_space {
        output.push(' ')
        pending_space = false
      }
      output.push(
        if options.map_separators && is_separator(ch) {
          ' '
        } else {
          ch
        },
      )
    }
  }
  let collapsed = chars_to_normalized_string(output)
  let trimmed = if options.trim {
    collapsed.trim().to_owned()
  } else {
    collapsed
  }
  let cased = if options.lowercase { trimmed.to_lower() } else { trimmed }
  let limited_chars = cased.to_array()
  let truncated = options.max_length > 0 &&
    limited_chars.length() > options.max_length
  let final_text = if truncated {
    chars_to_normalized_string(
      [
        for index in 0.. limited_chars[index]
      ],
    )
  } else {
    cased
  }
  let changed_count = count_changed_characters(input, final_text.to_array())
  {
    text: final_text,
    input_length: input.length(),
    output_length: final_text.length(),
    changed_count,
    whitespace_count,
    dropped_count,
    truncated,
  }
}

///|
/// Normalize one entry without changing its label or weight.
pub fn normalize_entry(item : Entry, options : NormalizationOptions) -> Entry {
  {
    keyword: normalize_for_lookup(item.keyword, options),
    label: if options.lowercase {
      item.label.to_lower()
    } else {
      item.label
    },
    weight: item.weight,
  }
}

///|
/// Normalize and remove entries whose keyword becomes empty.
pub fn normalize_entries(
  entries : ArrayView[Entry],
  options : NormalizationOptions,
) -> Array[Entry] {
  let output : Array[Entry] = []
  for item in entries {
    let normalized = normalize_entry(item, options)
    if normalized.keyword.length() > 0 {
      output.push(normalized)
    }
  }
  deduplicate_entries(output)
}

///|
/// Compile a dictionary after applying the same normalization policy to every
/// keyword. Callers should normalize queries with the same options.
pub fn compile_normalized(
  entries : ArrayView[Entry],
  options : NormalizationOptions,
) -> Lexicon {
  compile(normalize_entries(entries, options))
}

///|
/// Produce a normalized query key.
pub fn normalize_for_lookup(
  query : String,
  options : NormalizationOptions,
) -> String {
  normalize_with_report(query, options).text
}

///|
/// Look up an exact normalized key in a normalized lexicon.
pub fn normalized_lookup(
  lexicon : Lexicon,
  query : String,
  options : NormalizationOptions,
) -> Array[Entry] {
  lexicon.lookup(normalize_for_lookup(query, options))
}

///|
/// Return true when normalization changes a string.
pub fn needs_normalization(
  text : String,
  options : NormalizationOptions,
) -> Bool {
  normalize_for_lookup(text, options) != text
}

///|
/// Count whitespace characters using the same classifier as the normalizer.
pub fn count_normalization_whitespace(text : String) -> Int {
  let mut count = 0
  for ch in text {
    if is_normalization_whitespace(ch) {
      count += 1
    }
  }
  count
}

///|
/// Return the normalized keyword length of an entry.
pub fn normalized_keyword_length(
  item : Entry,
  options : NormalizationOptions,
) -> Int {
  normalize_for_lookup(item.keyword, options).length()
}

///|
/// Return a report for every entry, retaining source order.
pub(all) struct EntryNormalizationReport {
  original : Entry
  normalized : Entry
  changed : Bool
  issue_count : Int
} derive(Debug, Eq, ToJson)

///|
pub fn normalization_reports(
  entries : ArrayView[Entry],
  options : NormalizationOptions,
) -> Array[EntryNormalizationReport] {
  let output : Array[EntryNormalizationReport] = []
  for item in entries {
    let normalized = normalize_entry(item, options)
    output.push({
      original: item,
      normalized,
      changed: item != normalized,
      issue_count: validate_entry(normalized).length(),
    })
  }
  output
}

///|
/// Render normalization diagnostics as TSV.
pub fn normalization_report_tsv(
  entries : ArrayView[Entry],
  options : NormalizationOptions,
) -> String {
  let output = StringBuilder()
  output.write_string("original\t normalized\tchanged\tissues\n")
  for report in normalization_reports(entries, options) {
    let changed = if report.changed { "1" } else { "0" }
    output.write_string(
      "\{escape_dictionary_field(report.original.keyword)}\t\{escape_dictionary_field(report.normalized.keyword)}\t\{changed}\t\{report.issue_count}\n",
    )
  }
  output.to_string()
}

///|
/// Return a readable policy description.
pub fn normalization_description(options : NormalizationOptions) -> String {
  let output = StringBuilder()
  if options.lowercase {
    output.write_string("lowercase,")
  }
  if options.trim {
    output.write_string("trim,")
  }
  if options.collapse_whitespace {
    output.write_string("collapse-whitespace,")
  }
  if options.map_separators {
    output.write_string("map-separators,")
  }
  if options.drop_controls {
    output.write_string("drop-controls,")
  }
  if options.max_length > 0 {
    output.write_string("limit-\{options.max_length}")
  }
  output.to_string()
}

///|
fn is_normalization_whitespace(ch : Char) -> Bool {
  ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r'
}

///|
fn is_separator(ch : Char) -> Bool {
  ch == '-' || ch == '_' || ch == '/'
}

///|
fn chars_to_normalized_string(chars : Array[Char]) -> String {
  let output = StringBuilder()
  for ch in chars {
    output.write_char(ch)
  }
  output.to_string()
}

///|
fn count_changed_characters(
  original : Array[Char],
  normalized : Array[Char],
) -> Int {
  let shared = if original.length() < normalized.length() {
    original.length()
  } else {
    normalized.length()
  }
  let mut changed = if original.length() > normalized.length() {
    original.length() - normalized.length()
  } else {
    normalized.length() - original.length()
  }
  for index in 0..