///|
fn is_ascii_letter(code : UInt16) -> Bool {
(code >= 65 && code <= 90) || (code >= 97 && code <= 122)
}
///|
fn is_ascii_digit(code : UInt16) -> Bool {
code >= 48 && code <= 57
}
///|
fn is_name_whitespace(code : UInt16) -> Bool {
code == 9 || code == 10 || code == 13 || code == 32
}
///|
fn is_name_hyphen(code : UInt16) -> Bool {
code == 45 || code == 0x2010 || code == 0x2011
}
///|
fn is_name_apostrophe(code : UInt16) -> Bool {
code == 39 || code == 0x2019
}
///|
fn is_ascii_punctuation(code : UInt16) -> Bool {
(code >= 33 && code <= 47) ||
(code >= 58 && code <= 64) ||
(code >= 91 && code <= 96) ||
(code >= 123 && code <= 126)
}
///|
fn case_ascii_letter(code : UInt16, policy : CasePolicy) -> String {
let converted = match policy {
Preserve => code
Upper => if code >= 97 && code <= 122 { code - 32 } else { code }
Lower => if code >= 65 && code <= 90 { code + 32 } else { code }
}
converted.unsafe_to_char().to_string()
}
///|
fn append_separator(output : String) -> String {
if output.length() == 0 || output[output.length() - 1] == 32 {
output
} else {
output + " "
}
}
///|
fn trim_trailing_separator(output : String) -> String {
if output.length() > 0 && output[output.length() - 1] == 32 {
let mut trimmed = ""
for i = 0; i < output.length() - 1; i = i + 1 {
trimmed = trimmed + output[i].unsafe_to_char().to_string()
}
trimmed
} else {
output
}
}
///|
fn apply_character_policy(
output : String,
code : UInt16,
index : Int,
policy : CharacterPolicy,
separator_when_kept : Bool,
) -> Result[(String, Int), NormalizationError] {
match policy {
Reject => Err(RejectedCharacter(index, code))
Drop => Ok((output, 1))
Keep => {
let next = if separator_when_kept {
append_separator(output)
} else {
output + code.unsafe_to_char().to_string()
}
Ok((next, 0))
}
}
}
///|
/// Splits text on ASCII whitespace and ignores repeated separators.
pub fn tokenize_normalized(input : String) -> Array[String] {
let tokens : Array[String] = []
let mut current = ""
for i = 0; i < input.length(); i = i + 1 {
let code = input[i]
if is_name_whitespace(code) {
if current.length() > 0 {
tokens.push(current)
current = ""
}
} else {
current = current + code.unsafe_to_char().to_string()
}
}
if current.length() > 0 {
tokens.push(current)
}
tokens
}
///|
/// Joins tokens with one ASCII space.
pub fn join_tokens(tokens : Array[String]) -> String {
let mut output = ""
for i = 0; i < tokens.length(); i = i + 1 {
if i > 0 {
output = output + " "
}
output = output + tokens[i]
}
output
}
///|
/// Normalizes a Latin-script name according to an explicit policy.
pub fn normalize_name(
input : String,
config : NormalizationConfig,
) -> Result[NormalizationResult, NormalizationError] {
match validate_normalization_config(config) {
Err(error) => return Err(error)
Ok(_) => ()
}
if input.length() > config.max_input_length {
return Err(InputTooLong(input.length(), config.max_input_length))
}
let mut output = ""
let mut dropped = 0
let mut folded = 0
for i = 0; i < input.length(); i = i + 1 {
let code = input[i]
if is_ascii_letter(code) {
output = output + case_ascii_letter(code, config.case_policy)
continue
}
if is_ascii_digit(code) {
match apply_character_policy(output, code, i, config.digits, false) {
Err(error) => return Err(error)
Ok((next, count)) => {
output = next
dropped = dropped + count
}
}
continue
}
if is_name_whitespace(code) {
match apply_character_policy(output, code, i, config.whitespace, true) {
Err(error) => return Err(error)
Ok((next, count)) => {
output = next
dropped = dropped + count
}
}
continue
}
if is_name_hyphen(code) {
match apply_character_policy(output, code, i, config.hyphen, true) {
Err(error) => return Err(error)
Ok((next, count)) => {
output = next
dropped = dropped + count
}
}
continue
}
if is_name_apostrophe(code) {
match apply_character_policy(output, 39, i, config.apostrophe, false) {
Err(error) => return Err(error)
Ok((next, count)) => {
output = next
dropped = dropped + count
}
}
continue
}
if is_ascii_punctuation(code) {
match apply_character_policy(output, code, i, config.punctuation, false) {
Err(error) => return Err(error)
Ok((next, count)) => {
output = next
dropped = dropped + count
}
}
continue
}
if config.fold_latin_diacritics {
let replacement = folded_latin_for_case(code, config.case_policy)
if replacement.length() > 0 {
output = output + replacement
folded = folded + 1
continue
}
}
return Err(RejectedCharacter(i, code))
}
output = trim_trailing_separator(output)
if output.length() == 0 {
return Err(EmptyNormalizedInput)
}
let notices : Array[NormalizationNotice] = []
if dropped > 0 {
notices.push(CharactersDropped(dropped))
}
if folded > 0 {
notices.push(DiacriticsFolded(folded))
}
Ok({
original: input,
normalized: output,
tokens: tokenize_normalized(output),
notices,
truncated: false,
})
}