///|
pub fn normalize_encoding(name : String) -> String {
let value = name.trim().to_upper().replace_all(old="_", new="-")
match value {
"UTF8" | "UTF-8" => "UTF-8"
"ISO8859-1" | "ISO-8859-1" | "LATIN1" | "LATIN-1" => "ISO8859-1"
"ISO8859-2" | "ISO-8859-2" | "LATIN2" | "LATIN-2" => "ISO8859-2"
"ISO8859-15" | "ISO-8859-15" | "LATIN9" | "LATIN-9" => "ISO8859-15"
_ => value.to_owned()
}
}
///|
fn iso8859_2_extra(code : Int) -> Char? {
match code {
0xA0 => Some('\u{A0}')
0xA1 => Some('\u{104}')
0xA2 => Some('\u{2D8}')
0xA3 => Some('\u{141}')
0xA4 => Some('\u{A4}')
0xA5 => Some('\u{13D}')
0xA6 => Some('\u{15A}')
0xA7 => Some('\u{A7}')
0xA8 => Some('\u{A8}')
0xA9 => Some('\u{160}')
0xAA => Some('\u{15E}')
0xAB => Some('\u{164}')
0xAC => Some('\u{179}')
0xAD => Some('\u{AD}')
0xAE => Some('\u{17D}')
0xAF => Some('\u{17B}')
0xB0 => Some('\u{B0}')
0xB1 => Some('\u{105}')
0xB2 => Some('\u{2DB}')
0xB3 => Some('\u{142}')
0xB4 => Some('\u{B4}')
0xB5 => Some('\u{13E}')
0xB6 => Some('\u{15B}')
0xB7 => Some('\u{2C7}')
0xB8 => Some('\u{B8}')
0xB9 => Some('\u{161}')
0xBA => Some('\u{15F}')
0xBB => Some('\u{165}')
0xBC => Some('\u{17A}')
0xBD => Some('\u{2DD}')
0xBE => Some('\u{17E}')
0xBF => Some('\u{17C}')
0xC0 => Some('\u{154}')
0xC1 => Some('\u{C1}')
0xC2 => Some('\u{C2}')
0xC3 => Some('\u{102}')
0xC4 => Some('\u{C4}')
0xC5 => Some('\u{139}')
0xC6 => Some('\u{106}')
0xC7 => Some('\u{C7}')
0xC8 => Some('\u{10C}')
0xC9 => Some('\u{C9}')
0xCA => Some('\u{118}')
0xCB => Some('\u{CB}')
0xCC => Some('\u{11A}')
0xCD => Some('\u{CD}')
0xCE => Some('\u{CE}')
0xCF => Some('\u{10E}')
0xD0 => Some('\u{110}')
0xD1 => Some('\u{143}')
0xD2 => Some('\u{147}')
0xD3 => Some('\u{D3}')
0xD4 => Some('\u{D4}')
0xD5 => Some('\u{150}')
0xD6 => Some('\u{D6}')
0xD7 => Some('\u{D7}')
0xD8 => Some('\u{158}')
0xD9 => Some('\u{16E}')
0xDA => Some('\u{DA}')
0xDB => Some('\u{170}')
0xDC => Some('\u{DC}')
0xDD => Some('\u{DD}')
0xDE => Some('\u{162}')
0xDF => Some('\u{DF}')
0xE0 => Some('\u{155}')
0xE1 => Some('\u{E1}')
0xE2 => Some('\u{E2}')
0xE3 => Some('\u{103}')
0xE4 => Some('\u{E4}')
0xE5 => Some('\u{13A}')
0xE6 => Some('\u{107}')
0xE7 => Some('\u{E7}')
0xE8 => Some('\u{10D}')
0xE9 => Some('\u{E9}')
0xEA => Some('\u{119}')
0xEB => Some('\u{EB}')
0xEC => Some('\u{11B}')
0xED => Some('\u{ED}')
0xEE => Some('\u{EE}')
0xEF => Some('\u{10F}')
0xF0 => Some('\u{111}')
0xF1 => Some('\u{144}')
0xF2 => Some('\u{148}')
0xF3 => Some('\u{F3}')
0xF4 => Some('\u{F4}')
0xF5 => Some('\u{151}')
0xF6 => Some('\u{F6}')
0xF7 => Some('\u{F7}')
0xF8 => Some('\u{159}')
0xF9 => Some('\u{16F}')
0xFA => Some('\u{FA}')
0xFB => Some('\u{171}')
0xFC => Some('\u{FC}')
0xFD => Some('\u{FD}')
0xFE => Some('\u{163}')
0xFF => Some('\u{2D9}')
_ => None
}
}
///|
fn iso8859_15_extra(code : Int) -> Char? {
match code {
0xA4 => Some('\u{20AC}')
0xA6 => Some('\u{160}')
0xA8 => Some('\u{161}')
0xB4 => Some('\u{17D}')
0xB8 => Some('\u{17E}')
0xBC => Some('\u{152}')
0xBD => Some('\u{153}')
0xBE => Some('\u{178}')
_ => None
}
}
///|
fn decode_single_byte(bytes : Bytes, encoding : String) -> String {
let builder = StringBuilder()
for i in 0..
match iso8859_2_extra(code) {
Some(value) => value
None => code.unsafe_to_char()
}
"ISO8859-15" =>
match iso8859_15_extra(code) {
Some(value) => value
None => code.unsafe_to_char()
}
_ => code.unsafe_to_char()
}
builder.write_char(ch)
}
builder.to_string()
}
///|
pub fn decode_bytes(bytes : Bytes, encoding : String) -> String {
match normalize_encoding(encoding) {
"UTF-8" => @unicode.to_utf8_string(bytes)
value => decode_single_byte(bytes, value)
}
}
///|
fn is_valid_utf8(bytes : Bytes) -> Bool {
let mut index = 0
while index < bytes.length() {
let first = bytes[index].to_uint().reinterpret_as_int()
if first < 0x80 {
index += 1
} else if first >= 0xC2 && first <= 0xDF {
if index + 1 >= bytes.length() {
return false
}
let second = bytes[index + 1].to_uint().reinterpret_as_int()
if second < 0x80 || second > 0xBF {
return false
}
index += 2
} else if first >= 0xE0 && first <= 0xEF {
if index + 2 >= bytes.length() {
return false
}
let second = bytes[index + 1].to_uint().reinterpret_as_int()
let third = bytes[index + 2].to_uint().reinterpret_as_int()
if second < 0x80 || second > 0xBF || third < 0x80 || third > 0xBF {
return false
}
if first == 0xE0 && second < 0xA0 {
return false
}
if first == 0xED && second > 0x9F {
return false
}
index += 3
} else if first >= 0xF0 && first <= 0xF4 {
if index + 3 >= bytes.length() {
return false
}
let second = bytes[index + 1].to_uint().reinterpret_as_int()
let third = bytes[index + 2].to_uint().reinterpret_as_int()
let fourth = bytes[index + 3].to_uint().reinterpret_as_int()
if second < 0x80 ||
second > 0xBF ||
third < 0x80 ||
third > 0xBF ||
fourth < 0x80 ||
fourth > 0xBF {
return false
}
if first == 0xF0 && second < 0x90 {
return false
}
if first == 0xF4 && second > 0x8F {
return false
}
index += 4
} else {
return false
}
}
true
}
///|
pub fn detect_aff_encoding(bytes : Bytes) -> String {
let length = bytes.length()
let mut index = 0
while index < length {
let mut line_start = index
while line_start < length &&
(bytes[line_start] == b' ' || bytes[line_start] == b'\t') {
line_start += 1
}
let mut line_end = line_start
while line_end < length &&
bytes[line_end] != b'\n' &&
bytes[line_end] != b'\r' {
line_end += 1
}
if line_start + 4 <= line_end &&
bytes[line_start] == b'S' &&
bytes[line_start + 1] == b'E' &&
bytes[line_start + 2] == b'T' {
let mut cursor = line_start + 3
while cursor < line_end &&
(bytes[cursor] == b' ' || bytes[cursor] == b'\t') {
cursor += 1
}
let builder = StringBuilder()
while cursor < line_end && bytes[cursor] != b' ' && bytes[cursor] != b'\t' {
builder.write_char(
bytes[cursor].to_uint().reinterpret_as_int().unsafe_to_char(),
)
cursor += 1
}
let value = normalize_encoding(builder.to_string())
if !value.is_empty() {
return value
}
}
index = line_end + 1
}
let mut has_high_byte = false
for i in 0..= 128 {
has_high_byte = true
break
}
}
if has_high_byte {
if is_valid_utf8(bytes) {
return "UTF-8"
}
}
"ISO8859-1"
}