///|
priv enum TextCharset {
Ascii
Utf8
Utf16
Utf16Be
Utf16Le
}
///|
fn text_charset(name : String) -> TextCharset raise SbeError {
match name.to_lower() {
"ascii" | "us-ascii" => Ascii
"" | "utf-8" => Utf8
"utf-16" => Utf16
"utf-16be" => Utf16Be
"utf-16le" => Utf16Le
_ => raise Unsupported("characterEncoding: \{name}")
}
}
///|
fn is_high_surrogate(word : Int32) -> Bool {
word >= 0xd800 && word <= 0xdbff
}
///|
fn is_low_surrogate(word : Int32) -> Bool {
word >= 0xdc00 && word <= 0xdfff
}
///|
fn validate_text_units(
units : ArrayView[UInt16],
path : String,
) -> Unit raise SbeError {
for i = 0; i < units.length(); {
let word = units[i].to_int()
if is_high_surrogate(word) {
if i + 1 >= units.length() || !is_low_surrogate(units[i + 1].to_int()) {
raise InvalidValue(path, "unpaired high surrogate in text")
}
continue i + 2
}
if is_low_surrogate(word) {
raise InvalidValue(path, "unpaired low surrogate in text")
}
continue i + 1
}
}
///|
fn encode_text(
text : String,
character_encoding : String,
path : String,
limit~ : Int32 = 16777216,
) -> Bytes raise SbeError {
let charset = text_charset(character_encoding)
let units = text.code_units()
if units.length() > limit {
raise LimitExceeded("text exceeds byte limit: \{path}")
}
validate_text_units(units, path)
let bytes = match charset {
Ascii =>
Bytes::makei(units.length(), i => {
let word = units[i].to_int()
if word > 127 {
raise InvalidValue(path, "non-ASCII character in ASCII field")
}
word.to_byte()
})
Utf8 => @utf8.encode(text)
Utf16 | Utf16Be | Utf16Le => {
// Match UTF-16's big-endian default and BOM. Explicit endian
// encodings have no implicit BOM. An empty value has zero bytes.
let prefix = if charset is Utf16 && !units.is_empty() { 2 } else { 0 }
let size = prefix + units.length() * 2
if size > limit {
raise LimitExceeded("text exceeds byte limit: \{path}")
}
Bytes::makei(size, i => {
if i < prefix {
if i == 0 {
b'\xfe'
} else {
b'\xff'
}
} else {
let index = i - prefix
let word = units[index / 2].to_int()
let high_byte = if charset is Utf16Le {
index % 2 != 0
} else {
index % 2 == 0
}
if high_byte {
(word >> 8).to_byte()
} else {
(word & 255).to_byte()
}
}
})
}
}
if bytes.length() > limit {
raise LimitExceeded("text exceeds byte limit: \{path}")
}
bytes
}
///|
fn utf16_word(bytes : Bytes, index : Int32, little : Bool) -> Int32 {
let first = bytes[index].to_int()
let second = bytes[index + 1].to_int()
if little {
first | (second << 8)
} else {
(first << 8) | second
}
}
///|
fn decode_utf16(bytes : Bytes, charset : TextCharset) -> String raise SbeError {
if bytes.length() % 2 != 0 {
raise InvalidMessage("UTF-16 field has an odd byte length")
}
let (start, little) = if charset is Utf16 && bytes.length() >= 2 {
match utf16_word(bytes, 0, false) {
0xfeff => (2, false)
0xfffe => (2, true)
_ => (0, false)
}
} else {
(0, charset is Utf16Le)
}
let text = StringBuilder()
for i = start; i < bytes.length(); {
let first = utf16_word(bytes, i, little)
let (scalar, width) = if is_high_surrogate(first) {
if i + 3 >= bytes.length() {
raise InvalidMessage("truncated UTF-16 surrogate pair")
}
let second = utf16_word(bytes, i + 2, little)
if !is_low_surrogate(second) {
raise InvalidMessage("invalid UTF-16 surrogate pair")
}
(0x10000 + ((first - 0xd800) << 10) + second - 0xdc00, 4)
} else {
(first, 2)
}
guard scalar.to_char() is Some(char) else {
raise InvalidMessage("invalid UTF-16 scalar")
}
text.write_char(char)
continue i + width
}
text.to_string()
}
///|
fn decode_text(
bytes : Bytes,
character_encoding : String,
) -> String raise SbeError {
let charset = text_charset(character_encoding)
match charset {
Utf16 | Utf16Be | Utf16Le => decode_utf16(bytes, charset)
Ascii => {
for byte in bytes {
if byte.to_int() > 127 {
raise InvalidMessage("non-ASCII byte in ASCII field")
}
}
@utf8.decode_lossy(bytes)
}
Utf8 =>
@utf8.decode(bytes) catch {
_ => raise InvalidMessage("malformed UTF-8 in text field")
}
}
}
///|
fn decode_fixed_text(
bytes : Bytes,
character_encoding : String,
) -> String raise SbeError {
let unit_width = match text_charset(character_encoding) {
Utf16 | Utf16Be | Utf16Le => 2
Ascii | Utf8 => 1
}
let mut end = 0
while end + unit_width <= bytes.length() {
if bytes[end] == 0 && (unit_width == 1 || bytes[end + 1] == 0) {
break
}
end += unit_width
}
for index in end..