// Copyright 2024 International Digital Economy Academy
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Simplified, procedural-style decoding implementation
///| The Unicode Replacement Character, which is used to replace invalid or unrecognized sequences during lossy decoding.
/// https://unicode.org/charts/nameslist/n_FFF0.html
pub const U_REP = '\u{FFFD}'
// Result of trying to decode one character
priv enum DecoderResult {
Character(Char) // Successfully decoded character
NeedMoreBytes // Not enough bytes available for complete character
InvalidSequence(Int) // Invalid sequence, Int is number of bytes to skip
}
///| Create a decoder for the specified encoding
pub fn decoder(encoding : Encoding) -> Decoder {
// Use the existing Decoder struct but adapt it for our simplified approach
// We'll repurpose the fields:
// i = input buffer
// i_pos = input position
// t = temp buffer for partial characters
// t_len = temp buffer length
// t_need = encoding type (0=UTF8, 1=UTF16LE, 2=UTF16BE)
// k = dummy continuation (not used)
let encoding_num = match encoding {
UTF8 => 0
UTF16LE => 1
UTF16BE => 2
}
{
i: FixedArray::default(),
i_pos: 0,
t: FixedArray::make(4, Byte::default()),
t_len: 0,
t_need: encoding_num,
k: fn(_decoder) { End } // Dummy continuation, not used
}
}
///| Add input bytes to decoder's buffer
fn add_input(decoder : Decoder, input : @bytes.View) -> Unit {
if input.length() == 0 {
return
}
// Calculate remaining bytes in current buffer (i.length() - i_pos)
let remaining_bytes = decoder.i.length() - decoder.i_pos
let new_total_length = remaining_bytes + input.length()
// Create new buffer to hold all data
let new_buffer = FixedArray::make(new_total_length, Byte::default())
// Copy any remaining bytes from current buffer
if remaining_bytes > 0 {
decoder.i.blit_to(new_buffer, len=remaining_bytes, src_offset=decoder.i_pos)
}
// Add new input bytes
new_buffer.blit_from_bytesview(remaining_bytes, input)
// Update decoder
decoder.i = new_buffer
decoder.i_pos = 0
}
///| Check if decoder has any remaining input bytes
fn has_remaining_input(decoder : Decoder) -> Bool {
decoder.i_pos < decoder.i.length()
}
///| Get UTf-8 character length from first byte
fn get_utf8_char_length(first_byte : Int) -> Int {
if first_byte < 0x80 {
1 // ASCII character
} else if first_byte < 0xC0 {
0 // Invalid start byte (continuation byte)
} else if first_byte < 0xE0 {
2 // 2-byte character
} else if first_byte < 0xF0 {
3 // 3-byte character
} else if first_byte < 0xF8 {
4 // 4-byte character
} else {
0 // Invalid start byte
}
}
///| Try to decode one UTF-8 character from buffer at current position
fn try_decode_utf8_char(decoder : Decoder) -> DecoderResult {
let available_bytes = decoder.i.length() - decoder.i_pos
if available_bytes == 0 {
return NeedMoreBytes
}
let first_byte = decoder.i[decoder.i_pos].to_int()
let char_length = get_utf8_char_length(first_byte)
if char_length == 0 {
// Invalid first byte
return InvalidSequence(1)
}
if available_bytes < char_length {
// Not enough bytes available
return NeedMoreBytes
}
match char_length {
1 => {
// ASCII character
decoder.i_pos += 1
Character(first_byte.unsafe_to_char())
}
2 => {
// 2-byte UTF-8 character
let b1 = decoder.i[decoder.i_pos + 1].to_int()
if b1 >> 6 != 0b10 {
// Invalid continuation byte
InvalidSequence(1)
} else {
let codepoint = ((first_byte & 0x1F) << 6) | (b1 & 0x3F)
if codepoint < 0x80 {
// Overlong encoding
InvalidSequence(1)
} else {
decoder.i_pos += 2
Character(codepoint.unsafe_to_char())
}
}
}
3 => {
// 3-byte UTF-8 character
let b1 = decoder.i[decoder.i_pos + 1].to_int()
let b2 = decoder.i[decoder.i_pos + 2].to_int()
if b1 >> 6 != 0b10 || b2 >> 6 != 0b10 {
// Invalid continuation bytes
InvalidSequence(1)
} else {
let codepoint = ((first_byte & 0x0F) << 12) | ((b1 & 0x3F) << 6) | (b2 & 0x3F)
// Check for overlong sequences and surrogates
let valid = match first_byte {
0xE0 => b1 >= 0xA0 // No overlong
0xED => b1 <= 0x9F // No surrogates
_ => true
}
if not(valid) || codepoint < 0x800 {
InvalidSequence(1)
} else {
decoder.i_pos += 3
Character(codepoint.unsafe_to_char())
}
}
}
4 => {
// 4-byte UTF-8 character
let b1 = decoder.i[decoder.i_pos + 1].to_int()
let b2 = decoder.i[decoder.i_pos + 2].to_int()
let b3 = decoder.i[decoder.i_pos + 3].to_int()
if b1 >> 6 != 0b10 || b2 >> 6 != 0b10 || b3 >> 6 != 0b10 {
// Invalid continuation bytes
InvalidSequence(1)
} else {
let codepoint = ((first_byte & 0x07) << 18) |
((b1 & 0x3F) << 12) |
((b2 & 0x3F) << 6) |
(b3 & 0x3F)
// Check for overlong sequences and valid Unicode range
let valid = match first_byte {
0xF0 => b1 >= 0x90 // No overlong
0xF4 => b1 <= 0x8F // Within valid Unicode range
_ => first_byte <= 0xF4
}
if not(valid) || codepoint < 0x10000 || codepoint > 0x10FFFF {
InvalidSequence(1)
} else {
decoder.i_pos += 4
Character(codepoint.unsafe_to_char())
}
}
}
_ => InvalidSequence(1) // Should never happen
}
}
///| Try to decode one UTF-16LE character from buffer at current position
fn try_decode_utf16le_char(decoder : Decoder) -> DecoderResult {
let available_bytes = decoder.i.length() - decoder.i_pos
if available_bytes < 2 {
return NeedMoreBytes
}
let low_byte = decoder.i[decoder.i_pos].to_int()
let high_byte = decoder.i[decoder.i_pos + 1].to_int()
let code_unit = low_byte | (high_byte << 8)
// Check if high surrogate (need surrogate pair)
if code_unit >= 0xD800 && code_unit <= 0xDBFF {
if available_bytes < 4 {
return NeedMoreBytes
}
let low_byte2 = decoder.i[decoder.i_pos + 2].to_int()
let high_byte2 = decoder.i[decoder.i_pos + 3].to_int()
let low_surrogate = low_byte2 | (high_byte2 << 8)
if low_surrogate >= 0xDC00 && low_surrogate <= 0xDFFF {
// Valid surrogate pair
let codepoint = 0x10000 + ((code_unit - 0xD800) << 10) + (low_surrogate - 0xDC00)
decoder.i_pos += 4
Character(codepoint.unsafe_to_char())
} else {
// Invalid surrogate pair
InvalidSequence(2)
}
} else if code_unit >= 0xDC00 && code_unit <= 0xDFFF {
// Unexpected low surrogate
InvalidSequence(2)
} else {
// Regular BMP character
decoder.i_pos += 2
Character(code_unit.unsafe_to_char())
}
}
///| Try to decode one UTF-16BE character from buffer at current position
fn try_decode_utf16be_char(decoder : Decoder) -> DecoderResult {
let available_bytes = decoder.i.length() - decoder.i_pos
if available_bytes < 2 {
return NeedMoreBytes
}
let high_byte = decoder.i[decoder.i_pos].to_int()
let low_byte = decoder.i[decoder.i_pos + 1].to_int()
let code_unit = (high_byte << 8) | low_byte
// Check if high surrogate (need surrogate pair)
if code_unit >= 0xD800 && code_unit <= 0xDBFF {
if available_bytes < 4 {
return NeedMoreBytes
}
let high_byte2 = decoder.i[decoder.i_pos + 2].to_int()
let low_byte2 = decoder.i[decoder.i_pos + 3].to_int()
let low_surrogate = (high_byte2 << 8) | low_byte2
if low_surrogate >= 0xDC00 && low_surrogate <= 0xDFFF {
// Valid surrogate pair
let codepoint = 0x10000 + ((code_unit - 0xD800) << 10) + (low_surrogate - 0xDC00)
decoder.i_pos += 4
Character(codepoint.unsafe_to_char())
} else {
// Invalid surrogate pair
InvalidSequence(2)
}
} else if code_unit >= 0xDC00 && code_unit <= 0xDFFF {
// Unexpected low surrogate
InvalidSequence(2)
} else {
// Regular BMP character
decoder.i_pos += 2
Character(code_unit.unsafe_to_char())
}
}
///| Try to decode one character based on encoding
fn try_decode_char(decoder : Decoder) -> DecoderResult {
match decoder.t_need {
0 => try_decode_utf8_char(decoder) // UTF8
1 => try_decode_utf16le_char(decoder) // UTF16LE
2 => try_decode_utf16be_char(decoder) // UTF16BE
_ => try_decode_utf8_char(decoder) // fallback
}
}
///| Save remaining bytes to temp buffer for next decode call
fn save_remaining_bytes(decoder : Decoder) -> Unit {
let remaining = decoder.i.length() - decoder.i_pos
decoder.t_len = 0
let max_save = if remaining < 4 { remaining } else { 4 }
let mut i = 0
while i < max_save {
decoder.t[i] = decoder.i[decoder.i_pos + i]
i = i + 1
}
decoder.t_len = max_save
}
///| Process any bytes we have in temp buffer first
fn process_temp_buffer(decoder : Decoder, output : StringBuilder) -> Unit {
if decoder.t_len == 0 {
return
}
// Combine temp buffer with new input
let available_input = decoder.i.length() - decoder.i_pos
let total_available = decoder.t_len + available_input
if total_available == decoder.t_len {
// No new input, can't make progress
return
}
// Create combined buffer
let combined = FixedArray::make(total_available, Byte::default())
// Copy temp buffer first
let mut i = 0
while i < decoder.t_len {
combined[i] = decoder.t[i]
i = i + 1
}
// Copy input buffer
if available_input > 0 {
decoder.i.blit_to(combined,
len=available_input,
dst_offset=decoder.t_len,
src_offset=decoder.i_pos)
}
// Create temporary decoder to decode from combined buffer
let temp_decoder = {
i: combined,
i_pos: 0,
t: FixedArray::make(4, Byte::default()),
t_len: 0,
t_need: decoder.t_need,
k: decoder.k
}
match try_decode_char(temp_decoder) {
Character(ch) => {
output.write_char(ch)
// Update original decoder positions
let bytes_consumed = temp_decoder.i_pos
let consumed_from_input = bytes_consumed - decoder.t_len
if consumed_from_input > 0 {
decoder.i_pos += consumed_from_input
}
decoder.t_len = 0
}
NeedMoreBytes => ()
InvalidSequence(skip) => {
output.write_char(U_REP)
if skip <= decoder.t_len {
// Error was in temp buffer
let remaining_temp = decoder.t_len - skip
let mut i = 0
while i < remaining_temp {
decoder.t[i] = decoder.t[i + skip]
i = i + 1
}
decoder.t_len = remaining_temp
} else {
// Error spans temp and input
let skip_from_input = skip - decoder.t_len
decoder.i_pos += skip_from_input
decoder.t_len = 0
}
}
}
}
///| Main decoding loop - processes all available input
fn decode_loop(decoder : Decoder, output : StringBuilder, stream : Bool) -> Unit raise DecodingError {
while has_remaining_input(decoder) {
match try_decode_char(decoder) {
Character(ch) => {
output.write_char(ch)
}
NeedMoreBytes => {
if stream {
// Save remaining bytes for next call
save_remaining_bytes(decoder)
return
} else {
// End of input with incomplete character
let remaining = decoder.i.length() - decoder.i_pos
let error_bytes = FixedArray::make(remaining, Byte::default())
decoder.i.blit_to(error_bytes, len=remaining, src_offset=decoder.i_pos)
raise DecodingError::Truncated(Bytes::from_fixedarray(error_bytes, len=remaining))
}
}
InvalidSequence(skip) => {
raise DecodingError::Malformed({
let error_bytes = FixedArray::make(skip, Byte::default())
decoder.i.blit_to(error_bytes, len=skip, src_offset=decoder.i_pos)
Bytes::from_fixedarray(error_bytes, len=skip)
})
}
}
}
}
///| Lossy decoding loop - replaces invalid sequences with U_REP
fn decode_loop_lossy(decoder : Decoder, output : StringBuilder, stream : Bool) -> Unit {
while has_remaining_input(decoder) {
match try_decode_char(decoder) {
Character(ch) => {
output.write_char(ch)
}
NeedMoreBytes => {
if stream {
// Save remaining bytes for next call
save_remaining_bytes(decoder)
return
} else {
// End of input - just stop here
return
}
}
InvalidSequence(skip) => {
// Replace with replacement character and skip invalid bytes
output.write_char(U_REP)
decoder.i_pos += skip
}
}
}
}
///| Decode bytes and write result to StringBuilder
pub fn Decoder::decode_to(
self : Decoder,
input : @bytes.View,
output : StringBuilder,
stream~ : Bool = false
) -> Unit raise DecodingError {
// Add new input to our buffer
add_input(self, input)
// Process any partial character from previous call
process_temp_buffer(self, output)
// Process all available input
decode_loop(self, output, stream)
}
///| Decode bytes to string
pub fn Decoder::decode(
self : Decoder,
input : @bytes.View,
stream~ : Bool = false
) -> String raise DecodingError {
let builder = StringBuilder::new(size_hint=input.length())
self.decode_to(input, builder, stream~)
builder.to_string()
}
///| Decode bytes and write result to StringBuilder (lossy)
pub fn Decoder::decode_lossy_to(
self : Decoder,
input : @bytes.View,
output : StringBuilder,
stream~ : Bool = false
) -> Unit {
// Add new input to our buffer
add_input(self, input)
// Process any partial character from previous call (lossy)
process_temp_buffer(self, output)
// Process all available input (lossy)
decode_loop_lossy(self, output, stream)
}
///| Decode bytes to string (lossy)
pub fn Decoder::decode_lossy(
self : Decoder,
input : @bytes.View,
stream~ : Bool = false
) -> String {
let builder = StringBuilder::new(size_hint=input.length())
self.decode_lossy_to(input, builder, stream~)
builder.to_string()
}
///| Finish decoding and return any remaining content
pub fn Decoder::finish(self : Decoder) -> String raise DecodingError {
self.decode(b"", stream=false)
}
///| Finish decoding in lossy mode
pub fn Decoder::lossy_finish(self : Decoder) -> String {
self.decode_lossy(b"", stream=false)
}
// Legacy compatibility methods
///| Legacy method: same as decode
pub fn Decoder::consume(
self : Decoder,
input : @bytes.View
) -> String raise DecodingError {
self.decode(input, stream=false)
}
///| Legacy method: same as decode_lossy
pub fn Decoder::lossy_consume(
self : Decoder,
input : @bytes.View
) -> String {
self.decode_lossy(input, stream=false)
}
// Convenience functions
///| Decode bytes to string in one call
pub fn decode(
bytes : @bytes.View,
encoding~ : Encoding = UTF8
) -> String raise DecodingError {
let dec = decoder(encoding)
dec.decode(bytes)
}
///| Decode bytes and write to StringBuilder in one call
pub fn decode_to(
input : @bytes.View,
output : StringBuilder,
encoding~ : Encoding = UTF8
) -> Unit raise DecodingError {
let dec = decoder(encoding)
dec.decode_to(input, output)
}
///| Decode bytes to string in one call (lossy)
pub fn decode_lossy(
input : @bytes.View,
encoding~ : Encoding = UTF8
) -> String {
let dec = decoder(encoding)
dec.decode_lossy(input)
}
///| Decode bytes and write to StringBuilder in one call (lossy)
pub fn decode_lossy_to(
input : @bytes.View,
output : StringBuilder,
encoding~ : Encoding = UTF8
) -> Unit {
let dec = decoder(encoding)
dec.decode_lossy_to(input, output)
}