///|
fn product(values : Array[Int64]) -> Int64 raise Hdf5Error {
let mut result = 1L
for value in values {
guard value > 0L && value <= 0x7FFFFFFFL else {
raise InvalidField("invalid chunk dimension")
}
guard result <= 0x7FFFFFFFL / value else {
raise InvalidField("chunk element count is too large")
}
result = result * value
}
result
}
///|
/// One allocated raw chunk discovered while walking a chunk index. The address
/// is the stored HDF5 address, not a file offset.
priv struct RawChunkRecord {
offsets : Array[UInt64]
address : UInt64
size : UInt64
mask : UInt32
address_offset : Int32
address_width : Int32
stored_size_offset : Int32
stored_size_width : Int32
mask_offset : Int32
metadata_start : Int32
metadata_checksum_offset : Int32
page : RawChunkPage?
missing_block_index : Int32
}
///|
/// An unallocated chunk-index page whose reserved entry storage is writable.
/// Allocating the page also updates its bitmap bit and the checksum of the
/// metadata block that owns the bitmap. Fixed-array pages keep both the page
/// payload and bitmap in the same data block; extensible-array pages keep the
/// bitmap in a separate super block.
priv struct RawChunkPage {
start : Int32
entries : Int32
entry_size : Int32
checksum_offset : Int32
bitmap_offset : Int32
bitmap_mask : Byte
bitmap_metadata_start : Int32
bitmap_checksum_offset : Int32
address_width : Int32
stored_size_width : Int32
}
///|
fn push_raw_chunk_record(
records : Array[RawChunkRecord]?,
offsets : Array[UInt64],
address : UInt64,
size : UInt64,
) -> Unit {
match records {
Some(records) =>
records.push({
offsets,
address,
size,
mask: 0U,
address_offset: -1,
address_width: 0,
stored_size_offset: -1,
stored_size_width: 0,
mask_offset: -1,
metadata_start: -1,
metadata_checksum_offset: -1,
page: None,
missing_block_index: -1,
})
None => ()
}
}
///|
fn push_located_raw_chunk_record(
records : Array[RawChunkRecord]?,
offsets : Array[UInt64],
address : UInt64,
size : UInt64,
mask : UInt32,
address_offset : Int32,
address_width : Int32,
stored_size_offset : Int32,
stored_size_width : Int32,
metadata_start : Int32,
metadata_checksum_offset : Int32,
) -> Unit {
push_chunk_slot_record(
records, offsets, address, size, mask, address_offset, address_width, stored_size_offset,
stored_size_width, -1, metadata_start, metadata_checksum_offset,
)
}
///|
fn push_chunk_slot_record(
records : Array[RawChunkRecord]?,
offsets : Array[UInt64],
address : UInt64,
size : UInt64,
mask : UInt32,
address_offset : Int32,
address_width : Int32,
stored_size_offset : Int32,
stored_size_width : Int32,
mask_offset : Int32,
metadata_start : Int32,
metadata_checksum_offset : Int32,
) -> Unit {
push_chunk_page_slot_record(
records,
offsets,
address,
size,
mask,
address_offset,
address_width,
stored_size_offset,
stored_size_width,
mask_offset,
metadata_start,
metadata_checksum_offset,
None,
)
}
///|
fn push_chunk_page_slot_record(
records : Array[RawChunkRecord]?,
offsets : Array[UInt64],
address : UInt64,
size : UInt64,
mask : UInt32,
address_offset : Int32,
address_width : Int32,
stored_size_offset : Int32,
stored_size_width : Int32,
mask_offset : Int32,
metadata_start : Int32,
metadata_checksum_offset : Int32,
page : RawChunkPage?,
missing_block_index~ : Int32 = -1,
) -> Unit {
match records {
Some(records) =>
records.push({
offsets,
address,
size,
mask,
address_offset,
address_width,
stored_size_offset,
stored_size_width,
mask_offset,
metadata_start,
metadata_checksum_offset,
page,
missing_block_index,
})
None => ()
}
}
///|
fn chunk_byte_size(
dimensions : Array[UInt32],
element_size : Int32,
) -> Int32 raise Hdf5Error {
let count = product(dimensions.map(size => size.to_int64()))
guard element_size > 0 && count <= 0x7FFFFFFFL / element_size.to_int64() else {
raise InvalidField("chunk byte size is too large")
}
count.to_int() * element_size
}
///|
fn read_filtered_chunk(
bytes : Bytes,
address : UInt64,
base_address : UInt64,
stored_size : UInt64,
filters : Array[Filter],
mask : UInt32,
element_size : Int32,
expected_size : Int32,
) -> Bytes raise Hdf5Error {
guard stored_size > 0UL && stored_size <= 0x7FFFFFFFUL else {
raise InvalidField("invalid stored chunk size")
}
let data = Reader::at(bytes, address_to_offset(address, base_address)).read_bytes(
stored_size.to_int(),
)
decode_filters(data, filters, mask, element_size, expected_size)
}
///|
fn read_source_filtered_chunk(
source : &Hdf5Source,
address : UInt64,
base_address : UInt64,
stored_size : UInt64,
filters : Array[Filter],
mask : UInt32,
element_size : Int32,
expected_size : Int32,
) -> Bytes raise Hdf5Error {
guard stored_size > 0UL && stored_size <= 0x7FFFFFFFUL else {
raise InvalidField("invalid stored chunk size")
}
let offset = source_address_to_offset(address, base_address)
let data = source.read_at(offset, stored_size.to_int())
decode_filters(data, filters, mask, element_size, expected_size)
}
///|
fn selected_chunk_index(
index : UInt64,
offsets : Array[UInt64],
chunk_dimensions : Array[UInt32],
dataspace_dimensions : Array[UInt64],
) -> UInt64? raise Hdf5Error {
let mut remaining = index
let mut chunk_index = 0UL
let mut multiplier = 1UL
let mut dimension = dataspace_dimensions.length()
while dimension > 0 {
dimension = dimension - 1
let dataspace_size = dataspace_dimensions[dimension]
guard dataspace_size > 0UL else {
raise InvalidField("selected dataset dimension is empty")
}
let coordinate = remaining % dataspace_size
remaining = remaining / dataspace_size
if coordinate < offsets[dimension] ||
coordinate - offsets[dimension] >= chunk_dimensions[dimension].to_uint64() {
return None
}
chunk_index = chunk_index + (coordinate - offsets[dimension]) * multiplier
multiplier = multiplier * chunk_dimensions[dimension].to_uint64()
}
Some(chunk_index)
}
///|
fn chunk_contains_selection(
offsets : Array[UInt64],
chunk_dimensions : Array[UInt32],
dataspace_dimensions : Array[UInt64],
indices : Array[UInt64]?,
) -> Bool raise Hdf5Error {
guard indices is Some(selected) else { return true }
for index in selected {
if selected_chunk_index(
index, offsets, chunk_dimensions, dataspace_dimensions,
)
is Some(_) {
return true
}
}
false
}
///|
fn copy_chunk(
chunk_data : Bytes,
chunk_size : Int32,
offsets : Array[UInt64],
chunk_dimensions : Array[UInt32],
dataspace_dimensions : Array[UInt64],
element_size : Int32,
output : FixedArray[Byte],
selection : Array[UInt64]?,
) -> Unit raise Hdf5Error {
let ndims = chunk_dimensions.length()
let chunk_dimensions_i64 = chunk_dimensions.map(value => value.to_int64())
let chunk_count = product(chunk_dimensions_i64)
guard element_size > 0 &&
chunk_size >= 0 &&
chunk_size <= chunk_data.length() &&
chunk_count <= chunk_size.to_int64() / element_size.to_int64() else {
raise InvalidField("chunk is smaller than its declared shape")
}
if selection is Some(indices) {
for selected in 0.. 0 {
dimension = dimension - 1
let size = chunk_dimensions_i64[dimension]
coordinates[dimension] = remaining % size
remaining = remaining / size
}
let mut destination = 0L
let mut outside = false
for i in 0..= dataspace_dimensions[i].reinterpret_as_int64() {
outside = true
break
}
guard dataspace_dimensions[i] <= 0x7FFFFFFFUL &&
destination <=
(0x7FFFFFFFL - target) / dataspace_dimensions[i].reinterpret_as_int64() else {
raise InvalidField("chunk destination index is out of range")
}
destination = destination * dataspace_dimensions[i].reinterpret_as_int64() +
target
}
if outside {
continue
}
guard destination >= 0L && destination <= 0x7FFFFFFFL else {
raise InvalidField("chunk destination index is out of range")
}
let source_start = linear * element_size
guard element_size <= output.length() &&
destination <=
(output.length() - element_size).to_int64() / element_size.to_int64() else {
raise InvalidField("chunk destination exceeds output buffer")
}
let destination_start = destination.to_int() * element_size
output.blit_from_bytes(
destination_start, chunk_data, source_start, element_size,
)
}
}
///|
fn read_chunk_btree(
bytes : Bytes,
node_address : UInt64,
expected_level : Int32,
chunk_dimensions : Array[UInt32],
dataspace_dimensions : Array[UInt64],
element_size : Int32,
offset_size : Int32,
base_address : UInt64,
output : FixedArray[Byte],
filters : Array[Filter],
selection : Array[UInt64]?,
records~ : Array[RawChunkRecord]? = None,
) -> Unit raise Hdf5Error {
let offset = address_to_offset(node_address, base_address)
let reader = Reader::at(bytes, offset)
guard reader.read_bytes(4) == b"TREE" else {
raise InvalidField("expected chunk B-tree node")
}
guard reader.read_u8() == b'\x01' else {
raise InvalidField("B-tree node is not a raw-data chunk node")
}
let node_level = reader.read_u8().to_int()
if expected_level >= 0 {
guard node_level == expected_level else {
raise InvalidField("unexpected chunk B-tree level")
}
}
let entries_used = reader.read_u16_le().to_int()
let _left_sibling = reader.read_uint_le(offset_size)
let _right_sibling = reader.read_uint_le(offset_size)
let ndims = chunk_dimensions.length()
let key_dimensions = ndims + 1
for _ in 0.. 0 {
read_chunk_btree(
bytes,
child,
node_level - 1,
chunk_dimensions,
dataspace_dimensions,
element_size,
offset_size,
base_address,
output,
filters,
selection,
records~,
)
} else {
guard chunk_size >= 0 else { raise InvalidField("negative chunk size") }
push_located_raw_chunk_record(
records,
offsets.copy(),
child,
chunk_size.to_uint64(),
filter_mask,
address_offset,
offset_size,
stored_size_offset,
4,
-1,
-1,
)
if !chunk_contains_selection(
offsets, chunk_dimensions, dataspace_dimensions, selection,
) {
continue
}
let decoded = read_filtered_chunk(
bytes,
child,
base_address,
chunk_size.to_uint64(),
filters,
filter_mask,
element_size,
chunk_byte_size(chunk_dimensions, element_size),
)
copy_chunk(
decoded,
decoded.length(),
offsets,
chunk_dimensions,
dataspace_dimensions,
element_size,
output,
selection,
)
}
}
let _final_size = reader.read_u32_le()
let _final_filter_mask = reader.read_u32_le()
for _ in 0.. Unit raise Hdf5Error {
let offset = source_address_to_offset(node_address, base_address)
let prefix_size = 8 + offset_size * 2
let prefix = Reader::new(source.read_at(offset, prefix_size))
guard prefix.read_bytes(4) == b"TREE" else {
raise InvalidField("expected chunk B-tree node")
}
guard prefix.read_u8() == b'\x01' else {
raise InvalidField("B-tree node is not a raw-data chunk node")
}
let node_level = prefix.read_u8().to_int()
if expected_level >= 0 {
guard node_level == expected_level else {
raise InvalidField("unexpected chunk B-tree level")
}
}
let entries_used = prefix.read_u16_le().to_int()
let key_dimensions = chunk_dimensions.length() + 1
let key_size = 8 + key_dimensions * 8
let entry_size = key_size + offset_size
let total_size = prefix_size + entries_used * entry_size + key_size
let reader = Reader::new(source.read_at(offset, total_size))
reader.skip(prefix_size)
for _ in 0.. 0 {
read_source_chunk_btree(
source,
child,
node_level - 1,
chunk_dimensions,
dataspace_dimensions,
element_size,
offset_size,
base_address,
output,
filters,
selection,
)
} else {
guard chunk_size >= 0 else { raise InvalidField("negative chunk size") }
if !chunk_contains_selection(
offsets, chunk_dimensions, dataspace_dimensions, selection,
) {
continue
}
let decoded = read_source_filtered_chunk(
source,
child,
base_address,
chunk_size.to_uint64(),
filters,
filter_mask,
element_size,
chunk_byte_size(chunk_dimensions, element_size),
)
copy_chunk(
decoded,
decoded.length(),
offsets,
chunk_dimensions,
dataspace_dimensions,
element_size,
output,
selection,
)
}
}
}
///|
fn read_source_chunked_with_filters(
source : &Hdf5Source,
layout : DataLayout,
dataspace : Dataspace,
datatype : Datatype,
offset_size : Int32,
length_size : Int32,
base_address : UInt64,
fill_value : Bytes,
filters : Array[Filter],
selection : Array[UInt64],
) -> Bytes raise Hdf5Error {
guard dataspace.kind == Simple && dataspace.dimensions.length() > 0 else {
raise InvalidField("chunked datasets require a simple dataspace")
}
guard layout.chunk_dimensions.length() == dataspace.dimensions.length() else {
raise InvalidField("chunk rank does not match the dataspace rank")
}
chunk_byte_size(layout.chunk_dimensions, datatype.size) |> ignore
if layout.version == 4 {
guard layout.element_size == datatype.size else {
raise InvalidField("chunk element size does not match datatype")
}
match layout.chunk_index {
Single(_, _) => {
guard dataspace.maximum_dimensions == dataspace.dimensions &&
dataspace.dimensions ==
layout.chunk_dimensions.map(size => size.to_uint64()) else {
raise InvalidField("single chunk dimensions must match dataspace")
}
}
Implicit => {
guard filters.length() == 0 else {
raise InvalidField("implicit chunks cannot have filters")
}
}
BTreeV1 => raise InvalidField("v4 layout cannot use a v1 B-tree index")
FixedArray(_) | ExtensibleArray(_) | BTreeV2(_) => ()
}
}
let output = filled_payload(
selection.length().to_int64(),
datatype.size,
fill_value,
)
guard offset_size == 1 ||
offset_size == 2 ||
offset_size == 4 ||
offset_size == 8 else {
raise UnsupportedAddressSize(offset_size)
}
let undefined = if offset_size == 8 {
0xFFFFFFFFFFFFFFFFUL
} else {
(1UL << (offset_size * 8)) - 1UL
}
if layout.data_address == undefined {
return output.unsafe_reinterpret_as_bytes()
}
match layout.chunk_index {
BTreeV1 =>
read_source_chunk_btree(
source,
layout.data_address,
-1,
layout.chunk_dimensions,
dataspace.dimensions,
datatype.size,
offset_size,
base_address,
output,
filters,
Some(selection),
)
Single(_, _) | Implicit =>
read_source_direct_chunks(
source,
layout,
dataspace,
datatype,
base_address,
filters,
output,
Some(selection),
)
FixedArray(page_bits) =>
read_source_fixed_array_chunks(
source,
layout,
dataspace,
datatype,
offset_size,
length_size,
base_address,
filters,
output,
undefined,
page_bits,
Some(selection),
)
BTreeV2(config) =>
read_source_btree_v2_chunks(
source,
layout,
dataspace,
datatype,
offset_size,
length_size,
base_address,
filters,
output,
undefined,
config,
Some(selection),
)
ExtensibleArray(config) =>
read_source_extensible_array_chunks(
source,
layout,
dataspace,
datatype,
offset_size,
length_size,
base_address,
filters,
output,
undefined,
config,
Some(selection),
)
}
output.unsafe_reinterpret_as_bytes()
}
///|
/// Reconstructs an unfiltered chunked dataset into a contiguous buffer.
fn _read_chunked_payload(
bytes : Bytes,
layout : DataLayout,
dataspace : Dataspace,
datatype : Datatype,
offset_size : Int32,
base_address : UInt64,
fill_value~ : Bytes = b"",
length_size~ : Int32 = 8,
) -> Bytes raise Hdf5Error {
read_chunked_with_filters(
bytes,
layout,
dataspace,
datatype,
offset_size,
base_address,
fill_value,
[],
length_size,
)
}
///|
fn read_chunked_with_filters(
bytes : Bytes,
layout : DataLayout,
dataspace : Dataspace,
datatype : Datatype,
offset_size : Int32,
base_address : UInt64,
fill_value : Bytes,
filters : Array[Filter],
length_size : Int32,
selection~ : Array[UInt64]? = None,
records~ : Array[RawChunkRecord]? = None,
missing_blocks~ : Array[MissingExtensibleDataBlock]? = None,
missing_supers~ : Array[MissingExtensibleSuperBlock]? = None,
) -> Bytes raise Hdf5Error {
guard dataspace.kind == Simple && dataspace.dimensions.length() > 0 else {
raise InvalidField("chunked datasets require a simple dataspace")
}
guard layout.chunk_dimensions.length() == dataspace.dimensions.length() else {
raise InvalidField("chunk rank does not match the dataspace rank")
}
chunk_byte_size(layout.chunk_dimensions, datatype.size) |> ignore
if layout.version == 4 {
guard layout.element_size == datatype.size else {
raise InvalidField("chunk element size does not match datatype")
}
match layout.chunk_index {
Single(_, _) => {
guard dataspace.maximum_dimensions == dataspace.dimensions &&
dataspace.dimensions ==
layout.chunk_dimensions.map(size => size.to_uint64()) else {
raise InvalidField("single chunk dimensions must match dataspace")
}
}
Implicit => {
guard filters.length() == 0 else {
raise InvalidField("implicit chunks cannot have filters")
}
}
BTreeV1 => raise InvalidField("v4 layout cannot use a v1 B-tree index")
FixedArray(_) | ExtensibleArray(_) | BTreeV2(_) => ()
}
}
let count = dataspace.element_count()
guard count >= 0 && datatype.size > 0 else {
raise InvalidField("invalid chunked dataset size")
}
if selection is None && count > 0x7FFFFFFFL / datatype.size.to_int64() {
raise InvalidField("chunked dataset size overflow")
}
let output_count = match selection {
Some(indices) => indices.length().to_int64()
None => count
}
let output = filled_payload(output_count, datatype.size, fill_value)
guard offset_size == 1 ||
offset_size == 2 ||
offset_size == 4 ||
offset_size == 8 else {
raise UnsupportedAddressSize(offset_size)
}
let undefined = if offset_size == 8 {
0xFFFFFFFFFFFFFFFFUL
} else {
(1UL << (offset_size * 8)) - 1UL
}
if layout.data_address == undefined {
return output.unsafe_reinterpret_as_bytes()
}
match layout.chunk_index {
BTreeV1 =>
read_chunk_btree(
bytes,
layout.data_address,
-1,
layout.chunk_dimensions,
dataspace.dimensions,
datatype.size,
offset_size,
base_address,
output,
filters,
selection,
records~,
)
FixedArray(page_bits) =>
read_fixed_array_chunks(
bytes,
layout,
dataspace,
datatype,
offset_size,
length_size,
base_address,
filters,
output,
undefined,
page_bits,
selection,
records~,
)
ExtensibleArray(config) =>
read_extensible_array_chunks(
bytes,
layout,
dataspace,
datatype,
offset_size,
length_size,
base_address,
filters,
output,
undefined,
config,
selection,
records~,
missing_blocks~,
missing_supers~,
)
BTreeV2(config) =>
read_btree_v2_chunks(
bytes,
layout,
dataspace,
datatype,
offset_size,
length_size,
base_address,
filters,
output,
undefined,
config,
selection,
records~,
)
_ =>
read_direct_chunks(
bytes,
layout,
dataspace,
datatype,
base_address,
filters,
output,
selection,
records~,
)
}
output.unsafe_reinterpret_as_bytes()
}