///|
/// Values accepted by the HDF5 new-file writer.
#derive(@debug.Debug)
pub(all) enum Hdf5WriteValues {
Int8Values(Array[Int32])
Int16Values(Array[Int32])
Int32Values(Array[Int32])
Int64Values(Array[Int64])
UInt8Values(Array[UInt32])
UInt16Values(Array[UInt32])
UInt32Values(Array[UInt32])
UInt64Values(Array[UInt64])
/// Little-endian fixed-point values wider than 8 bytes.
BigIntegerValues(Array[@bigint.BigInt], Int32, Bool)
Float16Values(Array[Float])
Float32Values(Array[Float])
Float64Values(Array[Double])
FixedStringValues(Array[String], Int32)
VariableStringValues(Array[String])
ObjectReferenceValues(Array[String?])
Complex64Values(Array[(Float, Float)])
Complex128Values(Array[(Double, Double)])
StructuredValues(Datatype, Array[DatasetValue])
}
///|
pub extend Hdf5WriteValues with @debug.Debug::{to_repr}
///|
fn writer_datatype(
class : Int32,
size : Int32,
version~ : Int32 = 3,
signed~ : Bool = false,
little_endian~ : Bool = true,
reference_kind~ : Int32 = 0,
bit_offset~ : Int32 = 0,
bit_precision~ : Int32 = 0,
vlen_is_string~ : Bool = false,
opaque_tag~ : String = "",
enum_members~ : Array[EnumMember] = [],
compound_members~ : Array[CompoundMember] = [],
base_types~ : Array[Datatype] = [],
array_dimensions~ : Array[UInt64] = [],
) -> Datatype {
{
class,
version,
size,
signed,
little_endian,
vlen_is_string,
enum_members,
compound_members,
base_types,
array_dimensions,
bit_offset,
bit_precision: if bit_precision > 0 {
bit_precision
} else if class == 0 || class == 1 {
size * 8
} else {
0
},
float_sign_position: 0,
float_exponent_position: 0,
float_exponent_size: 0,
float_mantissa_position: 0,
float_mantissa_size: 0,
float_exponent_bias: 0,
float_normalization: 0,
string_padding: if class == 3 {
1
} else {
0
},
string_charset: if class == 3 || (class == 9 && vlen_is_string) {
1
} else {
0
},
opaque_tag,
reference_kind,
}
}
///|
fn writer_integer_datatype(
signed : Bool,
size : Int32,
little_endian : Bool,
precision : Int32,
offset : Int32,
) -> Datatype raise Hdf5Error {
guard size >= 1 &&
size <= 8191 &&
precision > 0 &&
precision.to_int64() <= size.to_int64() * 8L &&
offset >= 0 &&
offset.to_int64() + precision.to_int64() <= size.to_int64() * 8L else {
raise InvalidArgument("writer integer datatype layout is invalid")
}
writer_datatype(
0,
size,
signed~,
little_endian~,
bit_offset=offset,
bit_precision=precision,
)
}
///|
/// Creates a standard little-endian integer datatype for structured writes.
pub fn Datatype::write_integer(
signed : Bool,
size : Int32,
precision? : Int32,
offset? : Int32,
) -> Datatype raise Hdf5Error {
let precision = match precision {
Some(value) => value
None => size * 8
}
let offset = match offset {
Some(value) => value
None => 0
}
writer_integer_datatype(signed, size, true, precision, offset)
}
///|
/// Creates an integer datatype with an explicit byte order for structured writes.
pub fn Datatype::write_integer_with_order(
signed : Bool,
size : Int32,
little_endian : Bool,
precision? : Int32,
offset? : Int32,
) -> Datatype raise Hdf5Error {
let precision = match precision {
Some(value) => value
None => size * 8
}
let offset = match offset {
Some(value) => value
None => 0
}
writer_integer_datatype(signed, size, little_endian, precision, offset)
}
///|
/// Creates a standard little-endian IEEE floating-point datatype.
pub fn Datatype::write_float(size : Int32) -> Datatype raise Hdf5Error {
Datatype::write_float_with_order(size, true)
}
///|
/// Creates an IEEE floating-point datatype with an explicit byte order for
/// structured writes.
pub fn Datatype::write_float_with_order(
size : Int32,
little_endian : Bool,
) -> Datatype raise Hdf5Error {
guard size == 2 || size == 4 || size == 8 || size == 10 || size == 16 else {
raise InvalidArgument("writer floating-point datatype width is invalid")
}
let datatype = writer_datatype(1, size, little_endian~)
match size {
2 =>
{
..datatype,
float_sign_position: 15,
float_exponent_position: 10,
float_exponent_size: 5,
float_mantissa_position: 0,
float_mantissa_size: 10,
float_exponent_bias: 15,
float_normalization: 2,
}
4 =>
{
..datatype,
float_sign_position: 31,
float_exponent_position: 23,
float_exponent_size: 8,
float_mantissa_position: 0,
float_mantissa_size: 23,
float_exponent_bias: 127,
float_normalization: 2,
}
8 =>
{
..datatype,
float_sign_position: 63,
float_exponent_position: 52,
float_exponent_size: 11,
float_mantissa_position: 0,
float_mantissa_size: 52,
float_exponent_bias: 1023,
float_normalization: 2,
}
_ =>
{
..datatype,
float_sign_position: 79,
float_exponent_position: 64,
float_exponent_size: 15,
float_mantissa_position: 0,
float_mantissa_size: 63,
float_exponent_bias: 16383,
float_normalization: 2,
bit_precision: 80,
}
}
}
///|
/// Creates an HDF5 time datatype backed by an unsigned integer.
pub fn Datatype::write_time(
size : Int32,
little_endian~ : Bool = true,
) -> Datatype raise Hdf5Error {
guard size >= 1 && size <= 8191 else {
raise InvalidArgument("writer time datatype width is invalid")
}
writer_datatype(2, size, little_endian~, bit_precision=size * 8)
}
///|
/// Creates a bitfield datatype with a declared offset and precision.
pub fn Datatype::write_bitfield(
size : Int32,
precision : Int32,
offset~ : Int32 = 0,
little_endian~ : Bool = true,
) -> Datatype raise Hdf5Error {
guard size >= 1 &&
size <= 8191 &&
precision > 0 &&
precision.to_int64() <= size.to_int64() * 8L &&
offset >= 0 &&
offset.to_int64() + precision.to_int64() <= size.to_int64() * 8L else {
raise InvalidArgument("writer bitfield layout is invalid")
}
writer_datatype(
4,
size,
little_endian~,
bit_offset=offset,
bit_precision=precision,
)
}
///|
/// Creates an opaque datatype with an optional ASCII tag.
pub fn Datatype::write_opaque(
size : Int32,
tag : String,
) -> Datatype raise Hdf5Error {
guard size > 0 else {
raise InvalidArgument("writer opaque datatype width is invalid")
}
let encoded = @utf8.encode(tag)
for byte in encoded {
guard byte > b'\x00' && byte < b'\x80' else {
raise InvalidArgument("opaque datatype tag must be non-NUL ASCII")
}
}
let padded = if encoded.length() == 0 {
0
} else {
round_up_to_eight(encoded.length() + 1)
}
guard padded <= 255 else {
raise InvalidArgument("opaque datatype tag is too long")
}
writer_datatype(5, size, opaque_tag=tag)
}
///|
/// Creates a variable-length sequence datatype with a fixed-size base type.
pub fn Datatype::write_vlen_sequence(
base : Datatype,
) -> Datatype raise Hdf5Error {
Datatype::write_vlen_sequence_with_address_size(base, 8)
}
///|
/// Creates a variable-length sequence datatype for a file with the given
/// on-disk heap address width.
pub fn Datatype::write_vlen_sequence_with_address_size(
base : Datatype,
address_size : Int32,
) -> Datatype raise Hdf5Error {
guard address_size == 4 || address_size == 8 else {
raise InvalidArgument(
"writer variable-length address size must be 4 or 8 bytes",
)
}
guard !datatype_contains_vlen(base) else {
raise InvalidArgument("writer variable-length sequence base is invalid")
}
writer_datatype(9, address_size + 8, base_types=[base])
}
///|
/// Creates a variable-length UTF-8 string datatype for structured writes.
pub fn Datatype::write_vlen_string() -> Datatype {
writer_datatype(9, 16, vlen_is_string=true, base_types=[writer_datatype(0, 1)])
}
///|
/// Creates a variable-length UTF-8 string datatype for a file with the given
/// on-disk heap address width.
pub fn Datatype::write_vlen_string_with_address_size(
address_size : Int32,
) -> Datatype raise Hdf5Error {
guard address_size == 4 || address_size == 8 else {
raise InvalidArgument(
"writer variable-length address size must be 4 or 8 bytes",
)
}
writer_datatype(9, address_size + 8, vlen_is_string=true, base_types=[
writer_datatype(0, 1),
])
}
///|
/// Creates a fixed-width UTF-8 string datatype.
pub fn Datatype::write_fixed_string(width : Int32) -> Datatype raise Hdf5Error {
guard width > 0 && width <= 65535 else {
raise InvalidArgument("fixed string width is invalid")
}
writer_datatype(3, width)
}
///|
/// Creates a legacy object-reference datatype for structured writes.
pub fn Datatype::write_object_reference() -> Datatype {
writer_datatype(7, 8)
}
///|
/// Creates a legacy object-reference datatype for a file with the given
/// on-disk address width.
pub fn Datatype::write_object_reference_with_address_size(
address_size : Int32,
) -> Datatype raise Hdf5Error {
guard address_size == 4 || address_size == 8 else {
raise InvalidArgument("writer reference address size must be 4 or 8 bytes")
}
writer_datatype(7, address_size)
}
///|
/// Creates a legacy dataset-region reference datatype.
pub fn Datatype::write_region_reference() -> Datatype {
writer_datatype(7, 12, reference_kind=1)
}
///|
/// Creates a legacy dataset-region reference datatype for a file with the given
/// on-disk address width.
pub fn Datatype::write_region_reference_with_address_size(
address_size : Int32,
) -> Datatype raise Hdf5Error {
guard address_size == 4 || address_size == 8 else {
raise InvalidArgument("writer reference address size must be 4 or 8 bytes")
}
writer_datatype(7, address_size + 4, reference_kind=1)
}
///|
/// Creates a version-4 object-reference datatype for structured writes.
pub fn Datatype::write_version4_object_reference() -> Datatype {
writer_datatype(7, 18, version=4, reference_kind=2)
}
///|
/// Creates a version-4 object-reference datatype for a file with the given
/// on-disk address width.
pub fn Datatype::write_version4_object_reference_with_address_size(
address_size : Int32,
) -> Datatype raise Hdf5Error {
guard address_size == 4 || address_size == 8 else {
raise InvalidArgument("writer reference address size must be 4 or 8 bytes")
}
writer_datatype(7, address_size + 10, version=4, reference_kind=2)
}
///|
/// Creates a version-4 dataset-region reference datatype.
pub fn Datatype::write_version4_region_reference() -> Datatype {
writer_datatype(7, 18, version=4, reference_kind=3)
}
///|
/// Creates a version-4 dataset-region reference datatype for a file with the
/// given on-disk address width.
pub fn Datatype::write_version4_region_reference_with_address_size(
address_size : Int32,
) -> Datatype raise Hdf5Error {
guard address_size == 4 || address_size == 8 else {
raise InvalidArgument("writer reference address size must be 4 or 8 bytes")
}
writer_datatype(7, address_size + 10, version=4, reference_kind=3)
}
///|
/// Creates a version-4 attribute-reference datatype.
pub fn Datatype::write_version4_attribute_reference() -> Datatype {
writer_datatype(7, 18, version=4, reference_kind=4)
}
///|
/// Creates a version-4 attribute-reference datatype for a file with the given
/// on-disk address width.
pub fn Datatype::write_version4_attribute_reference_with_address_size(
address_size : Int32,
) -> Datatype raise Hdf5Error {
guard address_size == 4 || address_size == 8 else {
raise InvalidArgument("writer reference address size must be 4 or 8 bytes")
}
writer_datatype(7, address_size + 10, version=4, reference_kind=4)
}
///|
/// Creates a fixed-size array datatype.
pub fn Datatype::write_array(
base : Datatype,
dimensions : Array[UInt64],
) -> Datatype raise Hdf5Error {
guard dimensions.length() > 0 && dimensions.length() <= 32 else {
raise InvalidArgument("writer array datatype rank is invalid")
}
let size = datatype_array_size(dimensions, base.size)
writer_datatype(
10,
size,
base_types=[base],
array_dimensions=dimensions.copy(),
)
}
///|
/// Creates a class-11 complex datatype with a floating-point component type.
/// Class-11 complex is a post-profile extension; use compound or two-element
/// array forms for HDF5 1.14.6 common-file interoperability.
pub fn Datatype::write_complex(base : Datatype) -> Datatype raise Hdf5Error {
guard base.class == 1 && base.size > 0 && base.size <= 0x7FFFFFFF / 2 else {
raise InvalidArgument(
"writer complex datatype requires a floating-point base",
)
}
writer_datatype(11, base.size * 2, version=4, base_types=[base])
}
///|
/// Creates an enumeration datatype with an integer base type.
pub fn Datatype::write_enum(
base : Datatype,
members : Array[EnumMember],
) -> Datatype raise Hdf5Error {
guard base.class == 0 && members.length() > 0 && members.length() <= 65535 else {
raise InvalidArgument("writer enum requires an integer base and members")
}
writer_datatype(
8,
base.size,
signed=base.signed,
enum_members=members.copy(),
base_types=[base],
)
}
///|
/// Creates a packed compound datatype in member order.
pub fn Datatype::write_compound(
members : Array[(String, Datatype)],
) -> Datatype raise Hdf5Error {
guard members.length() > 0 && members.length() <= 65535 else {
raise InvalidArgument("writer compound datatype requires members")
}
let fields : Array[CompoundMember] = []
let mut size = 0
for (name, datatype) in members {
guard size <= 0x7FFFFFFF - datatype.size else {
raise InvalidArgument("writer compound datatype is too large")
}
fields.push({ name, offset: size, datatype, })
size = size + datatype.size
}
writer_datatype(6, size, compound_members=fields)
}
///|
/// Filters available for chunked writer datasets.
#derive(Eq, @debug.Debug)
pub(all) enum Hdf5WriteFilter {
Shuffle
Deflate(Int32)
Fletcher32
/// Removes insignificant datatype bits before later filters are applied.
Nbit
/// Integer bits or decimal digits for floats; zero selects automatic bits for integers.
ScaleOffset(Int32)
/// Registered caller-defined filter ID and its little-endian client data.
Custom(Int32, Array[UInt32])
}
///|
pub extend Hdf5WriteFilter with Eq::{not_equal, equal}
///|
pub extend Hdf5WriteFilter with @debug.Debug::{to_repr}
///|
priv enum WritableStorage {
WritableCompact
WritableContiguous
WritableChunked(Array[UInt32], Array[Hdf5WriteFilter])
WritableExtensible(Array[UInt32], Array[Hdf5WriteFilter], Array[Int32])
WritableExternal(Array[Hdf5ExternalSlot])
WritableVirtual(Array[Hdf5VirtualMapping])
}
///|
/// One external raw-data slot stored outside the HDF5 file.
pub(all) struct Hdf5ExternalSlot {
file_name : String
file_offset : UInt64
data_size : UInt64
}
///|
priv struct WritableChunkFilterMask {
coordinates : Array[UInt64]
filter_mask : UInt32
}
///|
priv struct WritableDataset {
path : String
dimensions : Array[UInt64]
maximum_dimensions : Array[UInt64]
values : Hdf5WriteValues
storage : WritableStorage
fill_value : Hdf5WriteValues?
chunk_filter_masks : Array[WritableChunkFilterMask]
}
///|
priv struct WritableCommittedDatatype {
path : String
datatype : Datatype
data : Bytes
}
///|
priv struct WritableAttribute {
object_path : String
name : String
dimensions : Array[UInt64]
values : Hdf5WriteValues
creation_order : UInt64
}
///|
priv struct WritableSoftLink {
parent_path : String
name : String
target : String
}
///|
priv struct WritableExternalLink {
parent_path : String
name : String
target_file : String
target_path : String
}
///|
priv struct WritableLinkOrder {
order : UInt64
}
///|
priv struct WritableLinkOrderCounter {
next_order : UInt64
}
///|
/// In-memory builder for a new HDF5 file.
pub struct Hdf5Writer {
priv groups : Array[String]
priv datasets : Array[WritableDataset]
priv committed_datatypes : Array[WritableCommittedDatatype]
priv attributes : Array[WritableAttribute]
priv soft_links : Array[WritableSoftLink]
priv external_links : Array[WritableExternalLink]
priv link_orders : Array[WritableLinkOrder]
priv link_order_counters : Array[WritableLinkOrderCounter]
priv soft_link_indices : @hashmap.HashMap[String, Int32]
priv external_link_indices : @hashmap.HashMap[String, Int32]
priv link_order_indices : @hashmap.HashMap[String, Int32]
priv link_order_counter_indices : @hashmap.HashMap[String, Int32]
priv mut share_attributes : Bool
priv mut share_fills : Bool
priv mut share_filters : Bool
priv mut user_block_size : Int32
priv mut user_block : Bytes
priv custom_filters : Array[&Hdf5CustomFilter]
priv external_sources : Array[Hdf5ExternalSource]
}
///|
priv enum WritableObjectKind {
WritableGroup
WritableDatasetObject(Int32)
WritableCommittedDatatype(Int32)
}
///|
priv struct WritableObject {
path : String
kind : WritableObjectKind
address : UInt64
raw_address : UInt64
dataspace : Dataspace?
}
///|
priv struct EncodedWriteValues {
datatype : Bytes
payload : Bytes
element_size : Int32
}
///|
priv struct EncodedChunkEntry {
allocated : Bool
data : Bytes
address : UInt64
filter_mask : UInt32
}
///|
priv struct WritableGlobalHeap {
address : UInt64
region_address : UInt64
strings : Array[String]
regions : Array[Bytes]
sequences : Array[Bytes]
external_references : Array[WritableExternalReference]
}
///|
priv struct WritableExternalReference {
value : DatasetValue
payload : Bytes
}
///|
#derive(Eq)
priv enum WritableDenseKind {
WritableDenseLinks
WritableDenseAttributes
}
///|
priv struct WritableDenseMetadata {
path : String
kind : WritableDenseKind
heap_address : UInt64
index_address : UInt64
order_index_address : UInt64
}
///|
priv struct WritableExternalHeap {
path : String
address : UInt64
}
///|
priv struct WritableVirtualHeap {
path : String
address : UInt64
}
///|
priv struct WritableDenseHeapEntry {
name : String
hash : UInt32
data : Bytes
creation_order : UInt64
flags : Byte
shared_id : Bytes?
}
///|
priv struct WritableSharedAttribute {
path : String
name : String
data : Bytes
id : Bytes
}
///|
priv struct WritableSharedFill {
path : String
data : Bytes
id : Bytes
}
///|
priv struct WritableSharedFilter {
path : String
data : Bytes
id : Bytes
}
///|
priv struct WritableSharedHeap {
message_types : Int32
address : UInt64
messages : Array[Bytes]
}
///|
priv struct WritableSharedMetadata {
extension_address : UInt64
table_address : UInt64
heaps : Array[WritableSharedHeap]
fills : Array[WritableSharedFill]
filters : Array[WritableSharedFilter]
attributes : Array[WritableSharedAttribute]
message_types : Int32
message_count : Int32
}
///|
priv struct WritableLayout {
objects : Array[WritableObject]
heap : WritableGlobalHeap
dense : Array[WritableDenseMetadata]
external : Array[WritableExternalHeap]
virtual_heaps : Array[WritableVirtualHeap]
shared : WritableSharedMetadata?
}
///|
/// Creates an empty writer containing the root group.
pub fn Hdf5Writer::new() -> Hdf5Writer {
{
groups: ["/"],
datasets: [],
committed_datatypes: [],
attributes: [],
soft_links: [],
external_links: [],
link_orders: [],
link_order_counters: [],
soft_link_indices: @hashmap.HashMap([]),
external_link_indices: @hashmap.HashMap([]),
link_order_indices: @hashmap.HashMap([]),
link_order_counter_indices: @hashmap.HashMap([]),
share_attributes: false,
share_fills: false,
share_filters: false,
user_block_size: 0,
user_block: b"",
custom_filters: [],
external_sources: [],
}
}
///|
/// Reserves a zero-filled user block before the HDF5 superblock.
///
/// HDF5 requires a user block to be either absent or a power-of-two size of at
/// least 512 bytes. The in-memory writer accepts every power of two that fits
/// its 32-bit buffer offsets, up to 1 GiB. It always emits a version-3
/// superblock, so the declared base address is the user-block size and all
/// stored addresses remain relative to that base.
pub fn Hdf5Writer::set_user_block(
self : Hdf5Writer,
size : Int32,
) -> Unit raise Hdf5Error {
guard size == 0 || (size >= 512 && (size & (size - 1)) == 0) else {
raise InvalidArgument(
"user block size must be zero or a power of two of at least 512 bytes",
)
}
self.user_block_size = size
self.user_block = b""
}
///|
/// Reserves a user block containing caller-provided prefix bytes.
///
/// The block size is the smallest permitted power of two from 512 bytes through
/// 1 GiB that can hold `content`; remaining bytes are zero-filled. An empty
/// byte string removes the user block.
pub fn Hdf5Writer::set_user_block_content(
self : Hdf5Writer,
content : Bytes,
) -> Unit raise Hdf5Error {
let size = user_block_size_for_content(content.length())
self.user_block_size = size
self.user_block = content
}
///|
fn user_block_size_for_content(length : Int32) -> Int32 raise Hdf5Error {
guard length >= 0 && length <= 1073741824 else {
raise InvalidArgument("user block content exceeds 1073741824 bytes")
}
let mut size = if length == 0 { 0 } else { 512 }
while size < length {
size = size * 2
}
size
}
///|
/// Registers a source that external version-4 reference values may target.
///
/// The reference value stores the caller-provided file name; registration is
/// matched after path normalization. Duplicate names are rejected.
pub fn Hdf5Writer::add_external_source(
self : Hdf5Writer,
name : String,
source : &Hdf5Source,
) -> Unit raise Hdf5Error {
let normalized = resolver_file_name(name)
guard normalized != resolver_root_name() else {
raise InvalidArgument(" is reserved for the writer output")
}
guard !self.external_sources.any(item => item.name == normalized) else {
raise InvalidArgument(
"external source is already registered: " + normalized,
)
}
self.external_sources.push({ name: normalized, source, })
}
///|
/// Enables or disables writing whole attributes through the shared-object
/// header message (SOHM) table. Attributes whose serialized message is at most
/// 65,535 bytes are shared.
pub fn Hdf5Writer::set_attribute_sharing(
self : Hdf5Writer,
enabled : Bool,
) -> Unit {
self.share_attributes = enabled
}
///|
/// Enables or disables writing complete fill-value messages through the SOHM
/// table. Fill messages at most 65,535 bytes are shared; larger messages remain in
/// the object header.
pub fn Hdf5Writer::set_fill_sharing(self : Hdf5Writer, enabled : Bool) -> Unit {
self.share_fills = enabled
}
///|
/// Enables or disables writing complete filter-pipeline messages through the
/// SOHM table. Pipelines with equal encoded messages reuse one heap ID.
pub fn Hdf5Writer::set_filter_sharing(
self : Hdf5Writer,
enabled : Bool,
) -> Unit {
self.share_filters = enabled
}
///|
/// Registers a caller-defined filter for subsequent chunked datasets. The
/// filter's `encode` method is used by `Custom` pipeline entries, and its
/// `decode` method can read the resulting file through `Hdf5File`.
pub fn Hdf5Writer::add_custom_filter(
self : Hdf5Writer,
filter : &Hdf5CustomFilter,
) -> Unit raise Hdf5Error {
register_custom_filter(self.custom_filters, filter)
}
///|
fn writer_write_bytes(buffer : @buffer.Buffer, bytes : Bytes) -> Unit {
buffer.write_bytes(bytes.get_view().unwrap())
}
///|
fn writer_write_zeros(buffer : @buffer.Buffer, count : Int32) -> Unit {
for _ in 0.. Unit raise Hdf5Error {
guard address <= 0x7FFFFFFFUL &&
base_address.to_int64() + address.reinterpret_as_int64() <= 0x7FFFFFFFL else {
raise UnsupportedFeature("file offset is too large")
}
let target = base_address + address.to_int()
guard target >= buffer.length() else {
raise InvalidField("HDF5 write offsets are out of order")
}
writer_write_zeros(buffer, target - buffer.length())
writer_write_bytes(buffer, payload)
}
///|
fn writer_align(value : Int32) -> Int32 {
(value + 7) / 8 * 8
}
///|
fn writer_path_segments(path : String) -> Array[String] raise Hdf5Error {
guard path.has_prefix("/") else {
raise InvalidArgument("writer paths must be absolute")
}
let segments : Array[String] = []
for raw in path.split("/") {
let segment = raw.to_owned()
if segment != "" {
guard segment != "." && segment != ".." else {
raise InvalidArgument("writer paths cannot contain dot segments")
}
let encoded = @utf8.encode(segment)
guard encoded.length() <= 65535 else {
raise InvalidArgument("writer path component is too long")
}
for byte in encoded {
guard byte != b'\x00' else {
raise InvalidArgument("writer paths cannot contain NUL")
}
}
segments.push(segment)
}
}
segments
}
///|
fn writer_path(segments : Array[String]) -> String {
let mut path = ""
for segment in segments {
path = path + "/" + segment
}
if path == "" {
"/"
} else {
path
}
}
///|
fn canonical_writer_path(path : String) -> String raise Hdf5Error {
writer_path(writer_path_segments(path))
}
///|
fn writer_parent_path(path : String) -> String raise Hdf5Error {
let segments = writer_path_segments(path)
guard segments.length() > 0 else {
raise InvalidArgument("root has no parent")
}
segments.pop() |> ignore
writer_path(segments)
}
///|
fn writer_basename(path : String) -> String raise Hdf5Error {
let segments = writer_path_segments(path)
guard segments.length() > 0 else { raise InvalidArgument("root has no name") }
segments[segments.length() - 1]
}
///|
fn writer_value_count(values : Hdf5WriteValues) -> Int32 {
match values {
Int8Values(items) => items.length()
Int16Values(items) => items.length()
Int32Values(items) => items.length()
Int64Values(items) => items.length()
UInt8Values(items) => items.length()
UInt16Values(items) => items.length()
UInt32Values(items) => items.length()
UInt64Values(items) => items.length()
BigIntegerValues(items, _, _) => items.length()
Float16Values(items) => items.length()
Float32Values(items) => items.length()
Float64Values(items) => items.length()
FixedStringValues(items, _) => items.length()
VariableStringValues(items) => items.length()
ObjectReferenceValues(items) => items.length()
Complex64Values(items) => items.length()
Complex128Values(items) => items.length()
StructuredValues(_, items) => items.length()
}
}
///|
fn writer_value_element_size(values : Hdf5WriteValues) -> Int32 {
match values {
Int8Values(_) | UInt8Values(_) => 1
Int16Values(_) | UInt16Values(_) => 2
Int32Values(_) | UInt32Values(_) | Float32Values(_) => 4
Int64Values(_) | UInt64Values(_) | Float64Values(_) => 8
BigIntegerValues(_, size, _) => size
Float16Values(_) => 2
FixedStringValues(_, width) => width
VariableStringValues(_) => 16
ObjectReferenceValues(_) => 8
Complex64Values(_) => 8
Complex128Values(_) => 16
StructuredValues(datatype, _) => datatype.size
}
}
///|
/// Returns the size HDF5 assigns to a datatype in memory. Legacy references
/// and variable-length fields use fixed native widths that can be wider than
/// their file representation in four-byte-address files.
fn writer_native_datatype_size(
datatype : Datatype,
depth~ : Int32 = 0,
) -> Int32 raise Hdf5Error {
guard depth < 16 else {
raise InvalidArgument("writer datatype is too deeply nested")
}
match datatype.class {
6 => {
let mut size = 0
for field in datatype.compound_members {
let field_size = writer_native_datatype_size(
field.datatype,
depth=depth + 1,
)
guard size <= 0x7FFFFFFF - field_size else {
raise InvalidArgument("native compound datatype is too large")
}
size = size + field_size
}
size
}
7 =>
if datatype.version >= 4 {
18
} else if datatype.reference_kind == 0 {
8
} else {
12
}
8 => writer_native_datatype_size(datatype.base_types[0], depth=depth + 1)
9 => if datatype.vlen_is_string { 8 } else { 16 }
10 => {
let mut size = writer_native_datatype_size(
datatype.base_types[0],
depth=depth + 1,
).to_int64()
for dimension in datatype.array_dimensions {
guard dimension > 0UL &&
size <= 0x7FFFFFFFL / dimension.reinterpret_as_int64() else {
raise InvalidArgument("native array datatype is too large")
}
size = size * dimension.reinterpret_as_int64()
}
size.to_int()
}
_ => datatype.size
}
}
///|
fn writer_native_value_size(values : Hdf5WriteValues) -> Int32 raise Hdf5Error {
match values {
VariableStringValues(_) | ObjectReferenceValues(_) => 8
StructuredValues(datatype, _) => writer_native_datatype_size(datatype)
_ => writer_value_element_size(values)
}
}
///|
fn copy_write_values(values : Hdf5WriteValues) -> Hdf5WriteValues {
match values {
Int8Values(items) => Int8Values(items.copy())
Int16Values(items) => Int16Values(items.copy())
Int32Values(items) => Int32Values(items.copy())
Int64Values(items) => Int64Values(items.copy())
UInt8Values(items) => UInt8Values(items.copy())
UInt16Values(items) => UInt16Values(items.copy())
UInt32Values(items) => UInt32Values(items.copy())
UInt64Values(items) => UInt64Values(items.copy())
BigIntegerValues(items, size, signed) =>
BigIntegerValues(items.copy(), size, signed)
Float16Values(items) => Float16Values(items.copy())
Float32Values(items) => Float32Values(items.copy())
Float64Values(items) => Float64Values(items.copy())
FixedStringValues(items, width) => FixedStringValues(items.copy(), width)
VariableStringValues(items) => VariableStringValues(items.copy())
ObjectReferenceValues(items) => ObjectReferenceValues(items.copy())
Complex64Values(items) => Complex64Values(items.copy())
Complex128Values(items) => Complex128Values(items.copy())
StructuredValues(datatype, items) =>
StructuredValues(
copy_writer_datatype(datatype),
items.map(copy_dataset_value),
)
}
}
///|
fn copy_writer_datatype(datatype : Datatype) -> Datatype {
{
class: datatype.class,
version: datatype.version,
size: datatype.size,
signed: datatype.signed,
little_endian: datatype.little_endian,
vlen_is_string: datatype.vlen_is_string,
enum_members: datatype.enum_members.map(enum_item => {
name: enum_item.name,
value: enum_item.value,
}),
compound_members: datatype.compound_members.map(field => {
name: field.name,
offset: field.offset,
datatype: copy_writer_datatype(field.datatype),
}),
base_types: datatype.base_types.map(copy_writer_datatype),
array_dimensions: datatype.array_dimensions.copy(),
bit_offset: datatype.bit_offset,
bit_precision: datatype.bit_precision,
float_sign_position: datatype.float_sign_position,
float_exponent_position: datatype.float_exponent_position,
float_exponent_size: datatype.float_exponent_size,
float_mantissa_position: datatype.float_mantissa_position,
float_mantissa_size: datatype.float_mantissa_size,
float_exponent_bias: datatype.float_exponent_bias,
float_normalization: datatype.float_normalization,
string_padding: datatype.string_padding,
string_charset: datatype.string_charset,
opaque_tag: datatype.opaque_tag,
reference_kind: datatype.reference_kind,
}
}
///|
fn copy_dataset_value(value : DatasetValue) -> DatasetValue {
match value {
Compound(fields) =>
Compound(
fields.map(field => {
name: field.name,
value: copy_dataset_value(field.value),
}),
)
Elements(items) => Elements(items.map(copy_dataset_value))
Sequence(items) => Sequence(items.map(copy_dataset_value))
Opaque(bytes) => Opaque(bytes)
Reference(bytes) => Reference(bytes)
AttributeReference(path, name) => AttributeReference(path, name)
RegionReference(path, selection) =>
RegionReference(path, copy_region_selection(selection))
_ => value
}
}
///|
fn copy_region_selection(
selection : Hdf5RegionSelection,
) -> Hdf5RegionSelection {
match selection {
RegionEmpty | RegionAll => selection
RegionPoints(points) => RegionPoints(points.map(point => point.copy()))
RegionHyperslab(start, stride, count, block) =>
RegionHyperslab(start.copy(), stride.copy(), count.copy(), block.copy())
RegionBlocks(blocks) =>
RegionBlocks(
blocks.map(block => {
start: block.start.copy(),
end: block.end.copy(),
}),
)
}
}
///|
fn writer_shape_count(dimensions : Array[UInt64]) -> Int32 raise Hdf5Error {
let mut count = 1UL
for dimension in dimensions {
guard dimension <= 0x7FFFFFFFUL else {
raise InvalidArgument("writer dimension is too large")
}
}
if dimensions.contains(0UL) {
return 0
}
for dimension in dimensions {
guard count <= 0x7FFFFFFFUL / dimension else {
raise InvalidArgument("writer dataspace is too large")
}
count = count * dimension
}
count.to_int()
}
///|
fn validate_writer_byte_size(
count : Int32,
size : Int32,
) -> Unit raise Hdf5Error {
guard size > 0 && count <= 0x7FFFFFFF / size else {
raise InvalidArgument("writer dataset byte size is too large")
}
}
///|
fn validate_writer_datatype_name(
name : String,
description : String,
) -> Unit raise Hdf5Error {
let encoded = @utf8.encode(name)
guard encoded.length() > 0 && encoded.length() <= 65535 else {
raise InvalidArgument(description + " is empty or too long")
}
for byte in encoded {
guard byte != b'\x00' else {
raise InvalidArgument(description + " contains NUL")
}
}
}
///|
fn validate_writer_datatype(
datatype : Datatype,
depth : Int32,
reference_address_size~ : Int32? = None,
) -> Unit raise Hdf5Error {
guard depth < 16 && datatype.size > 0 else {
raise InvalidArgument("writer datatype is invalid or too deeply nested")
}
match datatype.class {
0 => {
guard datatype.size >= 1 &&
datatype.size <= 8191 &&
datatype.bit_precision > 0 &&
datatype.bit_offset >= 0 &&
datatype.bit_offset.to_int64() + datatype.bit_precision.to_int64() <=
datatype.size.to_int64() * 8L else {
raise InvalidArgument("writer integer datatype is unsupported")
}
}
1 => {
guard datatype.size == 2 ||
datatype.size == 4 ||
datatype.size == 8 ||
is_x87_float_layout(datatype) else {
raise InvalidArgument("writer floating-point datatype is unsupported")
}
}
2 => {
guard datatype.size >= 1 &&
datatype.size <= 8191 &&
datatype.bit_precision > 0 &&
datatype.bit_precision.to_int64() <= datatype.size.to_int64() * 8L else {
raise InvalidArgument("writer time datatype is unsupported")
}
}
3 => {
guard datatype.size <= 65535 &&
datatype.string_padding == 1 &&
datatype.string_charset == 1 else {
raise InvalidArgument("writer string datatype is unsupported")
}
}
4 => {
guard datatype.size >= 1 &&
datatype.size <= 8191 &&
datatype.bit_precision > 0 &&
datatype.bit_offset >= 0 &&
datatype.bit_offset.to_int64() + datatype.bit_precision.to_int64() <=
datatype.size.to_int64() * 8L else {
raise InvalidArgument("writer bitfield datatype is unsupported")
}
}
5 => {
let encoded = @utf8.encode(datatype.opaque_tag)
for byte in encoded {
guard byte > b'\x00' && byte < b'\x80' else {
raise InvalidArgument("writer opaque datatype tag is invalid")
}
}
guard (if encoded.length() == 0 {
0
} else {
round_up_to_eight(encoded.length() + 1)
}) <=
255 else {
raise InvalidArgument("writer opaque datatype tag is too long")
}
}
6 => {
guard datatype.compound_members.length() > 0 &&
datatype.compound_members.length() <= 65535 else {
raise InvalidArgument("writer compound datatype requires members")
}
for index in 0..= 0 &&
field.offset <= datatype.size - field.datatype.size else {
raise InvalidArgument("compound member extends beyond datatype size")
}
for previous in 0.. {
let legacy_layout_supported = match reference_address_size {
Some(address_size) =>
datatype.version <= 3 &&
(
(datatype.reference_kind == 0 && datatype.size == address_size) ||
(datatype.reference_kind == 1 && datatype.size == address_size + 4)
)
None =>
datatype.version == 3 &&
(
(datatype.reference_kind == 0 && datatype.size == 8) ||
(datatype.reference_kind == 1 && datatype.size == 12)
)
}
guard legacy_layout_supported ||
(
datatype.version == 4 &&
datatype.size ==
(match reference_address_size {
Some(address_size) => address_size + 10
None => 18
}) &&
(
datatype.reference_kind == 2 ||
datatype.reference_kind == 3 ||
datatype.reference_kind == 4
)
) else {
raise InvalidArgument("writer reference datatype is unsupported")
}
}
8 => {
guard datatype.base_types.length() == 1 &&
datatype.enum_members.length() > 0 &&
datatype.enum_members.length() <= 65535 else {
raise InvalidArgument("writer enum datatype is invalid")
}
let base = datatype.base_types[0]
validate_writer_datatype(base, depth + 1, reference_address_size~)
guard base.class == 0 && base.size == datatype.size else {
raise InvalidArgument("writer enum base datatype is invalid")
}
for index in 0.. {
guard base.signed && base.size <= 8 else {
raise InvalidArgument(
"enum member signedness does not match its base",
)
}
SignedInteger(number)
}
EnumValue::Unsigned(number) => {
guard !base.signed && base.size <= 8 else {
raise InvalidArgument(
"enum member signedness does not match its base",
)
}
UnsignedInteger(number)
}
EnumValue::Big(number) => {
guard base.size > 8 else {
raise InvalidArgument(
"wide enum member does not match a narrow base",
)
}
BigInteger(number)
}
}
validate_writer_integer(integer_value, base)
for previous in 0.. {
let expected_size = match reference_address_size {
Some(address_size) => address_size + 8
None => 16
}
guard datatype.size == expected_size else {
raise InvalidArgument("writer variable-length datatype is invalid")
}
if datatype.vlen_is_string {
guard datatype.base_types.length() == 1 else {
raise InvalidArgument("writer variable-length string is invalid")
}
let base = datatype.base_types[0]
validate_writer_datatype(base, depth + 1, reference_address_size~)
guard base.class == 0 && base.size == 1 && !base.signed else {
raise InvalidArgument("writer variable-length string base is invalid")
}
} else {
guard datatype.base_types.length() == 1 else {
raise InvalidArgument("writer variable-length sequence is invalid")
}
let base = datatype.base_types[0]
guard !datatype_contains_vlen(base) else {
raise InvalidArgument(
"writer variable-length sequence base must be fixed-size",
)
}
validate_writer_datatype(base, depth + 1, reference_address_size~)
}
}
10 => {
guard datatype.base_types.length() == 1 &&
datatype.array_dimensions.length() > 0 &&
datatype.array_dimensions.length() <= 32 else {
raise InvalidArgument("writer array datatype is invalid")
}
let base = datatype.base_types[0]
validate_writer_datatype(base, depth + 1, reference_address_size~)
guard datatype_array_size(datatype.array_dimensions, base.size) ==
datatype.size else {
raise InvalidArgument(
"array datatype size does not match its base type",
)
}
}
11 => {
guard datatype.base_types.length() == 1 && datatype.size % 2 == 0 else {
raise InvalidArgument("writer complex datatype is invalid")
}
let base = datatype.base_types[0]
guard base.class == 1 && base.size * 2 == datatype.size else {
raise InvalidArgument(
"complex datatype size does not match its floating-point base",
)
}
validate_writer_datatype(base, depth + 1, reference_address_size~)
}
_ =>
raise InvalidArgument("writer structured datatype class is unsupported")
}
}
///|
fn validate_writer_integer(
value : DatasetValue,
datatype : Datatype,
) -> Unit raise Hdf5Error {
let precision = datatype.bit_precision
if datatype.size > 8 {
guard value is BigInteger(number) else {
raise InvalidArgument("wide structured integer has the wrong kind")
}
validate_big_integer(number, datatype.size, datatype.signed)
let one = @bigint.BigInt::from_int(1)
if datatype.signed {
let half = one.shl(precision - 1)
guard number.compare(half.neg()) >= 0 && number.compare(half) < 0 else {
raise InvalidArgument("structured signed integer is out of range")
}
} else {
let limit = one.shl(precision)
guard number.compare(@bigint.BigInt::from_int(0)) >= 0 &&
number.compare(limit) < 0 else {
raise InvalidArgument("structured unsigned integer is out of range")
}
}
} else if datatype.signed {
guard value is SignedInteger(number) else {
raise InvalidArgument("structured integer value has the wrong kind")
}
if precision < 64 {
let minimum = -(1L << (precision - 1))
let maximum = (1L << (precision - 1)) - 1L
guard number >= minimum && number <= maximum else {
raise InvalidArgument("structured signed integer is out of range")
}
}
} else {
guard value is UnsignedInteger(number) else {
raise InvalidArgument("structured integer value has the wrong kind")
}
if precision < 64 {
guard number < 1UL << precision else {
raise InvalidArgument("structured unsigned integer is out of range")
}
}
}
}
///|
fn validate_external_reference_file_name(name : String) -> Unit raise Hdf5Error {
let encoded = @utf8.encode(name)
guard encoded.length() > 0 && encoded.length() <= 65535 else {
raise InvalidArgument("external reference file name is invalid")
}
for byte in encoded {
guard byte != b'\x00' else {
raise InvalidArgument("external reference file names cannot contain NUL")
}
}
resolver_file_name(name) |> ignore
}
///|
fn validate_writer_reference_attribute_name(
name : String,
) -> Unit raise Hdf5Error {
let encoded = @utf8.encode(name)
guard encoded.length() > 0 && encoded.length() <= 65535 else {
raise InvalidArgument("writer attribute reference name is invalid")
}
}
///|
fn validate_structured_value(
value : DatasetValue,
datatype : Datatype,
depth : Int32,
) -> Unit raise Hdf5Error {
guard depth < 16 else {
raise InvalidArgument("writer value is too deeply nested")
}
match datatype.class {
0 => validate_writer_integer(value, datatype)
1 => {
guard value is FloatingPoint(_) else {
raise InvalidArgument(
"structured floating-point value has the wrong kind",
)
}
}
2 =>
if datatype.size > 8 {
guard value is BigTime(number) else {
raise InvalidArgument("wide structured time has the wrong kind")
}
validate_big_integer(number, datatype.size, false)
guard number.bit_length() <= datatype.bit_precision else {
raise InvalidArgument("structured time is out of range")
}
} else {
guard value is Time(number) else {
raise InvalidArgument("structured time value has the wrong kind")
}
if datatype.bit_precision < 64 {
guard number < 1UL << datatype.bit_precision else {
raise InvalidArgument("structured time is out of range")
}
}
}
3 => {
guard value is Text(text) else {
raise InvalidArgument("structured string value has the wrong kind")
}
guard @utf8.encode(text).length() <= datatype.size else {
raise InvalidArgument("structured string exceeds its width")
}
}
4 =>
if datatype.size > 8 {
guard value is BigBitfield(number) else {
raise InvalidArgument("wide structured bitfield has the wrong kind")
}
validate_big_integer(number, datatype.size, false)
guard number.bit_length() <= datatype.bit_precision else {
raise InvalidArgument("structured bitfield is out of range")
}
} else {
guard value is Bitfield(number) else {
raise InvalidArgument("structured bitfield value has the wrong kind")
}
if datatype.bit_precision < 64 {
guard number < 1UL << datatype.bit_precision else {
raise InvalidArgument("structured bitfield is out of range")
}
}
}
5 => {
guard value is Opaque(bytes) else {
raise InvalidArgument("structured opaque value has the wrong kind")
}
guard bytes.length() == datatype.size else {
raise InvalidArgument("structured opaque value has the wrong size")
}
}
6 => {
guard value is Compound(fields) &&
fields.length() == datatype.compound_members.length() else {
raise InvalidArgument("compound value does not match its datatype")
}
for index in 0..
if datatype.reference_kind == 0 || datatype.reference_kind == 2 {
match value {
ObjectReference(path) =>
match path {
Some(target) => canonical_writer_path(target) |> ignore
None => ()
}
ExternalObjectReference(file_name, path) => {
guard datatype.version >= 4 else {
raise InvalidArgument(
"external references require a version-4 datatype",
)
}
validate_external_reference_file_name(file_name)
canonical_writer_path(path) |> ignore
}
_ =>
raise InvalidArgument("object reference value has the wrong kind")
}
} else if datatype.reference_kind == 1 || datatype.reference_kind == 3 {
match value {
RegionReference(path, _) =>
match path {
Some(target) => canonical_writer_path(target) |> ignore
None => ()
}
ExternalRegionReference(file_name, path, _) => {
guard datatype.version >= 4 && datatype.reference_kind == 3 else {
raise InvalidArgument(
"external region references require a version-4 datatype",
)
}
validate_external_reference_file_name(file_name)
canonical_writer_path(path) |> ignore
}
_ =>
raise InvalidArgument("region reference value has the wrong kind")
}
} else {
match value {
AttributeReference(path, name) =>
match path {
Some(target) => {
canonical_writer_path(target) |> ignore
validate_writer_reference_attribute_name(name)
}
None => ()
}
ExternalAttributeReference(file_name, path, name) => {
guard datatype.version >= 4 && datatype.reference_kind == 4 else {
raise InvalidArgument(
"external attribute references require a version-4 datatype",
)
}
validate_external_reference_file_name(file_name)
canonical_writer_path(path) |> ignore
validate_writer_reference_attribute_name(name)
}
_ =>
raise InvalidArgument(
"attribute reference value has the wrong kind",
)
}
}
8 => {
guard value is Enumeration(name) else {
raise InvalidArgument("enum value has the wrong kind")
}
guard datatype.enum_members.search_by(enum_item => enum_item.name == name)
is Some(_) else {
raise InvalidArgument("enum value does not match any member")
}
}
9 =>
if datatype.vlen_is_string {
guard value is Text(_) else {
raise InvalidArgument(
"variable-length string value has the wrong kind",
)
}
} else {
guard value is Sequence(items) else {
raise InvalidArgument(
"variable-length sequence value has the wrong kind",
)
}
let base = datatype.base_types[0]
for item in items {
validate_structured_value(item, base, depth + 1)
}
}
10 => {
guard value is Elements(items) else {
raise InvalidArgument("array value has the wrong kind")
}
let base = datatype.base_types[0]
guard items.length() == datatype.size / base.size else {
raise InvalidArgument(
"array value element count does not match its datatype",
)
}
for item in items {
validate_structured_value(item, base, depth + 1)
}
}
11 => {
guard value is Complex(_, _) && datatype.base_types.length() == 1 else {
raise InvalidArgument(
"structured complex value or datatype has the wrong kind",
)
}
}
_ =>
raise InvalidArgument("writer structured datatype class is unsupported")
}
}
///|
fn validate_write_values(
dimensions : Array[UInt64],
values : Hdf5WriteValues,
reference_address_size~ : Int32? = None,
) -> Unit raise Hdf5Error {
guard dimensions.length() <= 32 else {
raise InvalidArgument("writer dataspace rank exceeds 32")
}
guard writer_shape_count(dimensions) == writer_value_count(values) else {
raise InvalidArgument("writer value count does not match dataspace")
}
match values {
FixedStringValues(items, width) => {
guard width > 0 && width <= 65535 else {
raise InvalidArgument("fixed string width is invalid")
}
for item in items {
guard @utf8.encode(item).length() <= width else {
raise InvalidArgument("fixed string value exceeds its width")
}
}
validate_writer_byte_size(items.length(), width)
}
VariableStringValues(items) => {
validate_writer_byte_size(items.length(), 16)
for item in items {
guard @utf8.encode(item).length() <= 0x7FFFFFFF else {
raise InvalidArgument("variable string value is too large")
}
}
}
ObjectReferenceValues(items) => {
validate_writer_byte_size(items.length(), 8)
for item in items {
match item {
Some(path) => canonical_writer_path(path) |> ignore
None => ()
}
}
}
Int8Values(items) => {
validate_writer_byte_size(items.length(), 1)
for item in items {
guard item >= -128 && item <= 127 else {
raise InvalidArgument("Int8 writer value is out of range")
}
}
}
Int16Values(items) => {
validate_writer_byte_size(items.length(), 2)
for item in items {
guard item >= -32768 && item <= 32767 else {
raise InvalidArgument("Int16 writer value is out of range")
}
}
}
UInt8Values(items) => {
validate_writer_byte_size(items.length(), 1)
for item in items {
guard item <= 255U else {
raise InvalidArgument("UInt8 writer value is out of range")
}
}
}
UInt16Values(items) => {
validate_writer_byte_size(items.length(), 2)
for item in items {
guard item <= 65535U else {
raise InvalidArgument("UInt16 writer value is out of range")
}
}
}
Int64Values(items) => validate_writer_byte_size(items.length(), 8)
UInt64Values(items) => validate_writer_byte_size(items.length(), 8)
BigIntegerValues(items, size, signed) => {
validate_writer_byte_size(items.length(), size)
for item in items {
validate_big_integer(item, size, signed)
}
}
Float64Values(items) => validate_writer_byte_size(items.length(), 8)
Float16Values(items) => validate_writer_byte_size(items.length(), 2)
Complex64Values(items) => validate_writer_byte_size(items.length(), 8)
Complex128Values(items) => validate_writer_byte_size(items.length(), 16)
Int32Values(items) => validate_writer_byte_size(items.length(), 4)
UInt32Values(items) => validate_writer_byte_size(items.length(), 4)
Float32Values(items) => validate_writer_byte_size(items.length(), 4)
StructuredValues(datatype, items) => {
validate_writer_datatype(datatype, 0, reference_address_size~)
validate_writer_byte_size(items.length(), datatype.size)
for item in items {
validate_structured_value(item, datatype, 0)
}
}
}
}
///|
fn Hdf5Writer::has_object(self : Hdf5Writer, path : String) -> Bool {
self.groups.contains(path) ||
self.datasets.search_by(dataset => dataset.path == path) is Some(_) ||
self.committed_datatypes.search_by(datatype => datatype.path == path)
is Some(_)
}
///|
fn writer_soft_link_path(parent_path : String, name : String) -> String {
if parent_path == "/" {
"/" + name
} else {
parent_path + "/" + name
}
}
///|
fn Hdf5Writer::has_soft_link(self : Hdf5Writer, path : String) -> Bool {
self.soft_link_indices.contains(path)
}
///|
fn Hdf5Writer::has_external_link(self : Hdf5Writer, path : String) -> Bool {
self.external_link_indices.contains(path)
}
///|
fn Hdf5Writer::has_entry(self : Hdf5Writer, path : String) -> Bool {
self.has_object(path) ||
self.has_soft_link(path) ||
self.has_external_link(path)
}
///|
fn validate_writer_link_name(
name : String,
kind : String,
) -> Unit raise Hdf5Error {
let encoded = @utf8.encode(name)
guard encoded.length() > 0 else {
raise InvalidArgument(kind + " name is empty")
}
guard name.split("/").count() == 1 else {
raise InvalidArgument(kind + " names cannot contain '/'")
}
guard name != "." && name != ".." else {
raise InvalidArgument(kind + " names cannot be dot segments")
}
guard encoded.length() <= 65535 else {
raise InvalidArgument(kind + " name is too long")
}
for byte in encoded {
guard byte != b'\x00' else {
raise InvalidArgument(kind + " names cannot contain NUL")
}
}
}
///|
fn validate_writer_link_string(
value : String,
description : String,
) -> Unit raise Hdf5Error {
let encoded = @utf8.encode(value)
guard encoded.length() > 0 else {
raise InvalidArgument(description + " is empty")
}
guard encoded.length() <= 65535 else {
raise InvalidArgument(description + " is too long")
}
for byte in encoded {
guard byte != b'\x00' else {
raise InvalidArgument(description + " cannot contain NUL")
}
}
}
///|
fn validate_soft_link_target(target : String) -> Unit raise Hdf5Error {
let encoded = @utf8.encode(target)
guard encoded.length() > 0 else {
raise InvalidArgument("soft link target is empty")
}
guard encoded.length() <= 65535 else {
raise InvalidArgument("soft link target is too long")
}
for byte in encoded {
guard byte != b'\x00' else {
raise InvalidArgument("soft link targets cannot contain NUL")
}
}
}
///|
fn validate_writer_link_message_size(
name : String,
info_size : Int32,
kind : String,
) -> Unit raise Hdf5Error {
let encoded_name = @utf8.encode(name)
guard 16 + encoded_name.length() + info_size <= 65535 else {
raise InvalidArgument(kind + " message is too long")
}
}
///|
fn Hdf5Writer::record_link(
self : Hdf5Writer,
path : String,
) -> Unit raise Hdf5Error {
guard path != "/" else { return }
if self.link_order_indices.contains(path) {
return
}
let parent_path = writer_parent_path(path)
let counter = match self.link_order_counter_indices.get(parent_path) {
Some(index) => index
None => {
self.link_order_counters.push({ next_order: 0UL, })
let index = self.link_order_counters.length() - 1
self.link_order_counter_indices.set(parent_path, index)
index
}
}
let order = self.link_order_counters[counter].next_order
guard order < 0xFFFFFFFFFFFFFFFFUL else {
raise InvalidArgument("writer has too many links")
}
self.link_orders.push({ order, })
self.link_order_indices.set(path, self.link_orders.length() - 1)
self.link_order_counters[counter] = { next_order: order + 1UL, }
}
///|
fn Hdf5Writer::link_creation_order(
self : Hdf5Writer,
path : String,
) -> UInt64 raise Hdf5Error {
match self.link_order_indices.get(path) {
Some(index) => self.link_orders[index].order
None => raise InvalidField("writer link has no creation order")
}
}
///|
fn Hdf5Writer::ensure_groups(
self : Hdf5Writer,
path : String,
) -> Unit raise Hdf5Error {
let segments = writer_path_segments(path)
let mut current = ""
for segment in segments {
current = current + "/" + segment
guard self.datasets.search_by(dataset => dataset.path == current) is None else {
raise InvalidArgument("writer path conflicts with a dataset")
}
guard self.committed_datatypes.search_by(datatype => {
datatype.path == current
})
is None else {
raise InvalidArgument("writer path conflicts with a committed datatype")
}
guard !self.has_soft_link(current) else {
raise InvalidArgument("writer path conflicts with a soft link")
}
guard !self.has_external_link(current) else {
raise InvalidArgument("writer path conflicts with an external link")
}
if !self.groups.contains(current) {
self.record_link(current)
self.groups.push(current)
}
}
}
///|
/// Adds a group and any missing parent groups.
pub fn Hdf5Writer::add_group(
self : Hdf5Writer,
path : String,
) -> Unit raise Hdf5Error {
self.ensure_groups(canonical_writer_path(path))
}
///|
/// Adds a named committed datatype at `path`.
///
/// Datasets and unshared attributes whose encoded datatype matches this type
/// reference it through a committed-datatype shared message.
pub fn Hdf5Writer::add_committed_datatype(
self : Hdf5Writer,
path : String,
datatype : Datatype,
) -> Unit raise Hdf5Error {
let normalized = canonical_writer_path(path)
guard normalized != "/" else {
raise InvalidArgument("a datatype cannot replace the root group")
}
guard !self.has_entry(normalized) else {
raise InvalidArgument("writer object path already exists")
}
validate_writer_datatype(datatype, 0)
let data = encode_writer_datatype(datatype)
self.ensure_groups(writer_parent_path(normalized))
self.record_link(normalized)
self.committed_datatypes.push({ path: normalized, datatype, data, })
}
///|
/// Adds a soft link at `parent_path`/`name` pointing to `target`.
///
/// Absolute targets start at the root group. Relative targets are resolved
/// from the group containing the link.
pub fn Hdf5Writer::add_soft_link(
self : Hdf5Writer,
parent_path : String,
name : String,
target : String,
) -> Unit raise Hdf5Error {
validate_writer_link_name(name, "soft link")
validate_soft_link_target(target)
validate_writer_link_message_size(
name,
@utf8.encode(target).length(),
"soft link",
)
let normalized_parent = canonical_writer_path(parent_path)
let link_path = writer_soft_link_path(normalized_parent, name)
guard !self.has_entry(link_path) else {
raise InvalidArgument("writer object path already exists")
}
self.ensure_groups(normalized_parent)
self.record_link(link_path)
self.soft_links.push({ parent_path: normalized_parent, name, target, })
self.soft_link_indices.set(link_path, self.soft_links.length() - 1)
}
///|
/// Adds an external link at `parent_path`/`name`.
///
/// `target_file` is resolved by the HDF5 reader relative to the file
/// containing the link. `target_path` names an object inside that file.
pub fn Hdf5Writer::add_external_link(
self : Hdf5Writer,
parent_path : String,
name : String,
target_file : String,
target_path : String,
) -> Unit raise Hdf5Error {
validate_writer_link_name(name, "external link")
validate_writer_link_string(target_file, "external link target file")
validate_writer_link_string(target_path, "external link target path")
validate_writer_link_message_size(
name,
5 + @utf8.encode(target_file).length() + @utf8.encode(target_path).length(),
"external link",
)
let normalized_parent = canonical_writer_path(parent_path)
let link_path = writer_soft_link_path(normalized_parent, name)
guard !self.has_entry(link_path) else {
raise InvalidArgument("writer object path already exists")
}
self.ensure_groups(normalized_parent)
self.record_link(link_path)
self.external_links.push({
parent_path: normalized_parent,
name,
target_file,
target_path,
})
self.external_link_indices.set(link_path, self.external_links.length() - 1)
}
///|
/// Adds a contiguous dataset. Empty `dimensions` denotes a scalar dataspace.
/// An optional fill value must be a single value of the dataset datatype.
pub fn Hdf5Writer::add_dataset(
self : Hdf5Writer,
path : String,
dimensions : Array[UInt64],
values : Hdf5WriteValues,
fill_value? : Hdf5WriteValues,
) -> Unit raise Hdf5Error {
let normalized = canonical_writer_path(path)
guard normalized != "/" else {
raise InvalidArgument("a dataset cannot replace the root group")
}
guard !self.has_entry(normalized) else {
raise InvalidArgument("writer object path already exists")
}
validate_write_values(dimensions, values)
let copied_fill = validated_write_fill(values, fill_value)
self.ensure_groups(writer_parent_path(normalized))
self.record_link(normalized)
self.datasets.push({
path: normalized,
dimensions: dimensions.copy(),
maximum_dimensions: dimensions.copy(),
values: copy_write_values(values),
storage: WritableContiguous,
fill_value: copied_fill,
chunk_filter_masks: [],
})
}
///|
/// Adds a compact dataset whose values are stored inline in the object header.
/// Empty `dimensions` denotes a scalar dataspace. An optional fill value must
/// be a single value of the dataset datatype.
pub fn Hdf5Writer::add_compact_dataset(
self : Hdf5Writer,
path : String,
dimensions : Array[UInt64],
values : Hdf5WriteValues,
fill_value? : Hdf5WriteValues,
) -> Unit raise Hdf5Error {
let normalized = canonical_writer_path(path)
guard normalized != "/" else {
raise InvalidArgument("a dataset cannot replace the root group")
}
guard !self.has_entry(normalized) else {
raise InvalidArgument("writer object path already exists")
}
validate_write_values(dimensions, values)
let count = selection_element_count(dimensions)
guard count >= 0 &&
count <= 65524L / writer_value_element_size(values).to_int64() else {
raise InvalidArgument("compact dataset values exceed 65524 bytes")
}
let copied_fill = validated_write_fill(values, fill_value)
self.ensure_groups(writer_parent_path(normalized))
self.record_link(normalized)
self.datasets.push({
path: normalized,
dimensions: dimensions.copy(),
maximum_dimensions: dimensions.copy(),
values: copy_write_values(values),
storage: WritableCompact,
fill_value: copied_fill,
chunk_filter_masks: [],
})
}
///|
fn validate_external_slots(
slots : Array[Hdf5ExternalSlot],
expected_size : UInt64,
) -> Unit raise Hdf5Error {
guard slots.length() > 0 && slots.length() <= 65535 else {
raise InvalidArgument("external dataset requires 1 to 65535 slots")
}
let mut total = 0UL
for slot in slots {
let encoded = @utf8.encode(slot.file_name)
guard encoded.length() > 0 && encoded.length() <= 65535 else {
raise InvalidArgument("external data file name is empty or too long")
}
for byte in encoded {
guard byte != b'\x00' else {
raise InvalidArgument("external data file name cannot contain NUL")
}
}
guard slot.data_size > 0UL else {
raise InvalidArgument("external data slot is empty")
}
guard slot.file_offset <= 0xFFFFFFFFFFFFFFFFUL - slot.data_size else {
raise InvalidArgument("external data file offset is too large")
}
guard total <= 0xFFFFFFFFFFFFFFFFUL - slot.data_size else {
raise InvalidArgument("external data size is too large")
}
total = total + slot.data_size
}
guard total == expected_size else {
raise InvalidArgument("external data slots do not match the dataset size")
}
}
///|
/// Adds an unfiltered dataset whose raw bytes live in external files.
///
/// Slots are concatenated in caller order to form the row-major element
/// stream. Their sizes must add up exactly to the dataset byte size.
pub fn Hdf5Writer::add_external_dataset(
self : Hdf5Writer,
path : String,
dimensions : Array[UInt64],
values : Hdf5WriteValues,
slots : Array[Hdf5ExternalSlot],
fill_value? : Hdf5WriteValues,
) -> Unit raise Hdf5Error {
let normalized = canonical_writer_path(path)
guard normalized != "/" else {
raise InvalidArgument("a dataset cannot replace the root group")
}
guard !self.has_entry(normalized) else {
raise InvalidArgument("writer object path already exists")
}
validate_write_values(dimensions, values)
guard !writer_values_contain_vlen(values) else {
raise UnsupportedFeature(
"external storage cannot hold variable-length values",
)
}
let expected_size = writer_shape_count(dimensions).to_int64() *
writer_value_element_size(values).to_int64()
guard expected_size >= 0 else {
raise InvalidArgument("external dataset size is too large")
}
validate_external_slots(slots, expected_size.reinterpret_as_uint64())
let copied_fill = validated_write_fill(values, fill_value)
self.ensure_groups(writer_parent_path(normalized))
self.record_link(normalized)
self.datasets.push({
path: normalized,
dimensions: dimensions.copy(),
maximum_dimensions: dimensions.copy(),
values: copy_write_values(values),
storage: WritableExternal(slots.copy()),
fill_value: copied_fill,
chunk_filter_masks: [],
})
}
///|
fn same_write_datatype(
left : Hdf5WriteValues,
right : Hdf5WriteValues,
reference_address_size~ : Int32? = None,
) -> Bool raise Hdf5Error {
match (left, right) {
(Int8Values(_), Int8Values(_)) => true
(Int16Values(_), Int16Values(_)) => true
(Int32Values(_), Int32Values(_)) => true
(Int64Values(_), Int64Values(_)) => true
(UInt8Values(_), UInt8Values(_)) => true
(UInt16Values(_), UInt16Values(_)) => true
(UInt32Values(_), UInt32Values(_)) => true
(UInt64Values(_), UInt64Values(_)) => true
(
BigIntegerValues(_, left_size, left_signed),
BigIntegerValues(_, right_size, right_signed),
) => left_size == right_size && left_signed == right_signed
(Float16Values(_), Float16Values(_)) => true
(Float32Values(_), Float32Values(_)) => true
(Float64Values(_), Float64Values(_)) => true
(FixedStringValues(_, left_width), FixedStringValues(_, right_width)) =>
left_width == right_width
(VariableStringValues(_), VariableStringValues(_)) => true
(ObjectReferenceValues(_), ObjectReferenceValues(_)) => true
(Complex64Values(_), Complex64Values(_)) => true
(Complex128Values(_), Complex128Values(_)) => true
(StructuredValues(left_type, _), StructuredValues(right_type, _)) =>
encode_writer_datatype(
left_type,
reference_address_size=reference_address_size.unwrap_or(8),
) ==
encode_writer_datatype(
right_type,
reference_address_size=reference_address_size.unwrap_or(8),
)
_ => false
}
}
///|
fn copy_write_filters(
filters : Array[Hdf5WriteFilter],
) -> Array[Hdf5WriteFilter] {
let result : Array[Hdf5WriteFilter] = []
for filter in filters {
match filter {
Custom(id, parameters) => result.push(Custom(id, parameters.copy()))
_ => result.push(filter)
}
}
result
}
///|
fn validate_write_filters(
filters : Array[Hdf5WriteFilter],
custom_filters : Array[&Hdf5CustomFilter],
) -> Unit raise Hdf5Error {
guard filters.length() <= 32 else {
raise InvalidArgument("too many filters")
}
let mut saw_shuffle = false
let mut saw_deflate = false
let mut saw_fletcher = false
let mut saw_scaleoffset = false
let mut saw_nbit = false
let mut saw_any = false
let mut saw_non_shuffle = false
for filter in filters {
match filter {
Shuffle => {
guard !saw_shuffle && !saw_any else {
raise InvalidArgument("shuffle must be the first writer filter")
}
saw_shuffle = true
saw_any = true
}
Deflate(level) => {
guard !saw_deflate && !saw_fletcher && level >= 0 && level <= 9 else {
raise InvalidArgument("invalid deflate writer filter")
}
saw_deflate = true
saw_any = true
saw_non_shuffle = true
}
Fletcher32 => {
guard !saw_fletcher else {
raise InvalidArgument("duplicate Fletcher32 writer filter")
}
saw_fletcher = true
saw_any = true
saw_non_shuffle = true
}
Nbit => {
guard !saw_nbit && !saw_non_shuffle else {
raise InvalidArgument(
"N-bit must precede lossless filters and follow only shuffle",
)
}
saw_nbit = true
saw_any = true
saw_non_shuffle = true
}
ScaleOffset(minbits) => {
guard !saw_scaleoffset && !saw_any else {
raise InvalidArgument("scale-offset must be the first writer filter")
}
guard minbits >= 0 else {
raise InvalidArgument("invalid scale-offset writer filter")
}
saw_scaleoffset = true
saw_any = true
saw_non_shuffle = true
}
Custom(id, parameters) => {
validate_custom_filter_id(id)
let custom = find_custom_filter(custom_filters, id)
guard parameters.length() <= 65535 else {
raise InvalidArgument("custom filter has too many parameters")
}
if id >= 256 {
guard @utf8.encode(custom.name()).length() <= 65535 else {
raise InvalidArgument("custom filter name is too long")
}
}
saw_any = true
saw_non_shuffle = true
}
}
}
}
///|
fn writer_values_contain_vlen(values : Hdf5WriteValues) -> Bool {
match values {
VariableStringValues(_) => true
StructuredValues(datatype, _) => datatype_contains_vlen(datatype)
_ => false
}
}
///|
fn writer_values_datatype(values : Hdf5WriteValues) -> Datatype raise Hdf5Error {
match values {
Int8Values(_) => Datatype::write_integer(true, 1)
Int16Values(_) => Datatype::write_integer(true, 2)
Int32Values(_) => Datatype::write_integer(true, 4)
Int64Values(_) => Datatype::write_integer(true, 8)
UInt8Values(_) => Datatype::write_integer(false, 1)
UInt16Values(_) => Datatype::write_integer(false, 2)
UInt32Values(_) => Datatype::write_integer(false, 4)
UInt64Values(_) => Datatype::write_integer(false, 8)
BigIntegerValues(_, size, signed) => Datatype::write_integer(signed, size)
Float16Values(_) => Datatype::write_float(2)
Float32Values(_) => Datatype::write_float(4)
Float64Values(_) => Datatype::write_float(8)
FixedStringValues(_, width) => Datatype::write_fixed_string(width)
VariableStringValues(_) => Datatype::write_vlen_string()
ObjectReferenceValues(_) => Datatype::write_object_reference()
Complex64Values(_) =>
Datatype::write_compound([
("r", Datatype::write_float(4)),
("i", Datatype::write_float(4)),
])
Complex128Values(_) =>
Datatype::write_compound([
("r", Datatype::write_float(8)),
("i", Datatype::write_float(8)),
])
StructuredValues(datatype, _) => datatype
}
}
///|
fn nbit_datatype_supported(datatype : Datatype, depth : Int32) -> Bool {
if depth >= 16 || datatype.size <= 0 {
return false
}
match datatype.class {
0 | 1 | 2 | 3 | 4 | 5 | 7 | 8 | 9 => true
6 =>
datatype.compound_members.length() > 0 &&
datatype.compound_members.all(field => {
nbit_datatype_supported(field.datatype, depth + 1)
})
10 =>
datatype.base_types.length() == 1 &&
nbit_datatype_supported(datatype.base_types[0], depth + 1)
_ => false
}
}
///|
fn nbit_datatype_needs_compression(
datatype : Datatype,
depth~ : Int32 = 0,
) -> Bool raise Hdf5Error {
guard depth < 16 else {
raise InvalidArgument("N-bit datatype is too deeply nested")
}
match datatype.class {
0 | 1 =>
datatype.bit_offset != 0 || datatype.bit_precision != datatype.size * 8
6 =>
datatype.compound_members.any(field => {
nbit_datatype_needs_compression(field.datatype, depth=depth + 1)
})
10 =>
nbit_datatype_needs_compression(datatype.base_types[0], depth=depth + 1)
_ => false
}
}
///|
fn nbit_append_client_node(
datatype : Datatype,
parameters : Array[UInt32],
depth : Int32,
) -> Bool raise Hdf5Error {
guard depth < 16 else {
raise InvalidArgument("N-bit datatype is too deeply nested")
}
match datatype.class {
0 | 1 => {
guard datatype.size > 0 &&
datatype.bit_precision > 0 &&
datatype.bit_offset >= 0 &&
datatype.bit_offset.to_int64() + datatype.bit_precision.to_int64() <=
datatype.size.to_int64() * 8L else {
raise InvalidArgument("invalid N-bit atomic datatype")
}
parameters.push(1U)
parameters.push(datatype.size.reinterpret_as_uint())
parameters.push(if datatype.little_endian { 0U } else { 1U })
parameters.push(datatype.bit_precision.reinterpret_as_uint())
parameters.push(datatype.bit_offset.reinterpret_as_uint())
datatype.bit_offset != 0 || datatype.bit_precision != datatype.size * 8
}
2 | 3 | 4 | 5 | 7 | 8 | 9 => {
parameters.push(4U)
parameters.push(datatype.size.reinterpret_as_uint())
false
}
6 => {
guard datatype.compound_members.length() > 0 &&
datatype.compound_members.length() <= 0xFFF else {
raise InvalidArgument("invalid N-bit compound datatype")
}
parameters.push(3U)
parameters.push(datatype.size.reinterpret_as_uint())
parameters.push(datatype.compound_members.length().reinterpret_as_uint())
let mut needs_compression = false
for field in datatype.compound_members {
guard field.offset >= 0 &&
field.offset <= datatype.size - field.datatype.size else {
raise InvalidArgument("invalid N-bit compound member")
}
parameters.push(field.offset.reinterpret_as_uint())
if nbit_append_client_node(field.datatype, parameters, depth + 1) {
needs_compression = true
}
}
needs_compression
}
10 => {
guard datatype.base_types.length() == 1 &&
datatype.array_dimensions.length() > 0 else {
raise InvalidArgument("invalid N-bit array datatype")
}
parameters.push(2U)
parameters.push(datatype.size.reinterpret_as_uint())
nbit_append_client_node(datatype.base_types[0], parameters, depth + 1)
}
_ => raise UnsupportedFeature("N-bit does not support this datatype")
}
}
///|
fn nbit_client_data(
datatype : Datatype,
chunk_elements : Int32,
) -> Array[UInt32] raise Hdf5Error {
guard chunk_elements > 0 && nbit_datatype_supported(datatype, 0) else {
raise UnsupportedFeature("N-bit does not support this datatype")
}
let parameters : Array[UInt32] = []
if datatype.class == 0 ||
datatype.class == 1 ||
datatype.class == 6 ||
datatype.class == 10 {
parameters.push(0U)
parameters.push(0U)
parameters.push(chunk_elements.reinterpret_as_uint())
let needs_compression = nbit_append_client_node(datatype, parameters, 0)
parameters[0] = parameters.length().reinterpret_as_uint()
parameters[1] = if needs_compression { 0U } else { 1U }
} else {
parameters.push(3U)
parameters.push(1U)
parameters.push(chunk_elements.reinterpret_as_uint())
}
guard parameters.length() <= 4096 else {
raise UnsupportedFeature("N-bit datatype needs too many parameters")
}
parameters
}
///|
fn validate_writer_filters_for_values(
values : Hdf5WriteValues,
filters : Array[Hdf5WriteFilter],
) -> Unit raise Hdf5Error {
let contains_vlen = writer_values_contain_vlen(values)
let has_nbit = filters.any(filter => filter is Nbit)
let has_shuffle = filters.any(filter => filter is Shuffle)
if has_nbit {
let datatype = writer_values_datatype(values)
guard nbit_datatype_supported(datatype, 0) else {
raise UnsupportedFeature("N-bit does not support this datatype")
}
if has_shuffle && nbit_datatype_needs_compression(datatype) {
raise InvalidArgument("shuffle with N-bit requires full-precision fields")
}
}
for filter in filters {
match filter {
Fletcher32 => {
guard !contains_vlen else {
raise UnsupportedFeature(
"Fletcher32 is not supported for variable-length datasets",
)
}
}
ScaleOffset(scale) => {
let (element_size, _, datatype_class) = writer_scaleoffset_layout(
values,
)
guard scale >= 0 else {
raise InvalidArgument("invalid scale-offset writer filter")
}
if datatype_class == 0 {
guard scale <= element_size * 8 else {
raise InvalidArgument(
"scale-offset minimum bits exceed the datatype width",
)
}
} else {
let maximum_digits = if element_size == 4 { 38 } else { 308 }
guard scale <= maximum_digits else {
raise InvalidArgument(
"scale-offset decimal digits exceed the datatype precision",
)
}
for other in filters {
match other {
Fletcher32 =>
raise InvalidArgument(
"Fletcher32 cannot be combined with floating-point scale-offset",
)
_ => ()
}
}
}
}
_ => ()
}
}
}
///|
fn writer_chunk_intersects_shape(
index : Int32,
grid : Array[Int32],
order : Array[Int32],
dimensions : Array[UInt64],
chunk_dimensions : Array[UInt32],
) -> Bool {
let coordinates : Array[Int32] = Array::make(grid.length(), 0)
let mut remaining = index
let mut order_index = order.length()
while order_index > 0 {
order_index = order_index - 1
let axis = order[order_index]
let count = grid[axis]
if count <= 0 {
return false
}
coordinates[axis] = remaining % count
remaining = remaining / count
}
for axis in 0..=
dimensions[axis] {
return false
}
}
true
}
///|
fn validate_writer_chunk_size(
chunk_dimensions : Array[UInt32],
element_size : Int32,
) -> Unit raise Hdf5Error {
let mut chunk_bytes = element_size.to_int64()
for dimension in chunk_dimensions {
guard chunk_bytes <= 0x7FFFFFFFL / dimension.to_int64() else {
raise InvalidArgument("writer chunk is too large")
}
chunk_bytes = chunk_bytes * dimension.to_int64()
}
}
///|
fn validated_write_fill(
values : Hdf5WriteValues,
fill_value : Hdf5WriteValues?,
reference_address_size~ : Int32? = None,
) -> Hdf5WriteValues? raise Hdf5Error {
match fill_value {
Some(fill) => {
guard writer_value_count(fill) == 1 &&
same_write_datatype(values, fill, reference_address_size~) else {
raise InvalidArgument(
"fill value must be one value of the dataset type",
)
}
validate_write_values([], fill, reference_address_size~)
Some(copy_write_values(fill))
}
None => None
}
}
///|
/// Adds a finite chunked dataset using a fixed-array chunk index.
///
/// Chunks containing only the fill value are left unallocated. Filters are
/// applied in the supplied HDF5 pipeline order.
pub fn Hdf5Writer::add_chunked_dataset(
self : Hdf5Writer,
path : String,
dimensions : Array[UInt64],
chunk_dimensions : Array[UInt32],
values : Hdf5WriteValues,
fill_value? : Hdf5WriteValues,
filters~ : Array[Hdf5WriteFilter] = [],
) -> Unit raise Hdf5Error {
let normalized = canonical_writer_path(path)
guard normalized != "/" else {
raise InvalidArgument("a dataset cannot replace the root group")
}
guard !self.has_entry(normalized) else {
raise InvalidArgument("writer object path already exists")
}
validate_write_values(dimensions, values)
guard dimensions.length() > 0 &&
chunk_dimensions.length() == dimensions.length() else {
raise InvalidArgument("chunk dimensions must match a simple dataspace")
}
for axis in 0.. 0UL && dimension > 0U && dimension <= 0x7FFFFFFFU else {
raise InvalidArgument("chunk dimension is invalid")
}
}
chunk_grid(dimensions, chunk_dimensions) |> ignore
validate_writer_chunk_size(
chunk_dimensions,
writer_value_element_size(values),
)
validate_write_filters(filters, self.custom_filters)
validate_writer_filters_for_values(values, filters)
let copied_fill = validated_write_fill(values, fill_value)
self.ensure_groups(writer_parent_path(normalized))
self.record_link(normalized)
self.datasets.push({
path: normalized,
dimensions: dimensions.copy(),
maximum_dimensions: dimensions.copy(),
values: copy_write_values(values),
storage: WritableChunked(
chunk_dimensions.copy(),
copy_write_filters(filters),
),
fill_value: copied_fill,
chunk_filter_masks: [],
})
}
///|
fn Hdf5Writer::writer_dataset_index(
self : Hdf5Writer,
path : String,
) -> Int32 raise Hdf5Error {
let normalized = canonical_writer_path(path)
match self.datasets.search_by(dataset => dataset.path == normalized) {
Some(index) => index
None => raise InvalidArgument("writer dataset does not exist")
}
}
///|
fn writer_dataset_chunk_grid(
dataset : WritableDataset,
) -> (Array[UInt32], Array[Hdf5WriteFilter], Array[Int32], Array[UInt64]) raise Hdf5Error {
match dataset.storage {
WritableChunked(dimensions, filters) => {
let order : Array[Int32] = []
for axis in 0.. {
let order : Array[Int32] = []
if unlimited_axes.length() == 1 {
order.push(unlimited_axes[0])
} else {
for axis in 0..
raise InvalidArgument("dataset does not use chunked storage")
}
}
///|
fn writer_chunk_coordinates(
chunk_index : Int32,
grid : Array[Int32],
order : Array[Int32],
) -> Array[UInt64] raise Hdf5Error {
guard chunk_index >= 0 else {
raise InvalidArgument("chunk index is outside the dataset grid")
}
let coordinates = Array::make(order.length(), 0UL)
let mut remaining = chunk_index
let mut position = order.length()
while position > 0 {
position = position - 1
let axis = order[position]
let count = grid[axis]
guard count > 0 else {
raise InvalidArgument("chunk index is outside the dataset grid")
}
coordinates[axis] = (remaining % count).to_uint64()
remaining = remaining / count
}
guard remaining == 0 else {
raise InvalidArgument("chunk index is outside the dataset grid")
}
coordinates
}
///|
/// Sets the HDF5 filter mask for one chunk of an existing chunked dataset.
///
/// Bit `n` set means filter `n` in the dataset pipeline is skipped for that
/// chunk. The mask is stored in the fixed-array or extensible-array chunk index.
pub fn Hdf5Writer::set_chunk_filter_mask(
self : Hdf5Writer,
path : String,
chunk_index : Int32,
filter_mask : UInt32,
) -> Unit raise Hdf5Error {
let index = self.writer_dataset_index(path)
let dataset = self.datasets[index]
let (chunk_dimensions, filters, order, grid_dimensions) = writer_dataset_chunk_grid(
dataset,
)
guard filters.length() > 0 else {
raise InvalidArgument("dataset has no filters for a chunk mask")
}
guard filters.length() == 32 || filter_mask >> filters.length() == 0U else {
raise InvalidArgument("chunk filter mask exceeds the filter pipeline")
}
let (grid, chunk_count) = chunk_grid(grid_dimensions, chunk_dimensions)
guard chunk_index >= 0 && chunk_index < chunk_count else {
raise InvalidArgument("chunk index is outside the dataset grid")
}
let coordinates = writer_chunk_coordinates(chunk_index, grid, order)
let masks = dataset.chunk_filter_masks.copy()
if masks.search_by(entry => entry.coordinates == coordinates)
is Some(mask_index) {
masks[mask_index] = { coordinates, filter_mask, }
} else {
masks.push({ coordinates, filter_mask, })
}
self.datasets[index] = {
path: dataset.path,
dimensions: dataset.dimensions,
maximum_dimensions: dataset.maximum_dimensions,
values: dataset.values,
storage: dataset.storage,
fill_value: dataset.fill_value,
chunk_filter_masks: masks,
}
}
///|
fn dataset_chunk_filter_mask(
dataset : WritableDataset,
coordinates : Array[UInt64],
) -> UInt32 {
match
dataset.chunk_filter_masks.search_by(entry => {
entry.coordinates == coordinates
}) {
Some(index) => dataset.chunk_filter_masks[index].filter_mask
None => 0U
}
}
///|
fn validate_extensible_dimensions(
dimensions : Array[UInt64],
maximum_dimensions : Array[UInt64?],
chunk_dimensions : Array[UInt32],
) -> (Array[UInt64], Array[Int32]) raise Hdf5Error {
guard dimensions.length() > 0 &&
maximum_dimensions.length() == dimensions.length() &&
chunk_dimensions.length() == dimensions.length() else {
raise InvalidArgument(
"extensible dimensions must have matching nonzero ranks",
)
}
let encoded_maximum : Array[UInt64] = []
let unlimited_axes : Array[Int32] = []
for axis in 0.. 0U &&
chunk_dimensions[axis] <= 0x7FFFFFFFU else {
raise InvalidArgument("extensible dataset dimension is invalid")
}
match maximum_dimensions[axis] {
Some(maximum) => {
guard maximum > 0UL &&
maximum <= 0x7FFFFFFFUL &&
dimensions[axis] <= maximum else {
raise InvalidArgument(
"extensible dataset maximum dimension is invalid",
)
}
encoded_maximum.push(maximum)
}
None => {
unlimited_axes.push(axis)
encoded_maximum.push(0xFFFFFFFFFFFFFFFFUL)
}
}
}
guard unlimited_axes.length() > 0 else {
raise InvalidArgument(
"extensible dataset requires at least one unlimited dimension",
)
}
(encoded_maximum, unlimited_axes)
}
///|
/// Adds a chunked dataset with one or more unlimited dimensions.
///
/// A `None` maximum marks an unlimited axis. One unlimited axis uses an
/// extensible-array index; multiple unlimited axes use a v2 B-tree.
pub fn Hdf5Writer::add_extensible_dataset(
self : Hdf5Writer,
path : String,
dimensions : Array[UInt64],
maximum_dimensions : Array[UInt64?],
chunk_dimensions : Array[UInt32],
values : Hdf5WriteValues,
fill_value? : Hdf5WriteValues,
filters~ : Array[Hdf5WriteFilter] = [],
) -> Unit raise Hdf5Error {
let normalized = canonical_writer_path(path)
guard normalized != "/" else {
raise InvalidArgument("a dataset cannot replace the root group")
}
guard !self.has_entry(normalized) else {
raise InvalidArgument("writer object path already exists")
}
validate_write_values(dimensions, values)
let (encoded_maximum, unlimited_axes) = validate_extensible_dimensions(
dimensions, maximum_dimensions, chunk_dimensions,
)
validate_writer_chunk_size(
chunk_dimensions,
writer_value_element_size(values),
)
validate_write_filters(filters, self.custom_filters)
validate_writer_filters_for_values(values, filters)
let copied_fill = validated_write_fill(values, fill_value)
let grid_dimensions = dimensions.copy()
for axis in 0.. ignore
self.ensure_groups(writer_parent_path(normalized))
self.record_link(normalized)
self.datasets.push({
path: normalized,
dimensions: dimensions.copy(),
maximum_dimensions: encoded_maximum,
values: copy_write_values(values),
storage: WritableExtensible(
chunk_dimensions.copy(),
copy_write_filters(filters),
unlimited_axes,
),
fill_value: copied_fill,
chunk_filter_masks: [],
})
}
///|
fn default_structured_value(
datatype : Datatype,
) -> DatasetValue raise Hdf5Error {
match datatype.class {
0 =>
if datatype.size > 8 {
BigInteger(@bigint.BigInt::from_int(0))
} else if datatype.signed {
SignedInteger(0L)
} else {
UnsignedInteger(0UL)
}
1 => FloatingPoint(0.0)
2 =>
if datatype.size > 8 {
BigTime(@bigint.BigInt::from_int(0))
} else {
Time(0UL)
}
3 => Text("")
4 =>
if datatype.size > 8 {
BigBitfield(@bigint.BigInt::from_int(0))
} else {
Bitfield(0UL)
}
5 => Opaque(Bytes::make(datatype.size, b'\x00'))
6 =>
Compound(
datatype.compound_members.map(field => {
name: field.name,
value: default_structured_value(field.datatype),
}),
)
7 =>
if datatype.reference_kind == 1 || datatype.reference_kind == 3 {
RegionReference(None, RegionEmpty)
} else if datatype.reference_kind == 4 {
AttributeReference(None, "")
} else {
ObjectReference(None)
}
8 => {
let zero = datatype.enum_members.search_by(enum_item => {
enum_item.value ==
(if datatype.signed {
EnumValue::Signed(0L)
} else {
EnumValue::Unsigned(0UL)
})
})
guard zero is Some(index) else {
raise InvalidArgument("enum datatype has no zero-valued default fill")
}
Enumeration(datatype.enum_members[index].name)
}
9 => if datatype.vlen_is_string { Text("") } else { Sequence([]) }
10 => {
let base = datatype.base_types[0]
Elements(
Array::make(datatype.size / base.size, default_structured_value(base)),
)
}
11 => Complex(0.0, 0.0)
_ =>
raise InvalidArgument("writer structured datatype class is unsupported")
}
}
///|
fn default_write_fill(
values : Hdf5WriteValues,
) -> Hdf5WriteValues raise Hdf5Error {
match values {
Int8Values(_) => Int8Values([0])
Int16Values(_) => Int16Values([0])
Int32Values(_) => Int32Values([0])
Int64Values(_) => Int64Values([0L])
UInt8Values(_) => UInt8Values([0U])
UInt16Values(_) => UInt16Values([0U])
UInt32Values(_) => UInt32Values([0U])
UInt64Values(_) => UInt64Values([0UL])
BigIntegerValues(_, size, signed) =>
BigIntegerValues([@bigint.BigInt::from_int(0)], size, signed)
Float16Values(_) => Float16Values([0.0])
Float32Values(_) => Float32Values([0.0])
Float64Values(_) => Float64Values([0.0])
FixedStringValues(_, width) => FixedStringValues([""], width)
VariableStringValues(_) => VariableStringValues([""])
ObjectReferenceValues(_) => ObjectReferenceValues([None])
Complex64Values(_) => Complex64Values([(0.0, 0.0)])
Complex128Values(_) => Complex128Values([(0.0, 0.0)])
StructuredValues(datatype, _) =>
StructuredValues(datatype, [default_structured_value(datatype)])
}
}
///|
fn writer_linear_destination(
source_index : Int32,
source_dimensions : Array[UInt64],
destination_dimensions : Array[UInt64],
offsets : Array[UInt64],
) -> Int32? {
let rank = source_dimensions.length()
if offsets.length() != rank {
return None
}
let coordinates = Array::make(rank, 0UL)
let mut remaining = source_index.to_uint64()
let mut axis = rank
while axis > 0 {
axis = axis - 1
let width = source_dimensions[axis]
if width == 0UL {
return None
}
coordinates[axis] = remaining % width
remaining = remaining / width
}
let mut destination = 0UL
for dimension in 0..= destination_dimensions[dimension] {
return None
}
destination = destination * destination_dimensions[dimension] + coordinate
}
Some(destination.to_int())
}
///|
fn repeated_write_values(
template : Hdf5WriteValues,
fill : Hdf5WriteValues,
count : Int32,
) -> Hdf5WriteValues raise Hdf5Error {
match (template, fill) {
(Int8Values(_), Int8Values(items)) =>
Int8Values(Array::make(count, items[0]))
(Int16Values(_), Int16Values(items)) =>
Int16Values(Array::make(count, items[0]))
(Int32Values(_), Int32Values(items)) =>
Int32Values(Array::make(count, items[0]))
(Int64Values(_), Int64Values(items)) =>
Int64Values(Array::make(count, items[0]))
(UInt8Values(_), UInt8Values(items)) =>
UInt8Values(Array::make(count, items[0]))
(UInt16Values(_), UInt16Values(items)) =>
UInt16Values(Array::make(count, items[0]))
(UInt32Values(_), UInt32Values(items)) =>
UInt32Values(Array::make(count, items[0]))
(UInt64Values(_), UInt64Values(items)) =>
UInt64Values(Array::make(count, items[0]))
(BigIntegerValues(_, size, signed), BigIntegerValues(items, _, _)) =>
BigIntegerValues(Array::make(count, items[0]), size, signed)
(Float16Values(_), Float16Values(items)) =>
Float16Values(Array::make(count, items[0]))
(Float32Values(_), Float32Values(items)) =>
Float32Values(Array::make(count, items[0]))
(Float64Values(_), Float64Values(items)) =>
Float64Values(Array::make(count, items[0]))
(FixedStringValues(_, width), FixedStringValues(items, _)) =>
FixedStringValues(Array::make(count, items[0]), width)
(VariableStringValues(_), VariableStringValues(items)) =>
VariableStringValues(Array::make(count, items[0]))
(ObjectReferenceValues(_), ObjectReferenceValues(items)) =>
ObjectReferenceValues(Array::make(count, items[0]))
(Complex64Values(_), Complex64Values(items)) =>
Complex64Values(Array::make(count, items[0]))
(Complex128Values(_), Complex128Values(items)) =>
Complex128Values(Array::make(count, items[0]))
(StructuredValues(datatype, _), StructuredValues(_, items)) =>
StructuredValues(datatype, Array::make(count, items[0]))
_ =>
raise InvalidArgument("fill value must be one value of the dataset type")
}
}
///|
/// Changes an extensible dataset shape while preserving its overlapping data.
pub fn Hdf5Writer::resize_dataset(
self : Hdf5Writer,
path : String,
dimensions : Array[UInt64],
) -> Unit raise Hdf5Error {
let index = self.writer_dataset_index(path)
let dataset = self.datasets[index]
let (chunk_dimensions, filters, unlimited_axes) = match dataset.storage {
WritableExtensible(chunks, pipeline, axes) => (chunks, pipeline, axes)
_ => raise InvalidArgument("only extensible datasets can be resized")
}
guard dimensions.length() == dataset.dimensions.length() else {
raise InvalidArgument("resized dataset rank does not match")
}
for axis in 0.. ignore
let fill = match dataset.fill_value {
Some(value) => value
None => default_write_fill(dataset.values)
}
let values = overlay_write_values(
repeated_write_values(dataset.values, fill, writer_shape_count(dimensions)),
dataset.values,
dataset.dimensions,
dimensions,
Array::make(dimensions.length(), 0UL),
false,
)
self.datasets[index] = {
path: dataset.path,
dimensions: dimensions.copy(),
maximum_dimensions: dataset.maximum_dimensions,
values,
storage: WritableExtensible(chunk_dimensions, filters, unlimited_axes),
fill_value: dataset.fill_value,
chunk_filter_masks: dataset.chunk_filter_masks,
}
}
///|
fn[T] overlay_array(
output : Array[T],
items : Array[T],
source_dimensions : Array[UInt64],
destination_dimensions : Array[UInt64],
offsets : Array[UInt64],
strict : Bool,
) -> Array[T] raise Hdf5Error {
for index in 0.. output[destination] = items[index]
None =>
if strict {
raise InvalidArgument(
"appended value is outside the destination shape",
)
}
}
}
output
}
///|
fn overlay_write_values(
target : Hdf5WriteValues,
source : Hdf5WriteValues,
source_dimensions : Array[UInt64],
destination_dimensions : Array[UInt64],
offsets : Array[UInt64],
strict : Bool,
) -> Hdf5WriteValues raise Hdf5Error {
match (target, source) {
(Int8Values(output), Int8Values(items)) =>
Int8Values(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
(Int16Values(output), Int16Values(items)) =>
Int16Values(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
(Int32Values(output), Int32Values(items)) =>
Int32Values(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
(Int64Values(output), Int64Values(items)) =>
Int64Values(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
(UInt8Values(output), UInt8Values(items)) =>
UInt8Values(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
(UInt16Values(output), UInt16Values(items)) =>
UInt16Values(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
(UInt32Values(output), UInt32Values(items)) =>
UInt32Values(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
(UInt64Values(output), UInt64Values(items)) =>
UInt64Values(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
(Float16Values(output), Float16Values(items)) =>
Float16Values(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
(Float32Values(output), Float32Values(items)) =>
Float32Values(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
(Float64Values(output), Float64Values(items)) =>
Float64Values(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
(StructuredValues(datatype, output), StructuredValues(_, items)) =>
StructuredValues(
datatype,
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
(FixedStringValues(output, width), FixedStringValues(items, source_width)) => {
guard width == source_width else {
raise InvalidArgument("appended values must match the dataset type")
}
FixedStringValues(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
width,
)
}
(VariableStringValues(output), VariableStringValues(items)) =>
VariableStringValues(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
(ObjectReferenceValues(output), ObjectReferenceValues(items)) =>
ObjectReferenceValues(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
(Complex64Values(output), Complex64Values(items)) =>
Complex64Values(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
(Complex128Values(output), Complex128Values(items)) =>
Complex128Values(
overlay_array(
output, items, source_dimensions, destination_dimensions, offsets, strict,
),
)
_ => raise InvalidArgument("appended values must match the dataset type")
}
}
///|
/// Appends a block at the trailing corner of the dataset's unlimited axes.
///
/// Fixed axes must match the current dataset shape. Each unlimited axis grows
/// by the corresponding block extent. A one-unlimited-axis dataset therefore
/// appends along that axis, while a multi-unlimited dataset grows every
/// unlimited axis and places the block at the old shape's trailing corner.
pub fn Hdf5Writer::append_dataset(
self : Hdf5Writer,
path : String,
dimensions : Array[UInt64],
values : Hdf5WriteValues,
) -> Unit raise Hdf5Error {
let index = self.writer_dataset_index(path)
let dataset = self.datasets[index]
let unlimited_axes = match dataset.storage {
WritableExtensible(_, _, axes) => axes
_ => raise InvalidArgument("only extensible datasets can be appended")
}
guard dimensions.length() == dataset.dimensions.length() else {
raise InvalidArgument("appended dataset rank does not match")
}
validate_write_values(dimensions, values)
guard same_write_datatype(dataset.values, values) else {
raise InvalidArgument("appended values must match the dataset type")
}
let old_dimensions = dataset.dimensions.copy()
let new_dimensions = old_dimensions.copy()
let offsets = Array::make(dimensions.length(), 0UL)
for axis in 0.. Unit raise Hdf5Error {
let index = self.writer_dataset_index(path)
let dataset = self.datasets[index]
let unlimited_axes = match dataset.storage {
WritableExternal(_) =>
raise InvalidArgument("external datasets cannot be patched in a builder")
WritableVirtual(_) =>
raise InvalidArgument("virtual datasets cannot be patched in a builder")
WritableExtensible(_, _, axes) => axes
WritableChunked(_, _) | WritableCompact | WritableContiguous => []
}
let rank = dataset.dimensions.length()
guard offsets.length() == rank && dimensions.length() == rank else {
raise InvalidArgument("dataset slab rank does not match the dataset")
}
validate_write_values(dimensions, values)
guard same_write_datatype(dataset.values, values) else {
raise InvalidArgument("slab values must match the dataset type")
}
let target_dimensions = dataset.dimensions.copy()
for axis in 0.. target_dimensions[axis] {
if unlimited_axes.contains(axis) {
target_dimensions[axis] = end
} else {
raise InvalidArgument("dataset slab extends beyond a fixed dimension")
}
}
}
if target_dimensions != dataset.dimensions {
self.resize_dataset(path, target_dimensions)
}
let resized = self.datasets[index]
self.datasets[index] = {
path: resized.path,
dimensions: resized.dimensions,
maximum_dimensions: resized.maximum_dimensions,
values: overlay_write_values(
resized.values,
values,
dimensions,
resized.dimensions,
offsets,
true,
),
storage: resized.storage,
fill_value: resized.fill_value,
chunk_filter_masks: resized.chunk_filter_masks,
}
}
///|
/// Adds an inline attribute to an existing group or dataset.
pub fn Hdf5Writer::add_attribute(
self : Hdf5Writer,
object_path : String,
name : String,
dimensions : Array[UInt64],
values : Hdf5WriteValues,
) -> Unit raise Hdf5Error {
let normalized = canonical_writer_path(object_path)
guard self.has_object(normalized) else {
raise InvalidArgument("attribute target does not exist")
}
let encoded_name = @utf8.encode(name)
guard encoded_name.length() > 0 && encoded_name.length() < 65535 else {
raise InvalidArgument("attribute name is empty or too long")
}
for byte in encoded_name {
guard byte != b'\x00' else {
raise InvalidArgument("attribute names cannot contain NUL")
}
}
guard self.attributes.search_by(attribute => {
attribute.object_path == normalized && attribute.name == name
})
is None else {
raise InvalidArgument("attribute name already exists")
}
validate_write_values(dimensions, values)
let creation_order = self.object_attributes(normalized).length().to_uint64()
self.attributes.push({
object_path: normalized,
name,
dimensions: dimensions.copy(),
values: copy_write_values(values),
creation_order,
})
}
///|
fn write_uint_width(
buffer : @buffer.Buffer,
value : UInt64,
size : Int32,
little_endian~ : Bool = true,
) -> Unit {
for index in 0..> shift) & 0xFFUL).to_byte())
}
}
///|
fn validate_big_integer(
value : @bigint.BigInt,
size : Int32,
signed : Bool,
) -> Unit raise Hdf5Error {
guard size >= 1 && size <= 8191 else {
raise InvalidArgument("writer integer width is invalid")
}
let bits = size * 8
if signed {
let limit = @bigint.BigInt::from_int(1).shl(bits - 1)
guard value.compare(limit.neg()) >= 0 && value.compare(limit) < 0 else {
raise InvalidArgument("writer signed integer is out of range")
}
} else {
let limit = @bigint.BigInt::from_int(1).shl(bits)
guard value.compare(@bigint.BigInt::from_int(0)) >= 0 &&
value.compare(limit) < 0 else {
raise InvalidArgument("writer unsigned integer is out of range")
}
}
}
///|
fn write_big_integer_width(
buffer : @buffer.Buffer,
value : @bigint.BigInt,
size : Int32,
signed : Bool,
little_endian~ : Bool = true,
) -> Unit raise Hdf5Error {
validate_big_integer(value, size, signed)
let magnitude = if value.compare(@bigint.BigInt::from_int(0)) < 0 {
@bigint.BigInt::from_int(1).shl(size * 8).add(value)
} else {
value
}
let encoded = magnitude.to_octets(length=size)
for index in 0.. Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=12)
let flags = (if signed { 8 } else { 0 }) | (if little_endian { 0 } else { 1 })
let precision = if bit_precision > 0 { bit_precision } else { size * 8 }
buffer.write_byte(b'\x10')
buffer.write_byte(flags.to_byte())
buffer.write_byte(b'\x00')
buffer.write_byte(b'\x00')
buffer.write_uint_le(size.reinterpret_as_uint())
buffer.write_uint16_le(bit_offset.to_uint16())
buffer.write_uint16_le(precision.to_uint16())
buffer.to_bytes()
}
///|
fn encode_float_datatype(size : Int32, little_endian~ : Bool = true) -> Bytes {
let (sign_position, exponent_position, exponent_size, mantissa_size, bias) = if size ==
2 {
(15, 10, 5, 10, 15U)
} else if size == 4 {
(31, 23, 8, 23, 127U)
} else {
(63, 52, 11, 52, 1023U)
}
let buffer = @buffer.Buffer::Buffer(size_hint=20)
buffer.write_byte(b'\x11')
buffer.write_byte(if little_endian { b'\x20' } else { b'\x21' })
buffer.write_byte(sign_position.to_byte())
buffer.write_byte(b'\x00')
buffer.write_uint_le(size.reinterpret_as_uint())
buffer.write_uint16_le(0)
buffer.write_uint16_le((size * 8).to_uint16())
buffer.write_byte(exponent_position.to_byte())
buffer.write_byte(exponent_size.to_byte())
buffer.write_byte(b'\x00')
buffer.write_byte(mantissa_size.to_byte())
buffer.write_uint_le(bias)
buffer.to_bytes()
}
///|
fn encode_x87_float_datatype(size : Int32, little_endian~ : Bool) -> Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=20)
buffer.write_byte(b'\x11')
buffer.write_byte(if little_endian { b'\x20' } else { b'\x21' })
buffer.write_byte(b'\x4f')
buffer.write_byte(b'\x00')
buffer.write_uint_le(size.reinterpret_as_uint())
buffer.write_uint16_le(0)
buffer.write_uint16_le(80)
buffer.write_byte(b'\x40')
buffer.write_byte(b'\x0f')
buffer.write_byte(b'\x00')
buffer.write_byte(b'\x3f')
buffer.write_uint_le(16383U)
buffer.to_bytes()
}
///|
fn writer_x87_fields(number : Double) -> (UInt32, UInt64) {
let bits = number.reinterpret_as_uint64()
let sign = ((bits >> 63) & 1UL).to_uint() << 15
let exponent = ((bits >> 52) & 0x7FFUL).to_int()
let fraction = bits & 0xFFFFFFFFFFFFFUL
if exponent == 0x7FF {
let significand = if fraction == 0UL {
1UL << 63
} else {
(1UL << 63) | (fraction << 11)
}
return (sign | 0x7FFFU, significand)
}
if exponent == 0 && fraction == 0UL {
return (sign, 0UL)
}
if exponent == 0 {
let mut highest = 51
while highest > 0 && (fraction & (1UL << highest)) == 0UL {
highest = highest - 1
}
let unbiased = highest - 1074
let extended_exponent = unbiased + 16383
let significand = fraction << (63 - highest)
return (sign | extended_exponent.reinterpret_as_uint(), significand)
}
let extended_exponent = exponent - 1023 + 16383
let significand = (1UL << 63) | (fraction << 11)
(sign | extended_exponent.reinterpret_as_uint(), significand)
}
///|
fn encode_x87_float_value(
buffer : @buffer.Buffer,
number : Double,
size : Int32,
little_endian : Bool,
) -> Unit {
let (sign_exponent, significand) = writer_x87_fields(number)
if little_endian {
buffer.write_uint64_le(significand)
buffer.write_uint16_le(sign_exponent.to_uint16())
writer_write_zeros(buffer, size - 10)
} else {
writer_write_zeros(buffer, size - 10)
buffer.write_uint16_be(sign_exponent.to_uint16())
for shift in [56, 48, 40, 32, 24, 16, 8, 0] {
buffer.write_byte(((significand >> shift) & 0xFFUL).to_byte())
}
}
}
///|
fn writer_float16_bits(value : Float) -> UInt32 {
let bits = value.reinterpret_as_uint()
let sign = (bits >> 16) & 0x8000U
let exponent = ((bits >> 23) & 0xFFU).reinterpret_as_int()
let mantissa = bits & 0x7FFFFFU
if exponent == 0xFF {
return if mantissa == 0U {
sign | 0x7C00U
} else {
sign | 0x7E00U | (mantissa >> 13)
}
}
let half_exponent = exponent - 127 + 15
if half_exponent >= 0x1F {
return sign | 0x7C00U
}
if half_exponent <= 0 {
if half_exponent < -10 {
return sign
}
let normalized = mantissa | 0x800000U
let shift = 14 - half_exponent
let truncated = normalized >> shift
let remainder = normalized & ((1U << shift) - 1U)
let halfway = 1U << (shift - 1)
let rounded = if remainder > halfway ||
(remainder == halfway && (truncated & 1U) == 1U) {
truncated + 1U
} else {
truncated
}
return sign | rounded
}
let truncated = mantissa >> 13
let remainder = mantissa & 0x1FFFU
let rounded = if remainder > 0x1000U ||
(remainder == 0x1000U && (truncated & 1U) == 1U) {
truncated + 1U
} else {
truncated
}
if rounded == 0x400U {
let carried = half_exponent + 1
if carried >= 0x1F {
return sign | 0x7C00U
}
return sign | (carried.reinterpret_as_uint() << 10)
}
sign | (half_exponent.reinterpret_as_uint() << 10) | rounded
}
///|
fn writer_float16_bits_from_double(value : Double) -> UInt32 {
if value.is_nan() {
return writer_float16_bits(Float::from_double(value))
}
let sign = if value.reinterpret_as_uint64() >> 63 != 0UL {
0x8000U
} else {
0U
}
let magnitude = Double::abs(value)
if magnitude.is_inf() {
return sign | 0x7C00U
}
if magnitude == 0.0 {
return sign
}
let candidate = writer_float16_bits(Float::from_double(magnitude)) & 0x7FFFU
if candidate == 0x7C00U {
return if magnitude >= 65520.0 { sign | 0x7C00U } else { sign | 0x7BFFU }
}
let candidate_value = float16_to_float(candidate).to_double()
if magnitude == candidate_value {
return sign | candidate
}
let neighbor = if candidate_value < magnitude {
if candidate == 0x7BFFU {
return if magnitude >= 65520.0 {
sign | 0x7C00U
} else {
sign | candidate
}
}
candidate + 1U
} else {
if candidate == 0U {
return sign
}
candidate - 1U
}
let neighbor_value = float16_to_float(neighbor).to_double()
let midpoint = (candidate_value + neighbor_value) / 2.0
let selected = if magnitude > midpoint {
neighbor
} else if magnitude < midpoint {
candidate
} else if (candidate & 1U) == 0U {
candidate
} else {
neighbor
}
sign | selected
}
///|
fn encode_writer_datatype(
datatype : Datatype,
reference_address_size~ : Int32 = 8,
) -> Bytes raise Hdf5Error {
validate_writer_datatype(
datatype,
0,
reference_address_size=Some(reference_address_size),
)
match datatype.class {
0 =>
encode_integer_datatype(
datatype.signed,
datatype.size,
little_endian=datatype.little_endian,
bit_offset=datatype.bit_offset,
bit_precision=datatype.bit_precision,
)
1 =>
if datatype.size == 10 || datatype.size == 16 {
encode_x87_float_datatype(
datatype.size,
little_endian=datatype.little_endian,
)
} else {
encode_float_datatype(
datatype.size,
little_endian=datatype.little_endian,
)
}
2 => {
let buffer = @buffer.Buffer::Buffer(size_hint=12)
buffer.write_byte(b'\x12')
buffer.write_byte(if datatype.little_endian { b'\x00' } else { b'\x01' })
buffer.write_byte(b'\x00')
buffer.write_byte(b'\x00')
buffer.write_uint_le(datatype.size.reinterpret_as_uint())
buffer.write_uint16_le(datatype.bit_precision.to_uint16())
buffer.to_bytes()
}
3 => encode_string_datatype(datatype.size)
4 => {
let buffer = @buffer.Buffer::Buffer(size_hint=14)
buffer.write_byte(b'\x14')
buffer.write_byte(if datatype.little_endian { b'\x00' } else { b'\x01' })
buffer.write_byte(b'\x00')
buffer.write_byte(b'\x00')
buffer.write_uint_le(datatype.size.reinterpret_as_uint())
buffer.write_uint16_le(datatype.bit_offset.to_uint16())
buffer.write_uint16_le(datatype.bit_precision.to_uint16())
buffer.to_bytes()
}
5 => {
let tag = @utf8.encode(datatype.opaque_tag)
let padded = if tag.length() == 0 {
0
} else {
round_up_to_eight(tag.length() + 1)
}
let buffer = @buffer.Buffer::Buffer(size_hint=8 + padded)
buffer.write_byte(b'\x15')
buffer.write_byte(padded.to_byte())
buffer.write_byte(b'\x00')
buffer.write_byte(b'\x00')
buffer.write_uint_le(datatype.size.reinterpret_as_uint())
if padded > 0 {
writer_write_bytes(buffer, tag)
writer_write_zeros(buffer, padded - tag.length())
}
buffer.to_bytes()
}
6 => {
let buffer = @buffer.Buffer::Buffer()
buffer.write_byte(b'\x46')
buffer.write_byte((datatype.compound_members.length() & 0xFF).to_byte())
buffer.write_byte((datatype.compound_members.length() >> 8).to_byte())
buffer.write_byte(b'\x00')
buffer.write_uint_le(datatype.size.reinterpret_as_uint())
let offset_width = datatype_offset_width(datatype.size)
for field in datatype.compound_members {
writer_write_bytes(buffer, @utf8.encode(field.name))
buffer.write_byte(b'\x00')
write_uint_width(buffer, field.offset.to_uint64(), offset_width)
writer_write_bytes(
buffer,
encode_writer_datatype(field.datatype, reference_address_size~),
)
}
buffer.to_bytes()
}
7 =>
encode_reference_datatype(
datatype.size,
datatype.reference_kind,
datatype.version,
)
8 => {
let buffer = @buffer.Buffer::Buffer()
buffer.write_byte(b'\x38')
buffer.write_byte((datatype.enum_members.length() & 0xFF).to_byte())
buffer.write_byte((datatype.enum_members.length() >> 8).to_byte())
buffer.write_byte(b'\x00')
buffer.write_uint_le(datatype.size.reinterpret_as_uint())
writer_write_bytes(
buffer,
encode_writer_datatype(datatype.base_types[0], reference_address_size~),
)
for enum_item in datatype.enum_members {
writer_write_bytes(buffer, @utf8.encode(enum_item.name))
buffer.write_byte(b'\x00')
}
for enum_item in datatype.enum_members {
match enum_item.value {
EnumValue::Signed(value) =>
write_uint_width(
buffer,
value.reinterpret_as_uint64(),
datatype.size,
little_endian=datatype.base_types[0].little_endian,
)
EnumValue::Unsigned(value) =>
write_uint_width(
buffer,
value,
datatype.size,
little_endian=datatype.base_types[0].little_endian,
)
EnumValue::Big(value) =>
write_big_integer_width(
buffer,
value,
datatype.size,
datatype.base_types[0].signed,
little_endian=datatype.base_types[0].little_endian,
)
}
}
buffer.to_bytes()
}
9 => {
let buffer = @buffer.Buffer::Buffer()
buffer.write_byte(((1 << 4) | 9).to_byte())
buffer.write_byte(
if datatype.vlen_is_string {
(1 | (datatype.string_padding << 4)).to_byte()
} else {
b'\x00'
},
)
buffer.write_byte(
if datatype.vlen_is_string {
datatype.string_charset.to_byte()
} else {
b'\x00'
},
)
buffer.write_byte(b'\x00')
buffer.write_uint_le(datatype.size.reinterpret_as_uint())
if datatype.vlen_is_string {
writer_write_bytes(
buffer,
encode_writer_datatype(
datatype.base_types[0],
reference_address_size~,
),
)
} else {
writer_write_bytes(
buffer,
encode_writer_datatype(
datatype.base_types[0],
reference_address_size~,
),
)
}
buffer.to_bytes()
}
10 => {
let buffer = @buffer.Buffer::Buffer()
buffer.write_byte(b'\x4a')
buffer.write_byte(b'\x00')
buffer.write_byte(b'\x00')
buffer.write_byte(b'\x00')
buffer.write_uint_le(datatype.size.reinterpret_as_uint())
buffer.write_byte(datatype.array_dimensions.length().to_byte())
for dimension in datatype.array_dimensions {
buffer.write_uint_le(dimension.to_uint())
}
writer_write_bytes(
buffer,
encode_writer_datatype(datatype.base_types[0], reference_address_size~),
)
buffer.to_bytes()
}
11 => {
let buffer = @buffer.Buffer::Buffer()
buffer.write_byte(((datatype.version << 4) | 11).to_byte())
buffer.write_byte(b'\x01')
buffer.write_byte(b'\x00')
buffer.write_byte(b'\x00')
buffer.write_uint_le(datatype.size.reinterpret_as_uint())
writer_write_bytes(
buffer,
encode_writer_datatype(datatype.base_types[0], reference_address_size~),
)
buffer.to_bytes()
}
_ => raise InvalidField("writer structured datatype class is unsupported")
}
}
///|
fn encode_structured_value(
datatype : Datatype,
value : DatasetValue,
heap~ : WritableGlobalHeap = empty_writable_heap(),
objects~ : Array[WritableObject] = [],
reference_address_size~ : Int32 = 8,
vlen_address_size~ : Int32 = 0,
) -> Bytes raise Hdf5Error {
let vlen_width = if vlen_address_size > 0 {
vlen_address_size
} else {
reference_address_size
}
validate_structured_value(value, datatype, 0)
let buffer = @buffer.Buffer::Buffer(size_hint=datatype.size)
match datatype.class {
0 =>
if datatype.size > 8 {
match value {
BigInteger(number) => {
let one = @bigint.BigInt::from_int(1)
let modulus = one.shl(datatype.size * 8)
let raw = if number.compare(@bigint.BigInt::from_int(0)) < 0 {
modulus.add(number)
} else {
number
}
let mask = one.shl(datatype.bit_precision).sub(one)
let stored = raw.land(mask).shl(datatype.bit_offset)
write_big_integer_width(
buffer,
stored,
datatype.size,
false,
little_endian=datatype.little_endian,
)
}
_ => writer_write_zeros(buffer, datatype.size)
}
} else {
let raw = match value {
SignedInteger(number) => number.reinterpret_as_uint64()
UnsignedInteger(number) => number
_ => 0UL
}
let masked = if datatype.bit_precision == 64 {
raw
} else {
raw & ((1UL << datatype.bit_precision) - 1UL)
}
write_uint_width(
buffer,
masked << datatype.bit_offset,
datatype.size,
little_endian=datatype.little_endian,
)
}
1 =>
match value {
FloatingPoint(number) =>
if datatype.size == 10 || datatype.size == 16 {
encode_x87_float_value(
buffer,
number,
datatype.size,
datatype.little_endian,
)
} else if datatype.size == 2 {
let bits = writer_float16_bits_from_double(number).to_uint16()
if datatype.little_endian {
buffer.write_uint16_le(bits)
} else {
buffer.write_uint16_be(bits)
}
} else if datatype.size == 4 {
if datatype.little_endian {
buffer.write_float_le(Float::from_double(number))
} else {
buffer.write_float_be(Float::from_double(number))
}
} else if datatype.little_endian {
buffer.write_double_le(number)
} else {
buffer.write_double_be(number)
}
_ => ()
}
2 =>
match value {
Time(number) =>
write_uint_width(
buffer,
number,
datatype.size,
little_endian=datatype.little_endian,
)
BigTime(number) =>
write_big_integer_width(
buffer,
number,
datatype.size,
false,
little_endian=datatype.little_endian,
)
_ => ()
}
3 =>
match value {
Text(text) => {
let encoded = @utf8.encode(text)
writer_write_bytes(buffer, encoded)
let remaining = datatype.size - encoded.length()
if datatype.string_padding == 0 && remaining > 0 {
buffer.write_byte(b'\x00')
writer_write_zeros(buffer, remaining - 1)
} else if datatype.string_padding == 2 {
for _ in 0.. ()
}
4 =>
match value {
Bitfield(number) =>
write_uint_width(
buffer,
number << datatype.bit_offset,
datatype.size,
little_endian=datatype.little_endian,
)
BigBitfield(number) =>
write_big_integer_width(
buffer,
number.shl(datatype.bit_offset),
datatype.size,
false,
little_endian=datatype.little_endian,
)
_ => ()
}
5 =>
match value {
Opaque(bytes) => writer_write_bytes(buffer, bytes)
_ => ()
}
6 =>
match value {
Compound(fields) => {
let output = FixedArray::make(datatype.size, b'\x00')
for index in 0.. ()
}
7 =>
if datatype.version >= 4 {
match (datatype.reference_kind, value) {
(2, ObjectReference(path)) =>
match path {
None => writer_write_zeros(buffer, datatype.size)
Some(target) => {
let address = writer_reference_address(objects, target)
buffer.write_byte(b'\x02')
buffer.write_byte(b'\x00')
buffer.write_byte(reference_address_size.to_byte())
write_uint_width(buffer, address, reference_address_size)
writer_write_zeros(
buffer,
datatype.size - 3 - reference_address_size,
)
}
}
(2, ExternalObjectReference(_, _)) =>
match writable_external_reference_slot(heap, value) {
None => writer_write_zeros(buffer, datatype.size)
Some((payload, index)) => {
guard payload.length() <= 0x7FFFFFFF else {
raise InvalidArgument(
"version-4 external object reference payload is too large",
)
}
buffer.write_byte(b'\x02')
buffer.write_byte(b'\x01')
buffer.write_uint_le(payload.length().reinterpret_as_uint())
write_uint_width(
buffer,
heap.region_address,
reference_address_size,
)
buffer.write_uint_le(index.reinterpret_as_uint())
}
}
(3, RegionReference(path, selection)) =>
match path {
None => writer_write_zeros(buffer, datatype.size)
Some(target) => {
let payload = writer_region_reference_payload(
objects,
target,
selection,
version4=true,
reference_address_size~,
)
guard payload.length() <= 0x7FFFFFFF else {
raise InvalidArgument(
"version-4 region reference payload is too large",
)
}
buffer.write_byte(b'\x03')
buffer.write_byte(b'\x00')
buffer.write_uint_le(payload.length().reinterpret_as_uint())
write_uint_width(
buffer,
heap.region_address,
reference_address_size,
)
buffer.write_uint_le(
writable_region_heap_index(heap, payload).reinterpret_as_uint(),
)
}
}
(3, ExternalRegionReference(_, _, _)) =>
match writable_external_reference_slot(heap, value) {
None => writer_write_zeros(buffer, datatype.size)
Some((payload, index)) => {
buffer.write_byte(b'\x03')
buffer.write_byte(b'\x01')
buffer.write_uint_le(payload.length().reinterpret_as_uint())
write_uint_width(
buffer,
heap.region_address,
reference_address_size,
)
buffer.write_uint_le(index.reinterpret_as_uint())
}
}
(4, AttributeReference(path, name)) =>
match path {
None => writer_write_zeros(buffer, datatype.size)
Some(target) => {
let payload = writer_attribute_reference_payload(
objects,
target,
name,
reference_address_size~,
)
guard payload.length() <= 0x7FFFFFFF else {
raise InvalidArgument(
"version-4 attribute reference payload is too large",
)
}
buffer.write_byte(b'\x04')
buffer.write_byte(b'\x00')
buffer.write_uint_le(payload.length().reinterpret_as_uint())
write_uint_width(
buffer,
heap.region_address,
reference_address_size,
)
buffer.write_uint_le(
writable_region_heap_index(heap, payload).reinterpret_as_uint(),
)
}
}
(4, ExternalAttributeReference(_, _, _)) =>
match writable_external_reference_slot(heap, value) {
None => writer_write_zeros(buffer, datatype.size)
Some((payload, index)) => {
buffer.write_byte(b'\x04')
buffer.write_byte(b'\x01')
buffer.write_uint_le(payload.length().reinterpret_as_uint())
write_uint_width(
buffer,
heap.region_address,
reference_address_size,
)
buffer.write_uint_le(index.reinterpret_as_uint())
}
}
_ => ()
}
} else if datatype.reference_kind == 0 {
match value {
ObjectReference(path) => {
let address = match path {
None => 0UL
Some(target) => writer_reference_address(objects, target)
}
write_uint_width(buffer, address, datatype.size)
}
_ => ()
}
} else {
match value {
RegionReference(path, selection) =>
match path {
None => writer_write_zeros(buffer, datatype.size)
Some(target) => {
let payload = writer_region_reference_payload(
objects,
target,
selection,
reference_address_size~,
)
write_uint_width(
buffer,
heap.region_address,
reference_address_size,
)
buffer.write_uint_le(
writable_region_heap_index(heap, payload).reinterpret_as_uint(),
)
}
}
_ => ()
}
}
8 =>
match value {
Enumeration(name) => {
let index = datatype.enum_members
.search_by(enum_item => enum_item.name == name)
.unwrap()
match datatype.enum_members[index].value {
EnumValue::Signed(number) =>
write_uint_width(
buffer,
number.reinterpret_as_uint64(),
datatype.size,
little_endian=datatype.base_types[0].little_endian,
)
EnumValue::Unsigned(number) =>
write_uint_width(
buffer,
number,
datatype.size,
little_endian=datatype.base_types[0].little_endian,
)
EnumValue::Big(number) =>
write_big_integer_width(
buffer,
number,
datatype.size,
datatype.base_types[0].signed,
little_endian=datatype.base_types[0].little_endian,
)
}
}
_ => ()
}
9 =>
if datatype.vlen_is_string {
match value {
Text(text) =>
if @utf8.encode(text).length() == 0 {
buffer.write_uint_le(0U)
write_uint_width(buffer, 0UL, vlen_width)
buffer.write_uint_le(0U)
} else {
let encoded = @utf8.encode(text)
buffer.write_uint_le(encoded.length().reinterpret_as_uint())
write_uint_width(buffer, heap.address, vlen_width)
buffer.write_uint_le(
writable_heap_index(heap, text).reinterpret_as_uint(),
)
}
_ => ()
}
} else {
match value {
Sequence(items) =>
if items.length() == 0 {
buffer.write_uint_le(0U)
write_uint_width(buffer, 0UL, vlen_width)
buffer.write_uint_le(0U)
} else {
let encoded = @buffer.Buffer::Buffer()
let base = datatype.base_types[0]
for item in items {
writer_write_bytes(
encoded,
encode_structured_value(
base,
item,
heap~,
objects~,
reference_address_size~,
vlen_address_size~,
),
)
}
let data = encoded.to_bytes()
buffer.write_uint_le(items.length().reinterpret_as_uint())
write_uint_width(buffer, heap.address, vlen_width)
buffer.write_uint_le(
writable_sequence_heap_index(heap, data).reinterpret_as_uint(),
)
}
_ => ()
}
}
10 =>
match value {
Elements(items) =>
for item in items {
writer_write_bytes(
buffer,
encode_structured_value(
datatype.base_types[0],
item,
heap~,
objects~,
reference_address_size~,
vlen_address_size~,
),
)
}
_ => ()
}
11 =>
match (datatype.base_types[0], value) {
(base, Complex(real, imaginary)) => {
writer_write_bytes(
buffer,
encode_structured_value(
base,
FloatingPoint(real),
heap~,
objects~,
reference_address_size~,
vlen_address_size~,
),
)
writer_write_bytes(
buffer,
encode_structured_value(
base,
FloatingPoint(imaginary),
heap~,
objects~,
reference_address_size~,
vlen_address_size~,
),
)
}
_ => ()
}
_ => raise InvalidField("writer structured datatype class is unsupported")
}
guard buffer.length() == datatype.size else {
raise InvalidField("structured value size does not match its datatype")
}
buffer.to_bytes()
}
///|
fn encode_string_datatype(width : Int32) -> Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=8)
buffer.write_byte(b'\x13')
buffer.write_byte(b'\x11')
buffer.write_byte(b'\x00')
buffer.write_byte(b'\x00')
buffer.write_uint_le(width.reinterpret_as_uint())
buffer.to_bytes()
}
///|
fn encode_variable_string_datatype(size : Int32) -> Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=16)
buffer.write_byte(b'\x19')
buffer.write_byte(b'\x01')
buffer.write_byte(b'\x01')
buffer.write_byte(b'\x00')
buffer.write_uint_le(size.reinterpret_as_uint())
writer_write_bytes(buffer, encode_integer_datatype(false, 1))
buffer.to_bytes()
}
///|
fn encode_reference_datatype(
size : Int32,
reference_kind : Int32,
version : Int32,
) -> Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=8)
let encoded_version = if version >= 4 { version } else { 1 }
buffer.write_byte(((encoded_version << 4) | 7).to_byte())
let flags = if version >= 4 { 0x10 | reference_kind } else { reference_kind }
buffer.write_byte(flags.to_byte())
buffer.write_byte(b'\x00')
buffer.write_byte(b'\x00')
buffer.write_uint_le(size.reinterpret_as_uint())
buffer.to_bytes()
}
///|
fn empty_writable_heap() -> WritableGlobalHeap {
{
address: 0UL,
region_address: 0UL,
strings: [],
regions: [],
sequences: [],
external_references: [],
}
}
///|
fn writable_heap_index(
heap : WritableGlobalHeap,
value : String,
) -> Int32 raise Hdf5Error {
match heap.strings.search_by(item => item == value) {
Some(index) => index + 1
None => {
guard heap.address == 0UL else {
raise InvalidField("variable string is missing from the global heap")
}
0
}
}
}
///|
fn writable_sequence_heap_index(
heap : WritableGlobalHeap,
value : Bytes,
) -> Int32 raise Hdf5Error {
match heap.sequences.search_by(item => item == value) {
Some(index) => heap.strings.length() + index + 1
None => {
guard heap.address == 0UL else {
raise InvalidField("variable sequence is missing from the global heap")
}
0
}
}
}
///|
fn writable_region_heap_index(
heap : WritableGlobalHeap,
value : Bytes,
) -> Int32 raise Hdf5Error {
match heap.regions.search_by(item => item == value) {
Some(index) => index + 1
None => {
guard heap.address == 0UL else {
raise InvalidField("region reference is missing from the global heap")
}
0
}
}
}
///|
fn writable_external_reference_slot(
heap : WritableGlobalHeap,
value : DatasetValue,
) -> (Bytes, Int32)? raise Hdf5Error {
match heap.external_references.search_by(item => item.value == value) {
Some(index) => {
let payload = heap.external_references[index].payload
Some((payload, writable_region_heap_index(heap, payload)))
}
None =>
if heap.region_address == 0UL && heap.external_references.length() == 0 {
None
} else {
raise InvalidField("external reference is missing from the global heap")
}
}
}
///|
fn encode_complex_datatype(size : Int32) -> Bytes raise Hdf5Error {
let component = Datatype::write_float(size)
encode_writer_datatype(
Datatype::write_compound([("r", component), ("i", component)]),
)
}
///|
fn encode_write_values(
values : Hdf5WriteValues,
heap~ : WritableGlobalHeap = empty_writable_heap(),
objects~ : Array[WritableObject] = [],
reference_address_size~ : Int32 = 8,
vlen_address_size~ : Int32 = 0,
) -> EncodedWriteValues raise Hdf5Error {
let vlen_width = if vlen_address_size > 0 {
vlen_address_size
} else {
reference_address_size
}
let payload = @buffer.Buffer::Buffer()
let (datatype, element_size) = match values {
Int8Values(items) => {
for item in items {
payload.write_byte(item.to_byte())
}
(encode_integer_datatype(true, 1), 1)
}
Int16Values(items) => {
for item in items {
payload.write_int16_le(Int16::from_int(item))
}
(encode_integer_datatype(true, 2), 2)
}
Int32Values(items) => {
for item in items {
payload.write_int_le(item)
}
(encode_integer_datatype(true, 4), 4)
}
Int64Values(items) => {
for item in items {
payload.write_int64_le(item)
}
(encode_integer_datatype(true, 8), 8)
}
UInt8Values(items) => {
for item in items {
payload.write_byte(item.to_byte())
}
(encode_integer_datatype(false, 1), 1)
}
UInt16Values(items) => {
for item in items {
payload.write_uint16_le(item.to_uint16())
}
(encode_integer_datatype(false, 2), 2)
}
UInt32Values(items) => {
for item in items {
payload.write_uint_le(item)
}
(encode_integer_datatype(false, 4), 4)
}
UInt64Values(items) => {
for item in items {
payload.write_uint64_le(item)
}
(encode_integer_datatype(false, 8), 8)
}
BigIntegerValues(items, size, signed) => {
for item in items {
write_big_integer_width(payload, item, size, signed)
}
(encode_integer_datatype(signed, size), size)
}
Float16Values(items) => {
for item in items {
payload.write_uint16_le(writer_float16_bits(item).to_uint16())
}
(encode_float_datatype(2), 2)
}
Float32Values(items) => {
for item in items {
payload.write_float_le(item)
}
(encode_float_datatype(4), 4)
}
Float64Values(items) => {
for item in items {
payload.write_double_le(item)
}
(encode_float_datatype(8), 8)
}
FixedStringValues(items, width) => {
for item in items {
let encoded = @utf8.encode(item)
writer_write_bytes(payload, encoded)
writer_write_zeros(payload, width - encoded.length())
}
(encode_string_datatype(width), width)
}
VariableStringValues(items) => {
let vlen_size = 4 + vlen_width + 4
for item in items {
let encoded = @utf8.encode(item)
payload.write_uint_le(encoded.length().reinterpret_as_uint())
if encoded.length() == 0 {
write_uint_width(payload, 0UL, vlen_width)
payload.write_uint_le(0U)
} else {
write_uint_width(payload, heap.address, vlen_width)
payload.write_uint_le(
writable_heap_index(heap, item).reinterpret_as_uint(),
)
}
}
(encode_variable_string_datatype(vlen_size), vlen_size)
}
ObjectReferenceValues(items) => {
for item in items {
let address = match item {
None => 0UL
Some(path) => writer_reference_address(objects, path)
}
write_uint_width(payload, address, reference_address_size)
}
(
encode_reference_datatype(reference_address_size, 0, 3),
reference_address_size,
)
}
Complex64Values(items) => {
for (real, imaginary) in items {
payload.write_float_le(real)
payload.write_float_le(imaginary)
}
(encode_complex_datatype(4), 8)
}
Complex128Values(items) => {
for (real, imaginary) in items {
payload.write_double_le(real)
payload.write_double_le(imaginary)
}
(encode_complex_datatype(8), 16)
}
StructuredValues(datatype, items) => {
for item in items {
writer_write_bytes(
payload,
encode_structured_value(
datatype,
item,
heap~,
objects~,
reference_address_size~,
vlen_address_size~,
),
)
}
(encode_writer_datatype(datatype, reference_address_size~), datatype.size)
}
}
{ datatype, payload: payload.to_bytes(), element_size, }
}
///|
fn encode_dataspace(
dimensions : Array[UInt64],
maximum_dimensions? : Array[UInt64],
) -> Bytes {
let buffer = @buffer.Buffer::Buffer(
size_hint=4 +
dimensions.length() * (if maximum_dimensions is Some(_) { 16 } else { 8 }),
)
buffer.write_byte(b'\x02')
buffer.write_byte(dimensions.length().to_byte())
buffer.write_byte(
if maximum_dimensions is Some(_) {
b'\x01'
} else {
b'\x00'
},
)
buffer.write_byte(if dimensions.length() == 0 { b'\x00' } else { b'\x01' })
for dimension in dimensions {
buffer.write_uint64_le(dimension)
}
if maximum_dimensions is Some(maximum) {
for dimension in maximum {
buffer.write_uint64_le(dimension)
}
}
buffer.to_bytes()
}
///|
fn encode_contiguous_layout(address : UInt64, size : Int32) -> Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=18)
buffer.write_byte(b'\x03')
buffer.write_byte(b'\x01')
buffer.write_uint64_le(address)
buffer.write_uint64_le(size.to_uint64())
buffer.to_bytes()
}
///|
fn encode_compact_layout(payload : Bytes) -> Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=4 + payload.length())
buffer.write_byte(b'\x03')
buffer.write_byte(b'\x00')
buffer.write_uint16_le(payload.length().to_uint16())
writer_write_bytes(buffer, payload)
buffer.to_bytes()
}
///|
fn encode_external_data_layout(
size : UInt64,
offset_size~ : Int32 = 8,
length_size~ : Int32 = 8,
) -> Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=2 + offset_size + length_size)
buffer.write_byte(b'\x03')
buffer.write_byte(b'\x01')
write_uint_width(buffer, 0xFFFFFFFFFFFFFFFFUL, offset_size)
write_uint_width(buffer, size, length_size)
buffer.to_bytes()
}
///|
fn encode_virtual_layout(
heap_address : UInt64,
heap_index : Int32,
offset_size~ : Int32 = 8,
) -> Bytes raise Hdf5Error {
guard heap_index > 0 else {
raise InvalidArgument("virtual mapping heap index must be positive")
}
let buffer = @buffer.Buffer::Buffer(size_hint=2 + offset_size + 4)
buffer.write_byte(b'\x04')
buffer.write_byte(b'\x03')
write_uint_width(buffer, heap_address, offset_size)
buffer.write_uint_le(heap_index.reinterpret_as_uint())
buffer.to_bytes()
}
///|
fn external_slot_total(slots : Array[Hdf5ExternalSlot]) -> UInt64 {
let mut total = 0UL
for slot in slots {
total = total + slot.data_size
}
total
}
///|
fn external_name_offsets(
slots : Array[Hdf5ExternalSlot],
) -> (Array[UInt64], Int32) raise Hdf5Error {
let names : Array[String] = []
let offsets : Array[UInt64] = []
let mut data_size = 8
for slot in slots {
match names.search_by(name => name == slot.file_name) {
Some(index) => offsets.push(offsets[index])
None => {
let encoded = @utf8.encode(slot.file_name)
guard encoded.length() < 0x7FFFFFFF - data_size else {
raise InvalidArgument("external data file names are too long")
}
names.push(slot.file_name)
offsets.push(data_size.to_uint64())
data_size = data_size + encoded.length() + 1
}
}
}
(offsets, writer_align(data_size))
}
///|
fn encode_external_local_heap(
slots : Array[Hdf5ExternalSlot],
address : UInt64,
offset_size~ : Int32 = 8,
length_size~ : Int32 = 8,
) -> Bytes raise Hdf5Error {
let (_, data_size) = external_name_offsets(slots)
let data = FixedArray::make(data_size, b'\x00')
let names : Array[String] = []
let mut cursor = 8
for slot in slots {
if names.search_by(name => name == slot.file_name) is None {
let encoded = @utf8.encode(slot.file_name)
data.blit_from_bytes(cursor, encoded, 0, encoded.length())
cursor = cursor + encoded.length() + 1
names.push(slot.file_name)
}
}
let header_length = 8 + 2 * length_size + offset_size
let data_start = writer_align(header_length)
let buffer = @buffer.Buffer::Buffer(size_hint=data_start + data_size)
writer_write_bytes(buffer, b"HEAP")
buffer.write_byte(b'\x00')
writer_write_zeros(buffer, 3)
write_uint_width(buffer, data_size.to_uint64(), length_size)
write_uint_width(buffer, 1UL, length_size)
write_uint_width(buffer, address + data_start.to_uint64(), offset_size)
writer_write_zeros(buffer, data_start - buffer.length())
writer_write_bytes(buffer, data.unsafe_reinterpret_as_bytes())
buffer.to_bytes()
}
///|
fn encode_external_data_message(
slots : Array[Hdf5ExternalSlot],
heap_address : UInt64,
offset_size~ : Int32 = 8,
length_size~ : Int32 = 8,
) -> Bytes raise Hdf5Error {
let (offsets, _) = external_name_offsets(slots)
guard slots.length() <= 65535 else {
raise InvalidArgument("external dataset requires at most 65535 slots")
}
let buffer = @buffer.Buffer::Buffer()
buffer.write_byte(b'\x01')
writer_write_zeros(buffer, 3)
buffer.write_uint16_le(slots.length().to_uint16())
buffer.write_uint16_le(slots.length().to_uint16())
write_uint_width(buffer, heap_address, offset_size)
for index in 0.. Bytes {
let mut largest = element_size.to_uint64()
for dimension in chunk_dimensions {
if dimension.to_uint64() > largest {
largest = dimension.to_uint64()
}
}
let mut width = 1
let mut remaining = largest >> 8
while remaining > 0UL {
width = width + 1
remaining = remaining >> 8
}
let buffer = @buffer.Buffer::Buffer(
size_hint=8 + (chunk_dimensions.length() + 1) * width + offset_size,
)
buffer.write_byte(b'\x04')
buffer.write_byte(b'\x02')
buffer.write_byte(b'\x00')
buffer.write_byte((chunk_dimensions.length() + 1).to_byte())
buffer.write_byte(width.to_byte())
for dimension in chunk_dimensions {
for shift in 0..> (shift * 8)) & 0xFFUL).to_byte(),
)
}
}
for shift in 0..> (shift * 8)) & 0xFFUL).to_byte(),
)
}
if btree_v2 {
buffer.write_byte(b'\x05')
buffer.write_uint_le(writer_btree_v2_node_size.reinterpret_as_uint())
buffer.write_byte(writer_btree_v2_split_percent.to_byte())
buffer.write_byte(writer_btree_v2_merge_percent.to_byte())
} else if extensible {
buffer.write_byte(b'\x04')
buffer.write_byte(b'\x20')
buffer.write_byte(b'\x04')
buffer.write_byte(b'\x04')
buffer.write_byte(b'\x10')
buffer.write_byte(b'\x0a')
} else {
buffer.write_byte(b'\x03')
buffer.write_byte(b'\x0a')
}
write_uint_width(buffer, address, offset_size)
buffer.to_bytes()
}
///|
fn encode_fill_value(
fill : Hdf5WriteValues,
heap~ : WritableGlobalHeap,
objects~ : Array[WritableObject],
reference_address_size~ : Int32 = 8,
vlen_address_size~ : Int32 = 0,
) -> Bytes raise Hdf5Error {
let encoded = encode_write_values(
fill,
heap~,
objects~,
reference_address_size~,
vlen_address_size~,
)
let stored_size = if writer_native_value_size(fill) > encoded.element_size {
writer_native_value_size(fill)
} else {
encoded.element_size
}
let buffer = @buffer.Buffer::Buffer(size_hint=6 + stored_size)
buffer.write_byte(b'\x03')
buffer.write_byte(b'\x2b')
buffer.write_uint_le(stored_size.reinterpret_as_uint())
writer_write_bytes(buffer, encoded.payload)
writer_write_zeros(buffer, stored_size - encoded.payload.length())
buffer.to_bytes()
}
///|
fn writer_scaleoffset_layout(
values : Hdf5WriteValues,
) -> (Int32, Bool, Int32) raise Hdf5Error {
match values {
Int8Values(_) => (1, true, 0)
Int16Values(_) => (2, true, 0)
Int32Values(_) => (4, true, 0)
Int64Values(_) => (8, true, 0)
UInt8Values(_) => (1, false, 0)
UInt16Values(_) => (2, false, 0)
UInt32Values(_) => (4, false, 0)
UInt64Values(_) => (8, false, 0)
BigIntegerValues(_, _, _) =>
raise UnsupportedFeature(
"scale-offset does not support integers wider than 8 bytes",
)
Float32Values(_) => (4, false, 1)
Float64Values(_) => (8, false, 1)
_ =>
raise UnsupportedFeature(
"scale-offset writer filter requires fixed-size numeric values",
)
}
}
///|
fn scaleoffset_read_raw(data : Bytes, offset : Int32, size : Int32) -> UInt64 {
let mut value = 0UL
for index in 0.. UInt64 {
if size == 8 {
0xFFFFFFFFFFFFFFFFUL
} else {
(1UL << (size * 8)) - 1UL
}
}
///|
fn scaleoffset_ceil_log2(value : UInt64) -> Int32 {
let mut remaining = value - 1UL
let mut bits = 0
while remaining > 0UL {
bits = bits + 1
remaining = remaining >> 1
}
bits
}
///|
fn scaleoffset_min_max(
data : Bytes,
element_size : Int32,
signed : Bool,
fill_raw : UInt64,
) -> (UInt64, UInt64) {
let bits = element_size * 8
let count = data.length() / element_size
let mut found = false
let mut min_raw = 0UL
let mut max_raw = 0UL
let mut min_value = 0L
let mut max_value = 0L
for index in 0.. max_value {
max_value = value
max_raw = raw
}
}
} else if !found {
min_raw = raw
max_raw = raw
found = true
} else {
if raw < min_raw {
min_raw = raw
}
if raw > max_raw {
max_raw = raw
}
}
}
(min_raw, max_raw)
}
///|
fn scaleoffset_client_data(
scale : Int32,
datatype_class : Int32,
element_size : Int32,
signed : Bool,
chunk_elements : Int32,
fill : Bytes,
) -> Array[UInt32] {
let parameters : Array[UInt32] = Array::make(20, 0U)
parameters[0] = if datatype_class == 0 { 2U } else { 0U }
parameters[1] = scale.reinterpret_as_uint()
parameters[2] = chunk_elements.reinterpret_as_uint()
parameters[3] = datatype_class.reinterpret_as_uint()
parameters[4] = element_size.reinterpret_as_uint()
parameters[5] = if signed { 1U } else { 0U }
parameters[7] = 1U
for index in 0.. Int64 {
let truncated = value.to_int64()
let fraction = value - truncated.to_double()
if fraction >= 0.5 {
truncated + 1L
} else if fraction <= -0.5 {
truncated - 1L
} else {
truncated
}
}
///|
fn scaleoffset_finite32(value : Float) -> Bool {
!value.is_nan() && !value.is_inf()
}
///|
fn scaleoffset_finite64(value : Double) -> Bool {
!value.is_nan() && !value.is_inf()
}
///|
fn scaleoffset_write_packed_chunk(
minbits : Int32,
min_raw : UInt64,
packed : FixedArray[UInt64],
) -> Bytes raise Hdf5Error {
let count = packed.length()
guard minbits >= 0 && minbits < 64 else {
raise InvalidArgument("invalid scale-offset minimum bits")
}
let output_size = 21L + count.to_int64() * minbits.to_int64() / 8L + 1L
guard output_size <= 0x7FFFFFFFL else {
raise InvalidField("scale-offset writer chunk is too large")
}
let output = FixedArray::make(output_size.to_int(), b'\x00')
let encoded_bits = minbits.reinterpret_as_uint()
for index in 0..<4 {
output[index] = ((encoded_bits >> (index * 8)) & 0xFFU).to_byte()
}
output[4] = 8
for index in 0..<8 {
output[5 + index] = ((min_raw >> (index * 8)) & 0xFFUL).to_byte()
}
let mut bit_index = 0
for index in 0.. 0 {
let byte_index = bit_index / 8
let bit_offset = bit_index % 8
let available = 8 - bit_offset
let take = if remaining < available { remaining } else { available }
let part = (value >> (remaining - take)) & ((1UL << take) - 1UL)
output[21 + byte_index] = (output[21 + byte_index].to_uint() |
(part.to_uint() << (available - take))).to_byte()
bit_index = bit_index + take
remaining = remaining - take
}
}
output.unsafe_reinterpret_as_bytes()
}
///|
fn encode_scaleoffset_float32_chunk(
data : Bytes,
digits : Int32,
fill : Bytes,
) -> Bytes raise Hdf5Error {
guard data.length() % 4 == 0 && fill.length() == 4 else {
raise InvalidField("invalid float32 scale-offset writer chunk")
}
guard digits >= 0 && digits <= 38 else {
raise InvalidArgument("invalid float32 scale-offset decimal digits")
}
let count = data.length() / 4
let fill_value = Float::reinterpret_from_uint(fill.unsafe_read_uint32_le(0))
guard scaleoffset_finite32(fill_value) else {
raise UnsupportedFeature("scale-offset fill value must be finite")
}
let factor = scaleoffset_float_factor32(digits)
let tolerance = @math.powf(Float(10.0), -Float::from_int(digits))
guard scaleoffset_finite32(factor) && scaleoffset_finite32(tolerance) else {
raise UnsupportedFeature("float32 scale-offset factor is out of range")
}
let mut found = false
let mut minimum = Float(0.0)
let mut maximum = Float(0.0)
for index in 0.. maximum {
maximum = value
}
}
}
let difference = maximum * factor - minimum * factor
guard scaleoffset_finite32(difference) &&
difference >= Float(0.0) &&
difference.to_double() < 9223372036854775808.0 else {
raise UnsupportedFeature("float32 scale-offset range is out of range")
}
let rounded_difference = scaleoffset_round_away(difference.to_double())
guard rounded_difference >= 0L &&
rounded_difference.to_double() < 9223372036854775808.0 else {
raise UnsupportedFeature("float32 scale-offset range is out of range")
}
let span = rounded_difference.reinterpret_as_uint64() + 1UL
let minbits = scaleoffset_ceil_log2(span + 1UL)
guard minbits < 32 else {
raise UnsupportedFeature(
"scale-offset automatic bits require full precision",
)
}
let marker = (1UL << minbits) - 1UL
let minimum_scaled = minimum * factor
guard scaleoffset_finite32(minimum_scaled) else {
raise UnsupportedFeature("float32 scale-offset range is out of range")
}
let packed = FixedArray::make(count, 0UL)
for index in 0..= Float(0.0) &&
scaled.to_double() < 9223372036854775808.0 else {
raise UnsupportedFeature("float32 scale-offset value is out of range")
}
let quantized = scaleoffset_round_away(scaled.to_double())
guard quantized >= 0L && quantized.reinterpret_as_uint64() <= marker else {
raise InvalidField("float32 scale-offset value is out of range")
}
packed[index] = quantized.reinterpret_as_uint64()
}
}
scaleoffset_write_packed_chunk(
minbits,
minimum.reinterpret_as_uint().to_uint64(),
packed,
)
}
///|
fn encode_scaleoffset_float64_chunk(
data : Bytes,
digits : Int32,
fill : Bytes,
) -> Bytes raise Hdf5Error {
guard data.length() % 8 == 0 && fill.length() == 8 else {
raise InvalidField("invalid float64 scale-offset writer chunk")
}
guard digits >= 0 && digits <= 308 else {
raise InvalidArgument("invalid float64 scale-offset decimal digits")
}
let count = data.length() / 8
let fill_value = fill.unsafe_read_uint64_le(0).reinterpret_as_double()
guard scaleoffset_finite64(fill_value) else {
raise UnsupportedFeature("scale-offset fill value must be finite")
}
let factor = scaleoffset_float_factor64(digits)
let tolerance = @math.pow(10.0, -Double::from_int(digits))
guard scaleoffset_finite64(factor) && scaleoffset_finite64(tolerance) else {
raise UnsupportedFeature("float64 scale-offset factor is out of range")
}
let mut found = false
let mut minimum = 0.0
let mut maximum = 0.0
for index in 0.. maximum {
maximum = value
}
}
}
let difference = maximum * factor - minimum * factor
guard scaleoffset_finite64(difference) &&
difference >= 0.0 &&
difference < 9223372036854775808.0 else {
raise UnsupportedFeature("float64 scale-offset range is out of range")
}
let rounded_difference = scaleoffset_round_away(difference)
guard rounded_difference >= 0L &&
rounded_difference.to_double() < 9223372036854775808.0 else {
raise UnsupportedFeature("float64 scale-offset range is out of range")
}
let span = rounded_difference.reinterpret_as_uint64() + 1UL
let minbits = scaleoffset_ceil_log2(span + 1UL)
guard minbits < 64 else {
raise UnsupportedFeature(
"scale-offset automatic bits require full precision",
)
}
let marker = (1UL << minbits) - 1UL
let minimum_scaled = minimum * factor
guard scaleoffset_finite64(minimum_scaled) else {
raise UnsupportedFeature("float64 scale-offset range is out of range")
}
let packed = FixedArray::make(count, 0UL)
for index in 0..= 0.0 &&
scaled < 9223372036854775808.0 else {
raise UnsupportedFeature("float64 scale-offset value is out of range")
}
let quantized = scaleoffset_round_away(scaled)
guard quantized >= 0L && quantized.reinterpret_as_uint64() <= marker else {
raise InvalidField("float64 scale-offset value is out of range")
}
packed[index] = quantized.reinterpret_as_uint64()
}
}
scaleoffset_write_packed_chunk(
minbits,
minimum.reinterpret_as_uint64(),
packed,
)
}
///|
fn encode_scaleoffset_chunk(
data : Bytes,
scale : Int32,
element_size : Int32,
datatype_class : Int32,
signed : Bool,
fill : Bytes,
) -> Bytes raise Hdf5Error {
guard element_size >= 1 &&
element_size <= 8 &&
data.length() % element_size == 0 &&
fill.length() == element_size else {
raise InvalidField("invalid scale-offset writer chunk")
}
if datatype_class == 1 {
if element_size == 4 {
return encode_scaleoffset_float32_chunk(data, scale, fill)
}
if element_size == 8 {
return encode_scaleoffset_float64_chunk(data, scale, fill)
}
raise UnsupportedFeature(
"scale-offset floating-point width is not supported",
)
}
let bits = element_size * 8
if scale == bits {
return data
}
let count = data.length() / element_size
let fill_raw = scaleoffset_read_raw(fill, 0, element_size)
let (min_raw, max_raw) = scaleoffset_min_max(
data, element_size, signed, fill_raw,
)
let value_mask = scaleoffset_bit_mask(element_size)
let difference = (max_raw - min_raw) & value_mask
let minbits = if scale == 0 {
let span = difference + 1UL
let computed = scaleoffset_ceil_log2(span + 1UL)
guard computed < bits else {
raise UnsupportedFeature(
"scale-offset automatic bits require full precision",
)
}
computed
} else {
scale
}
guard minbits >= 0 && minbits < bits else {
raise InvalidArgument("invalid scale-offset minimum bits")
}
let output_size = 21L + count.to_int64() * minbits.to_int64() / 8L + 1L
guard output_size <= 0x7FFFFFFFL else {
raise InvalidField("scale-offset writer chunk is too large")
}
let output = FixedArray::make(output_size.to_int(), b'\x00')
let encoded_bits = minbits.reinterpret_as_uint()
for index in 0..<4 {
output[index] = ((encoded_bits >> (index * 8)) & 0xFFU).to_byte()
}
output[4] = 8
for index in 0..<8 {
output[5 + index] = ((min_raw >> (index * 8)) & 0xFFUL).to_byte()
}
let packed_mask = if minbits == 0 { 0UL } else { (1UL << minbits) - 1UL }
let mut bit_index = 0
for index in 0.. 0 {
let byte_index = bit_index / 8
let bit_offset = bit_index % 8
let available = 8 - bit_offset
let take = if remaining < available { remaining } else { available }
let part = (packed >> (remaining - take)) & ((1UL << take) - 1UL)
output[21 + byte_index] = (output[21 + byte_index].to_uint() |
(part.to_uint() << (available - take))).to_byte()
bit_index = bit_index + take
remaining = remaining - take
}
}
output.unsafe_reinterpret_as_bytes()
}
///|
fn writer_chunk_element_count(
chunk_dimensions : Array[UInt32],
) -> Int32 raise Hdf5Error {
let mut count = 1L
for dimension in chunk_dimensions {
guard dimension > 0U && count <= 0x7FFFFFFFL / dimension.to_int64() else {
raise InvalidField("writer chunk is too large")
}
count = count * dimension.to_int64()
}
count.to_int()
}
///|
fn encode_filter_pipeline(
filters : Array[Hdf5WriteFilter],
element_size : Int32,
chunk_elements : Int32,
values : Hdf5WriteValues,
fill : Bytes,
custom_filters : Array[&Hdf5CustomFilter],
) -> Bytes raise Hdf5Error {
let buffer = @buffer.Buffer::Buffer()
buffer.write_byte(b'\x02')
buffer.write_byte(filters.length().to_byte())
for filter in filters {
match filter {
Shuffle => {
buffer.write_uint16_le(2)
buffer.write_uint16_le(0)
buffer.write_uint16_le(1)
buffer.write_uint_le(element_size.reinterpret_as_uint())
}
Deflate(level) => {
buffer.write_uint16_le(1)
buffer.write_uint16_le(0)
buffer.write_uint16_le(1)
buffer.write_uint_le(level.reinterpret_as_uint())
}
Fletcher32 => {
buffer.write_uint16_le(3)
buffer.write_uint16_le(0)
buffer.write_uint16_le(0)
}
Nbit => {
let datatype = writer_values_datatype(values)
guard datatype.size == element_size else {
raise InvalidField("N-bit datatype size mismatch")
}
let parameters = nbit_client_data(datatype, chunk_elements)
buffer.write_uint16_le(5)
buffer.write_uint16_le(0)
buffer.write_uint16_le(parameters.length().to_uint16())
for parameter in parameters {
buffer.write_uint_le(parameter)
}
}
ScaleOffset(minbits) => {
let (size, signed, datatype_class) = writer_scaleoffset_layout(values)
guard size == element_size else {
raise InvalidField("scale-offset datatype size mismatch")
}
let parameters = scaleoffset_client_data(
minbits, datatype_class, size, signed, chunk_elements, fill,
)
buffer.write_uint16_le(6)
buffer.write_uint16_le(1)
buffer.write_uint16_le(20)
for parameter in parameters {
buffer.write_uint_le(parameter)
}
}
Custom(id, parameters) => {
let custom = find_custom_filter(custom_filters, id)
buffer.write_uint16_le(id.to_uint16())
let name = if id >= 256 { @utf8.encode(custom.name()) } else { b"" }
if id >= 256 {
guard name.length() <= 65535 else {
raise InvalidField("custom filter name is too long")
}
buffer.write_uint16_le(name.length().to_uint16())
}
buffer.write_uint16_le(0)
guard parameters.length() <= 65535 else {
raise InvalidField("custom filter has too many parameters")
}
buffer.write_uint16_le(parameters.length().to_uint16())
if name.length() > 0 {
writer_write_bytes(buffer, name)
}
for parameter in parameters {
buffer.write_uint_le(parameter)
}
}
}
}
buffer.to_bytes()
}
///|
fn shuffle_chunk(data : Bytes, element_size : Int32) -> Bytes {
let elements = data.length() / element_size
let output = FixedArray::make(data.length(), b'\x00')
for byte_index in 0.. Bytes {
let mut first = 0U
let mut second = 0U
let mut offset = 0
while offset < data.length() {
let high = data[offset].to_uint() << 8
let low = if offset + 1 < data.length() {
data[offset + 1].to_uint()
} else {
0U
}
first = first + high + low
first = (first & 0xFFFFU) + (first >> 16)
second = second + first
second = (second & 0xFFFFU) + (second >> 16)
offset = offset + 2
}
first = (first & 0xFFFFU) + (first >> 16)
second = (second & 0xFFFFU) + (second >> 16)
let checksum = (second << 16) | first
let legacy = ((checksum & 0x00FF00FFU) << 8) | ((checksum & 0xFF00FF00U) >> 8)
let buffer = @buffer.Buffer::Buffer(size_hint=data.length() + 4)
writer_write_bytes(buffer, data)
buffer.write_uint_le(legacy)
buffer.to_bytes()
}
///|
fn encode_chunk_filters(
data : Bytes,
filters : Array[Hdf5WriteFilter],
filter_mask : UInt32,
element_size : Int32,
datatype : Datatype,
values : Hdf5WriteValues,
fill : Bytes,
custom_filters : Array[&Hdf5CustomFilter],
) -> Bytes raise Hdf5Error {
let expected_size = data.length()
let mut output = data
for index in 0.. shuffle_chunk(output, element_size)
Deflate(level) => @zlib.zlib(output[:], level~)
Fletcher32 => add_fletcher32(output)
Nbit => {
guard datatype.size == element_size &&
output.length() % element_size == 0 else {
raise InvalidField("N-bit datatype size mismatch")
}
let parameters = nbit_client_data(
datatype,
output.length() / element_size,
)
encode_nbit(output, parameters, output.length())
}
ScaleOffset(minbits) => {
let (size, signed, datatype_class) = writer_scaleoffset_layout(values)
guard size == element_size else {
raise InvalidField("scale-offset datatype size mismatch")
}
encode_scaleoffset_chunk(
output, minbits, size, datatype_class, signed, fill,
)
}
Custom(id, parameters) => {
let encoded = find_custom_filter(custom_filters, id).encode(
output, parameters, element_size, expected_size,
)
guard encoded.length() > 0 else {
raise InvalidField("custom filter produced an empty chunk")
}
encoded
}
}
}
output
}
///|
fn write_repeated_element(output : FixedArray[Byte], element : Bytes) -> Unit {
let count = output.length() / element.length()
for index in 0.. Bool {
for byte_index in 0.. (Array[Int32], Int32) raise Hdf5Error {
let grid : Array[Int32] = []
let mut total = 1L
for axis in 0.. Array[EncodedChunkEntry] raise Hdf5Error {
let (chunk_dimensions, filters, order, grid_dimensions) = writer_dataset_chunk_grid(
dataset,
)
let encoded = encode_write_values(
dataset.values,
heap~,
objects~,
reference_address_size~,
vlen_address_size~,
)
let fill = match dataset.fill_value {
Some(value) =>
encode_write_values(
value,
heap~,
objects~,
reference_address_size~,
vlen_address_size~,
).payload
None => Bytes::make(encoded.element_size, b'\x00')
}
let (grid, chunk_count) = chunk_grid(grid_dimensions, chunk_dimensions)
let mut chunk_elements = 1L
for dimension in chunk_dimensions {
guard chunk_elements <= 0x7FFFFFFFL / dimension.to_int64() else {
raise InvalidField("writer chunk is too large")
}
chunk_elements = chunk_elements * dimension.to_int64()
}
guard chunk_elements <= 0x7FFFFFFFL / encoded.element_size.to_int64() else {
raise InvalidField("writer chunk is too large")
}
let chunk_size = (chunk_elements * encoded.element_size.to_int64()).to_int()
let chunk_buffers : Array[FixedArray[Byte]] = []
let allocated = Array::make(chunk_count, false)
let always_allocate = writer_values_contain_vlen(dataset.values)
for _ in 0.. 0 {
axis = axis - 1
let dimension = dataset.dimensions[axis].reinterpret_as_int64()
coordinates[axis] = (remaining % dimension).to_int()
remaining = remaining / dimension
}
let mut chunk_index = 0
let mut within_index = 0
for order_index in 0.. Bytes raise Hdf5Error {
let (_, filters) = match dataset.storage {
WritableChunked(dimensions, filters) => (dimensions, filters)
WritableExtensible(dimensions, filters, _) => (dimensions, filters)
WritableCompact
| WritableContiguous
| WritableExternal(_)
| WritableVirtual(_) =>
raise InvalidField("expected chunked writer dataset")
}
let entries = encode_dataset_chunks(
dataset,
heap,
objects,
custom_filters,
reference_address_size~,
vlen_address_size~,
)
let filtered = filters.length() > 0
let entry_size = if filtered { offset_size + 6 } else { offset_size }
let page_bits = 10
let page_entries = 1 << page_bits
let paged = entries.length() > page_entries
let page_count = if paged {
(entries.length() + page_entries - 1) / page_entries
} else {
1
}
let bitmap_size = if paged { (page_count + 7) / 8 } else { 0 }
let header_length = 8 + length_size + offset_size + 4
let block_address = writer_align(start.to_int() + header_length)
let stored_start = file_offset_to_stored_address(start, base_address)
let stored_block_address = file_offset_to_stored_address(
block_address.to_uint64(),
base_address,
)
let block_length = 6 +
offset_size +
bitmap_size +
(if paged { 4 + page_count * 4 } else { 4 }) +
entries.length() * entry_size
let mut chunk_address = writer_align(block_address + block_length)
for index in 0..= 8 {
b'\xff'
} else {
(0xFF << (8 - remaining)).to_byte()
},
)
}
block.write_uint_le(metadata_checksum(block.to_bytes(), 0, block.length()))
}
let mut entry_index = 0
for _ in 0.. Bool {
if start >= entries.length().to_int64() {
return false
}
let end = if start + count < entries.length().to_int64() {
start + count
} else {
entries.length().to_int64()
}
for index in start.to_int().. Unit raise Hdf5Error {
let allocated = index < entries.length().to_int64() &&
entries[index.to_int()].allocated
write_uint_width(
buffer,
if allocated {
file_offset_to_stored_address(
entries[index.to_int()].address,
base_address,
)
} else {
0xFFFFFFFFFFFFFFFFUL
},
offset_size,
)
if filtered {
let size = if allocated { entries[index.to_int()].data.length() } else { 0 }
guard size <= 65535 else {
raise InvalidField("filtered writer chunk is too large")
}
buffer.write_uint16_le(size.to_uint16())
buffer.write_uint_le(
if allocated {
entries[index.to_int()].filter_mask
} else {
0U
},
)
}
}
///|
fn writer_extensible_data_block_length(
count : Int64,
entry_size : Int32,
page_bits : Int32,
offset_size : Int32,
block_offset_size : Int32,
) -> Int64 {
let page_size = 1L << page_bits
if count <= page_size {
6L +
offset_size.to_int64() +
block_offset_size.to_int64() +
count * entry_size.to_int64() +
4L
} else {
let pages = (count + page_size - 1L) / page_size
6L +
offset_size.to_int64() +
block_offset_size.to_int64() +
4L +
count * entry_size.to_int64() +
pages * 4L
}
}
///|
fn encode_writer_extensible_data_block(
header_address : UInt64,
entries : Array[EncodedChunkEntry],
entry_start : Int64,
count : Int64,
block_offset : UInt64,
filtered : Bool,
page_bits : Int32,
offset_size~ : Int32,
block_offset_size~ : Int32 = 4,
base_address~ : UInt64,
) -> Bytes raise Hdf5Error {
let page_size = 1L << page_bits
let paged = count > page_size
let block = @buffer.Buffer::Buffer()
writer_write_bytes(block, b"EADB")
block.write_byte(b'\x00')
block.write_byte(if filtered { b'\x01' } else { b'\x00' })
write_uint_width(
block,
file_offset_to_stored_address(header_address, base_address),
offset_size,
)
write_uint_width(block, block_offset, block_offset_size)
if paged {
block.write_uint_le(metadata_checksum(block.to_bytes(), 0, block.length()))
}
let pages = if paged { (count + page_size - 1L) / page_size } else { 1L }
for page in 0.. Bytes raise Hdf5Error {
let (_, filters, unlimited_axes) = match dataset.storage {
WritableExtensible(dimensions, filters, axes) => (dimensions, filters, axes)
_ => raise InvalidField("expected extensible writer dataset")
}
guard unlimited_axes.length() == 1 else {
raise InvalidField(
"extensible-array storage requires one unlimited dimension",
)
}
let entries = encode_dataset_chunks(
dataset,
heap,
objects,
custom_filters,
reference_address_size~,
vlen_address_size~,
)
let filtered = filters.length() > 0
let entry_size = if filtered { offset_size + 6 } else { offset_size }
let max_bits = 32
let index_elements = 4
let min_elements = 16
let min_pointers = 4
let page_bits = 10
let first_super = 2 * array_log2(min_pointers)
let levels = 1 + max_bits - array_log2(min_elements)
let direct_count = 2 * (min_pointers - 1)
let header_length = 12 + length_size * 6 + offset_size + 4
let index_address = start + header_length.to_uint64()
let index_length = 6 +
offset_size +
index_elements * entry_size +
(direct_count + levels - first_super) * offset_size +
4
let direct_addresses = Array::make(direct_count, 0xFFFFFFFFFFFFFFFFUL)
let block_addresses : Array[UInt64] = []
let block_starts : Array[Int64] = []
let block_counts : Array[Int64] = []
let block_offsets : Array[UInt64] = []
let super_addresses = Array::make(levels - first_super, 0xFFFFFFFFFFFFFFFFUL)
let super_starts : Array[Int64] = []
let super_counts : Array[Int64] = []
let super_block_addresses : Array[Array[UInt64]] = []
let super_bitmaps : Array[Bytes] = []
let mut secondary_block_count = 0L
let mut secondary_block_size = 0L
let mut data_block_count = 0L
let mut data_block_size = 0L
let mut realized_elements = index_elements.to_int64()
let mut metadata_end = writer_align(index_address.to_int() + index_length)
let mut level_start = 0L
let mut direct_start = 0
for level in 0..= entries.length().to_int64() {
break
}
if writer_extensible_range_allocated(entries, entry_start, count) {
let block_address = metadata_end.to_uint64()
direct_addresses[direct_start + block_index] = block_address
block_addresses.push(block_address)
block_starts.push(entry_start)
block_counts.push(count)
block_offsets.push(
(level_start +
(direct_start.to_int64() + block_index.to_int64()) * count).reinterpret_as_uint64(),
)
let block_length = writer_extensible_data_block_length(
count, entry_size, page_bits, offset_size, 4,
)
data_block_count = data_block_count + 1L
data_block_size = data_block_size + block_length
realized_elements = realized_elements + count
metadata_end = writer_align(metadata_end + block_length.to_int())
}
}
level_start = level_start + blocks * count
direct_start = direct_start + blocks.to_int()
}
for level in first_super..= entries.length().to_int64() {
break
}
let blocks = 1L << (level / 2)
let count = min_elements.to_int64() << ((level + 1) / 2)
if writer_extensible_range_allocated(
entries,
index_elements.to_int64() + level_start,
blocks * count,
) {
let page_size = 1L << page_bits
let pages = if count > page_size { count / page_size } else { 0L }
let bitmap_bytes = (pages + 7L) / 8L
let bitmap = Array::make((blocks * bitmap_bytes).to_int(), b'\x00')
let addresses = Array::make(blocks.to_int(), 0xFFFFFFFFFFFFFFFFUL)
let super_address = metadata_end.to_uint64()
super_addresses[level - first_super] = super_address
let super_length = 6L +
offset_size.to_int64() +
4L +
blocks * bitmap_bytes +
blocks * offset_size.to_int64() +
4L
secondary_block_count = secondary_block_count + 1L
secondary_block_size = secondary_block_size + super_length
metadata_end = writer_align(metadata_end + super_length.to_int())
for block_index in 0.. 0L {
for page in 0..> (bit % 8L).to_int())).to_byte()
}
}
}
let block_length = writer_extensible_data_block_length(
count, entry_size, page_bits, offset_size, 4,
)
data_block_count = data_block_count + 1L
data_block_size = data_block_size + block_length
realized_elements = realized_elements + count
metadata_end = writer_align(metadata_end + block_length.to_int())
}
}
super_starts.push(level_start)
super_counts.push(count)
super_block_addresses.push(addresses)
super_bitmaps.push(Bytes::from_array(bitmap))
}
level_start = level_start + blocks * count
}
guard entries.length().to_int64() <= index_elements.to_int64() + level_start else {
raise InvalidField(
"writer extensible-array index exceeds configured capacity",
)
}
let mut chunk_address = metadata_end
for index in 0.. Unit {
for shift in 0..> (shift * 8)) & 0xFFUL).to_byte())
}
}
///|
fn writer_btree_capacities(
node_size : Int32,
record_size : Int32,
offset_size : Int32,
) -> (Array[Int32], Array[Int32], Int32) raise Hdf5Error {
let leaf_capacity = (node_size - 10) / record_size
guard leaf_capacity > 0 && leaf_capacity <= 65535 else {
raise InvalidArgument("writer v2 B-tree record is too large")
}
let count_width = btree_integer_width(leaf_capacity.to_uint64())
let capacities : Array[Int32] = [leaf_capacity]
let total_widths : Array[Int32] = [0]
let mut cumulative = leaf_capacity.to_uint64()
for level in 1..<32 {
let pointer_size = offset_size + count_width + total_widths[level - 1]
let capacity = (node_size - 10 - pointer_size) /
(record_size + pointer_size)
if capacity <= 0 {
break
}
if cumulative >
(0xFFFFFFFFFFFFFFFFUL - capacity.to_uint64()) / (capacity + 1).to_uint64() {
break
}
cumulative = cumulative * (capacity + 1).to_uint64() + capacity.to_uint64()
capacities.push(capacity)
total_widths.push(btree_integer_width(cumulative))
}
(capacities, total_widths, count_width)
}
///|
fn writer_btree_max_records(
depth : Int32,
capacities : Array[Int32],
) -> UInt64 raise Hdf5Error {
guard depth >= 0 && depth < capacities.length() else {
raise InvalidArgument("writer v2 B-tree depth is too large")
}
let mut maximum = capacities[0].to_uint64()
for level in 1..<=depth {
let capacity = capacities[level].to_uint64()
guard maximum <= (0xFFFFFFFFFFFFFFFFUL - capacity) / (capacity + 1UL) else {
raise InvalidArgument("writer v2 B-tree is too large")
}
maximum = maximum * (capacity + 1UL) + capacity
}
maximum
}
///|
fn writer_btree_depth(
count : UInt64,
capacities : Array[Int32],
) -> Int32 raise Hdf5Error {
guard count > 0UL else { return 0 }
for depth in 0.. Array[Bytes] {
let result : Array[Bytes] = []
for index in start.. Int32 raise Hdf5Error {
if depth == 0 {
guard records.length() <= capacities[0] else {
raise InvalidArgument("writer v2 B-tree leaf is too large")
}
let index = nodes.length()
nodes.push({
depth: 0,
records,
children: [],
total: records.length().to_uint64(),
address: 0UL,
})
return index
}
let capacity = capacities[depth]
let child_maximum = writer_btree_max_records(depth - 1, capacities)
let minimum_child_size = if depth - 1 == 0 { 1 } else { 2 }
let count = records.length().to_int64()
let mut group_count = 0
for offset in 0..= minimum_total && count <= maximum_total {
group_count = groups
break
}
}
guard group_count > 0 else {
raise InvalidArgument("writer v2 B-tree node cannot be balanced")
}
let sizes : Array[Int32] = Array::make(group_count, minimum_child_size)
let remaining = records.length() - minimum_child_size * group_count
let extra_each = remaining / group_count
let extra_remainder = remaining % group_count
for group in 0.. Bytes raise Hdf5Error {
let buffer = @buffer.Buffer::Buffer(
size_hint=offset_size +
(if filtered { size_width + 4 } else { 0 }) +
length_size * coordinates.length(),
)
write_uint_width(
buffer,
file_offset_to_stored_address(entry.address, base_address),
offset_size,
)
if filtered {
writer_write_uint_width(buffer, entry.data.length().to_uint64(), size_width)
writer_write_uint_width(buffer, entry.filter_mask.to_uint64(), 4)
}
for coordinate in coordinates {
write_uint_width(buffer, coordinate, length_size)
}
buffer.to_bytes()
}
///|
fn writer_btree_coordinates(
index : Int32,
grid : Array[Int32],
) -> Array[UInt64] {
let coordinates = Array::make(grid.length(), 0UL)
let mut remaining = index
let mut axis = grid.length()
while axis > 0 {
axis = axis - 1
let width = grid[axis]
coordinates[axis] = (remaining % width).to_uint64()
remaining = remaining / width
}
coordinates
}
///|
fn encode_writer_btree_header(
root : UInt64,
root_count : Int32,
depth : Int32,
total : UInt64,
node_size : Int32,
record_size : Int32,
client : Int32,
offset_size~ : Int32 = 8,
length_size~ : Int32 = 8,
base_address~ : UInt64 = 0UL,
) -> Bytes raise Hdf5Error {
let buffer = @buffer.Buffer::Buffer(
size_hint=16 + offset_size + 2 + length_size + 4,
)
writer_write_bytes(buffer, b"BTHD")
buffer.write_byte(b'\x00')
buffer.write_byte(client.to_byte())
buffer.write_uint_le(node_size.reinterpret_as_uint())
buffer.write_uint16_le(record_size.to_uint16())
buffer.write_uint16_le(depth.to_uint16())
buffer.write_byte(writer_btree_v2_split_percent.to_byte())
buffer.write_byte(writer_btree_v2_merge_percent.to_byte())
write_uint_width(
buffer,
if root == 0xFFFFFFFFFFFFFFFFUL {
root
} else {
file_offset_to_stored_address(root, base_address)
},
offset_size,
)
buffer.write_uint16_le(root_count.to_uint16())
write_uint_width(buffer, total, length_size)
buffer.write_uint_le(metadata_checksum(buffer.to_bytes(), 0, buffer.length()))
buffer.to_bytes()
}
///|
fn encode_writer_btree_node(
node : WriterBTreeV2Node,
nodes : Array[WriterBTreeV2Node],
node_size : Int32,
record_size : Int32,
count_width : Int32,
total_widths : Array[Int32],
client : Int32,
offset_size~ : Int32 = 8,
base_address~ : UInt64 = 0UL,
) -> Bytes raise Hdf5Error {
let buffer = @buffer.Buffer::Buffer(size_hint=node_size)
writer_write_bytes(buffer, if node.depth == 0 { b"BTLF" } else { b"BTIN" })
buffer.write_byte(b'\x00')
buffer.write_byte(client.to_byte())
for record in node.records {
guard record.length() == record_size else {
raise InvalidField("writer v2 B-tree record size changed")
}
writer_write_bytes(buffer, record)
}
if node.depth > 0 {
for child_index in node.children {
let child = nodes[child_index]
write_uint_width(
buffer,
file_offset_to_stored_address(child.address, base_address),
offset_size,
)
writer_write_uint_width(
buffer,
child.records.length().to_uint64(),
count_width,
)
if node.depth > 1 {
writer_write_uint_width(
buffer,
child.total,
total_widths[node.depth - 1],
)
}
}
}
guard buffer.length() + 4 <= node_size else {
raise InvalidArgument("writer v2 B-tree node exceeds its configured size")
}
buffer.write_uint_le(metadata_checksum(buffer.to_bytes(), 0, buffer.length()))
writer_write_zeros(buffer, node_size - buffer.length())
buffer.to_bytes()
}
///|
fn encode_btree_v2_chunk_storage(
start : UInt64,
dataset : WritableDataset,
heap : WritableGlobalHeap,
objects : Array[WritableObject],
custom_filters : Array[&Hdf5CustomFilter],
offset_size~ : Int32 = 8,
length_size~ : Int32 = 8,
base_address~ : UInt64 = 0UL,
reference_address_size~ : Int32 = 8,
vlen_address_size~ : Int32 = 0,
) -> Bytes raise Hdf5Error {
let (chunk_dimensions, filters, unlimited_axes) = match dataset.storage {
WritableExtensible(chunks, pipeline, axes) => (chunks, pipeline, axes)
_ => raise InvalidField("expected extensible writer dataset")
}
guard unlimited_axes.length() > 1 else {
raise InvalidField(
"v2 B-tree storage requires multiple unlimited dimensions",
)
}
let entries = encode_dataset_chunks(
dataset,
heap,
objects,
custom_filters,
reference_address_size~,
vlen_address_size~,
)
let grid_dimensions = dataset.dimensions.copy()
for axis in 0.. 0
let chunk_size = writer_chunk_element_count(chunk_dimensions) *
writer_value_element_size(dataset.values)
let mut maximum_stored = 0
for entry in entries {
if entry.allocated && entry.data.length() > maximum_stored {
maximum_stored = entry.data.length()
}
}
let size_width = if filtered {
let largest = if maximum_stored > chunk_size {
maximum_stored
} else {
chunk_size
}
let width = btree_integer_width(largest.to_uint64()) + 1
if width > 8 {
8
} else {
width
}
} else {
0
}
let record_size = offset_size +
(if filtered { size_width + 4 } else { 0 }) +
length_size * dataset.dimensions.length()
let (capacities, total_widths, count_width) = writer_btree_capacities(
writer_btree_v2_node_size, record_size, offset_size,
)
let indices : Array[Int32] = []
let coordinates : Array[Array[UInt64]] = []
let placeholder : Array[Bytes] = []
for index in 0.. ignore
draft.length()
}
let node_start = writer_align(header_length)
let data_start = writer_align(
node_start + node_count * writer_btree_v2_node_size,
)
let updated = entries.copy()
let mut address = start + data_start.to_uint64()
for index in indices {
updated[index] = {
allocated: true,
data: entries[index].data,
address,
filter_mask: entries[index].filter_mask,
}
address = writer_align(address.to_int() + entries[index].data.length()).to_uint64()
}
let records : Array[Bytes] = []
for entry_index in 0.. 0 {
depth = writer_btree_depth(records.length().to_uint64(), capacities)
root = writer_btree_build_node(records, depth, capacities, nodes)
guard nodes.length() == node_count else {
raise InvalidField("writer v2 B-tree shape changed during encoding")
}
for index in 0..= 0 {
nodes[root].address
} else {
0xFFFFFFFFFFFFFFFFUL
}
let root_count = if root >= 0 { nodes[root].records.length() } else { 0 }
let header = encode_writer_btree_header(
root_address,
root_count,
depth,
records.length().to_uint64(),
writer_btree_v2_node_size,
record_size,
client,
offset_size~,
length_size~,
base_address~,
)
let total_chunks = address.to_int() - start.to_int()
let output = @buffer.Buffer::Buffer(size_hint=total_chunks)
writer_write_bytes(output, header)
writer_write_zeros(output, node_start - output.length())
for index in 0.. Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=18)
buffer.write_byte(b'\x00')
buffer.write_byte(b'\x00')
buffer.write_uint64_le(0xFFFFFFFFFFFFFFFFUL)
buffer.write_uint64_le(0xFFFFFFFFFFFFFFFFUL)
buffer.to_bytes()
}
///|
fn encode_hard_link(
name : String,
address : UInt64,
creation_order : UInt64,
track_order : Bool,
address_size~ : Int32 = 8,
) -> Bytes {
let encoded = @utf8.encode(name)
let buffer = @buffer.Buffer::Buffer(
size_hint=12 + address_size + encoded.length(),
)
buffer.write_byte(b'\x01')
buffer.write_byte(if track_order { b'\x15' } else { b'\x11' })
if track_order {
buffer.write_uint64_le(creation_order)
}
buffer.write_byte(b'\x01')
buffer.write_uint16_le(encoded.length().to_uint16())
writer_write_bytes(buffer, encoded)
write_uint_width(buffer, address, address_size)
buffer.to_bytes()
}
///|
fn encode_soft_link(
name : String,
target : String,
creation_order : UInt64,
track_order : Bool,
) -> Bytes {
let encoded_name = @utf8.encode(name)
let encoded_target = @utf8.encode(target)
let buffer = @buffer.Buffer::Buffer(
size_hint=8 + encoded_name.length() + encoded_target.length(),
)
buffer.write_byte(b'\x01')
buffer.write_byte(if track_order { b'\x1D' } else { b'\x19' })
buffer.write_byte(b'\x01')
if track_order {
buffer.write_uint64_le(creation_order)
}
buffer.write_byte(b'\x01')
buffer.write_uint16_le(encoded_name.length().to_uint16())
writer_write_bytes(buffer, encoded_name)
buffer.write_uint16_le(encoded_target.length().to_uint16())
writer_write_bytes(buffer, encoded_target)
buffer.to_bytes()
}
///|
fn encode_external_link(
name : String,
target_file : String,
target_path : String,
creation_order : UInt64,
track_order : Bool,
) -> Bytes raise Hdf5Error {
let encoded_name = @utf8.encode(name)
let encoded_file = @utf8.encode(target_file)
let encoded_path = @utf8.encode(target_path)
let info_size = 1 + encoded_file.length() + 1 + encoded_path.length() + 1
guard info_size <= 65535 else {
raise InvalidArgument("external link information is too long")
}
let buffer = @buffer.Buffer::Buffer(
size_hint=8 + encoded_name.length() + info_size,
)
buffer.write_byte(b'\x01')
buffer.write_byte(if track_order { b'\x1D' } else { b'\x19' })
buffer.write_byte(b'\x40')
if track_order {
buffer.write_uint64_le(creation_order)
}
buffer.write_byte(b'\x01')
buffer.write_uint16_le(encoded_name.length().to_uint16())
writer_write_bytes(buffer, encoded_name)
buffer.write_uint16_le(info_size.to_uint16())
buffer.write_byte(b'\x00')
writer_write_bytes(buffer, encoded_file)
buffer.write_byte(b'\x00')
writer_write_bytes(buffer, encoded_path)
buffer.write_byte(b'\x00')
buffer.to_bytes()
}
///|
fn Hdf5Writer::encode_attribute(
self : Hdf5Writer,
attribute : WritableAttribute,
heap : WritableGlobalHeap,
objects : Array[WritableObject],
) -> Bytes raise Hdf5Error {
let encoded = encode_write_values(attribute.values, heap~, objects~)
let name = @utf8.encode(attribute.name)
let dataspace = encode_dataspace(attribute.dimensions)
let (datatype, flags) = match
self.committed_datatype_address(encoded.datatype, objects) {
Some(address) => (encode_writer_committed_message(address), b'\x01')
None => (encoded.datatype, b'\x00')
}
let buffer = @buffer.Buffer::Buffer(
size_hint=9 +
name.length() +
datatype.length() +
dataspace.length() +
encoded.payload.length(),
)
buffer.write_byte(b'\x03')
buffer.write_byte(flags)
buffer.write_uint16_le((name.length() + 1).to_uint16())
buffer.write_uint16_le(datatype.length().to_uint16())
buffer.write_uint16_le(dataspace.length().to_uint16())
buffer.write_byte(b'\x01')
writer_write_bytes(buffer, name)
buffer.write_byte(b'\x00')
writer_write_bytes(buffer, datatype)
writer_write_bytes(buffer, dataspace)
writer_write_bytes(buffer, encoded.payload)
buffer.to_bytes()
}
///|
fn encode_compact_attribute_info() -> Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=18)
buffer.write_byte(b'\x00')
buffer.write_byte(b'\x00')
buffer.write_uint64_le(0xFFFFFFFFFFFFFFFFUL)
buffer.write_uint64_le(0xFFFFFFFFFFFFFFFFUL)
buffer.to_bytes()
}
///|
let writer_dense_threshold : Int32 = 8
///|
let writer_dense_max_managed_size : Int32 = 65535
///|
let writer_dense_max_link_records : Int32 = 0x7FFFFFFF
///|
let writer_dense_max_attribute_records : Int32 = 65535
///|
let writer_dense_btree_node_size : Int32 = 512
///|
fn encode_dense_storage_info(
group : Bool,
max_creation_order : UInt64,
heap_address : UInt64,
index_address : UInt64,
order_index_address : UInt64,
offset_size~ : Int32 = 8,
) -> Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=10 + 3 * offset_size)
buffer.write_byte(b'\x00')
buffer.write_byte(b'\x03')
if group {
buffer.write_uint64_le(max_creation_order)
} else {
buffer.write_uint16_le(max_creation_order.to_uint16())
}
write_uint_width(buffer, heap_address, offset_size)
write_uint_width(buffer, index_address, offset_size)
write_uint_width(buffer, order_index_address, offset_size)
buffer.to_bytes()
}
///|
fn writer_dense_max_creation_order(
entries : Array[WritableDenseHeapEntry],
) -> UInt64 {
let mut maximum = 0UL
for entry in entries {
if entry.creation_order > maximum {
maximum = entry.creation_order
}
}
maximum
}
///|
fn writer_name_hash(name : String) -> UInt32 {
let encoded = @utf8.encode(name)
metadata_checksum(encoded, 0, encoded.length())
}
///|
fn Hdf5Writer::dense_link_entries(
self : Hdf5Writer,
object : WritableObject,
objects : Array[WritableObject],
track_order~ : Bool,
) -> Array[WritableDenseHeapEntry] raise Hdf5Error {
let entries : Array[WritableDenseHeapEntry] = []
for child in objects {
if child.path != "/" && writer_parent_path(child.path) == object.path {
let name = writer_basename(child.path)
entries.push({
name,
hash: writer_name_hash(name),
data: encode_hard_link(
name,
child.address,
self.link_creation_order(child.path),
track_order,
),
creation_order: self.link_creation_order(child.path),
flags: b'\x00',
shared_id: None,
})
}
}
for link in self.soft_links {
if link.parent_path == object.path {
entries.push({
name: link.name,
hash: writer_name_hash(link.name),
data: encode_soft_link(
link.name,
link.target,
self.link_creation_order(
writer_soft_link_path(link.parent_path, link.name),
),
track_order,
),
creation_order: self.link_creation_order(
writer_soft_link_path(link.parent_path, link.name),
),
flags: b'\x00',
shared_id: None,
})
}
}
for link in self.external_links {
if link.parent_path == object.path {
entries.push({
name: link.name,
hash: writer_name_hash(link.name),
data: encode_external_link(
link.name,
link.target_file,
link.target_path,
self.link_creation_order(
writer_soft_link_path(link.parent_path, link.name),
),
track_order,
),
creation_order: self.link_creation_order(
writer_soft_link_path(link.parent_path, link.name),
),
flags: b'\x00',
shared_id: None,
})
}
}
entries
}
///|
fn writer_shared_attribute(
attributes : Array[WritableSharedAttribute],
path : String,
name : String,
) -> WritableSharedAttribute? {
match
attributes.search_by(attribute => {
attribute.path == path && attribute.name == name
}) {
Some(index) => Some(attributes[index])
None => None
}
}
///|
fn writer_shared_fill(
fills : Array[WritableSharedFill],
path : String,
) -> WritableSharedFill? {
match fills.search_by(fill => fill.path == path) {
Some(index) => Some(fills[index])
None => None
}
}
///|
fn writer_shared_filter(
filters : Array[WritableSharedFilter],
path : String,
) -> WritableSharedFilter? {
match filters.search_by(filter => filter.path == path) {
Some(index) => Some(filters[index])
None => None
}
}
///|
fn encode_writer_shared_message(heap_id : Bytes) -> Bytes raise Hdf5Error {
guard heap_id.length() == 8 else {
raise InvalidField("writer shared-message heap ID has an invalid size")
}
let buffer = @buffer.Buffer::Buffer(size_hint=10)
buffer.write_byte(b'\x03')
buffer.write_byte(b'\x01')
writer_write_bytes(buffer, heap_id)
buffer.to_bytes()
}
///|
fn encode_writer_committed_message(address : UInt64) -> Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=10)
buffer.write_byte(b'\x03')
buffer.write_byte(b'\x02')
buffer.write_uint64_le(address)
buffer.to_bytes()
}
///|
fn Hdf5Writer::committed_datatype_address(
self : Hdf5Writer,
datatype_data : Bytes,
objects : Array[WritableObject],
) -> UInt64? raise Hdf5Error {
match
self.committed_datatypes.search_by(datatype => {
datatype.data == datatype_data
}) {
Some(index) =>
Some(
find_writable_object(objects, self.committed_datatypes[index].path).address,
)
None => None
}
}
///|
fn Hdf5Writer::dense_attribute_entries(
self : Hdf5Writer,
path : String,
heap : WritableGlobalHeap,
objects : Array[WritableObject],
shared~ : Array[WritableSharedAttribute] = [],
) -> Array[WritableDenseHeapEntry] raise Hdf5Error {
let entries : Array[WritableDenseHeapEntry] = []
for attribute in self.object_attributes(path) {
let (data, flags, shared_id) = match
writer_shared_attribute(shared, path, attribute.name) {
Some(shared_attribute) =>
(
encode_writer_shared_message(shared_attribute.id),
shared_message_flag.to_byte(),
Some(shared_attribute.id),
)
None => (self.encode_attribute(attribute, heap, objects), b'\x00', None)
}
entries.push({
name: attribute.name,
hash: writer_name_hash(attribute.name),
data,
creation_order: attribute.creation_order,
flags,
shared_id,
})
}
entries
}
///|
fn writer_dense_entries_fit(
entries : Array[WritableDenseHeapEntry],
kind~ : WritableDenseKind,
) -> Bool {
if entries.length() <= writer_dense_threshold {
return false
}
match kind {
WritableDenseLinks => entries.length() <= writer_dense_max_link_records
WritableDenseAttributes =>
entries.length() <= writer_dense_max_attribute_records
}
}
///|
fn writer_sort_dense_entries(entries : Array[WritableDenseHeapEntry]) -> Unit {
entries.sort_by((left, right) => {
if left.hash < right.hash {
-1
} else if left.hash > right.hash {
1
} else if left.name < right.name {
-1
} else if left.name > right.name {
1
} else {
0
}
})
}
///|
fn writer_next_power_of_two(
value : Int32,
minimum : Int32,
) -> Int32 raise Hdf5Error {
let mut result = minimum
while result < value {
guard result <= 0x3FFFFFFF else {
raise InvalidField("writer dense metadata is too large")
}
result = result * 2
}
result
}
///|
fn writer_dense_heap_header_size(
offset_size : Int32,
length_size : Int32,
filter_info_size~ : Int32 = 0,
) -> Int32 {
22 +
length_size * 12 +
offset_size * 3 +
(if filter_info_size > 0 { length_size + 4 + filter_info_size } else { 0 }) +
metadata_checksum_size
}
///|
fn encode_writer_dense_heap_header(
root_address : UInt64,
id_length : Int32,
start_block_size : Int32,
managed_size : Int32,
max_direct_size : Int32,
max_heap_bits : Int32,
record_count : Int32,
max_managed_size : Int32,
table_width : Int32,
offset_size~ : Int32 = 8,
length_size~ : Int32 = 8,
next_huge_id~ : UInt64 = 0UL,
huge_btree_address~ : UInt64? = None,
huge_object_size~ : UInt64 = 0UL,
huge_objects~ : UInt64 = 0UL,
current_rows~ : Int32 = 0,
starting_rows~ : Int32 = 1,
filtered_root_size~ : UInt64 = 0UL,
filtered_root_mask~ : UInt32 = 0U,
encoded_filters~ : Bytes = b"",
) -> Bytes {
let buffer = @buffer.Buffer::Buffer(
size_hint=writer_dense_heap_header_size(
offset_size,
length_size,
filter_info_size=encoded_filters.length(),
),
)
writer_write_bytes(buffer, b"FRHP")
buffer.write_byte(b'\x00')
buffer.write_uint16_le(id_length.to_uint16())
buffer.write_uint16_le(encoded_filters.length().to_uint16())
buffer.write_byte(b'\x02')
buffer.write_uint_le(max_managed_size.reinterpret_as_uint())
write_uint_width(buffer, next_huge_id, length_size)
write_uint_width(
buffer,
match huge_btree_address {
Some(address) => address
None => heap_undefined_address(offset_size)
},
offset_size,
)
write_uint_width(buffer, 0UL, length_size)
write_uint_width(buffer, heap_undefined_address(offset_size), offset_size)
write_uint_width(buffer, managed_size.to_uint64(), length_size)
write_uint_width(buffer, managed_size.to_uint64(), length_size)
write_uint_width(buffer, 0UL, length_size)
write_uint_width(buffer, record_count.to_uint64(), length_size)
write_uint_width(buffer, huge_object_size, length_size)
write_uint_width(buffer, huge_objects, length_size)
write_uint_width(buffer, 0UL, length_size)
write_uint_width(buffer, 0UL, length_size)
buffer.write_uint16_le(table_width.to_uint16())
write_uint_width(buffer, start_block_size.to_uint64(), length_size)
write_uint_width(buffer, max_direct_size.to_uint64(), length_size)
buffer.write_uint16_le(max_heap_bits.to_uint16())
buffer.write_uint16_le(starting_rows.to_uint16())
write_uint_width(buffer, root_address, offset_size)
buffer.write_uint16_le(current_rows.to_uint16())
if encoded_filters.length() > 0 {
write_uint_width(buffer, filtered_root_size, length_size)
buffer.write_uint_le(filtered_root_mask)
writer_write_bytes(buffer, encoded_filters)
}
buffer.write_uint_le(metadata_checksum(buffer.to_bytes(), 0, buffer.length()))
buffer.to_bytes()
}
///|
fn patched_zeroed_checksum(bytes : Bytes, checksum_offset : Int32) -> Bytes {
let checksum = checksum_lookup3(
bytes,
0,
bytes.length(),
checksum_offset,
metadata_checksum_size,
0U,
)
let output = @buffer.Buffer::Buffer(size_hint=bytes.length())
output.write_bytes(bytes.get_view(start=0, end=checksum_offset).unwrap())
output.write_uint_le(checksum)
output.write_bytes(
bytes
.get_view(
start=checksum_offset + metadata_checksum_size,
end=bytes.length(),
)
.unwrap(),
)
output.to_bytes()
}
///|
fn writer_heap_direct_header_size(
offset_size : Int32,
block_offset_size : Int32,
) -> Int32 {
9 + offset_size + block_offset_size
}
///|
fn writer_heap_block_size_at(
index : Int32,
start_block_size : Int32,
table_width : Int32,
max_direct_size : Int32,
) -> Int32 raise Hdf5Error {
guard index >= 0 && table_width > 0 && start_block_size > 0 else {
raise InvalidField("invalid writer fractal heap block index")
}
let row = index / table_width
let shift = if row > 1 { row - 1 } else { 0 }
guard shift <= 30 else {
raise InvalidArgument("writer fractal heap block is too large")
}
let size = start_block_size << shift
guard size > 0 && size <= max_direct_size else {
raise InvalidArgument("writer fractal heap exceeds its direct block limit")
}
size
}
///|
fn writer_heap_max_direct_rows(
start_block_size : Int32,
max_direct_size : Int32,
) -> Int32 raise Hdf5Error {
let start_bits = heap_log2(
start_block_size.to_uint64(),
"writer fractal heap starting block size",
)
let direct_bits = heap_log2(
max_direct_size.to_uint64(),
"writer fractal heap maximum direct block size",
)
guard direct_bits >= start_bits else {
raise InvalidField("writer fractal heap direct block size is too small")
}
direct_bits - start_bits + 2
}
///|
fn writer_heap_indirect_block_size(
offset_size : Int32,
block_offset_size : Int32,
table_width : Int32,
rows : Int32,
filtered~ : Bool = false,
length_size~ : Int32 = 8,
) -> Int32 raise Hdf5Error {
guard rows > 0 &&
table_width > 0 &&
rows.to_int64() * table_width.to_int64() <= 0x7FFFFFFFL else {
raise InvalidArgument("writer fractal heap indirect block is too large")
}
let entry_size = offset_size + (if filtered { length_size + 4 } else { 0 })
5 +
offset_size +
block_offset_size +
rows * table_width * entry_size +
metadata_checksum_size
}
///|
fn encode_writer_heap_indirect_block(
heap_address : UInt64,
offset_size : Int32,
block_offset_size : Int32,
table_width : Int32,
rows : Int32,
max_direct_rows : Int32,
direct_entries : Array[(UInt64, UInt64, UInt32)],
filtered~ : Bool = false,
length_size~ : Int32 = 8,
) -> Bytes raise Hdf5Error {
guard rows > 0 &&
table_width > 0 &&
max_direct_rows > 0 &&
rows <= max_direct_rows &&
direct_entries.length() > 0 &&
direct_entries.length() <= rows * table_width else {
raise InvalidField("invalid writer fractal heap indirect block")
}
let entries = rows * table_width
let entry_size = offset_size + (if filtered { length_size + 4 } else { 0 })
let content_size = 5 + offset_size + block_offset_size + entries * entry_size
let buffer = @buffer.Buffer::Buffer(
size_hint=content_size + metadata_checksum_size,
)
writer_write_bytes(buffer, b"FHIB")
buffer.write_byte(b'\x00')
write_uint_width(buffer, heap_address, offset_size)
write_uint_width(buffer, 0UL, block_offset_size)
let undefined = heap_undefined_address(offset_size)
for index in 0.. Bytes raise Hdf5Error {
guard element_size > 0 && element_size <= data.length() else {
raise InvalidField("invalid metadata shuffle element size")
}
let elements = data.length() / element_size
let tail_start = elements * element_size
let output = FixedArray::make(data.length(), b'\x00')
for byte_index in 0.. Bytes raise Hdf5Error {
let expected_size = data.length()
let mut output = data
for filter in filters {
output = match filter.id {
1 => {
guard filter.parameters.length() == 1 && filter.parameters[0] <= 9U else {
raise InvalidField("invalid metadata deflate filter parameters")
}
@zlib.zlib(output[:], level=filter.parameters[0].reinterpret_as_int())
}
2 => {
guard filter.parameters.length() == 1 &&
filter.parameters[0] > 0U &&
filter.parameters[0] <= 0x7FFFFFFFU else {
raise InvalidField("invalid metadata shuffle filter parameters")
}
writer_shuffle_metadata_block(
output,
filter.parameters[0].reinterpret_as_int(),
)
}
3 => {
guard filter.parameters.length() == 0 else {
raise InvalidField("invalid metadata Fletcher32 filter parameters")
}
add_fletcher32(output)
}
6 =>
raise UnsupportedFeature(
"scale-offset cannot encode fractal heap metadata blocks",
)
_ =>
match filter.decoder {
Some(custom) => {
let encoded = custom.encode(
output,
filter.parameters,
0,
expected_size,
)
guard encoded.length() > 0 else {
raise InvalidField(
"custom filter produced an empty metadata block",
)
}
encoded
}
None =>
raise UnsupportedFeature(
"filter " + filter.id.to_string() + " is not supported",
)
}
}
guard output.length() > 0 && output.length() <= 0x7FFFFFFF else {
raise InvalidField("filtered metadata heap block has an invalid size")
}
}
output
}
///|
fn encode_writer_dense_heap_storage(
entries : Array[WritableDenseHeapEntry],
heap_address : UInt64,
id_length : Int32,
minimum_block_size : Int32,
max_heap_bits : Int32,
offset_size~ : Int32 = 8,
length_size~ : Int32 = 8,
max_managed_size~ : Int32 = writer_dense_max_managed_size,
table_width~ : Int32 = 4,
metadata_filters~ : Array[Filter] = [],
encoded_filters~ : Bytes = b"",
) -> (Bytes, Array[Bytes]) raise Hdf5Error {
guard id_length > 1 &&
id_length <= 255 &&
minimum_block_size > 0 &&
max_heap_bits > 0 &&
max_heap_bits <= 64 &&
table_width > 0 &&
max_managed_size > 0 &&
id_length - 1 < offset_size + length_size else {
raise InvalidField("invalid writer fractal heap configuration")
}
guard (metadata_filters.length() == 0) == (encoded_filters.length() == 0) &&
encoded_filters.length() <= 65535 else {
raise InvalidField("invalid writer fractal heap filter pipeline")
}
let filtered = encoded_filters.length() > 0
let block_offset_size = (max_heap_bits + 7) / 8
let managed_offset_size = block_offset_size
let ids : Array[Bytes] = []
for _ in 0.. {
guard id.length() == id_length else {
raise InvalidField(
"writer shared-message heap ID has an invalid size",
)
}
ids[index] = id
}
None => {
let length = entries[index].data.length()
guard length > 0 else {
raise InvalidField("writer fractal heap object is empty")
}
if length <= max_managed_size {
managed.push(index)
if length > maximum_managed_object {
maximum_managed_object = length
}
} else {
huge.push(index)
guard huge_data_size <= 0x7FFFFFFF - length else {
raise InvalidArgument(
"writer fractal heap huge objects are too large",
)
}
huge_data_size = huge_data_size + length
}
}
}
}
let direct_header_size = writer_heap_direct_header_size(
offset_size, block_offset_size,
)
let mut managed_data_size = 0
for index in managed {
managed_data_size = managed_data_size + entries[index].data.length()
}
let single_block = managed_data_size + direct_header_size <= 65536
let mut start_block_size = minimum_block_size
let mut max_direct_size = 65536
if managed.length() > 0 && single_block {
start_block_size = writer_next_power_of_two(
managed_data_size + direct_header_size,
minimum_block_size,
)
max_direct_size = writer_next_power_of_two(start_block_size, 65536)
} else if managed.length() > 0 {
start_block_size = writer_next_power_of_two(
if maximum_managed_object + direct_header_size > minimum_block_size {
maximum_managed_object + direct_header_size
} else {
minimum_block_size
},
minimum_block_size,
)
max_direct_size = writer_next_power_of_two(
if managed_data_size > start_block_size {
managed_data_size
} else {
start_block_size
},
if start_block_size > 65536 {
start_block_size
} else {
65536
},
)
} else {
max_direct_size = writer_next_power_of_two(start_block_size, 65536)
}
let managed_length_size = heap_byte_width(
if max_direct_size.to_uint64() < max_managed_size.to_uint64() {
max_direct_size.to_uint64()
} else {
max_managed_size.to_uint64()
},
)
let managed_id_length = 1 + managed_offset_size + managed_length_size
guard managed_id_length <= id_length else {
raise InvalidField("writer fractal heap managed IDs are too large")
}
let max_direct_rows = writer_heap_max_direct_rows(
start_block_size, max_direct_size,
)
guard max_direct_rows <= 0xFFFF / table_width else {
raise InvalidArgument("writer fractal heap has too many direct rows")
}
let max_blocks = max_direct_rows * table_width
let block_sizes : Array[Int32] = []
let block_of_entry = Array::make(entries.length(), -1)
let offset_of_entry = Array::make(entries.length(), 0)
if managed.length() > 0 {
let mut block_index = 0
let mut cursor = direct_header_size
for entry_index in managed {
let length = entries[entry_index].data.length()
while true {
if block_index >= block_sizes.length() {
guard block_index < max_blocks else {
raise InvalidArgument(
"writer fractal heap needs more direct blocks than its limit",
)
}
block_sizes.push(
writer_heap_block_size_at(
block_index, start_block_size, table_width, max_direct_size,
),
)
cursor = direct_header_size
}
let size = block_sizes[block_index]
if cursor + length <= size {
block_of_entry[entry_index] = block_index
offset_of_entry[entry_index] = cursor
cursor = cursor + length
break
}
block_index = block_index + 1
cursor = direct_header_size
}
}
}
let block_count = block_sizes.length()
let use_indirect = block_count > 0 && !single_block
let rows = if use_indirect {
(block_count + table_width - 1) / table_width
} else {
0
}
guard rows <= max_direct_rows else {
raise InvalidArgument(
"writer fractal heap indirect block has too many rows",
)
}
let heap_header_size = writer_dense_heap_header_size(
offset_size,
length_size,
filter_info_size=encoded_filters.length(),
)
let indirect_block_size = if use_indirect {
writer_heap_indirect_block_size(
offset_size,
block_offset_size,
table_width,
rows,
filtered~,
length_size~,
)
} else {
0
}
let block_offsets : Array[UInt64] = []
let mut managed_space_size = 0UL
for size in block_sizes {
block_offsets.push(managed_space_size)
managed_space_size = managed_space_size + size.to_uint64()
}
let huge_id_size = id_length - 1
for index in managed {
let block_index = block_of_entry[index]
guard block_index >= 0 && block_index < block_count else {
raise InvalidField("writer fractal heap object has no block")
}
let offset = block_offsets[block_index] + offset_of_entry[index].to_uint64()
let id = @buffer.Buffer::Buffer(size_hint=id_length)
id.write_byte(b'\x00')
write_uint_width(id, offset, managed_offset_size)
write_uint_width(
id,
entries[index].data.length().to_uint64(),
managed_length_size,
)
while id.length() < id_length {
id.write_byte(b'\x00')
}
ids[index] = id.to_bytes()
}
for index in 0.. 0 else {
raise InvalidField("writer fractal heap stored direct block is empty")
}
block_addresses.push(heap_address + relative.to_uint64())
block_stored_sizes.push(block.length().to_uint64())
relative = relative + block.length()
}
let huge_blocks : Array[Bytes] = []
let mut huge_relative = writer_align(relative)
let huge_addresses : Array[UInt64] = []
for entry_index in huge {
let stored = if filtered {
encode_metadata_heap_filters(entries[entry_index].data, metadata_filters)
} else {
entries[entry_index].data
}
huge_relative = writer_align(huge_relative)
huge_addresses.push(heap_address + huge_relative.to_uint64())
huge_blocks.push(stored)
huge_relative = huge_relative + stored.length()
}
let huge_btree_relative = writer_align(huge_relative)
let huge_record_size = offset_size +
(if filtered { length_size * 3 + 4 } else { length_size * 2 })
let huge_records : Array[Bytes] = []
for index in 0.. 0 {
encode_writer_metadata_btree(
if filtered {
2
} else {
1
},
writer_dense_btree_node_size,
huge_record_size,
huge_records,
huge_btree_address,
offset_size~,
length_size~,
)
} else {
b""
}
let root_address = if block_count == 0 {
heap_undefined_address(offset_size)
} else if use_indirect {
heap_address + heap_header_size.to_uint64()
} else {
block_addresses[0]
}
let header = encode_writer_dense_heap_header(
root_address,
id_length,
if block_count > 0 {
start_block_size
} else {
minimum_block_size
},
managed_space_size.to_int(),
max_direct_size,
max_heap_bits,
managed.length(),
max_managed_size,
table_width,
offset_size~,
length_size~,
next_huge_id=(huge.length() + 1).to_uint64(),
huge_btree_address=if huge.length() > 0 {
Some(huge_btree_address)
} else {
None
},
huge_object_size=huge_data_size.to_uint64(),
huge_objects=huge.length().to_uint64(),
current_rows=rows,
filtered_root_size=if block_count > 0 && !use_indirect {
block_stored_sizes[0]
} else {
0UL
},
filtered_root_mask=0U,
encoded_filters~,
)
let output = @buffer.Buffer::Buffer(
size_hint=heap_header_size +
indirect_block_size +
(relative - heap_header_size - indirect_block_size) +
huge_btree_relative +
huge_btree.length(),
)
writer_write_bytes(output, header)
if use_indirect {
let direct_entries : Array[(UInt64, UInt64, UInt32)] = []
for index in 0.. WriterMetadataBTreeShape raise Hdf5Error {
guard record_size > 0 &&
node_size > 10 &&
offset_size >= 1 &&
offset_size <= 8 &&
count >= 0 else {
raise InvalidField("invalid writer metadata B-tree shape")
}
let leaf_capacity = (node_size - 10) / record_size
guard leaf_capacity > 0 && leaf_capacity <= 65535 else {
raise InvalidField("writer metadata B-tree leaf capacity is out of range")
}
let capacities : Array[Int32] = [leaf_capacity]
let count_widths : Array[Int32] = [0]
let total_widths : Array[Int32] = [0]
let totals : Array[UInt64] = [leaf_capacity.to_uint64()]
let mut depth = 0
while totals[depth] < count.to_uint64() {
guard depth < 31 else {
raise InvalidArgument("writer metadata B-tree needs too many levels")
}
let level = depth + 1
let count_width = metadata_count_width(capacities[level - 1].to_uint64())
let total_width = if level > 1 {
metadata_count_width(totals[level - 1])
} else {
0
}
let pointer_size = offset_size + count_width + total_width
let capacity = (node_size - 10 - pointer_size) /
(record_size + pointer_size)
guard capacity > 0 && capacity <= 65535 else {
raise InvalidField(
"writer metadata B-tree internal capacity is out of range",
)
}
guard totals[level - 1] <=
(0xFFFFFFFFFFFFFFFFUL - capacity.to_uint64()) / (capacity + 1).to_uint64() else {
raise InvalidArgument("writer metadata B-tree subtree count overflow")
}
let total = totals[level - 1] * (capacity + 1).to_uint64() +
capacity.to_uint64()
capacities.push(capacity)
count_widths.push(count_width)
total_widths.push(total_width)
totals.push(total)
depth = level
}
{ capacities, count_widths, total_widths, depth, }
}
///|
fn encode_writer_metadata_btree_header(
node_type : Int32,
node_size : Int32,
record_size : Int32,
root_address : UInt64,
root_count : Int32,
depth : Int32,
total : UInt64,
offset_size~ : Int32 = 8,
length_size~ : Int32 = 8,
) -> Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=22 + offset_size + length_size)
writer_write_bytes(buffer, b"BTHD")
buffer.write_byte(b'\x00')
buffer.write_byte(node_type.to_byte())
buffer.write_uint_le(node_size.reinterpret_as_uint())
buffer.write_uint16_le(record_size.to_uint16())
buffer.write_uint16_le(depth.to_uint16())
buffer.write_byte(b'\x64')
buffer.write_byte(b'\x28')
write_uint_width(buffer, root_address, offset_size)
buffer.write_uint16_le(root_count.to_uint16())
write_uint_width(buffer, total, length_size)
buffer.write_uint_le(metadata_checksum(buffer.to_bytes(), 0, buffer.length()))
buffer.to_bytes()
}
///|
fn encode_writer_metadata_btree_node(
node : WriterBTreeV2Node,
nodes : Array[WriterBTreeV2Node],
shape : WriterMetadataBTreeShape,
node_type : Int32,
node_size : Int32,
record_size : Int32,
offset_size : Int32,
) -> Bytes raise Hdf5Error {
let buffer = @buffer.Buffer::Buffer(size_hint=node_size)
writer_write_bytes(buffer, if node.depth == 0 { b"BTLF" } else { b"BTIN" })
buffer.write_byte(b'\x00')
buffer.write_byte(node_type.to_byte())
for record in node.records {
guard record.length() == record_size else {
raise InvalidField("writer dense B-tree record size mismatch")
}
writer_write_bytes(buffer, record)
}
if node.depth > 0 {
for child_index in node.children {
let child = nodes[child_index]
writer_write_uint_width(buffer, child.address, offset_size)
writer_write_uint_width(
buffer,
child.records.length().to_uint64(),
shape.count_widths[node.depth],
)
if node.depth > 1 {
writer_write_uint_width(
buffer,
child.total,
shape.total_widths[node.depth],
)
}
}
}
guard buffer.length() + metadata_checksum_size <= node_size else {
raise InvalidField("writer dense B-tree node is too small")
}
buffer.write_uint_le(metadata_checksum(buffer.to_bytes(), 0, buffer.length()))
writer_write_zeros(buffer, node_size - buffer.length())
buffer.to_bytes()
}
///|
fn encode_writer_metadata_btree(
node_type : Int32,
node_size : Int32,
record_size : Int32,
records : Array[Bytes],
start_address : UInt64,
offset_size~ : Int32 = 8,
length_size~ : Int32 = 8,
) -> Bytes raise Hdf5Error {
let header_size = 22 + offset_size + length_size
if records.length() == 0 {
return encode_writer_metadata_btree_header(
node_type,
node_size,
record_size,
heap_undefined_address(offset_size),
0,
0,
0UL,
offset_size~,
length_size~,
)
}
guard records.length() <= 0x7FFFFFFF else {
raise InvalidArgument("writer metadata B-tree has too many records")
}
let shape = writer_metadata_btree_shape(
record_size,
node_size,
offset_size,
records.length(),
)
let nodes : Array[WriterBTreeV2Node] = []
let root = writer_btree_build_node(
records,
shape.depth,
shape.capacities,
nodes,
)
guard nodes[root].records.length() <= 65535 else {
raise InvalidArgument("writer metadata B-tree root has too many records")
}
for index in 0.. Array[WritableSharedAttribute] raise Hdf5Error {
let attributes : Array[WritableSharedAttribute] = []
if !self.share_attributes {
return attributes
}
let empty_id = Bytes::make(8, b'\x00')
for object in objects {
for attribute in self.object_attributes(object.path) {
let data = self.encode_attribute(attribute, heap, objects)
if data.length() <= writer_shared_max_managed_size {
attributes.push({
path: object.path,
name: attribute.name,
data,
id: empty_id,
})
}
}
}
guard attributes.length() <= 65535 else {
raise InvalidArgument("writer has too many shared attributes")
}
attributes
}
///|
fn Hdf5Writer::shared_fill_candidates(
self : Hdf5Writer,
objects : Array[WritableObject],
heap : WritableGlobalHeap,
) -> Array[WritableSharedFill] raise Hdf5Error {
let fills : Array[WritableSharedFill] = []
if !self.share_fills {
return fills
}
let empty_id = Bytes::make(8, b'\x00')
for object in objects {
match object.kind {
WritableDatasetObject(index) => {
let dataset = self.datasets[index]
match dataset.fill_value {
Some(fill) => {
let data = encode_fill_value(fill, heap~, objects~)
if data.length() <= writer_shared_max_managed_size {
fills.push({ path: object.path, data, id: empty_id, })
}
}
None => ()
}
}
WritableGroup | WritableCommittedDatatype(_) => ()
}
}
fills
}
///|
fn Hdf5Writer::shared_filter_candidates(
self : Hdf5Writer,
objects : Array[WritableObject],
heap : WritableGlobalHeap,
) -> Array[WritableSharedFilter] raise Hdf5Error {
let filters : Array[WritableSharedFilter] = []
if !self.share_filters {
return filters
}
let empty_id = Bytes::make(8, b'\x00')
for object in objects {
match object.kind {
WritableDatasetObject(index) => {
let dataset = self.datasets[index]
let (chunk_dimensions, pipeline) = match dataset.storage {
WritableChunked(dimensions, filters) => (dimensions, filters)
WritableExtensible(dimensions, filters, _) => (dimensions, filters)
WritableCompact
| WritableContiguous
| WritableExternal(_)
| WritableVirtual(_) => continue
}
if pipeline.length() == 0 {
continue
}
let encoded = encode_write_values(dataset.values, heap~, objects~)
let fill = match dataset.fill_value {
Some(fill) => encode_write_values(fill, heap~, objects~).payload
None => Bytes::make(encoded.element_size, b'\x00')
}
let data = encode_filter_pipeline(
pipeline,
encoded.element_size,
writer_chunk_element_count(chunk_dimensions),
dataset.values,
fill,
self.custom_filters,
)
if data.length() <= writer_shared_max_managed_size {
filters.push({ path: object.path, data, id: empty_id, })
}
}
WritableGroup | WritableCommittedDatatype(_) => ()
}
}
guard filters.length() <= 65535 else {
raise InvalidArgument("writer has too many shared filters")
}
filters
}
///|
fn encode_writer_shared_heap(
heap_address : UInt64,
messages : Array[Bytes],
) -> (Bytes, Array[Bytes]) raise Hdf5Error {
guard messages.length() > 0 else { return (b"", []) }
for message in messages {
guard message.length() <= writer_shared_max_managed_size else {
raise InvalidField("writer shared message exceeds its managed heap")
}
}
let entries = messages.map(data => {
name: "",
hash: 0U,
data,
creation_order: 0UL,
flags: b'\x00',
shared_id: None,
})
encode_writer_dense_heap_storage(
entries,
heap_address,
8,
1024,
40,
max_managed_size=writer_shared_max_managed_size,
table_width=4,
)
}
///|
fn writer_shared_table_size(index_count : Int32) -> Int32 {
4 + index_count * (14 + 16) + metadata_checksum_size
}
///|
fn encode_writer_shared_table(heaps : Array[WritableSharedHeap]) -> Bytes {
let buffer = @buffer.Buffer::Buffer(
size_hint=writer_shared_table_size(heaps.length()),
)
writer_write_bytes(buffer, b"SMTB")
for heap in heaps {
buffer.write_byte(b'\x00')
buffer.write_byte(b'\x00')
buffer.write_uint16_le(heap.message_types.to_uint16())
buffer.write_uint_le(0U)
buffer.write_uint16_le(65535U.to_uint16())
buffer.write_uint16_le(0U.to_uint16())
buffer.write_uint16_le(heap.messages.length().to_uint16())
buffer.write_uint64_le(0xFFFFFFFFFFFFFFFFUL)
buffer.write_uint64_le(heap.address)
}
buffer.write_uint_le(metadata_checksum(buffer.to_bytes(), 0, buffer.length()))
buffer.to_bytes()
}
///|
fn encode_writer_superblock_extension(
table_address : UInt64,
index_count : Int32,
) -> Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=40)
buffer.write_byte(b'\x01')
buffer.write_byte(b'\x00')
buffer.write_uint16_le(1U.to_uint16())
buffer.write_uint_le(1U)
buffer.write_uint_le(24U)
buffer.write_uint_le(0U)
buffer.write_uint16_le(0x000FU.to_uint16())
buffer.write_uint16_le(16U.to_uint16())
buffer.write_byte(b'\x00')
writer_write_zeros(buffer, 3)
buffer.write_byte(b'\x00')
buffer.write_uint64_le(table_address)
buffer.write_byte(index_count.to_byte())
writer_write_zeros(buffer, 6)
buffer.to_bytes()
}
///|
fn Hdf5Writer::encode_dense_metadata(
self : Hdf5Writer,
object : WritableObject,
objects : Array[WritableObject],
heap : WritableGlobalHeap,
heap_address : UInt64,
kind : WritableDenseKind,
shared~ : Array[WritableSharedAttribute] = [],
) -> (Bytes, Bytes, Bytes) raise Hdf5Error {
let group = match kind {
WritableDenseLinks => true
WritableDenseAttributes => false
}
let entries = if group {
self.dense_link_entries(object, objects, track_order=true)
} else {
self.dense_attribute_entries(object.path, heap, objects, shared~)
}
guard writer_dense_entries_fit(entries, kind~) else {
raise InvalidField("dense metadata is not available for this object")
}
writer_sort_dense_entries(entries)
let record_size = if group { 11 } else { 17 }
let max_heap_bits = if group { 32 } else { 40 }
let id_length = if group { 7 } else { 8 }
let minimum_block_size = if group { 512 } else { 1024 }
let (heap_bytes, ids) = encode_writer_dense_heap_storage(
entries, heap_address, id_length, minimum_block_size, max_heap_bits,
)
let index_address = heap_address + heap_bytes.length().to_uint64()
let records : Array[Bytes] = []
for index in 0.. {
let left_order = entries[left].creation_order
let right_order = entries[right].creation_order
if left_order < right_order {
-1
} else if left_order > right_order {
1
} else if entries[left].name < entries[right].name {
-1
} else if entries[left].name > entries[right].name {
1
} else {
0
}
})
let order_record_size = if group { 15 } else { 13 }
let order_records : Array[Bytes] = []
for index in order_indices {
let order_record = @buffer.Buffer::Buffer(size_hint=order_record_size)
if group {
order_record.write_uint64_le(entries[index].creation_order)
writer_write_bytes(order_record, ids[index])
} else {
writer_write_bytes(order_record, ids[index])
order_record.write_byte(entries[index].flags)
order_record.write_uint_le(entries[index].creation_order.to_uint())
}
guard order_record.length() == order_record_size else {
raise InvalidField("writer dense order B-tree record size mismatch")
}
order_records.push(order_record.to_bytes())
}
let index_bytes = encode_writer_metadata_btree(
if group {
5
} else {
8
},
writer_dense_btree_node_size,
record_size,
records,
index_address,
)
let order_index_address = index_address + index_bytes.length().to_uint64()
let order_bytes = encode_writer_metadata_btree(
if group {
6
} else {
9
},
writer_dense_btree_node_size,
order_record_size,
order_records,
order_index_address,
)
(heap_bytes, index_bytes, order_bytes)
}
///|
fn writer_dense_addresses(
dense : Array[WritableDenseMetadata],
path : String,
kind : WritableDenseKind,
) -> (UInt64, UInt64, UInt64) {
match
dense.search_by(metadata => metadata.path == path && metadata.kind == kind) {
Some(index) =>
(
dense[index].heap_address,
dense[index].index_address,
dense[index].order_index_address,
)
None => (0xFFFFFFFFFFFFFFFFUL, 0xFFFFFFFFFFFFFFFFUL, 0xFFFFFFFFFFFFFFFFUL)
}
}
///|
fn writer_external_heap_address(
external : Array[WritableExternalHeap],
path : String,
) -> UInt64 {
match external.search_by(heap => heap.path == path) {
Some(index) => external[index].address
None => 0xFFFFFFFFFFFFFFFFUL
}
}
///|
fn writer_virtual_heap_address(
virtual_heaps : Array[WritableVirtualHeap],
path : String,
) -> UInt64 {
match virtual_heaps.search_by(heap => heap.path == path) {
Some(index) => virtual_heaps[index].address
None => 0xFFFFFFFFFFFFFFFFUL
}
}
///|
fn write_header_message(
buffer : @buffer.Buffer,
message_type : Int32,
data : Bytes,
) -> Unit raise Hdf5Error {
write_header_message_with_order(
buffer,
message_type,
data,
creation_order=false,
)
}
///|
fn write_header_message_with_order(
buffer : @buffer.Buffer,
message_type : Int32,
data : Bytes,
creation_order~ : Bool,
) -> Unit raise Hdf5Error {
guard data.length() <= 65535 else {
raise InvalidField("object header message is too large")
}
buffer.write_byte(message_type.to_byte())
buffer.write_uint16_le(data.length().to_uint16())
buffer.write_byte(b'\x00')
if creation_order {
buffer.write_uint16_le(0)
}
writer_write_bytes(buffer, data)
}
///|
fn write_header_message_with_flags(
buffer : @buffer.Buffer,
message_type : Int32,
data : Bytes,
flags : Byte,
) -> Unit raise Hdf5Error {
guard data.length() <= 65535 else {
raise InvalidField("object header message is too large")
}
buffer.write_byte(message_type.to_byte())
buffer.write_uint16_le(data.length().to_uint16())
buffer.write_byte(flags)
writer_write_bytes(buffer, data)
}
///|
fn split_object_header_messages(
payload : Bytes,
) -> Array[Bytes] raise Hdf5Error {
let messages : Array[Bytes] = []
let reader = Reader::new(payload)
while reader.remaining() > 0 {
let start = reader.position()
guard reader.remaining() >= 4 else {
raise InvalidField("incomplete writer object header message")
}
let _message_type = reader.read_u8()
let size = reader.read_u16_le().to_int()
let _flags = reader.read_u8()
reader.skip(size)
messages.push(
payload.get_view(start~, end=reader.position()).unwrap().to_owned(),
)
}
messages
}
///|
fn message_with_creation_order(message : Bytes) -> Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=message.length() + 2)
writer_write_bytes(
buffer,
message.get_view(start=0, end=4).unwrap().to_owned(),
)
buffer.write_uint16_le(0)
writer_write_bytes(
buffer,
message.get_view(start=4, end=message.length()).unwrap().to_owned(),
)
buffer.to_bytes()
}
///|
fn object_header_messages_size(messages : Array[Bytes]) -> Int32 {
let mut size = 0
for message in messages {
size = size + message.length()
}
size
}
///|
fn plan_object_header_chunks(messages : Array[Bytes]) -> Array[Array[Bytes]] {
let chunks : Array[Array[Bytes]] = []
let mut index = 0
let mut initial = true
while index < messages.length() {
let capacity = if initial { 65535 } else { 1024 * 1024 }
let mut remaining_size = 0
for position in index.. Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=offset_size + length_size)
write_uint_width(buffer, address, offset_size)
write_uint_width(buffer, length, length_size)
buffer.to_bytes()
}
///|
fn encode_object_header(
payload : Bytes,
start : UInt64,
creation_order~ : Bool = false,
offset_size~ : Int32 = 8,
length_size~ : Int32 = 8,
) -> Bytes raise Hdf5Error {
let raw_messages = split_object_header_messages(payload)
let messages = if creation_order {
raw_messages.map(message_with_creation_order)
} else {
raw_messages
}
let chunks = plan_object_header_chunks(messages)
let message_header_size = if creation_order { 6 } else { 4 }
let continuation_data_size = offset_size + length_size
let initial_message_size = object_header_messages_size(chunks[0])
let initial_continuation_size = if chunks.length() > 1 {
message_header_size + continuation_data_size
} else {
0
}
let initial_unpadded = initial_message_size + initial_continuation_size
let minimum_initial_chunk_body : Int32 = 128
let initial_padding = if initial_unpadded + message_header_size <=
minimum_initial_chunk_body {
minimum_initial_chunk_body - initial_unpadded
} else {
0
}
let chunk_bodies : Array[Int32] = []
for index in 0.. 1 {
write_header_message_with_order(
initial,
msg_object_continuation,
encode_object_continuation(
addresses[1],
(8 + chunk_bodies[1]).to_uint64(),
offset_size~,
length_size~,
),
creation_order~,
)
}
if initial_padding > 0 {
let nil_size = initial_padding - message_header_size
write_header_message_with_order(
initial,
0,
Bytes::make(nil_size, b'\x00'),
creation_order~,
)
}
guard initial.length() <= 65535 else {
raise InvalidField("object header chunk is too large")
}
guard initial.length() == chunk_bodies[0] else {
raise InvalidField("object header padding size mismatch")
}
writer_write_bytes(output, b"OHDR")
output.write_byte(b'\x02')
output.write_byte(if creation_order { b'\x0D' } else { b'\x01' })
output.write_uint16_le(initial.length().to_uint16())
writer_write_bytes(output, initial.to_bytes())
output.write_uint_le(metadata_checksum(output.to_bytes(), 0, output.length()))
for index in 1.. Array[WritableAttribute] {
self.attributes.filter(attribute => attribute.object_path == path)
}
///|
fn find_writable_object(
objects : Array[WritableObject],
path : String,
) -> WritableObject raise Hdf5Error {
match objects.search_by(object => object.path == path) {
Some(index) => objects[index]
None => raise InvalidField("writer object was not assigned an address")
}
}
///|
fn writer_reference_address(
objects : Array[WritableObject],
path : String,
) -> UInt64 raise Hdf5Error {
let canonical = canonical_writer_path(path)
match objects.search_by(object => object.path == canonical) {
Some(index) => objects[index].address
None =>
raise InvalidArgument(
"object reference target does not exist: " + canonical,
)
}
}
///|
fn region_selection_max_value(selection : Hdf5RegionSelection) -> UInt64 {
let mut maximum = 0UL
let observe = fn(value : UInt64) { if value > maximum { maximum = value } }
match selection {
RegionEmpty | RegionAll => ()
RegionPoints(points) => {
observe(points.length().to_uint64())
for point in points {
for coordinate in point {
observe(coordinate)
}
}
}
RegionHyperslab(start, stride, count, block) =>
for values in [start, stride, count, block] {
for value in values {
observe(value)
}
}
RegionBlocks(blocks) => {
observe(blocks.length().to_uint64())
for block in blocks {
for value in block.start {
observe(value)
}
for value in block.end {
observe(value)
}
}
}
}
maximum
}
///|
fn region_selection_encoding_width(selection : Hdf5RegionSelection) -> Int32 {
let maximum = region_selection_max_value(selection)
if maximum <= 0xFFFFUL {
2
} else if maximum <= 0xFFFFFFFFUL {
4
} else {
8
}
}
///|
fn encode_region_selection_body(
selection : Hdf5RegionSelection,
rank : Int32,
) -> Bytes raise Hdf5Error {
guard rank >= 0 && rank <= 32 else {
raise InvalidArgument("region selection rank is outside 0..32")
}
let buffer = @buffer.Buffer::Buffer()
match selection {
RegionEmpty | RegionAll => {
buffer.write_uint_le(if selection is RegionAll { 3U } else { 0U })
buffer.write_uint_le(1U)
buffer.write_uint_le(0U)
buffer.write_uint_le(0U)
}
RegionPoints(points) => {
guard points.length() <= 0x7FFFFFFF else {
raise InvalidArgument("region reference contains too many points")
}
for point in points {
guard point.length() == rank else {
raise InvalidArgument(
"region point rank does not match the selection",
)
}
}
let width = region_selection_encoding_width(selection)
buffer.write_uint_le(1U)
buffer.write_uint_le(2U)
buffer.write_byte(width.to_byte())
buffer.write_uint_le(rank.reinterpret_as_uint())
write_uint_width(buffer, points.length().to_uint64(), width)
for point in points {
for coordinate in point {
write_uint_width(buffer, coordinate, width)
}
}
}
RegionHyperslab(start, stride, count, block) => {
guard start.length() == rank &&
stride.length() == rank &&
count.length() == rank &&
block.length() == rank else {
raise InvalidArgument(
"region hyperslab rank does not match the selection",
)
}
let width = region_selection_encoding_width(selection)
buffer.write_uint_le(2U)
buffer.write_uint_le(3U)
buffer.write_byte(b'\x01')
buffer.write_byte(width.to_byte())
buffer.write_uint_le(rank.reinterpret_as_uint())
for axis in 0.. {
guard blocks.length() <= 0x7FFFFFFF else {
raise InvalidArgument("region reference contains too many blocks")
}
for block in blocks {
guard block.start.length() == rank && block.end.length() == rank else {
raise InvalidArgument(
"region block rank does not match the selection",
)
}
}
let width = region_selection_encoding_width(selection)
buffer.write_uint_le(2U)
buffer.write_uint_le(3U)
buffer.write_byte(b'\x00')
buffer.write_byte(width.to_byte())
buffer.write_uint_le(rank.reinterpret_as_uint())
write_uint_width(buffer, blocks.length().to_uint64(), width)
for block in blocks {
for axis in 0.. Bytes raise Hdf5Error {
validate_region_selection(dataspace, selection)
encode_region_selection_body(selection, dataspace.dimensions.length())
}
///|
fn writer_region_reference_payload(
objects : Array[WritableObject],
path : String,
selection : Hdf5RegionSelection,
version4~ : Bool = false,
reference_address_size~ : Int32 = 8,
) -> Bytes raise Hdf5Error {
let canonical = canonical_writer_path(path)
let object = match objects.search_by(item => item.path == canonical) {
Some(index) => objects[index]
None =>
raise InvalidArgument(
"region reference target does not exist: " + canonical,
)
}
guard object.kind is WritableDatasetObject(_) else {
raise InvalidArgument(
"region reference target is not a dataset: " + canonical,
)
}
let dataspace = match object.dataspace {
Some(dataspace) => dataspace
None => raise InvalidField("writer dataset has no dataspace")
}
guard dataspace.kind != Null else {
raise InvalidArgument("region reference target has a null dataspace")
}
let buffer = @buffer.Buffer::Buffer()
if version4 {
buffer.write_byte(b'\x08')
}
write_uint_width(buffer, object.address, reference_address_size)
if version4 {
let encoded_selection = encode_region_selection(dataspace, selection)
buffer.write_uint_le(encoded_selection.length().reinterpret_as_uint())
buffer.write_uint_le(dataspace.dimensions.length().reinterpret_as_uint())
writer_write_bytes(buffer, encoded_selection)
} else {
writer_write_bytes(buffer, encode_region_selection(dataspace, selection))
}
buffer.to_bytes()
}
///|
fn writer_attribute_reference_payload(
objects : Array[WritableObject],
path : String,
name : String,
reference_address_size~ : Int32 = 8,
) -> Bytes raise Hdf5Error {
let canonical = canonical_writer_path(path)
let object = match objects.search_by(item => item.path == canonical) {
Some(index) => objects[index]
None =>
raise InvalidArgument(
"attribute reference target does not exist: " + canonical,
)
}
let encoded_name = @utf8.encode(name)
guard encoded_name.length() > 0 && encoded_name.length() <= 65535 else {
raise InvalidArgument("writer attribute reference name is invalid")
}
let buffer = @buffer.Buffer::Buffer(
size_hint=3 + reference_address_size + encoded_name.length(),
)
buffer.write_byte(b'\x08')
write_uint_width(buffer, object.address, reference_address_size)
buffer.write_uint16_le(encoded_name.length().to_uint16())
writer_write_bytes(buffer, encoded_name)
buffer.to_bytes()
}
///|
fn writer_external_reference_file(
sources : Array[Hdf5ExternalSource],
custom_filters : Array[&Hdf5CustomFilter],
name : String,
) -> (String, Hdf5File) raise Hdf5Error {
let normalized = resolver_file_name(name)
let registered = match sources.search_by(item => item.name == normalized) {
Some(index) => sources[index]
None =>
raise InvalidArgument(
"external reference file is not registered: " + name,
)
}
let mut file = Hdf5File::parse_source(registered.source)
for filter in custom_filters {
file = file.with_custom_filter(filter)
}
(registered.name, file)
}
///|
fn writer_external_reference_prefix(
sources : Array[Hdf5ExternalSource],
custom_filters : Array[&Hdf5CustomFilter],
file_name : String,
path : String,
) -> (Hdf5File, Hdf5ReferenceTarget, Int32, Bytes) raise Hdf5Error {
let (_, target_file) = writer_external_reference_file(
sources, custom_filters, file_name,
)
let node = target_file.resolve(path)
let encoded_name = @utf8.encode(file_name)
guard encoded_name.length() > 0 && encoded_name.length() <= 65535 else {
raise InvalidArgument("external reference file name is invalid")
}
guard encoded_name.to_array().all(byte => byte != b'\x00') else {
raise InvalidArgument("external reference file names cannot contain NUL")
}
let offset_size = target_file.address_size()
let prefix = @buffer.Buffer::Buffer(
size_hint=1 + offset_size + 2 + encoded_name.length(),
)
prefix.write_byte(offset_size.to_byte())
write_uint_width(prefix, node.address, offset_size)
prefix.write_uint16_le(encoded_name.length().to_uint16())
writer_write_bytes(prefix, encoded_name)
(
target_file,
{ address: node.address, kind: node.kind, path: Some(node.path), },
offset_size,
prefix.to_bytes(),
)
}
///|
fn writer_external_object_reference_payload(
sources : Array[Hdf5ExternalSource],
custom_filters : Array[&Hdf5CustomFilter],
file_name : String,
path : String,
) -> Bytes raise Hdf5Error {
let (_, _, _, payload) = writer_external_reference_prefix(
sources, custom_filters, file_name, path,
)
payload
}
///|
fn writer_external_region_reference_payload(
sources : Array[Hdf5ExternalSource],
custom_filters : Array[&Hdf5CustomFilter],
file_name : String,
path : String,
selection : Hdf5RegionSelection,
) -> Bytes raise Hdf5Error {
let (target_file, target, offset_size, prefix) = writer_external_reference_prefix(
sources, custom_filters, file_name, path,
)
guard target.kind == Dataset else {
raise InvalidArgument(
"external region reference target is not a dataset: " +
file_name +
":" +
path,
)
}
let metadata = read_dataset_metadata(
target_file.object_header_at(target.address),
offset_size,
target_file.superblock.length_size,
)
guard metadata.dataspace.kind != Null else {
raise InvalidArgument(
"external region reference target has a null dataspace",
)
}
let encoded_selection = encode_region_selection(metadata.dataspace, selection)
guard encoded_selection.length() <= 0x7FFFFFFF else {
raise UnsupportedFeature("external region reference selection is too large")
}
let buffer = @buffer.Buffer::Buffer()
writer_write_bytes(buffer, prefix)
buffer.write_uint_le(encoded_selection.length().reinterpret_as_uint())
buffer.write_uint_le(
metadata.dataspace.dimensions.length().reinterpret_as_uint(),
)
writer_write_bytes(buffer, encoded_selection)
buffer.to_bytes()
}
///|
fn writer_external_attribute_reference_payload(
sources : Array[Hdf5ExternalSource],
custom_filters : Array[&Hdf5CustomFilter],
file_name : String,
path : String,
name : String,
) -> Bytes raise Hdf5Error {
let (target_file, _, _, prefix) = writer_external_reference_prefix(
sources, custom_filters, file_name, path,
)
let encoded_name = @utf8.encode(name)
guard encoded_name.length() > 0 && encoded_name.length() <= 65535 else {
raise InvalidArgument("external attribute reference name is invalid")
}
guard target_file
.attributes(path)
.search_by(attribute => attribute.name == name)
is Some(_) else {
raise InvalidArgument(
"external attribute reference attribute does not exist: " +
file_name +
":" +
path +
"." +
name,
)
}
let buffer = @buffer.Buffer::Buffer()
writer_write_bytes(buffer, prefix)
buffer.write_uint16_le(encoded_name.length().to_uint16())
writer_write_bytes(buffer, encoded_name)
buffer.to_bytes()
}
///|
fn Hdf5Writer::object_payload(
self : Hdf5Writer,
object : WritableObject,
objects : Array[WritableObject],
heap : WritableGlobalHeap,
dense : Array[WritableDenseMetadata],
external : Array[WritableExternalHeap],
virtual_heaps : Array[WritableVirtualHeap],
aux_addresses_final : Bool,
shared~ : Array[WritableSharedAttribute] = [],
fills~ : Array[WritableSharedFill] = [],
shared_filters~ : Array[WritableSharedFilter] = [],
) -> Bytes raise Hdf5Error {
let buffer = @buffer.Buffer::Buffer()
match object.kind {
WritableGroup => {
let entries = self.dense_link_entries(object, objects, track_order=true)
if writer_dense_entries_fit(entries, kind=WritableDenseLinks) {
let (heap_address, index_address, order_index_address) = writer_dense_addresses(
dense,
object.path,
WritableDenseLinks,
)
write_header_message(
buffer,
msg_link_info,
encode_dense_storage_info(
true,
writer_dense_max_creation_order(entries),
heap_address,
index_address,
order_index_address,
),
)
} else {
write_header_message(buffer, msg_link_info, encode_link_info())
let compact_entries = self.dense_link_entries(
object,
objects,
track_order=false,
)
for entry in compact_entries {
write_header_message(buffer, msg_link, entry.data)
}
}
}
WritableCommittedDatatype(index) =>
write_header_message(
buffer,
msg_datatype,
encode_writer_datatype(self.committed_datatypes[index].datatype),
)
WritableDatasetObject(index) => {
let dataset = self.datasets[index]
let encoded = encode_write_values(dataset.values, heap~, objects~)
write_header_message(
buffer,
msg_dataspace,
match dataset.storage {
WritableExtensible(_, _, _) =>
encode_dataspace(
dataset.dimensions,
maximum_dimensions=dataset.maximum_dimensions,
)
WritableVirtual(_) =>
if virtual_dataset_needs_extended_dataspace(
dataset.dimensions,
dataset.maximum_dimensions,
) {
encode_dataspace(
dataset.dimensions,
maximum_dimensions=dataset.maximum_dimensions,
)
} else {
encode_dataspace(dataset.dimensions)
}
_ => encode_dataspace(dataset.dimensions)
},
)
match self.committed_datatype_address(encoded.datatype, objects) {
Some(address) =>
write_header_message_with_flags(
buffer,
msg_datatype,
encode_writer_committed_message(address),
shared_message_flag.to_byte(),
)
None => write_header_message(buffer, msg_datatype, encoded.datatype)
}
match dataset.fill_value {
Some(fill) =>
match writer_shared_fill(fills, object.path) {
Some(shared_fill) =>
write_header_message_with_flags(
buffer,
msg_fill_value,
encode_writer_shared_message(shared_fill.id),
shared_message_flag.to_byte(),
)
None =>
write_header_message(
buffer,
msg_fill_value,
encode_fill_value(fill, heap~, objects~),
)
}
None => ()
}
let filter_fill = match dataset.fill_value {
Some(fill) => encode_write_values(fill, heap~, objects~).payload
None => Bytes::make(encoded.element_size, b'\x00')
}
match dataset.storage {
WritableCompact => {
guard encoded.payload.length() <= 65524 else {
raise InvalidField("compact writer dataset exceeds 65524 bytes")
}
write_header_message(
buffer,
msg_data_layout,
encode_compact_layout(encoded.payload),
)
}
WritableContiguous => {
let address = if encoded.payload.length() == 0 {
0xFFFFFFFFFFFFFFFFUL
} else {
object.raw_address
}
write_header_message(
buffer,
msg_data_layout,
encode_contiguous_layout(address, encoded.payload.length()),
)
}
WritableExternal(slots) => {
let heap_address = if aux_addresses_final {
let address = writer_external_heap_address(external, object.path)
guard address != 0xFFFFFFFFFFFFFFFFUL else {
raise InvalidField("external data heap was not assigned")
}
address
} else {
0UL
}
write_header_message(
buffer,
msg_external_data_files,
encode_external_data_message(slots, heap_address),
)
write_header_message(
buffer,
msg_data_layout,
encode_external_data_layout(external_slot_total(slots)),
)
}
WritableVirtual(_) => {
let heap_address = if aux_addresses_final {
let address = writer_virtual_heap_address(
virtual_heaps,
object.path,
)
guard address != 0xFFFFFFFFFFFFFFFFUL else {
raise InvalidField("virtual mapping heap was not assigned")
}
address
} else {
0UL
}
write_header_message(
buffer,
msg_data_layout,
encode_virtual_layout(heap_address, 1),
)
}
WritableChunked(chunk_dimensions, filters) => {
if filters.length() > 0 {
match writer_shared_filter(shared_filters, object.path) {
Some(shared_filter) =>
write_header_message_with_flags(
buffer,
msg_filter_pipeline,
encode_writer_shared_message(shared_filter.id),
shared_message_flag.to_byte(),
)
None =>
write_header_message(
buffer,
msg_filter_pipeline,
encode_filter_pipeline(
filters,
encoded.element_size,
writer_chunk_element_count(chunk_dimensions),
dataset.values,
filter_fill,
self.custom_filters,
),
)
}
}
write_header_message(
buffer,
msg_data_layout,
encode_chunked_layout(
object.raw_address,
chunk_dimensions,
encoded.element_size,
),
)
}
WritableExtensible(chunk_dimensions, filters, unlimited_axes) => {
if filters.length() > 0 {
match writer_shared_filter(shared_filters, object.path) {
Some(shared_filter) =>
write_header_message_with_flags(
buffer,
msg_filter_pipeline,
encode_writer_shared_message(shared_filter.id),
shared_message_flag.to_byte(),
)
None =>
write_header_message(
buffer,
msg_filter_pipeline,
encode_filter_pipeline(
filters,
encoded.element_size,
writer_chunk_element_count(chunk_dimensions),
dataset.values,
filter_fill,
self.custom_filters,
),
)
}
}
write_header_message(
buffer,
msg_data_layout,
encode_chunked_layout(
object.raw_address,
chunk_dimensions,
encoded.element_size,
extensible=unlimited_axes.length() == 1,
btree_v2=unlimited_axes.length() > 1,
),
)
}
}
}
}
let attributes = self.object_attributes(object.path)
if attributes.length() > 0 {
let entries = self.dense_attribute_entries(
object.path,
heap,
objects,
shared~,
)
if writer_dense_entries_fit(entries, kind=WritableDenseAttributes) {
let (heap_address, index_address, order_index_address) = writer_dense_addresses(
dense,
object.path,
WritableDenseAttributes,
)
write_header_message(
buffer,
msg_attribute_info,
encode_dense_storage_info(
false,
writer_dense_max_creation_order(entries),
heap_address,
index_address,
order_index_address,
),
)
} else {
write_header_message(
buffer,
msg_attribute_info,
encode_compact_attribute_info(),
)
for entry in entries {
write_header_message_with_flags(
buffer,
msg_attribute,
entry.data,
entry.flags,
)
}
}
}
buffer.to_bytes()
}
///|
fn initial_writable_objects(writer : Hdf5Writer) -> Array[WritableObject] {
let objects : Array[WritableObject] = []
for group in writer.groups {
objects.push({
path: group,
kind: WritableGroup,
address: 0UL,
raw_address: 0UL,
dataspace: None,
})
}
for index in 0.. Unit raise Hdf5Error {
match values {
VariableStringValues(items) =>
for item in items {
if item != "" && !output.contains(item) {
guard output.length() < 65535 else {
raise InvalidField("writer global heap has too many objects")
}
output.push(item)
}
}
_ => ()
}
}
///|
fn collect_heap_structured_strings(
output : Array[String],
datatype : Datatype,
value : DatasetValue,
) -> Unit raise Hdf5Error {
match datatype.class {
6 =>
match value {
Compound(fields) =>
for index in 0.. ()
}
9 =>
if datatype.vlen_is_string {
match value {
Text(text) =>
if text != "" && !output.contains(text) {
guard output.length() < 65535 else {
raise InvalidField("writer global heap has too many objects")
}
output.push(text)
}
_ => ()
}
} else {
match value {
Sequence(items) =>
for item in items {
collect_heap_structured_strings(
output,
datatype.base_types[0],
item,
)
}
_ => ()
}
}
10 =>
match value {
Elements(items) =>
for item in items {
collect_heap_structured_strings(
output,
datatype.base_types[0],
item,
)
}
_ => ()
}
_ => ()
}
}
///|
fn collect_heap_sequences(
output : Array[Bytes],
datatype : Datatype,
value : DatasetValue,
heap~ : WritableGlobalHeap,
objects~ : Array[WritableObject],
reference_address_size~ : Int32 = 8,
vlen_address_size~ : Int32 = 0,
) -> Unit raise Hdf5Error {
match datatype.class {
6 =>
match value {
Compound(fields) =>
for index in 0.. ()
}
9 =>
if !datatype.vlen_is_string {
match value {
Sequence(items) =>
if items.length() > 0 {
let base = datatype.base_types[0]
let encoded = @buffer.Buffer::Buffer()
for item in items {
writer_write_bytes(
encoded,
encode_structured_value(
base,
item,
heap~,
objects~,
reference_address_size~,
vlen_address_size~,
),
)
}
let data = encoded.to_bytes()
if !output.contains(data) {
guard output.length() < 65535 else {
raise InvalidField("writer global heap has too many objects")
}
output.push(data)
}
}
_ => ()
}
}
10 =>
match value {
Elements(items) =>
for item in items {
collect_heap_sequences(
output,
datatype.base_types[0],
item,
heap~,
objects~,
reference_address_size~,
vlen_address_size~,
)
}
_ => ()
}
_ => ()
}
}
///|
fn collect_heap_regions(
output : Array[Bytes],
datatype : Datatype,
value : DatasetValue,
objects : Array[WritableObject],
attributes : Array[WritableAttribute],
reference_address_size~ : Int32 = 8,
external_sources~ : Array[Hdf5ExternalSource] = [],
custom_filters~ : Array[&Hdf5CustomFilter] = [],
external_references~ : Array[WritableExternalReference] = [],
) -> Unit raise Hdf5Error {
match datatype.class {
6 =>
match value {
Compound(fields) =>
for index in 0.. ()
}
7 =>
if datatype.reference_kind == 1 || datatype.reference_kind == 3 {
match value {
RegionReference(Some(target), selection) => {
let payload = writer_region_reference_payload(
objects,
target,
selection,
version4=datatype.reference_kind == 3,
reference_address_size~,
)
if !output.contains(payload) {
guard output.length() < 65535 else {
raise InvalidField("writer global heap has too many objects")
}
output.push(payload)
}
}
ExternalRegionReference(file_name, path, selection) if datatype.reference_kind ==
3 => {
let payload = writer_external_region_reference_payload(
external_sources, custom_filters, file_name, path, selection,
)
if !output.contains(payload) {
guard output.length() < 65535 else {
raise InvalidField("writer global heap has too many objects")
}
output.push(payload)
}
if external_references.search_by(item => item.value == value)
is None {
external_references.push({ value, payload, })
}
}
_ => ()
}
} else if datatype.reference_kind == 4 {
match value {
AttributeReference(Some(target), name) => {
let canonical = canonical_writer_path(target)
guard attributes.search_by(attribute => {
attribute.object_path == canonical && attribute.name == name
})
is Some(_) else {
raise InvalidArgument(
"attribute reference attribute does not exist: " +
canonical +
"." +
name,
)
}
let payload = writer_attribute_reference_payload(
objects,
canonical,
name,
reference_address_size~,
)
if !output.contains(payload) {
guard output.length() < 65535 else {
raise InvalidField("writer global heap has too many objects")
}
output.push(payload)
}
}
ExternalAttributeReference(file_name, path, name) => {
let payload = writer_external_attribute_reference_payload(
external_sources, custom_filters, file_name, path, name,
)
if !output.contains(payload) {
guard output.length() < 65535 else {
raise InvalidField("writer global heap has too many objects")
}
output.push(payload)
}
if external_references.search_by(item => item.value == value)
is None {
external_references.push({ value, payload, })
}
}
_ => ()
}
} else if datatype.reference_kind == 2 {
match value {
ExternalObjectReference(file_name, path) => {
let payload = writer_external_object_reference_payload(
external_sources, custom_filters, file_name, path,
)
if !output.contains(payload) {
guard output.length() < 65535 else {
raise InvalidField("writer global heap has too many objects")
}
output.push(payload)
}
if external_references.search_by(item => item.value == value)
is None {
external_references.push({ value, payload, })
}
}
_ => ()
}
}
9 =>
match value {
Sequence(items) =>
for item in items {
collect_heap_regions(
output,
datatype.base_types[0],
item,
objects,
attributes,
reference_address_size~,
external_sources~,
custom_filters~,
external_references~,
)
}
_ => ()
}
10 =>
match value {
Elements(items) =>
for item in items {
collect_heap_regions(
output,
datatype.base_types[0],
item,
objects,
attributes,
reference_address_size~,
external_sources~,
custom_filters~,
external_references~,
)
}
_ => ()
}
_ => ()
}
}
///|
fn Hdf5Writer::global_heap_strings(
self : Hdf5Writer,
) -> Array[String] raise Hdf5Error {
let strings : Array[String] = []
for dataset in self.datasets {
collect_heap_strings(strings, dataset.values)
match dataset.values {
StructuredValues(datatype, items) =>
for item in items {
collect_heap_structured_strings(strings, datatype, item)
}
_ => ()
}
match dataset.fill_value {
Some(fill) => {
collect_heap_strings(strings, fill)
match fill {
StructuredValues(datatype, items) =>
for item in items {
collect_heap_structured_strings(strings, datatype, item)
}
_ => ()
}
}
None => ()
}
}
for attribute in self.attributes {
collect_heap_strings(strings, attribute.values)
match attribute.values {
StructuredValues(datatype, items) =>
for item in items {
collect_heap_structured_strings(strings, datatype, item)
}
_ => ()
}
}
strings
}
///|
fn Hdf5Writer::global_heap_sequences(
self : Hdf5Writer,
heap~ : WritableGlobalHeap,
objects~ : Array[WritableObject],
) -> Array[Bytes] raise Hdf5Error {
let sequences : Array[Bytes] = []
for dataset in self.datasets {
match dataset.values {
StructuredValues(datatype, items) =>
for item in items {
collect_heap_sequences(sequences, datatype, item, heap~, objects~)
}
_ => ()
}
match dataset.fill_value {
Some(StructuredValues(datatype, items)) =>
for item in items {
collect_heap_sequences(sequences, datatype, item, heap~, objects~)
}
_ => ()
}
}
for attribute in self.attributes {
match attribute.values {
StructuredValues(datatype, items) =>
for item in items {
collect_heap_sequences(sequences, datatype, item, heap~, objects~)
}
_ => ()
}
}
sequences
}
///|
fn Hdf5Writer::global_heap_regions(
self : Hdf5Writer,
objects~ : Array[WritableObject],
) -> (Array[Bytes], Array[WritableExternalReference]) raise Hdf5Error {
let regions : Array[Bytes] = []
let external_references : Array[WritableExternalReference] = []
for dataset in self.datasets {
match dataset.values {
StructuredValues(datatype, items) =>
for item in items {
collect_heap_regions(
regions,
datatype,
item,
objects,
self.attributes,
external_sources=self.external_sources,
custom_filters=self.custom_filters,
external_references~,
)
}
_ => ()
}
match dataset.fill_value {
Some(StructuredValues(datatype, items)) =>
for item in items {
collect_heap_regions(
regions,
datatype,
item,
objects,
self.attributes,
external_sources=self.external_sources,
custom_filters=self.custom_filters,
external_references~,
)
}
_ => ()
}
}
for attribute in self.attributes {
match attribute.values {
StructuredValues(datatype, items) =>
for item in items {
collect_heap_regions(
regions,
datatype,
item,
objects,
self.attributes,
external_sources=self.external_sources,
custom_filters=self.custom_filters,
external_references~,
)
}
_ => ()
}
}
(regions, external_references)
}
///|
fn encode_heap_objects(
objects : Array[Bytes],
length_size~ : Int32 = 8,
) -> Bytes raise Hdf5Error {
guard length_size == 4 || length_size == 8 else {
raise InvalidField("global heap length size must be 4 or 8 bytes")
}
let object_count = objects.length()
if object_count == 0 {
return b""
}
guard object_count <= 65535 else {
raise InvalidField("writer global heap has too many objects")
}
let mut used_size = 16
for item in objects {
let length = item.length()
guard length <= 0x7FFFFFF8 else {
raise InvalidField("writer global heap object is too large")
}
guard used_size <= 0x7FFFFFFF - 16 - round_up_to_eight(length) else {
raise InvalidField("writer global heap is too large")
}
used_size = used_size + 16 + round_up_to_eight(length)
}
let collection_size_64 = (used_size.to_int64() + 16L + 4095L) / 4096L * 4096L
guard collection_size_64 <= 0x7FFFFFFFL else {
raise InvalidField("writer global heap is too large")
}
let collection_size = collection_size_64.to_int()
let buffer = @buffer.Buffer::Buffer(size_hint=collection_size)
writer_write_bytes(buffer, b"GCOL")
buffer.write_byte(b'\x01')
writer_write_zeros(buffer, 3)
write_uint_width(buffer, collection_size.to_uint64(), length_size)
let mut object_index = 0
for item in objects {
object_index = object_index + 1
buffer.write_uint16_le(object_index.to_uint16())
buffer.write_uint16_le(0)
buffer.write_uint_le(0U)
write_uint_width(buffer, item.length().to_uint64(), length_size)
writer_write_bytes(buffer, item)
writer_write_zeros(buffer, round_up_to_eight(item.length()) - item.length())
}
buffer.write_uint16_le(0)
buffer.write_uint16_le(0)
buffer.write_uint_le(0U)
write_uint_width(
buffer,
(collection_size - used_size).to_uint64(),
length_size,
)
writer_write_zeros(buffer, collection_size - buffer.length())
buffer.to_bytes()
}
///|
fn encode_global_heap(
heap : WritableGlobalHeap,
length_size~ : Int32 = 8,
) -> Bytes raise Hdf5Error {
let objects : Array[Bytes] = []
for item in heap.strings {
objects.push(@utf8.encode(item))
}
for item in heap.sequences {
objects.push(item)
}
encode_heap_objects(objects, length_size~)
}
///|
fn encode_region_heap(
heap : WritableGlobalHeap,
length_size~ : Int32 = 8,
) -> Bytes raise Hdf5Error {
encode_heap_objects(heap.regions, length_size~)
}
///|
fn assign_writable_addresses(
writer : Hdf5Writer,
templates : Array[WritableObject],
) -> WritableLayout raise Hdf5Error {
let empty_heap = empty_writable_heap()
let shared_candidates = writer.shared_attribute_candidates(
templates, empty_heap,
)
let shared_fills = writer.shared_fill_candidates(templates, empty_heap)
let shared_filters = writer.shared_filter_candidates(templates, empty_heap)
let shared_message_types = (if shared_candidates.length() > 0 {
1 << 0x0C
} else {
0
}) |
(if shared_fills.length() > 0 { 1 << 5 } else { 0 }) |
(if shared_filters.length() > 0 { 1 << 0x0B } else { 0 })
let has_shared_messages = shared_candidates.length() > 0 ||
shared_fills.length() > 0 ||
shared_filters.length() > 0
let lengths : Array[Int32] = []
for object in templates {
lengths.push(
encode_object_header(
writer.object_payload(
object,
templates,
empty_heap,
[],
[],
[],
false,
shared=shared_candidates,
fills=shared_fills,
shared_filters~,
),
0UL,
creation_order=writer_dense_entries_fit(
writer.dense_attribute_entries(
object.path,
empty_heap,
templates,
shared=shared_candidates,
),
kind=WritableDenseAttributes,
),
).length(),
)
}
let objects : Array[WritableObject] = []
let mut offset = if has_shared_messages { 88 } else { 48 }
for index in 0.. 0 {
offset.to_uint64()
} else {
0UL
}
if regions.length() > 0 {
offset = writer_align(offset + encode_heap_objects(regions).length())
}
let base_heap = {
address: if strings.length() > 0 {
offset.to_uint64()
} else {
0UL
},
region_address,
strings,
regions,
sequences: [],
external_references,
}
let sequences = writer.global_heap_sequences(heap=base_heap, objects~)
let heap = {
address: if base_heap.address == 0UL && sequences.length() > 0 {
offset.to_uint64()
} else {
base_heap.address
},
region_address,
strings,
regions,
sequences,
external_references,
}
if heap.strings.length() > 0 || heap.sequences.length() > 0 {
offset = writer_align(offset + encode_global_heap(heap).length())
}
let dense : Array[WritableDenseMetadata] = []
for object in objects {
let candidates : Array[(WritableDenseKind, Array[WritableDenseHeapEntry])] = []
match object.kind {
WritableGroup =>
candidates.push(
(
WritableDenseLinks,
writer.dense_link_entries(object, objects, track_order=true),
),
)
WritableDatasetObject(_) | WritableCommittedDatatype(_) => ()
}
candidates.push(
(
WritableDenseAttributes,
writer.dense_attribute_entries(
object.path,
heap,
objects,
shared=shared_candidates,
),
),
)
for (kind, dense_entries) in candidates {
if writer_dense_entries_fit(dense_entries, kind~) {
let (placeholder_heap, placeholder_index, placeholder_order_index) = writer.encode_dense_metadata(
object,
objects,
heap,
0UL,
kind,
shared=shared_candidates,
)
let heap_address = offset.to_uint64()
let index_address = heap_address + placeholder_heap.length().to_uint64()
dense.push({
path: object.path,
kind,
heap_address,
index_address,
order_index_address: index_address +
placeholder_index.length().to_uint64(),
})
offset = writer_align(
offset +
placeholder_heap.length() +
placeholder_index.length() +
placeholder_order_index.length(),
)
}
}
}
let external : Array[WritableExternalHeap] = []
for object in objects {
match object.kind {
WritableDatasetObject(dataset_index) => {
let dataset = writer.datasets[dataset_index]
match dataset.storage {
WritableExternal(slots) => {
let address = offset.to_uint64()
external.push({ path: object.path, address, })
offset = writer_align(
offset + encode_external_local_heap(slots, address).length(),
)
}
_ => ()
}
}
WritableGroup | WritableCommittedDatatype(_) => ()
}
}
let virtual_heaps : Array[WritableVirtualHeap] = []
for object in objects {
match object.kind {
WritableDatasetObject(dataset_index) => {
let dataset = writer.datasets[dataset_index]
match dataset.storage {
WritableVirtual(mappings) => {
let address = offset.to_uint64()
virtual_heaps.push({ path: object.path, address, })
offset = writer_align(
offset +
encode_heap_objects([
encode_virtual_mapping_block(
dataset.dimensions,
dataset.maximum_dimensions,
mappings,
8,
),
]).length(),
)
}
_ => ()
}
}
WritableGroup | WritableCommittedDatatype(_) => ()
}
}
for index in 0.. {
let dataset = writer.datasets[dataset_index]
let has_storage = match dataset.storage {
WritableCompact => false
WritableContiguous =>
encode_write_values(dataset.values, objects~).payload.length() > 0
WritableChunked(_, _) => true
WritableExtensible(_, _, _) => true
WritableExternal(_) => false
WritableVirtual(_) => false
}
if has_storage {
objects[index] = {
path: objects[index].path,
kind: objects[index].kind,
address: objects[index].address,
raw_address: offset.to_uint64(),
dataspace: objects[index].dataspace,
}
offset = match dataset.storage {
WritableCompact => offset
WritableContiguous =>
writer_align(
offset +
encode_write_values(dataset.values, heap~, objects~).payload.length(),
)
WritableChunked(_, _) =>
writer_align(
offset +
encode_fixed_array_storage(
offset.to_uint64(),
dataset,
heap,
objects,
writer.custom_filters,
).length(),
)
WritableExtensible(_, _, unlimited_axes) => {
let payload_length = if unlimited_axes.length() == 1 {
encode_extensible_array_storage(
offset.to_uint64(),
dataset,
heap,
objects,
writer.custom_filters,
).length()
} else {
encode_btree_v2_chunk_storage(
offset.to_uint64(),
dataset,
heap,
objects,
writer.custom_filters,
).length()
}
writer_align(offset + payload_length)
}
WritableExternal(_) => offset
WritableVirtual(_) => offset
}
}
}
WritableGroup | WritableCommittedDatatype(_) => ()
}
}
let shared = if has_shared_messages {
let table_address = offset.to_uint64()
let attributes : Array[WritableSharedAttribute] = []
let attribute_messages : Array[Bytes] = []
let attribute_message_indices : Array[Int32] = []
for candidate in shared_candidates {
let attribute = match
writer
.object_attributes(candidate.path)
.search_by(attribute => attribute.name == candidate.name) {
Some(index) => writer.object_attributes(candidate.path)[index]
None => raise InvalidField("writer shared attribute disappeared")
}
let data = writer.encode_attribute(attribute, heap, objects)
guard data.length() <= writer_shared_max_managed_size else {
raise InvalidField("writer shared attribute changed size")
}
let message_index = match
attribute_messages.search_by(message => message == data) {
Some(index) => index
None => {
attribute_messages.push(data)
attribute_messages.length() - 1
}
}
attributes.push({
path: candidate.path,
name: candidate.name,
data,
id: b"",
})
attribute_message_indices.push(message_index)
}
let fills : Array[WritableSharedFill] = []
let fill_messages : Array[Bytes] = []
let fill_message_indices : Array[Int32] = []
for candidate in shared_fills {
let dataset = writer.datasets[writer.writer_dataset_index(candidate.path)]
let fill = match dataset.fill_value {
Some(fill) => fill
None => raise InvalidField("writer shared fill disappeared")
}
let data = encode_fill_value(fill, heap~, objects~)
guard data.length() <= writer_shared_max_managed_size else {
raise InvalidField("writer shared fill changed size")
}
let message_index = match
fill_messages.search_by(message => message == data) {
Some(index) => index
None => {
fill_messages.push(data)
fill_messages.length() - 1
}
}
fills.push({ path: candidate.path, data, id: b"", })
fill_message_indices.push(message_index)
}
let filters : Array[WritableSharedFilter] = []
let filter_messages : Array[Bytes] = []
let filter_message_indices : Array[Int32] = []
for candidate in shared_filters {
let dataset = writer.datasets[writer.writer_dataset_index(candidate.path)]
let (chunk_dimensions, pipeline) = match dataset.storage {
WritableChunked(dimensions, filters) => (dimensions, filters)
WritableExtensible(dimensions, filters, _) => (dimensions, filters)
WritableCompact
| WritableContiguous
| WritableExternal(_)
| WritableVirtual(_) =>
raise InvalidField("writer shared filter disappeared")
}
guard pipeline.length() > 0 else {
raise InvalidField("writer shared filter disappeared")
}
let encoded = encode_write_values(dataset.values, heap~, objects~)
let fill = match dataset.fill_value {
Some(fill) => encode_write_values(fill, heap~, objects~).payload
None => Bytes::make(encoded.element_size, b'\x00')
}
let data = encode_filter_pipeline(
pipeline,
encoded.element_size,
writer_chunk_element_count(chunk_dimensions),
dataset.values,
fill,
writer.custom_filters,
)
guard data.length() <= writer_shared_max_managed_size else {
raise InvalidField("writer shared filter changed size")
}
let message_index = match
filter_messages.search_by(message => message == data) {
Some(index) => index
None => {
filter_messages.push(data)
filter_messages.length() - 1
}
}
filters.push({ path: candidate.path, data, id: b"", })
filter_message_indices.push(message_index)
}
guard fill_messages.length() <= 65535 &&
filter_messages.length() <= 65535 &&
attribute_messages.length() <= 65535 else {
raise InvalidArgument("writer shared-message index has too many entries")
}
let index_count = (if fill_messages.length() > 0 { 1 } else { 0 }) +
(if filter_messages.length() > 0 { 1 } else { 0 }) +
(if attribute_messages.length() > 0 { 1 } else { 0 })
let mut next_address = writer_align(
table_address.to_int() + writer_shared_table_size(index_count),
)
let heaps : Array[WritableSharedHeap] = []
let mut fill_ids : Array[Bytes] = []
if fill_messages.length() > 0 {
let heap_address = next_address.to_uint64()
let (heap_bytes, ids) = encode_writer_shared_heap(
heap_address, fill_messages,
)
fill_ids = ids
heaps.push({
message_types: 1 << 5,
address: heap_address,
messages: fill_messages,
})
next_address = writer_align(next_address + heap_bytes.length())
}
let mut filter_ids : Array[Bytes] = []
if filter_messages.length() > 0 {
let heap_address = next_address.to_uint64()
let (heap_bytes, ids) = encode_writer_shared_heap(
heap_address, filter_messages,
)
filter_ids = ids
heaps.push({
message_types: 1 << 0x0B,
address: heap_address,
messages: filter_messages,
})
next_address = writer_align(next_address + heap_bytes.length())
}
let mut attribute_ids : Array[Bytes] = []
if attribute_messages.length() > 0 {
let heap_address = next_address.to_uint64()
let (heap_bytes, ids) = encode_writer_shared_heap(
heap_address, attribute_messages,
)
attribute_ids = ids
heaps.push({
message_types: 1 << 0x0C,
address: heap_address,
messages: attribute_messages,
})
next_address = writer_align(next_address + heap_bytes.length())
}
let resolved_fills : Array[WritableSharedFill] = []
for index in 0.. Bytes {
let buffer = @buffer.Buffer::Buffer(size_hint=48)
writer_write_bytes(buffer, b"\x89HDF\r\n\x1a\n")
buffer.write_byte(b'\x03')
buffer.write_byte(b'\x08')
buffer.write_byte(b'\x08')
buffer.write_byte(b'\x00')
buffer.write_uint64_le(base_address)
buffer.write_uint64_le(extension_address)
buffer.write_uint64_le(eof)
buffer.write_uint64_le(root_address)
buffer.write_uint_le(metadata_checksum(buffer.to_bytes(), 0, buffer.length()))
buffer.to_bytes()
}
///|
/// Encodes the builder as a complete HDF5 file.
pub fn Hdf5Writer::encode(self : Hdf5Writer) -> Bytes raise Hdf5Error {
let layout = assign_writable_addresses(self, initial_writable_objects(self))
let objects = layout.objects
let shared_attributes = match layout.shared {
Some(shared) => shared.attributes
None => []
}
let shared_fills = match layout.shared {
Some(shared) => shared.fills
None => []
}
let shared_filters = match layout.shared {
Some(shared) => shared.filters
None => []
}
let encoded_headers : Array[Bytes] = []
let mut eof = 48
for object in objects {
let creation_order = layout.dense.search_by(metadata => {
metadata.path == object.path && metadata.kind == WritableDenseAttributes
})
is Some(_)
let header = encode_object_header(
self.object_payload(
object,
objects,
layout.heap,
layout.dense,
layout.external,
layout.virtual_heaps,
true,
shared=shared_attributes,
fills=shared_fills,
shared_filters~,
),
object.address,
creation_order~,
)
encoded_headers.push(header)
eof = writer_align(object.address.to_int() + header.length())
}
let region_heap_payload = encode_region_heap(layout.heap)
if region_heap_payload.length() > 0 {
eof = writer_align(
layout.heap.region_address.to_int() + region_heap_payload.length(),
)
}
let heap_payload = encode_global_heap(layout.heap)
if heap_payload.length() > 0 {
eof = writer_align(layout.heap.address.to_int() + heap_payload.length())
}
let dense_payloads : Array[(UInt64, Bytes)] = []
for metadata in layout.dense {
let object = find_writable_object(objects, metadata.path)
let (heap_bytes, index_bytes, order_index_bytes) = self.encode_dense_metadata(
object,
objects,
layout.heap,
metadata.heap_address,
metadata.kind,
shared=shared_attributes,
)
guard heap_bytes.length().to_uint64() ==
metadata.index_address - metadata.heap_address else {
raise InvalidField("dense metadata address changed during encoding")
}
guard index_bytes.length().to_uint64() ==
metadata.order_index_address - metadata.index_address else {
raise InvalidField("dense order index address changed during encoding")
}
dense_payloads.push((metadata.heap_address, heap_bytes))
dense_payloads.push((metadata.index_address, index_bytes))
dense_payloads.push((metadata.order_index_address, order_index_bytes))
eof = writer_align(
metadata.order_index_address.to_int() + order_index_bytes.length(),
)
}
let external_payloads : Array[(UInt64, Bytes)] = []
for metadata in layout.external {
let dataset = self.datasets[self.writer_dataset_index(metadata.path)]
let slots = match dataset.storage {
WritableExternal(slots) => slots
_ => raise InvalidField("external data heap has the wrong dataset kind")
}
let payload = encode_external_local_heap(slots, metadata.address)
external_payloads.push((metadata.address, payload))
eof = writer_align(metadata.address.to_int() + payload.length())
}
let virtual_payloads : Array[(UInt64, Bytes)] = []
for metadata in layout.virtual_heaps {
let dataset = self.datasets[self.writer_dataset_index(metadata.path)]
let mappings = match dataset.storage {
WritableVirtual(mappings) => mappings
_ => raise InvalidField("virtual mapping heap has the wrong dataset kind")
}
let payload = encode_heap_objects([
encode_virtual_mapping_block(
dataset.dimensions,
dataset.maximum_dimensions,
mappings,
8,
),
])
virtual_payloads.push((metadata.address, payload))
eof = writer_align(metadata.address.to_int() + payload.length())
}
let raw_payloads : Array[(UInt64, Bytes)] = []
for object in objects {
match object.kind {
WritableDatasetObject(index) => {
let dataset = self.datasets[index]
let payload = match dataset.storage {
WritableCompact => b""
WritableContiguous =>
encode_write_values(dataset.values, heap=layout.heap, objects~).payload
WritableChunked(_, _) =>
encode_fixed_array_storage(
object.raw_address,
dataset,
layout.heap,
objects,
self.custom_filters,
)
WritableExtensible(_, _, unlimited_axes) =>
if unlimited_axes.length() == 1 {
encode_extensible_array_storage(
object.raw_address,
dataset,
layout.heap,
objects,
self.custom_filters,
)
} else {
encode_btree_v2_chunk_storage(
object.raw_address,
dataset,
layout.heap,
objects,
self.custom_filters,
)
}
WritableExternal(_) => b""
WritableVirtual(_) => b""
}
if payload.length() > 0 {
raw_payloads.push((object.raw_address, payload))
eof = writer_align(object.raw_address.to_int() + payload.length())
}
}
WritableGroup | WritableCommittedDatatype(_) => ()
}
}
let shared_payloads : Array[(UInt64, Bytes)] = []
if layout.shared is Some(shared) {
let mut message_count = 0
let mut message_types = 0
for heap in shared.heaps {
message_count = message_count + heap.messages.length()
message_types = message_types | heap.message_types
}
guard message_count == shared.message_count &&
message_types == shared.message_types else {
raise InvalidField("shared-message count changed during encoding")
}
let table = encode_writer_shared_table(shared.heaps)
guard table.length().to_uint64() <=
shared.heaps[0].address - shared.table_address else {
raise InvalidField("shared-message table overlaps its first heap")
}
shared_payloads.push((shared.table_address, table))
for heap in shared.heaps {
let (heap_bytes, ids) = encode_writer_shared_heap(
heap.address,
heap.messages,
)
guard ids.length() == heap.messages.length() else {
raise InvalidField(
"shared-message heap ID count changed during encoding",
)
}
for index in 0.. fill.data == message) {
Some(fill_index) => shared.fills[fill_index].id
None => raise InvalidField("shared fill message disappeared")
}
} else if heap.message_types == 1 << 0x0B {
match shared.filters.search_by(filter => filter.data == message) {
Some(filter_index) => shared.filters[filter_index].id
None => raise InvalidField("shared filter message disappeared")
}
} else {
match
shared.attributes.search_by(attribute => attribute.data == message) {
Some(attribute_index) => shared.attributes[attribute_index].id
None => raise InvalidField("shared attribute message disappeared")
}
}
guard ids[index] == expected else {
raise InvalidField("shared-message heap ID changed during encoding")
}
}
shared_payloads.push((heap.address, heap_bytes))
let end = writer_align(heap.address.to_int() + heap_bytes.length())
if end > eof {
eof = end
}
}
for index in 1.. shared.extension_address
None => 0xFFFFFFFFFFFFFFFFUL
},
file_length.to_uint64(),
base_address=self.user_block_size.to_uint64(),
),
)
if layout.shared is Some(shared) {
writer_write_at(
output,
self.user_block_size,
shared.extension_address,
encode_writer_superblock_extension(
shared.table_address,
shared.heaps.length(),
),
)
}
for index in 0.. 0 {
writer_write_at(
output,
self.user_block_size,
layout.heap.region_address,
region_heap_payload,
)
}
if heap_payload.length() > 0 {
writer_write_at(
output,
self.user_block_size,
layout.heap.address,
heap_payload,
)
}
for (address, payload) in dense_payloads {
writer_write_at(output, self.user_block_size, address, payload)
}
for (address, payload) in external_payloads {
writer_write_at(output, self.user_block_size, address, payload)
}
for (address, payload) in virtual_payloads {
writer_write_at(output, self.user_block_size, address, payload)
}
for (address, payload) in raw_payloads {
writer_write_at(output, self.user_block_size, address, payload)
}
for (address, payload) in shared_payloads {
writer_write_at(output, self.user_block_size, address, payload)
}
guard file_length >= output.length() else {
raise InvalidField("HDF5 output exceeds its declared end-of-file address")
}
writer_write_zeros(output, file_length - output.length())
output.to_bytes()
}