///|
/// One decoded tile from a dataset. `offset` is the tile origin in dataset
/// coordinates and `dimensions` is the edge-clipped tile shape.
#derive(@debug.Debug)
pub(all) struct DatasetChunk {
  offset : Array[UInt64]
  dimensions : Array[UInt64]
  values : DatasetValues
}

///|
pub extend DatasetChunk with @debug.Debug::{to_repr}

///|
/// Iterates over a dataset in caller-selected tiles without materializing the
/// complete dataset. Storage chunks that intersect a tile are decoded at most
/// once per `next` call.
pub struct DatasetChunkIterator {
  priv file : Hdf5File
  priv header : ObjectHeader
  priv metadata : DatasetMetadata
  priv tile_dimensions : Array[UInt64]
  priv mut next_index : Int64
  priv total_tiles : Int64
  priv resolver : Hdf5Resolver?
  priv referring_file : String
}

///|
fn dataset_tile_count(
  dataspace : Dataspace,
  tile_dimensions : Array[UInt64],
) -> Int64 raise Hdf5Error {
  guard dataspace.kind != Null || tile_dimensions.length() == 0 else {
    raise InvalidArgument("null datasets do not accept tile dimensions")
  }
  if dataspace.kind == Null {
    return 0L
  }
  let rank = dataspace.dimensions.length()
  guard tile_dimensions.length() == rank else {
    raise InvalidArgument("tile rank does not match dataset rank")
  }
  let mut tile_elements = 1L
  for axis in 0.. 0UL else {
      raise InvalidArgument("tile dimensions must be positive")
    }
    guard tile <= 0x7FFFFFFFUL else {
      raise InvalidArgument("tile dimension is too large")
    }
    guard tile_elements <= 0x7FFFFFFFFFFFFFFFL / tile.reinterpret_as_int64() else {
      raise InvalidArgument("tile element count overflow")
    }
    tile_elements = tile_elements * tile.reinterpret_as_int64()
  }
  guard tile_elements <= 0x7FFFFFFFL else {
    raise InvalidArgument("tile contains too many elements")
  }
  if rank == 0 || dataspace.dimensions.contains(0UL) {
    return if rank == 0 { 1L } else { 0L }
  }
  let mut tiles = 1L
  for axis in 0.. DatasetChunkIterator raise Hdf5Error {
  let total_tiles = dataset_tile_count(metadata.dataspace, tile_dimensions)
  {
    file,
    header,
    metadata,
    tile_dimensions: tile_dimensions.copy(),
    next_index: 0L,
    total_tiles,
    resolver,
    referring_file,
  }
}

///|
/// Creates a bounded-memory iterator over a dataset.
pub fn Hdf5File::chunk_iterator(
  self : Hdf5File,
  path : String,
  tile_dimensions : Array[UInt64],
) -> DatasetChunkIterator raise Hdf5Error {
  let node = self.resolve(path)
  guard node.kind == Dataset else {
    raise InvalidField("object is not a dataset: " + path)
  }
  let header = self.object_header_at(node.address)
  let metadata = read_dataset_metadata(
    header,
    self.superblock.offset_size,
    self.superblock.length_size,
  )
  make_dataset_chunk_iterator(self, header, metadata, tile_dimensions, None, "")
}

///|
fn DatasetChunkIterator::read_selection(
  self : DatasetChunkIterator,
  selection : DatasetSelection,
) -> DatasetSelectionResult raise Hdf5Error {
  if self.metadata.layout.layout_class == 3 {
    let (dimensions, indices) = selection_indices(
      self.metadata.dataspace,
      selection,
    )
    return self.file.read_virtual_dataset_indices(
      self.header,
      self.metadata,
      dimensions,
      indices,
      self.resolver,
      self.referring_file,
    )
  }
  if self.header.first(msg_external_data_files) is Some(_) {
    let resolver = match self.resolver {
      Some(resolver) => resolver
      None =>
        raise UnsupportedFeature(
          "external dataset storage requires Hdf5Resolver",
        )
    }
    return resolver.read_external_dataset_selection(
      self.referring_file,
      self.file,
      self.header,
      self.metadata,
      selection,
    )
  }
  let file = self.file
  match file.source {
    Some(source) =>
      read_source_dataset_selection(
        source,
        self.header,
        self.metadata,
        selection,
        file.superblock.offset_size,
        file.superblock.length_size,
        file.superblock.base_address,
        custom_filters=file.custom_filters,
      )
    None =>
      read_dataset_selection(
        file.bytes,
        self.header,
        self.metadata,
        selection,
        file.superblock.offset_size,
        file.superblock.length_size,
        file.superblock.base_address,
        custom_filters=file.custom_filters,
      )
  }
}

///|
/// Returns the next tile, or `None` after the dataset has been consumed.
/// Callers can stop early by no longer calling `next`.
pub fn DatasetChunkIterator::next(
  self : DatasetChunkIterator,
) -> DatasetChunk? raise Hdf5Error {
  if self.next_index >= self.total_tiles {
    return None
  }
  let tile_index = self.next_index
  self.next_index = self.next_index + 1L
  let rank = self.metadata.dataspace.dimensions.length()
  if rank == 0 {
    let selected = self.read_selection(Hyperslab([], [], []))
    return Some({ offset: [], dimensions: [], values: selected.values, })
  }
  let offset : Array[UInt64] = Array::make(rank, 0UL)
  let dimensions : Array[UInt64] = Array::make(rank, 0UL)
  let mut remaining = tile_index
  let mut axis = rank - 1
  while axis >= 0 {
    let dimension = self.metadata.dataspace.dimensions[axis]
    let tile = self.tile_dimensions[axis]
    let mut grid = dimension / tile
    if dimension % tile != 0UL {
      grid = grid + 1UL
    }
    let grid_count = grid.reinterpret_as_int64()
    let coordinate = remaining % grid_count
    remaining = remaining / grid_count
    let start = coordinate.reinterpret_as_uint64() * tile
    let available = dimension - start
    offset[axis] = start
    dimensions[axis] = if available < tile { available } else { tile }
    axis = axis - 1
  }
  let stride : Array[UInt64] = Array::make(rank, 1UL)
  let selected = self.read_selection(
    Hyperslab(offset.copy(), dimensions.copy(), stride),
  )
  Some({ offset, dimensions, values: selected.values, })
}

///|
/// One page of a dataset selection. `offset` is the zero-based ordinal of the
/// first value inside the original selection; values are returned in selection
/// order as a flat page.
#derive(@debug.Debug)
pub(all) struct DatasetSelectionPage {
  offset : UInt64
  datatype : Datatype
  values : DatasetValues
}

///|
pub extend DatasetSelectionPage with @debug.Debug::{to_repr}

///|
/// Iterates over a dataset selection in bounded-memory pages. Hyperslab
/// ordinals are generated lazily, so creating an iterator for a very large
/// selection does not allocate an index array for the complete result.
pub struct DatasetSelectionIterator {
  priv file : Hdf5File
  priv header : ObjectHeader
  priv metadata : DatasetMetadata
  priv selection : DatasetSelection
  priv page_size : Int32
  priv total : Int64
  priv mut next_offset : Int64
  priv resolver : Hdf5Resolver?
  priv referring_file : String
}

///|
fn copy_dataset_selection(selection : DatasetSelection) -> DatasetSelection {
  match selection {
    Hyperslab(start, count, stride) =>
      Hyperslab(start.copy(), count.copy(), stride.copy())
    Points(points) => {
      let copied : Array[Array[UInt64]] = []
      for point in points {
        copied.push(point.copy())
      }
      Points(copied)
    }
  }
}

///|
fn make_dataset_selection_iterator(
  file : Hdf5File,
  header : ObjectHeader,
  metadata : DatasetMetadata,
  selection : DatasetSelection,
  page_size : Int32,
  resolver : Hdf5Resolver?,
  referring_file : String,
) -> DatasetSelectionIterator raise Hdf5Error {
  guard page_size > 0 else {
    raise InvalidArgument("selection page size must be positive")
  }
  let copied = copy_dataset_selection(selection)
  let total = selection_element_total(metadata.dataspace, copied)
  {
    file,
    header,
    metadata,
    selection: copied,
    page_size,
    total,
    next_offset: 0L,
    resolver,
    referring_file,
  }
}

///|
/// Creates a bounded-memory iterator over an arbitrary dataset selection.
/// Use `next` to retrieve pages of at most `page_size` values.
pub fn Hdf5File::selection_iterator(
  self : Hdf5File,
  path : String,
  selection : DatasetSelection,
  page_size : Int32,
) -> DatasetSelectionIterator raise Hdf5Error {
  let node = self.resolve(path)
  guard node.kind == Dataset else {
    raise InvalidField("object is not a dataset: " + path)
  }
  let header = self.object_header_at(node.address)
  let metadata = read_dataset_metadata(
    header,
    self.superblock.offset_size,
    self.superblock.length_size,
  )
  make_dataset_selection_iterator(
    self,
    header,
    metadata,
    selection,
    page_size,
    None,
    "",
  )
}

///|
/// Creates a bounded-memory selection iterator through the resolver, including
/// contiguous external raw-data storage.
pub fn Hdf5Resolver::selection_iterator(
  self : Hdf5Resolver,
  path : String,
  selection : DatasetSelection,
  page_size : Int32,
) -> DatasetSelectionIterator raise Hdf5Error {
  let node = self.resolve(path)
  guard node.kind == Dataset else {
    raise InvalidField("object is not a dataset: " + path)
  }
  let file = self.open(node.file)
  let header = file.object_header_at(node.address)
  let metadata = read_dataset_metadata(
    header,
    file.superblock.offset_size,
    file.superblock.length_size,
  )
  make_dataset_selection_iterator(
    file,
    header,
    metadata,
    selection,
    page_size,
    Some(self),
    node.file,
  )
}

///|
fn DatasetSelectionIterator::read_page(
  self : DatasetSelectionIterator,
  indices : Array[UInt64],
) -> DatasetSelectionResult raise Hdf5Error {
  let dimensions = [indices.length().to_uint64()]
  if self.metadata.layout.layout_class == 3 {
    return self.file.read_virtual_dataset_indices(
      self.header,
      self.metadata,
      dimensions,
      indices,
      self.resolver,
      self.referring_file,
    )
  }
  if self.header.first(msg_external_data_files) is Some(_) {
    let resolver = match self.resolver {
      Some(resolver) => resolver
      None =>
        raise UnsupportedFeature(
          "external dataset storage requires Hdf5Resolver",
        )
    }
    return resolver.read_external_dataset_indices(
      self.referring_file,
      self.file,
      self.header,
      self.metadata,
      dimensions,
      indices,
    )
  }
  let file = self.file
  match file.source {
    Some(source) =>
      read_source_dataset_indices(
        source,
        self.header,
        self.metadata,
        dimensions,
        indices,
        file.superblock.offset_size,
        file.superblock.length_size,
        file.superblock.base_address,
        custom_filters=file.custom_filters,
      )
    None =>
      read_dataset_indices(
        file.bytes,
        self.header,
        self.metadata,
        dimensions,
        indices,
        file.superblock.offset_size,
        file.superblock.length_size,
        file.superblock.base_address,
        custom_filters=file.custom_filters,
      )
  }
}

///|
/// Returns the next selection page, or `None` after the selection has been
/// consumed. Callers can stop early without reading the remaining pages.
pub fn DatasetSelectionIterator::next(
  self : DatasetSelectionIterator,
) -> DatasetSelectionPage? raise Hdf5Error {
  if self.next_offset >= self.total {
    return None
  }
  let offset = self.next_offset
  let remaining = self.total - offset
  let page_size = if remaining < self.page_size.to_int64() {
    remaining.to_int()
  } else {
    self.page_size
  }
  let indices : Array[UInt64] = []
  for ordinal in 0..