From 04f69fd6271f1162617cc12fe71c96aeb23e2d7e Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Sun, 26 Jul 2026 17:22:10 +0000
Subject: [PATCH 01/17] Initial plan


From 5f101ffab90b6e6a27ecdd8f67bf07682b4adf23 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Sun, 26 Jul 2026 17:27:38 +0000
Subject: [PATCH 02/17] Add from_parquet_3d reader for multi-dimension skims in
 parquet format

---
 docs/api.rst                   |   1 +
 sharrow/dataset.py             | 271 +++++++++++++++++++++++++++++++++
 sharrow/tests/test_datasets.py | 135 ++++++++++++++++
 3 files changed, 407 insertions(+)

diff --git a/docs/api.rst b/docs/api.rst
index 6389a2c..14a8d20 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -89,6 +89,7 @@ These functions can be found in the :py:mod:`sharrow.dataset` module.
 .. autofunction:: sharrow.dataset.from_table
 .. autofunction:: sharrow.dataset.from_omx
 .. autofunction:: sharrow.dataset.from_omx_3d
+.. autofunction:: sharrow.dataset.from_parquet_3d
 .. autofunction:: sharrow.dataset.from_zarr
 .. autofunction:: sharrow.dataset.from_named_objects
 
diff --git a/sharrow/dataset.py b/sharrow/dataset.py
index ec13a6b..e761095 100755
--- a/sharrow/dataset.py
+++ b/sharrow/dataset.py
@@ -674,6 +674,277 @@ def reload_from_omx_3d(
             h.close()
 
 
+def _parquet_layout(labels_0, labels_1):
+    """
+    Determine the layout of a two dimensional index given as two columns.
+
+    Parameters
+    ----------
+    labels_0, labels_1 : array-like
+        The values of the two index columns, which together identify the
+        position of each row of a table in a two dimensional array.
+
+    Returns
+    -------
+    layout : {"row-major", "column-major", "unsorted-dense", "sparse"}
+    index_0, index_1 : array-like or None
+        The unique labels for each dimension, in the order they appear in
+        the resulting array.  These are None if the layout is "sparse", as
+        the arrangement of a sparse table is resolved elsewhere.
+    """
+    n_rows = len(labels_0)
+    unique_0 = pd.unique(labels_0)
+    unique_1 = pd.unique(labels_1)
+    n_0 = len(unique_0)
+    n_1 = len(unique_1)
+    if n_0 * n_1 != n_rows:
+        return "sparse", None, None
+
+    # row-major: the first index changes slowly, the second changes quickly
+    row_major_0 = labels_0[::n_1]
+    row_major_1 = labels_1[:n_1]
+    if len(row_major_0) == n_0 and np.array_equal(
+        labels_0, np.repeat(row_major_0, n_1)
+    ):
+        if np.array_equal(labels_1, np.tile(row_major_1, n_0)):
+            return "row-major", row_major_0, row_major_1
+
+    # column-major: the second index changes slowly, the first changes quickly
+    column_major_0 = labels_0[:n_0]
+    column_major_1 = labels_1[::n_0]
+    if len(column_major_1) == n_1 and np.array_equal(
+        labels_1, np.repeat(column_major_1, n_0)
+    ):
+        if np.array_equal(labels_0, np.tile(column_major_0, n_1)):
+            return "column-major", column_major_0, column_major_1
+
+    # The number of rows matches a dense array, but the rows are not in
+    # either dense ordering.  This is only actually dense if every possible
+    # pair of labels appears exactly once.
+    codes_0 = pd.Index(unique_0).get_indexer(labels_0)
+    codes_1 = pd.Index(unique_1).get_indexer(labels_1)
+    flat = codes_0.astype(np.int64) * n_1 + codes_1
+    if len(np.unique(flat)) == n_rows:
+        return "unsorted-dense", unique_0, unique_1
+    return "sparse", None, None
+
+
+def _parquet_column_to_numpy(table, name):
+    return table.column(name).combine_chunks().to_numpy(zero_copy_only=False)
+
+
+def _parquet_data_names(schema_names, index_names, ignore):
+    data_names = [i for i in schema_names if i not in index_names]
+    if ignore is not None:
+        if isinstance(ignore, str):
+            ignore = [ignore]
+        data_names = [i for i in data_names if not _should_ignore(ignore, i)]
+    return data_names
+
+
+def _read_one_parquet_3d(filename, index_names, ignore):
+    """
+    Read the matrix tables in one parquet file into two dimensional arrays.
+
+    Parameters
+    ----------
+    filename : path-like
+        The parquet file to read.
+    index_names : tuple[str, str]
+        The names of the columns in the parquet file that give the position
+        of each row in the two dimensional arrays.
+    ignore : list-like or None
+        Regular expressions for matrix table names to skip.
+
+    Returns
+    -------
+    arrays : dict[str, array-like]
+        Two dimensional arrays, one for each matrix table in the file.
+    index_0, index_1 : array-like
+        The labels for each dimension of the arrays.
+    """
+    import pyarrow.parquet as pq
+
+    pf = pq.ParquetFile(filename)
+    schema_names = pf.schema_arrow.names
+    for i in index_names:
+        if i not in schema_names:
+            raise KeyError(f"index column {i!r} not found in {filename}")
+    data_names = _parquet_data_names(schema_names, index_names, ignore)
+
+    index_table = pf.read(columns=list(index_names))
+    labels_0 = _parquet_column_to_numpy(index_table, index_names[0])
+    labels_1 = _parquet_column_to_numpy(index_table, index_names[1])
+    layout, index_0, index_1 = _parquet_layout(labels_0, labels_1)
+    logger.info(f"parquet file {filename} has a {layout} layout")
+    del index_table, labels_0, labels_1
+
+    if layout == "unsorted-dense":
+        raise ValueError(
+            f"the data in {filename} is dense but is not sorted into "
+            f"row-major or column-major order"
+        )
+
+    if layout == "sparse":
+        # fall back to the generic xarray loader for sparse data
+        df = pf.read(columns=list(index_names) + data_names).to_pandas()
+        ds = df.set_index(list(index_names)).to_xarray()
+        arrays = {k: ds[k].to_numpy() for k in data_names}
+        return arrays, ds[index_names[0]].to_numpy(), ds[index_names[1]].to_numpy()
+
+    n_0 = len(index_0)
+    n_1 = len(index_1)
+    arrays = {}
+    for k in data_names:
+        content = _parquet_column_to_numpy(pf.read(columns=[k]), k)
+        if layout == "row-major":
+            arrays[k] = content.reshape(n_0, n_1)
+        else:
+            arrays[k] = content.reshape(n_1, n_0).transpose()
+    return arrays, index_0, index_1
+
+
+def from_parquet_3d(
+    parquet,
+    index_names=("otaz", "dtaz", "time_period"),
+    *,
+    time_periods=None,
+    time_period_sep="__",
+    max_float_precision=32,
+    ignore=None,
+):
+    """
+    Create a Dataset from parquet file(s) with an implicit third dimension.
+
+    The parquet file(s) should contain two index columns, which give the
+    position of each row in the two "native" dimensions, plus any number of
+    other columns, each of which gives the values of one matrix table.  The
+    matrix tables are named in the same manner as they would be in an OMX
+    file, including using a separator (typically a double underscore) to
+    identify time periods, which are assembled into a third dimension.
+
+    Parameters
+    ----------
+    parquet : path-like or Iterable[path-like]
+        The parquet file(s) to read.  When multiple files are given, the
+        matrix tables from all files are combined into a single dataset.
+        Each file is checked independently for its data layout, so the
+        index columns need not be in the same order in every file.
+    index_names : tuple, default ("otaz", "dtaz", "time_period")
+        Should be a tuple of length 3, giving the names of the three
+        dimensions.  The first two names are the names of the index columns
+        in the parquet file(s), the last is the name of the implicit
+        dimension that is created by parsing matrix table names.
+    time_periods : list-like, optional
+        A list of index values from which the third dimension is constructed
+        for all variables with a third dimension.  Required if any matrix
+        table name contains `time_period_sep`.
+    time_period_sep : str, default "__" (double underscore)
+        The presence of this separator within the name of any matrix table
+        indicates that table is to be considered a page in a three
+        dimensional variable.  The portion of the name preceding the first
+        instance of this separator is the name of the resulting variable,
+        and the portion of the name after the first instance of this
+        separator is the label of the position for this page, which should
+        appear in `time_periods`.
+    max_float_precision : int, default 32
+        When loading, reduce all floats to this level of precision,
+        generally to save memory if they were stored as double precision but
+        that level of detail is unneeded in the present application.
+    ignore : str or list-like, optional
+        A list of regular expressions that will be used to filter out
+        variables from the dataset.  If any of the regular expressions
+        match the name of a variable, that variable will not be included
+        in the loaded dataset.
+
+    Returns
+    -------
+    Dataset
+    """
+    if isinstance(parquet, (str, Path)) or not isinstance(parquet, Iterable):
+        parquet = [parquet]
+
+    if len(index_names) != 3:
+        raise ValueError("index_names must have length 3")
+
+    time_periods_map = None
+    if time_periods is not None:
+        time_periods = list(time_periods)
+        time_periods_map = {t: n for n, t in enumerate(time_periods)}
+
+    index_0 = None
+    index_1 = None
+    content = {}
+    pending_3d = {}
+
+    for filename in parquet:
+        arrays, file_index_0, file_index_1 = _read_one_parquet_3d(
+            filename, tuple(index_names[:2]), ignore
+        )
+        if index_0 is None:
+            index_0 = file_index_0
+            index_1 = file_index_1
+        elif not (
+            np.array_equal(index_0, file_index_0)
+            and np.array_equal(index_1, file_index_1)
+        ):
+            # the labels in this file are not in the same order as the
+            # labels in the first file, so rearrange this file's data
+            take_0 = pd.Index(file_index_0).get_indexer(index_0)
+            take_1 = pd.Index(file_index_1).get_indexer(index_1)
+            if (take_0 < 0).any() or (take_1 < 0).any():
+                raise ValueError(
+                    f"the index labels in {filename} do not match those in "
+                    f"the other parquet file(s)"
+                )
+            arrays = {k: v[take_0][:, take_1] for k, v in arrays.items()}
+
+        for k, v in arrays.items():
+            if time_period_sep in k:
+                base_k, time_k = k.split(time_period_sep, 1)
+                if time_periods_map is None:
+                    raise ValueError("must give time periods explicitly")
+                if time_k not in time_periods_map:
+                    raise KeyError(f"time period {time_k!r} not in time_periods")
+                if base_k not in pending_3d:
+                    pending_3d[base_k] = [None] * len(time_periods)
+                pending_3d[base_k][time_periods_map[time_k]] = v
+            else:
+                content[k] = xr.DataArray(
+                    v,
+                    dims=index_names[:2],
+                    coords={
+                        index_names[0]: index_0,
+                        index_names[1]: index_1,
+                    },
+                )
+
+    for base_k, arrs in pending_3d.items():
+        prototype = None
+        for i in arrs:
+            if i is not None:
+                prototype = i
+                break
+        if prototype is None:
+            raise ValueError("no prototype")
+        arrs_ = [(i if i is not None else np.zeros_like(prototype)) for i in arrs]
+        content[base_k] = xr.DataArray(
+            np.stack(arrs_, axis=-1),
+            dims=index_names,
+            coords={
+                index_names[0]: index_0,
+                index_names[1]: index_1,
+                index_names[2]: time_periods,
+            },
+        )
+
+    for i in content:
+        if np.issubdtype(content[i].dtype, np.floating):
+            if content[i].dtype.itemsize > max_float_precision / 8:
+                content[i] = content[i].astype(f"float{max_float_precision}")
+    return xr.Dataset(content)
+
+
 def from_amx(
     amx,
     index_names=("otaz", "dtaz"),
diff --git a/sharrow/tests/test_datasets.py b/sharrow/tests/test_datasets.py
index 18c721b..0e1f746 100644
--- a/sharrow/tests/test_datasets.py
+++ b/sharrow/tests/test_datasets.py
@@ -178,3 +178,138 @@ def test_dataarray_iloc():
     z = arr2d.iloc[dict(s=slice(1, 2), p=slice(2, 4))]
 
     xr.testing.assert_equal(z, xr.DataArray([[20, 28]], dims=["s", "p"]))
+
+
+def _skims_dataframe(zones=(11, 22, 33, 44), order="row-major"):
+    """Create a dense skims dataframe for parquet testing."""
+    n = len(zones)
+    otaz = np.repeat(np.asarray(zones), n)
+    dtaz = np.tile(np.asarray(zones), n)
+    df = pd.DataFrame(
+        {
+            "otaz": otaz,
+            "dtaz": dtaz,
+            "DIST": (otaz * 1000 + dtaz).astype(np.float32),
+            "TIME__AM": (otaz * 10 + dtaz).astype(np.float32),
+            "TIME__PM": (otaz * 10 + dtaz + 0.5).astype(np.float32),
+        }
+    )
+    if order == "column-major":
+        df = df.sort_values(["dtaz", "otaz"]).reset_index(drop=True)
+    return df
+
+
+def _expected_skims(zones=(11, 22, 33, 44)):
+    df = _skims_dataframe(zones)
+    return df.set_index(["otaz", "dtaz"]).to_xarray()
+
+
+def test_from_parquet_3d_row_major():
+    with tempfile.TemporaryDirectory() as tempdir:
+        f = Path(tempdir).joinpath("skims.parquet")
+        _skims_dataframe().to_parquet(f, index=False)
+        skims = sh.dataset.from_parquet_3d(f, time_periods=["AM", "PM"])
+    expected = _expected_skims()
+    assert skims["DIST"].dims == ("otaz", "dtaz")
+    assert skims["TIME"].dims == ("otaz", "dtaz", "time_period")
+    assert skims.coords["otaz"].values == approx(np.asarray([11, 22, 33, 44]))
+    assert skims.coords["dtaz"].values == approx(np.asarray([11, 22, 33, 44]))
+    assert list(skims.coords["time_period"].values) == ["AM", "PM"]
+    assert skims["DIST"].values == approx(expected["DIST"].values)
+    assert skims["TIME"].sel(time_period="AM").values == approx(
+        expected["TIME__AM"].values
+    )
+    assert skims["TIME"].sel(time_period="PM").values == approx(
+        expected["TIME__PM"].values
+    )
+
+
+def test_from_parquet_3d_column_major():
+    with tempfile.TemporaryDirectory() as tempdir:
+        f = Path(tempdir).joinpath("skims.parquet")
+        _skims_dataframe(order="column-major").to_parquet(f, index=False)
+        skims = sh.dataset.from_parquet_3d(f, time_periods=["AM", "PM"])
+    expected = _expected_skims()
+    assert skims["DIST"].values == approx(expected["DIST"].values)
+    assert skims["TIME"].sel(time_period="PM").values == approx(
+        expected["TIME__PM"].values
+    )
+
+
+def test_from_parquet_3d_sparse():
+    expected = _expected_skims()
+    df = _skims_dataframe()
+    # drop some rows to make the data sparse, and shuffle the remainder
+    df = df.drop(index=[1, 7]).sample(frac=1.0, random_state=42)
+    with tempfile.TemporaryDirectory() as tempdir:
+        f = Path(tempdir).joinpath("skims.parquet")
+        df.to_parquet(f, index=False)
+        skims = sh.dataset.from_parquet_3d(f, time_periods=["AM", "PM"])
+    assert skims["DIST"].dims == ("otaz", "dtaz")
+    assert skims.coords["otaz"].values == approx(np.asarray([11, 22, 33, 44]))
+    dist = skims["DIST"].values
+    assert np.isnan(dist[0, 1])
+    assert np.isnan(dist[1, 3])
+    valid = ~np.isnan(dist)
+    assert dist[valid] == approx(expected["DIST"].values[valid])
+
+
+def test_from_parquet_3d_unsorted_dense():
+    df = _skims_dataframe()
+    # a dense but improperly sorted table should raise an error
+    df = df.sample(frac=1.0, random_state=42)
+    with tempfile.TemporaryDirectory() as tempdir:
+        f = Path(tempdir).joinpath("skims.parquet")
+        df.to_parquet(f, index=False)
+        with pytest.raises(ValueError):
+            sh.dataset.from_parquet_3d(f, time_periods=["AM", "PM"])
+
+
+def test_from_parquet_3d_multiple_files():
+    expected = _expected_skims()
+    df = _skims_dataframe()
+    with tempfile.TemporaryDirectory() as tempdir:
+        f1 = Path(tempdir).joinpath("skims1.parquet")
+        f2 = Path(tempdir).joinpath("skims2.parquet")
+        df[["otaz", "dtaz", "DIST", "TIME__AM"]].to_parquet(f1, index=False)
+        # the second file is written in column-major order, to check that
+        # each file is inspected independently
+        df[["otaz", "dtaz", "TIME__PM"]].sort_values(["dtaz", "otaz"]).to_parquet(
+            f2, index=False
+        )
+        skims = sh.dataset.from_parquet_3d([f1, f2], time_periods=["AM", "PM"])
+    assert skims["DIST"].values == approx(expected["DIST"].values)
+    assert skims["TIME"].sel(time_period="AM").values == approx(
+        expected["TIME__AM"].values
+    )
+    assert skims["TIME"].sel(time_period="PM").values == approx(
+        expected["TIME__PM"].values
+    )
+
+
+def test_from_parquet_3d_mismatched_index_order():
+    expected = _expected_skims()
+    df = _skims_dataframe()
+    df2 = _skims_dataframe(zones=(44, 33, 22, 11))
+    with tempfile.TemporaryDirectory() as tempdir:
+        f1 = Path(tempdir).joinpath("skims1.parquet")
+        f2 = Path(tempdir).joinpath("skims2.parquet")
+        df[["otaz", "dtaz", "DIST"]].to_parquet(f1, index=False)
+        df2[["otaz", "dtaz", "TIME__AM", "TIME__PM"]].to_parquet(f2, index=False)
+        skims = sh.dataset.from_parquet_3d([f1, f2], time_periods=["AM", "PM"])
+    assert skims.coords["otaz"].values == approx(np.asarray([11, 22, 33, 44]))
+    assert skims["DIST"].values == approx(expected["DIST"].values)
+    assert skims["TIME"].sel(time_period="AM").values == approx(
+        expected["TIME__AM"].values
+    )
+
+
+def test_from_parquet_3d_ignore():
+    with tempfile.TemporaryDirectory() as tempdir:
+        f = Path(tempdir).joinpath("skims.parquet")
+        _skims_dataframe().to_parquet(f, index=False)
+        skims = sh.dataset.from_parquet_3d(
+            f, time_periods=["AM", "PM"], ignore="TIME.*"
+        )
+    assert "TIME" not in skims.variables
+    assert "DIST" in skims.variables

From b92e56c40db30720103e34374a4a114db695e046 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Sun, 26 Jul 2026 17:31:04 +0000
Subject: [PATCH 03/17] Improve pyarrow compatibility in parquet reader

---
 sharrow/dataset.py | 78 +++++++++++++++++++++++++---------------------
 1 file changed, 42 insertions(+), 36 deletions(-)

diff --git a/sharrow/dataset.py b/sharrow/dataset.py
index e761095..ae869ce 100755
--- a/sharrow/dataset.py
+++ b/sharrow/dataset.py
@@ -730,7 +730,13 @@ def _parquet_layout(labels_0, labels_1):
 
 
 def _parquet_column_to_numpy(table, name):
-    return table.column(name).combine_chunks().to_numpy(zero_copy_only=False)
+    column = table.column(name)
+    if isinstance(column, pa.ChunkedArray):
+        column = column.combine_chunks()
+    if isinstance(column, pa.ChunkedArray):
+        # older versions of pyarrow return a ChunkedArray from combine_chunks
+        return column.to_numpy()
+    return column.to_numpy(zero_copy_only=False)
 
 
 def _parquet_data_names(schema_names, index_names, ignore):
@@ -765,43 +771,43 @@ def _read_one_parquet_3d(filename, index_names, ignore):
     """
     import pyarrow.parquet as pq
 
-    pf = pq.ParquetFile(filename)
-    schema_names = pf.schema_arrow.names
-    for i in index_names:
-        if i not in schema_names:
-            raise KeyError(f"index column {i!r} not found in {filename}")
-    data_names = _parquet_data_names(schema_names, index_names, ignore)
-
-    index_table = pf.read(columns=list(index_names))
-    labels_0 = _parquet_column_to_numpy(index_table, index_names[0])
-    labels_1 = _parquet_column_to_numpy(index_table, index_names[1])
-    layout, index_0, index_1 = _parquet_layout(labels_0, labels_1)
-    logger.info(f"parquet file {filename} has a {layout} layout")
-    del index_table, labels_0, labels_1
-
-    if layout == "unsorted-dense":
-        raise ValueError(
-            f"the data in {filename} is dense but is not sorted into "
-            f"row-major or column-major order"
-        )
+    with pq.ParquetFile(filename) as pf:
+        schema_names = pf.schema_arrow.names
+        for i in index_names:
+            if i not in schema_names:
+                raise KeyError(f"index column {i!r} not found in {filename}")
+        data_names = _parquet_data_names(schema_names, index_names, ignore)
+
+        index_table = pf.read(columns=list(index_names))
+        labels_0 = _parquet_column_to_numpy(index_table, index_names[0])
+        labels_1 = _parquet_column_to_numpy(index_table, index_names[1])
+        layout, index_0, index_1 = _parquet_layout(labels_0, labels_1)
+        logger.info(f"parquet file {filename} has a {layout} layout")
+        del index_table, labels_0, labels_1
+
+        if layout == "unsorted-dense":
+            raise ValueError(
+                f"the data in {filename} is dense but is not sorted into "
+                f"row-major or column-major order"
+            )
 
-    if layout == "sparse":
-        # fall back to the generic xarray loader for sparse data
-        df = pf.read(columns=list(index_names) + data_names).to_pandas()
-        ds = df.set_index(list(index_names)).to_xarray()
-        arrays = {k: ds[k].to_numpy() for k in data_names}
-        return arrays, ds[index_names[0]].to_numpy(), ds[index_names[1]].to_numpy()
+        if layout == "sparse":
+            # fall back to the generic xarray loader for sparse data
+            df = pf.read(columns=list(index_names) + data_names).to_pandas()
+            ds = df.set_index(list(index_names)).to_xarray()
+            arrays = {k: ds[k].to_numpy() for k in data_names}
+            return arrays, ds[index_names[0]].to_numpy(), ds[index_names[1]].to_numpy()
 
-    n_0 = len(index_0)
-    n_1 = len(index_1)
-    arrays = {}
-    for k in data_names:
-        content = _parquet_column_to_numpy(pf.read(columns=[k]), k)
-        if layout == "row-major":
-            arrays[k] = content.reshape(n_0, n_1)
-        else:
-            arrays[k] = content.reshape(n_1, n_0).transpose()
-    return arrays, index_0, index_1
+        n_0 = len(index_0)
+        n_1 = len(index_1)
+        arrays = {}
+        for k in data_names:
+            content = _parquet_column_to_numpy(pf.read(columns=[k]), k)
+            if layout == "row-major":
+                arrays[k] = content.reshape(n_0, n_1)
+            else:
+                arrays[k] = content.reshape(n_1, n_0).transpose()
+        return arrays, index_0, index_1
 
 
 def from_parquet_3d(

From 7e630380e8f5745af729590adb81a0d03dd8e399 Mon Sep 17 00:00:00 2001
From: Jeffrey Newman <jeff@driftless.xyz>
Date: Tue, 28 Jul 2026 10:09:52 -0500
Subject: [PATCH 04/17] Update pandera version in run-tests.yml

---
 .github/workflows/run-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 763e7e8..1eeeb41 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -205,7 +205,7 @@ jobs:
                          "cytoolz==0.12.2" "dask==2023.11.*" "isort==5.12.0" \
                          "multimethod<2.0" "nbmake==1.4.6" "numba==0.57.*" \
                          "numpy==1.24.*" "openmatrix==0.3.5.0" "orca==1.8" \
-                         "pandera>=0.15,<0.18.1" "pandas==2.2.*" "platformdirs==3.2.*" \
+                         "pandera>=0.30" "pandas==2.2.*" "platformdirs==3.2.*" \
                          "psutil==5.9.*" "pyarrow==11.*" "pydantic==2.6.*" "pypyr==5.8.*" \
                          "tables>=3.9" "pytest==7.2.*" "pytest-cov" "pytest-regressions" \
                          "pyyaml==6.*" "requests==2.28.*" "ruff" "scikit-learn==1.2.*" \

From 27eb18e0fbf6840e9bd28326e0b8b26efd38d28c Mon Sep 17 00:00:00 2001
From: Jeffrey Newman <jeff@driftless.xyz>
Date: Tue, 28 Jul 2026 10:12:48 -0500
Subject: [PATCH 05/17] Update numpy version constraint in run-tests.yml

---
 .github/workflows/run-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 1eeeb41..776d447 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -204,7 +204,7 @@ jobs:
           uv pip install "black==22.12.0" "coveralls==3.3.1" \
                          "cytoolz==0.12.2" "dask==2023.11.*" "isort==5.12.0" \
                          "multimethod<2.0" "nbmake==1.4.6" "numba==0.57.*" \
-                         "numpy==1.24.*" "openmatrix==0.3.5.0" "orca==1.8" \
+                         "numpy>=2.0,<3" "openmatrix==0.3.5.0" "orca==1.8" \
                          "pandera>=0.30" "pandas==2.2.*" "platformdirs==3.2.*" \
                          "psutil==5.9.*" "pyarrow==11.*" "pydantic==2.6.*" "pypyr==5.8.*" \
                          "tables>=3.9" "pytest==7.2.*" "pytest-cov" "pytest-regressions" \

From 34ae26aff43b59cf13665d32ec65a16cfc6a856e Mon Sep 17 00:00:00 2001
From: Jeff Newman <jeff@driftless.xyz>
Date: Tue, 28 Jul 2026 15:47:13 -0500
Subject: [PATCH 06/17] loosen reqs in test

---
 .github/workflows/run-tests.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 776d447..da4ee8c 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -202,14 +202,14 @@ jobs:
           uv venv
           source .venv/bin/activate
           uv pip install "black==22.12.0" "coveralls==3.3.1" \
-                         "cytoolz==0.12.2" "dask==2023.11.*" "isort==5.12.0" \
-                         "multimethod<2.0" "nbmake==1.4.6" "numba==0.57.*" \
-                         "numpy>=2.0,<3" "openmatrix==0.3.5.0" "orca==1.8" \
-                         "pandera>=0.30" "pandas==2.2.*" "platformdirs==3.2.*" \
-                         "psutil==5.9.*" "pyarrow==11.*" "pydantic==2.6.*" "pypyr==5.8.*" \
-                         "tables>=3.9" "pytest==7.2.*" "pytest-cov" "pytest-regressions" \
-                         "pyyaml==6.*" "requests==2.28.*" "ruff" "scikit-learn==1.2.*" \
-                         "sharrow>=2.9.1" "simwrapper>1.7" "sparse" "xarray==2025.01.*" \
+                         "cytoolz==0.12.2" "dask>=2023.11.*" "isort>=5.12.0" \
+                         "multimethod<2.0" "nbmake>=1.4.6" "numba>=0.57.*" \
+                         "numpy>=2,<3" "openmatrix==0.3.5.0" \
+                         "pandera>=0.30" "pandas>=2,<3" "platformdirs>=3.2.*" \
+                         "psutil>=5.9.*" "pyarrow>=11.*" "pydantic>=2.6.*,<3" "pypyr>=5.8.*" \
+                         "tables>=3.9" "pytest>=7.2.*" "pytest-cov" "pytest-regressions" \
+                         "pyyaml>=6.*" "requests>=2.28.*" "ruff" "scikit-learn>=1.2" \
+                         "simwrapper>1.7" "sparse" "xarray>=2025" \
                          "zarr>=2,<3" "zstandard" \
                          ./sharrow ./activitysim
 

From 1d320a2bd8a050c985e64d391251aaf90c10c8c8 Mon Sep 17 00:00:00 2001
From: Jeff Newman <jeff@driftless.xyz>
Date: Tue, 28 Jul 2026 15:50:58 -0500
Subject: [PATCH 07/17] fix dep pin

---
 .github/workflows/run-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index da4ee8c..f22e9c0 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -202,7 +202,7 @@ jobs:
           uv venv
           source .venv/bin/activate
           uv pip install "black==22.12.0" "coveralls==3.3.1" \
-                         "cytoolz==0.12.2" "dask>=2023.11.*" "isort>=5.12.0" \
+                         "cytoolz==0.12.2" "dask>=2023.11" "isort>=5.12.0" \
                          "multimethod<2.0" "nbmake>=1.4.6" "numba>=0.57.*" \
                          "numpy>=2,<3" "openmatrix==0.3.5.0" \
                          "pandera>=0.30" "pandas>=2,<3" "platformdirs>=3.2.*" \

From 258ff722972a742302603db8ec89381c43f716c6 Mon Sep 17 00:00:00 2001
From: Jeff Newman <jeff@driftless.xyz>
Date: Tue, 28 Jul 2026 15:53:45 -0500
Subject: [PATCH 08/17] nix all the stars

---
 .github/workflows/run-tests.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index f22e9c0..67c3714 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -203,12 +203,12 @@ jobs:
           source .venv/bin/activate
           uv pip install "black==22.12.0" "coveralls==3.3.1" \
                          "cytoolz==0.12.2" "dask>=2023.11" "isort>=5.12.0" \
-                         "multimethod<2.0" "nbmake>=1.4.6" "numba>=0.57.*" \
+                         "multimethod<2.0" "nbmake>=1.4.6" "numba>=0.57" \
                          "numpy>=2,<3" "openmatrix==0.3.5.0" \
-                         "pandera>=0.30" "pandas>=2,<3" "platformdirs>=3.2.*" \
-                         "psutil>=5.9.*" "pyarrow>=11.*" "pydantic>=2.6.*,<3" "pypyr>=5.8.*" \
-                         "tables>=3.9" "pytest>=7.2.*" "pytest-cov" "pytest-regressions" \
-                         "pyyaml>=6.*" "requests>=2.28.*" "ruff" "scikit-learn>=1.2" \
+                         "pandera>=0.30" "pandas>=2,<3" "platformdirs>=3.2" \
+                         "psutil>=5.9" "pyarrow>=11.0" "pydantic>=2.6,<3" "pypyr>=5.8" \
+                         "tables>=3.9" "pytest>=7.2" "pytest-cov" "pytest-regressions" \
+                         "pyyaml>=6" "requests>=2.28" "ruff" "scikit-learn>=1.2" \
                          "simwrapper>1.7" "sparse" "xarray>=2025" \
                          "zarr>=2,<3" "zstandard" \
                          ./sharrow ./activitysim

From 54518dd97fb12e7ccd4b383fc7f5d61166244764 Mon Sep 17 00:00:00 2001
From: Jeff Newman <jeff@driftless.xyz>
Date: Tue, 28 Jul 2026 15:59:03 -0500
Subject: [PATCH 09/17] update coveralls because ugh

---
 .github/workflows/run-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 67c3714..ba0528f 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -201,7 +201,7 @@ jobs:
         run: |
           uv venv
           source .venv/bin/activate
-          uv pip install "black==22.12.0" "coveralls==3.3.1" \
+          uv pip install "black==22.12.0" "coveralls>=4" \
                          "cytoolz==0.12.2" "dask>=2023.11" "isort>=5.12.0" \
                          "multimethod<2.0" "nbmake>=1.4.6" "numba>=0.57" \
                          "numpy>=2,<3" "openmatrix==0.3.5.0" \

From e1ab0915581dddc9807d228e86fc21fff722f9a6 Mon Sep 17 00:00:00 2001
From: Jeff Newman <jeff@driftless.xyz>
Date: Tue, 28 Jul 2026 22:11:10 -0500
Subject: [PATCH 10/17] faster omx reader

---
 envs/development.yml           |   3 +
 envs/testing.yml               |   2 +
 pyproject.toml                 |   3 +
 sharrow/dataset.py             | 222 ++++++++++++------
 sharrow/omx_reader.py          | 399 +++++++++++++++++++++++++++++++++
 sharrow/tests/test_datasets.py | 138 ++++++++++++
 6 files changed, 698 insertions(+), 69 deletions(-)
 create mode 100644 sharrow/omx_reader.py

diff --git a/envs/development.yml b/envs/development.yml
index b3496ec..31dd4ad 100644
--- a/envs/development.yml
+++ b/envs/development.yml
@@ -8,6 +8,9 @@ dependencies:
   # required for testing
   - dask
   - filelock
+  - h5py
+  - hdf5plugin
+  - python-blosc2
   - ruff
   - jupyter
   - nbmake
diff --git a/envs/testing.yml b/envs/testing.yml
index 4202294..5da1b97 100644
--- a/envs/testing.yml
+++ b/envs/testing.yml
@@ -23,6 +23,8 @@ dependencies:
   - nbmake
   - openmatrix
   - h5py
+  - python-blosc2
+  - hdf5plugin
   - zarr
   - pip:
     - larch>=6
diff --git a/pyproject.toml b/pyproject.toml
index 03f506f..e9c0913 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -13,6 +13,9 @@ dependencies = [
     "numpy >= 1.19",
     "pandas >= 1.2",
     "pyarrow",
+    "h5py",
+    "blosc2",
+    "hdf5plugin",
     "xarray",
     "numba >= 0.57",
     "numexpr",
diff --git a/sharrow/dataset.py b/sharrow/dataset.py
index ae869ce..c0f4dcf 100755
--- a/sharrow/dataset.py
+++ b/sharrow/dataset.py
@@ -16,6 +16,7 @@
 import xarray as xr
 from xarray import DataArray, Dataset
 
+from . import omx_reader
 from .accessors import register_dataset_method
 from .aster import extract_all_name_tokens
 from .categorical import _Categorical  # noqa
@@ -289,6 +290,19 @@ def from_table(
     return result
 
 
+def _group_names(grp) -> list[str]:
+    """List the child node names of a pytables or h5py group."""
+    try:
+        return list(grp._v_children)  # pytables
+    except AttributeError:
+        return list(grp.keys())  # h5py
+
+
+def omx_file_name(omx) -> str | None:
+    """Resolve the on-disk filename of an OMX-like object, if possible."""
+    return omx_reader.h5_filename(omx)
+
+
 def from_omx(
     omx: openmatrix.File,
     index_names=("otaz", "dtaz"),
@@ -326,26 +340,46 @@ def from_omx(
     -------
     Dataset
     """
-    # handle both larch.OMX and openmatrix.open_file versions
+    import h5py
+
+    # handle larch.OMX, openmatrix.open_file, and h5py.File versions
     if "lar" in type(omx).__module__:
         omx_data = omx.data
         omx_lookup = omx.lookup
         omx_shape = omx.shape
+    elif isinstance(omx, h5py.File):
+        omx_data = omx["data"]
+        omx_lookup = omx["lookup"]
+        omx_shape = tuple(int(i) for i in omx.attrs["SHAPE"])
     else:
         omx_data = omx.root["data"]
         omx_lookup = omx.root["lookup"]
         omx_shape = omx.shape()
 
-    arrays = {}
     if renames is None:
-        for k in omx_data._v_children:
-            arrays[k] = omx_data[k][:]
+        data_names = _group_names(omx_data)
+        rename_pairs = [(k, k) for k in data_names]
     elif isinstance(renames, dict):
-        for new_k, old_k in renames.items():
-            arrays[new_k] = omx_data[old_k][:]
+        rename_pairs = list(renames.items())
     else:
-        for k in renames:
-            arrays[k] = omx_data[k][:]
+        rename_pairs = [(k, k) for k in renames]
+
+    arrays = {}
+    filename = omx_file_name(omx)
+    if filename is not None:
+        # fast path: parallel chunk decoding via h5py
+        import concurrent.futures
+
+        with h5py.File(filename, "r") as f5:
+            f5_data = f5["data"]
+            with concurrent.futures.ThreadPoolExecutor() as pool:
+                for new_k, old_k in rename_pairs:
+                    arrays[new_k] = omx_reader.read_dataset(
+                        f5_data[old_k], executor=pool
+                    )
+    else:
+        for new_k, old_k in rename_pairs:
+            arrays[new_k] = omx_data[old_k][:]
     d = {
         "dims": index_names,
         "data_vars": {k: {"dims": index_names, "data": arrays[k]} for k in arrays},
@@ -400,6 +434,14 @@ def _should_ignore(ignore, x):
     return False
 
 
+def _fast_load_omx_array(filename, name):
+    """Load one matrix table from an OMX file, decoding chunks in parallel."""
+    import h5py
+
+    with h5py.File(filename, "r") as f:
+        return omx_reader.read_dataset(f["data"][name])
+
+
 def from_omx_3d(
     omx: openmatrix.File | str | Iterable[openmatrix.File | str],
     index_names=("otaz", "dtaz", "time_period"),
@@ -471,10 +513,15 @@ def from_omx_3d(
     omx = use_file_handles
 
     try:
-        # handle both larch.OMX and openmatrix.open_file versions
+        import h5py
+
+        # handle larch.OMX, openmatrix.open_file, and h5py.File versions
         if "larch" in type(omx[0]).__module__:
             omx_shape = omx[0].shape
             omx_lookup = omx[0].lookup
+        elif isinstance(omx[0], h5py.File):
+            omx_shape = tuple(int(i) for i in omx[0].attrs["SHAPE"])
+            omx_lookup = omx[0]["lookup"]
         else:
             omx_shape = omx[0].shape()
             omx_lookup = omx[0].root["lookup"]
@@ -483,15 +530,32 @@ def from_omx_3d(
         for n, i in enumerate(omx):
             if "larch" in type(i).__module__:
                 omx_data.append(i.data)
-                for k in i.data._v_children:
-                    omx_data_map[k] = n
+            elif isinstance(i, h5py.File):
+                omx_data.append(i["data"])
             else:
                 omx_data.append(i.root["data"])
-                for k in i.root["data"]._v_children:
-                    omx_data_map[k] = n
+            for k in _group_names(omx_data[-1]):
+                omx_data_map[k] = n
+
+        omx_filenames = [omx_file_name(i) for i in omx]
 
         import dask.array
 
+        def _lazy_omx_array(k):
+            # Build a lazy dask array for one matrix table.  When the source
+            # file is on disk, defer to the parallel chunk-decoding reader;
+            # otherwise fall back to wrapping the open file handle's node.
+            n = omx_data_map[k]
+            filename = omx_filenames[n]
+            node = omx_data[n][k]
+            if filename is None:
+                return dask.array.from_array(node)
+            return dask.array.from_delayed(
+                dask.delayed(_fast_load_omx_array)(filename, k),
+                shape=tuple(node.shape),
+                dtype=node.dtype,
+            )
+
         data_names = list(omx_data_map.keys())
         if ignore is not None:
             if isinstance(ignore, str):
@@ -500,16 +564,17 @@ def from_omx_3d(
         n1, n2 = omx_shape
         if indexes is None:
             # default reads mapping if only one lookup is included, otherwise one-based
-            if len(omx_lookup._v_children) == 1:
+            lookup_names = _group_names(omx_lookup)
+            if len(lookup_names) == 1:
                 ranger = None
-                indexes = list(omx_lookup._v_children)[0]
+                indexes = lookup_names[0]
             else:
                 ranger = one_based
         elif indexes == "one-based":
             ranger = one_based
         elif indexes == "zero-based":
             ranger = zero_based
-        elif indexes in set(omx_lookup._v_children):
+        elif indexes in set(_group_names(omx_lookup)):
             ranger = None
         else:
             raise NotImplementedError(
@@ -534,12 +599,10 @@ def from_omx_3d(
                 base_k, time_k = k.split(time_period_sep, 1)
                 if base_k not in pending_3d:
                     pending_3d[base_k] = [None] * len(time_periods)
-                pending_3d[base_k][time_periods_map[time_k]] = dask.array.from_array(
-                    omx_data[omx_data_map[k]][k]
-                )
+                pending_3d[base_k][time_periods_map[time_k]] = _lazy_omx_array(k)
             else:
                 content[k] = xr.DataArray(
-                    dask.array.from_array(omx_data[omx_data_map[k]][k]),
+                    _lazy_omx_array(k),
                     dims=index_names[:2],
                     coords={
                         index_names[0]: r1,
@@ -613,61 +676,82 @@ def reload_from_omx_3d(
     if isinstance(ignore, str):
         ignore = [ignore]
 
-    use_file_handles = []
-    opened_file_handles = []
-    for filename in omx:
-        if isinstance(filename, (str, Path)):
-            import openmatrix
+    import concurrent.futures
 
-            h = openmatrix.open_file(filename)
-            opened_file_handles.append(h)
-            use_file_handles.append(h)
-        else:
-            use_file_handles.append(filename)
-    omx = use_file_handles
+    import h5py
 
     bytes_loaded = 0
+    t0 = time.time()
+
+    def _load_into(raw, dset, executor):
+        """Fill the array `raw` from the h5py or pytables dataset `dset`."""
+        if isinstance(dset, h5py.Dataset):
+            if raw.dtype == dset.dtype:
+                omx_reader.read_dataset(dset, out=raw, executor=executor)
+            else:
+                raw[:, :] = omx_reader.read_dataset(dset, executor=executor)
+        else:
+            raw[:, :] = dset[:, :]
+
+    def _load_one(dset, data_name, filter_note, executor):
+        nonlocal bytes_loaded
+        t1 = time.time()
+        if time_period_sep in data_name:
+            data_name_x, data_name_t = data_name.split(time_period_sep, 1)
+            if data_name_x not in dataset:
+                logger.info(
+                    f"skipping {data_name} because {data_name_x} not in dataset"
+                )
+                return
+            if len(dataset[data_name_x].dims) != 3:
+                raise ValueError(
+                    f"dataset variable {data_name_x} has "
+                    f"{len(dataset[data_name_x].dims)} dimensions, expected 3"
+                )
+            raw = dataset[data_name_x].sel(time_period=data_name_t).data
+        else:
+            if len(dataset[data_name].dims) != 2:
+                raise ValueError(
+                    f"dataset variable {data_name} has "
+                    f"{len(dataset[data_name].dims)} dimensions, expected 2"
+                )
+            raw = dataset[data_name].data
+        _load_into(raw, dset, executor)
+        bytes_loaded += raw.nbytes
+        logger.debug(
+            f"loaded {data_name} ({filter_note}) to dataset "
+            f"in {time.time() - t1:.2f}s, {si_units(bytes_loaded)}"
+        )
 
+    opened_file_handles = []
     try:
-        t0 = time.time()
-        for filename, f in zip(omx, use_file_handles):
-            if isinstance(filename, str):
-                logger.info(f"loading into dataset from {filename}")
-            for data_name in f.root.data._v_children:
-                if _should_ignore(ignore, data_name):
-                    logger.info(f"ignoring {data_name}")
-                    continue
-                t1 = time.time()
-                filters = f.root.data[data_name].filters
-                filter_note = f"{filters.complib}/{filters.complevel}"
-
-                if time_period_sep in data_name:
-                    data_name_x, data_name_t = data_name.split(time_period_sep, 1)
-                    if data_name_x not in dataset:
-                        logger.info(
-                            f"skipping {data_name} because {data_name_x} not in dataset"
-                        )
-                        continue
-                    if len(dataset[data_name_x].dims) != 3:
-                        raise ValueError(
-                            f"dataset variable {data_name_x} has "
-                            f"{len(dataset[data_name_x].dims)} dimensions, expected 3"
-                        )
-                    raw = dataset[data_name_x].sel(time_period=data_name_t).data
-                    raw[:, :] = f.root.data[data_name][:, :]
+        with concurrent.futures.ThreadPoolExecutor() as pool:
+            for source in omx:
+                filename = omx_file_name(source)
+                if filename is not None:
+                    # fast path: parallel chunk decoding via h5py
+                    logger.info(f"loading into dataset from {filename}")
+                    f5 = h5py.File(filename, "r")
+                    opened_file_handles.append(f5)
+                    data_group = f5["data"]
+                    for data_name in data_group.keys():
+                        if _should_ignore(ignore, data_name):
+                            logger.info(f"ignoring {data_name}")
+                            continue
+                        dset = data_group[data_name]
+                        filter_note = f"{dset.compression}/{dset.compression_opts}"
+                        _load_one(dset, data_name, filter_note, pool)
                 else:
-                    if len(dataset[data_name].dims) != 2:
-                        raise ValueError(
-                            f"dataset variable {data_name} has "
-                            f"{len(dataset[data_name].dims)} dimensions, expected 2"
-                        )
-                    raw = dataset[data_name].data
-                    raw[:, :] = f.root.data[data_name][:, :]
-                bytes_loaded += raw.nbytes
-                logger.debug(
-                    f"loaded {data_name} ({filter_note}) to dataset "
-                    f"in {time.time() - t1:.2f}s, {si_units(bytes_loaded)}"
-                )
+                    # source is an open file handle with no resolvable
+                    # on-disk filename; read through the handle directly
+                    f = source
+                    for data_name in f.root.data._v_children:
+                        if _should_ignore(ignore, data_name):
+                            logger.info(f"ignoring {data_name}")
+                            continue
+                        filters = f.root.data[data_name].filters
+                        filter_note = f"{filters.complib}/{filters.complevel}"
+                        _load_one(f.root.data[data_name], data_name, filter_note, pool)
         logger.info(f"loading to dataset complete in {time.time() - t0:.2f}s")
     finally:
         for h in opened_file_handles:
diff --git a/sharrow/omx_reader.py b/sharrow/omx_reader.py
new file mode 100644
index 0000000..0abd4b8
--- /dev/null
+++ b/sharrow/omx_reader.py
@@ -0,0 +1,399 @@
+"""Fast, low-memory readers for HDF5-backed OMX matrix files.
+
+The functions in this module bypass the HDF5 library's single-threaded
+decompression path.  Raw (still compressed) chunks are read from the file one
+at a time and handed to a thread pool, where the HDF5 filter pipeline is undone
+by extension code that releases the GIL.  Decompressed chunks are written
+directly into the destination array, so peak memory usage is the size of the
+result plus a small, bounded number of in-flight chunks.
+
+If a dataset uses a filter this module does not know how to invert, reading
+transparently falls back to h5py, which will use the registered HDF5 filter
+plugins instead.
+
+This module is adapted from the ``omx_fast_reader`` module of the `wring
+<https://github.com/driftlesslabs/wring>`_ project.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+import os
+import zlib
+from collections.abc import Sequence
+
+import blosc2
+import h5py
+import hdf5plugin  # noqa: F401  (registers blosc/blosc2/zstd/etc HDF5 filters)
+import numpy as np
+
+__all__ = [
+    "read_dataset",
+    "h5_filename",
+]
+
+H5Z_FILTER_DEFLATE = 1
+H5Z_FILTER_SHUFFLE = 2
+H5Z_FILTER_FLETCHER32 = 3
+H5Z_FILTER_BLOSC = 32001
+H5Z_FILTER_BLOSC2 = 32026
+
+#: Filters this module can invert without calling into the HDF5 filter pipeline.
+SUPPORTED_FILTERS = frozenset(
+    {
+        H5Z_FILTER_DEFLATE,
+        H5Z_FILTER_SHUFFLE,
+        H5Z_FILTER_FLETCHER32,
+        H5Z_FILTER_BLOSC,
+        H5Z_FILTER_BLOSC2,
+    }
+)
+
+
+def h5_filename(omx) -> str | None:
+    """Resolve the on-disk filename of an OMX-like object.
+
+    Parameters
+    ----------
+    omx : str, os.PathLike, h5py.File, tables.File, or larch.OMX
+        An OMX file reference: a path, an h5py file, a pytables file (which
+        includes ``openmatrix.File``), or a larch OMX object.
+
+    Returns
+    -------
+    str or None
+        The path of the underlying file, or None if it cannot be determined.
+    """
+    if isinstance(omx, (str, os.PathLike)):
+        return os.fspath(omx)
+    # h5py.File, tables.File (and openmatrix.File), and larch.OMX all expose
+    # the underlying file path as a `filename` attribute.
+    filename = getattr(omx, "filename", None)
+    if isinstance(filename, (str, os.PathLike)):
+        filename = os.fspath(filename)
+        if os.path.exists(filename):
+            return filename
+    return None
+
+
+def _unshuffle(data: bytes, element_size: int) -> bytes:
+    """Invert the HDF5 shuffle filter.
+
+    The shuffle filter de-interleaves the bytes of each element, so that all
+    first bytes are stored together, then all second bytes, and so on.
+
+    Parameters
+    ----------
+    data : bytes
+        The shuffled byte stream.
+    element_size : int
+        Size in bytes of one array element.
+
+    Returns
+    -------
+    bytes
+        The original (un-shuffled) byte stream.
+    """
+    if element_size <= 1:
+        return data
+    n_elements = len(data) // element_size
+    n_shuffled = n_elements * element_size
+    shuffled = np.frombuffer(data, dtype=np.uint8, count=n_shuffled)
+    unshuffled = shuffled.reshape(element_size, n_elements).T.tobytes()
+    # HDF5 leaves any trailing bytes that do not fill a whole element untouched
+    return unshuffled + data[n_shuffled:]
+
+
+def _read_filter_pipeline(dset: h5py.Dataset) -> list:
+    """Read the ordered HDF5 filter pipeline of a dataset.
+
+    Parameters
+    ----------
+    dset : h5py.Dataset
+        The dataset to inspect.
+
+    Returns
+    -------
+    list[tuple[int, tuple[int, ...]]]
+        Filters in the order they were applied when the data was written,
+        each as a ``(filter_id, cd_values)`` pair.
+    """
+    plist = dset.id.get_create_plist()
+    pipeline = []
+    for i in range(plist.get_nfilters()):
+        filter_id, _flags, cd_values, _name = plist.get_filter(i)
+        pipeline.append((int(filter_id), tuple(int(c) for c in cd_values)))
+    return pipeline
+
+
+def _pipeline_is_supported(pipeline: Sequence) -> bool:
+    """Report whether every filter in `pipeline` can be inverted by this module."""
+    return all(filter_id in SUPPORTED_FILTERS for filter_id, _ in pipeline)
+
+
+def _decode_blosc2_cframe(data: bytes):
+    """Decode a chunk written by the Blosc2 HDF5 filter.
+
+    Depending on how the data was written, the chunk may be an n-dimensional
+    ``b2nd`` container, a plain blosc2 super-chunk, or a bare blosc2 chunk.
+    All three are self-describing, so each form is tried in turn.
+
+    Parameters
+    ----------
+    data : bytes
+        The raw chunk as stored in the HDF5 file.
+
+    Returns
+    -------
+    bytes or numpy.ndarray
+        The decompressed payload.
+    """
+    try:
+        return blosc2.ndarray_from_cframe(data)[:]
+    except (RuntimeError, ValueError):
+        pass
+    try:
+        return blosc2.schunk_from_cframe(data)[:]
+    except (RuntimeError, ValueError):
+        pass
+    return blosc2.decompress(data)
+
+
+def _decompress_chunk(
+    raw_bytes: bytes,
+    filter_mask: int,
+    pipeline: Sequence,
+    dtype: np.dtype,
+    chunk_shape: tuple,
+) -> np.ndarray:
+    """Undo a dataset's filter pipeline for one raw chunk.
+
+    Parameters
+    ----------
+    raw_bytes : bytes
+        The raw, still-filtered bytes of the chunk as stored in the file.
+    filter_mask : int
+        Bit mask returned by ``read_direct_chunk``; a set bit means the filter
+        at that position in the pipeline was skipped for this chunk.
+    pipeline : Sequence
+        The dataset filter pipeline, in write order.
+    dtype : numpy.dtype
+        Element type of the dataset.
+    chunk_shape : tuple[int, ...]
+        Shape of a full chunk.
+
+    Returns
+    -------
+    numpy.ndarray
+        The decoded chunk, with shape `chunk_shape` and dtype `dtype`.
+
+    Raises
+    ------
+    NotImplementedError
+        If the pipeline contains a filter this module cannot invert.
+    """
+    data = raw_bytes
+
+    # Filters are applied in order when writing, so undo them in reverse order.
+    for position, (filter_id, cd_values) in reversed(list(enumerate(pipeline))):
+        if filter_mask & (1 << position):
+            # This filter was skipped for this particular chunk
+            continue
+        if isinstance(data, np.ndarray):
+            # Only the innermost (last) filter may emit an array instead of bytes
+            data = data.tobytes()
+        if filter_id == H5Z_FILTER_DEFLATE:
+            data = zlib.decompress(data)
+        elif filter_id == H5Z_FILTER_SHUFFLE:
+            element_size = (cd_values[0] if cd_values else 0) or dtype.itemsize
+            data = _unshuffle(data, element_size)
+        elif filter_id == H5Z_FILTER_FLETCHER32:
+            # Trailing 4-byte checksum, not verified here
+            data = data[:-4]
+        elif filter_id == H5Z_FILTER_BLOSC:
+            data = blosc2.decompress(data)
+        elif filter_id == H5Z_FILTER_BLOSC2:
+            data = _decode_blosc2_cframe(data)
+        else:
+            raise NotImplementedError(f"unsupported HDF5 filter id {filter_id}")
+
+    if isinstance(data, np.ndarray):
+        if data.dtype != dtype:
+            data = data.view(dtype)
+        return data.reshape(chunk_shape)
+    return np.frombuffer(data, dtype=dtype).reshape(chunk_shape)
+
+
+def _write_chunk(
+    out: np.ndarray,
+    offset: tuple,
+    raw_bytes: bytes,
+    filter_mask: int,
+    pipeline: Sequence,
+    chunk_shape: tuple,
+) -> None:
+    """Decode one raw chunk and write it into its place in `out`.
+
+    Chunks map onto disjoint regions of `out`, so this is safe to call
+    concurrently from several threads.
+
+    Parameters
+    ----------
+    out : numpy.ndarray
+        Destination array, shaped like the full dataset.
+    offset : tuple[int, ...]
+        Index in the dataset of the chunk's first element.
+    raw_bytes : bytes
+        Raw, still-filtered chunk bytes.
+    filter_mask : int
+        Per-chunk filter mask from ``read_direct_chunk``.
+    pipeline : Sequence
+        The dataset filter pipeline, in write order.
+    chunk_shape : tuple[int, ...]
+        Shape of a full chunk.
+    """
+    chunk = _decompress_chunk(raw_bytes, filter_mask, pipeline, out.dtype, chunk_shape)
+
+    # Edge chunks are stored padded out to the full chunk shape, so the part
+    # that actually lands in the dataset may be smaller than the chunk itself.
+    target = tuple(
+        slice(start, min(start + size, extent))
+        for start, size, extent in zip(offset, chunk_shape, out.shape)
+    )
+    source = tuple(slice(0, s.stop - s.start) for s in target)
+    out[target] = chunk[source]
+
+
+def _fallback_read(dset: h5py.Dataset, out: np.ndarray) -> None:
+    """Read `dset` into `out` using h5py's ordinary (serial) read path."""
+    if out.flags.c_contiguous:
+        dset.read_direct(out)
+    else:
+        out[...] = dset[()]
+
+
+def _load_dataset_into(
+    dset: h5py.Dataset,
+    out: np.ndarray,
+    executor: concurrent.futures.ThreadPoolExecutor,
+    max_pending: int,
+) -> None:
+    """Fill `out` with the contents of `dset`, decoding chunks in parallel.
+
+    Falls back to a plain h5py read when the dataset is not chunked, is not
+    fully written, or uses a filter that cannot be inverted here.
+
+    Parameters
+    ----------
+    dset : h5py.Dataset
+        Source dataset.
+    out : numpy.ndarray
+        Destination array; must have the same shape and dtype as `dset`.
+    executor : concurrent.futures.ThreadPoolExecutor
+        Pool used to decode chunks.
+    max_pending : int
+        Maximum number of raw chunks held in memory awaiting decoding.
+    """
+    chunk_shape = dset.chunks
+    if chunk_shape is None or dset.size == 0:
+        _fallback_read(dset, out)
+        return
+
+    pipeline = _read_filter_pipeline(dset)
+    if not _pipeline_is_supported(pipeline):
+        _fallback_read(dset, out)
+        return
+
+    dset_id = dset.id
+    try:
+        num_chunks = dset_id.get_num_chunks()
+    except (OSError, AttributeError, ValueError):
+        _fallback_read(dset, out)
+        return
+
+    expected_chunks = 1
+    for extent, size in zip(dset.shape, chunk_shape):
+        expected_chunks *= -(-extent // size)
+    if num_chunks != expected_chunks:
+        # Sparsely allocated dataset; let HDF5 supply the fill values.
+        _fallback_read(dset, out)
+        return
+
+    pending = set()
+    for i in range(num_chunks):
+        offset = dset_id.get_chunk_info(i).chunk_offset
+        filter_mask, raw_bytes = dset_id.read_direct_chunk(offset)
+
+        if len(pending) >= max_pending:
+            done, pending = concurrent.futures.wait(
+                pending, return_when=concurrent.futures.FIRST_COMPLETED
+            )
+            for future in done:
+                future.result()
+
+        pending.add(
+            executor.submit(
+                _write_chunk,
+                out,
+                offset,
+                raw_bytes,
+                filter_mask,
+                pipeline,
+                chunk_shape,
+            )
+        )
+
+    for future in concurrent.futures.as_completed(pending):
+        future.result()
+
+
+def _max_pending(max_workers: int | None) -> int:
+    """Compute how many raw chunks may queue up, given a worker count."""
+    workers = max_workers or (os.cpu_count() or 1)
+    return max(2 * workers, 4)
+
+
+def read_dataset(
+    dset: h5py.Dataset,
+    *,
+    out: np.ndarray | None = None,
+    executor: concurrent.futures.ThreadPoolExecutor | None = None,
+    max_workers: int | None = None,
+) -> np.ndarray:
+    """Read an already-open HDF5 dataset, decoding its chunks in parallel.
+
+    Parameters
+    ----------
+    dset : h5py.Dataset
+        The dataset to read.
+    out : numpy.ndarray, optional
+        Destination array, which must have the same shape and dtype as
+        `dset`.  It may be a non-contiguous view (e.g. a slice of a larger
+        array).  If not given, a new array is allocated.
+    executor : concurrent.futures.ThreadPoolExecutor, optional
+        Pool used to decode chunks.  If not given, a temporary pool is created
+        and shut down before returning.
+    max_workers : int, optional
+        Size of the temporary thread pool.  Ignored when `executor` is given.
+
+    Returns
+    -------
+    numpy.ndarray
+        The dataset contents (same object as `out` when given).
+    """
+    if out is None:
+        out = np.empty(dset.shape, dtype=dset.dtype)
+    else:
+        if tuple(out.shape) != tuple(dset.shape):
+            raise ValueError(
+                f"out has shape {out.shape}, expected {dset.shape}"
+            )
+        if out.dtype != dset.dtype:
+            raise ValueError(f"out has dtype {out.dtype}, expected {dset.dtype}")
+    if executor is not None:
+        workers = getattr(executor, "_max_workers", None)
+        _load_dataset_into(dset, out, executor, _max_pending(workers))
+    else:
+        with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
+            _load_dataset_into(dset, out, pool, _max_pending(max_workers))
+    return out
diff --git a/sharrow/tests/test_datasets.py b/sharrow/tests/test_datasets.py
index 0e1f746..051df15 100644
--- a/sharrow/tests/test_datasets.py
+++ b/sharrow/tests/test_datasets.py
@@ -313,3 +313,141 @@ def test_from_parquet_3d_ignore():
         )
     assert "TIME" not in skims.variables
     assert "DIST" in skims.variables
+
+
+def _write_compressed_omx(path, matrices, complib="zlib", complevel=7, shuffle=True):
+    """Write an OMX file with compressed, oddly-chunked matrix tables."""
+    import tables
+
+    filters = tables.Filters(complevel=complevel, complib=complib, shuffle=shuffle)
+    n1, n2 = next(iter(matrices.values())).shape
+    with openmatrix.open_file(path, mode="w") as out:
+        for name, arr in matrices.items():
+            out.create_carray(
+                "/data",
+                name,
+                obj=arr,
+                filters=filters,
+                # chunk shape that does not evenly divide the array,
+                # to exercise edge-chunk handling
+                chunkshape=(7, 7),
+            )
+        out.create_carray("/lookup", "taz", obj=np.arange(11, 11 + n1))
+        shp = np.empty(2, dtype=int)
+        shp[0], shp[1] = n1, n2
+        out.root._v_attrs.SHAPE = shp
+
+
+def _random_matrices(n=25, seed=42):
+    rng = np.random.default_rng(seed)
+    return {
+        "DIST": rng.random((n, n)),
+        "TIME__AM": (rng.random((n, n)) * 100).astype(np.float32),
+        "TIME__PM": (rng.random((n, n)) * 100).astype(np.float32),
+        "COUNTS": rng.integers(0, 9999, (n, n)).astype(np.int32),
+    }
+
+
+def test_from_omx_compressed_zlib():
+    matrices = _random_matrices()
+    with tempfile.TemporaryDirectory() as tempdir:
+        f = Path(tempdir).joinpath("skims.omx")
+        _write_compressed_omx(f, matrices)
+        with openmatrix.open_file(f, mode="r") as back:
+            ds = sh.dataset.from_omx(back, indexes="taz")
+            ds_renamed = sh.dataset.from_omx(
+                back, indexes="taz", renames={"distance": "DIST"}
+            )
+            ds_limited = sh.dataset.from_omx(back, indexes="taz", renames=["COUNTS"])
+    for name, arr in matrices.items():
+        assert ds[name].dtype == arr.dtype
+        np.testing.assert_array_equal(ds[name].values, arr)
+    assert ds.coords["otaz"].values == approx(np.arange(11, 36))
+    np.testing.assert_array_equal(ds_renamed["distance"].values, matrices["DIST"])
+    assert sorted(ds_limited.data_vars) == ["COUNTS"]
+
+
+def test_from_omx_compressed_h5py_handle():
+    import h5py
+
+    matrices = _random_matrices()
+    with tempfile.TemporaryDirectory() as tempdir:
+        f = Path(tempdir).joinpath("skims.omx")
+        _write_compressed_omx(f, matrices)
+        with h5py.File(f, mode="r") as back:
+            ds = sh.dataset.from_omx(back, indexes="taz")
+    for name, arr in matrices.items():
+        np.testing.assert_array_equal(ds[name].values, arr)
+    assert ds.coords["otaz"].values == approx(np.arange(11, 36))
+
+
+def test_from_omx_3d_compressed_zlib():
+    matrices = _random_matrices()
+    with tempfile.TemporaryDirectory() as tempdir:
+        f = Path(tempdir).joinpath("skims.omx")
+        _write_compressed_omx(f, matrices)
+        skims = sh.dataset.from_omx_3d(
+            str(f),
+            time_periods=["AM", "PM"],
+            max_float_precision=64,
+        ).compute()
+        np.testing.assert_array_equal(skims["DIST"].values, matrices["DIST"])
+        np.testing.assert_array_equal(
+            skims["TIME"].sel(time_period="AM").values, matrices["TIME__AM"]
+        )
+        np.testing.assert_array_equal(
+            skims["TIME"].sel(time_period="PM").values, matrices["TIME__PM"]
+        )
+        assert skims["TIME"].dtype == np.float32
+        assert skims.coords["otaz"].values == approx(np.arange(11, 36))
+
+        # also via an already-open file handle
+        with openmatrix.open_file(f, mode="r") as back:
+            skims2 = sh.dataset.from_omx_3d(
+                back,
+                time_periods=["AM", "PM"],
+                max_float_precision=64,
+            )
+        # data remains readable after the handle is closed
+        xr.testing.assert_equal(skims, skims2.compute())
+
+
+def test_reload_from_omx_3d_compressed():
+    matrices = _random_matrices()
+    with tempfile.TemporaryDirectory() as tempdir:
+        f = Path(tempdir).joinpath("skims.omx")
+        _write_compressed_omx(f, matrices)
+        expected = sh.dataset.from_omx_3d(
+            str(f), time_periods=["AM", "PM"], max_float_precision=32
+        ).compute()
+        blank = xr.zeros_like(expected)
+        assert not blank.equals(expected)
+        sh.dataset.reload_from_omx_3d(blank, [str(f)])
+        xr.testing.assert_equal(blank, expected)
+        # with ignore
+        blank2 = xr.zeros_like(expected)
+        sh.dataset.reload_from_omx_3d(blank2, [str(f)], ignore=["COUNTS"])
+        assert (blank2["COUNTS"].values == 0).all()
+        np.testing.assert_array_equal(blank2["DIST"].values, expected["DIST"].values)
+
+
+def test_from_omx_compressed_blosc():
+    import h5py
+    import hdf5plugin
+
+    rng = np.random.default_rng(7)
+    arr = rng.random((25, 25))
+    with tempfile.TemporaryDirectory() as tempdir:
+        f = Path(tempdir).joinpath("skims.omx")
+        with h5py.File(f, mode="w") as out:
+            out.create_dataset(
+                "data/DIST",
+                data=arr,
+                chunks=(7, 7),
+                **hdf5plugin.Blosc(cname="lz4", clevel=5),
+            )
+            out.create_dataset("lookup/taz", data=np.arange(11, 36))
+            out.attrs["SHAPE"] = np.asarray([25, 25], dtype=int)
+        with h5py.File(f, mode="r") as back:
+            ds = sh.dataset.from_omx(back, indexes="taz")
+    np.testing.assert_array_equal(ds["DIST"].values, arr)

From 7a6dacbc401db17ec5b54f3f56da429ae6136e38 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 29 Jul 2026 03:13:06 +0000
Subject: [PATCH 11/17] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 sharrow/omx_reader.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/sharrow/omx_reader.py b/sharrow/omx_reader.py
index 0abd4b8..e9321b3 100644
--- a/sharrow/omx_reader.py
+++ b/sharrow/omx_reader.py
@@ -385,9 +385,7 @@ def read_dataset(
         out = np.empty(dset.shape, dtype=dset.dtype)
     else:
         if tuple(out.shape) != tuple(dset.shape):
-            raise ValueError(
-                f"out has shape {out.shape}, expected {dset.shape}"
-            )
+            raise ValueError(f"out has shape {out.shape}, expected {dset.shape}")
         if out.dtype != dset.dtype:
             raise ValueError(f"out has dtype {out.dtype}, expected {dset.dtype}")
     if executor is not None:

From deab3a869d76b6fe32e967f4e25f6d9825efad50 Mon Sep 17 00:00:00 2001
From: Jeff Newman <jeff@driftless.xyz>
Date: Wed, 29 Jul 2026 10:19:56 -0500
Subject: [PATCH 12/17] update uv lockfile

---
 uv.lock | 131 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 130 insertions(+), 1 deletion(-)

diff --git a/uv.lock b/uv.lock
index 28b95d7..ba7d0af 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1,5 +1,5 @@
 version = 1
-revision = 2
+revision = 3
 requires-python = ">=3.9"
 resolution-markers = [
     "python_full_version >= '3.12'",
@@ -665,6 +665,127 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/47/71/70db47e4f6ce3e5c37a607355f80da8860a33226be640226ac52cb05ef2e/fsspec-2025.9.0-py3-none-any.whl", hash = "sha256:530dc2a2af60a414a832059574df4a6e10cce927f6f4a78209390fe38955cfb7", size = 199289, upload-time = "2025-09-02T19:10:47.708Z" },
 ]
 
+[[package]]
+name = "h5py"
+version = "3.14.0"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version < '3.10'",
+]
+dependencies = [
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/5d/57/dfb3c5c3f1bf5f5ef2e59a22dec4ff1f3d7408b55bfcefcfb0ea69ef21c6/h5py-3.14.0.tar.gz", hash = "sha256:2372116b2e0d5d3e5e705b7f663f7c8d96fa79a4052d250484ef91d24d6a08f4", size = 424323, upload-time = "2025-06-06T14:06:15.01Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/52/89/06cbb421e01dea2e338b3154326523c05d9698f89a01f9d9b65e1ec3fb18/h5py-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:24df6b2622f426857bda88683b16630014588a0e4155cba44e872eb011c4eaed", size = 3332522, upload-time = "2025-06-06T14:04:13.775Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/e7/6c860b002329e408348735bfd0459e7b12f712c83d357abeef3ef404eaa9/h5py-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6ff2389961ee5872de697054dd5a033b04284afc3fb52dc51d94561ece2c10c6", size = 2831051, upload-time = "2025-06-06T14:04:18.206Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/cd/3dd38cdb7cc9266dc4d85f27f0261680cb62f553f1523167ad7454e32b11/h5py-3.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:016e89d3be4c44f8d5e115fab60548e518ecd9efe9fa5c5324505a90773e6f03", size = 4324677, upload-time = "2025-06-06T14:04:23.438Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/45/e1a754dc7cd465ba35e438e28557119221ac89b20aaebef48282654e3dc7/h5py-3.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1223b902ef0b5d90bcc8a4778218d6d6cd0f5561861611eda59fa6c52b922f4d", size = 4557272, upload-time = "2025-06-06T14:04:28.863Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/06/f9506c1531645829d302c420851b78bb717af808dde11212c113585fae42/h5py-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:852b81f71df4bb9e27d407b43071d1da330d6a7094a588efa50ef02553fa7ce4", size = 2866734, upload-time = "2025-06-06T14:04:33.5Z" },
+    { url = "https://files.pythonhosted.org/packages/61/1b/ad24a8ce846cf0519695c10491e99969d9d203b9632c4fcd5004b1641c2e/h5py-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f30dbc58f2a0efeec6c8836c97f6c94afd769023f44e2bb0ed7b17a16ec46088", size = 3352382, upload-time = "2025-06-06T14:04:37.95Z" },
+    { url = "https://files.pythonhosted.org/packages/36/5b/a066e459ca48b47cc73a5c668e9924d9619da9e3c500d9fb9c29c03858ec/h5py-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:543877d7f3d8f8a9828ed5df6a0b78ca3d8846244b9702e99ed0d53610b583a8", size = 2852492, upload-time = "2025-06-06T14:04:42.092Z" },
+    { url = "https://files.pythonhosted.org/packages/08/0c/5e6aaf221557314bc15ba0e0da92e40b24af97ab162076c8ae009320a42b/h5py-3.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c497600c0496548810047257e36360ff551df8b59156d3a4181072eed47d8ad", size = 4298002, upload-time = "2025-06-06T14:04:47.106Z" },
+    { url = "https://files.pythonhosted.org/packages/21/d4/d461649cafd5137088fb7f8e78fdc6621bb0c4ff2c090a389f68e8edc136/h5py-3.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:723a40ee6505bd354bfd26385f2dae7bbfa87655f4e61bab175a49d72ebfc06b", size = 4516618, upload-time = "2025-06-06T14:04:52.467Z" },
+    { url = "https://files.pythonhosted.org/packages/db/0c/6c3f879a0f8e891625817637fad902da6e764e36919ed091dc77529004ac/h5py-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d2744b520440a996f2dae97f901caa8a953afc055db4673a993f2d87d7f38713", size = 2874888, upload-time = "2025-06-06T14:04:56.95Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/77/8f651053c1843391e38a189ccf50df7e261ef8cd8bfd8baba0cbe694f7c3/h5py-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e0045115d83272090b0717c555a31398c2c089b87d212ceba800d3dc5d952e23", size = 3312740, upload-time = "2025-06-06T14:05:01.193Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/10/20436a6cf419b31124e59fefc78d74cb061ccb22213226a583928a65d715/h5py-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6da62509b7e1d71a7d110478aa25d245dd32c8d9a1daee9d2a42dba8717b047a", size = 2829207, upload-time = "2025-06-06T14:05:05.061Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/19/c8bfe8543bfdd7ccfafd46d8cfd96fce53d6c33e9c7921f375530ee1d39a/h5py-3.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:554ef0ced3571366d4d383427c00c966c360e178b5fb5ee5bb31a435c424db0c", size = 4708455, upload-time = "2025-06-06T14:05:11.528Z" },
+    { url = "https://files.pythonhosted.org/packages/86/f9/f00de11c82c88bfc1ef22633557bfba9e271e0cb3189ad704183fc4a2644/h5py-3.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cbd41f4e3761f150aa5b662df991868ca533872c95467216f2bec5fcad84882", size = 4929422, upload-time = "2025-06-06T14:05:18.399Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/6d/6426d5d456f593c94b96fa942a9b3988ce4d65ebaf57d7273e452a7222e8/h5py-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:bf4897d67e613ecf5bdfbdab39a1158a64df105827da70ea1d90243d796d367f", size = 2862845, upload-time = "2025-06-06T14:05:23.699Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/c2/7efe82d09ca10afd77cd7c286e42342d520c049a8c43650194928bcc635c/h5py-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:aa4b7bbce683379b7bf80aaba68e17e23396100336a8d500206520052be2f812", size = 3289245, upload-time = "2025-06-06T14:05:28.24Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/31/f570fab1239b0d9441024b92b6ad03bb414ffa69101a985e4c83d37608bd/h5py-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ef9603a501a04fcd0ba28dd8f0995303d26a77a980a1f9474b3417543d4c6174", size = 2807335, upload-time = "2025-06-06T14:05:31.997Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/ce/3a21d87896bc7e3e9255e0ad5583ae31ae9e6b4b00e0bcb2a67e2b6acdbc/h5py-3.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8cbaf6910fa3983c46172666b0b8da7b7bd90d764399ca983236f2400436eeb", size = 4700675, upload-time = "2025-06-06T14:05:37.38Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/ec/86f59025306dcc6deee5fda54d980d077075b8d9889aac80f158bd585f1b/h5py-3.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d90e6445ab7c146d7f7981b11895d70bc1dd91278a4f9f9028bc0c95e4a53f13", size = 4921632, upload-time = "2025-06-06T14:05:43.464Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/6d/0084ed0b78d4fd3e7530c32491f2884140d9b06365dac8a08de726421d4a/h5py-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:ae18e3de237a7a830adb76aaa68ad438d85fe6e19e0d99944a3ce46b772c69b3", size = 2852929, upload-time = "2025-06-06T14:05:47.659Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/ac/9ea82488c8790ee5b6ad1a807cd7dc3b9dadfece1cd0e0e369f68a7a8937/h5py-3.14.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f5cc1601e78027cedfec6dd50efb4802f018551754191aeb58d948bd3ec3bd7a", size = 3345097, upload-time = "2025-06-06T14:05:51.984Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/bc/a172ecaaf287e3af2f837f23b470b0a2229c79555a0da9ac8b5cc5bed078/h5py-3.14.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5e59d2136a8b302afd25acdf7a89b634e0eb7c66b1a211ef2d0457853768a2ef", size = 2843320, upload-time = "2025-06-06T14:05:55.754Z" },
+    { url = "https://files.pythonhosted.org/packages/66/40/b423b57696514e05aa7bb06150ef96667d0e0006cc6de7ab52c71734ab51/h5py-3.14.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:573c33ad056ac7c1ab6d567b6db9df3ffc401045e3f605736218f96c1e0490c6", size = 4326368, upload-time = "2025-06-06T14:06:00.782Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/07/e088f89f04fdbe57ddf9de377f857158d3daa38cf5d0fb20ef9bd489e313/h5py-3.14.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccbe17dc187c0c64178f1a10aa274ed3a57d055117588942b8a08793cc448216", size = 4559686, upload-time = "2025-06-06T14:06:07.416Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/e4/fb8032d0e5480b1db9b419b5b50737b61bb3c7187c49d809975d62129fb0/h5py-3.14.0-cp39-cp39-win_amd64.whl", hash = "sha256:4f025cf30ae738c4c4e38c7439a761a71ccfcce04c2b87b2a2ac64e8c5171d43", size = 2877166, upload-time = "2025-06-06T14:06:13.05Z" },
+]
+
+[[package]]
+name = "h5py"
+version = "3.16.0"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version >= '3.12'",
+    "python_full_version == '3.11.*'",
+    "python_full_version == '3.10.*'",
+]
+dependencies = [
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/db/33/acd0ce6863b6c0d7735007df01815403f5589a21ff8c2e1ee2587a38f548/h5py-3.16.0.tar.gz", hash = "sha256:a0dbaad796840ccaa67a4c144a0d0c8080073c34c76d5a6941d6818678ef2738", size = 446526, upload-time = "2026-03-06T13:49:08.07Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3a/6b/231413e58a787a89b316bb0d1777da3c62257e4797e09afd8d17ad3549dc/h5py-3.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e06f864bedb2c8e7c1358e6c73af48519e317457c444d6f3d332bb4e8fa6d7d9", size = 3724137, upload-time = "2026-03-06T13:47:35.242Z" },
+    { url = "https://files.pythonhosted.org/packages/74/f9/557ce3aad0fe8471fb5279bab0fc56ea473858a022c4ce8a0b8f303d64e9/h5py-3.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ec86d4fffd87a0f4cb3d5796ceb5a50123a2a6d99b43e616e5504e66a953eca3", size = 3090112, upload-time = "2026-03-06T13:47:37.634Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/f5/e15b3d0dc8a18e56409a839e6468d6fb589bc5207c917399c2e0706eeb44/h5py-3.16.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:86385ea895508220b8a7e45efa428aeafaa586bd737c7af9ee04661d8d84a10d", size = 4844847, upload-time = "2026-03-06T13:47:39.811Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/92/a8851d936547efe30cc0ce5245feac01f3ec6171f7899bc3f775c72030b3/h5py-3.16.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:8975273c2c5921c25700193b408e28d6bdd0111c37468b2d4e25dcec4cd1d84d", size = 5065352, upload-time = "2026-03-06T13:47:41.489Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/ae/f2adc5d0ca9626db3277a3d87516e124cbc5d0eea0bd79bc085702d04f2c/h5py-3.16.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1677ad48b703f44efc9ea0c3ab284527f81bc4f318386aaaebc5fede6bbae56f", size = 4839173, upload-time = "2026-03-06T13:47:43.586Z" },
+    { url = "https://files.pythonhosted.org/packages/64/0b/e0c8c69da1d8838da023a50cd3080eae5d475691f7636b35eff20bb6ef20/h5py-3.16.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7c4dd4cf5f0a4e36083f73172f6cfc25a5710789269547f132a20975bfe2434c", size = 5076216, upload-time = "2026-03-06T13:47:45.315Z" },
+    { url = "https://files.pythonhosted.org/packages/66/35/d88fd6718832133c885004c61ceeeb24dbd6397ef877dbed6b3a64d6a286/h5py-3.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:bdef06507725b455fccba9c16529121a5e1fbf56aa375f7d9713d9e8ff42454d", size = 3183639, upload-time = "2026-03-06T13:47:47.041Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/95/a825894f3e45cbac7554c4e97314ce886b233a20033787eda755ca8fecc7/h5py-3.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:719439d14b83f74eeb080e9650a6c7aa6d0d9ea0ca7f804347b05fac6fbf18af", size = 3721663, upload-time = "2026-03-06T13:47:49.599Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/3b/38ff88b347c3e346cda1d3fc1b65a7aa75d40632228d8b8a5d7b58508c24/h5py-3.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c3f0a0e136f2e95dd0b67146abb6668af4f1a69c81ef8651a2d316e8e01de447", size = 3087630, upload-time = "2026-03-06T13:47:51.249Z" },
+    { url = "https://files.pythonhosted.org/packages/98/a8/2594cef906aee761601eff842c7dc598bea2b394a3e1c00966832b8eeb7c/h5py-3.16.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:a6fbc5367d4046801f9b7db9191b31895f22f1c6df1f9987d667854cac493538", size = 4823472, upload-time = "2026-03-06T13:47:53.085Z" },
+    { url = "https://files.pythonhosted.org/packages/52/a0/c1f604538ff6db22a0690be2dc44ab59178e115f63c917794e529356ab23/h5py-3.16.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:fb1720028d99040792bb2fb31facb8da44a6f29df7697e0b84f0d79aff2e9bd3", size = 5027150, upload-time = "2026-03-06T13:47:55.043Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/fd/301739083c2fc4fd89950f9bcfce75d6e14b40b0ca3d40e48a8993d1722c/h5py-3.16.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:314b6054fe0b1051c2b0cb2df5cbdab15622fb05e80f202e3b6a5eee0d6fe365", size = 4814544, upload-time = "2026-03-06T13:47:56.893Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/42/2193ed41ccee78baba8fcc0cff2c925b8b9ee3793305b23e1f22c20bf4c7/h5py-3.16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ffbab2fedd6581f6aa31cf1639ca2cb86e02779de525667892ebf4cc9fd26434", size = 5034013, upload-time = "2026-03-06T13:47:59.01Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/20/e6c0ff62ca2ad1a396a34f4380bafccaaf8791ff8fccf3d995a1fc12d417/h5py-3.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:17d1f1630f92ad74494a9a7392ab25982ce2b469fc62da6074c0ce48366a2999", size = 3191673, upload-time = "2026-03-06T13:48:00.626Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/48/239cbe352ac4f2b8243a8e620fa1a2034635f633731493a7ff1ed71e8658/h5py-3.16.0-cp311-cp311-win_arm64.whl", hash = "sha256:85b9c49dd58dc44cf70af944784e2c2038b6f799665d0dcbbc812a26e0faa859", size = 2673834, upload-time = "2026-03-06T13:48:02.579Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/c0/5d4119dba94093bbafede500d3defd2f5eab7897732998c04b54021e530b/h5py-3.16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c5313566f4643121a78503a473f0fb1e6dcc541d5115c44f05e037609c565c4d", size = 3685604, upload-time = "2026-03-06T13:48:04.198Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/42/c84efcc1d4caebafb1ecd8be4643f39c85c47a80fe254d92b8b43b1eadaf/h5py-3.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:42b012933a83e1a558c673176676a10ce2fd3759976a0fedee1e672d1e04fc9d", size = 3061940, upload-time = "2026-03-06T13:48:05.783Z" },
+    { url = "https://files.pythonhosted.org/packages/89/84/06281c82d4d1686fde1ac6b0f307c50918f1c0151062445ab3b6fa5a921d/h5py-3.16.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ff24039e2573297787c3063df64b60aab0591980ac898329a08b0320e0cf2527", size = 5198852, upload-time = "2026-03-06T13:48:07.482Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/e9/1a19e42cd43cc1365e127db6aae85e1c671da1d9a5d746f4d34a50edb577/h5py-3.16.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:dfc21898ff025f1e8e67e194965a95a8d4754f452f83454538f98f8a3fcb207e", size = 5405250, upload-time = "2026-03-06T13:48:09.628Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/8e/9790c1655eabeb85b92b1ecab7d7e62a2069e53baefd58c98f0909c7a948/h5py-3.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:698dd69291272642ffda44a0ecd6cd3bda5faf9621452d255f57ce91487b9794", size = 5190108, upload-time = "2026-03-06T13:48:11.26Z" },
+    { url = "https://files.pythonhosted.org/packages/51/d7/ab693274f1bd7e8c5f9fdd6c7003a88d59bedeaf8752716a55f532924fbb/h5py-3.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2b2c02b0a160faed5fb33f1ba8a264a37ee240b22e049ecc827345d0d9043074", size = 5419216, upload-time = "2026-03-06T13:48:13.322Z" },
+    { url = "https://files.pythonhosted.org/packages/03/c1/0976b235cf29ead553e22f2fb6385a8252b533715e00d0ae52ed7b900582/h5py-3.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:96b422019a1c8975c2d5dadcf61d4ba6f01c31f92bbde6e4649607885fe502d6", size = 3182868, upload-time = "2026-03-06T13:48:15.759Z" },
+    { url = "https://files.pythonhosted.org/packages/14/d9/866b7e570b39070f92d47b0ff1800f0f8239b6f9e45f02363d7112336c1f/h5py-3.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:39c2838fb1e8d97bcf1755e60ad1f3dd76a7b2a475928dc321672752678b96db", size = 2653286, upload-time = "2026-03-06T13:48:17.279Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/9e/6142ebfda0cb6e9349c091eae73c2e01a770b7659255248d637bec54a88b/h5py-3.16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:370a845f432c2c9619db8eed334d1e610c6015796122b0e57aa46312c22617d9", size = 3671808, upload-time = "2026-03-06T13:48:19.737Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/65/5e088a45d0f43cd814bc5bec521c051d42005a472e804b1a36c48dada09b/h5py-3.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:42108e93326c50c2810025aade9eac9d6827524cdccc7d4b75a546e5ab308edb", size = 3045837, upload-time = "2026-03-06T13:48:21.854Z" },
+    { url = "https://files.pythonhosted.org/packages/da/1e/6172269e18cc5a484e2913ced33339aad588e02ba407fafd00d369e22ef3/h5py-3.16.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:099f2525c9dcf28de366970a5fb34879aab20491589fa89ce2863a84218bb524", size = 5193860, upload-time = "2026-03-06T13:48:24.071Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/98/ef2b6fe2903e377cbe870c3b2800d62552f1e3dbe81ce49e1923c53d1c5c/h5py-3.16.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9300ad32dea9dfc5171f94d5f6948e159ed93e4701280b0f508773b3f582f402", size = 5400417, upload-time = "2026-03-06T13:48:25.728Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/81/5b62d760039eed64348c98129d17061fdfc7839fc9c04eaaad6dee1004e4/h5py-3.16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:171038f23bccddfc23f344cadabdfc9917ff554db6a0d417180d2747fe4c75a7", size = 5185214, upload-time = "2026-03-06T13:48:27.436Z" },
+    { url = "https://files.pythonhosted.org/packages/28/c4/532123bcd9080e250696779c927f2cb906c8bf3447df98f5ceb8dcded539/h5py-3.16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7e420b539fb6023a259a1b14d4c9f6df8cf50d7268f48e161169987a57b737ff", size = 5414598, upload-time = "2026-03-06T13:48:29.49Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/d9/a27997f84341fc0dfcdd1fe4179b6ba6c32a7aa880fdb8c514d4dad6fba3/h5py-3.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:18f2bbcd545e6991412253b98727374c356d67caa920e68dc79eab36bf5fedad", size = 3175509, upload-time = "2026-03-06T13:48:31.131Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/23/bb8647521d4fd770c30a76cfc6cb6a2f5495868904054e92f2394c5a78ff/h5py-3.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:656f00e4d903199a1d58df06b711cf3ca632b874b4207b7dbec86185b5c8c7d4", size = 2647362, upload-time = "2026-03-06T13:48:33.411Z" },
+    { url = "https://files.pythonhosted.org/packages/48/3c/7fcd9b4c9eed82e91fb15568992561019ae7a829d1f696b2c844355d95dd/h5py-3.16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9c9d307c0ef862d1cd5714f72ecfafe0a5d7529c44845afa8de9f46e5ba8bd65", size = 3678608, upload-time = "2026-03-06T13:48:35.183Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/b7/9366ed44ced9b7ef357ab48c94205280276db9d7f064aa3012a97227e966/h5py-3.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8c1eff849cdd53cbc73c214c30ebdb6f1bb8b64790b4b4fc36acdb5e43570210", size = 3054773, upload-time = "2026-03-06T13:48:37.139Z" },
+    { url = "https://files.pythonhosted.org/packages/58/a5/4964bc0e91e86340c2bbda83420225b2f770dcf1eb8a39464871ad769436/h5py-3.16.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:e2c04d129f180019e216ee5f9c40b78a418634091c8782e1f723a6ca3658b965", size = 5198886, upload-time = "2026-03-06T13:48:38.879Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/16/d905e7f53e661ce2c24686c38048d8e2b750ffc4350009d41c4e6c6c9826/h5py-3.16.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e4360f15875a532bc7b98196c7592ed4fc92672a57c0a621355961cafb17a6dd", size = 5404883, upload-time = "2026-03-06T13:48:41.324Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/f2/58f34cb74af46d39f4cd18ea20909a8514960c5a3e5b92fd06a28161e0a8/h5py-3.16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3fae9197390c325e62e0a1aa977f2f62d994aa87aab182abbea85479b791197c", size = 5192039, upload-time = "2026-03-06T13:48:43.117Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/ca/934a39c24ce2e2db017268c08da0537c20fa0be7e1549be3e977313fc8f5/h5py-3.16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:43259303989ac8adacc9986695b31e35dba6fd1e297ff9c6a04b7da5542139cc", size = 5421526, upload-time = "2026-03-06T13:48:44.838Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/14/615a450205e1b56d16c6783f5ccd116cde05550faad70ae077c955654a75/h5py-3.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:fa48993a0b799737ba7fd21e2350fa0a60701e58180fae9f2de834bc39a147ab", size = 3183263, upload-time = "2026-03-06T13:48:47.117Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/48/a6faef5ed632cae0c65ac6b214a6614a0b510c3183532c521bdb0055e117/h5py-3.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:1897a771a7f40d05c262fc8f37376ec37873218544b70216872876c627640f63", size = 2663450, upload-time = "2026-03-06T13:48:48.707Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/32/0c8bb8aedb62c772cf7c1d427c7d1951477e8c2835f872bc0a13d1f85f86/h5py-3.16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:15922e485844f77c0b9d275396d435db3baa58292a9c2176a386e072e0cf2491", size = 3760693, upload-time = "2026-03-06T13:48:50.453Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/1f/fcc5977d32d6387c5c9a694afee716a5e20658ac08b3ff24fdec79fb05f2/h5py-3.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:df02dd29bd247f98674634dfe41f89fd7c16ba3d7de8695ec958f58404a4e618", size = 3181305, upload-time = "2026-03-06T13:48:52.221Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/a1/af87f64b9f986889884243643621ebbd4ac72472ba8ec8cec891ac8e2ca1/h5py-3.16.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:0f456f556e4e2cebeebd9d66adf8dc321770a42593494a0b6f0af54a7567b242", size = 5074061, upload-time = "2026-03-06T13:48:54.089Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/d0/146f5eaff3dc246a9c7f6e5e4f42bd45cc613bce16693bcd4d1f7c958bf5/h5py-3.16.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:3e6cb3387c756de6a9492d601553dffea3fe11b5f22b443aac708c69f3f55e16", size = 5279216, upload-time = "2026-03-06T13:48:56.75Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/9d/12a13424f1e604fc7df9497b73c0356fb78c2fb206abd7465ce47226e8fd/h5py-3.16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8389e13a1fd745ad2856873e8187fd10268b2d9677877bb667b41aebd771d8b7", size = 5070068, upload-time = "2026-03-06T13:48:59.169Z" },
+    { url = "https://files.pythonhosted.org/packages/41/8c/bbe98f813722b4873818a8db3e15aa3e625b59278566905ac439725e8070/h5py-3.16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:346df559a0f7dcb31cf8e44805319e2ab24b8957c45e7708ce503b2ec79ba725", size = 5300253, upload-time = "2026-03-06T13:49:02.033Z" },
+    { url = "https://files.pythonhosted.org/packages/32/9e/87e6705b4d6890e7cecdf876e2a7d3e40654a2ae37482d79a6f1b87f7b92/h5py-3.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:4c6ab014ab704b4feaa719ae783b86522ed0bf1f82184704ed3c9e4e3228796e", size = 3381671, upload-time = "2026-03-06T13:49:04.351Z" },
+    { url = "https://files.pythonhosted.org/packages/96/91/9fad90cfc5f9b2489c7c26ad897157bce82f0e9534a986a221b99760b23b/h5py-3.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:faca8fb4e4319c09d83337adc80b2ca7d5c5a343c2d6f1b6388f32cfecca13c1", size = 2740706, upload-time = "2026-03-06T13:49:06.347Z" },
+]
+
+[[package]]
+name = "hdf5plugin"
+version = "7.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "h5py", version = "3.14.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "h5py", version = "3.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3d/64/0fc6b68e5bc671e7b81d67b930fbed3a4e8a2a92dc4af0f7282b2a2ff988/hdf5plugin-7.0.0.tar.gz", hash = "sha256:e6e6b1f8b0c4d2ca87e616ddc31d08330b36207e466357040f95269e5e0401c8", size = 68284761, upload-time = "2026-06-25T20:59:40.102Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/03/5a/00d0f491d420d491b5134ee044cd8ead5103382d88f4c8aee623258a6348/hdf5plugin-7.0.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:e0ff0a81e6319575ffc8bf230b8df43f3f19f6fe96843e113e04c5ba9f7f3141", size = 6941923, upload-time = "2026-06-25T20:59:24.517Z" },
+    { url = "https://files.pythonhosted.org/packages/96/ad/b28f4102e619c262b29bfcd13ccf6799e26f9ff113d2fdce19050ae29448/hdf5plugin-7.0.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:97ea4ff6114223c5e8ccce7e23cc6cd398b58c02f4e988e967aeea914fcb8030", size = 6259223, upload-time = "2026-06-25T20:59:26.246Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/f4/67263173ff61c49800eec4f16b4d84e6a39170be8d4871d4eb0d540b71ee/hdf5plugin-7.0.0-py3-none-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f3ba9f4e2a370340b45e7042d1b1b1577ab259c1ddf00956264836fa33052ef", size = 42789778, upload-time = "2026-06-25T20:59:28.438Z" },
+    { url = "https://files.pythonhosted.org/packages/75/d2/66673e2d0ef8499d08dd7061caa194e4ddec12df73ab512431c60538f675/hdf5plugin-7.0.0-py3-none-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1a4cb2207ec3ac538fc728e4596e9e49aed6c4487a641071fb370c04a361e7bf", size = 45295544, upload-time = "2026-06-25T20:59:31.765Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/0b/855e50e27eab8338c71c3157672c065cd2a2ab38887b3ea4bf128cbd89a1/hdf5plugin-7.0.0-py3-none-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4ad4ab0d3367699d132e61b1cc382a0c640a7dba56536e5105508205ebbe8762", size = 45012013, upload-time = "2026-06-25T20:59:35.029Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/14/32cf2aae083c74b95678875e11d25d0cb9e51a33d27f4218eb9aeacfcd70/hdf5plugin-7.0.0-py3-none-win_amd64.whl", hash = "sha256:2e052af8d7848e8bac92646584617503a08bb9b466cfa810a49ecd93e89b7ffa", size = 3523827, upload-time = "2026-06-25T20:59:37.758Z" },
+]
+
 [[package]]
 name = "idna"
 version = "3.11"
@@ -2843,10 +2964,15 @@ wheels = [
 name = "sharrow"
 source = { editable = "." }
 dependencies = [
+    { name = "blosc2", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "blosc2", version = "3.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
     { name = "dask", version = "2024.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
     { name = "dask", version = "2025.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
     { name = "filelock", version = "3.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
     { name = "filelock", version = "3.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+    { name = "h5py", version = "3.14.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "h5py", version = "3.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+    { name = "hdf5plugin" },
     { name = "networkx", version = "3.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
     { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
@@ -2884,8 +3010,11 @@ dev = [
 
 [package.metadata]
 requires-dist = [
+    { name = "blosc2" },
     { name = "dask" },
     { name = "filelock" },
+    { name = "h5py" },
+    { name = "hdf5plugin" },
     { name = "networkx" },
     { name = "numba", specifier = ">=0.57" },
     { name = "numexpr" },

From e4950674164fcd0f2226f67429706f27d497e785 Mon Sep 17 00:00:00 2001
From: Jeff Newman <jeff@driftless.xyz>
Date: Wed, 29 Jul 2026 11:36:46 -0500
Subject: [PATCH 13/17] Update dask version requirement to 2026 in
 run-tests.yml

---
 .github/workflows/run-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index ba0528f..388f04b 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -202,7 +202,7 @@ jobs:
           uv venv
           source .venv/bin/activate
           uv pip install "black==22.12.0" "coveralls>=4" \
-                         "cytoolz==0.12.2" "dask>=2023.11" "isort>=5.12.0" \
+                         "cytoolz==0.12.2" "dask>=2026" "isort>=5.12.0" \
                          "multimethod<2.0" "nbmake>=1.4.6" "numba>=0.57" \
                          "numpy>=2,<3" "openmatrix==0.3.5.0" \
                          "pandera>=0.30" "pandas>=2,<3" "platformdirs>=3.2" \

From dad18bfe34f2f887a4dfd939d456430233d981ed Mon Sep 17 00:00:00 2001
From: Jeff Newman <jeff@driftless.xyz>
Date: Wed, 29 Jul 2026 15:20:27 -0500
Subject: [PATCH 14/17] Reduce memory pressure in activitysim-examples CI jobs

The two activitysim-examples jobs have been failing with exit code 143
(runner terminated due to memory exhaustion). Mitigations:

- run each collected test in its own pytest process so memory is fully
  released between model runs
- restore/save the persistent sharrow flow cache (including numba
  on-disk cache files) across CI runs, so warm runs skip most numba
  compilation; saved even on failure to seed the cache
- set MALLOC_ARENA_MAX=2 to limit glibc arena memory retention and pin
  NUMBA_NUM_THREADS to the runner core count
- add a background memory monitor that samples system memory every 15s
  and reports a peak-usage summary at job end

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .github/workflows/run-tests.yml | 65 ++++++++++++++++++++++++++++++++-
 1 file changed, 64 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 388f04b..33f556e 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -155,6 +155,11 @@ jobs:
     env:
       python-version: "3.10"
       label: linux-64
+      # limit glibc malloc arenas, which can otherwise retain large amounts
+      # of memory freed during numba compilation
+      MALLOC_ARENA_MAX: "2"
+      # standard GitHub-hosted ubuntu runners have 4 cores
+      NUMBA_NUM_THREADS: "4"
     strategy:
       matrix:
         include:
@@ -197,6 +202,34 @@ jobs:
       - name: Set cache date for year and month
         run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV
 
+      - name: Start memory monitor
+        # samples system memory every 15 seconds so that peak usage can be
+        # reviewed after the job completes (or is killed by the runner)
+        run: |
+          cat > /tmp/mem-monitor.sh <<'EOF'
+          while true; do
+            ts=$(date +%H:%M:%S)
+            read -r used avail <<< "$(free -m | awk '/^Mem:/ {print $3, $7}')"
+            top=$(ps -eo rss=,comm= --sort=-rss | head -1 | awk '{printf "%s_MB=%d", $2, $1/1024}')
+            echo "$ts used_MB=$used avail_MB=$avail top_process_$top"
+            sleep 15
+          done
+          EOF
+          nohup bash /tmp/mem-monitor.sh >> /tmp/mem-monitor.log 2>&1 &
+          echo $! > /tmp/mem-monitor.pid
+
+      - name: Restore sharrow flow cache
+        # restores previously compiled sharrow flows (including numba on-disk
+        # cache files), which greatly reduces both the time and the peak
+        # memory needed to run the examples
+        uses: actions/cache/restore@v4
+        with:
+          path: ~/.cache/ActivitySim
+          key: sharrow-flows-${{ matrix.region-repo }}-${{ env.DATE }}-${{ github.run_id }}
+          restore-keys: |
+            sharrow-flows-${{ matrix.region-repo }}-${{ env.DATE }}-
+            sharrow-flows-${{ matrix.region-repo }}-
+
       - name: Create Virtual Env
         run: |
           uv venv
@@ -226,7 +259,37 @@ jobs:
           path: '${{ matrix.region-repo }}'
 
       - name: Test ${{ matrix.region }}
+        # each collected test is run in its own process, so that memory is
+        # fully released between tests; running them all in a single process
+        # accumulates enough memory to trigger an OOM kill (exit code 143)
+        # on standard GitHub-hosted runners
         run: |
           source .venv/bin/activate
           cd ${{ matrix.region-repo }}
-          python -m pytest ./test
+          readarray -t tests < <(python -m pytest ./test --collect-only -q | grep '::')
+          echo "collected ${#tests[@]} tests"
+          rc=0
+          for t in "${tests[@]}"; do
+            echo "::group::pytest $t"
+            python -m pytest "$t" || rc=1
+            echo "::endgroup::"
+          done
+          exit $rc
+
+      - name: Save sharrow flow cache
+        # saved even when the tests fail, so that flows compiled during a
+        # failed run still warm the cache for subsequent runs
+        if: always()
+        uses: actions/cache/save@v4
+        with:
+          path: ~/.cache/ActivitySim
+          key: sharrow-flows-${{ matrix.region-repo }}-${{ env.DATE }}-${{ github.run_id }}
+
+      - name: Memory monitor report
+        if: always()
+        run: |
+          kill $(cat /tmp/mem-monitor.pid) 2>/dev/null || true
+          echo "== system memory usage over time (15s samples) =="
+          cat /tmp/mem-monitor.log
+          awk '{ for (i=1; i<=NF; i++) if ($i ~ /^used_MB=/) { v=substr($i,9)+0; if (v>max) max=v } }
+               END { print "== peak used_MB=" max " ==" }' /tmp/mem-monitor.log

From 3e2e51085703eb5b65e2a372b07c1b7b4cc48402 Mon Sep 17 00:00:00 2001
From: Jeff Newman <jeff@driftless.xyz>
Date: Wed, 29 Jul 2026 16:17:41 -0500
Subject: [PATCH 15/17] Add swap, in-step memory watchdog for
 activitysim-examples jobs

Iteration 2: per-test process isolation confirmed the two non-sharrow
MTC tests pass in under a minute each, but the sharrow-enabled test
exhausts runner memory on its own during cold-cache numba compilation
(~47 min in). The abrupt runner shutdown also skipped the if:always()
steps, so the flow cache was never saved and the memory report was lost.

Changes:
- add an 8GB swapfile to absorb the transient compilation memory peak
- stream memory samples into the live step log so the data survives an
  abrupt runner shutdown
- add a watchdog that stops pytest gracefully when memory (incl. swap)
  is nearly exhausted, so the cache-save step still runs; each attempt
  then saves whatever flows it compiled, and subsequent runs resume
  from that warm cache instead of starting over

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .github/workflows/run-tests.yml | 72 ++++++++++++++++++++++-----------
 1 file changed, 49 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 33f556e..f34e206 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -202,21 +202,18 @@ jobs:
       - name: Set cache date for year and month
         run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV
 
-      - name: Start memory monitor
-        # samples system memory every 15 seconds so that peak usage can be
-        # reviewed after the job completes (or is killed by the runner)
+      - name: Add swap space
+        # numba compilation of sharrow flows has a large transient memory
+        # peak; extra swap absorbs the spike instead of the runner being
+        # terminated by the host when it exhausts physical memory
         run: |
-          cat > /tmp/mem-monitor.sh <<'EOF'
-          while true; do
-            ts=$(date +%H:%M:%S)
-            read -r used avail <<< "$(free -m | awk '/^Mem:/ {print $3, $7}')"
-            top=$(ps -eo rss=,comm= --sort=-rss | head -1 | awk '{printf "%s_MB=%d", $2, $1/1024}')
-            echo "$ts used_MB=$used avail_MB=$avail top_process_$top"
-            sleep 15
-          done
-          EOF
-          nohup bash /tmp/mem-monitor.sh >> /tmp/mem-monitor.log 2>&1 &
-          echo $! > /tmp/mem-monitor.pid
+          sudo swapon --show || true
+          df -h /mnt
+          sudo fallocate -l 8G /mnt/extra-swapfile
+          sudo chmod 600 /mnt/extra-swapfile
+          sudo mkswap /mnt/extra-swapfile
+          sudo swapon /mnt/extra-swapfile
+          free -h
 
       - name: Restore sharrow flow cache
         # restores previously compiled sharrow flows (including numba on-disk
@@ -259,21 +256,52 @@ jobs:
           path: '${{ matrix.region-repo }}'
 
       - name: Test ${{ matrix.region }}
-        # each collected test is run in its own process, so that memory is
-        # fully released between tests; running them all in a single process
-        # accumulates enough memory to trigger an OOM kill (exit code 143)
-        # on standard GitHub-hosted runners
+        # Each collected test runs in its own process, so memory is fully
+        # released between tests. A watchdog stops a test gracefully when
+        # system memory (incl. swap) is nearly exhausted: a controlled
+        # failure (unlike a runner shutdown) lets later steps still run, in
+        # particular saving the sharrow flow cache so a subsequent attempt
+        # resumes from the already-compiled flows instead of starting over.
         run: |
           source .venv/bin/activate
           cd ${{ matrix.region-repo }}
+
+          mem_line() {
+            free -m | awk '/^Mem:/ {u=$3; a=$7} /^Swap:/ {s=$4} END {print "used_MB=" u " avail_MB=" a " swap_free_MB=" s}'
+          }
+          free_total() {
+            free -m | awk '/^Mem:/ {a=$7} /^Swap:/ {s=$4} END {print a + s}'
+          }
+
+          # stream memory samples into the live log so the data survives
+          # even if the runner is terminated abruptly
+          ( while true; do echo "[mem] $(date +%H:%M:%S) $(mem_line)" | tee -a /tmp/mem-monitor.log; sleep 30; done ) &
+          monitor_pid=$!
+
           readarray -t tests < <(python -m pytest ./test --collect-only -q | grep '::')
           echo "collected ${#tests[@]} tests"
           rc=0
           for t in "${tests[@]}"; do
             echo "::group::pytest $t"
-            python -m pytest "$t" || rc=1
+            python -m pytest "$t" &
+            test_pid=$!
+            while kill -0 "$test_pid" 2>/dev/null; do
+              if [ "$(free_total)" -lt 1024 ]; then
+                echo "[watchdog] $(date +%H:%M:%S) memory nearly exhausted ($(mem_line)); stopping test"
+                kill "$test_pid" 2>/dev/null || true
+                sleep 30
+              fi
+              sleep 5
+            done
+            if wait "$test_pid"; then
+              echo "[ok] $t"
+            else
+              echo "[failed] $t (exit code $?)"
+              rc=1
+            fi
             echo "::endgroup::"
           done
+          kill "$monitor_pid" 2>/dev/null || true
           exit $rc
 
       - name: Save sharrow flow cache
@@ -288,8 +316,6 @@ jobs:
       - name: Memory monitor report
         if: always()
         run: |
-          kill $(cat /tmp/mem-monitor.pid) 2>/dev/null || true
-          echo "== system memory usage over time (15s samples) =="
-          cat /tmp/mem-monitor.log
+          echo "== peak system memory usage =="
           awk '{ for (i=1; i<=NF; i++) if ($i ~ /^used_MB=/) { v=substr($i,9)+0; if (v>max) max=v } }
-               END { print "== peak used_MB=" max " ==" }' /tmp/mem-monitor.log
+               END { print "peak used_MB=" max }' /tmp/mem-monitor.log || true

From 85545f8c758ae963d048380a87657df6f91e9ac7 Mon Sep 17 00:00:00 2001
From: Jeff Newman <jeff@driftless.xyz>
Date: Thu, 30 Jul 2026 09:49:58 -0500
Subject: [PATCH 16/17] uv not conda

---
 docs/intro.md | 52 +++++++++++++++++++--------------------------------
 1 file changed, 19 insertions(+), 33 deletions(-)

diff --git a/docs/intro.md b/docs/intro.md
index f131d20..54ef5f3 100644
--- a/docs/intro.md
+++ b/docs/intro.md
@@ -8,34 +8,21 @@ optimized, runnable code.
 
 ## Installation
 
-Sharrow will soon be available through conda-forge.
+Install Sharrow from [PyPI](https://pypi.org/project/sharrow/) in your `uv` project:
 
 ```shell
-conda install sharrow -c conda-forge
+uv add sharrow
 ```
 
-You can also install from the source code using setuptools and pip.  A number of
-the dependencies are not pure Python packages, and so it's highly recommended that
-you create an environment containing all those dependencies first, and then install
-sharrow itself.  To do so with conda, you can run:
+`uv` resolves and installs Sharrow and its dependencies.
 
-```shell
-conda install -c conda-forge numpy pandas xarray numba pyarrow numexpr filelock
-```
-
-Then clone the [repository](https://github.com/camsys/sharrow), and then from
-the root directory run
-
-```shell
-pip install -e .
-```
+## Development Installation
 
-Alternatively, you can install sharrow plus all the dependencies (including
-additional optional dependencies for development and testing) in a conda environment,
-using the `envs/development.yml` environment to create a `sh-dev` environment:
+To work from source, clone the [repository](https://github.com/activitysim/sharrow)
+and, from its root directory, create a development environment:
 
 ```shell
-conda env create -f envs/development.yml
+uv sync
 ```
 
 ## Testing
@@ -43,17 +30,21 @@ conda env create -f envs/development.yml
 Sharrow includes unit tests both in the `sharrow/tests` directory and embedded
 in the user documentation under `docs`.
 
-To run the test suite after installing sharrow, install (via pypi or conda) pytest and nbmake,
-and run `pytest` in the root directory of the sharrow repository.
+To run the test suite, install the development dependencies and run the following
+from the root directory of the Sharrow repository:
+
+```shell
+uv sync
+uv run pytest
+```
 
 
 ## Code Formatting
 
 Sharrow uses several tools to ensure a consistent code format throughout the project:
 
-- [Black](https://black.readthedocs.io/en/stable/) for standardized code formatting,
-- [Flake8](http://flake8.pycqa.org/en/latest/) for general code quality,
-- [isort](https://github.com/timothycrosley/isort) for standardized order in imports, and
+- [Ruff](https://docs.astral.sh/ruff/) for standardized code formatting, import
+  sorting, and code quality,
 - [nbstripout](https://github.com/kynan/nbstripout) to ensure notebooks are committed
   to the GitHub repository without bulky outputs included.
 
@@ -71,16 +62,11 @@ with `git commit --no-verify`.
 
 ## Building the Documentation
 
-The docs for sharrow are built using [Jupyter Book](https://jupyterbook.org).
-You can install Jupyter Book [via `pip`](https://pip.pypa.io/en/stable/):
-
-```shell
-pip install -U jupyter-book
-```
-or via [`conda-forge`](https://conda-forge.org/):
+The docs for sharrow are built using [Jupyter Book](https://jupyterbook.org). Install
+it with `uv`:
 
 ```shell
-conda install -c conda-forge jupyter-book
+uv tool install jupyter-book
 ```
 
 Then to build the docs, in the root directory of the sharrow repository run

From 99a2856e06b5cedf8804977ad9f09d759d1dfa72 Mon Sep 17 00:00:00 2001
From: Jeff Newman <jeff@driftless.xyz>
Date: Thu, 30 Jul 2026 10:30:59 -0500
Subject: [PATCH 17/17] add loading skims page

---
 docs/_toc.yml                        |   1 +
 docs/walkthrough/loading-skims.ipynb | 303 +++++++++++++++++++++++++++
 2 files changed, 304 insertions(+)
 create mode 100644 docs/walkthrough/loading-skims.ipynb

diff --git a/docs/_toc.yml b/docs/_toc.yml
index 5a18f79..79bec8b 100644
--- a/docs/_toc.yml
+++ b/docs/_toc.yml
@@ -11,4 +11,5 @@ chapters:
       - file: walkthrough/two-dim
       - file: walkthrough/encoding
       - file: walkthrough/sparse
+      - file: walkthrough/loading-skims
   - file: api
diff --git a/docs/walkthrough/loading-skims.ipynb b/docs/walkthrough/loading-skims.ipynb
new file mode 100644
index 0000000..ad8d601
--- /dev/null
+++ b/docs/walkthrough/loading-skims.ipynb
@@ -0,0 +1,303 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "0",
+   "metadata": {},
+   "source": [
+    "# Loading Skims from Files\n",
+    "\n",
+    "Sharrow can load skim data from several common file formats. This walkthrough\n",
+    "uses the same small `autotime` skim to demonstrate three layouts:\n",
+    "\n",
+    "- Zarr stores the three dimensions natively.\n",
+    "- OMX stores one matrix per time period, with the period encoded in the matrix\n",
+    "  name.\n",
+    "- Parquet stores one column per time period, with named origin and destination\n",
+    "  columns identifying the matrix indexes.\n",
+    "\n",
+    "The example creates temporary files so that it can be run without downloading\n",
+    "any data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "from tempfile import TemporaryDirectory\n",
+    "\n",
+    "import numpy as np\n",
+    "import openmatrix\n",
+    "import pandas as pd\n",
+    "import xarray as xr\n",
+    "\n",
+    "import sharrow as sh"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2",
+   "metadata": {},
+   "source": [
+    "## Example skim data\n",
+    "\n",
+    "The source dataset has native `otaz`, `dtaz`, and `timeperiod` dimensions.\n",
+    "The values are deliberately small and distinct so that the loaded data can be\n",
+    "checked easily."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "zones = np.array([101, 102, 103])\n",
+    "timeperiods = [\"AM\", \"PM\"]\n",
+    "autotime = np.array(\n",
+    "    [\n",
+    "        [[1.0, 1.5], [2.0, 2.5], [3.0, 3.5]],\n",
+    "        [[4.0, 4.5], [5.0, 5.5], [6.0, 6.5]],\n",
+    "        [[7.0, 7.5], [8.0, 8.5], [9.0, 9.5]],\n",
+    "    ],\n",
+    "    dtype=np.float32,\n",
+    ")\n",
+    "source_skims = xr.Dataset(\n",
+    "    {\"autotime\": ((\"otaz\", \"dtaz\", \"timeperiod\"), autotime)},\n",
+    "    coords={\"otaz\": zones, \"dtaz\": zones, \"timeperiod\": timeperiods},\n",
+    ")\n",
+    "source_skims"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "temporary_directory = TemporaryDirectory()\n",
+    "data_directory = Path(temporary_directory.name)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5",
+   "metadata": {},
+   "source": [
+    "## Zarr: native three-dimensional data\n",
+    "\n",
+    "Zarr preserves the dimensions and coordinates in the store. `from_zarr` reads\n",
+    "the `_ARRAY_DIMENSIONS` metadata written by xarray, so no dimension mapping is\n",
+    "needed when loading."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "zarr_path = data_directory / \"skims.zarr\"\n",
+    "source_skims.to_zarr(zarr_path, mode=\"w\")\n",
+    "\n",
+    "zarr_skims = sh.dataset.from_zarr(zarr_path)\n",
+    "zarr_skims"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {
+    "tags": [
+     "remove-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "xr.testing.assert_identical(zarr_skims.compute(), source_skims)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8",
+   "metadata": {},
+   "source": [
+    "## OMX: time period in matrix names\n",
+    "\n",
+    "The openmatrix (OMX) standard defines a data format that stores two-dimensional matrix data. To represent a third dimension, one approach is to store a matrix for each time period and include the time period after a double underscore in the matrix name, such as `autotime__AM`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "omx_path = data_directory / \"skims.omx\"\n",
+    "with openmatrix.open_file(omx_path, mode=\"w\") as omx_file:\n",
+    "    for timeperiod in timeperiods:\n",
+    "        omx_file.create_carray(\n",
+    "            \"/data\",\n",
+    "            f\"autotime__{timeperiod}\",\n",
+    "            obj=source_skims.autotime.sel(timeperiod=timeperiod).values,\n",
+    "        )\n",
+    "    omx_file.create_carray(\"/lookup\", \"taz\", obj=zones)\n",
+    "    omx_file.root._v_attrs.SHAPE = np.array([len(zones), len(zones)])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "10",
+   "metadata": {},
+   "source": [
+    "Sharrow includes the code needed to read this format of stored data into a three dimension dataset."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "omx_skims = sh.dataset.from_omx_3d(\n",
+    "    str(omx_path),\n",
+    "    index_names=(\"otaz\", \"dtaz\", \"timeperiod\"),\n",
+    "    time_periods=timeperiods,\n",
+    ")\n",
+    "omx_skims"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12",
+   "metadata": {
+    "tags": [
+     "remove-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "xr.testing.assert_identical(omx_skims.compute(), source_skims)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "13",
+   "metadata": {},
+   "source": [
+    "## Parquet: named indexes and time-period columns\n",
+    "\n",
+    "Sometimes, skim data is stored as a table. This is especially convenient when passing \n",
+    "data to tools which prefer tabular data and don't natively handle multi-dimensional\n",
+    "tensors. The parquet format is widely used for tabular data and can be used to\n",
+    "store skims as a table. Here, `origin` and `destination` name the\n",
+    "first two dimensions, while `autotime__AM` and `autotime__PM` encode the third\n",
+    "dimension in their column names. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "origin, destination = np.meshgrid(zones, zones, indexing=\"ij\")\n",
+    "parquet_data = pd.DataFrame(\n",
+    "    {\n",
+    "        \"origin\": origin.ravel(),\n",
+    "        \"destination\": destination.ravel(),\n",
+    "        \"autotime__AM\": source_skims.autotime.sel(timeperiod=\"AM\").values.ravel(),\n",
+    "        \"autotime__PM\": source_skims.autotime.sel(timeperiod=\"PM\").values.ravel(),\n",
+    "    }\n",
+    ")\n",
+    "parquet_path = data_directory / \"skims.parquet\"\n",
+    "parquet_data.to_parquet(parquet_path, index=False)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "15",
+   "metadata": {},
+   "source": [
+    "Sharrow can also read this format of data back into the preferred three dimension dataset in memory. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "parquet_skims = sh.dataset.from_parquet_3d(\n",
+    "    parquet_path,\n",
+    "    index_names=(\"origin\", \"destination\", \"timeperiod\"),\n",
+    "    time_periods=timeperiods,\n",
+    ")\n",
+    "parquet_skims"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "17",
+   "metadata": {
+    "tags": [
+     "remove-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "expected_parquet_skims = source_skims.rename({\"otaz\": \"origin\", \"dtaz\": \"destination\"})\n",
+    "xr.testing.assert_identical(parquet_skims, expected_parquet_skims)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18",
+   "metadata": {
+    "tags": [
+     "remove-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "temporary_directory.cleanup()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv (3.13.12)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}