From ae8577be2950f167042e6195cc5ed8b554075da6 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 3 Sep 2026 13:37:57 +0200 Subject: [PATCH 01/89] Unify lazy caching for remote arrays --- doc/guides/remote_arrays.md | 34 +++++++-- examples/s3-cat2-access.py | 108 +++++++++++++++++++++++++++ src/blosc2/c2array.py | 5 ++ src/blosc2/schunk.py | 63 ++++++++++++---- tests/ndarray/test_c2array_blocks.py | 82 ++++++++++++++++++++ 5 files changed, 274 insertions(+), 18 deletions(-) create mode 100644 examples/s3-cat2-access.py diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index 2d4a03a3f..d0e43b7b6 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -7,7 +7,7 @@ A Blosc2 array that lives on a server does not have to be downloaded to be used. | Where the array lives | How to open it | |---|---| | Any URL fsspec reaches — `s3://`, `gs://`, `https://`, `zip://`… | `blosc2.open(url, lazy=True)` | -| A [Caterva2](https://ironarray.io/caterva2) server | `blosc2.C2Array(path, urlbase=...)` | +| A [Caterva2](https://ironarray.io/caterva2) server | `blosc2.open(blosc2.URLPath(path, urlbase=...), lazy=True)` | | Anything else | A `read_range()` of your own — see [Your own transport](#your-own-transport) | ```python @@ -16,9 +16,11 @@ import blosc2 # An object store, a web server, a zip on either of them a = blosc2.open("s3://bucket/big.b2nd", lazy=True) -# A Caterva2 server -b = blosc2.C2Array( - "@public/examples/lung-jpeg2000_10x.b2nd", urlbase="https://cat2.cloud/demo" +# A Caterva2 server; add lazy=True for an automatic Proxy cache +b = blosc2.open( + blosc2.URLPath( + "@public/examples/lung-jpeg2000_10x.b2nd", urlbase="https://cat2.cloud/demo" + ) ) a.shape, a.dtype # metadata only; nothing was downloaded @@ -46,7 +48,9 @@ p[10:12, 500:600] # fetched from the server, and written to lung-cache.b2nd That file is an ordinary Blosc2 array holding only the pieces you touched — a few hundred bytes for a freshly opened proxy over a 64 MB dataset, growing as you read. It is a normal `.b2nd`: copy it, ship it, open it with {func}`blosc2.open`. With `mode="a"` a later run picks up where the last one left off. -{func}`blosc2.open` builds the proxy for you and offers the same choice under another name — `cache_storage=` a directory for a cache on disk, nothing for one in memory: +{func}`blosc2.open` builds the proxy for either kind of remote source and offers +the same choice under another name — `cache_storage=` for a cache on disk, +nothing for one in memory: ```python url = "s3://bucket/big.b2nd" @@ -60,6 +64,25 @@ a = blosc2.open(url, lazy=True, cache_storage="./b2cache") a[100:110, :50] # no request ``` +The same interface works for Caterva2: + +```python +url = blosc2.URLPath("@personal/run.b2nd") + +with blosc2.c2context( + urlbase="https://cat2.cloud/demo", + username="me@example.com", + password="secret", +): + a = blosc2.open(url, lazy=True, cache_storage="./b2cache") + a[100:110, :50] +``` + +For authenticated Caterva2 datasets, `cache_storage` must be private to the +current user. Applications serving multiple users must use a separate cache +directory for each user; sharing one between users is not supported. Reopen a +private cache inside an equivalent authenticated {func}`c2context`. + ## Only what a slice touches A chunk is the unit a container is compressed in, and it can be several megabytes. Fetching a whole one to read a corner of it is most of the cost of a remote read, so Blosc2 fetches **blocks** — the smaller pieces a chunk is built from — whenever a slice lands in a small part of a large chunk. @@ -223,5 +246,6 @@ Four things to get right: - {doc}`Tutorial 6 <../tutorials/06.remote_proxy>` — the same ground at a slower pace, with output. - `examples/ndarray/rw-fsspec.py` — every way of reading and writing an fsspec URL, runnable. +- `examples/s3-cat2-access.py` — the same dataset and cache API through HTTPS/fsspec and Caterva2, with timings. - `examples/c2array-traffic.py` — what a remote slice costs in bytes, and what blocks and the cache save, runnable. - {ref}`C2Array`, {ref}`FsspecNDSource`, {ref}`ByteRangeNDSource`, {ref}`Proxy`, {ref}`Traffic` — the reference pages. diff --git a/examples/s3-cat2-access.py b/examples/s3-cat2-access.py new file mode 100644 index 000000000..c39733b46 --- /dev/null +++ b/examples/s3-cat2-access.py @@ -0,0 +1,108 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Compare lazy access to the same array through fsspec and Caterva2. + +The HTTPS path needs the fsspec HTTP dependencies. Install them with: + + pip install "blosc2[fsspec]" aiohttp + +By default, caches are kept under ``./s3-cat2-cache``. Run the example again to +see the first data access served by the cache left by the previous process. +""" + +import argparse +from pathlib import Path +from time import perf_counter + +import numpy as np + +import blosc2 + +CATERVA2_URL = blosc2.URLPath( + "@public/examples/cube-1k-1k-1k.b2nd", + urlbase="https://cat2.cloud/demo", +) +# The same contents are published in this bucket with a ``-2`` suffix. +FSSPEC_URL = "https://blosc2.s3.us-west-001.backblazeb2.com/cube-1k-1k-1k-2.b2nd" +SLICE = np.s_[100:110, 200:300, 400:500] + + +def traffic_text(traffic: blosc2.Traffic | None) -> str: + if traffic is None: + return "traffic unavailable" + return f"{traffic.requests} requests, {traffic.nbytes / 2**20:.3f} MiB" + + +def size_text(size: int) -> str: + return f"{size / 2**20:.3f} MiB" + + +def benchmark(label: str, urlpath, cache_storage: Path) -> np.ndarray: + cache_existed = cache_storage.is_dir() and any(cache_storage.glob("*.b2nd")) + + start = perf_counter() + array = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + open_time = perf_counter() - start + + metadata = (array.shape, array.dtype, array.chunks, array.blocks) + cache_path = Path(array.urlpath).resolve() + + array.traffic.reset() + start = perf_counter() + data = array[SLICE] + first_read_time = perf_counter() - start + first_traffic = traffic_text(array.traffic) + cache_size = cache_path.stat().st_size + + # Open a fresh remote handle over the same on-disk cache. This demonstrates + # that cached data survives the Proxy object, not merely one array access. + del array + start = perf_counter() + reopened = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + reopen_time = perf_counter() - start + + reopened.traffic.reset() + start = perf_counter() + cached = reopened[SLICE] + cached_read_time = perf_counter() - start + cached_traffic = traffic_text(reopened.traffic) + np.testing.assert_array_equal(cached, data) + + print(f"\n{label}") + print(f" metadata: shape={metadata[0]}, dtype={metadata[1]}") + print(f" chunks={metadata[2]}, blocks={metadata[3]}") + print(f" persistent cache: {cache_path} ({'existing' if cache_existed else 'new'})") + print(f" open and remote metadata setup: {open_time:.6f} s") + print(f" first data slice this run: {first_read_time:.6f} s ({first_traffic})") + print(f" cache size after slice: {size_text(cache_size)}") + print(f" reopen persistent cache: {reopen_time:.6f} s") + print(f" same slice after reopen: {cached_read_time:.6f} s ({cached_traffic})") + return data + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--cache-dir", + type=Path, + default=Path("s3-cat2-cache"), + help="persistent cache root (default: ./s3-cat2-cache)", + ) + args = parser.parse_args() + root = args.cache_dir + + print(f"Persistent cache root: {root.resolve()}") + print("Run this command again to reuse these cache files.") + cat2_data = benchmark("Caterva2", CATERVA2_URL, root / "caterva2") + fsspec_data = benchmark("fsspec over HTTPS", FSSPEC_URL, root / "fsspec") + np.testing.assert_array_equal(cat2_data, fsspec_data) + print("\nBoth services returned identical data.") + + +if __name__ == "__main__": + main() diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index 9db9583eb..a2745a155 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -1670,6 +1670,11 @@ def __init__(self, path: str, /, urlbase: str | None = None, auth_token: str | N Create an instance of a remote data file (aka :ref:`C2Array `) urlpath. This is meant to be used in the :func:`blosc2.open` function. + Passing this object to :func:`blosc2.open` returns a :ref:`C2Array`. With + ``lazy=True`` it instead returns a :ref:`Proxy`, using an in-memory cache + by default or a persistent cache when ``cache_storage`` is provided. + Authenticated users sharing a machine must use separate cache directories. + The parameters are the same as for the :meth:`C2Array.__init__`. """ diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 1db3449ca..402456fae 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -1987,11 +1987,17 @@ def _lazy_fsspec_proxy( # None leaves the default where it belongs, on the source itself kwargs = {} if max_concurrency is None else {"max_concurrency": max_concurrency} src = blosc2.FsspecNDSource(urlpath, **kwargs) + return _lazy_remote_proxy(src, urlpath, cache_storage) + + +def _lazy_remote_proxy(src, identity: str, cache_storage: str | pathlib.Path | None): + """Wrap a remote source in a memory or persistent cache.""" if cache_storage is None: return blosc2.Proxy(src) - path = fsspec_cache_path(urlpath, cache_storage, ".b2nd") - if os.path.exists(path) and _cache_stamp(path) != src.stamp: + path = fsspec_cache_path(identity, cache_storage, ".b2nd") + stamp = getattr(src, "stamp", None) + if os.path.exists(path) and _cache_stamp(path) != stamp: # The remote frame was replaced, which makes every cached chunk -- and # every offset they were fetched by -- meaningless blosc2.remove_urlpath(path) @@ -2000,6 +2006,34 @@ def _lazy_fsspec_proxy( return blosc2.Proxy(src, urlpath=path, mode="a") +def _open_c2_urlpath(urlpath: blosc2.URLPath, mode: str, offset: int, kwargs: dict): + """Open a Caterva2 array directly, or through the same lazy cache API as fsspec.""" + if mode != "r": + raise NotImplementedError(f"Caterva2 arrays can only be opened with mode='r', not {mode!r}") + if offset != 0: + raise NotImplementedError("offset is not supported for Caterva2 arrays") + + cache_storage = kwargs.pop("cache_storage", None) + max_concurrency = kwargs.pop("max_concurrency", None) + lazy = kwargs.pop("lazy", False) + requested = [key for key, value in kwargs.items() if value is not None] + if requested: + raise NotImplementedError(f"{', '.join(requested)} is not supported for Caterva2 arrays") + + if not lazy: + if cache_storage is not None: + raise NotImplementedError("cache_storage for a Caterva2 array requires lazy=True") + if max_concurrency is not None: + raise NotImplementedError("max_concurrency is only supported with lazy=True") + return blosc2.C2Array(urlpath.path, urlbase=urlpath.urlbase, auth_token=urlpath.auth_token) + + src = blosc2.C2Array(urlpath.path, urlbase=urlpath.urlbase, auth_token=urlpath.auth_token) + if max_concurrency is not None: + src.max_concurrency = max_concurrency + identity = f"caterva2:{blosc2.c2array._server_url(src.urlbase, src.path)}" + return _lazy_remote_proxy(src, identity, cache_storage) + + def _cache_stamp(path: str): """The remote stamp a cached proxy container was built against, if any. @@ -2098,8 +2132,9 @@ def open( Open modes also define the allowed persistence side effects: - - ``'r'`` never writes to the persistent object or any sidecar/cache file. - Query acceleration and other execution caches remain process-local only. + - ``'r'`` never writes to the persistent object. It writes a local cache + only when ``cache_storage`` explicitly requests one; query acceleration + and other implicit execution caches remain process-local only. - ``'a'`` and ``'w'`` may persist explicit user-visible changes such as data, metadata, and index maintenance, but execution caches and query memoization still remain process-local only. @@ -2108,7 +2143,8 @@ def open( (e.g. in a file containing several such objects). kwargs: dict, optional lazy: bool, optional - Only for fsspec URLs: return a :ref:`Proxy` that leaves the container + For fsspec URLs and Caterva2 :ref:`URLPath` objects, return a + :ref:`Proxy` that leaves the container where it is and reads what a slice touches, in range requests, instead of transferring the whole thing. Contiguous frames holding an :ref:`NDArray` only. A slice landing in a small part of a large chunk @@ -2125,7 +2161,8 @@ def open( hide, where the pool costs about 10 microseconds per chunk and saves nothing. cache_storage: str | pathlib.Path, optional - Only for fsspec URLs: a directory holding this container's local + For fsspec URLs and lazy Caterva2 :ref:`URLPath` objects, a directory + holding this container's local copy — the whole thing, or just the chunks and blocks ``lazy`` has fetched so far. Either way a later run starts from what is already there, and the copy is discarded when the remote no longer matches @@ -2157,7 +2194,8 @@ def open( Returns ------- - out: :ref:`SChunk`, :ref:`NDArray`, :ref:`C2Array`, :ref:`DictStore`, :ref:`EmbedStore`, or :ref:`TreeStore` + out: :ref:`SChunk`, :ref:`NDArray`, :ref:`C2Array`, :ref:`Proxy`, + :ref:`DictStore`, :ref:`EmbedStore`, or :ref:`TreeStore` The object found in the path. Notes @@ -2169,7 +2207,10 @@ def open( :class:`LazyArray`, exiting the context is currently a no-op. * If :paramref:`urlpath` is a :ref:`URLPath` instance, :paramref:`mode` - must be 'r', :paramref:`offset` must be 0, and kwargs cannot be passed. + must be 'r' and :paramref:`offset` must be 0. Without ``lazy=True`` it + returns a :ref:`C2Array`; with ``lazy=True`` it returns a :ref:`Proxy`, + optionally persisted under ``cache_storage``. Authenticated users sharing + a machine must use separate cache directories. * fsspec URLs need the ``fsspec`` extra (``pip install "blosc2[fsspec]"``) and the driver for the protocol (``s3fs``, ``gcsfs``...), which fsspec asks for @@ -2234,11 +2275,7 @@ def open( True """ if isinstance(urlpath, blosc2.URLPath): - if mode != "r" or offset != 0 or kwargs != {}: - raise NotImplementedError( - "Cannot open a C2Array with mode != 'r', or offset != 0 or some kwargs" - ) - return blosc2.C2Array(urlpath.path, urlbase=urlpath.urlbase, auth_token=urlpath.auth_token) + return _open_c2_urlpath(urlpath, mode, offset, kwargs) if isinstance(urlpath, pathlib.PurePath): urlpath = str(urlpath) diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index 4ca8deb40..a5fce661d 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -324,6 +324,88 @@ def _bytes(srv, endpoint): return sum(n for kind, _, n in srv.log if kind == endpoint) +def test_open_urlpath_lazy_memory_cache(server, any_chunk_wants_blocks): + data = _incompressible((200, 200)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) + urlpath = blosc2.URLPath(array.path, urlbase=array.urlbase) + + proxy = blosc2.open(urlpath, lazy=True, max_concurrency=3) + + assert isinstance(proxy, blosc2.Proxy) + assert isinstance(proxy.src, blosc2.C2Array) + assert proxy.src.max_concurrency == 3 + assert proxy.urlpath is None + + result = proxy[0:5, 0:10] + served = len(srv.log) + assert np.array_equal(result, data[0:5, 0:10]) + assert np.array_equal(proxy[0:5, 0:10], result) + assert len(srv.log) == served + + +def test_open_urlpath_lazy_persistent_cache(tmp_path, server, any_chunk_wants_blocks): + data = _incompressible((200, 200)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) + urlpath = blosc2.URLPath(array.path, urlbase=array.urlbase) + cache_storage = tmp_path / "cache" + + proxy = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) + fetches = sum(endpoint == "fetch" for endpoint, _, _ in srv.log) + del proxy + + proxy = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) + assert sum(endpoint == "fetch" for endpoint, _, _ in srv.log) == fetches + assert len(list(cache_storage.glob("*.b2nd"))) == 1 + + +def test_open_urlpath_lazy_uses_c2context_without_persisting_token(tmp_path, server): + token = "session=secret" + data = _incompressible((20, 20)) + array, _ = server(data, chunks=(10, 20), blocks=(5, 10), cookie=token) + urlpath = blosc2.URLPath(array.path) + cache_storage = tmp_path / "cache" + + with blosc2.c2context(urlbase=array.urlbase, auth_token=token): + proxy = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + assert np.array_equal(proxy[0:5, 0:5], data[0:5, 0:5]) + assert proxy.schunk.meta["proxy-source"]["urlpath"][2] is None + + cache = next(cache_storage.glob("*.b2nd")) + reopened = blosc2.open(cache, mode="a") + assert np.array_equal(reopened[0:5, 0:5], data[0:5, 0:5]) + + +def test_open_urlpath_lazy_rebuilds_stale_cache(tmp_path, server, any_chunk_wants_blocks): + data = _incompressible((200, 200)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) + urlpath = blosc2.URLPath(array.path, urlbase=array.urlbase) + cache_storage = tmp_path / "cache" + + proxy = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) + del proxy + + other = _incompressible((200, 200), seed=1) + _replace(srv, other, chunks=(100, 200), blocks=(10, 20)) + + proxy = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + assert np.array_equal(proxy[0:5, 0:10], other[0:5, 0:10]) + + +def test_open_urlpath_cache_options_need_lazy(tmp_path, server): + data = _incompressible((20, 20)) + array, _ = server(data, chunks=(10, 20), blocks=(5, 10)) + urlpath = blosc2.URLPath(array.path, urlbase=array.urlbase) + + assert isinstance(blosc2.open(urlpath), blosc2.C2Array) + with pytest.raises(NotImplementedError, match=r"cache_storage.*lazy=True"): + blosc2.open(urlpath, cache_storage=tmp_path) + with pytest.raises(NotImplementedError, match=r"max_concurrency.*lazy=True"): + blosc2.open(urlpath, max_concurrency=2) + + def test_blocks_are_read_over_ranges(server, any_chunk_wants_blocks): data = _incompressible((200, 200)) array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) From d7d20ccf9a8d9a678151bd5c23a8bbff9098d83d Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 3 Sep 2026 13:47:47 +0200 Subject: [PATCH 02/89] Optimize lazy HTTPS opening --- doc/guides/remote_arrays.md | 5 ++++ src/blosc2/proxy_source.py | 60 +++++++++++++++++++++++++++++++++++-- tests/test_fsspec.py | 46 ++++++++++++++++++++++++++-- 3 files changed, 105 insertions(+), 6 deletions(-) diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index d0e43b7b6..23eda6634 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -29,6 +29,11 @@ a[100:110, :50] # a NumPy array, fetched now `https://` means a plain web server — nginx, a CDN, an S3 website endpoint — anything that answers a `Range` request. A Caterva2 server is *not* reached that way: it names its datasets by root and path, so use {ref}`C2Array`. +A lazy HTTP(S) open takes its frame metadata and remote identity from the same +initial range response, using `ETag` when the server provides one and falling +back to `Last-Modified` and object size. Thus opening needs one network round +trip, and a persistent cache can still detect when the object is replaced. + ## The cache Wrap either of those in a {ref}`Proxy` and what you read is kept: diff --git a/src/blosc2/proxy_source.py b/src/blosc2/proxy_source.py index 8f825d5fb..957834bd7 100644 --- a/src/blosc2/proxy_source.py +++ b/src/blosc2/proxy_source.py @@ -1280,7 +1280,9 @@ def __init__(self, urlpath: str, max_concurrency: int = REMOTE_MAX_CONCURRENCY): fsspec = _import_fsspec(urlpath) fs, path = fsspec.url_to_fs(urlpath) - if fs.isdir(path): + protocols = (fs.protocol,) if isinstance(fs.protocol, str) else fs.protocol + self._http = bool({"http", "https"} & set(protocols)) + if not self._http and fs.isdir(path): raise NotImplementedError( f"{urlpath} is a directory (a sparse frame or a store), which cannot be read " "chunk by chunk; open it with cache_storage= instead" @@ -1290,15 +1292,67 @@ def __init__(self, urlpath: str, max_concurrency: int = REMOTE_MAX_CONCURRENCY): # has gone stale -- and chunk offsets from a replaced frame are garbage. # fsspec's own token, rather than a tuple of the metadata fields we guess # a backend exposes: memory:// has no mtime, which left it size-only. - self.stamp = fs.ukey(path) + # HTTPFileSystem.isdir() sends a GET before the range read below, making + # a lazy open pay two serial network round trips. Its ukey is only a + # hash of the URL and options, so it needs no request either. Capture + # ETag/Last-Modified from the first range response instead: one request + # supplies both the frame header and a stronger identity for the cache. + if self._http: + from fsspec.utils import tokenize + + self.stamp = tokenize(path, fs.kwargs, fs.protocol) + else: + self.stamp = fs.ukey(path) + self._capture_http_headers = self._http super().__init__(urlpath, max_concurrency) def read_range(self, offset: int, size: int) -> bytes: - data = self._fs.cat_file(self._path, start=offset, end=offset + size) + if self._capture_http_headers: + from fsspec.asyn import sync + + data, headers = sync( + self._fs.loop, + _http_cat_file_with_headers, + self._fs, + self._path, + offset, + offset + size, + ) + self.stamp = _http_stamp(self.stamp, headers) + self._capture_http_headers = False + else: + data = self._fs.cat_file(self._path, start=offset, end=offset + size) self.traffic.charge(len(data)) return data +async def _http_cat_file_with_headers(fs, url: str, start: int, end: int): + """HTTPFileSystem.cat_file(), returning the response headers as well.""" + kwargs = fs.kwargs.copy() + headers = kwargs.pop("headers", {}).copy() + headers["Range"] = await fs._process_limits(url, start, end) + kwargs["headers"] = headers + session = await fs.set_session() + async with session.get(fs.encode_url(url), **kwargs) as response: + data = await response.read() + fs._raise_not_found_for_status(response, url) + response_headers = {key.lower(): value for key, value in response.headers.items()} + return data, response_headers + + +def _http_stamp(url_stamp: str, headers: Mapping[str, str]) -> str: + """Combine a URL identity with the strongest validators on an HTTP response.""" + if etag := headers.get("etag"): + return f"{url_stamp}:etag:{etag}" + + modified = headers.get("last-modified", "") + content_range = headers.get("content-range", "") + size = content_range.rpartition("/")[2] if "/" in content_range else headers.get("content-length", "") + if modified or size: + return f"{url_stamp}:modified:{modified}:size:{size}" + return url_stamp + + def convert_dtype(dt: str | DTypeLike): """ Attempts to convert to blosc2.dtype (i.e. numpy dtype) diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index 2edac6d5a..cd0499f8e 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -8,6 +8,7 @@ import contextlib import functools +import hashlib import http.server import os import pathlib @@ -548,17 +549,53 @@ def test_http_url_is_read_through_fsspec(tmp_path): root.mkdir() blosc2.asarray(data, chunks=(50, 200), blocks=(10, 100), urlpath=str(root / "big.b2nd")) - with _ranged_server(root) as urlbase: + with _ranged_server(root) as (urlbase, requests): whole = blosc2.open(f"{urlbase}/big.b2nd") # fetched in one go, as s3:// is assert np.array_equal(whole[:], data) + requests.clear() lazy = blosc2.open(f"{urlbase}/big.b2nd", lazy=True, cache_storage=str(tmp_path / "cs")) assert isinstance(lazy, blosc2.Proxy) assert isinstance(lazy.src, blosc2.FsspecNDSource) - assert lazy.src.stamp is not None # so a cache of it can tell it has moved + assert ":etag:" in lazy.src.stamp + assert requests == ["bytes=0-8191"] # metadata and identity, one round trip assert np.array_equal(lazy[3:5, 100:120], data[3:5, 100:120]) +def test_http_lazy_cache_rebuilt_when_remote_changes(tmp_path): + pytest.importorskip("aiohttp") + path = tmp_path / "www" + path.mkdir() + frame = path / "changing.b2nd" + first = np.arange(40_000, dtype="i4").reshape(200, 200) + second = first + 1 + blosc2.asarray(first, chunks=(50, 200), blocks=(10, 100), urlpath=frame) + + with _ranged_server(path) as (urlbase, _): + url = f"{urlbase}/{frame.name}" + cache = tmp_path / "cache" + lazy = blosc2.open(url, lazy=True, cache_storage=cache) + assert np.array_equal(lazy[3:5, 100:120], first[3:5, 100:120]) + del lazy + + blosc2.asarray(second, chunks=(50, 200), blocks=(10, 100), urlpath=frame, mode="w") + lazy = blosc2.open(url, lazy=True, cache_storage=cache) + assert np.array_equal(lazy[3:5, 100:120], second[3:5, 100:120]) + + +def test_http_stamp_prefers_etag_and_falls_back_to_modified_size(): + stamp = blosc2.proxy_source._http_stamp( + "url", + {"etag": '"abc"', "last-modified": "yesterday", "content-range": "bytes 0-7/100"}, + ) + assert stamp == 'url:etag:"abc"' + + stamp = blosc2.proxy_source._http_stamp( + "url", {"last-modified": "yesterday", "content-range": "bytes 0-7/100"} + ) + assert stamp == "url:modified:yesterday:size:100" + + @contextlib.contextmanager def _ranged_server(root): """A web server over *root* that honours `Range`, which the stock one does not.""" @@ -571,6 +608,7 @@ def log_message(self, *args): def do_GET(self): span = self.headers.get("Range") + self.server.requests.append(span) if not span: return super().do_GET() body = (root / self.path.lstrip("/")).read_bytes() @@ -581,15 +619,17 @@ def do_GET(self): self.send_header("Content-Range", f"bytes {first}-{last}/{len(body)}") self.send_header("Accept-Ranges", "bytes") self.send_header("Content-Length", str(len(part))) + self.send_header("ETag", hashlib.sha256(body).hexdigest()) self.end_headers() self.wfile.write(part) return None handler = functools.partial(Ranged, directory=str(root)) server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + server.requests = [] threading.Thread(target=server.serve_forever, daemon=True).start() try: - yield f"http://127.0.0.1:{server.server_address[1]}" + yield f"http://127.0.0.1:{server.server_address[1]}", server.requests finally: server.shutdown() server.server_close() From 65fdd56a3570a1611036e36f40fed382b3de8e30 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 3 Sep 2026 13:57:41 +0200 Subject: [PATCH 03/89] Clarify fsspec HTTPS example --- doc/getting_started/installation.rst | 9 +++++---- doc/guides/remote_arrays.md | 2 +- .../{s3-cat2-access.py => fsspec-cat2-access.py} | 14 +++++++------- pyproject.toml | 9 +++++---- 4 files changed, 18 insertions(+), 16 deletions(-) rename examples/{s3-cat2-access.py => fsspec-cat2-access.py} (88%) diff --git a/doc/getting_started/installation.rst b/doc/getting_started/installation.rst index b0b086a04..b4df7d7ed 100644 --- a/doc/getting_started/installation.rst +++ b/doc/getting_started/installation.rst @@ -42,10 +42,10 @@ grouped into *extras* that you opt into with the ``blosc2[extra]`` syntax: :doc:`../guides/parquet_to_blosc2`. * - ``fsspec`` - Reading and writing single-file containers through any `fsspec - `_ URL. The driver for each - protocol is a separate install (``s3fs`` for ``s3://``, ``gcsfs`` for - ``gs://``, ``adlfs`` for ``abfs://``...), and credentials are configured - through the driver, not through blosc2. + `_ URL. The HTTP(S) driver is + included. Other protocol drivers are separate installs (``s3fs`` for + ``s3://``, ``gcsfs`` for ``gs://``, ``adlfs`` for ``abfs://``...), and + credentials are configured through the driver, not through blosc2. Install one or more extras by listing them in brackets (quote the argument in shells like ``zsh`` that treat brackets specially): @@ -55,6 +55,7 @@ argument in shells like ``zsh`` that treat brackets specially): pip install "blosc2[tui]" # the b2view terminal browser pip install "blosc2[hires]" # b2view + its high-res view (h key) pip install "blosc2[parquet]" # the Parquet converter + pip install "blosc2[fsspec]" # fsspec URLs, including HTTP(S) pip install "blosc2[fsspec]" s3fs # fsspec URLs, plus the S3 driver pip install "blosc2[tui,parquet]" # several at once diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index 23eda6634..78497f093 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -251,6 +251,6 @@ Four things to get right: - {doc}`Tutorial 6 <../tutorials/06.remote_proxy>` — the same ground at a slower pace, with output. - `examples/ndarray/rw-fsspec.py` — every way of reading and writing an fsspec URL, runnable. -- `examples/s3-cat2-access.py` — the same dataset and cache API through HTTPS/fsspec and Caterva2, with timings. +- `examples/fsspec-cat2-access.py` — the same dataset and cache API through HTTPS/fsspec and Caterva2, with timings. - `examples/c2array-traffic.py` — what a remote slice costs in bytes, and what blocks and the cache save, runnable. - {ref}`C2Array`, {ref}`FsspecNDSource`, {ref}`ByteRangeNDSource`, {ref}`Proxy`, {ref}`Traffic` — the reference pages. diff --git a/examples/s3-cat2-access.py b/examples/fsspec-cat2-access.py similarity index 88% rename from examples/s3-cat2-access.py rename to examples/fsspec-cat2-access.py index c39733b46..94da9cc52 100644 --- a/examples/s3-cat2-access.py +++ b/examples/fsspec-cat2-access.py @@ -7,11 +7,11 @@ """Compare lazy access to the same array through fsspec and Caterva2. -The HTTPS path needs the fsspec HTTP dependencies. Install them with: +The HTTPS path needs the fsspec extra. Install it with: - pip install "blosc2[fsspec]" aiohttp + pip install "blosc2[fsspec]" -By default, caches are kept under ``./s3-cat2-cache``. Run the example again to +By default, caches are kept under ``./fsspec-cat2-cache``. Run the example again to see the first data access served by the cache left by the previous process. """ @@ -27,8 +27,8 @@ "@public/examples/cube-1k-1k-1k.b2nd", urlbase="https://cat2.cloud/demo", ) -# The same contents are published in this bucket with a ``-2`` suffix. -FSSPEC_URL = "https://blosc2.s3.us-west-001.backblazeb2.com/cube-1k-1k-1k-2.b2nd" +# The same contents are published in this Backblaze B2 bucket with a ``-2`` suffix. +FSSPEC_URL = "https://f001.backblazeb2.com/file/blosc2/cube-1k-1k-1k-2.b2nd" SLICE = np.s_[100:110, 200:300, 400:500] @@ -90,8 +90,8 @@ def main() -> None: parser.add_argument( "--cache-dir", type=Path, - default=Path("s3-cat2-cache"), - help="persistent cache root (default: ./s3-cat2-cache)", + default=Path("fsspec-cat2-cache"), + help="persistent cache root (default: ./fsspec-cat2-cache)", ) args = parser.parse_args() root = args.cache_dir diff --git a/pyproject.toml b/pyproject.toml index f52615b1c..b5703539d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,10 +59,11 @@ tui = ["textual", "textual-plotext"] # Adds the high-res 'h' view on top of [tui], rendering a real matplotlib image # (kitty/iTerm2/sixel, or half-cells elsewhere) — matplotlib is the heavy part. hires = ["blosc2[tui]", "textual-image", "matplotlib"] -# Read/write single-file containers through any fsspec URL (s3://, gs://, zip://, -# memory://...). The protocol backends (s3fs, gcsfs, adlfs...) are the caller's -# install: `pip install "blosc2[fsspec]" s3fs`. -fsspec = ["fsspec"] +# Read/write single-file containers through any fsspec URL (https://, s3://, +# gs://, zip://, memory://...). HTTP support is included; the other protocol +# backends (s3fs, gcsfs, adlfs...) are the caller's install: +# `pip install "blosc2[fsspec]" s3fs`. +fsspec = ["fsspec[http]"] [project.scripts] parquet-to-blosc2 = "blosc2.cli.parquet_to_blosc2:main" From 6c6e5a43b417d42c71375f254770103b7254514d Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 3 Sep 2026 18:59:06 +0200 Subject: [PATCH 04/89] Avoid redundant Caterva2 metadata refresh --- examples/fsspec-cat2-access.py | 6 +++++- src/blosc2/proxy.py | 15 +++++++++++---- src/blosc2/schunk.py | 13 +++++++++---- tests/ndarray/test_c2array_blocks.py | 9 +++++++-- 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/examples/fsspec-cat2-access.py b/examples/fsspec-cat2-access.py index 94da9cc52..8cb4e277b 100644 --- a/examples/fsspec-cat2-access.py +++ b/examples/fsspec-cat2-access.py @@ -23,12 +23,16 @@ import blosc2 +# Using the Caterva2 API CATERVA2_URL = blosc2.URLPath( "@public/examples/cube-1k-1k-1k.b2nd", urlbase="https://cat2.cloud/demo", ) +# ...and also using the fsspec path via fetch URL in Caterva2 +FSSPEC_URL = "https://cat2.cloud/demo/api/fetch/@public/examples/cube-1k-1k-1k.b2nd" # The same contents are published in this Backblaze B2 bucket with a ``-2`` suffix. -FSSPEC_URL = "https://f001.backblazeb2.com/file/blosc2/cube-1k-1k-1k-2.b2nd" +# FSSPEC_URL = "https://f001.backblazeb2.com/file/blosc2/cube-1k-1k-1k-2.b2nd" + SLICE = np.s_[100:110, 200:300, 400:500] diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index 5f7481c26..7d76d3378 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -68,7 +68,13 @@ class Proxy(blosc2.Operand): """ def __init__( - self, src: ProxySource or ProxyNDSource, urlpath: str | None = None, mode="a", **kwargs: dict + self, + src: ProxySource or ProxyNDSource, + urlpath: str | None = None, + mode="a", + *, + _refresh_source: bool = True, + **kwargs: dict, ): """ Create a new :ref:`Proxy` to serve as a cache to save accessed chunks locally. @@ -146,9 +152,10 @@ def __init__( # that has outlived someone else's writes would hand over a stamp the # cache still matches and a set of bytes it no longer does. Sources whose # bytes cannot move underneath them do not offer this and are not asked - refresh = getattr(self.src, "refresh_stamp", None) - if refresh is not None: - refresh() + if _refresh_source: + refresh = getattr(self.src, "refresh_stamp", None) + if refresh is not None: + refresh() if self._cache is None and mode == "a" and urlpath is not None and os.path.exists(urlpath): # Reuse the cache left by an earlier run: whatever was fetched then is diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 402456fae..045c47ea5 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -1990,10 +1990,12 @@ def _lazy_fsspec_proxy( return _lazy_remote_proxy(src, urlpath, cache_storage) -def _lazy_remote_proxy(src, identity: str, cache_storage: str | pathlib.Path | None): +def _lazy_remote_proxy( + src, identity: str, cache_storage: str | pathlib.Path | None, *, source_fresh: bool = False +): """Wrap a remote source in a memory or persistent cache.""" if cache_storage is None: - return blosc2.Proxy(src) + return blosc2.Proxy(src, _refresh_source=not source_fresh) path = fsspec_cache_path(identity, cache_storage, ".b2nd") stamp = getattr(src, "stamp", None) @@ -2003,7 +2005,7 @@ def _lazy_remote_proxy(src, identity: str, cache_storage: str | pathlib.Path | N blosc2.remove_urlpath(path) # Proxy stamps the cache with src.stamp itself, and refuses one built against # other bytes; removing it above is what turns that refusal into a refetch - return blosc2.Proxy(src, urlpath=path, mode="a") + return blosc2.Proxy(src, urlpath=path, mode="a", _refresh_source=not source_fresh) def _open_c2_urlpath(urlpath: blosc2.URLPath, mode: str, offset: int, kwargs: dict): @@ -2031,7 +2033,10 @@ def _open_c2_urlpath(urlpath: blosc2.URLPath, mode: str, offset: int, kwargs: di if max_concurrency is not None: src.max_concurrency = max_concurrency identity = f"caterva2:{blosc2.c2array._server_url(src.urlbase, src.path)}" - return _lazy_remote_proxy(src, identity, cache_storage) + # C2Array's constructor has just read api/info. That response supplies both + # the geometry and the stamp against which the cache is checked, so asking + # for it again in Proxy.__init__ only adds a second serial round trip. + return _lazy_remote_proxy(src, identity, cache_storage, source_fresh=True) def _cache_stamp(path: str): diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index a5fce661d..0c57d3c7e 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -329,8 +329,10 @@ def test_open_urlpath_lazy_memory_cache(server, any_chunk_wants_blocks): array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) urlpath = blosc2.URLPath(array.path, urlbase=array.urlbase) + srv.log.clear() proxy = blosc2.open(urlpath, lazy=True, max_concurrency=3) + assert [endpoint for endpoint, _, _ in srv.log] == ["info"] assert isinstance(proxy, blosc2.Proxy) assert isinstance(proxy.src, blosc2.C2Array) assert proxy.src.max_concurrency == 3 @@ -349,14 +351,17 @@ def test_open_urlpath_lazy_persistent_cache(tmp_path, server, any_chunk_wants_bl urlpath = blosc2.URLPath(array.path, urlbase=array.urlbase) cache_storage = tmp_path / "cache" + srv.log.clear() proxy = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + assert [endpoint for endpoint, _, _ in srv.log] == ["info"] assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) - fetches = sum(endpoint == "fetch" for endpoint, _, _ in srv.log) del proxy + srv.log.clear() proxy = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + assert [endpoint for endpoint, _, _ in srv.log] == ["info"] assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) - assert sum(endpoint == "fetch" for endpoint, _, _ in srv.log) == fetches + assert [endpoint for endpoint, _, _ in srv.log] == ["info"] assert len(list(cache_storage.glob("*.b2nd"))) == 1 From bca3cf413555eeb9b576317dc16a991b5000b293 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 4 Sep 2026 07:41:57 +0200 Subject: [PATCH 05/89] Clarify and extend remote array caching --- doc/getting_started/installation.rst | 2 +- doc/guides/remote_arrays.md | 223 ++++++++++++--------------- doc/reference/c2array.rst | 18 ++- doc/reference/fsspecndsource.rst | 5 + doc/tutorials/06.remote_proxy.ipynb | 2 +- examples/fsspec-cat2-access.py | 8 +- examples/ndarray/rw-fsspec.py | 4 +- src/blosc2/c2array.py | 2 +- src/blosc2/proxy_source.py | 6 +- src/blosc2/schunk.py | 127 ++++++++++----- tests/ndarray/test_c2array_blocks.py | 41 +++-- tests/test_fsspec.py | 91 +++++++---- tests/test_fsspec_s3.py | 4 +- 13 files changed, 307 insertions(+), 226 deletions(-) diff --git a/doc/getting_started/installation.rst b/doc/getting_started/installation.rst index b4df7d7ed..36f3213c5 100644 --- a/doc/getting_started/installation.rst +++ b/doc/getting_started/installation.rst @@ -60,7 +60,7 @@ argument in shells like ``zsh`` that treat brackets specially): pip install "blosc2[tui,parquet]" # several at once With the ``fsspec`` extra, :func:`blosc2.open` accepts any fsspec URL, chained -ones included, and reads it whole, through a local cache (``cache_storage=``), or +ones included, and reads it whole, through a local cache (``cache_dir=``), or by fetching only the chunks and blocks a slice touches (``lazy=True``); see :func:`blosc2.open` and :ref:`FsspecNDSource` for what each mode supports. ``examples/ndarray/rw-fsspec.py`` walks through all three plus the write side, diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index 78497f093..4010b67fa 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -1,163 +1,156 @@ # Working with Remote Arrays -A Blosc2 array that lives on a server does not have to be downloaded to be used. Blosc2 opens it where it is, fetches only the pieces a slice touches, and keeps those in a local cache so the next run starts from them. +Blosc2 can open an array without downloading it first. Metadata is read at open time; array data is fetched only when a slice needs it and is then kept in a local cache. -## Three ways in +## Choose a remote route -| Where the array lives | How to open it | -|---|---| -| Any URL fsspec reaches — `s3://`, `gs://`, `https://`, `zip://`… | `blosc2.open(url, lazy=True)` | -| A [Caterva2](https://ironarray.io/caterva2) server | `blosc2.open(blosc2.URLPath(path, urlbase=...), lazy=True)` | -| Anything else | A `read_range()` of your own — see [Your own transport](#your-own-transport) | +The argument passed to {func}`blosc2.open` selects the route: + +| Argument | Route | What it names | +|---|---|---| +| A URL string such as `s3://...` or `https://...` | fsspec | A byte-addressable, standalone `.b2nd` file | +| A {ref}`URLPath` | Caterva2 | One array-like dataset on a Caterva2 server | ```python import blosc2 -# An object store, a web server, a zip on either of them +# fsspec: an object store or plain web server a = blosc2.open("s3://bucket/big.b2nd", lazy=True) -# A Caterva2 server; add lazy=True for an automatic Proxy cache +# Caterva2: a dataset identified by root and path b = blosc2.open( blosc2.URLPath( - "@public/examples/lung-jpeg2000_10x.b2nd", urlbase="https://cat2.cloud/demo" - ) + "@public/examples/lung-jpeg2000_10x.b2nd", + urlbase="https://cat2.cloud/demo", + ), + lazy=True, ) -a.shape, a.dtype # metadata only; nothing was downloaded -a[100:110, :50] # a NumPy array, fetched now +a.shape, a.dtype # metadata is available immediately +a[100:110, :50] # data is fetched now ``` -`https://` means a plain web server — nginx, a CDN, an S3 website endpoint — anything that answers a `Range` request. A Caterva2 server is *not* reached that way: it names its datasets by root and path, so use {ref}`C2Array`. +A `URLPath` always means Caterva2. If its `urlbase` is omitted, the server comes from {func}`blosc2.c2context` or `BLOSC_C2URLBASE`. Other transports can be added with a custom {ref}`ByteRangeNDSource`; see [Use your own transport](#use-your-own-transport). -A lazy HTTP(S) open takes its frame metadata and remote identity from the same -initial range response, using `ETag` when the server provides one and falling -back to `Last-Modified` and object size. Thus opening needs one network round -trip, and a persistent cache can still detect when the object is replaced. +### What each route supports -## The cache +Both routes return a {ref}`Proxy` when opened with `lazy=True`, so slicing and caching work the same way. Their sources differ: -Wrap either of those in a {ref}`Proxy` and what you read is kept: +| Remote object | fsspec URL | Caterva2 `URLPath` | +|---|---|---| +| Standalone contiguous `.b2nd` | Yes | Yes | +| HDF5 dataset | No | Yes | +| NDArray leaf inside `.b2z` | No | Yes | +| Lazy or computed array | No | Yes | +| Whole `.b2z` `TreeStore` or `DictStore` | No | No; open one array-like leaf | -```python -p = blosc2.Proxy(b) # cache in memory, gone when the proxy is -p[10:12, 500:600] # fetched from the server, and kept -p[10:12, 500:600] # read from the cache, no request at all -``` +fsspec supplies byte ranges. Python-Blosc2 parses the `.b2nd` frame to discover its geometry and chunk offsets, making this route direct and efficient for standalone arrays. + +Caterva2 understands dataset paths, array metadata, and slicing. It can therefore expose array-like data that is not stored as a standalone Blosc2 frame, as well as apply authentication or server-side computation. Use Caterva2's navigation API to find a leaf in a remote hierarchy, then open that leaf with a `URLPath`. + +`lazy=True` changes when data is fetched; it does not expand the formats supported by either route. + +## Choose a cache -Where that cache lives is yours to choose, and it is the one decision to make here. Say nothing and it is memory: fast, and it dies with the proxy, which is all a single process reading a slice twice needs. Name a file with `urlpath=` and the cache outlives the run: +Every lazy open creates a cache. By default it lives in memory and disappears with the proxy: ```python -p = blosc2.Proxy(b, urlpath="lung-cache.b2nd", mode="a") -p[10:12, 500:600] # fetched from the server, and written to lung-cache.b2nd +a = blosc2.open("s3://bucket/big.b2nd", lazy=True) +a[10:12, 500:600] # fetched and cached +a[10:12, 500:600] # served from memory ``` -That file is an ordinary Blosc2 array holding only the pieces you touched — a few hundred bytes for a freshly opened proxy over a 64 MB dataset, growing as you read. It is a normal `.b2nd`: copy it, ship it, open it with {func}`blosc2.open`. With `mode="a"` a later run picks up where the last one left off. - -{func}`blosc2.open` builds the proxy for either kind of remote source and offers -the same choice under another name — `cache_storage=` for a cache on disk, -nothing for one in memory: +Set `cache_dir` to let Blosc2 manage a cache file inside a directory: ```python url = "s3://bucket/big.b2nd" -# First run: the slice is fetched, and lands under ./b2cache dir -a = blosc2.open(url, lazy=True, cache_storage="./b2cache") -a[100:110, :50] +a = blosc2.open(url, lazy=True, cache_dir="./b2cache") +a[100:110, :50] # fetched and stored under ./b2cache -# A later run, a different process: same call, served from ./b2cache -a = blosc2.open(url, lazy=True, cache_storage="./b2cache") +# A later process can reuse the same cache. +a = blosc2.open(url, lazy=True, cache_dir="./b2cache") a[100:110, :50] # no request ``` -The same interface works for Caterva2: +Use `cache_path` instead when the cache should have an exact filename: ```python -url = blosc2.URLPath("@personal/run.b2nd") - -with blosc2.c2context( - urlbase="https://cat2.cloud/demo", - username="me@example.com", - password="secret", -): - a = blosc2.open(url, lazy=True, cache_storage="./b2cache") - a[100:110, :50] +a = blosc2.open(url, lazy=True, cache_path="big-cache.b2nd") ``` -For authenticated Caterva2 datasets, `cache_storage` must be private to the -current user. Applications serving multiple users must use a separate cache -directory for each user; sharing one between users is not supported. Reopen a -private cache inside an equivalent authenticated {func}`c2context`. +In both cases, the cache is an ordinary `.b2nd` array that starts small and grows as regions are read. `cache_dir` and `cache_path` are mutually exclusive. -## Only what a slice touches +Authenticated Caterva2 caches must be private to one user. Reopen them under an equivalent authenticated {func}`blosc2.c2context`; do not share a cache directory between users. -A chunk is the unit a container is compressed in, and it can be several megabytes. Fetching a whole one to read a corner of it is most of the cost of a remote read, so Blosc2 fetches **blocks** — the smaller pieces a chunk is built from — whenever a slice lands in a small part of a large chunk. +## Only what a slice touches -You do not ask for this; it happens when it pays. For example: +Blosc2 arrays are compressed in chunks, which are divided into smaller blocks. For a small slice, fetching only its blocks can avoid transferring most of a large chunk. -- On S3, block reads are **5–17x faster** on arrays with multi-megabyte chunks, and **2–5x** on 1 MB ones. -- On cat2.cloud's `kevlar-tomo.b2nd`, a corner slice costs **0.031 MB instead of 2.723 MB**, and a slice touching ten chunks takes **0.14 s against 1.01 s**. +![A proxy fetches missing regions from the remote array into its local cache. The fetch method returns the cache container, while indexing returns only the requested values.](../tutorials/images/remote_proxy.png) -It is never a loss. A slice wanting more than half a chunk's blocks is wanting the chunk, and a fetch that would skip too little to pay for the extra round trip is made whole — both answered from metadata already in hand, before anything is read. Where blocks are not available the read falls back to whole chunks by itself: that happens for a dataset a Caterva2 server *computes* rather than stores (a lazy expression, an HDF5 leaf, a `.b2z` member), and for a server that stops honouring ranges. +Purple regions are cached; red regions are still remote. The grid is schematic: where byte ranges are available, the fetched regions can be blocks within a chunk. `fetch()` fills and returns the cache container, whereas indexing returns only the requested values. -Fetches also overlap: a lazy proxy runs 8 at a time by default. Pass `max_concurrency=1` for a local protocol with no latency to hide. +The proxy chooses blocks or whole chunks automatically. It fetches a whole chunk when most of its blocks are needed or when the source cannot expose block ranges, as with computed Caterva2 datasets. Independent reads overlap, with up to eight concurrent requests by default; use `max_concurrency=1` when concurrency does not help. -A step other than 1 needs a proxy — a bare {ref}`C2Array` refuses one. Through a proxy it is placed on the block grid like any other key: `p[::2]` reads the blocks holding the coordinates it selects and no others, and `[::-1]` costs what its forward twin does. What that saves is `min(step, block extent along that axis)`, so it is nothing where blocks already span the axis whole — a step along the last dimension, usually — and the step's own factor where they do not. On `kevlar-tomo.b2nd`, whose blocks are one row deep, `[::2]` halves the read and `[::5]` cuts it fivefold. +Stepped slices also use the block grid. For example, `p[::5]` can reduce transfers along an axis whose blocks do not already span that axis. A bare {ref}`C2Array` does not accept stepped slices; its proxy does. -### Seeing byte savings +### Measure network traffic -Wall time will not show you any of this: on a fast link a block read and a whole-chunk read take about as long and differ by the compression ratio in *bytes*. Bytes are also what a metered link and a shared server uplink run out of, so they are counted for you. {ref}`C2Array` and {ref}`Proxy` each carry a {ref}`Traffic` under `traffic` — cumulative requests and bytes, tallied at the transport, so the frame index and block offsets are in it too: +{ref}`C2Array` and remote {ref}`Proxy` objects expose cumulative request and byte counts through {ref}`Traffic`: ```python -b = blosc2.C2Array( - "@public/examples/kevlar-tomo.b2nd", urlbase="https://cat2.cloud/demo" +source = blosc2.C2Array( + "@public/examples/kevlar-tomo.b2nd", + urlbase="https://cat2.cloud/demo", ) -p = blosc2.Proxy(b) +p = blosc2.Proxy(source) p.traffic.reset() corner = p[0, :100, :100] -print(p.traffic) # Traffic(requests=4, nbytes=57767) +print(p.traffic) # requests and bytes fetched p.traffic.reset() -p[0, :100, :100] # the same slice, from the cache +p[0, :100, :100] print(p.traffic) # Traffic(requests=0, nbytes=0) ``` -Take two readings and subtract, or `reset()` between them. `Proxy.traffic` is `None` over a local array — nothing crosses a wire there, and a zero would say the traffic was free rather than that it was never measured. `examples/c2array-traffic.py` runs the whole comparison against cat2.cloud's `kevlar-tomo.b2nd`: a 100x100 corner costs 0.055 MB against 1.296 MB for the chunk holding it — 23.5x — and nothing at all on the second read. +Use `reset()` or subtract two readings to measure one operation. `Proxy.traffic` is `None` for a local source because no network transport exists. + +`examples/c2array-traffic.py` compares block, chunk, and cached reads against a live Caterva2 dataset. -## Scattered points +## Retrieve scattered points -A list of coordinates, or a boolean mask, is not a box — but every point it picks still lives in exactly one block, so it is placed on the block grid as exactly as a slice is: +A proxy maps coordinate arrays and boolean masks to the blocks that contain their selected points: ```python -p[rows, :100] # rows is an array of three indices: three blocks, not three chunks -p[mask] # a mask picks coordinates too, and costs the same +p[rows, :100] +p[mask] ``` -Nine scattered points of a 900³ array cost **236 KB in 19 requests** through a proxy, against 1.81 MB for the chunks holding them. +For Caterva2, a bare {ref}`C2Array` can be substantially more efficient: it sends the coordinates to the server, which returns only the selected values. Prefer direct `C2Array` indexing for sparse, one-off point retrieval; prefer a proxy when reuse through a local cache matters. -However, a {ref}`C2Array` does better with no proxy at all: the coordinates go to the server, which gathers the points and sends back those alone — **271 bytes in one request** for the same nine. When you need efficient scattered retrievals, C2Array+Caterva2 is your best friend. +## Handle remote changes -## When the remote changes underneath +A persistent cache records the source identity when one is available. On a later `blosc2.open()` with the same `cache_dir` or `cache_path`, a mismatched cache is discarded and rebuilt automatically. -A cache is only good while the bytes it was filled from are still there. Sources that can name their bytes — an fsspec URL by its token, a Caterva2 array by an identifier the server keeps — are checked against what the cache recorded: +When constructing a proxy directly in append mode, a mismatch is reported instead: ```python -p = blosc2.Proxy(src, urlpath="cache.b2nd", mode="a") -# ValueError: the cache at cache.b2nd was built against different remote bytes; -# pass mode='w' to fetch them anew +p = blosc2.Proxy(source, urlpath="cache.b2nd", mode="a") +# ValueError if cache.b2nd belongs to different remote bytes ``` -`mode="w"` starts the cache empty and refetches. For a source that cannot name its bytes, the cache is adopted on geometry alone — same shape, dtype and partitioning — so an array rewritten in place while its geometry stayed the same is served from the cache as it was. Use `mode="w"` when that is a possibility. +Use `mode="w"` to start that cache again. If a source cannot provide an identity, compatibility is checked only from shape, dtype, chunks, and blocks. Use a fresh cache when such a source may have changed without changing its geometry. -## Filling an array from several writers +## Fill a Caterva2 array concurrently -A Caterva2 array can be *written*, one chunk at a time, by as many processes as it has chunks. Lay the array out empty first — {func}`blosc2.uninit` writes a couple of hundred bytes whatever the shape — upload it to the server, then have each writer post the chunks it owns: +Several writers can fill one Caterva2 array when each chunk is written at most once. First create and upload an uninitialized array with its final geometry: ```python import blosc2 import numpy as np -# Once, before the writers start: an empty array of the final geometry blosc2.uninit( (1_000_000,), dtype=np.float64, @@ -167,50 +160,39 @@ blosc2.uninit( ) ``` -Upload it with the client that comes with Caterva2: - ```sh cat2-client upload run.b2nd @personal/run.b2nd ``` -Then each writer opens it and posts its own chunks: +Each writer compresses and posts the chunks it owns: ```python import math -import blosc2 - a = blosc2.C2Array("@personal/run.b2nd", urlbase="https://cat2.cloud/demo") -itemsize = a.dtype.itemsize chunk = blosc2.compress2( - data, typesize=itemsize, blocksize=math.prod(a.blocks) * itemsize + data, + typesize=a.dtype.itemsize, + blocksize=math.prod(a.blocks) * a.dtype.itemsize, ) -a.update_chunk(nchunk, chunk) -``` - -Each slot is written once. A second write to the same slot raises {class}`blosc2.ChunkAlreadyWritten`, and that refusal is the whole of the coordination — two writers that both think they own a chunk are sorted out by the array, with no lease, lock or registry between them. The loser drops its chunk and moves on: -```python try: a.update_chunk(nchunk, chunk) except blosc2.ChunkAlreadyWritten: - pass # someone else got there first + pass # another writer completed this slot ``` -Writing into an empty slot appends to the file and moves no other chunk, which is what makes a fill cheap and lets a reader follow one without its cached positions going wrong. {meth}`C2Array.written_chunks() ` says how far it has got, straight out of the file's own index — no endpoint of its own, about 2.5 ms over HTTP: +The server serializes updates, and {meth}`C2Array.written_chunks() ` reports progress from the array's index: ```python -written = a.written_chunks() # one bool per chunk -print(f"{written.sum()}/{written.size} chunks in") +written = a.written_chunks() for nchunk in np.flatnonzero(~written): - ... # the work still to do, after a crash + ... # chunks still missing after a restart ``` -What this buys: the server serializes the writes themselves, so what overlaps is the round trip — which over a network is nearly all of the cost. Against a real server, a fill went from **244 ms per chunk serially to 32 ms with 8 writers, 7.6x**. Over loopback, where there is no round trip to hide, it is 1.0x. +## Use your own transport -## Your own transport - -If your frames live somewhere fsspec does not reach — per-request credentials, a signing proxy, a database column, an in-house gateway — supply one method and you get everything above: +Subclass {ref}`ByteRangeNDSource` when the frame lives behind a transport that fsspec cannot use: ```python import boto3 @@ -219,18 +201,18 @@ import blosc2 class S3Source(blosc2.ByteRangeNDSource): def __init__(self, bucket, key): - self._s3 = boto3.client("s3") - self._bucket, self._key = bucket, key - self.stamp = self._s3.head_object(Bucket=bucket, Key=key)["ETag"] + self.s3 = boto3.client("s3") + self.bucket, self.key = bucket, key + self.stamp = self.s3.head_object(Bucket=bucket, Key=key)["ETag"] super().__init__(f"s3://{bucket}/{key}") def read_range(self, offset, size): - answer = self._s3.get_object( - Bucket=self._bucket, - Key=self._key, + response = self.s3.get_object( + Bucket=self.bucket, + Key=self.key, Range=f"bytes={offset}-{offset + size - 1}", ) - data = answer["Body"].read() + data = response["Body"].read() self.traffic.charge(len(data)) return data @@ -238,19 +220,14 @@ class S3Source(blosc2.ByteRangeNDSource): a = blosc2.Proxy(S3Source("bucket", "big.b2nd"), urlpath="cache.b2nd", mode="a") ``` -(For plain S3 you would just use `blosc2.open("s3://bucket/big.b2nd", lazy=True)`; this is the shape of the thing.) - -Four things to get right: +Initialize the transport before `super().__init__()`, because the base constructor immediately reads the frame header. Make `read_range()` thread-safe, set `stamp` so persistent caches can detect changes, and charge the bytes read so traffic measurements remain accurate. -- **Set up the transport before `super().__init__()`.** The base constructor calls `read_range()` straight away to read the file's header. -- **`read_range()` must be thread-safe.** It is called from a thread pool so fetches can overlap. A boto3 *client* is fine; a `Session` or resource is not. -- **Set `stamp` if you can.** It is what lets a cache tell that the remote has changed. Without it the cache is kept on geometry alone. -- **Charge what you read.** End `read_range()` with `self.traffic.charge(len(data))` and your source is counted like the built-in ones — see [Seeing byte savings](#seeing-byte-savings). Skip it and `traffic` reads zero forever, which looks like a free transport rather than an uncounted one. +For ordinary S3 access, use `blosc2.open("s3://bucket/big.b2nd", lazy=True)`; the custom class only illustrates the transport contract. ## See also -- {doc}`Tutorial 6 <../tutorials/06.remote_proxy>` — the same ground at a slower pace, with output. -- `examples/ndarray/rw-fsspec.py` — every way of reading and writing an fsspec URL, runnable. -- `examples/fsspec-cat2-access.py` — the same dataset and cache API through HTTPS/fsspec and Caterva2, with timings. -- `examples/c2array-traffic.py` — what a remote slice costs in bytes, and what blocks and the cache save, runnable. -- {ref}`C2Array`, {ref}`FsspecNDSource`, {ref}`ByteRangeNDSource`, {ref}`Proxy`, {ref}`Traffic` — the reference pages. +- {doc}`Tutorial 6 <../tutorials/06.remote_proxy>` — a step-by-step introduction with output. +- `examples/ndarray/rw-fsspec.py` — fsspec reading and writing examples. +- `examples/fsspec-cat2-access.py` — one dataset and cache through fsspec and Caterva2. +- `examples/c2array-traffic.py` — block, chunk, and cached transfer sizes. +- {ref}`C2Array`, {ref}`FsspecNDSource`, {ref}`ByteRangeNDSource`, {ref}`Proxy`, and {ref}`Traffic` — API reference pages. diff --git a/doc/reference/c2array.rst b/doc/reference/c2array.rst index 9f8caaef2..2085ce11e 100644 --- a/doc/reference/c2array.rst +++ b/doc/reference/c2array.rst @@ -3,16 +3,26 @@ C2Array ======= -This is a class for remote arrays. This kind of array can also work as operand on a LazyExpr, LazyUDF or reduction. +This is a class for one array-like dataset addressed through a Caterva2 server. +The dataset may be a standalone ``.b2nd`` array, an HDF5 dataset, an NDArray +leaf inside a ``.b2z`` store, or a lazy/computed array. A ``C2Array`` does not +represent or navigate a whole remote ``TreeStore`` or ``DictStore``; use +Caterva2 to select a leaf and open that leaf's path. This kind of array can also +work as an operand on a LazyExpr, LazyUDF or reduction. :ref:`URLPath` is +Caterva2-only, including when its ``urlbase`` is omitted and inherited from +:func:`blosc2.c2context`. + +For a comparison with byte-oriented fsspec access, see +:doc:`Working with Remote Arrays <../guides/remote_arrays>`. Wrapped in a :ref:`Proxy`, a stored remote array is read at block granularity: the proxy asks for the blocks a slice touches rather than the chunks they live in, which for a multi-megabyte chunk is a small fraction of the bytes. That rests on the server serving the dataset from a file, ``Range`` header and auth cookie both honoured; a dataset it computes instead (a lazy expression, an -HDF5 leaf) is fetched a whole chunk at a time, as everything was before. Which -one this is takes at most one request to find out, and is decided once -- -:meth:`C2Array.block_source` is what answers it. +HDF5 leaf, or a ``.b2z`` member) is fetched a whole chunk at a time, as +everything was before. Which one this is takes at most one request to find out, +and is decided once -- :meth:`C2Array.block_source` is what answers it. A stored remote array can also be *filled*, by as many writers at once as it has chunks. The array is laid out first -- ``blosc2.uninit`` writes a couple of diff --git a/doc/reference/fsspecndsource.rst b/doc/reference/fsspecndsource.rst index deb2a6fb6..8b53dfff0 100644 --- a/doc/reference/fsspecndsource.rst +++ b/doc/reference/fsspecndsource.rst @@ -7,6 +7,11 @@ A :ref:`ByteRangeNDSource` that serves the chunks of a Blosc2 frame living behind an fsspec URL, reading each one with a range request instead of transferring the whole container. Everything about the frame format, block granularity included, lives in the base class; this adds the fsspec transport. +The URL must name a standalone, contiguous ``.b2nd`` NDArray frame. It cannot +name an HDF5 dataset, a member inside a ``.b2z`` store, a sparse directory +container, or a computed array: fsspec provides bytes, not dataset semantics. +For the Caterva2 alternative and a capability comparison, see +:doc:`Working with Remote Arrays <../guides/remote_arrays>`. For other sources, see :ref:`ProxyNDSource` and :ref:`ProxySource`. ``examples/ndarray/rw-fsspec.py`` is a runnable walkthrough of this and the diff --git a/doc/tutorials/06.remote_proxy.ipynb b/doc/tutorials/06.remote_proxy.ipynb index 9c1cf8a98..41d8f6bc8 100644 --- a/doc/tutorials/06.remote_proxy.ipynb +++ b/doc/tutorials/06.remote_proxy.ipynb @@ -38,7 +38,7 @@ "metadata": {}, "source": [ "## ``C2Array`` class\n", - "Before we look at proxies, it is first necessary to understand how to use Blosc2 to work with remote data, via the ``C2Array`` class. The class implements a (limited) version of the NDArray interface of which we have already seen a lot in previous tutorials. However, it is really a local pointer to a remote array (stored e.g. on a remote server). This means that we can refer to the data, access certain attribute information about it, download portions of the data and even define it in computational expressions, without having to download the entire array into local memory or disk. This is particularly useful when working with large datasets that cannot fit into memory or would take far too long to transfer over the network.\n", + "Before we look at proxies, it is first necessary to understand how to use Blosc2 to work with remote data, via the ``C2Array`` class. The class implements a (limited) version of the NDArray interface of which we have already seen a lot in previous tutorials. However, it is really a local pointer to a remote array (stored e.g. on a remote server). This means that we can refer to the data, access certain attribute information about it, download portions of the data and even define it in computational expressions, without having to download the entire array into local memory or disk. This is particularly useful when working with large datasets that cannot fit into memory or would take far too long to transfer over the network. The [Working with Remote Arrays](https://www.blosc.org/python-blosc2/guides/remote_arrays.html#choosing-between-fsspec-and-caterva2) guide explains when to use Caterva2's semantic dataset route instead of a byte-oriented fsspec URL.\n", "\n", "However, one limitation of this approach is that every time one wants to download a slice of the dataset, the data is fetched over the network - even if the same slice has been downloaded before. This can lead to inefficiencies, especially when working with large datasets or when the same data is accessed multiple times. Proxies offer a solution to this, whilst still preserving the low storage requirements of the ``C2Array`` class.\n", "\n", diff --git a/examples/fsspec-cat2-access.py b/examples/fsspec-cat2-access.py index 8cb4e277b..78e21cfbf 100644 --- a/examples/fsspec-cat2-access.py +++ b/examples/fsspec-cat2-access.py @@ -46,11 +46,11 @@ def size_text(size: int) -> str: return f"{size / 2**20:.3f} MiB" -def benchmark(label: str, urlpath, cache_storage: Path) -> np.ndarray: - cache_existed = cache_storage.is_dir() and any(cache_storage.glob("*.b2nd")) +def benchmark(label: str, urlpath, cache_dir: Path) -> np.ndarray: + cache_existed = cache_dir.is_dir() and any(cache_dir.glob("*.b2nd")) start = perf_counter() - array = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + array = blosc2.open(urlpath, lazy=True, cache_dir=cache_dir) open_time = perf_counter() - start metadata = (array.shape, array.dtype, array.chunks, array.blocks) @@ -67,7 +67,7 @@ def benchmark(label: str, urlpath, cache_storage: Path) -> np.ndarray: # that cached data survives the Proxy object, not merely one array access. del array start = perf_counter() - reopened = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + reopened = blosc2.open(urlpath, lazy=True, cache_dir=cache_dir) reopen_time = perf_counter() - start reopened.traffic.reset() diff --git a/examples/ndarray/rw-fsspec.py b/examples/ndarray/rw-fsspec.py index c7aa286ca..1d7c09da0 100644 --- a/examples/ndarray/rw-fsspec.py +++ b/examples/ndarray/rw-fsspec.py @@ -43,7 +43,7 @@ # starts from the copy that is already there. Cached copies are checked # against the remote on every open, so a replaced array is never served # from a stale cache. - c = blosc2.open(urlpath, cache_storage=cachedir, mmap_mode="r") + c = blosc2.open(urlpath, cache_dir=cachedir, mmap_mode="r") print(f"read cached: {c.shape} (mmapped from {cachedir})") np.testing.assert_array_equal(c[:], a[:]) @@ -51,7 +51,7 @@ # is and each slice fetches only what it touches -- the chunks it lands in, # or just the blocks inside them when the chunks are large enough for that # to pay. This is what you want for an array too big to download. - d = blosc2.open(urlpath, lazy=True, cache_storage=cachedir) + d = blosc2.open(urlpath, lazy=True, cache_dir=cachedir) print(f"read lazy: {type(d).__name__} {d.shape} {d.dtype}") # Only the two chunks covering rows 15..25 are fetched here diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index a2745a155..9286894b9 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -1672,7 +1672,7 @@ def __init__(self, path: str, /, urlbase: str | None = None, auth_token: str | N Passing this object to :func:`blosc2.open` returns a :ref:`C2Array`. With ``lazy=True`` it instead returns a :ref:`Proxy`, using an in-memory cache - by default or a persistent cache when ``cache_storage`` is provided. + by default or a persistent cache when ``cache_dir`` is provided. Authenticated users sharing a machine must use separate cache directories. The parameters are the same as for the :meth:`C2Array.__init__`. diff --git a/src/blosc2/proxy_source.py b/src/blosc2/proxy_source.py index 957834bd7..a775d1629 100644 --- a/src/blosc2/proxy_source.py +++ b/src/blosc2/proxy_source.py @@ -734,7 +734,7 @@ def __init__( except KeyError: raise NotImplementedError( f"{urlpath} has no b2nd metalayer, so it is a plain SChunk rather than an " - "NDArray; read it whole or with cache_storage= instead" + "NDArray; read it whole or with cache_dir= instead" ) from None if dtype_format != 0: raise NotImplementedError(f"unsupported dtype format {dtype_format} in {urlpath}") @@ -1262,7 +1262,7 @@ class FsspecNDSource(ByteRangeNDSource): This is what ``blosc2.open(url, lazy=True)`` builds; wrap it in a :ref:`Proxy` by hand when the cache belongs at a path of your choosing - rather than inside ``cache_storage``:: + rather than inside ``cache_dir``:: src = blosc2.FsspecNDSource("s3://bucket/big.b2nd") a = blosc2.Proxy(src, urlpath="big-cache.b2nd", mode="a") @@ -1285,7 +1285,7 @@ def __init__(self, urlpath: str, max_concurrency: int = REMOTE_MAX_CONCURRENCY): if not self._http and fs.isdir(path): raise NotImplementedError( f"{urlpath} is a directory (a sparse frame or a store), which cannot be read " - "chunk by chunk; open it with cache_storage= instead" + "chunk by chunk; open it with cache_dir= instead" ) self._fs, self._path = fs, path # Identifies the remote bytes, so a cache built against them can tell it diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 045c47ea5..ac23c6a16 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -10,6 +10,7 @@ import builtins import os import pathlib +import warnings import weakref import zipfile from collections import namedtuple @@ -1975,29 +1976,62 @@ def _finalize_special_open(special, urlpath, mode): return special +def _remote_cache_options(kwargs: dict) -> tuple[str | pathlib.Path | None, str | pathlib.Path | None]: + """Pop the public remote-cache options, including the deprecated alias.""" + legacy_present = "cache_storage" in kwargs + cache_storage = kwargs.pop("cache_storage", None) + cache_dir = kwargs.pop("cache_dir", None) + cache_path = kwargs.pop("cache_path", None) + + if legacy_present: + warnings.warn( + "cache_storage is deprecated; use cache_dir instead", + DeprecationWarning, + stacklevel=4, + ) + + selected = [value for value in (cache_storage, cache_dir, cache_path) if value is not None] + if len(selected) > 1: + raise ValueError("cache_storage, cache_dir, and cache_path are mutually exclusive") + return (cache_dir if cache_dir is not None else cache_storage), cache_path + + def _lazy_fsspec_proxy( - urlpath: str, cache_storage: str | pathlib.Path | None, max_concurrency: int | None = None + urlpath: str, + cache_dir: str | pathlib.Path | None, + cache_path: str | pathlib.Path | None, + max_concurrency: int | None = None, ): """Wrap a remote frame in a Proxy that fetches chunks on demand. - Without `cache_storage` the fetched chunks live in memory and die with the - proxy; with it they go to a container under that directory, so a later run - starts from what this one pulled. + Without a cache location the fetched chunks live in memory and die with the + proxy. Otherwise they go to `cache_path`, or to a derived name under + `cache_dir`, so a later run starts from what this one pulled. """ # None leaves the default where it belongs, on the source itself kwargs = {} if max_concurrency is None else {"max_concurrency": max_concurrency} src = blosc2.FsspecNDSource(urlpath, **kwargs) - return _lazy_remote_proxy(src, urlpath, cache_storage) + return _lazy_remote_proxy(src, urlpath, cache_dir, cache_path) def _lazy_remote_proxy( - src, identity: str, cache_storage: str | pathlib.Path | None, *, source_fresh: bool = False + src, + identity: str, + cache_dir: str | pathlib.Path | None, + cache_path: str | pathlib.Path | None, + *, + source_fresh: bool = False, ): """Wrap a remote source in a memory or persistent cache.""" - if cache_storage is None: + if cache_dir is None and cache_path is None: return blosc2.Proxy(src, _refresh_source=not source_fresh) - path = fsspec_cache_path(identity, cache_storage, ".b2nd") + if cache_path is not None: + path = os.fspath(cache_path) + if os.path.isdir(path): + raise ValueError("cache_path must name a file, not a directory") + else: + path = fsspec_cache_path(identity, cache_dir, ".b2nd") stamp = getattr(src, "stamp", None) if os.path.exists(path) and _cache_stamp(path) != stamp: # The remote frame was replaced, which makes every cached chunk -- and @@ -2015,7 +2049,7 @@ def _open_c2_urlpath(urlpath: blosc2.URLPath, mode: str, offset: int, kwargs: di if offset != 0: raise NotImplementedError("offset is not supported for Caterva2 arrays") - cache_storage = kwargs.pop("cache_storage", None) + cache_dir, cache_path = _remote_cache_options(kwargs) max_concurrency = kwargs.pop("max_concurrency", None) lazy = kwargs.pop("lazy", False) requested = [key for key, value in kwargs.items() if value is not None] @@ -2023,8 +2057,8 @@ def _open_c2_urlpath(urlpath: blosc2.URLPath, mode: str, offset: int, kwargs: di raise NotImplementedError(f"{', '.join(requested)} is not supported for Caterva2 arrays") if not lazy: - if cache_storage is not None: - raise NotImplementedError("cache_storage for a Caterva2 array requires lazy=True") + if cache_dir is not None or cache_path is not None: + raise NotImplementedError("cache_dir and cache_path for a Caterva2 array require lazy=True") if max_concurrency is not None: raise NotImplementedError("max_concurrency is only supported with lazy=True") return blosc2.C2Array(urlpath.path, urlbase=urlpath.urlbase, auth_token=urlpath.auth_token) @@ -2036,7 +2070,7 @@ def _open_c2_urlpath(urlpath: blosc2.URLPath, mode: str, offset: int, kwargs: di # C2Array's constructor has just read api/info. That response supplies both # the geometry and the stamp against which the cache is checked, so asking # for it again in Proxy.__init__ only adds a second serial round trip. - return _lazy_remote_proxy(src, identity, cache_storage, source_fresh=True) + return _lazy_remote_proxy(src, identity, cache_dir, cache_path, source_fresh=True) def _cache_stamp(path: str): @@ -2056,9 +2090,9 @@ def _cache_stamp(path: str): def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): """Open a container living behind an fsspec URL. - Without `cache_storage`, the whole object is fetched in one go and rebuilt in + Without `cache_dir`, the whole object is fetched in one go and rebuilt in memory, which is the right thing for a one-shot read of a small container but - only works for single-file ones. With `cache_storage`, the container is + only works for single-file ones. With `cache_dir`, the container is materialized under that directory and opened as an ordinary local path, so every format, `mmap_mode` and `offset` work. With `lazy`, nothing is fetched up front and each slice pulls just the chunks it needs. @@ -2066,7 +2100,7 @@ def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): if mode != "r": raise NotImplementedError(f"fsspec URLs can only be opened with mode='r', not {mode!r}") - cache_storage = kwargs.pop("cache_storage", None) + cache_dir, cache_path = _remote_cache_options(kwargs) max_concurrency = kwargs.pop("max_concurrency", None) if kwargs.pop("lazy", False): if offset != 0: @@ -2074,25 +2108,28 @@ def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): requested = [k for k, v in kwargs.items() if v is not None] if requested: raise NotImplementedError(f"{', '.join(requested)} is not supported with lazy=True") - return _lazy_fsspec_proxy(urlpath, cache_storage, max_concurrency) + return _lazy_fsspec_proxy(urlpath, cache_dir, cache_path, max_concurrency) + + if cache_path is not None: + raise NotImplementedError("cache_path is only supported with lazy=True") if max_concurrency is not None: # Nothing is fetched chunk by chunk here, so there is nothing to overlap raise NotImplementedError("max_concurrency is only supported with lazy=True") - if cache_storage is not None: - return open(localize_fsspec_url(urlpath, cache_storage), mode, offset, **kwargs) + if cache_dir is not None: + return open(localize_fsspec_url(urlpath, cache_dir), mode, offset, **kwargs) if offset != 0: - raise NotImplementedError("offset on an fsspec URL requires passing cache_storage=") + raise NotImplementedError("offset on an fsspec URL requires passing cache_dir=") # Unset options (dparams=None and friends) are not a request for anything requested = [k for k, v in kwargs.items() if v is not None] if requested: - raise NotImplementedError(f"{', '.join(requested)} on an fsspec URL requires passing cache_storage=") + raise NotImplementedError(f"{', '.join(requested)} on an fsspec URL requires passing cache_dir=") if urlpath.split("?", 1)[0].split("#", 1)[0].endswith(".b2d"): raise NotImplementedError( "directory containers (.b2d, sparse frames) on an fsspec URL require " - "passing cache_storage= to fetch them locally first" + "passing cache_dir= to fetch them locally first" ) with fsspec_open(urlpath, "rb") as f: return blosc2.from_cframe(f.read()) @@ -2124,8 +2161,10 @@ def open( ---------- urlpath: str | pathlib.Path | :ref:`URLPath` The path where the :ref:`SChunk` (or :ref:`NDArray`) - is stored. If it is a remote Caterva2 array, a :ref:`URLPath` must be passed: - a server names its datasets by root and path rather than by URL. + is stored. :ref:`URLPath` is exclusively a Caterva2 dataset reference, + including when its ``urlbase`` is omitted and inherited from + :func:`c2context`; a server names its datasets by root and path rather + than by URL. Any URL with a scheme (``s3://``, ``gs://``, ``https://``, ``zip://``, ``memory://``...) is opened through fsspec; see the `Notes` section for the limits. @@ -2138,7 +2177,7 @@ def open( Open modes also define the allowed persistence side effects: - ``'r'`` never writes to the persistent object. It writes a local cache - only when ``cache_storage`` explicitly requests one; query acceleration + only when ``cache_dir`` or ``cache_path`` explicitly requests one; query acceleration and other implicit execution caches remain process-local only. - ``'a'`` and ``'w'`` may persist explicit user-visible changes such as data, metadata, and index maintenance, but execution caches and query memoization @@ -2148,16 +2187,17 @@ def open( (e.g. in a file containing several such objects). kwargs: dict, optional lazy: bool, optional - For fsspec URLs and Caterva2 :ref:`URLPath` objects, return a - :ref:`Proxy` that leaves the container - where it is and reads what a slice touches, in range requests, - instead of transferring the whole thing. Contiguous frames holding an - :ref:`NDArray` only. A slice landing in a small part of a large chunk - costs only the *blocks* it touches, which for the partitions - :func:`blosc2.asarray` picks by default can be a hundredth of the - chunk; chunks small enough to be one cheap request are still fetched - whole. What arrives is kept in memory, or in ``cache_storage`` when - that is given as well. + For an fsspec URL, return a :ref:`Proxy` over a standalone, + contiguous :ref:`NDArray` frame and read the byte ranges a slice + touches. For a Caterva2 :ref:`URLPath`, return a :ref:`Proxy` over + one array-like dataset; stored ``.b2nd`` arrays use byte ranges when + available, while HDF5 datasets, ``.b2z`` leaves, and computed arrays + fall back to semantic chunk requests. Neither form opens a whole + remote store hierarchy. A slice landing in a small part of a large + chunk costs only the *blocks* it touches when ranges are available; + chunks small enough to be one cheap request are still fetched whole. + What arrives is kept in memory, under ``cache_dir``, or at the exact + ``cache_path`` when either is given. max_concurrency: int, optional Only with ``lazy``: how many fetches to run at once, in a thread pool. A slice against an object store is almost entirely round-trip @@ -2165,14 +2205,19 @@ def open( bearable. Defaults to 8; pass 1 for a protocol with no latency to hide, where the pool costs about 10 microseconds per chunk and saves nothing. - cache_storage: str | pathlib.Path, optional - For fsspec URLs and lazy Caterva2 :ref:`URLPath` objects, a directory - holding this container's local + cache_dir: str | pathlib.Path, optional + For fsspec URLs and lazy Caterva2 :ref:`URLPath` objects, a directory holding this container's local copy — the whole thing, or just the chunks and blocks ``lazy`` has fetched so far. Either way a later run starts from what is already there, and the copy is discarded when the remote no longer matches it. There is no default on purpose, so nothing writes to a disk you did not name. + cache_path: str | pathlib.Path, optional + With ``lazy=True``, the exact file to use for the remote array's + persistent proxy cache. Mutually exclusive with ``cache_dir``. + cache_storage: str | pathlib.Path, optional + Deprecated alias for ``cache_dir``. Mutually exclusive with + ``cache_dir`` and ``cache_path``. mmap_mode: str, optional If set, the file will be memory-mapped instead of using the default I/O functions and the `mode` argument will be ignored. @@ -2214,8 +2259,8 @@ def open( * If :paramref:`urlpath` is a :ref:`URLPath` instance, :paramref:`mode` must be 'r' and :paramref:`offset` must be 0. Without ``lazy=True`` it returns a :ref:`C2Array`; with ``lazy=True`` it returns a :ref:`Proxy`, - optionally persisted under ``cache_storage``. Authenticated users sharing - a machine must use separate cache directories. + optionally persisted under ``cache_dir`` or at ``cache_path``. + Authenticated users sharing a machine must use separate caches. * fsspec URLs need the ``fsspec`` extra (``pip install "blosc2[fsspec]"``) and the driver for the protocol (``s3fs``, ``gcsfs``...), which fsspec asks for @@ -2223,8 +2268,8 @@ def open( ``mode != 'r'`` always raises, as object stores have no rename and no locks. A plain URL read rebuilds the object from a cframe held in memory, so it covers ``.b2nd``, ``.b2f`` and ``.b2e`` only -- a ``.b2z`` store is a zip - archive rather than a cframe, and needs ``cache_storage`` like the - directory formats do. ``cache_storage`` and ``lazy`` above lift that, each + archive rather than a cframe, and needs ``cache_dir`` like the directory + formats do. ``cache_dir`` and ``lazy`` above lift that, each in its own way. * Persistent data handling follows a strict no-hidden-writes rule: diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index 0c57d3c7e..50eeb3c64 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -349,20 +349,37 @@ def test_open_urlpath_lazy_persistent_cache(tmp_path, server, any_chunk_wants_bl data = _incompressible((200, 200)) array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) urlpath = blosc2.URLPath(array.path, urlbase=array.urlbase) - cache_storage = tmp_path / "cache" + cache_dir = tmp_path / "cache" srv.log.clear() - proxy = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + proxy = blosc2.open(urlpath, lazy=True, cache_dir=cache_dir) assert [endpoint for endpoint, _, _ in srv.log] == ["info"] assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) del proxy srv.log.clear() - proxy = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + proxy = blosc2.open(urlpath, lazy=True, cache_dir=cache_dir) assert [endpoint for endpoint, _, _ in srv.log] == ["info"] assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) assert [endpoint for endpoint, _, _ in srv.log] == ["info"] - assert len(list(cache_storage.glob("*.b2nd"))) == 1 + assert len(list(cache_dir.glob("*.b2nd"))) == 1 + + +def test_open_urlpath_lazy_exact_cache_path(tmp_path, server, any_chunk_wants_blocks): + data = _incompressible((200, 200)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) + urlpath = blosc2.URLPath(array.path, urlbase=array.urlbase) + cache_path = tmp_path / "chosen.b2nd" + + proxy = blosc2.open(urlpath, lazy=True, cache_path=cache_path) + assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) + assert proxy.urlpath == str(cache_path) + del proxy + + srv.log.clear() + proxy = blosc2.open(urlpath, lazy=True, cache_path=cache_path) + assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) + assert [endpoint for endpoint, _, _ in srv.log] == ["info"] def test_open_urlpath_lazy_uses_c2context_without_persisting_token(tmp_path, server): @@ -370,14 +387,14 @@ def test_open_urlpath_lazy_uses_c2context_without_persisting_token(tmp_path, ser data = _incompressible((20, 20)) array, _ = server(data, chunks=(10, 20), blocks=(5, 10), cookie=token) urlpath = blosc2.URLPath(array.path) - cache_storage = tmp_path / "cache" + cache_dir = tmp_path / "cache" with blosc2.c2context(urlbase=array.urlbase, auth_token=token): - proxy = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + proxy = blosc2.open(urlpath, lazy=True, cache_dir=cache_dir) assert np.array_equal(proxy[0:5, 0:5], data[0:5, 0:5]) assert proxy.schunk.meta["proxy-source"]["urlpath"][2] is None - cache = next(cache_storage.glob("*.b2nd")) + cache = next(cache_dir.glob("*.b2nd")) reopened = blosc2.open(cache, mode="a") assert np.array_equal(reopened[0:5, 0:5], data[0:5, 0:5]) @@ -386,16 +403,16 @@ def test_open_urlpath_lazy_rebuilds_stale_cache(tmp_path, server, any_chunk_want data = _incompressible((200, 200)) array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) urlpath = blosc2.URLPath(array.path, urlbase=array.urlbase) - cache_storage = tmp_path / "cache" + cache_dir = tmp_path / "cache" - proxy = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + proxy = blosc2.open(urlpath, lazy=True, cache_dir=cache_dir) assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) del proxy other = _incompressible((200, 200), seed=1) _replace(srv, other, chunks=(100, 200), blocks=(10, 20)) - proxy = blosc2.open(urlpath, lazy=True, cache_storage=cache_storage) + proxy = blosc2.open(urlpath, lazy=True, cache_dir=cache_dir) assert np.array_equal(proxy[0:5, 0:10], other[0:5, 0:10]) @@ -405,8 +422,8 @@ def test_open_urlpath_cache_options_need_lazy(tmp_path, server): urlpath = blosc2.URLPath(array.path, urlbase=array.urlbase) assert isinstance(blosc2.open(urlpath), blosc2.C2Array) - with pytest.raises(NotImplementedError, match=r"cache_storage.*lazy=True"): - blosc2.open(urlpath, cache_storage=tmp_path) + with pytest.raises(NotImplementedError, match=r"cache_dir.*lazy=True"): + blosc2.open(urlpath, cache_dir=tmp_path) with pytest.raises(NotImplementedError, match=r"max_concurrency.*lazy=True"): blosc2.open(urlpath, max_concurrency=2) diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index cd0499f8e..93f138a35 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -122,26 +122,26 @@ def test_chained_url(tmp_path): @pytest.mark.parametrize("mode", ["a", "w"]) def test_mode_not_supported(mode): with pytest.raises(NotImplementedError): - blosc2.open("memory://x.b2nd", mode=mode, cache_storage="/tmp/nope") + blosc2.open("memory://x.b2nd", mode=mode, cache_dir="/tmp/nope") def test_offset_needs_cache(): - with pytest.raises(NotImplementedError, match="cache_storage"): + with pytest.raises(NotImplementedError, match="cache_dir"): blosc2.open("memory://x.b2nd", offset=32) def test_mmap_needs_cache(): - with pytest.raises(NotImplementedError, match="cache_storage"): + with pytest.raises(NotImplementedError, match="cache_dir"): blosc2.open("memory://x.b2nd", mmap_mode="r") def test_dir_container_needs_cache(): - with pytest.raises(NotImplementedError, match="cache_storage"): + with pytest.raises(NotImplementedError, match="cache_dir"): blosc2.open("memory://store.b2d") def test_dir_container_with_query_needs_cache(): - with pytest.raises(NotImplementedError, match="cache_storage"): + with pytest.raises(NotImplementedError, match="cache_dir"): blosc2.open("memory://store.b2d?version=1") @@ -150,7 +150,7 @@ def test_cached_open(tmp_path): with fsspec.open("memory://c.b2nd", "wb") as f: f.write(a.to_cframe()) - b = blosc2.open("memory://c.b2nd", cache_storage=tmp_path) + b = blosc2.open("memory://c.b2nd", cache_dir=tmp_path) assert np.array_equal(b[:], a[:]) assert any(tmp_path.iterdir()) @@ -161,7 +161,7 @@ def test_cached_open_is_local(tmp_path): with fsspec.open("memory://m.b2nd", "wb") as f: f.write(a.to_cframe()) - b = blosc2.open("memory://m.b2nd", cache_storage=tmp_path, mmap_mode="r") + b = blosc2.open("memory://m.b2nd", cache_dir=tmp_path, mmap_mode="r") assert np.array_equal(b[:], a[:]) @@ -177,20 +177,20 @@ def test_cache_hit_avoids_refetch(tmp_path, monkeypatch): memfs, "_open", lambda self, path, *a, **kw: (fetches.append(path), orig(self, path, *a, **kw))[1] ) - blosc2.open("memory://h.b2nd", cache_storage=tmp_path) + blosc2.open("memory://h.b2nd", cache_dir=tmp_path) assert len(fetches) == 1 - blosc2.open("memory://h.b2nd", cache_storage=tmp_path) + blosc2.open("memory://h.b2nd", cache_dir=tmp_path) assert len(fetches) == 1 def test_cache_refetches_when_remote_changes(tmp_path): with fsspec.open("memory://s.b2nd", "wb") as f: f.write(blosc2.arange(10, dtype="i4").to_cframe()) - assert blosc2.open("memory://s.b2nd", cache_storage=tmp_path).shape == (10,) + assert blosc2.open("memory://s.b2nd", cache_dir=tmp_path).shape == (10,) with fsspec.open("memory://s.b2nd", "wb") as f: f.write(blosc2.arange(20, dtype="i4").to_cframe()) - assert blosc2.open("memory://s.b2nd", cache_storage=tmp_path).shape == (20,) + assert blosc2.open("memory://s.b2nd", cache_dir=tmp_path).shape == (20,) def test_cached_dict_store(tmp_path): @@ -201,7 +201,7 @@ def test_cached_dict_store(tmp_path): dstore["/b"] = blosc2.arange(5, dtype="f8") fsspec.filesystem("memory").put(localstore, "memory://store.b2d", recursive=True) - with blosc2.open("memory://store.b2d", cache_storage=tmp_path / "cache") as dstore: + with blosc2.open("memory://store.b2d", cache_dir=tmp_path / "cache") as dstore: assert sorted(dstore.keys()) == ["/a", "/b"] assert np.array_equal(dstore["/a"][:], np.arange(10, dtype="i4")) @@ -213,14 +213,14 @@ def test_cached_dir_refetches_when_remote_changes(tmp_path): with blosc2.DictStore(localstore, mode="w") as dstore: dstore["/a"] = blosc2.arange(10, dtype="i4") memfs.put(localstore, "memory://d.b2d", recursive=True) - with blosc2.open("memory://d.b2d", cache_storage=cache) as dstore: + with blosc2.open("memory://d.b2d", cache_dir=cache) as dstore: assert list(dstore.keys()) == ["/a"] with blosc2.DictStore(localstore, mode="a") as dstore: dstore["/b"] = blosc2.arange(5, dtype="i4") memfs.rm("/d.b2d", recursive=True) memfs.put(localstore, "memory://d.b2d", recursive=True) - with blosc2.open("memory://d.b2d", cache_storage=cache) as dstore: + with blosc2.open("memory://d.b2d", cache_dir=cache) as dstore: assert sorted(dstore.keys()) == ["/a", "/b"] @@ -229,7 +229,7 @@ def test_cached_sparse_frame(tmp_path): a = blosc2.arange(1000, dtype="i4", chunks=(100,), urlpath=localpath, mode="w", contiguous=False) fsspec.filesystem("memory").put(localpath, "memory://sparse.b2nd", recursive=True) - b = blosc2.open("memory://sparse.b2nd", cache_storage=tmp_path / "cache") + b = blosc2.open("memory://sparse.b2nd", cache_dir=tmp_path / "cache") assert np.array_equal(b[:], a[:]) @@ -470,7 +470,7 @@ def test_lazy_rejects_directories(tmp_path): localpath = str(tmp_path / "sparse.b2nd") blosc2.arange(0, 1000, dtype="i4", chunks=(100,), urlpath=localpath, mode="w", contiguous=False) fsspec.filesystem("memory").put(localpath, "memory://sparse.b2nd", recursive=True) - with pytest.raises(NotImplementedError, match="cache_storage"): + with pytest.raises(NotImplementedError, match="cache_dir"): blosc2.open("memory://sparse.b2nd", lazy=True) @@ -480,7 +480,7 @@ def test_lazy_not_a_frame(): blosc2.open("memory://junk.b2nd", lazy=True) -def test_lazy_with_cache_storage(tmp_path, monkeypatch): +def test_lazy_with_cache_dir(tmp_path, monkeypatch): a = blosc2.arange(0, 1000, dtype="i4", chunks=(100,)) url = _put("lazycache.b2nd", a) @@ -492,19 +492,46 @@ def test_lazy_with_cache_storage(tmp_path, monkeypatch): lambda self, nchunk: (fetched.append(nchunk), orig(self, nchunk))[1], ) - p = blosc2.open(url, lazy=True, cache_storage=tmp_path) + p = blosc2.open(url, lazy=True, cache_dir=tmp_path) assert np.array_equal(p[0:100], a[0:100]) assert fetched == [0] del p # A later run starts from the chunks the previous one pulled - p = blosc2.open(url, lazy=True, cache_storage=tmp_path) + p = blosc2.open(url, lazy=True, cache_dir=tmp_path) assert np.array_equal(p[0:100], a[0:100]) assert fetched == [0] assert np.array_equal(p[500:600], a[500:600]) assert fetched == [0, 5] +def test_lazy_with_exact_cache_path(tmp_path): + a = blosc2.arange(0, 1000, dtype="i4", chunks=(100,)) + url = _put("exactcache.b2nd", a) + cache_path = tmp_path / "chosen.b2nd" + + p = blosc2.open(url, lazy=True, cache_path=cache_path) + assert np.array_equal(p[0:100], a[0:100]) + assert p.urlpath == str(cache_path) + assert cache_path.is_file() + + q = blosc2.open(url, lazy=True, cache_path=cache_path) + assert np.array_equal(q[0:100], a[0:100]) + + +def test_remote_cache_options_are_mutually_exclusive(tmp_path): + url = _put("exclusivecache.b2nd", blosc2.arange(0, 10)) + with pytest.raises(ValueError, match="mutually exclusive"): + blosc2.open(url, lazy=True, cache_dir=tmp_path, cache_path=tmp_path / "cache.b2nd") + + +def test_cache_storage_is_deprecated(tmp_path): + url = _put("legacycache.b2nd", blosc2.arange(0, 10)) + with pytest.warns(DeprecationWarning, match="use cache_dir"): + p = blosc2.open(url, lazy=True, cache_storage=tmp_path) + assert np.array_equal(p[:], np.arange(10)) + + def test_lazy_cache_rebuilt_when_remote_changes(tmp_path): # Uncompressed, so both frames are byte-for-byte the same size: the stamp # cannot fall back to comparing sizes and get this right by luck @@ -514,14 +541,14 @@ def test_lazy_cache_rebuilt_when_remote_changes(tmp_path): assert len(a.to_cframe()) == len(b.to_cframe()) url = _put("lazystale.b2nd", a) - p = blosc2.open(url, lazy=True, cache_storage=tmp_path) + p = blosc2.open(url, lazy=True, cache_dir=tmp_path) assert np.array_equal(p[0:100], a[0:100]) del p # Replacing the frame invalidates both the cached chunks and the offsets # they were fetched by, so the cache must be thrown away rather than reused _put("lazystale.b2nd", b) - p = blosc2.open(url, lazy=True, cache_storage=tmp_path) + p = blosc2.open(url, lazy=True, cache_dir=tmp_path) assert np.array_equal(p[0:100], b[0:100]) @@ -554,7 +581,7 @@ def test_http_url_is_read_through_fsspec(tmp_path): assert np.array_equal(whole[:], data) requests.clear() - lazy = blosc2.open(f"{urlbase}/big.b2nd", lazy=True, cache_storage=str(tmp_path / "cs")) + lazy = blosc2.open(f"{urlbase}/big.b2nd", lazy=True, cache_dir=str(tmp_path / "cs")) assert isinstance(lazy, blosc2.Proxy) assert isinstance(lazy.src, blosc2.FsspecNDSource) assert ":etag:" in lazy.src.stamp @@ -574,12 +601,12 @@ def test_http_lazy_cache_rebuilt_when_remote_changes(tmp_path): with _ranged_server(path) as (urlbase, _): url = f"{urlbase}/{frame.name}" cache = tmp_path / "cache" - lazy = blosc2.open(url, lazy=True, cache_storage=cache) + lazy = blosc2.open(url, lazy=True, cache_dir=cache) assert np.array_equal(lazy[3:5, 100:120], first[3:5, 100:120]) del lazy blosc2.asarray(second, chunks=(50, 200), blocks=(10, 100), urlpath=frame, mode="w") - lazy = blosc2.open(url, lazy=True, cache_storage=cache) + lazy = blosc2.open(url, lazy=True, cache_dir=cache) assert np.array_equal(lazy[3:5, 100:120], second[3:5, 100:120]) @@ -645,7 +672,7 @@ def test_zip_store_needs_cache(tmp_path): with pytest.raises(RuntimeError): blosc2.open("memory://t.b2z") - with blosc2.open("memory://t.b2z", cache_storage=tmp_path / "cache") as tstore: + with blosc2.open("memory://t.b2z", cache_dir=tmp_path / "cache") as tstore: assert np.array_equal(tstore["/a"][:], np.arange(10, dtype="i4")) @@ -729,7 +756,7 @@ def test_cached_container_keeps_its_extension(tmp_path): del estore fsspec.filesystem("memory").pipe_file("/e.b2e", pathlib.Path(localpath).read_bytes()) - opened = blosc2.open("memory://e.b2e", cache_storage=tmp_path / "cache") + opened = blosc2.open("memory://e.b2e", cache_dir=tmp_path / "cache") assert isinstance(opened, blosc2.EmbedStore) assert np.array_equal(opened["/a"][:], np.arange(10, dtype="i4")) @@ -747,23 +774,23 @@ def test_lazy_empty_array(tmp_path): def test_lazy_cache_rebuilt_when_corrupt(tmp_path): # An interrupted run can leave a half-written cache behind; the whole point of - # cache_storage is surviving across runs, so it has to be discarded, not fatal + # cache_dir is surviving across runs, so it has to be discarded, not fatal a = blosc2.arange(100, dtype="i4", chunks=(10,)) fsspec.filesystem("memory").pipe_file("/c.b2nd", a.to_cframe()) - with blosc2.open("memory://c.b2nd", lazy=True, cache_storage=tmp_path) as b: + with blosc2.open("memory://c.b2nd", lazy=True, cache_dir=tmp_path) as b: assert np.array_equal(b[:10], a[:10]) cache = next(p for p in tmp_path.iterdir() if p.suffix == ".b2nd") cache.write_bytes(cache.read_bytes()[:50]) - with blosc2.open("memory://c.b2nd", lazy=True, cache_storage=tmp_path) as b: + with blosc2.open("memory://c.b2nd", lazy=True, cache_dir=tmp_path) as b: assert np.array_equal(b[:], a[:]) def test_max_concurrency_needs_lazy(tmp_path): fsspec.filesystem("memory").pipe_file("/m.b2nd", blosc2.arange(10, dtype="i4").to_cframe()) with pytest.raises(NotImplementedError, match="max_concurrency"): - blosc2.open("memory://m.b2nd", cache_storage=tmp_path, max_concurrency=4) + blosc2.open("memory://m.b2nd", cache_dir=tmp_path, max_concurrency=4) def test_storage_mapping_is_normalized(tmp_path): @@ -1337,13 +1364,13 @@ def test_lazy_eviction_survives_a_reopen(tmp_path, monkeypatch, any_chunk_wants_ cache = str(tmp_path / "evicted-cache") reads, _ = _traffic(monkeypatch) - p = blosc2.open(url, lazy=True, cache_storage=cache) + p = blosc2.open(url, lazy=True, cache_dir=cache) assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) fetched = len(reads) p.schunk.update_special(0, blosc2.SpecialValue.UNINIT) del p - q = blosc2.open(url, lazy=True, cache_storage=cache) + q = blosc2.open(url, lazy=True, cache_dir=cache) assert np.array_equal(q[0:5, 0:10], data[0:5, 0:10]) assert len(reads) > fetched diff --git a/tests/test_fsspec_s3.py b/tests/test_fsspec_s3.py index f4f8d583a..61b148d54 100644 --- a/tests/test_fsspec_s3.py +++ b/tests/test_fsspec_s3.py @@ -81,9 +81,9 @@ def test_save_and_open_whole(stored): assert np.array_equal(blosc2.open(urlpath)[:], a[:]) -def test_cache_storage(stored, tmp_path): +def test_cache_dir(stored, tmp_path): urlpath, a = stored - b = blosc2.open(urlpath, cache_storage=tmp_path, mmap_mode="r") + b = blosc2.open(urlpath, cache_dir=tmp_path, mmap_mode="r") assert np.array_equal(b[:], a[:]) From eb214c197a65a4d482393a1db5640e079b92b46c Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 4 Sep 2026 08:03:25 +0200 Subject: [PATCH 06/89] Reconstruct remote sources from proxy caches --- doc/guides/remote_arrays.md | 18 +++++++++++++++++ src/blosc2/proxy.py | 17 +++++++++++++--- src/blosc2/schunk.py | 29 +++++++++++++++++++++------- tests/ndarray/test_c2array_blocks.py | 7 ++++++- tests/test_fsspec.py | 28 +++++++++++++++++++++++++++ 5 files changed, 88 insertions(+), 11 deletions(-) diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index 4010b67fa..4da6247fc 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -83,6 +83,24 @@ In both cases, the cache is an ordinary `.b2nd` array that starts small and grow Authenticated Caterva2 caches must be private to one user. Reopen them under an equivalent authenticated {func}`blosc2.c2context`; do not share a cache directory between users. +### Reopen a cache file independently + +A persistent cache records enough information to reconstruct built-in fsspec and Caterva2 sources. When its filename is known, it can therefore be opened without repeating the original remote URL: + +```python +# Created earlier with cache_path="big-cache.b2nd" +a = blosc2.open("big-cache.b2nd", mode="a") + +a[100:110, :50] # cached data stays local +a[500:510, :50] # missing data is fetched from the recorded source and cached +``` + +This cache is operational, but not necessarily self-contained. Regions not fetched previously still require the original source. Opening with `mode="a"` lets newly fetched regions extend the cache; the default `mode="r"` keeps the cache file unchanged. + +Independent reopening works for fsspec URLs, Caterva2 datasets, and persistent local Blosc2 sources. The required runtime environment must still be available: fsspec backends and their configuration must be installed, local source paths must remain valid, and authenticated Caterva2 caches must be reopened inside an equivalent {func}`blosc2.c2context`. Caterva2 credentials are not stored in the cache file. + +An arbitrary custom {ref}`ProxyNDSource` cannot be reconstructed because its Python class and runtime state are not serialized. In that case, recreate the source explicitly and attach the existing cache with `blosc2.Proxy(source, urlpath="big-cache.b2nd", mode="a")`. + ## Only what a slice touches Blosc2 arrays are compressed in chunks, which are divided into smaller blocks. For a small slice, fetching only its blocks can avoid transferring most of a large chunk. diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index 7d76d3378..c2b1160e7 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -172,15 +172,26 @@ def __init__( fresh = self._cache is None if fresh: meta_val = { + "source_kind": None, "local_abspath": None, "urlpath": None, "caterva2_env": caterva2_env, } container = getattr(self.src, "schunk", self.src) - if hasattr(container, "urlpath"): - meta_val["local_abspath"] = container.urlpath + if isinstance(self.src, blosc2.FsspecNDSource): + meta_val["source_kind"] = "fsspec" + meta_val["urlpath"] = self.src.urlpath + # Keep the legacy field populated so older readers still + # reopen this cache, albeit through their eager URL path. + meta_val["local_abspath"] = self.src.urlpath elif isinstance(self.src, blosc2.C2Array): - meta_val["urlpath"] = (self.src.path, self.src.urlbase, self.src.auth_token) + meta_val["source_kind"] = "caterva2" + # Authentication belongs to the reopening process, not to a + # portable cache file. C2Array resolves it again from c2context. + meta_val["urlpath"] = (self.src.path, self.src.urlbase, None) + elif hasattr(container, "urlpath"): + meta_val["source_kind"] = "local" + meta_val["local_abspath"] = container.urlpath meta = {"proxy-source": meta_val} if hasattr(self.src, "shape"): self._cache = blosc2.empty( diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index ac23c6a16..7fe4925d7 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -1891,9 +1891,25 @@ def process_opened_object(res): if "proxy-source" in meta: proxy_cache = res proxy_src = meta["proxy-source"] + source_kind = proxy_src.get("source_kind") + if source_kind == "fsspec": + src = blosc2.FsspecNDSource(proxy_src["urlpath"]) + return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) + if source_kind == "caterva2": + src = blosc2.C2Array( + proxy_src["urlpath"][0], proxy_src["urlpath"][1], proxy_src["urlpath"][2] + ) + return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) if proxy_src["local_abspath"] is not None: - src = blosc2.open(proxy_src["local_abspath"], mode="r") - return blosc2.Proxy(src, _cache=proxy_cache) + source_path = proxy_src["local_abspath"] + # Older FsspecNDSource caches recorded their URL in the field that + # otherwise names a local source. Preserve those caches while + # restoring their lazy byte-range behavior. + if source_kind is None and is_fsspec_url(source_path): + src = blosc2.FsspecNDSource(source_path) + else: + src = blosc2.open(source_path, mode="r") + return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) elif proxy_src["urlpath"] is not None: src = blosc2.C2Array(proxy_src["urlpath"][0], proxy_src["urlpath"][1], proxy_src["urlpath"][2]) return blosc2.Proxy(src, _cache=proxy_cache) @@ -2279,11 +2295,10 @@ def open( caller; runtime caches are not serialized back to disk. * If the original object saved in :paramref:`urlpath` is a :ref:`Proxy`, - this function will only return a :ref:`Proxy` if its source is a local - :ref:`SChunk`, :ref:`NDArray` or a remote :ref:`C2Array`. Otherwise, - it will return the Python-Blosc2 container used to cache the data which - can be a :ref:`SChunk` or a :ref:`NDArray` and may not have all the data - initialized (e.g. if the user has not accessed to it yet). + this function reconstructs sources backed by a persistent local + :ref:`SChunk` or :ref:`NDArray`, an fsspec URL, or a remote + :ref:`C2Array`. Custom proxy sources must be recreated explicitly because + their Python class and runtime state are not stored in the cache. * When opening a :ref:`LazyExpr` keep in mind the note above regarding operands. diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index 50eeb3c64..cf11c85a5 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -374,12 +374,17 @@ def test_open_urlpath_lazy_exact_cache_path(tmp_path, server, any_chunk_wants_bl proxy = blosc2.open(urlpath, lazy=True, cache_path=cache_path) assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) assert proxy.urlpath == str(cache_path) + assert proxy.schunk.meta["proxy-source"]["source_kind"] == "caterva2" del proxy srv.log.clear() - proxy = blosc2.open(urlpath, lazy=True, cache_path=cache_path) + proxy = blosc2.open(cache_path, mode="a") + assert isinstance(proxy, blosc2.Proxy) + assert isinstance(proxy.src, blosc2.C2Array) assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) assert [endpoint for endpoint, _, _ in srv.log] == ["info"] + assert np.array_equal(proxy[100:105, 0:10], data[100:105, 0:10]) + assert len(srv.log) > 1 def test_open_urlpath_lazy_uses_c2context_without_persisting_token(tmp_path, server): diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index 93f138a35..29d41d8d5 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -519,6 +519,34 @@ def test_lazy_with_exact_cache_path(tmp_path): assert np.array_equal(q[0:100], a[0:100]) +def test_exact_cache_path_reopens_as_lazy_fsspec_proxy(tmp_path, monkeypatch): + a = blosc2.arange(0, 1000, dtype="i4", chunks=(100,)) + url = _put("independentcache.b2nd", a) + cache_path = tmp_path / "independent.b2nd" + p = blosc2.open(url, lazy=True, cache_path=cache_path) + assert np.array_equal(p[0:100], a[0:100]) + source_meta = p.schunk.meta["proxy-source"] + assert source_meta["source_kind"] == "fsspec" + assert source_meta["urlpath"] == url + del p + + fetched = [] + orig = blosc2.FsspecNDSource.get_chunk + monkeypatch.setattr( + blosc2.FsspecNDSource, + "get_chunk", + lambda self, nchunk: (fetched.append(nchunk), orig(self, nchunk))[1], + ) + + reopened = blosc2.open(cache_path, mode="a") + assert isinstance(reopened, blosc2.Proxy) + assert isinstance(reopened.src, blosc2.FsspecNDSource) + assert np.array_equal(reopened[0:100], a[0:100]) + assert fetched == [] + assert np.array_equal(reopened[500:600], a[500:600]) + assert fetched == [5] + + def test_remote_cache_options_are_mutually_exclusive(tmp_path): url = _put("exclusivecache.b2nd", blosc2.arange(0, 10)) with pytest.raises(ValueError, match="mutually exclusive"): From 266e8e8fc3c45a83e3445ea81b4845ddacf70361 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 4 Sep 2026 08:16:25 +0200 Subject: [PATCH 07/89] Count metadata in remote traffic --- doc/guides/remote_arrays.md | 2 +- examples/c2array-traffic.py | 7 +++---- examples/fsspec-cat2-access.py | 15 +++++++++------ src/blosc2/c2array.py | 25 ++++++++++++++++++------- src/blosc2/proxy.py | 7 +++---- tests/ndarray/test_c2array_blocks.py | 6 ++++-- 6 files changed, 38 insertions(+), 24 deletions(-) diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index 4da6247fc..5aec9e9ae 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -115,7 +115,7 @@ Stepped slices also use the block grid. For example, `p[::5]` can reduce transfe ### Measure network traffic -{ref}`C2Array` and remote {ref}`Proxy` objects expose cumulative request and byte counts through {ref}`Traffic`: +{ref}`C2Array` and remote {ref}`Proxy` objects expose cumulative request and byte counts through {ref}`Traffic`. The count starts when the remote source is opened, so it includes metadata as well as array data: ```python source = blosc2.C2Array( diff --git a/examples/c2array-traffic.py b/examples/c2array-traffic.py index 60b323b76..ee8b42835 100644 --- a/examples/c2array-traffic.py +++ b/examples/c2array-traffic.py @@ -11,7 +11,7 @@ # the whole chunk: on a fast link the two take about as long, and differ by the # compression ratio in bytes. Bytes are also what a metered link and a shared # server uplink actually run out of, so they are what `Traffic` counts -- at the -# transport, so the frame index and block offsets are in the tally too. +# transport, so metadata, the frame index, and block offsets are in the tally too. import blosc2 @@ -27,9 +27,8 @@ def cost(traffic): array = blosc2.C2Array(path, urlbase=urlbase) print(f"{path}: shape={array.shape} chunks={array.chunks} blocks={array.blocks}") -# Opening a handle costs one `api/info` call, which is metadata rather than data -# and is deliberately not counted -- no slice can avoid it, and no choice of -# granularity changes it. +# Opening a handle costs one `api/info` call, included so this is a complete +# account of everything that crossed the wire. print(f"after opening: {array.traffic}") # -- A proxy reads through the block path, so it pays for what a slice touches. diff --git a/examples/fsspec-cat2-access.py b/examples/fsspec-cat2-access.py index 78e21cfbf..4f685a0fd 100644 --- a/examples/fsspec-cat2-access.py +++ b/examples/fsspec-cat2-access.py @@ -39,7 +39,8 @@ def traffic_text(traffic: blosc2.Traffic | None) -> str: if traffic is None: return "traffic unavailable" - return f"{traffic.requests} requests, {traffic.nbytes / 2**20:.3f} MiB" + request_word = "request" if traffic.requests == 1 else "requests" + return f"{traffic.requests} {request_word}, {traffic.nbytes / 2**20:.3f} MiB" def size_text(size: int) -> str: @@ -52,6 +53,7 @@ def benchmark(label: str, urlpath, cache_dir: Path) -> np.ndarray: start = perf_counter() array = blosc2.open(urlpath, lazy=True, cache_dir=cache_dir) open_time = perf_counter() - start + open_traffic = traffic_text(array.traffic) metadata = (array.shape, array.dtype, array.chunks, array.blocks) cache_path = Path(array.urlpath).resolve() @@ -69,6 +71,7 @@ def benchmark(label: str, urlpath, cache_dir: Path) -> np.ndarray: start = perf_counter() reopened = blosc2.open(urlpath, lazy=True, cache_dir=cache_dir) reopen_time = perf_counter() - start + reopen_traffic = traffic_text(reopened.traffic) reopened.traffic.reset() start = perf_counter() @@ -81,11 +84,11 @@ def benchmark(label: str, urlpath, cache_dir: Path) -> np.ndarray: print(f" metadata: shape={metadata[0]}, dtype={metadata[1]}") print(f" chunks={metadata[2]}, blocks={metadata[3]}") print(f" persistent cache: {cache_path} ({'existing' if cache_existed else 'new'})") - print(f" open and remote metadata setup: {open_time:.6f} s") - print(f" first data slice this run: {first_read_time:.6f} s ({first_traffic})") - print(f" cache size after slice: {size_text(cache_size)}") - print(f" reopen persistent cache: {reopen_time:.6f} s") - print(f" same slice after reopen: {cached_read_time:.6f} s ({cached_traffic})") + print(f" {'open + remote metadata:':<27}{open_time * 1000:.0f} ms ({open_traffic})") + print(f" {'first data slice:':<27}{first_read_time * 1000:.0f} ms ({first_traffic})") + print(f" {'cache after slice:':<27}{size_text(cache_size)}") + print(f" {'reopen + remote metadata:':<27}{reopen_time * 1000:.0f} ms ({reopen_traffic})") + print(f" {'same slice after reopen:':<27}{cached_read_time * 1000:.0f} ms ({cached_traffic})") return data diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index 9286894b9..c14db8303 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -264,9 +264,11 @@ def login(username, password, urlbase): return "=".join(list(resp.cookies.items())[0]) -def info(path, urlbase, params=None, headers=None, model=None, auth_token=None): +def info(path, urlbase, params=None, headers=None, model=None, auth_token=None, traffic=None): url = _server_url(urlbase, f"api/info/{path}") response = _xget(url, params, headers, auth_token) + if traffic is not None: + traffic.charge(len(response.content)) json = response.json() return json if model is None else model(**json) @@ -771,10 +773,9 @@ def __init__(self, path: str, /, urlbase: str | None = None, auth_token: str | N """Bytes and requests this handle has read off the server; see :ref:`Traffic`. Cumulative since the array was opened, counted at the transport, so the - frame index and the block offsets are in it as well as the data, and the - `api/info` call that opened this handle is not. Whichever endpoint the - read used is in it too, and the block source built later is handed this - same tally, so one counter answers for the array however it is read. + opening `api/info` response, frame index, block offsets, and data are all + included. Whichever endpoint serves a read uses this same tally, so one + counter answers for the array however it is read. What a slice cost is the difference between two readings, or one reading after :meth:`Traffic.reset`. `examples/c2array-traffic.py` is a runnable @@ -784,7 +785,12 @@ def __init__(self, path: str, /, urlbase: str | None = None, auth_token: str | N # Try to 'open' the remote path try: - self.meta = info(self.path, self.urlbase, auth_token=self.auth_token) + self.meta = info( + self.path, + self.urlbase, + auth_token=self.auth_token, + traffic=self.traffic, + ) except _httpx().HTTPStatusError as err: # HTTPStatusError only (not the broader HTTPError, which also covers # connection-level failures): a 404 means "not found", a connection @@ -1163,7 +1169,12 @@ def _reread_meta(self) -> None: """ with self._meta_lock: seen = self._meta_epoch - meta = info(self.path, self.urlbase, auth_token=self.auth_token) + meta = info( + self.path, + self.urlbase, + auth_token=self.auth_token, + traffic=self.traffic, + ) with self._meta_lock: if self._meta_epoch != seen: return diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index c2b1160e7..61fb73c93 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -251,10 +251,9 @@ def traffic(self) -> "blosc2.proxy_source.Traffic | None": """What this proxy has read off its source, or None for a local one. Cumulative bytes and requests since the source was opened, counted at the - transport, so the frame index and the block offsets are in it as well as - the data, and the metadata call that opened the handle is not. What a - slice cost in traffic is the difference between two readings of this, or - one reading after :meth:`Traffic.reset`. + transport, including metadata, frame indexes, block offsets, and data. + What a slice cost in traffic is the difference between two readings of + this, or one reading after :meth:`Traffic.reset`. It is what says whether block granularity is doing anything for a given dataset and access pattern: whole chunks and blocks of them take similar diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index cf11c85a5..8511f5f20 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -955,14 +955,16 @@ def test_traffic_counts_what_crossed_the_wire(server, any_chunk_wants_blocks): array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) p = blosc2.Proxy(array, mode="w") assert p.traffic is array.traffic # the array's tally, not a second one + info_requests = sum(kind == "info" for kind, _, _ in srv.log) + assert p.traffic.requests == info_requests + assert p.traffic.nbytes == _bytes(srv, "info") p.traffic.reset() assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) blocks = (p.traffic.requests, p.traffic.nbytes) assert blocks[0] > 0 assert blocks[1] > 0 - # What the server logged for the data endpoints is what was counted; the - # `api/info` that opened the handle is metadata and is deliberately not + # After the reset, what the server logged for the data endpoints is what was counted. served = [(kind, nbytes) for kind, _, nbytes in srv.log if kind != "info"] assert blocks[0] == len(served) assert blocks[1] <= sum(nbytes for _, nbytes in served) From 5200b647b5eee5109693e9c86e6a7964af90496f Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 4 Sep 2026 11:50:00 +0200 Subject: [PATCH 08/89] Report persistent cache status --- examples/fsspec-cat2-access.py | 5 ++--- src/blosc2/proxy.py | 11 +++++++++++ src/blosc2/schunk.py | 21 +++++++++++++-------- tests/test_fsspec.py | 3 +++ 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/examples/fsspec-cat2-access.py b/examples/fsspec-cat2-access.py index 4f685a0fd..547c963d6 100644 --- a/examples/fsspec-cat2-access.py +++ b/examples/fsspec-cat2-access.py @@ -48,8 +48,6 @@ def size_text(size: int) -> str: def benchmark(label: str, urlpath, cache_dir: Path) -> np.ndarray: - cache_existed = cache_dir.is_dir() and any(cache_dir.glob("*.b2nd")) - start = perf_counter() array = blosc2.open(urlpath, lazy=True, cache_dir=cache_dir) open_time = perf_counter() - start @@ -57,6 +55,7 @@ def benchmark(label: str, urlpath, cache_dir: Path) -> np.ndarray: metadata = (array.shape, array.dtype, array.chunks, array.blocks) cache_path = Path(array.urlpath).resolve() + cache_status = array.cache_status array.traffic.reset() start = perf_counter() @@ -83,7 +82,7 @@ def benchmark(label: str, urlpath, cache_dir: Path) -> np.ndarray: print(f"\n{label}") print(f" metadata: shape={metadata[0]}, dtype={metadata[1]}") print(f" chunks={metadata[2]}, blocks={metadata[3]}") - print(f" persistent cache: {cache_path} ({'existing' if cache_existed else 'new'})") + print(f" persistent cache: {cache_path} ({cache_status})") print(f" {'open + remote metadata:':<27}{open_time * 1000:.0f} ms ({open_traffic})") print(f" {'first data slice:':<27}{first_read_time * 1000:.0f} ms ({first_traffic})") print(f" {'cache after slice:':<27}{size_text(cache_size)}") diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index 61fb73c93..0c14a40b2 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -128,6 +128,7 @@ def __init__( """ self.src = src self.urlpath = urlpath + self._cache_status = None if kwargs is None: kwargs = {} self._cache = kwargs.pop("_cache", None) @@ -274,6 +275,16 @@ def traffic(self) -> "blosc2.proxy_source.Traffic | None": """ return getattr(self.src, "traffic", None) + @property + def cache_status(self) -> str | None: + """How the persistent cache was handled when this proxy was opened. + + This is ``"created"``, ``"reused"``, or ``"invalidated/rebuilt"`` for + a remote proxy opened with ``cache_dir`` or ``cache_path``. It is + ``None`` for proxies without a managed persistent cache. + """ + return self._cache_status + def __enter__(self) -> "Proxy": """Enter a context manager and return this proxy.""" return self diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 7fe4925d7..928e1f413 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -1896,9 +1896,7 @@ def process_opened_object(res): src = blosc2.FsspecNDSource(proxy_src["urlpath"]) return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) if source_kind == "caterva2": - src = blosc2.C2Array( - proxy_src["urlpath"][0], proxy_src["urlpath"][1], proxy_src["urlpath"][2] - ) + src = blosc2.C2Array(proxy_src["urlpath"][0], proxy_src["urlpath"][1], proxy_src["urlpath"][2]) return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) if proxy_src["local_abspath"] is not None: source_path = proxy_src["local_abspath"] @@ -2049,13 +2047,20 @@ def _lazy_remote_proxy( else: path = fsspec_cache_path(identity, cache_dir, ".b2nd") stamp = getattr(src, "stamp", None) - if os.path.exists(path) and _cache_stamp(path) != stamp: - # The remote frame was replaced, which makes every cached chunk -- and - # every offset they were fetched by -- meaningless - blosc2.remove_urlpath(path) + cache_status = "created" + if os.path.exists(path): + if _cache_stamp(path) != stamp: + # The remote frame was replaced, which makes every cached chunk -- and + # every offset they were fetched by -- meaningless + blosc2.remove_urlpath(path) + cache_status = "invalidated/rebuilt" + else: + cache_status = "reused" # Proxy stamps the cache with src.stamp itself, and refuses one built against # other bytes; removing it above is what turns that refusal into a refetch - return blosc2.Proxy(src, urlpath=path, mode="a", _refresh_source=not source_fresh) + proxy = blosc2.Proxy(src, urlpath=path, mode="a", _refresh_source=not source_fresh) + proxy._cache_status = cache_status + return proxy def _open_c2_urlpath(urlpath: blosc2.URLPath, mode: str, offset: int, kwargs: dict): diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index 29d41d8d5..a1b5f63b9 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -493,12 +493,14 @@ def test_lazy_with_cache_dir(tmp_path, monkeypatch): ) p = blosc2.open(url, lazy=True, cache_dir=tmp_path) + assert p.cache_status == "created" assert np.array_equal(p[0:100], a[0:100]) assert fetched == [0] del p # A later run starts from the chunks the previous one pulled p = blosc2.open(url, lazy=True, cache_dir=tmp_path) + assert p.cache_status == "reused" assert np.array_equal(p[0:100], a[0:100]) assert fetched == [0] assert np.array_equal(p[500:600], a[500:600]) @@ -577,6 +579,7 @@ def test_lazy_cache_rebuilt_when_remote_changes(tmp_path): # they were fetched by, so the cache must be thrown away rather than reused _put("lazystale.b2nd", b) p = blosc2.open(url, lazy=True, cache_dir=tmp_path) + assert p.cache_status == "invalidated/rebuilt" assert np.array_equal(p[0:100], b[0:100]) From 408899dc1e959940168b53904ac439d8a0cf61d9 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 4 Sep 2026 11:54:33 +0200 Subject: [PATCH 09/89] Update C2Array test doubles for traffic --- tests/ndarray/test_c2array_async.py | 1 + tests/test_b2objects.py | 2 +- tests/test_objectarray.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ndarray/test_c2array_async.py b/tests/ndarray/test_c2array_async.py index 4556eaa5f..a11887680 100644 --- a/tests/ndarray/test_c2array_async.py +++ b/tests/ndarray/test_c2array_async.py @@ -18,6 +18,7 @@ class _FakeResponse: def __init__(self, json_data): self._json = json_data + self.content = b"" def raise_for_status(self): pass diff --git a/tests/test_b2objects.py b/tests/test_b2objects.py index 6317d34e6..08d6a52a2 100644 --- a/tests/test_b2objects.py +++ b/tests/test_b2objects.py @@ -32,7 +32,7 @@ def _make_c2array( ): dtype = np.dtype(dtype) - def fake_info(path_, urlbase_, params=None, headers=None, model=None, auth_token=None): + def fake_info(path_, urlbase_, params=None, headers=None, model=None, auth_token=None, traffic=None): return { "shape": list(shape), "chunks": list(chunks), diff --git a/tests/test_objectarray.py b/tests/test_objectarray.py index f86a2c31b..53c526c59 100644 --- a/tests/test_objectarray.py +++ b/tests/test_objectarray.py @@ -58,7 +58,7 @@ def _make_nested_blosc2_objects(): def _make_c2array(monkeypatch, path="@public/examples/ds-1d.b2nd", urlbase="https://cat2.cloud/demo/"): - def fake_info(path_, urlbase_, params=None, headers=None, model=None, auth_token=None): + def fake_info(path_, urlbase_, params=None, headers=None, model=None, auth_token=None, traffic=None): return {"schunk": {"cparams": dict(blosc2.cparams_dflts)}} monkeypatch.setattr(blosc2_c2array, "info", fake_info) From 47fd16514bdee06550ab41ef4874890fb64586de Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 4 Sep 2026 12:12:49 +0200 Subject: [PATCH 10/89] Reduce default test suite runtime --- tests/ctable/test_ctable_indexing.py | 6 ++-- tests/ctable/test_dictionary_column.py | 39 ++++++++------------------ tests/ctable/test_utf8.py | 1 + tests/test_ctable_cframe.py | 3 +- tests/test_locking.py | 10 +++---- 5 files changed, 23 insertions(+), 36 deletions(-) diff --git a/tests/ctable/test_ctable_indexing.py b/tests/ctable/test_ctable_indexing.py index c4dae75a1..dd3e00604 100644 --- a/tests/ctable/test_ctable_indexing.py +++ b/tests/ctable/test_ctable_indexing.py @@ -848,7 +848,7 @@ class _IncrRow: i: int = blosc2.field(blosc2.int64(), chunks=(2000,), blocks=(500,)) -def _build_incr_data(n=9000): +def _build_incr_data(n=2250): rng = np.random.default_rng(7) f = (rng.standard_normal(n) * 50).astype(np.float32) f[rng.integers(0, n, n // 100)] = np.nan # exercise NaN flags @@ -867,6 +867,7 @@ def _summary_sidecars(table): return out +@pytest.mark.heavy def test_incremental_summary_matches_ooc_build(tmp_path): """The incremental per-block accumulator (folded during the write phase) must produce SUMMARY sidecars byte-identical to the out-of-core @@ -901,10 +902,11 @@ def test_incremental_summary_matches_ooc_build(tmp_path): assert np.allclose(a["max"], b["max"], equal_nan=True) +@pytest.mark.heavy def test_incremental_summary_stale_on_inplace(tmp_path): """An in-place column write before close must invalidate the accumulator so the builder falls back to a correct full rescan.""" - f, i = _build_incr_data(n=4000) + f, i = _build_incr_data(n=2250) path = str(tmp_path / "upd.b2z") with blosc2.CTable(_IncrRow, urlpath=path, mode="w") as t: t.extend({"f": f, "i": i}) diff --git a/tests/ctable/test_dictionary_column.py b/tests/ctable/test_dictionary_column.py index f7cf95bb0..da53396a5 100644 --- a/tests/ctable/test_dictionary_column.py +++ b/tests/ctable/test_dictionary_column.py @@ -600,7 +600,6 @@ def test_dictionary_column_comparisons_are_elementwise(): It used to return a plain ``False`` — silently wrong rather than an error. """ - import numpy as np @dataclass class Row: @@ -625,7 +624,6 @@ def test_dictionary_ne_predicate_matches_live_rows(): Negating afterwards turned every dead capacity slot True, which then failed with an IndexError when used to select rows. """ - import numpy as np @dataclass class Row: @@ -637,11 +635,9 @@ class Row: t._flush_varlen_columns() assert sorted(t[t["c"] != "a1"]["c"][:]) == sorted(v for v in values if v != "a1") - assert len(t[t["c"] == "a1"]["c"][:]) == 13 - # A value no row carries: nothing matches, everything differs. - assert len(t[t["c"] == "absent"]["c"][:]) == 0 + # A value no row carries differs from every live row, but not from padded + # capacity slots. assert len(t[t["c"] != "absent"]["c"][:]) == len(values) - assert np.asarray((t["c"] != "a1")[:]).sum() == 26 def test_dictionary_index_answers_equality(tmp_path): @@ -652,27 +648,16 @@ class Row: c: str = blosc2.field(blosc2.dictionary()) values = ["pear", "apple", "cherry", "apple", "banana"] - results = {} - for tag in ("scan", "index"): - t = CTable(Row, urlpath=str(tmp_path / f"{tag}.b2t"), mode="w") - t.extend({"c": values}, validate=False) - t._flush_varlen_columns() - if tag == "index": - t.create_index("c", kind="full") - assert t["c"]._dictionary_index_mask("apple") is not None - # A value absent from the dictionary still answers, matching nothing. - assert not t["c"]._dictionary_index_mask("absent").any() - results[tag] = { - probe: ( - sorted(t[t["c"] == probe]["c"][:]), - sorted(t[t["c"] != probe]["c"][:]), - ) - for probe in ("apple", "pear", "absent") - } - del t - - assert results["index"] == results["scan"] - assert results["scan"]["apple"][0] == ["apple", "apple"] + t = CTable(Row, urlpath=str(tmp_path / "indexed.b2t"), mode="w") + t.extend({"c": values}, validate=False) + t._flush_varlen_columns() + t.create_index("c", kind="full") + assert t["c"]._dictionary_index_mask("apple") is not None + # A value absent from the dictionary still answers, matching nothing. + assert not t["c"]._dictionary_index_mask("absent").any() + assert sorted(t[t["c"] == "apple"]["c"][:]) == ["apple", "apple"] + assert list(t[t["c"] == "absent"]["c"][:]) == [] + del t def test_dictionary_index_spans_deleted_rows(tmp_path): diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index d4b6af359..33ef053c9 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -2299,6 +2299,7 @@ def test_constructors_string_dtype_reject_nd(): blosc2.zeros(3, dtype=STRING_DTYPE, urlpath="unused.b2nd") +@pytest.mark.heavy @pytest.mark.skipif( blosc2.IS_WASM, reason="peak-memory scaling is not measurable under Pyodide: its noise floor " diff --git a/tests/test_ctable_cframe.py b/tests/test_ctable_cframe.py index cc1205e95..c072a0fa5 100644 --- a/tests/test_ctable_cframe.py +++ b/tests/test_ctable_cframe.py @@ -203,8 +203,7 @@ class R: p = pathlib.Path(tempfile.mkdtemp()) / "t.b2z" t = blosc2.CTable(R, urlpath=str(p), mode="w", compact=True) - for i in range(50): - t.append((i, f"n{i}")) + t.extend([(i, f"n{i}") for i in range(50)]) t.close() t = blosc2.open(p) cf = t.to_cframe() diff --git a/tests/test_locking.py b/tests/test_locking.py index 16dbf05a0..86718bd37 100644 --- a/tests/test_locking.py +++ b/tests/test_locking.py @@ -225,7 +225,7 @@ def test_cross_process_hammer(tmp_path): urlpath = tmp_path / "schunk-hammer.b2frame" create_schunk(urlpath, contiguous=False, locking=True) - iters = 500 + iters = 150 writer = subprocess.Popen( [sys.executable, "-c", WRITER_SCRIPT, str(urlpath), str(NCHUNKS), str(CHUNK_NITEMS), str(iters)] ) @@ -353,7 +353,7 @@ def test_cross_process_multiwriter_update(tmp_path): # owner's last-written value. urlpath = tmp_path / "schunk-multiwriter-update.b2frame" nwriters = 4 - iters = 60 + iters = 20 schunk = create_schunk(urlpath, contiguous=False, locking=True) nchunks = schunk.nchunks del schunk @@ -1332,10 +1332,10 @@ def slow_sync(self): dstore = blosc2.DictStore(path, mode="w", threshold=500, locking=True) dstore["/hot"] = np.arange(100) - writer = subprocess.Popen([sys.executable, "-c", DSTORE_OVERWRITER, path, "300"]) + writer = subprocess.Popen([sys.executable, "-c", DSTORE_OVERWRITER, path, "100"]) try: nreads = 0 - while writer.poll() is None and nreads < 60: + while writer.poll() is None and nreads < 20: data = dstore["/hot"][:] # Each round writes arange(i, i + 100); a torn read breaks the run assert np.array_equal(data, np.arange(data[0], data[0] + 100)) @@ -1345,7 +1345,7 @@ def slow_sync(self): writer.kill() writer.wait() - assert nreads == 60 + assert nreads == 20 dstore._closed = True From 1d0e2b97f6185865896bc3245e63aa7a2d8d82f8 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 4 Sep 2026 14:49:02 +0200 Subject: [PATCH 11/89] Add persistable RemoteProxy --- doc/guides/remote_arrays.md | 41 +- doc/reference/classes.rst | 2 + doc/reference/msgpack_serialization.rst | 17 +- doc/reference/ref.rst | 1 + doc/reference/remoteproxy.rst | 93 +++ plans/remote-proxy.md | 817 ++++++++++++++++++++++++ src/blosc2/__init__.py | 11 + src/blosc2/b2objects.py | 8 +- src/blosc2/c2array.py | 4 +- src/blosc2/core.py | 4 +- src/blosc2/lazyexpr.py | 27 +- src/blosc2/msgpack_utils.py | 8 +- src/blosc2/ndarray.py | 2 +- src/blosc2/objectarray.py | 8 +- src/blosc2/proxy.py | 95 ++- src/blosc2/ref.py | 29 +- src/blosc2/remote_proxy.py | 443 +++++++++++++ src/blosc2/schunk.py | 83 ++- tests/test_remote_proxy.py | 348 ++++++++++ 19 files changed, 2005 insertions(+), 36 deletions(-) create mode 100644 doc/reference/remoteproxy.rst create mode 100644 plans/remote-proxy.md create mode 100644 src/blosc2/remote_proxy.py create mode 100644 tests/test_remote_proxy.py diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index 5aec9e9ae..a01a0abcb 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -83,6 +83,45 @@ In both cases, the cache is an ordinary `.b2nd` array that starts small and grow Authenticated Caterva2 caches must be private to one user. Reopen them under an equivalent authenticated {func}`blosc2.c2context`; do not share a cache directory between users. +### Persist a reference instead of a cache + +Use {ref}`RemoteProxy` when the `.b2nd` file itself should remain a small, +immutable reference to the remote array rather than become its cache: + +```python +remote = blosc2.RemoteProxy( + "s3://bucket/big.b2nd", + cache_policy=blosc2.CachePolicy.NONE, +) +remote.save("big-reference.b2nd") +``` + +The saved object contains source and geometry metadata but no fetched chunks or +credentials. It reopens with `CachePolicy.NONE`, so repeated reads contact the +source again and never mutate the reference file. This is also true when a +`RemoteProxy` using `MEMORY` or `DISK` caching is saved: those runtime caches +are not part of the portable reference. + +`RemoteProxy` also supports bounded runtime caching. Memory caches retain at +most 256 MiB of compressed payload by default; disk caches are unlimited unless +an explicit bound is supplied: + +```python +remote = blosc2.RemoteProxy( + "s3://bucket/big.b2nd", + cache_policy=blosc2.CachePolicy.DISK, + cache_path="big-cache.b2nd", + max_cache_bytes=2 * 2**30, +) +``` + +Opening the saved reference itself returns `CachePolicy.NONE`. Runtime caching +must be selected again explicitly from its `urlpath`; it is never inferred from +the reference carrier. + +The bound is applied after each operation. It does not limit the temporary +working set or a NumPy result requested by the caller. + ### Reopen a cache file independently A persistent cache records enough information to reconstruct built-in fsspec and Caterva2 sources. When its filename is known, it can therefore be opened without repeating the original remote URL: @@ -248,4 +287,4 @@ For ordinary S3 access, use `blosc2.open("s3://bucket/big.b2nd", lazy=True)`; th - `examples/ndarray/rw-fsspec.py` — fsspec reading and writing examples. - `examples/fsspec-cat2-access.py` — one dataset and cache through fsspec and Caterva2. - `examples/c2array-traffic.py` — block, chunk, and cached transfer sizes. -- {ref}`C2Array`, {ref}`FsspecNDSource`, {ref}`ByteRangeNDSource`, {ref}`Proxy`, and {ref}`Traffic` — API reference pages. +- {ref}`C2Array`, {ref}`FsspecNDSource`, {ref}`ByteRangeNDSource`, {ref}`Proxy`, {ref}`RemoteProxy`, and {ref}`Traffic` — API reference pages. diff --git a/doc/reference/classes.rst b/doc/reference/classes.rst index d06646d04..a09bd06f6 100644 --- a/doc/reference/classes.rst +++ b/doc/reference/classes.rst @@ -108,6 +108,7 @@ codecs, filters, and remote paths. SpecialValue Tuner FPAccuracy + CachePolicy URLPath @@ -138,6 +139,7 @@ container APIs above. list_array objectarray proxy + remoteproxy proxysource proxyndsource byterangendsource diff --git a/doc/reference/msgpack_serialization.rst b/doc/reference/msgpack_serialization.rst index 9807a5fbe..6755b4501 100644 --- a/doc/reference/msgpack_serialization.rst +++ b/doc/reference/msgpack_serialization.rst @@ -22,6 +22,7 @@ The following objects are serialized by value using - ``ObjectArray`` - ``BatchArray`` - ``EmbedStore`` +- ``RemoteProxy`` Structured objects ------------------ @@ -40,12 +41,14 @@ Currently implemented structured kinds are: - ``"ref"`` - ``"c2array"`` +- ``"remote_proxy"`` +- ``"fsspec"`` - ``"urlpath"`` - ``"dictstore_key"`` - ``"lazyexpr"`` - ``"lazyudf"`` -The ``"urlpath"``, ``"dictstore_key"``, and ``"c2array"`` reference forms map +The ``"urlpath"``, ``"dictstore_key"``, ``"c2array"``, and ``"fsspec"`` reference forms map directly onto the public :class:`blosc2.Ref` type. ``C2Array`` @@ -58,6 +61,16 @@ Remote arrays are serialized as lightweight references with: Authentication data is intentionally not serialized. +``RemoteProxy`` +--------------- + +Remote proxies use a metadata-only carrier containing a versioned Caterva2 or +fsspec source descriptor. Saving a live proxy is reference-only: its carrier +reopens with :attr:`blosc2.CachePolicy.NONE`, regardless of whether the live +proxy used ``MEMORY`` or ``DISK`` caching. The policy is encoded on disk as the +stable string ``"none"``; runtime cache contents, local cache paths, fetched +data, and credentials are intentionally not serialized. + Persistent local operands ------------------------- @@ -93,6 +106,7 @@ Only durable reference-style operands are supported: - persistent local Blosc2 operands reopenable from ``urlpath`` - remote ``C2Array`` operands +- ``RemoteProxy`` operands for fsspec or Caterva2 references - ``DictStore`` members reopenable from ``(.b2d|.b2z, key)`` Purely in-memory operands are intentionally rejected. This keeps msgpack @@ -119,6 +133,7 @@ Supported operands are the same durable reference-style operands used for - persistent local Blosc2 operands reopenable from ``urlpath`` - remote ``C2Array`` operands +- ``RemoteProxy`` operands for fsspec or Caterva2 references - ``DictStore`` members reopenable from ``(.b2d|.b2z, key)`` Plain Python ``LazyUDF`` callables are intentionally not serialized by diff --git a/doc/reference/ref.rst b/doc/reference/ref.rst index 120f89cf0..6a3489765 100644 --- a/doc/reference/ref.rst +++ b/doc/reference/ref.rst @@ -14,6 +14,7 @@ Currently supported reference kinds are: - ``"urlpath"`` for persistent local objects - ``"dictstore_key"`` for members inside ``.b2d`` / ``.b2z`` ``DictStore`` containers - ``"c2array"`` for remote ``C2Array`` objects +- ``"fsspec"`` for fsspec sources used by ``RemoteProxy`` objects Use :meth:`Ref.open` to resolve a reference back into a live object. diff --git a/doc/reference/remoteproxy.rst b/doc/reference/remoteproxy.rst new file mode 100644 index 000000000..26ac6cb16 --- /dev/null +++ b/doc/reference/remoteproxy.rst @@ -0,0 +1,93 @@ +.. _RemoteProxy: + +RemoteProxy +=========== + +``RemoteProxy`` is a persistable reference to one remote B2ND array. It accepts +an fsspec URL or a Caterva2 :ref:`URLPath` and separates the portable reference +from any runtime data cache. + +The default policy is :attr:`blosc2.CachePolicy.NONE`: each operation reads the +remote data it needs and no fetched data is retained afterwards. Saving the +object writes only its source descriptor and array geometry, never fetched data +or credentials. + +.. code-block:: python + + remote = blosc2.RemoteProxy( + "s3://public-bucket/dataset.b2nd", + cache_policy=blosc2.CachePolicy.NONE, + ) + remote.save("dataset-reference.b2nd") + +Runtime caching is available through :attr:`blosc2.CachePolicy.MEMORY` and +:attr:`blosc2.CachePolicy.DISK`. Memory caches retain at most 256 MiB of +compressed payload by default. Disk caches are unlimited by default, but both +can take an explicit ``max_cache_bytes`` bound. The bound is enforced after an +operation completes and therefore does not limit its temporary working set or +returned NumPy array. + +.. code-block:: python + + remote = blosc2.RemoteProxy( + "s3://public-bucket/dataset.b2nd", + cache_policy=blosc2.CachePolicy.DISK, + cache_path="dataset-cache.b2nd", + max_cache_bytes=2 * 2**30, + ) + +Regardless of its runtime policy, :meth:`RemoteProxy.save +` and :meth:`RemoteProxy.to_cframe +` produce a reference-only object that reopens +with :attr:`blosc2.CachePolicy.NONE`. Local cache paths and authentication data are not +serialized. + +To cache again after reopening a reference, opt into a runtime policy when +constructing a new proxy from its source: + +.. code-block:: python + + reference = blosc2.open("dataset-reference.b2nd") + cached = blosc2.RemoteProxy( + reference.urlpath, + cache_policy=blosc2.CachePolicy.MEMORY, + ) + +.. warning:: + + Resolving an uploaded remote reference makes the receiving server perform + an outbound request. Caterva2 installations must reject these references by + default unless administrators configure allowed protocols, destinations, + credentials, redirects, and resource limits. Client-side URL checks are not + a server security boundary. + +.. autoclass:: blosc2.RemoteProxy + + .. automethod:: __init__ + .. automethod:: __getitem__ + .. automethod:: get_chunk + .. automethod:: aget_chunk + .. automethod:: save + .. automethod:: to_cframe + .. autoattribute:: shape + .. autoattribute:: dtype + .. autoattribute:: ndim + .. autoattribute:: chunks + .. autoattribute:: blocks + .. autoattribute:: cparams + .. autoattribute:: nbytes + .. autoattribute:: info + .. autoattribute:: cache_bytes + .. autoattribute:: cache_policy + .. autoattribute:: max_cache_bytes + .. autoattribute:: cache_path + .. autoattribute:: cache_status + .. autoattribute:: source + .. autoattribute:: traffic + .. autoattribute:: urlpath + +CachePolicy +----------- + +.. autoclass:: blosc2.CachePolicy + :members: diff --git a/plans/remote-proxy.md b/plans/remote-proxy.md new file mode 100644 index 000000000..54c67170c --- /dev/null +++ b/plans/remote-proxy.md @@ -0,0 +1,817 @@ +# Plan: Persistable `RemoteProxy` + +## Motivation + +Python-Blosc2 can already access remote B2ND arrays lazily: + +```python +array = blosc2.open( + remote_urlpath, + lazy=True, + cache_path="mycache.b2nd", +) +``` + +The resulting `mycache.b2nd` can be uploaded to Caterva2 and acts as a proxy, +but it is fundamentally a **persistent cache**. As remote chunks are read, +the file acquires compressed data plus cache bookkeeping. This is useful for +offline reuse, but it is not the right representation when the desired object +is only a small, immutable reference to a remote array. + +The missing abstraction is a persistable proxy whose stored B2ND file contains +the remote source description and array geometry, but never becomes the data +cache. When uploaded to Caterva2, reads should be forwarded to the referenced +Caterva2 or fsspec source under an explicit server-side security policy. + +## Decision Summary + +Introduce a new public `RemoteProxy` type rather than extending +`SimpleProxy` or broadening `C2Array`: + +- `SimpleProxy` remains the generic, non-persistable adapter for arbitrary + array-like Python objects. +- `C2Array` remains the direct Caterva2 client object and its existing + persistence format remains supported. +- `Proxy` remains the implementation for reusable memory and disk caches. +- `RemoteProxy` is a persistable, backend-neutral remote reference with an + explicit `CachePolicy`. + +The public cache policy should be an enum: + +```python +class CachePolicy(Enum): + NONE = "none" + MEMORY = "memory" + DISK = "disk" +``` + +Python cannot parse `CachePolicy.None` because `None` is a keyword, so the +public spelling must be `blosc2.CachePolicy.NONE`. Uppercase enum members also +match the enum style already used by Python-Blosc2. + +## Implementation Status + +The Python-Blosc2 client implementation on this branch covers the core +RemoteProxy design and the first four implementation phases: + +- `CachePolicy.NONE`, `MEMORY`, and `DISK` are public and validated at runtime. +- `RemoteProxy` supports Caterva2 `URLPath`/`C2Array` sources and contiguous + single-file fsspec B2ND URLs. +- `NONE` uses direct Caterva2 indexing or operation-scoped fsspec assembly; + `MEMORY` and `DISK` reuse the existing `Proxy` machinery. +- Memory caches default to a 256 MiB post-operation compressed-payload bound; + disk caches are unlimited by default and support the same optional LRU bound. +- Remote lazy open accepts explicit cache policies and limits while preserving + the pre-existing `blosc2.open(..., lazy=True)` `Proxy` behavior when neither + is supplied. +- Reference-only `remote_proxy` carriers, strict source/geometry validation, + authentication omission, fsspec `Ref` values, LazyExpr operands, and + ObjectArray/BatchArray msgpack round trips are implemented and documented. +- Client-side URL safety checks reject local filesystem URLs, chained fsspec + URLs, user information, fragments, and credential-like query parameters. + +The following remain future work or deliberate follow-ups: + +- Opening a local reference carrier with an explicit runtime policy (for + example, `blosc2.open("reference.b2nd", cache_policy=...)`) is not yet a + convenience path. Reconstructing `RemoteProxy(reference.urlpath, ...)` is + the current explicit opt-in. +- Cache-oriented `fetch()`/`afetch()` methods are not exposed on `RemoteProxy` + under `NONE`; a separate materialization API can be designed later. +- Caterva2 server-side discovery, default-deny protocol/destination policy, + credential selection, SSRF protection, resource limits, reference-cycle + handling, and tenant isolation remain to be implemented in Caterva2. +- Pinned reference semantics, broader fsspec/server protocol allowlists, and + any `C2Array.save(as_remote_proxy=True)` convenience are future decisions. + +## Goals + +The first implementation should allow this workflow: + +```python +proxy = blosc2.RemoteProxy( + "s3://example/dataset.b2nd", + cache_policy=blosc2.CachePolicy.NONE, +) +proxy.save("dataset-proxy.b2nd") + +# Upload dataset-proxy.b2nd to a suitably configured Caterva2 server. +``` + +It should also work for a Caterva2 source: + +```python +proxy = blosc2.RemoteProxy( + blosc2.URLPath( + "@public/dataset.b2nd", + urlbase="https://example.org/caterva2", + ), + cache_policy=blosc2.CachePolicy.NONE, +) +``` + +The saved file should: + +- be metadata-sized rather than proportional to the remote array +- reopen as `RemoteProxy` +- preserve shape, dtype, chunks, blocks, and the safe source descriptor +- serve slices and chunks from either Caterva2 or supported fsspec URLs +- remain byte-for-byte unchanged after reads under `CachePolicy.NONE` +- never contain credentials + +## Non-Goals For The First Version + +- Persisting arbitrary `SimpleProxy` sources or Python callables. +- Serializing fsspec filesystem instances or arbitrary `storage_options`. +- Embedding authentication tokens, cookies, cloud keys, or signed credentials. +- Proxying every fsspec object type. Initially support a single remote + contiguous B2ND frame; remote directory stores such as `.b2d` can be added + after their object and authorization semantics are designed. +- Making an uploaded descriptor safe merely through client-side URL + validation. Caterva2 must enforce its own resolution policy. +- Changing the existing on-disk cache proxy format. + +## Why `RemoteProxy` Should Be A Separate Type + +### Do not make `SimpleProxy` persistable + +`SimpleProxy` accepts a broad array-like object with `shape`, `dtype`, and +`__getitem__`. Most such objects have no stable or safe reconstruction recipe. +Making the class conditionally persistable only for Caterva2 and fsspec inputs +would give one public type two substantially different contracts and invite +accidental serialization of arbitrary Python state. + +`RemoteProxy` can instead require a constrained, versioned source descriptor. +This makes persistence an invariant of the type rather than a special case. + +### Do not make `C2Array` backend-neutral + +`C2Array` models Caterva2 operations and authentication. Teaching it about +fsspec URLs would mix the Caterva2 protocol with byte-range filesystem access. +It would also leave no natural home for the cache policy shared by both +backends. + +### Reuse implementation, not identity + +`RemoteProxy` should delegate to existing components: + +- `C2Array` for Caterva2 reads +- `FsspecNDSource` for fsspec metadata, chunks, and byte ranges +- `Proxy` for reusable memory or disk caching + +It should not duplicate those implementations, and it should not itself be a +subclass of `SimpleProxy` unless that inheritance remains strictly an internal +convenience with no effect on serialization. + +## Public API + +### Construction + +Proposed primary constructor: + +```python +proxy = blosc2.RemoteProxy( + urlpath, + cache_policy=blosc2.CachePolicy.NONE, + cache_path=None, + cache_dir=None, + max_cache_bytes=None, +) +``` + +The omitted `max_cache_bytes` value uses the policy-dependent default; an +explicit `None` requests an unlimited MEMORY or DISK cache. + +The constructor should discover the source kind and remote array metadata. A +future explicit `source_kind=` escape hatch can be added if URL recognition is +ambiguous, but should not be needed initially. + +`RemoteProxy` should expose at least: + +- `shape`, `dtype`, `chunks`, `blocks`, and `cparams` +- `urlpath` and a read-only normalized source descriptor +- `cache_policy` +- synchronous `__getitem__` and `get_chunk()` +- asynchronous counterparts where the selected backend supports them +- traffic information compatible with the current remote-access diagnostics +- `to_cframe()` and `save(urlpath)` + +It should participate as an expression operand in the same way as `C2Array` +and other remote array-like operands. + +### Integration with `blosc2.open` + +Once the explicit class is stable, extend the existing remote open path: + +```python +blosc2.open( + remote_urlpath, + lazy=True, + cache_policy=blosc2.CachePolicy.NONE, +) +``` + +Recommended compatibility mapping: + +| Arguments | Effective policy | Result | +| --- | --- | --- | +| `lazy=True` only | `MEMORY` | Preserve current process-local lazy cache behavior | +| `lazy=True, cache_policy=NONE` | `NONE` | `RemoteProxy` with no retained data cache | +| `lazy=True, cache_policy=MEMORY` | `MEMORY` | `RemoteProxy` backed by the current memory `Proxy` | +| `lazy=True, cache_path=...` | `DISK` | Preserve current persistent-cache behavior | +| `lazy=True, cache_dir=...` | `DISK` | Preserve current persistent-cache behavior | +| `lazy=True, cache_policy=DISK, cache_path/cache_dir=...` | `DISK` | Explicit persistent cache | + +Validation rules: + +- `NONE` or `MEMORY` combined with `cache_path`/`cache_dir` is an error. +- `DISK` without a cache location is an error unless a documented automatic + cache-location policy is deliberately introduced. +- `max_cache_bytes` is invalid with `NONE`, defaults to 256 MiB with `MEMORY`, + and defaults to unlimited with `DISK`. A positive explicit value bounds + either memory or disk cache payload. The implementation needs an internal + sentinel to distinguish an omitted policy-dependent default from explicit + `None`, which means unlimited. +- Supplying both `cache_path` and `cache_dir` continues to follow the existing + validation rule. +- The Python API should require a `CachePolicy` instance rather than expose + several string aliases. The serialized payload uses stable lowercase string + values so it is independent of Python enum internals. + +The default constructor policy for an explicit `RemoteProxy` should be `NONE`, +because its defining purpose is a reference-only proxy. The default for the +pre-existing `blosc2.open(..., lazy=True)` call should remain `MEMORY` to avoid +a silent performance regression. + +## Precise Cache Semantics + +The word "none" must describe retained cache state, not prohibit every +temporary buffer. fsspec range reads need somewhere to assemble compressed +blocks for a slice. + +### `CachePolicy.NONE` + +- No fetched chunk or block is retained between independent operations. +- No fetched bitmap, chunk payload, or cache index is written to the carrier. +- The serialized carrier is never used as scratch space. +- Temporary buffers may exist in memory for the duration of one operation. +- If an operation requires an NDArray workspace, it is operation-scoped and + discarded before returning. +- Repeating the same data read is expected to contact the remote source again. +- Metadata may be retained in the live Python object's immutable fields; + reopening the carrier does not imply a data fetch. + +### `CachePolicy.MEMORY` + +- Fetched data may be retained for the lifetime of the Python object. +- Repeating a covered read should be served without remote data traffic. +- Retained compressed payload is limited to 256 MiB by default. The caller may + select another positive `max_cache_bytes`, or explicitly request an + unlimited cache with `None` through an API representation that distinguishes + it from an omitted policy-dependent default. +- After each operation, least-recently-used chunks are evicted until the + retained payload is within the bound. +- `save()` still writes a reference-only carrier with persisted policy `NONE`, + not a snapshot of this process-local cache. +- Closing or dropping the object loses the cache. + +### `CachePolicy.DISK` + +- Reuse the existing persistent `Proxy` cache behavior and format. +- The disk cache is unlimited by default for compatibility and because its + purpose is cross-process reuse. An explicit positive `max_cache_bytes` + enables the same post-operation LRU bound as the memory cache. +- Eviction replaces whole cached chunks with `UNINIT`, clears their fetched + bitmap entries, and shrinks the live `.b2nd` payload. Compact contiguous + files may need to move later compressed data, so frequent disk eviction can + be more expensive than memory eviction. +- The disk cache and a reference carrier are distinct concepts. +- `RemoteProxy.save()` should always save the reference-only representation and + normalize the persisted policy to `NONE`. Memory and disk policies describe + the live process, not portable behavior to impose on another machine. +- The configured cache lives at `cache_path` or under `cache_dir`; it is not the + destination passed to `RemoteProxy.save()`. +- Existing `Proxy` cache files continue reopening as `Proxy`, not + `RemoteProxy`. + +This distinction prevents a supposedly portable descriptor from silently +growing or containing a partial snapshot because it happened to be read before +upload. + +### Meaning and accounting of `max_cache_bytes` + +The bound applies after an operation completes and covers retained compressed +cache payload, including partial chunks and duplicated hot partial-block +payloads. It does not bound: + +- the compressed working set needed to complete the current operation +- in-flight concurrent responses +- decompression and assembly buffers +- the NumPy result returned to the caller +- total process RSS, because an allocator may retain freed arenas + +Eviction must happen only after the requested result has been assembled. The +current proxy sequence fetches all required regions before reading the result +from its cache, so evicting during that fetch could discard an early chunk and +produce an incorrect result. + +LRU granularity is one chunk even when only some blocks in that chunk are +cached. Cache hits refresh recency as well as remote fetches. A chunk larger +than the bound may be used for the current operation and then evicted, leaving +the retained payload below the limit. + +For a reopened disk cache, exact recency from a previous process need not be +persisted initially. Existing fetched chunks are seeded in deterministic chunk +order as older than chunks touched by the new process. This preserves the +bound and correctness without rewriting LRU metadata after every read; it only +reduces eviction quality immediately after reopen. + +## Source Model + +Only explicitly supported, reconstructable source kinds should be serialized. +Extend the reference model with a versioned fsspec source kind while retaining +the existing Caterva2 reference kind. + +Conceptually: + +```python +RemoteSource = Caterva2SourceRef | FsspecSourceRef +``` + +Suggested descriptors: + +```python +{ + "kind": "caterva2", + "version": 1, + "path": "@public/dataset.b2nd", + "urlbase": "https://example.org/caterva2", +} +``` + +```python +{ + "kind": "fsspec", + "version": 1, + "urlpath": "s3://public-bucket/dataset.b2nd", +} +``` + +The source reference must contain only location and format information. It +must not include headers, bearer tokens, passwords, signed query parameters, +filesystem objects, or arbitrary fsspec keyword arguments. + +The current `Ref.from_object(Proxy)` behavior should not be changed as part of +this feature: persisted lazy-expression operands may rely on it unwrapping to +the proxy cache. Add a dedicated remote-source encoder/resolver instead of +silently changing generic `Ref` semantics. + +## Serialized B2 Object Format + +Use the existing B2 object carrier mechanism with a new object kind, +`remote_proxy`. The carrier is an empty, structurally valid NDArray containing +array geometry in its normal metadata and a versioned B2 object payload in +variable-length metadata. + +Example payload: + +```python +{ + "kind": "remote_proxy", + "version": 1, + "source": { + "kind": "fsspec", + "version": 1, + "urlpath": "s3://public-bucket/dataset.b2nd", + }, + "cache_policy": "none", +} +``` + +The carrier should include: + +- shape and dtype +- chunk and block geometry +- compression parameters needed to interpret fetched chunks +- the `remote_proxy` payload + +Version 1 always writes `"cache_policy": "none"`. This field makes the +reference-only behavior explicit and leaves room for future policy negotiation, +but a carrier must not request server memory or disk use based on the creating +process's runtime policy. A caller reopening locally can select a runtime cache +policy through an explicit future open override; without one, it remains +`NONE`. + +It must not include: + +- `proxy-source` +- `proxy-fetched` or `proxy-index` +- fetched compressed chunks +- access credentials or client configuration + +`process_opened_object()` should dispatch `b2o.kind == "remote_proxy"` to the +new decoder. The existing `proxy-source` check must continue to identify +legacy/current persistent cache proxies before generic B2 object dispatch. + +### Source identity and mutation + +Version 1 may optionally store a non-secret source stamp such as an ETag, +content length, or backend revision when available. It should not rely on one +being available for every backend. + +Recommended initial semantics are a **floating reference with structural +validation**: + +- reads see the source's current contents +- the source must still match the carrier's shape, dtype, chunks, and blocks +- a mismatch raises a clear stale-reference error before returning data + +A future pinned mode can require an exact source stamp for reproducibility. It +should be a separate, explicit option rather than an accidental consequence of +metadata captured at creation time. + +## Runtime Design + +### Common `RemoteProxy` layer + +`RemoteProxy` owns: + +- the normalized source descriptor +- immutable array geometry captured in the carrier +- the selected `CachePolicy` +- a backend adapter +- optional memory/disk cache state according to policy +- the policy-dependent retained-cache limit and chunk-level LRU state + +The public layer validates the source metadata against the carrier and provides +consistent indexing, persistence, traffic reporting, and error behavior. + +### Caterva2 backend + +For `NONE`, delegate slices and chunks directly to `C2Array`; no assembly cache +is necessary. For `MEMORY` and `DISK`, either continue direct delegation when +it satisfies the operation or wrap it with the existing cache proxy machinery. +The policy must still determine whether results are reusable between calls. + +The serialized source should reuse the existing `C2Array` reference fields, +including the rule that authentication tokens are not persisted. + +### fsspec backend + +Use `FsspecNDSource` for metadata, chunk, and range access. Because this source +does not provide general `__getitem__`, slicing needs an assembly layer: + +- `NONE`: construct an in-memory operation-scoped workspace for the requested + slice/chunks, return the result, then discard the workspace +- `MEMORY`: use one in-memory `Proxy` cache retained by `RemoteProxy` +- `DISK`: use the existing persistent `Proxy` at the configured cache location, + with optional post-operation LRU eviction + +The first implementation should favor correctness and a clean policy boundary. +An optimized no-cache slice assembler can replace the operation-scoped proxy +later without changing the public or serialized formats. + +### Fetch APIs + +The first implementation deliberately keeps cache-oriented `fetch()` and +`afetch()` on `Proxy` rather than exposing them on `RemoteProxy`. This avoids +an ambiguous operation under `CachePolicy.NONE`: indexing and `get_chunk()` +return results while retaining no reusable data, whereas a future named +materialization API can explicitly return a new `NDArray` without changing the +reference carrier. Any such API must not silently convert a no-cache proxy +into a retained cache. + +## Caterva2 Server Contract + +Serving an uploaded `RemoteProxy` requires explicit support in Caterva2. The +Python-Blosc2 carrier is only a descriptor; Caterva2 is responsible for deciding +whether and how it may be resolved. + +Recommended request lifecycle: + +1. Upload stores the carrier as an immutable reference object. +2. Catalog and shape/dtype inspection may use the carrier metadata without an + outbound request. +3. On first data access, the server parses and validates the descriptor against + its configured remote-source policy. +4. It resolves current remote metadata and verifies the carrier geometry. +5. It performs only the byte ranges or slices needed for the request. +6. Any temporary assembly state lives outside the uploaded dataset and is + discarded for `NONE`. +7. The carrier's contents, size, and modification time remain unchanged. + +An installation may optionally validate reachability at upload time, but that +cannot replace validation at read time: DNS, redirects, credentials, and the +remote object can all change later. + +Until Caterva2 implements the security and resource controls below, it should +reject `remote_proxy` carriers by default rather than resolve arbitrary URLs. + +## Security Requirements + +An uploaded remote reference asks the server to make outbound requests chosen +by a client. This is an SSRF and resource-exhaustion boundary, not merely a new +file type. + +### Protocol and destination policy + +Caterva2 should: + +- deny all source protocols by default +- enable only administrator-configured protocols, for example `https` or `s3` +- reject local and process-oriented schemes such as `file`, `memory`, and + arbitrary chained fsspec URLs +- optionally allow only configured hosts, ports, buckets, and key prefixes +- resolve hostnames and reject loopback, link-local, private, multicast, and + cloud-instance-metadata destinations unless explicitly authorized +- recheck the destination after every redirect and cap redirect count +- protect against DNS rebinding by validating the actual connection target, + not only the submitted hostname +- normalize URLs before policy checks to prevent parser or encoding bypasses + +Support for fsspec protocol chaining such as archive-over-network URLs should +be out of scope initially because every layer expands the policy surface. + +### Credentials + +- Never serialize client credentials in the B2 object. +- Strip or reject user-info, sensitive query parameters, custom headers, + cookies, tokens, and arbitrary `storage_options` at creation and upload. +- Configure server credentials out of band. +- Scope credentials to the smallest allowed host, bucket, and prefix. +- Select credentials from the validated destination, never from untrusted + descriptor-provided provider names. +- Avoid reflecting secrets or sensitive internal response bodies in errors. + +Public Caterva2 references should continue to work without a persisted auth +token. Private references require credentials available to the resolving +server; client credentials cannot make an uploaded proxy portable safely. + +### Resource limits + +Caterva2 should configure and enforce: + +- connection, read, and total request timeouts +- maximum redirects, range requests, retries, and concurrency per operation +- maximum bytes fetched for metadata and for one user request +- maximum rank, shape, logical `nbytes`, chunk count, and metadata size +- decompression and expansion limits before allocating output buffers +- per-user or per-tenant rate and bandwidth limits +- cancellation of upstream requests when the client request is cancelled + +Carrier geometry is untrusted input and must be validated before multiplication +or allocation. + +### Reference graphs + +A remote target may itself be another proxy, possibly pointing back to the +original object. The server must enforce: + +- maximum proxy depth and total remote hops +- cycle detection using normalized source identities +- rejection of direct or indirect self-references +- one cumulative resource budget across the entire reference chain + +### Tenant isolation and observability + +- Never share authenticated sessions or memory caches across security + principals unless the cache key includes the full authorization context. +- Log descriptor identity, resolved destination, bytes, request count, timing, + and policy decision without logging secrets. +- Expose actionable but non-sensitive failures for denied destinations, + unavailable credentials, stale geometry, and exhausted limits. + +## Compatibility And Migration + +- Existing persisted `C2Array` carriers keep their current kind and decoder. +- `C2Array.save()` remains unchanged initially; it may delegate internally in a + later cleanup but should not silently start emitting `remote_proxy`. +- Existing persistent cache files using `proxy-source` keep their current + format and reopen behavior. +- `SimpleProxy` remains non-persistable. +- Existing `blosc2.open(remote, lazy=True)` behavior remains memory-cached. +- Existing `cache_path` and `cache_dir` calls remain disk-cached. +- The new B2 object kind must fail clearly on older readers, as other unknown B2 + object kinds do, without being mistaken for an ordinary empty NDArray. +- Stored policy values are stable lowercase strings; decoder code maps these to + enum members and rejects unknown values rather than guessing. + +## Proposed Code Organization + +### New module + +Add `src/blosc2/remote_proxy.py` containing: + +- `RemoteProxy` +- source normalization and safe descriptor validation used by the client +- backend selection +- policy-specific runtime adapters +- B2 object payload encode/decode helpers where this avoids import cycles + +Client-side validation improves error messages and prevents accidentally +writing credentials, but must be documented as distinct from Caterva2's +authoritative server-side policy. + +### Existing modules + +- `src/blosc2/__init__.py` + - define or re-export `CachePolicy` + - export `RemoteProxy` + - add both to `__all__` +- `src/blosc2/ref.py` + - add a versioned fsspec reference representation or a dedicated + remote-source reference helper + - do not change generic `Proxy` unwrapping semantics +- `src/blosc2/b2objects.py` + - encode and decode the `remote_proxy` B2 object kind + - build the metadata-only carrier +- `src/blosc2/schunk.py` + - recognize the new B2 object during `blosc2.open()` dispatch + - preserve precedence of persistent cache-proxy detection +- `src/blosc2/proxy.py` + - expose reusable internal assembly/cache pieces only as needed + - leave `SimpleProxy`'s public contract unchanged +- `src/blosc2/c2array.py` + - expose any small backend-neutral hooks needed by `RemoteProxy` + - retain its existing public persistence behavior + +If defining `CachePolicy` in `__init__.py` creates import cycles, place it in a +small non-private core module and re-export it from `blosc2`. The public name +and enum values are the compatibility surface, not its physical module. + +## Implementation Phases + +### Phase 0: Settle contracts + +- Confirm that `NONE` means no reusable data across independent operations, + while permitting operation-scoped memory. +- Decide `fetch()`/`afetch()` behavior under `NONE`. +- Confirm floating-reference semantics and structural validation. +- Agree on the Caterva2 protocol/host policy and initially supported fsspec + schemes. +- Version and document the `remote_proxy` payload before writing code. + +### Phase 1: Enum and source descriptors + +- Add and export `CachePolicy` with `NONE`, `MEMORY`, and `DISK`. +- Add normalized Caterva2 and fsspec descriptor creation. +- Reject secrets and unsupported URL constructions. +- Add round-trip tests for descriptors and enum payload values. + +### Phase 2: Runtime `RemoteProxy` + +- Implement construction and metadata discovery. +- Implement Caterva2 reads for all policies. +- Implement fsspec reads with operation-scoped assembly for `NONE`. +- Reuse existing memory and persistent `Proxy` caching for the other policies. +- Add traffic accounting and synchronous/asynchronous behavior. +- Verify source geometry before serving data. +- Add shared chunk-level LRU accounting and post-operation eviction for bounded + memory and disk caches. + +### Phase 3: Persistence + +- Add `remote_proxy` carrier encoding, `to_cframe()`, and `save()`. +- Add open-time dispatch and decoding. +- Guarantee reference-only persistence independently of runtime cache state, + normalizing the saved policy to `NONE`. +- Verify that reads never mutate a `NONE` carrier. + +### Phase 4: `blosc2.open` integration + +- Add `cache_policy` to the remote lazy-open path. +- Preserve old defaults and infer `DISK` from existing cache-location + arguments. +- Add conflict validation and focused regression tests. +- Keep the explicit `RemoteProxy` constructor available so descriptor creation + does not depend on overloaded `open()` behavior. + +### Phase 5: Caterva2 support + +- Add `remote_proxy` discovery and read dispatch to Caterva2. +- Implement default-deny protocol and destination configuration. +- Add credential selection outside the descriptor. +- Add SSRF, redirect, DNS, resource-limit, cycle, and tenant-isolation tests. +- Confirm the uploaded carrier is never mutated. + +This phase may live in the Caterva2 repository, but the feature should not be +presented as safe for arbitrary uploads until both sides are complete. + +### Phase 6: Documentation and examples + +- Add a `RemoteProxy` API page and include it in the reference toctree. +- Document the three cache policies and their lifetime guarantees. +- Add examples for a public Caterva2 source and an allowed fsspec HTTPS/S3 + source. +- Document that private sources use server-side credentials. +- Add a Caterva2 administrator guide for the security policy and operational + limits. + +## Test Plan + +### Unit tests + +- `CachePolicy` exports and serialized values. +- `RemoteProxy` metadata and indexing for mocked Caterva2 and fsspec sources. +- Descriptor normalization and rejection of credentials/unsupported schemes. +- B2 object cframe and file round trips for each supported source kind. +- Unknown payload versions, source kinds, and policy values fail clearly. +- Source geometry changes produce a stale-reference error. +- `RemoteProxy` works as a lazy-expression operand. + +### Cache-policy tests + +- Under `NONE`, two identical reads each cause remote data traffic. +- Under `NONE`, carrier size, bytes, mtime, and metadata are unchanged after + reads. +- Under `MEMORY`, an identical covered second read causes no remote data + traffic within the same object lifetime. +- The default memory cache retains at most 256 MiB of compressed payload after + each operation. +- A bounded memory cache evicts least-recently-used chunks and refetches them + when accessed again. +- Reopening a `MEMORY` carrier starts with an empty memory cache. +- Under `DISK`, a covered read survives close/reopen through the configured + cache path. +- A disk cache is unlimited by default; with an explicit bound its live payload + and file size fall after LRU eviction, allowing only fixed metadata overhead. +- Reopening a bounded disk cache preserves the limit and seeds deterministic + recency without requiring persistent timestamps. +- `RemoteProxy.save()` remains reference-only after memory or disk cache use. +- Invalid policy/cache-location combinations raise deterministic errors. + +### Compatibility tests + +- Existing `C2Array` cframes/files still reopen as `C2Array`. +- Existing `proxy-source` cache files still reopen as `Proxy` and preserve + fetched state. +- Existing remote `lazy=True` calls retain their current default caching. +- Existing lazy-expression serialization involving `Proxy` operands is + unchanged. +- The default non-network suite uses local mocks or a local range-capable HTTP + fixture; real services remain under the `network` marker. + +### Caterva2 security tests + +- Default-deny behavior for all remote descriptors. +- Allowed public destination succeeds. +- Local file, loopback, private/link-local addresses, cloud metadata endpoints, + forbidden ports, and disallowed buckets/prefixes are rejected. +- Redirect from an allowed URL to a denied destination is rejected. +- DNS rebinding or changed resolution is rejected at connection time. +- User-info, sensitive query data, custom credentials, and chained fsspec + protocols are rejected. +- Cycles and excessive reference depth are rejected. +- Byte, range, concurrency, timeout, decompression, and allocation limits are + enforced. +- One tenant cannot observe or reuse another tenant's authenticated cache or + session. + +## Acceptance Criteria + +The feature is ready when all of the following hold: + +1. A user can persist and reopen a `RemoteProxy` for both Caterva2 and one + supported fsspec-backed single-file B2ND source. +2. A `NONE` carrier stays metadata-sized and byte-for-byte unchanged after any + supported read. +3. Repeated reads demonstrate observably different traffic behavior for + `NONE`, `MEMORY`, and `DISK`. +4. Bounded memory and disk caches evict whole least-recently-used chunks after + each operation without affecting the returned result; the default bound is + 256 MiB for memory and unlimited for disk. +5. `RemoteProxy.save()` never serializes fetched data or credentials. +6. Legacy C2Array and persistent cache-proxy files keep their behavior. +7. Caterva2 rejects remote proxies by default and resolves them only through an + administrator-controlled destination and credential policy. +8. Geometry changes, unavailable credentials, denied destinations, and + resource-limit failures produce clear errors. +9. API and administrator documentation explain both caching semantics and the + outbound-request security boundary. + +## Future Considerations + +- Which fsspec schemes should a Caterva2 server implementation support? A + narrow starting set such as HTTPS and S3 is preferable to arbitrary plugins. +- When should pinned references be added, and what exact mismatch exception + should they raise? +- Should a future `C2Array.save(as_remote_proxy=True)` convenience exist, or is + the explicit `RemoteProxy(c2array.urlpath)` conversion clearer? +- Should local URLs remain accepted by Python-Blosc2 for testing while + Caterva2 rejects them, or should client-side construction enforce remote-only + schemes everywhere? + +## Initial Slice (Completed) + +The smallest end-to-end path was implemented as follows: + +1. `CachePolicy` and explicit `RemoteProxy` construction. +2. Public, single-file HTTPS/fsspec B2ND source. +3. `CachePolicy.NONE` with operation-scoped in-memory assembly. +4. `remote_proxy` cframe/file round trip. +5. Local mocked tests proving the carrier is immutable and repeated reads do + not reuse data. + +Caterva2-source support and memory/disk policy integration are also implemented +in the Python client. Caterva2 server resolution remains a separate follow-up +behind a default-deny configuration, as described in Phase 5. diff --git a/src/blosc2/__init__.py b/src/blosc2/__init__.py index 307e3bde0..5f4023811 100644 --- a/src/blosc2/__init__.py +++ b/src/blosc2/__init__.py @@ -230,6 +230,14 @@ class IndexKind(Enum): OPSI = "opsi" +class CachePolicy(Enum): + """Retention policy for data read through a remote proxy.""" + + NONE = "none" + MEMORY = "memory" + DISK = "disk" + + from .blosc2_ext import ( DEFINED_CODECS_STOP, EXTENDED_HEADER_LENGTH, @@ -608,6 +616,7 @@ def _raise(exc): jit, as_simpleproxy, ) +from .remote_proxy import RemoteProxy from . import linalg from .linalg import tensordot, vecdot, permute_dims, matrix_transpose, matmul, transpose, diagonal, outer from .utils import linalg_funcs as linalg_funcs_list @@ -873,6 +882,7 @@ def _raise(exc): "BatchArray", # Enums "Codec", + "CachePolicy", "DParams", "DictStore", "EmbedStore", @@ -896,6 +906,7 @@ def _raise(exc): "ProxyNDSource", "ProxySource", "Ref", + "RemoteProxy", "SChunk", "SimpleProxy", "SpecialValue", diff --git a/src/blosc2/b2objects.py b/src/blosc2/b2objects.py index 375a1c9f3..a525e2c76 100644 --- a/src/blosc2/b2objects.py +++ b/src/blosc2/b2objects.py @@ -116,7 +116,7 @@ def encode_b2object_payload(obj) -> dict[str, Any] | None: return None -def decode_b2object_payload(payload: dict[str, Any], *, carrier_path=None): +def decode_b2object_payload(payload: dict[str, Any], *, carrier_path=None, carrier=None): kind = payload.get("kind") version = payload.get("version") if version != _B2OBJECT_VERSION: @@ -124,6 +124,10 @@ def decode_b2object_payload(payload: dict[str, Any], *, carrier_path=None): if kind == "c2array": ref = blosc2.Ref.from_dict(payload) return ref.open() + if kind == "remote_proxy": + if carrier is None: + raise ValueError("A persisted RemoteProxy requires its B2ND carrier") + return blosc2.RemoteProxy._from_payload(payload, carrier) if kind == "lazyexpr": return decode_structured_lazyexpr(payload, carrier_path=carrier_path) if kind == "lazyudf": @@ -227,7 +231,7 @@ def open_b2object(obj): schunk = getattr(obj, "schunk", obj) if getattr(schunk, "urlpath", None) is not None: carrier_path = pathlib.Path(schunk.urlpath).parent - opened = decode_b2object_payload(payload, carrier_path=carrier_path) + opened = decode_b2object_payload(payload, carrier_path=carrier_path, carrier=obj) if isinstance(opened, blosc2.LazyExpr | blosc2.LazyUDF): opened.array = obj opened.schunk = schunk diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index c14db8303..575f3a3b1 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -1683,7 +1683,9 @@ def __init__(self, path: str, /, urlbase: str | None = None, auth_token: str | N Passing this object to :func:`blosc2.open` returns a :ref:`C2Array`. With ``lazy=True`` it instead returns a :ref:`Proxy`, using an in-memory cache - by default or a persistent cache when ``cache_dir`` is provided. + by default or a persistent cache when ``cache_dir`` is provided. Passing + ``cache_policy`` explicitly selects a :ref:`RemoteProxy` with the + requested retention policy. Authenticated users sharing a machine must use separate cache directories. The parameters are the same as for the :meth:`C2Array.__init__`. diff --git a/src/blosc2/core.py b/src/blosc2/core.py index feed6ebef..0982adf6f 100644 --- a/src/blosc2/core.py +++ b/src/blosc2/core.py @@ -2102,6 +2102,7 @@ def from_cframe( | blosc2.BatchArray | blosc2.ObjectArray | blosc2.C2Array + | blosc2.RemoteProxy ): """Create a :ref:`EmbedStore `, :ref:`NDArray `, :ref:`SChunk `, :ref:`BatchArray ` or :ref:`ObjectArray ` instance @@ -2120,7 +2121,8 @@ def from_cframe( Returns ------- out: :ref:`EmbedStore `, :ref:`NDArray `, :ref:`SChunk `, - :ref:`BatchArray ` or :ref:`ObjectArray ` + :ref:`BatchArray `, :ref:`ObjectArray `, or + :ref:`RemoteProxy ` A new instance of the appropriate type containing the data passed. See Also diff --git a/src/blosc2/lazyexpr.py b/src/blosc2/lazyexpr.py index 2a9cf8473..bfc6efa10 100644 --- a/src/blosc2/lazyexpr.py +++ b/src/blosc2/lazyexpr.py @@ -709,7 +709,8 @@ def save(self, **kwargs: Any) -> None: * All the operands of the LazyArray must be Python scalars, or :class:`blosc2.Array` objects. * If an operand is a :ref:`Proxy`, keep in mind that Python-Blosc2 will only be able to reopen it as such if its source is a :ref:`SChunk`, :ref:`NDArray` or a :ref:`C2Array` (see :func:`blosc2.open` notes - section for more info). + section for more info). A :ref:`RemoteProxy` is persisted as its + reference-only source descriptor. * This is currently only supported for :ref:`LazyExpr` and :ref:`LazyUDF` (including kernels decorated with :func:`blosc2.dsl_kernel`). * User metadata can be attached via :attr:`vlmeta`. For in-memory LazyArrays @@ -4700,10 +4701,19 @@ def info_items(self): items = [] items += [("type", f"{self.__class__.__name__}")] items += [("expression", self.expression)] - opsinfo = { - key: str(value) if value.schunk.urlpath is None else value.schunk.urlpath - for key, value in self.operands.items() - } + opsinfo = {} + for key, value in self.operands.items(): + if isinstance(value, blosc2.RemoteProxy): + urlpath = value.urlpath + opsinfo[key] = urlpath if isinstance(urlpath, str) else str(value) + continue + schunk = getattr(value, "schunk", None) + if schunk is not None: + urlpath = getattr(schunk, "urlpath", None) + opsinfo[key] = str(value) if urlpath is None else urlpath + else: + # C2Array is a remote reference without a local SChunk. + opsinfo[key] = getattr(value, "urlpath", str(value)) items += [("operands", opsinfo)] items += [("shape", self.shape)] items += [("dtype", self.dtype)] @@ -4732,6 +4742,9 @@ def _to_b2object_carrier(self, **kwargs): if isinstance(value, blosc2.C2Array): payload["operands"][key] = encode_b2object_payload(value) continue + if isinstance(value, blosc2.RemoteProxy): + payload["operands"][key] = blosc2.Ref.from_object(value).to_dict() + continue if isinstance(value, blosc2.Proxy): value = value._cache ref = getattr(value, "_blosc2_ref", None) @@ -4739,9 +4752,7 @@ def _to_b2object_carrier(self, **kwargs): payload["operands"][key] = ref.to_dict() continue if not hasattr(value, "schunk"): - raise ValueError( - "To save a LazyArray, all operands must be blosc2.NDArray or blosc2.C2Array objects" - ) + raise ValueError("To save a LazyArray, all operands must be persistent Blosc2 array objects") if value.schunk.urlpath is None: raise ValueError("To save a LazyArray, all operands must be stored on disk/network") operand_urlpath = Path(value.schunk.urlpath) diff --git a/src/blosc2/msgpack_utils.py b/src/blosc2/msgpack_utils.py index 79d14899d..204be2c4b 100644 --- a/src/blosc2/msgpack_utils.py +++ b/src/blosc2/msgpack_utils.py @@ -59,7 +59,13 @@ def _encode_msgpack_ext(obj): import blosc2 if isinstance( - obj, blosc2.NDArray | blosc2.SChunk | blosc2.ObjectArray | blosc2.BatchArray | blosc2.EmbedStore + obj, + blosc2.NDArray + | blosc2.SChunk + | blosc2.ObjectArray + | blosc2.BatchArray + | blosc2.EmbedStore + | blosc2.RemoteProxy, ): return ExtType(_BLOSC2_EXT_CODE, obj.to_cframe()) structured = _encode_structured_reference(obj) diff --git a/src/blosc2/ndarray.py b/src/blosc2/ndarray.py index 398a9defa..d2fb889a1 100644 --- a/src/blosc2/ndarray.py +++ b/src/blosc2/ndarray.py @@ -159,7 +159,7 @@ class Array(Protocol): This protocol describes the basic interface required by blosc2 arrays. It is implemented by blosc2 classes (:ref:`NDArray`, :ref:`NDField`, - :ref:`LazyArray`, :ref:`C2Array`, :ref:`ProxyNDSource`...) + :ref:`LazyArray`, :ref:`C2Array`, :ref:`RemoteProxy`, :ref:`ProxyNDSource`...) and is compatible with NumPy arrays and other array-like containers (e.g., PyTorch, TensorFlow, Dask, Zarr, ...). """ diff --git a/src/blosc2/objectarray.py b/src/blosc2/objectarray.py index fdde23cf1..326e5c748 100644 --- a/src/blosc2/objectarray.py +++ b/src/blosc2/objectarray.py @@ -36,7 +36,8 @@ class ObjectArray: Entries are serialized with msgpack before compression. Standard Python objects are supported, and Blosc2 containers such as :class:`blosc2.NDArray`, :class:`blosc2.SChunk`, :class:`blosc2.ObjectArray`, - :class:`blosc2.BatchArray`, and :class:`blosc2.EmbedStore` are serialized + :class:`blosc2.BatchArray`, :class:`blosc2.EmbedStore`, and + :class:`blosc2.RemoteProxy` are serialized transparently via :meth:`to_cframe` / :func:`blosc2.from_cframe`. Msgpack also supports structured Blosc2 reference objects. Currently this @@ -44,8 +45,9 @@ class ObjectArray: :class:`blosc2.LazyUDF` backed by :func:`blosc2.dsl_kernel`. Lazy expressions and supported lazy UDFs are serialized as recipes plus durable operand references, so only persistent local operands, - :class:`blosc2.C2Array` operands, and :class:`blosc2.DictStore` members are - supported. Purely in-memory operands are intentionally rejected. Plain + :class:`blosc2.C2Array` and :class:`blosc2.RemoteProxy` operands, and + :class:`blosc2.DictStore` members are supported. Purely in-memory operands + are intentionally rejected. Plain Python :class:`blosc2.LazyUDF` callables are not serialized by msgpack. """ diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index 0c14a40b2..cb1ea7034 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -12,6 +12,7 @@ import math import os import textwrap +from collections import OrderedDict from collections.abc import Sequence from concurrent.futures import ThreadPoolExecutor @@ -41,7 +42,14 @@ # vlmeta entries the proxy keeps its own state in: what it has fetched, and which # remote bytes the cache was filled from. A caller cannot write these. _RESERVED_VLMETA = frozenset( - {"proxy-fetched", "proxy-fetched-blocks", "proxy-fetched-bpc", "proxy-stamp", "proxy-index"} + { + "proxy-cache-sizes", + "proxy-fetched", + "proxy-fetched-blocks", + "proxy-fetched-bpc", + "proxy-stamp", + "proxy-index", + } ) # `jit` kwargs that tune *how* an expression is evaluated, not what container the @@ -52,6 +60,16 @@ _JIT_EXECUTION_TUNING_KWARGS = frozenset({"jit", "jit_backend", "fp_accuracy"}) +def _validate_max_cache_bytes(value: int | None) -> int | None: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError("max_cache_bytes must be a positive integer or None") + if value <= 0: + raise ValueError("max_cache_bytes must be a positive integer or None") + return value + + class Proxy(blosc2.Operand): """Proxy (with cache support) for an object following the :ref:`ProxySource` interface. @@ -132,6 +150,7 @@ def __init__( if kwargs is None: kwargs = {} self._cache = kwargs.pop("_cache", None) + self._max_cache_bytes = _validate_max_cache_bytes(kwargs.pop("_max_cache_bytes", None)) vlmeta = kwargs.pop("vlmeta", None) caterva2_env = kwargs.pop("caterva2_env", False) # Before anything is built or emptied: a call that is going to be refused @@ -244,6 +263,9 @@ def __init__( if self.urlpath is None: self.urlpath = getattr(self._schunk_cache, "urlpath", None) self._fetched = self._adopt_cache(fresh, self._schunk_cache.nchunks) + self._cache_sizes: dict[int, int] = {} + self._cache_lru = OrderedDict() + self._restore_cache_accounting() for key in vlmeta or (): self._schunk_cache.vlmeta[key] = vlmeta[key] @@ -425,6 +447,64 @@ def _sync_evictions(self) -> None: for n in range(base, base + self._blocks_per_chunk): self._fetched[n // 8] &= ~(1 << (n % 8)) self._hot_payloads.pop(info.nchunk, None) + self._cache_sizes.pop(info.nchunk, None) + self._cache_lru.pop(info.nchunk, None) + + def _restore_cache_accounting(self) -> None: + """Restore compressed-byte accounting for a bounded cache.""" + if self._max_cache_bytes is None: + return + stored = self._schunk_cache.vlmeta.get("proxy-cache-sizes", {}) + if not isinstance(stored, dict): + stored = {} + for nchunk in range(self._schunk_cache.nchunks): + base = nchunk * self._blocks_per_chunk + if not any( + self._fetched[n // 8] >> (n % 8) & 1 for n in range(base, base + self._blocks_per_chunk) + ): + continue + size = stored.get(nchunk, stored.get(str(nchunk))) + if not isinstance(size, int) or size < 0: + # Only legacy caches lack this metadata. Opting such a cache into + # a bound pays one compressed-chunk read per populated chunk once. + size = len(self._schunk_cache.get_chunk(nchunk)) + self._cache_sizes[nchunk] = size + self._cache_lru[nchunk] = None + + def _remember_cached(self, nchunk: int, size: int) -> None: + """Record the current compressed size and recency of one cached chunk.""" + if self._max_cache_bytes is None: + return + self._cache_sizes[nchunk] = size + self._cache_lru.pop(nchunk, None) + self._cache_lru[nchunk] = None + + def _retained_cache_bytes(self) -> int: + """Compressed bytes retained by a bounded cache, including hot duplicates.""" + hot = sum(len(payload) for blocks in self._hot_payloads.values() for payload in blocks.values()) + return sum(self._cache_sizes.values()) + hot + + def _enforce_cache_limit(self, item) -> None: + """Touch *item* and evict whole LRU chunks after its result is assembled.""" + if self._max_cache_bytes is None: + return + for nchunk in self._wanted_chunks(item): + if nchunk in self._cache_sizes: + self._cache_lru.move_to_end(nchunk) + + evicted = False + while self._retained_cache_bytes() > self._max_cache_bytes and self._cache_lru: + nchunk, _ = self._cache_lru.popitem(last=False) + self._cache_sizes.pop(nchunk, None) + self._hot_payloads.pop(nchunk, None) + self._schunk_cache.update_special(nchunk, blosc2.SpecialValue.UNINIT) + base = nchunk * self._blocks_per_chunk + for n in range(base, base + self._blocks_per_chunk): + self._fetched[n // 8] &= ~(1 << (n % 8)) + evicted = True + if evicted: + self._specialized = getattr(self._schunk_cache, "nspecialized", self._specialized) + self._save_fetched() def _plan(self, item): """Where *item* lands on the cache's grid, read once for a fetch. @@ -565,6 +645,10 @@ def _save_fetched(self) -> None: self._schunk_cache.vlmeta[self._fetched_key] = bytes(self._fetched) if self._blocks_per_chunk > 1: self._schunk_cache.vlmeta["proxy-fetched-bpc"] = self._blocks_per_chunk + if self._max_cache_bytes is not None: + self._schunk_cache.vlmeta["proxy-cache-sizes"] = { + str(nchunk): size for nchunk, size in self._cache_sizes.items() + } # Where the source read things to be, so the next run over this cache need # not ask again. Only for a source that can name the bytes it read: an # unstamped one cannot tell a replaced frame from the one these positions @@ -867,7 +951,9 @@ def _write_blocks(self, nchunk: int, payloads: dict[int, bytes], header: bytes) # the whole chunk, and it cannot be deferred or batched further, since the # cache is what the next read comes out of. Removing it needs the cache to # hold blocks apart from their chunk, which is a different container. - self._schunk_cache.update_chunk(nchunk, _splice_chunk(header, nblocks, kept)) + chunk = _splice_chunk(header, nblocks, kept) + self._schunk_cache.update_chunk(nchunk, chunk) + self._remember_cached(nchunk, len(chunk)) for nblock in payloads: self._mark_fetched(nchunk, nblock) if len(kept) < nblocks: # a chunk that is now complete will never be rewritten @@ -878,6 +964,7 @@ def _write_blocks(self, nchunk: int, payloads: dict[int, bytes], header: bytes) def _store_chunk(self, nchunk: int, chunk: bytes) -> None: """Put a whole chunk in the cache, dropping anything held about its blocks.""" self._schunk_cache.update_chunk(nchunk, chunk) + self._remember_cached(nchunk, len(chunk)) self._mark_fetched(nchunk) self._hot_payloads.pop(nchunk, None) @@ -1034,7 +1121,9 @@ def __getitem__(self, item: slice | list[slice]) -> np.ndarray: if getattr(self._schunk_cache, "mode", None) != "r" or "reading mode" not in str(exc): raise return self.src[item] - return self._cache[item] + result = self._cache[item] + self._enforce_cache_limit(item) + return result @property def dtype(self) -> np.dtype: diff --git a/src/blosc2/ref.py b/src/blosc2/ref.py index b1eda6b14..292f611bc 100644 --- a/src/blosc2/ref.py +++ b/src/blosc2/ref.py @@ -20,6 +20,7 @@ class Ref: - a persistent local Blosc2 object reopenable from ``urlpath`` - a member inside a :class:`blosc2.DictStore` - a remote :class:`blosc2.C2Array` + - an fsspec URL used by a :class:`blosc2.RemoteProxy` Instances can be created directly, from dictionaries via :meth:`from_dict`, or from supported objects via :meth:`from_object`. Use :meth:`open` to @@ -33,11 +34,18 @@ class Ref: urlbase: str | None = None def __post_init__(self) -> None: - if self.kind == "urlpath": + if self.kind in {"urlpath", "fsspec"}: if not isinstance(self.urlpath, str): - raise TypeError("Ref(kind='urlpath') requires a string 'urlpath'") + raise TypeError(f"Ref(kind={self.kind!r}) requires a string 'urlpath'") if self.key is not None or self.path is not None or self.urlbase is not None: - raise ValueError("Ref(kind='urlpath') only supports the 'urlpath' field") + raise ValueError(f"Ref(kind={self.kind!r}) only supports the 'urlpath' field") + if self.kind == "fsspec": + # Keep structured references subject to the same credential and + # portability checks as an explicit RemoteProxy. The import is + # local because Ref is imported before the public proxy module. + from blosc2.remote_proxy import _validate_persistable_url + + _validate_persistable_url(self.urlpath) return if self.kind == "dictstore_key": if not isinstance(self.urlpath, str): @@ -69,6 +77,10 @@ def dictstore_key(cls, urlpath: str, key: str) -> Ref: def c2array_ref(cls, path: str, urlbase: str | None = None) -> Ref: return cls(kind="c2array", path=path, urlbase=urlbase) + @classmethod + def fsspec_ref(cls, urlpath: str) -> Ref: + return cls(kind="fsspec", urlpath=urlpath) + @classmethod def from_dict(cls, payload: dict[str, Any]) -> Ref: if not isinstance(payload, dict): @@ -90,6 +102,11 @@ def from_object(cls, obj: Any) -> Ref: if isinstance(obj, blosc2.C2Array): return cls.c2array_ref(obj.path, obj.urlbase) + if isinstance(obj, blosc2.RemoteProxy): + source = obj.source + if source["kind"] == "caterva2": + return cls.c2array_ref(source["path"], source["urlbase"]) + return cls.fsspec_ref(source["urlpath"]) if isinstance(obj, blosc2.Proxy): obj = obj._cache ref = getattr(obj, "_blosc2_ref", None) @@ -100,11 +117,11 @@ def from_object(cls, obj: Any) -> Ref: if urlpath is None: raise ValueError("Durable Blosc2 references require operands to be stored on disk/network") return cls.urlpath_ref(urlpath) - raise TypeError("Durable Blosc2 references require NDArray, C2Array, or Proxy operands") + raise TypeError("Durable Blosc2 references require NDArray, C2Array, RemoteProxy, or Proxy operands") def to_dict(self) -> dict[str, Any]: payload = {"kind": self.kind, "version": 1} - if self.kind == "urlpath": + if self.kind in {"urlpath", "fsspec"}: payload["urlpath"] = self.urlpath elif self.kind == "dictstore_key": payload["urlpath"] = self.urlpath @@ -125,4 +142,6 @@ def open(self): return blosc2.DictStore(self.urlpath, mode="r")[self.key] if self.kind == "c2array": return blosc2.C2Array(self.path, urlbase=self.urlbase) + if self.kind == "fsspec": + return blosc2.RemoteProxy(self.urlpath) raise ValueError(f"Unsupported Ref kind: {self.kind!r}") diff --git a/src/blosc2/remote_proxy.py b/src/blosc2/remote_proxy.py new file mode 100644 index 000000000..18798d102 --- /dev/null +++ b/src/blosc2/remote_proxy.py @@ -0,0 +1,443 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Persistable references to remote arrays.""" + +from __future__ import annotations + +import math +import os +from urllib.parse import parse_qsl, urlsplit + +import numpy as np + +import blosc2 +from blosc2.b2objects import make_b2object_carrier, write_b2object_payload +from blosc2.info import InfoReporter, format_nbytes_info + +DEFAULT_MEMORY_CACHE_BYTES = 256 * 2**20 + + +class _PolicyDefault: + def __repr__(self) -> str: + return "" + + +_POLICY_DEFAULT = _PolicyDefault() +_SENSITIVE_QUERY_PARTS = ( + "credential", + "signature", + "signed", + "token", + "password", + "secret", + "key", + "sig", + "expires", +) + + +def _validate_persistable_url(url: str) -> None: + """Reject URL features that would put credentials in a portable carrier.""" + if "::" in url: + raise ValueError("RemoteProxy does not persist chained fsspec URLs") + parsed = urlsplit(url) + if not parsed.scheme: + raise ValueError("RemoteProxy requires a remote URL or a Caterva2 URLPath") + if parsed.scheme.lower() in {"file", "local"}: + raise ValueError("RemoteProxy does not persist local filesystem URLs") + if parsed.username is not None or parsed.password is not None: + raise ValueError("RemoteProxy URLs cannot contain user information") + if parsed.fragment: + raise ValueError("RemoteProxy URLs cannot contain fragments") + sensitive = [ + key + for key, _ in parse_qsl(parsed.query, keep_blank_values=True) + if any(part in key.lower() for part in _SENSITIVE_QUERY_PARTS) + ] + if sensitive: + raise ValueError("RemoteProxy URLs cannot contain credential-like query parameters") + + +def _normalize_limit(policy, value): + if policy is blosc2.CachePolicy.NONE: + if value is not _POLICY_DEFAULT: + raise ValueError("max_cache_bytes is not applicable to CachePolicy.NONE") + return None + if value is _POLICY_DEFAULT: + return DEFAULT_MEMORY_CACHE_BYTES if policy is blosc2.CachePolicy.MEMORY else None + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError("max_cache_bytes must be a positive integer or None") + if value <= 0: + raise ValueError("max_cache_bytes must be a positive integer or None") + return value + + +def _validate_max_concurrency(value: int | None) -> int | None: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError("max_concurrency must be a positive integer") + if value <= 0: + raise ValueError("max_concurrency must be a positive integer") + return value + + +class RemoteProxy(blosc2.Operand): + """A persistable reference to a Caterva2 or fsspec remote array. + + Unlike :class:`Proxy`, the object serialized by :meth:`save` is never the + data cache. It is a metadata-only carrier which reopens with + :attr:`CachePolicy.NONE`, regardless of the live proxy's runtime policy. + Memory and disk caches are runtime choices and are not serialized. + + Parameters + ---------- + urlpath: str, URLPath, or C2Array + A single-file B2ND URL opened through fsspec, or a Caterva2 array + reference. + cache_policy: CachePolicy + ``NONE`` retains no array data between operations. ``MEMORY`` retains + compressed data in memory. ``DISK`` retains it in ``cache_path`` or + under ``cache_dir``. + cache_path: str or path-like, optional + Exact persistent cache filename. Only valid with ``DISK`` and mutually + exclusive with ``cache_dir``. + cache_dir: str or path-like, optional + Directory in which a source-derived persistent cache filename is made. + Only valid with ``DISK``. + max_cache_bytes: int or None, optional + Post-operation compressed-payload bound. It defaults to 256 MiB for + ``MEMORY`` and unlimited for ``DISK``; explicit ``None`` means + unlimited. It is not applicable to ``NONE``. + max_concurrency: int, optional + Maximum number of independent remote fetches in flight. + """ + + def __init__( + self, + urlpath, + *, + cache_policy=blosc2.CachePolicy.NONE, + cache_path=None, + cache_dir=None, + max_cache_bytes=_POLICY_DEFAULT, + max_concurrency: int | None = None, + ): + if not isinstance(cache_policy, blosc2.CachePolicy): + raise TypeError("cache_policy must be a blosc2.CachePolicy instance") + if cache_dir is not None and cache_path is not None: + raise ValueError("cache_dir and cache_path are mutually exclusive") + if cache_policy is not blosc2.CachePolicy.DISK and (cache_dir is not None or cache_path is not None): + raise ValueError("cache_dir and cache_path require CachePolicy.DISK") + if cache_policy is blosc2.CachePolicy.DISK and cache_dir is None and cache_path is None: + raise ValueError("CachePolicy.DISK requires cache_dir or cache_path") + + self._cache_policy = cache_policy + self._cache_limit = _normalize_limit(cache_policy, max_cache_bytes) + max_concurrency = _validate_max_concurrency(max_concurrency) + self.src, self._source = self._open_source(urlpath, max_concurrency) + self._proxy = None + + if cache_policy is not blosc2.CachePolicy.NONE: + identity = self._source_identity() + self._proxy = blosc2.schunk._lazy_remote_proxy( + self.src, + identity, + cache_dir, + cache_path, + source_fresh=True, + max_cache_bytes=self._cache_limit, + ) + + @staticmethod + def _open_source(urlpath, max_concurrency): + if isinstance(urlpath, blosc2.C2Array): + src = urlpath + if src.urlbase is not None: + _validate_persistable_url(src.urlbase) + source = { + "kind": "caterva2", + "version": 1, + "path": src.path, + "urlbase": src.urlbase, + } + elif isinstance(urlpath, blosc2.URLPath): + if urlpath.urlbase is not None: + _validate_persistable_url(urlpath.urlbase) + src = blosc2.C2Array(urlpath.path, urlbase=urlpath.urlbase, auth_token=urlpath.auth_token) + source = { + "kind": "caterva2", + "version": 1, + "path": src.path, + "urlbase": src.urlbase, + } + elif isinstance(urlpath, str): + _validate_persistable_url(urlpath) + kwargs = {} if max_concurrency is None else {"max_concurrency": max_concurrency} + src = blosc2.FsspecNDSource(urlpath, **kwargs) + source = {"kind": "fsspec", "version": 1, "urlpath": urlpath} + else: + raise TypeError("RemoteProxy requires a URL string, URLPath, or C2Array") + + if max_concurrency is not None and isinstance(src, blosc2.C2Array): + src.max_concurrency = max_concurrency + return src, source + + def _source_identity(self) -> str: + if self._source["kind"] == "fsspec": + return self._source["urlpath"] + return f"caterva2:{blosc2.c2array._server_url(self.src.urlbase, self.src.path)}" + + def _validate_geometry(self, expected) -> None: + if expected is None: + return + actual = ( + tuple(self.src.shape), + np.dtype(self.src.dtype), + tuple(self.src.chunks), + tuple(self.src.blocks), + ) + normalized = ( + tuple(expected[0]), + np.dtype(expected[1]), + tuple(expected[2]), + tuple(expected[3]), + ) + if actual != normalized: + raise ValueError( + "RemoteProxy source geometry no longer matches its carrier: " + f"carrier={normalized}, source={actual}" + ) + + @property + def shape(self): + return tuple(self.src.shape) + + @property + def dtype(self): + return np.dtype(self.src.dtype) + + @property + def ndim(self) -> int: + """The number of dimensions in the remote array.""" + return len(self.shape) + + @property + def chunks(self): + return tuple(self.src.chunks) + + @property + def blocks(self): + return tuple(self.src.blocks) + + @property + def cache_policy(self) -> blosc2.CachePolicy: + """The immutable runtime retention policy.""" + return self._cache_policy + + @property + def max_cache_bytes(self) -> int | None: + """The immutable post-operation retained-cache bound.""" + return self._cache_limit + + @property + def cparams(self): + return self.src.cparams + + @property + def traffic(self): + return getattr(self.src, "traffic", None) + + @property + def nbytes(self) -> int: + """The uncompressed size of the remote array.""" + value = getattr(self.src, "nbytes", None) + return int(value) if value is not None else math.prod(self.shape) * self.dtype.itemsize + + @property + def info(self) -> InfoReporter: + """A printable summary of this remote reference.""" + return InfoReporter(self) + + @property + def info_items(self) -> list[tuple[str, object]]: + """The fields shown by :attr:`info`.""" + return [ + ("type", type(self).__name__), + ("source", self.source), + ("shape", self.shape), + ("chunks", self.chunks), + ("blocks", self.blocks), + ("dtype", self.dtype), + ("nbytes", format_nbytes_info(self.nbytes)), + ("cache_policy", self.cache_policy.name), + ("cache_bytes", format_nbytes_info(self.cache_bytes)), + ] + + @property + def source(self) -> dict: + """A copy of the credential-free source descriptor.""" + return dict(self._source) + + @property + def urlpath(self): + """The remote fsspec URL or credential-free Caterva2 URLPath.""" + if self._source["kind"] == "fsspec": + return self._source["urlpath"] + return blosc2.URLPath(self._source["path"], urlbase=self._source["urlbase"]) + + @property + def cache_path(self): + """The runtime disk cache path, or ``None`` for other policies.""" + if self._proxy is None or self.cache_policy is not blosc2.CachePolicy.DISK: + return None + return self._proxy.urlpath + + @property + def cache_status(self): + """How a persistent disk cache was handled, or ``None`` otherwise.""" + if self._proxy is None: + return None + return self._proxy.cache_status + + @property + def cache_bytes(self) -> int: + """Compressed bytes currently retained by the runtime cache.""" + if self._proxy is None: + return 0 + if self._proxy._max_cache_bytes is None: + return self._proxy.schunk.cbytes + return self._proxy._retained_cache_bytes() + + def __getitem__(self, item): + if self._proxy is not None: + return self._proxy[item] + if isinstance(self.src, blosc2.C2Array): + # Caterva2 can evaluate slices and fancy indices server-side. In + # particular, do not turn a no-cache C2 read into a chunk-by-chunk + # client assembly operation just to satisfy the fsspec backend. + return self.src[item] + # fsspec exposes chunk/range reads rather than NumPy indexing. Use an + # operation-scoped Proxy so its temporary assembly state is discarded + # as soon as this result is returned. + proxy = blosc2.Proxy(self.src, _refresh_source=False) + return proxy[item] + + def __len__(self) -> int: + """The length of the first dimension, like other array operands.""" + if not self.shape: + raise TypeError("len() of unsized object") + return self.shape[0] + + def _chunk_slice(self, nchunk: int): + grid = tuple(math.ceil(size / chunk) for size, chunk in zip(self.shape, self.chunks, strict=True)) + total = math.prod(grid) + if nchunk < 0 or nchunk >= total: + raise IndexError(f"nchunk must be in range [0, {total}), got {nchunk}") + coords = np.unravel_index(nchunk, grid) + return tuple( + slice(int(coord) * chunk, min((int(coord) + 1) * chunk, size)) + for coord, chunk, size in zip(coords, self.chunks, self.shape, strict=True) + ) + + def get_chunk(self, nchunk: int) -> bytes: + if self._proxy is None: + return self.src.get_chunk(nchunk) + item = self._chunk_slice(nchunk) + self._proxy.fetch(item) + chunk = self._proxy.schunk.get_chunk(nchunk) + self._proxy._enforce_cache_limit(item) + return chunk + + async def aget_chunk(self, nchunk: int) -> bytes: + if self._proxy is None: + method = getattr(self.src, "aget_chunk", None) + if method is None: + raise NotImplementedError("the remote source does not provide asynchronous chunk reads") + return await method(nchunk) + item = self._chunk_slice(nchunk) + await self._proxy.afetch(item) + chunk = self._proxy.schunk.get_chunk(nchunk) + self._proxy._enforce_cache_limit(item) + return chunk + + def _payload(self): + return { + "kind": "remote_proxy", + "version": 1, + "source": dict(self._source), + # Persistence is intentionally reference-only. Runtime cache state + # and local cache paths never cross this boundary. + "cache_policy": blosc2.CachePolicy.NONE.value, + } + + def _to_b2object_carrier(self, **kwargs): + array = make_b2object_carrier( + "remote_proxy", + self.shape, + self.dtype, + chunks=self.chunks, + blocks=self.blocks, + cparams=self.cparams, + **kwargs, + ) + write_b2object_payload(array, self._payload()) + return array + + def to_cframe(self) -> bytes: + """Serialize this reference, without cached array data, as a CFrame.""" + return self._to_b2object_carrier().to_cframe() + + def save(self, urlpath: str | os.PathLike, contiguous: bool = True, **kwargs) -> None: + """Persist this reference without its runtime cache or credentials.""" + urlpath = os.fspath(urlpath) + blosc2.blosc2_ext.check_access_mode(urlpath, "w") + kwargs.update(urlpath=urlpath, contiguous=contiguous, mode="w") + self._to_b2object_carrier(**kwargs) + + @classmethod + def _from_payload(cls, payload, carrier): + if set(payload) != {"kind", "version", "source", "cache_policy"}: + raise ValueError("persisted RemoteProxy payload contains unsupported fields") + if payload.get("cache_policy") != blosc2.CachePolicy.NONE.value: + raise ValueError("persisted RemoteProxy objects must use cache policy 'none'") + source = payload.get("source") + if not isinstance(source, dict) or source.get("version") != 1: + raise ValueError("unsupported RemoteProxy source descriptor") + source_kind = source.get("kind") + if source_kind == "fsspec": + if set(source) != {"kind", "version", "urlpath"}: + raise ValueError("fsspec RemoteProxy source descriptors contain unsupported fields") + urlpath = source.get("urlpath") + if not isinstance(urlpath, str): + raise TypeError("fsspec RemoteProxy sources require a string 'urlpath'") + elif source_kind == "caterva2": + if set(source) != {"kind", "version", "path", "urlbase"}: + raise ValueError("Caterva2 RemoteProxy source descriptors contain unsupported fields") + path = source.get("path") + urlbase = source.get("urlbase") + if not isinstance(path, str) or (urlbase is not None and not isinstance(urlbase, str)): + raise TypeError("Caterva2 RemoteProxy sources require string 'path' and 'urlbase' fields") + urlpath = blosc2.URLPath(path, urlbase=urlbase) + else: + raise ValueError(f"unsupported RemoteProxy source kind: {source_kind!r}") + expected = (carrier.shape, carrier.dtype, carrier.chunks, carrier.blocks) + obj = cls(urlpath, cache_policy=blosc2.CachePolicy.NONE) + obj._validate_geometry(expected) + return obj + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + return False + + def __str__(self): + return f"RemoteProxy({self._source_identity()!r}, cache_policy={self.cache_policy.name})" diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 928e1f413..51a105e57 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -2010,6 +2010,31 @@ def _remote_cache_options(kwargs: dict) -> tuple[str | pathlib.Path | None, str return (cache_dir if cache_dir is not None else cache_storage), cache_path +def _remote_proxy_options(kwargs, cache_dir, cache_path, max_concurrency): + """Return explicit RemoteProxy options, or None for the legacy lazy Proxy path.""" + policy_present = "cache_policy" in kwargs + limit_present = "max_cache_bytes" in kwargs + if not policy_present and not limit_present: + return None + policy = kwargs.pop("cache_policy", None) + limit = kwargs.pop("max_cache_bytes", None) + if not policy_present: + policy = ( + blosc2.CachePolicy.DISK + if cache_dir is not None or cache_path is not None + else blosc2.CachePolicy.MEMORY + ) + options = { + "cache_policy": policy, + "cache_dir": cache_dir, + "cache_path": cache_path, + "max_concurrency": max_concurrency, + } + if limit_present: + options["max_cache_bytes"] = limit + return options + + def _lazy_fsspec_proxy( urlpath: str, cache_dir: str | pathlib.Path | None, @@ -2035,10 +2060,15 @@ def _lazy_remote_proxy( cache_path: str | pathlib.Path | None, *, source_fresh: bool = False, + max_cache_bytes: int | None = None, ): """Wrap a remote source in a memory or persistent cache.""" if cache_dir is None and cache_path is None: - return blosc2.Proxy(src, _refresh_source=not source_fresh) + return blosc2.Proxy( + src, + _refresh_source=not source_fresh, + _max_cache_bytes=max_cache_bytes, + ) if cache_path is not None: path = os.fspath(cache_path) @@ -2058,7 +2088,13 @@ def _lazy_remote_proxy( cache_status = "reused" # Proxy stamps the cache with src.stamp itself, and refuses one built against # other bytes; removing it above is what turns that refusal into a refetch - proxy = blosc2.Proxy(src, urlpath=path, mode="a", _refresh_source=not source_fresh) + proxy = blosc2.Proxy( + src, + urlpath=path, + mode="a", + _refresh_source=not source_fresh, + _max_cache_bytes=max_cache_bytes, + ) proxy._cache_status = cache_status return proxy @@ -2073,17 +2109,23 @@ def _open_c2_urlpath(urlpath: blosc2.URLPath, mode: str, offset: int, kwargs: di cache_dir, cache_path = _remote_cache_options(kwargs) max_concurrency = kwargs.pop("max_concurrency", None) lazy = kwargs.pop("lazy", False) + remote_proxy_options = _remote_proxy_options(kwargs, cache_dir, cache_path, max_concurrency) requested = [key for key, value in kwargs.items() if value is not None] if requested: raise NotImplementedError(f"{', '.join(requested)} is not supported for Caterva2 arrays") if not lazy: + if remote_proxy_options is not None: + raise NotImplementedError("cache_policy and max_cache_bytes require lazy=True") if cache_dir is not None or cache_path is not None: raise NotImplementedError("cache_dir and cache_path for a Caterva2 array require lazy=True") if max_concurrency is not None: raise NotImplementedError("max_concurrency is only supported with lazy=True") return blosc2.C2Array(urlpath.path, urlbase=urlpath.urlbase, auth_token=urlpath.auth_token) + if remote_proxy_options is not None: + return blosc2.RemoteProxy(urlpath, **remote_proxy_options) + src = blosc2.C2Array(urlpath.path, urlbase=urlpath.urlbase, auth_token=urlpath.auth_token) if max_concurrency is not None: src.max_concurrency = max_concurrency @@ -2123,14 +2165,21 @@ def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): cache_dir, cache_path = _remote_cache_options(kwargs) max_concurrency = kwargs.pop("max_concurrency", None) - if kwargs.pop("lazy", False): + lazy = kwargs.pop("lazy", False) + remote_proxy_options = _remote_proxy_options(kwargs, cache_dir, cache_path, max_concurrency) + if lazy: if offset != 0: raise NotImplementedError("offset is not supported with lazy=True") requested = [k for k, v in kwargs.items() if v is not None] if requested: raise NotImplementedError(f"{', '.join(requested)} is not supported with lazy=True") + if remote_proxy_options is not None: + return blosc2.RemoteProxy(urlpath, **remote_proxy_options) return _lazy_fsspec_proxy(urlpath, cache_dir, cache_path, max_concurrency) + if remote_proxy_options is not None: + raise NotImplementedError("cache_policy and max_cache_bytes require lazy=True") + if cache_path is not None: raise NotImplementedError("cache_path is only supported with lazy=True") @@ -2167,6 +2216,7 @@ def open( | blosc2.BatchArray | blosc2.ObjectArray | blosc2.C2Array + | blosc2.RemoteProxy | blosc2.LazyArray | blosc2.Proxy | blosc2.DictStore @@ -2174,7 +2224,8 @@ def open( | blosc2.EmbedStore ): """Open a persistent :ref:`SChunk`, :ref:`NDArray`, a remote :ref:`C2Array`, - a :ref:`Proxy`, a :ref:`DictStore`, :ref:`EmbedStore`, or :ref:`TreeStore`. + :ref:`RemoteProxy`, :ref:`Proxy`, a :ref:`DictStore`, :ref:`EmbedStore`, or + :ref:`TreeStore`. See the `Notes` section for more info on opening `Proxy` objects. @@ -2239,6 +2290,17 @@ def open( cache_storage: str | pathlib.Path, optional Deprecated alias for ``cache_dir``. Mutually exclusive with ``cache_dir`` and ``cache_path``. + cache_policy: CachePolicy, optional + With ``lazy=True`` on a remote source, return a :ref:`RemoteProxy` + using the requested retention policy. ``NONE`` retains no data + between operations, ``MEMORY`` retains compressed data in memory, + and ``DISK`` requires ``cache_dir`` or ``cache_path``. When omitted, + the existing :ref:`Proxy` behavior is preserved. + max_cache_bytes: int or None, optional + With ``lazy=True``, enable a :ref:`RemoteProxy` and bound retained + compressed cache payload after each operation. The memory-policy + default is 256 MiB; disk is unlimited unless this is explicitly set. + This does not bound the current operation's working set or result. mmap_mode: str, optional If set, the file will be memory-mapped instead of using the default I/O functions and the `mode` argument will be ignored. @@ -2265,8 +2327,8 @@ def open( Returns ------- - out: :ref:`SChunk`, :ref:`NDArray`, :ref:`C2Array`, :ref:`Proxy`, - :ref:`DictStore`, :ref:`EmbedStore`, or :ref:`TreeStore` + out: :ref:`SChunk`, :ref:`NDArray`, :ref:`C2Array`, :ref:`RemoteProxy`, + :ref:`Proxy`, :ref:`DictStore`, :ref:`EmbedStore`, or :ref:`TreeStore` The object found in the path. Notes @@ -2274,13 +2336,16 @@ def open( * Returned objects can be used as context managers for API consistency. For objects with an explicit ``close()`` implementation, exiting the context will close/flush them; for logical handles such as regular - :class:`SChunk`, :class:`NDArray`, :class:`C2Array`, :class:`Proxy`, and - :class:`LazyArray`, exiting the context is currently a no-op. + :class:`SChunk`, :class:`NDArray`, :class:`C2Array`, :class:`RemoteProxy`, + :class:`Proxy`, and :class:`LazyArray`, exiting the context is currently a + no-op. * If :paramref:`urlpath` is a :ref:`URLPath` instance, :paramref:`mode` must be 'r' and :paramref:`offset` must be 0. Without ``lazy=True`` it returns a :ref:`C2Array`; with ``lazy=True`` it returns a :ref:`Proxy`, - optionally persisted under ``cache_dir`` or at ``cache_path``. + optionally persisted under ``cache_dir`` or at ``cache_path``. Supplying + ``cache_policy`` or ``max_cache_bytes`` explicitly selects a + :ref:`RemoteProxy` instead. Authenticated users sharing a machine must use separate caches. * fsspec URLs need the ``fsspec`` extra (``pip install "blosc2[fsspec]"``) and diff --git a/tests/test_remote_proxy.py b/tests/test_remote_proxy.py new file mode 100644 index 000000000..e01294de6 --- /dev/null +++ b/tests/test_remote_proxy.py @@ -0,0 +1,348 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +from __future__ import annotations + +import hashlib + +import fsspec +import numpy as np +import pytest + +import blosc2 +import blosc2.c2array as blosc2_c2array +from blosc2.b2objects import decode_b2object_payload + + +def _remote_array(name="remote-proxy.b2nd", *, nchunks=4, chunk_size=100_000): + data = np.random.default_rng(1).integers(0, 256, nchunks * chunk_size, dtype=np.uint8) + array = blosc2.asarray(data, chunks=(chunk_size,), blocks=(chunk_size,)) + url = f"memory://{name}" + fsspec.filesystem("memory").pipe_file(name, array.to_cframe()) + return url, data + + +def test_cache_policy_validation(tmp_path): + url, _ = _remote_array("policy.b2nd") + + none = blosc2.RemoteProxy(url) + assert none.cache_policy is blosc2.CachePolicy.NONE + assert none.max_cache_bytes is None + + memory = blosc2.RemoteProxy(url, cache_policy=blosc2.CachePolicy.MEMORY) + assert memory.max_cache_bytes == 256 * 2**20 + + disk = blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=tmp_path / "cache.b2nd", + ) + assert disk.max_cache_bytes is None + + with pytest.raises(TypeError, match="CachePolicy"): + blosc2.RemoteProxy(url, cache_policy="memory") + with pytest.raises(ValueError, match="not applicable"): + blosc2.RemoteProxy(url, max_cache_bytes=1) + with pytest.raises(ValueError, match="requires cache_dir or cache_path"): + blosc2.RemoteProxy(url, cache_policy=blosc2.CachePolicy.DISK) + with pytest.raises(ValueError, match="max_concurrency"): + blosc2.RemoteProxy(url, max_concurrency=0) + + +def test_remote_proxy_array_operand_interface(): + url, _ = _remote_array("operand-interface.b2nd", nchunks=1, chunk_size=100) + proxy = blosc2.RemoteProxy(url) + + assert proxy.ndim == 1 + assert len(proxy) == 100 + assert proxy.info is not None + assert dict(proxy.info_items)["cache_policy"] == "NONE" + + expression = blosc2.lazyexpr("a + 1", operands={"a": proxy}) + assert url in dict(expression.info_items)["operands"].values() + + +def test_open_selects_remote_proxy_only_for_explicit_policy(tmp_path): + url, data = _remote_array("open-policy.b2nd") + + legacy = blosc2.open(url, lazy=True) + assert isinstance(legacy, blosc2.Proxy) + assert not isinstance(legacy, blosc2.RemoteProxy) + + none = blosc2.open(url, lazy=True, cache_policy=blosc2.CachePolicy.NONE) + assert isinstance(none, blosc2.RemoteProxy) + assert none.cache_policy is blosc2.CachePolicy.NONE + np.testing.assert_array_equal(none[:100_000], data[:100_000]) + + memory = blosc2.open(url, lazy=True, max_cache_bytes=120_000) + assert isinstance(memory, blosc2.RemoteProxy) + assert memory.cache_policy is blosc2.CachePolicy.MEMORY + assert memory.max_cache_bytes == 120_000 + + disk = blosc2.open( + url, + lazy=True, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=tmp_path / "open-cache.b2nd", + max_cache_bytes=120_000, + ) + assert isinstance(disk, blosc2.RemoteProxy) + assert disk.cache_policy is blosc2.CachePolicy.DISK + + +def test_none_does_not_retain_remote_data(): + url, data = _remote_array("none.b2nd") + proxy = blosc2.RemoteProxy(url) + + proxy.traffic.reset() + np.testing.assert_array_equal(proxy[:100_000], data[:100_000]) + assert proxy.traffic.requests > 0 + assert proxy.cache_bytes == 0 + + proxy.traffic.reset() + np.testing.assert_array_equal(proxy[:100_000], data[:100_000]) + assert proxy.traffic.requests > 0 + assert proxy.cache_bytes == 0 + + +def test_memory_bound_evicts_lru_chunk(): + url, data = _remote_array("memory-bound.b2nd") + proxy = blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.MEMORY, + max_cache_bytes=120_000, + ) + + np.testing.assert_array_equal(proxy[:100_000], data[:100_000]) + np.testing.assert_array_equal(proxy[100_000:200_000], data[100_000:200_000]) + assert proxy.cache_bytes <= 120_000 + + proxy.traffic.reset() + np.testing.assert_array_equal(proxy[:100_000], data[:100_000]) + assert proxy.traffic.requests > 0 + assert proxy.cache_bytes <= 120_000 + + +def test_memory_bound_refreshes_lru_on_cache_hit(): + url, data = _remote_array("memory-lru.b2nd") + proxy = blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.MEMORY, + max_cache_bytes=220_000, + ) + + proxy[:100_000] + proxy[100_000:200_000] + proxy[:100_000] # chunk 0 is now newer than chunk 1 + proxy[200_000:300_000] + + proxy.traffic.reset() + np.testing.assert_array_equal(proxy[:100_000], data[:100_000]) + assert proxy.traffic.requests == 0 + + proxy.traffic.reset() + np.testing.assert_array_equal(proxy[100_000:200_000], data[100_000:200_000]) + assert proxy.traffic.requests > 0 + + +def test_disk_bound_is_optional_and_shrinks_cache(tmp_path): + url, data = _remote_array("disk-bound.b2nd") + cache_path = tmp_path / "bounded-cache.b2nd" + proxy = blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=cache_path, + max_cache_bytes=120_000, + ) + + for start in (0, 100_000, 200_000): + np.testing.assert_array_equal(proxy[start : start + 100_000], data[start : start + 100_000]) + assert proxy.cache_bytes <= 120_000 + assert cache_path.stat().st_size < 140_000 + + reopened = blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=cache_path, + max_cache_bytes=120_000, + ) + assert reopened.cache_status == "reused" + np.testing.assert_array_equal(reopened[300_000:400_000], data[300_000:400_000]) + assert reopened.cache_bytes <= 120_000 + + +def test_reference_roundtrip_is_none_and_does_not_mutate(tmp_path): + url, data = _remote_array("roundtrip.b2nd") + original = blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.MEMORY, + max_cache_bytes=120_000, + ) + original[:100_000] + + carrier = blosc2.ndarray_from_cframe(original.to_cframe()) + assert carrier.schunk.meta["b2o"] == {"kind": "remote_proxy", "version": 1} + assert carrier.schunk.vlmeta["b2o"] == { + "kind": "remote_proxy", + "version": 1, + "source": {"kind": "fsspec", "version": 1, "urlpath": url}, + "cache_policy": "none", + } + + path = tmp_path / "reference.b2nd" + original.save(path) + before = hashlib.sha256(path.read_bytes()).digest(), path.stat().st_size, path.stat().st_mtime_ns + restored = blosc2.open(path, mode="r") + assert isinstance(restored, blosc2.RemoteProxy) + assert restored.cache_policy is blosc2.CachePolicy.NONE + np.testing.assert_array_equal(restored[:100_000], data[:100_000]) + after = hashlib.sha256(path.read_bytes()).digest(), path.stat().st_size, path.stat().st_mtime_ns + assert after == before + + +def test_reference_rejects_changed_source_geometry(tmp_path): + url, _ = _remote_array("changed-geometry.b2nd", nchunks=1, chunk_size=100) + path = tmp_path / "changed-reference.b2nd" + blosc2.RemoteProxy(url).save(path) + + replacement = blosc2.arange(200, dtype=np.uint8, chunks=(100,), blocks=(100,)) + fsspec.filesystem("memory").pipe_file("changed-geometry.b2nd", replacement.to_cframe()) + with pytest.raises(ValueError, match="geometry no longer matches"): + blosc2.open(path, mode="r") + + +def test_reference_rejects_runtime_cache_policy_in_payload(): + url, _ = _remote_array("bad-persisted-policy.b2nd", nchunks=1, chunk_size=100) + carrier = blosc2.ndarray_from_cframe(blosc2.RemoteProxy(url).to_cframe()) + payload = dict(carrier.schunk.vlmeta["b2o"]) + payload["cache_policy"] = "memory" + + with pytest.raises(ValueError, match="must use cache policy 'none'"): + decode_b2object_payload(payload, carrier=carrier) + + +@pytest.mark.parametrize("field", ["auth_token", "storage_options"]) +def test_reference_rejects_secret_or_runtime_source_fields(field): + url, _ = _remote_array(f"bad-source-{field}.b2nd", nchunks=1, chunk_size=100) + carrier = blosc2.ndarray_from_cframe(blosc2.RemoteProxy(url).to_cframe()) + payload = dict(carrier.schunk.vlmeta["b2o"]) + payload["source"] = dict(payload["source"], **{field: "secret"}) + + with pytest.raises(ValueError, match="unsupported fields"): + decode_b2object_payload(payload, carrier=carrier) + + +def test_caterva2_reference_does_not_persist_auth(monkeypatch): + def fake_info(path, urlbase, params=None, headers=None, model=None, auth_token=None, traffic=None): + return { + "shape": [10], + "chunks": [5], + "blocks": [5], + "dtype": np.dtype(np.int32).str, + "schunk": { + "cparams": dict(blosc2.cparams_dflts), + "nbytes": 40, + "cbytes": 40, + "cratio": 1.0, + "blocksize": 20, + "vlmeta": {}, + }, + } + + monkeypatch.setattr(blosc2_c2array, "info", fake_info) + remote = blosc2.RemoteProxy( + blosc2.URLPath( + "@personal/private.b2nd", + urlbase="https://example.org/caterva2", + auth_token="secret", + ) + ) + carrier = blosc2.ndarray_from_cframe(remote.to_cframe()) + assert carrier.schunk.vlmeta["b2o"]["source"] == { + "kind": "caterva2", + "version": 1, + "path": "@personal/private.b2nd", + "urlbase": "https://example.org/caterva2/", + } + + restored = blosc2.from_cframe(remote.to_cframe()) + assert isinstance(restored, blosc2.RemoteProxy) + assert isinstance(restored.src, blosc2.C2Array) + assert restored.src.auth_token is None + + +def test_caterva2_no_cache_keeps_native_indexing(monkeypatch): + def fake_info(path, urlbase, params=None, headers=None, model=None, auth_token=None, traffic=None): + return { + "shape": [10], + "chunks": [5], + "blocks": [5], + "dtype": np.dtype(np.int32).str, + "schunk": {"cparams": dict(blosc2.cparams_dflts)}, + } + + calls = [] + + def fake_fetch_data(path, urlbase, params, auth_token=None, as_blosc2=False, traffic=None): + calls.append(params) + return np.arange(10, dtype=np.int32)[2:5] + + monkeypatch.setattr(blosc2_c2array, "info", fake_info) + monkeypatch.setattr(blosc2_c2array, "fetch_data", fake_fetch_data) + remote = blosc2.RemoteProxy( + blosc2.URLPath("@public/native-index.b2nd", urlbase="https://example.org/c2") + ) + + np.testing.assert_array_equal(remote[2:5], np.arange(10, dtype=np.int32)[2:5]) + assert calls == [{"slice_": "2:5"}] + + +def test_remote_proxy_is_a_persistable_lazyexpr_operand(): + url, data = _remote_array("operand.b2nd", nchunks=1, chunk_size=100) + remote = blosc2.RemoteProxy(url) + expression = blosc2.lazyexpr("a + 1", operands={"a": remote}) + + restored = blosc2.from_cframe(expression.to_cframe()) + assert any(isinstance(operand, blosc2.RemoteProxy) for operand in restored.operands.values()) + np.testing.assert_array_equal(restored[:], data + 1) + + +def test_objectarray_msgpack_supports_remote_proxy(): + url, data = _remote_array("objectarray-remote-proxy.b2nd", nchunks=1, chunk_size=100) + proxy = blosc2.RemoteProxy(url) + + objects = blosc2.ObjectArray() + objects.append(proxy) + restored = objects[0] + + assert isinstance(restored, blosc2.RemoteProxy) + assert restored.cache_policy is blosc2.CachePolicy.NONE + np.testing.assert_array_equal(restored[:], data) + + +@pytest.mark.parametrize( + "url", + [ + "https://user@example.org/data.b2nd", + "https://example.org/data.b2nd?token=secret", + "https://example.org/data.b2nd?sig=secret", + "https://example.org/data.b2nd#credentials", + "zip://data.b2nd::https://example.org/archive.zip", + "file:///private/data.b2nd", + ], +) +def test_persistence_rejects_credentials_and_chained_urls(url): + with pytest.raises(ValueError): + blosc2.RemoteProxy(url) + + +@pytest.mark.parametrize( + "url", ["https://example.org/data.b2nd?token=secret", "https://user@example.org/data.b2nd"] +) +def test_fsspec_refs_reject_credentials(url): + with pytest.raises(ValueError): + blosc2.Ref.fsspec_ref(url) From 38890c037b8437536b09675f9b4372c43eacba13 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 4 Sep 2026 18:06:45 +0200 Subject: [PATCH 12/89] Harden remote proxy source refresh --- doc/reference/remoteproxy.rst | 26 +++++- plans/remote-proxy.md | 15 +++- src/blosc2/c2array.py | 18 +++- src/blosc2/proxy_source.py | 46 +++++++++- src/blosc2/remote_proxy.py | 155 ++++++++++++++++++++++++--------- tests/test_fsspec.py | 29 +++++++ tests/test_remote_proxy.py | 156 ++++++++++++++++++++++++++++++++++ 7 files changed, 392 insertions(+), 53 deletions(-) diff --git a/doc/reference/remoteproxy.rst b/doc/reference/remoteproxy.rst index 26ac6cb16..957d51c6f 100644 --- a/doc/reference/remoteproxy.rst +++ b/doc/reference/remoteproxy.rst @@ -20,6 +20,23 @@ or credentials. ) remote.save("dataset-reference.b2nd") +A Caterva2 dataset is named with :class:`blosc2.URLPath` rather than an fsspec +URL: + +.. code-block:: python + + remote = blosc2.RemoteProxy( + blosc2.URLPath( + "@public/dataset.b2nd", + urlbase="https://example.org/caterva2", + ) + ) + +References are floating: before each data operation, ``RemoteProxy`` checks the +source identity and verifies that shape, dtype, chunks, and blocks still match +the captured geometry. A replacement with different geometry is rejected; +runtime memory or disk cache data is discarded when the source identity moves. + Runtime caching is available through :attr:`blosc2.CachePolicy.MEMORY` and :attr:`blosc2.CachePolicy.DISK`. Memory caches retain at most 256 MiB of compressed payload by default. Disk caches are unlimited by default, but both @@ -39,8 +56,13 @@ returned NumPy array. Regardless of its runtime policy, :meth:`RemoteProxy.save ` and :meth:`RemoteProxy.to_cframe ` produce a reference-only object that reopens -with :attr:`blosc2.CachePolicy.NONE`. Local cache paths and authentication data are not -serialized. +with :attr:`blosc2.CachePolicy.NONE`. Local cache paths and authentication data +are not serialized. + +Authentication supplied to a live Caterva2 source is deliberately omitted from +the carrier. A receiving server resolves private sources with credentials from +its own administrator-controlled destination mapping; client credentials never +travel with the reference. To cache again after reopening a reference, opt into a runtime policy when constructing a new proxy from its source: diff --git a/plans/remote-proxy.md b/plans/remote-proxy.md index 54c67170c..ea5b7631b 100644 --- a/plans/remote-proxy.md +++ b/plans/remote-proxy.md @@ -69,6 +69,14 @@ RemoteProxy design and the first four implementation phases: ObjectArray/BatchArray msgpack round trips are implemented and documented. - Client-side URL safety checks reject local filesystem URLs, chained fsspec URLs, user information, fragments, and credential-like query parameters. +- Floating references refresh their source identity before data operations, + reject geometry changes that happen after a carrier is opened, and discard + memory or disk cache state when the object at a source URL is replaced. +- Caterva2 now discovers direct `remote_proxy` carriers without resolving them + and denies data access by default. Its initial opt-in resolver is limited to + credential-free HTTPS destinations on an exact administrator allowlist; it + pins public DNS results, disables redirects, and bounds time, geometry, chunk + count, and concurrency. The following remain future work or deliberate follow-ups: @@ -78,9 +86,10 @@ The following remain future work or deliberate follow-ups: the current explicit opt-in. - Cache-oriented `fetch()`/`afetch()` methods are not exposed on `RemoteProxy` under `NONE`; a separate materialization API can be designed later. -- Caterva2 server-side discovery, default-deny protocol/destination policy, - credential selection, SSRF protection, resource limits, reference-cycle - handling, and tenant isolation remain to be implemented in Caterva2. +- Caterva2 credential selection, S3 support, cumulative fetched-byte budgets, + reference-chain resolution/cycle handling, and authenticated tenant-scoped + sessions remain to be implemented. Remote references embedded inside stored + expressions are rejected until they can use the same secure resolver. - Pinned reference semantics, broader fsspec/server protocol allowlists, and any `C2Array.save(as_remote_proxy=True)` convenience are future decisions. diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index 575f3a3b1..f37a33e2f 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -709,7 +709,15 @@ class C2Array(blosc2.Operand): thread-safe: they share one pooled HTTP client and hold no state of their own. """ - def __init__(self, path: str, /, urlbase: str | None = None, auth_token: str | None = None): + def __init__( + self, + path: str, + /, + urlbase: str | None = None, + auth_token: str | None = None, + *, + _traffic=None, + ): """Create an instance of a remote NDArray. Remote NDArrays can be accessed via HTTP from a Caterva2 server @@ -769,7 +777,7 @@ def __init__(self, path: str, /, urlbase: str | None = None, auth_token: str | N self._meta_lock = threading.Lock() # An index a `Proxy` handed over before the source existed; see _adopt_index self._pending_index = None - self.traffic = blosc2.proxy_source.Traffic() + self.traffic = _traffic if _traffic is not None else blosc2.proxy_source.Traffic() """Bytes and requests this handle has read off the server; see :ref:`Traffic`. Cumulative since the array was opened, counted at the transport, so the @@ -1205,7 +1213,7 @@ def _meta_complete(self) -> bool: vlmeta = self.meta.get("schunk", {}).get("vlmeta") or {} return vlmeta.get("fill_nonce") is not None and vlmeta.get("fill_state", "filling") != "filling" - def refresh_stamp(self) -> None: + def refresh_stamp(self, *, force: bool = False) -> None: """Look at the array again, so that :attr:`stamp` speaks for it now. `meta` is read when the handle is opened and, of itself, never again: a @@ -1216,8 +1224,10 @@ def refresh_stamp(self) -> None: One `api/info`, and none at all for an array already known to be complete -- nothing can write to one of those, so nothing it reports can move. + ``force=True`` is for a durable reference whose path may have been + replaced with another object after this handle was opened. """ - if self._meta_stale or not self._meta_complete: + if force or self._meta_stale or not self._meta_complete: self._reread_meta() # -- Block-granular reads. A :ref:`Proxy` uses these to fetch the blocks a diff --git a/src/blosc2/proxy_source.py b/src/blosc2/proxy_source.py index a775d1629..411c49f7c 100644 --- a/src/blosc2/proxy_source.py +++ b/src/blosc2/proxy_source.py @@ -1275,11 +1275,22 @@ class FsspecNDSource(ByteRangeNDSource): As in :ref:`ByteRangeNDSource`. """ - def __init__(self, urlpath: str, max_concurrency: int = REMOTE_MAX_CONCURRENCY): + def __init__( + self, + urlpath: str, + max_concurrency: int = REMOTE_MAX_CONCURRENCY, + *, + _filesystem=None, + _traffic=None, + ): from blosc2.core import _import_fsspec fsspec = _import_fsspec(urlpath) - fs, path = fsspec.url_to_fs(urlpath) + if _filesystem is None: + fs, path = fsspec.url_to_fs(urlpath) + else: + fs = _filesystem + path = fs._strip_protocol(urlpath) protocols = (fs.protocol,) if isinstance(fs.protocol, str) else fs.protocol self._http = bool({"http", "https"} & set(protocols)) if not self._http and fs.isdir(path): @@ -1300,11 +1311,38 @@ def __init__(self, urlpath: str, max_concurrency: int = REMOTE_MAX_CONCURRENCY): if self._http: from fsspec.utils import tokenize - self.stamp = tokenize(path, fs.kwargs, fs.protocol) + self._base_stamp = tokenize(path, fs.kwargs, fs.protocol) + self.stamp = self._base_stamp else: + self._base_stamp = None self.stamp = fs.ukey(path) self._capture_http_headers = self._http - super().__init__(urlpath, max_concurrency) + super().__init__(urlpath, max_concurrency, traffic=_traffic) + + def refresh_identity(self) -> None: + """Refresh the identity of the object currently stored at this URL. + + Object-store implementations expose this through ``ukey``. HTTP needs + an explicit metadata request because its fsspec ukey identifies only + the URL and options, not the response currently available there. + ``None`` means that the HTTP server supplied no validator strong enough + to justify retaining cached data across operations. + """ + if not self._http: + self.stamp = self._fs.ukey(self._path) + return + + info = self._fs.info(self._path) + etag = info.get("etag", info.get("ETag")) + if etag: + self.stamp = f"{self._base_stamp}:etag:{etag}" + return + modified = info.get("mtime", info.get("LastModified", info.get("last_modified"))) + size = info.get("size") + if modified is not None and size is not None: + self.stamp = f"{self._base_stamp}:modified:{modified}:size:{size}" + return + self.stamp = None def read_range(self, offset: int, size: int) -> bytes: if self._capture_http_headers: diff --git a/src/blosc2/remote_proxy.py b/src/blosc2/remote_proxy.py index 18798d102..d07e1f9d6 100644 --- a/src/blosc2/remote_proxy.py +++ b/src/blosc2/remote_proxy.py @@ -11,6 +11,7 @@ import math import os +import threading from urllib.parse import parse_qsl, urlsplit import numpy as np @@ -141,23 +142,48 @@ def __init__( self._cache_policy = cache_policy self._cache_limit = _normalize_limit(cache_policy, max_cache_bytes) - max_concurrency = _validate_max_concurrency(max_concurrency) - self.src, self._source = self._open_source(urlpath, max_concurrency) + self._max_concurrency = _validate_max_concurrency(max_concurrency) + self.src, self._source = self._open_source(urlpath, self._max_concurrency) + self._runtime_urlpath = self._runtime_source(urlpath) + self._expected_geometry = self._geometry(self.src) + self._expected_cparams = self.src.cparams + self._refresh_lock = threading.Lock() self._proxy = None if cache_policy is not blosc2.CachePolicy.NONE: - identity = self._source_identity() - self._proxy = blosc2.schunk._lazy_remote_proxy( - self.src, - identity, - cache_dir, - cache_path, - source_fresh=True, - max_cache_bytes=self._cache_limit, + self._proxy = self._make_cache_proxy(cache_dir, cache_path) + + def _runtime_source(self, original): + """Keep credentials in live process state, outside the descriptor.""" + if isinstance(self.src, blosc2.C2Array): + return blosc2.URLPath( + self.src.path, + urlbase=self.src.urlbase, + auth_token=self.src.auth_token, ) + return original + + @staticmethod + def _geometry(src): + return ( + tuple(src.shape), + np.dtype(src.dtype), + tuple(src.chunks), + tuple(src.blocks), + ) + + def _make_cache_proxy(self, cache_dir, cache_path): + return blosc2.schunk._lazy_remote_proxy( + self.src, + self._source_identity(), + cache_dir, + cache_path, + source_fresh=True, + max_cache_bytes=self._cache_limit, + ) @staticmethod - def _open_source(urlpath, max_concurrency): + def _open_source(urlpath, max_concurrency, *, traffic=None): if isinstance(urlpath, blosc2.C2Array): src = urlpath if src.urlbase is not None: @@ -171,7 +197,12 @@ def _open_source(urlpath, max_concurrency): elif isinstance(urlpath, blosc2.URLPath): if urlpath.urlbase is not None: _validate_persistable_url(urlpath.urlbase) - src = blosc2.C2Array(urlpath.path, urlbase=urlpath.urlbase, auth_token=urlpath.auth_token) + src = blosc2.C2Array( + urlpath.path, + urlbase=urlpath.urlbase, + auth_token=urlpath.auth_token, + _traffic=traffic, + ) source = { "kind": "caterva2", "version": 1, @@ -181,7 +212,7 @@ def _open_source(urlpath, max_concurrency): elif isinstance(urlpath, str): _validate_persistable_url(urlpath) kwargs = {} if max_concurrency is None else {"max_concurrency": max_concurrency} - src = blosc2.FsspecNDSource(urlpath, **kwargs) + src = blosc2.FsspecNDSource(urlpath, _traffic=traffic, **kwargs) source = {"kind": "fsspec", "version": 1, "urlpath": urlpath} else: raise TypeError("RemoteProxy requires a URL string, URLPath, or C2Array") @@ -195,15 +226,10 @@ def _source_identity(self) -> str: return self._source["urlpath"] return f"caterva2:{blosc2.c2array._server_url(self.src.urlbase, self.src.path)}" - def _validate_geometry(self, expected) -> None: + def _validate_geometry(self, expected, *, src=None) -> None: if expected is None: return - actual = ( - tuple(self.src.shape), - np.dtype(self.src.dtype), - tuple(self.src.chunks), - tuple(self.src.blocks), - ) + actual = self._geometry(self.src if src is None else src) normalized = ( tuple(expected[0]), np.dtype(expected[1]), @@ -216,13 +242,59 @@ def _validate_geometry(self, expected) -> None: f"carrier={normalized}, source={actual}" ) + def _prepare_read(self): + """Refresh source identity and return the backend for one operation.""" + with self._refresh_lock: + previous_stamp = getattr(self.src, "stamp", None) + refresh = getattr(self.src, "refresh_identity", None) + if refresh is None: + refresh = getattr(self.src, "refresh_stamp", None) + if refresh is not None: + if isinstance(self.src, blosc2.C2Array): + refresh(force=True) + else: + refresh() + + self._validate_geometry(self._expected_geometry) + current_stamp = getattr(self.src, "stamp", None) + source_changed = ( + previous_stamp is None or current_stamp is None or current_stamp != previous_stamp + ) + if source_changed: + cache_path = self.cache_path + if ( + current_stamp is None + and self.cache_policy is blosc2.CachePolicy.DISK + and cache_path is not None + and os.path.exists(cache_path) + ): + # With no source identity, an on-disk cache cannot prove + # that its payload belongs to what the URL serves now. + blosc2.remove_urlpath(cache_path) + fresh, _ = self._open_source( + self._runtime_urlpath, + self._max_concurrency, + traffic=self.traffic, + ) + if current_stamp is None and not isinstance(fresh, blosc2.C2Array): + # No stable validator means cached bytes cannot safely be + # carried from one independent operation to the next. + fresh.stamp = None + self._validate_geometry(self._expected_geometry, src=fresh) + self.src = fresh + if self.cache_policy is not blosc2.CachePolicy.NONE: + cache_dir = None + self._proxy = self._make_cache_proxy(cache_dir, cache_path) + + return self.src if self._proxy is None else self._proxy + @property def shape(self): - return tuple(self.src.shape) + return self._expected_geometry[0] @property def dtype(self): - return np.dtype(self.src.dtype) + return self._expected_geometry[1] @property def ndim(self) -> int: @@ -231,11 +303,11 @@ def ndim(self) -> int: @property def chunks(self): - return tuple(self.src.chunks) + return self._expected_geometry[2] @property def blocks(self): - return tuple(self.src.blocks) + return self._expected_geometry[3] @property def cache_policy(self) -> blosc2.CachePolicy: @@ -249,7 +321,7 @@ def max_cache_bytes(self) -> int | None: @property def cparams(self): - return self.src.cparams + return self._expected_cparams @property def traffic(self): @@ -317,17 +389,18 @@ def cache_bytes(self) -> int: return self._proxy._retained_cache_bytes() def __getitem__(self, item): - if self._proxy is not None: - return self._proxy[item] - if isinstance(self.src, blosc2.C2Array): + backend = self._prepare_read() + if isinstance(backend, blosc2.Proxy): + return backend[item] + if isinstance(backend, blosc2.C2Array): # Caterva2 can evaluate slices and fancy indices server-side. In # particular, do not turn a no-cache C2 read into a chunk-by-chunk # client assembly operation just to satisfy the fsspec backend. - return self.src[item] + return backend[item] # fsspec exposes chunk/range reads rather than NumPy indexing. Use an # operation-scoped Proxy so its temporary assembly state is discarded # as soon as this result is returned. - proxy = blosc2.Proxy(self.src, _refresh_source=False) + proxy = blosc2.Proxy(backend, _refresh_source=False) return proxy[item] def __len__(self) -> int: @@ -348,24 +421,26 @@ def _chunk_slice(self, nchunk: int): ) def get_chunk(self, nchunk: int) -> bytes: - if self._proxy is None: - return self.src.get_chunk(nchunk) + backend = self._prepare_read() + if not isinstance(backend, blosc2.Proxy): + return backend.get_chunk(nchunk) item = self._chunk_slice(nchunk) - self._proxy.fetch(item) - chunk = self._proxy.schunk.get_chunk(nchunk) - self._proxy._enforce_cache_limit(item) + backend.fetch(item) + chunk = backend.schunk.get_chunk(nchunk) + backend._enforce_cache_limit(item) return chunk async def aget_chunk(self, nchunk: int) -> bytes: - if self._proxy is None: - method = getattr(self.src, "aget_chunk", None) + backend = self._prepare_read() + if not isinstance(backend, blosc2.Proxy): + method = getattr(backend, "aget_chunk", None) if method is None: raise NotImplementedError("the remote source does not provide asynchronous chunk reads") return await method(nchunk) item = self._chunk_slice(nchunk) - await self._proxy.afetch(item) - chunk = self._proxy.schunk.get_chunk(nchunk) - self._proxy._enforce_cache_limit(item) + await backend.afetch(item) + chunk = backend.schunk.get_chunk(nchunk) + backend._enforce_cache_limit(item) return chunk def _payload(self): diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index a1b5f63b9..a883550bf 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -641,6 +641,27 @@ def test_http_lazy_cache_rebuilt_when_remote_changes(tmp_path): assert np.array_equal(lazy[3:5, 100:120], second[3:5, 100:120]) +def test_http_remote_proxy_checks_identity_without_refetching_cached_data(tmp_path): + pytest.importorskip("aiohttp") + path = tmp_path / "www" + path.mkdir() + data = np.arange(40_000, dtype="i4").reshape(200, 200) + blosc2.asarray(data, chunks=(50, 200), blocks=(10, 100), urlpath=path / "stable.b2nd") + + with _ranged_server(path) as (urlbase, requests): + remote = blosc2.RemoteProxy( + f"{urlbase}/stable.b2nd", + cache_policy=blosc2.CachePolicy.MEMORY, + ) + np.testing.assert_array_equal(remote[3:5, 100:120], data[3:5, 100:120]) + + requests.clear() + remote.traffic.reset() + np.testing.assert_array_equal(remote[3:5, 100:120], data[3:5, 100:120]) + assert requests == [] + assert remote.traffic.requests == 0 + + def test_http_stamp_prefers_etag_and_falls_back_to_modified_size(): stamp = blosc2.proxy_source._http_stamp( "url", @@ -682,6 +703,14 @@ def do_GET(self): self.wfile.write(part) return None + def do_HEAD(self): + body = (root / self.path.lstrip("/")).read_bytes() + self.send_response(200) + self.send_header("Accept-Ranges", "bytes") + self.send_header("Content-Length", str(len(body))) + self.send_header("ETag", hashlib.sha256(body).hexdigest()) + self.end_headers() + handler = functools.partial(Ranged, directory=str(root)) server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) server.requests = [] diff --git a/tests/test_remote_proxy.py b/tests/test_remote_proxy.py index e01294de6..eb57d269f 100644 --- a/tests/test_remote_proxy.py +++ b/tests/test_remote_proxy.py @@ -7,6 +7,7 @@ from __future__ import annotations +import asyncio import hashlib import fsspec @@ -215,6 +216,37 @@ def test_reference_rejects_changed_source_geometry(tmp_path): blosc2.open(path, mode="r") +def test_open_reference_rejects_geometry_changed_before_read(tmp_path): + url, _ = _remote_array("changed-after-open.b2nd", nchunks=1, chunk_size=100) + path = tmp_path / "changed-after-open-reference.b2nd" + blosc2.RemoteProxy(url).save(path) + restored = blosc2.open(path, mode="r") + + replacement = blosc2.arange(200, dtype=np.uint8, chunks=(100,), blocks=(100,)) + fsspec.filesystem("memory").pipe_file("changed-after-open.b2nd", replacement.to_cframe()) + + with pytest.raises(ValueError, match="geometry no longer matches"): + restored[:] + + +@pytest.mark.parametrize("policy", [blosc2.CachePolicy.MEMORY, blosc2.CachePolicy.DISK]) +def test_runtime_cache_is_invalidated_after_same_geometry_replacement(tmp_path, policy): + url, data = _remote_array(f"same-geometry-{policy.value}.b2nd", nchunks=1, chunk_size=100) + kwargs = ( + {"cache_path": tmp_path / f"{policy.value}-cache.b2nd"} if policy is blosc2.CachePolicy.DISK else {} + ) + proxy = blosc2.RemoteProxy(url, cache_policy=policy, **kwargs) + traffic = proxy.traffic + np.testing.assert_array_equal(proxy[:], data) + + replacement = np.arange(100, dtype=np.uint8) + array = blosc2.asarray(replacement, chunks=(100,), blocks=(100,)) + fsspec.filesystem("memory").pipe_file(f"same-geometry-{policy.value}.b2nd", array.to_cframe()) + + np.testing.assert_array_equal(proxy[:], replacement) + assert proxy.traffic is traffic + + def test_reference_rejects_runtime_cache_policy_in_payload(): url, _ = _remote_array("bad-persisted-policy.b2nd", nchunks=1, chunk_size=100) carrier = blosc2.ndarray_from_cframe(blosc2.RemoteProxy(url).to_cframe()) @@ -225,6 +257,23 @@ def test_reference_rejects_runtime_cache_policy_in_payload(): decode_b2object_payload(payload, carrier=carrier) +@pytest.mark.parametrize( + ("field", "value", "error"), + [ + ("kind", "unknown", "unsupported RemoteProxy source kind"), + ("version", 2, "unsupported RemoteProxy source descriptor"), + ], +) +def test_reference_rejects_unknown_source_descriptor(field, value, error): + url, _ = _remote_array(f"bad-source-{field}.b2nd", nchunks=1, chunk_size=100) + carrier = blosc2.ndarray_from_cframe(blosc2.RemoteProxy(url).to_cframe()) + payload = dict(carrier.schunk.vlmeta["b2o"]) + payload["source"] = dict(payload["source"], **{field: value}) + + with pytest.raises(ValueError, match=error): + decode_b2object_payload(payload, carrier=carrier) + + @pytest.mark.parametrize("field", ["auth_token", "storage_options"]) def test_reference_rejects_secret_or_runtime_source_fields(field): url, _ = _remote_array(f"bad-source-{field}.b2nd", nchunks=1, chunk_size=100) @@ -301,6 +350,46 @@ def fake_fetch_data(path, urlbase, params, auth_token=None, as_blosc2=False, tra assert calls == [{"slice_": "2:5"}] +@pytest.mark.parametrize("policy", [blosc2.CachePolicy.MEMORY, blosc2.CachePolicy.DISK]) +def test_caterva2_runtime_caches_reuse_chunks(monkeypatch, tmp_path, policy): + data = np.arange(10, dtype=np.int32) + local = blosc2.asarray(data, chunks=(5,), blocks=(5,)) + compressed = [local.schunk.get_chunk(i) for i in range(2)] + calls = [] + + def fake_info(path, urlbase, params=None, headers=None, model=None, auth_token=None, traffic=None): + return { + "shape": [10], + "chunks": [5], + "blocks": [5], + "dtype": data.dtype.str, + "mtime": 1, + "accept_ranges": "none", + "schunk": { + "cparams": dict(blosc2.cparams_dflts), + "cbytes": sum(map(len, compressed)), + "vlmeta": {}, + }, + } + + def fake_get_chunk(self, nchunk): + calls.append(nchunk) + return compressed[nchunk] + + monkeypatch.setattr(blosc2_c2array, "info", fake_info) + monkeypatch.setattr(blosc2.C2Array, "get_chunk", fake_get_chunk) + kwargs = {"cache_path": tmp_path / "caterva2-cache.b2nd"} if policy is blosc2.CachePolicy.DISK else {} + remote = blosc2.RemoteProxy( + blosc2.URLPath("@public/cache.b2nd", urlbase="https://example.org/c2"), + cache_policy=policy, + **kwargs, + ) + + np.testing.assert_array_equal(remote[:5], data[:5]) + np.testing.assert_array_equal(remote[:5], data[:5]) + assert calls == [0] + + def test_remote_proxy_is_a_persistable_lazyexpr_operand(): url, data = _remote_array("operand.b2nd", nchunks=1, chunk_size=100) remote = blosc2.RemoteProxy(url) @@ -324,6 +413,73 @@ def test_objectarray_msgpack_supports_remote_proxy(): np.testing.assert_array_equal(restored[:], data) +def test_batcharray_msgpack_supports_remote_proxy(): + url, data = _remote_array("batcharray-remote-proxy.b2nd", nchunks=1, chunk_size=100) + batches = blosc2.BatchArray() + batches.append([blosc2.RemoteProxy(url)]) + + restored = batches[0][0] + assert isinstance(restored, blosc2.RemoteProxy) + assert restored.cache_policy is blosc2.CachePolicy.NONE + np.testing.assert_array_equal(restored[:], data) + + +@pytest.mark.parametrize("policy", list(blosc2.CachePolicy)) +def test_get_chunk_for_each_policy(tmp_path, policy): + url, data = _remote_array(f"get-chunk-{policy.value}.b2nd", nchunks=2, chunk_size=100) + kwargs = {"cache_path": tmp_path / "get-chunk-cache.b2nd"} if policy is blosc2.CachePolicy.DISK else {} + proxy = blosc2.RemoteProxy(url, cache_policy=policy, **kwargs) + + chunk = proxy.get_chunk(1) + np.testing.assert_array_equal(np.frombuffer(blosc2.decompress2(chunk), dtype=np.uint8), data[100:]) + + +@pytest.mark.parametrize("policy", list(blosc2.CachePolicy)) +def test_aget_chunk_for_each_policy(tmp_path, policy): + url, data = _remote_array(f"aget-chunk-{policy.value}.b2nd", nchunks=2, chunk_size=100) + kwargs = {"cache_path": tmp_path / "aget-chunk-cache.b2nd"} if policy is blosc2.CachePolicy.DISK else {} + proxy = blosc2.RemoteProxy(url, cache_policy=policy, **kwargs) + + chunk = asyncio.run(proxy.aget_chunk(1)) + np.testing.assert_array_equal(np.frombuffer(blosc2.decompress2(chunk), dtype=np.uint8), data[100:]) + + +def test_disk_cache_survives_reopen_without_remote_data_traffic(tmp_path): + url, data = _remote_array("disk-reuse.b2nd", nchunks=2, chunk_size=100_000) + cache_path = tmp_path / "disk-reuse-cache.b2nd" + first = blosc2.RemoteProxy(url, cache_policy=blosc2.CachePolicy.DISK, cache_path=cache_path) + np.testing.assert_array_equal(first[:100_000], data[:100_000]) + + reopened = blosc2.RemoteProxy(url, cache_policy=blosc2.CachePolicy.DISK, cache_path=cache_path) + reopened.traffic.reset() + np.testing.assert_array_equal(reopened[:100_000], data[:100_000]) + assert reopened.traffic.requests == 0 + + +def test_save_after_disk_cache_use_remains_reference_only(tmp_path): + url, data = _remote_array("save-after-disk.b2nd", nchunks=2, chunk_size=100_000) + proxy = blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=tmp_path / "runtime-cache.b2nd", + ) + np.testing.assert_array_equal(proxy[:100_000], data[:100_000]) + + reference_path = tmp_path / "saved-reference.b2nd" + proxy.save(reference_path) + carrier = blosc2.ndarray_from_cframe(reference_path.read_bytes()) + assert carrier.schunk.vlmeta["b2o"]["cache_policy"] == "none" + assert "proxy-source" not in carrier.schunk.meta + + +def test_reference_size_is_independent_of_remote_payload(tmp_path): + url, _ = _remote_array("metadata-sized.b2nd", nchunks=20, chunk_size=100_000) + path = tmp_path / "metadata-sized-reference.b2nd" + blosc2.RemoteProxy(url).save(path) + + assert path.stat().st_size < 10_000 + + @pytest.mark.parametrize( "url", [ From 7e7753c0dd0a4abca79e1c06bab6c6f80c161244 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 5 Sep 2026 09:03:04 +0200 Subject: [PATCH 13/89] Make remote proxies self-caching --- doc/guides/remote_arrays.md | 27 +- doc/reference/msgpack_serialization.rst | 12 +- doc/reference/remoteproxy.rst | 46 ++- plans/remote-proxy-v2.md | 425 ++++++++++++++++++++++++ plans/remote-proxy.md | 182 ++++++++-- src/blosc2/__init__.py | 1 - src/blosc2/lazyexpr.py | 2 +- src/blosc2/proxy.py | 4 +- src/blosc2/remote_proxy.py | 163 +++++---- src/blosc2/schunk.py | 28 +- tests/test_fsspec.py | 3 +- tests/test_remote_proxy.py | 177 +++++----- 12 files changed, 830 insertions(+), 240 deletions(-) create mode 100644 plans/remote-proxy-v2.md diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index a01a0abcb..fa3ea3c92 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -83,10 +83,10 @@ In both cases, the cache is an ordinary `.b2nd` array that starts small and grow Authenticated Caterva2 caches must be private to one user. Reopen them under an equivalent authenticated {func}`blosc2.c2context`; do not share a cache directory between users. -### Persist a reference instead of a cache +### Persist a self-caching remote proxy -Use {ref}`RemoteProxy` when the `.b2nd` file itself should remain a small, -immutable reference to the remote array rather than become its cache: +Use {ref}`RemoteProxy` when the `.b2nd` file should carry a portable remote +descriptor and, optionally, its own bounded persistent cache: ```python remote = blosc2.RemoteProxy( @@ -96,15 +96,13 @@ remote = blosc2.RemoteProxy( remote.save("big-reference.b2nd") ``` -The saved object contains source and geometry metadata but no fetched chunks or -credentials. It reopens with `CachePolicy.NONE`, so repeated reads contact the -source again and never mutate the reference file. This is also true when a -`RemoteProxy` using `MEMORY` or `DISK` caching is saved: those runtime caches -are not part of the portable reference. +The saved object contains source and geometry metadata but no credentials. With +`CachePolicy.NONE`, repeated reads contact the source and do not mutate the +carrier. -`RemoteProxy` also supports bounded runtime caching. Memory caches retain at -most 256 MiB of compressed payload by default; disk caches are unlimited unless -an explicit bound is supplied: +`RemoteProxy` also supports bounded disk caching. The proxy carrier itself is +the cache, with a finite 256 MiB compressed-payload bound by default or an +explicit bound: ```python remote = blosc2.RemoteProxy( @@ -115,9 +113,10 @@ remote = blosc2.RemoteProxy( ) ``` -Opening the saved reference itself returns `CachePolicy.NONE`. Runtime caching -must be selected again explicitly from its `urlpath`; it is never inferred from -the reference carrier. +Saving and serializing preserve valid warm chunks by default. Pass +`include_cache=False` to `save()` or `to_cframe()` to export a cold copy without +mutating the warm carrier. A saved disk proxy opened with `mode="a"` retains +misses in itself; `mode="r"` can use warm chunks but does not retain misses. The bound is applied after each operation. It does not limit the temporary working set or a NumPy result requested by the caller. diff --git a/doc/reference/msgpack_serialization.rst b/doc/reference/msgpack_serialization.rst index 6755b4501..ee23905e9 100644 --- a/doc/reference/msgpack_serialization.rst +++ b/doc/reference/msgpack_serialization.rst @@ -64,12 +64,12 @@ Authentication data is intentionally not serialized. ``RemoteProxy`` --------------- -Remote proxies use a metadata-only carrier containing a versioned Caterva2 or -fsspec source descriptor. Saving a live proxy is reference-only: its carrier -reopens with :attr:`blosc2.CachePolicy.NONE`, regardless of whether the live -proxy used ``MEMORY`` or ``DISK`` caching. The policy is encoded on disk as the -stable string ``"none"``; runtime cache contents, local cache paths, fetched -data, and credentials are intentionally not serialized. +Remote proxies use a B2ND carrier containing a versioned Caterva2 or fsspec +source descriptor, a ``"none"`` or ``"disk"`` cache policy, and the finite +disk-cache limit. A disk-caching carrier may also contain fetched compressed +chunks and its cache bookkeeping. Saving includes valid warm chunks by default; +``include_cache=False`` produces a cold carrier. Local paths, live filesystem +objects, and credentials are intentionally not serialized. Persistent local operands ------------------------- diff --git a/doc/reference/remoteproxy.rst b/doc/reference/remoteproxy.rst index 957d51c6f..cfef56412 100644 --- a/doc/reference/remoteproxy.rst +++ b/doc/reference/remoteproxy.rst @@ -3,14 +3,13 @@ RemoteProxy =========== -``RemoteProxy`` is a persistable reference to one remote B2ND array. It accepts -an fsspec URL or a Caterva2 :ref:`URLPath` and separates the portable reference -from any runtime data cache. +``RemoteProxy`` is a persistable proxy for one remote B2ND array. It accepts an +fsspec URL or a Caterva2 :ref:`URLPath`. With disk caching enabled, its B2ND +carrier is both the portable descriptor and the bounded compressed-data cache. The default policy is :attr:`blosc2.CachePolicy.NONE`: each operation reads the -remote data it needs and no fetched data is retained afterwards. Saving the -object writes only its source descriptor and array geometry, never fetched data -or credentials. +remote data it needs and no fetched data is retained afterwards. Saving such an +object writes only its source descriptor and array geometry. .. code-block:: python @@ -35,12 +34,11 @@ URL: References are floating: before each data operation, ``RemoteProxy`` checks the source identity and verifies that shape, dtype, chunks, and blocks still match the captured geometry. A replacement with different geometry is rejected; -runtime memory or disk cache data is discarded when the source identity moves. +cached disk data is invalidated when the source identity moves. -Runtime caching is available through :attr:`blosc2.CachePolicy.MEMORY` and -:attr:`blosc2.CachePolicy.DISK`. Memory caches retain at most 256 MiB of -compressed payload by default. Disk caches are unlimited by default, but both -can take an explicit ``max_cache_bytes`` bound. The bound is enforced after an +Persistent caching is available through :attr:`blosc2.CachePolicy.DISK`. +Disk caches have a finite 256 MiB compressed-payload bound by default and can +take an explicit ``max_cache_bytes`` bound. The bound is enforced after an operation completes and therefore does not limit its temporary working set or returned NumPy array. @@ -53,27 +51,23 @@ returned NumPy array. max_cache_bytes=2 * 2**30, ) -Regardless of its runtime policy, :meth:`RemoteProxy.save -` and :meth:`RemoteProxy.to_cframe -` produce a reference-only object that reopens -with :attr:`blosc2.CachePolicy.NONE`. Local cache paths and authentication data -are not serialized. +By default, :meth:`RemoteProxy.save ` and +:meth:`RemoteProxy.to_cframe ` include valid warm +chunks. Pass ``include_cache=False`` for a cold carrier without changing the +warm original. The cache policy and limit remain in both forms; local paths and +authentication data are not serialized. Authentication supplied to a live Caterva2 source is deliberately omitted from -the carrier. A receiving server resolves private sources with credentials from -its own administrator-controlled destination mapping; client credentials never -travel with the reference. +the carrier. Caterva2's first server implementation resolves public HTTPS +sources only; client credentials never travel with the proxy. -To cache again after reopening a reference, opt into a runtime policy when -constructing a new proxy from its source: +Open a disk-caching carrier in append mode to let misses populate that same +file. Read-only mode can use warm chunks but does not retain misses: .. code-block:: python - reference = blosc2.open("dataset-reference.b2nd") - cached = blosc2.RemoteProxy( - reference.urlpath, - cache_policy=blosc2.CachePolicy.MEMORY, - ) + cached = blosc2.open("dataset-cache.b2nd", mode="a") + cached[100:200] .. warning:: diff --git a/plans/remote-proxy-v2.md b/plans/remote-proxy-v2.md new file mode 100644 index 000000000..5fe1920bc --- /dev/null +++ b/plans/remote-proxy-v2.md @@ -0,0 +1,425 @@ +# Plan: Self-Caching RemoteProxy + +## Status + +Implemented and validated in Python-Blosc2 and Caterva2. The product and +persistence contracts are settled; the phase checklist and test matrix below +describe the delivered version-1 behavior. + +## Purpose + +Allow Caterva2 to host a small `.b2nd` proxy for a public remote B2ND array. +When Caterva2 reads missing regions, the proxy may retain the fetched compressed +chunks inside that same `.b2nd` file, up to a configured storage limit. + +The proxy file is therefore a mutable persistent cache, not an immutable remote +reference plus a separate server cache. + +## Scope + +The first version supports: + +- direct RemoteProxy `.b2nd` files +- one public, contiguous B2ND source available over HTTPS +- public S3 objects through their HTTPS object URLs +- operation with embedded caching disabled or enabled +- a bound on compressed chunk payload retained in each proxy file +- reuse of cached chunks after process and server restarts +- warm-carrier and cache-free export/download forms +- invalidation when the remote object identity changes +- Caterva2's existing `api/info`, `api/fetch`, `api/chunk`, and physical + download endpoints + +The first version does not support: + +- credentials, signed URLs, cookies, custom headers, or private sources +- native `s3://` resolution in Caterva2 +- redirects or arbitrary fsspec protocol chains +- remote references embedded in expressions or other object graphs +- cumulative network-byte or request-count budgets +- a new Caterva2 HTTP endpoint for proxy creation or cache management +- a separate RemoteProxy-specific server quota + +Ordinary connection timeouts, remote-read concurrency, destination policy, and +structural validation remain in scope. + +## Core Decisions + +### The carrier is the cache + +The uploaded `.b2nd` contains both: + +- stable identity: versioned source descriptor and array geometry +- mutable state: fetched compressed chunks, source stamp, fetched bitmap, + compressed-size accounting, and LRU bookkeeping + +Reads may change the file's contents, size, modification time, and cache +metadata. The source descriptor and geometry must not change as a consequence +of reading or eviction. + +### Cache data is disposable + +Cached chunks are an optimization, never the authority for array identity. +Deleting every cached chunk leaves a valid proxy. If the source identity changes, +the cache is emptied logically before any old chunk can be served. + +### Reuse the existing Proxy cache engine + +`blosc2.Proxy` already implements the required persistent-cache mechanisms: + +- UNINIT chunks for cache misses +- chunk/block fetched bitmaps +- remote source stamps +- compressed-byte accounting +- whole-chunk LRU eviction +- reopening and adopting a valid cache + +RemoteProxy should reuse these mechanics with its own carrier passed as the +Proxy cache. It should not implement a second cache engine. + +### RemoteProxy has no memory-cache policy + +RemoteProxy supports only `CachePolicy.NONE` and `CachePolicy.DISK`. A retained +RemoteProxy cache belongs in its carrier, so a `MEMORY` policy would violate the +self-caching artifact model and could multiply memory use across Caterva2 +workers and hosted proxies. + +Remove `CachePolicy.MEMORY` while the API is still unreleased. Existing +`blosc2.open(remote, lazy=True)` process-local memory caching continues to +return the established generic `Proxy`; users who explicitly need an ephemeral +memory cache can also construct that type directly. Caterva2 therefore never +needs to translate, reject, or impose a quota on a persisted memory-cache +policy because no valid RemoteProxy carrier can contain one. + +### The proxy specifies caching; the customer quota bounds storage + +The persisted proxy specifies whether it retains misses and its maximum cached +compressed payload. This behavior travels with the proxy instead of being +selected independently by each Caterva2 server. + +Each customer owns a virtual Caterva2 server whose users share one state +directory and one server-wide quota. Embedded proxy-cache growth is charged to +that existing customer quota just like other stored data. No user-level +attribution or separate RemoteProxy quota is needed. + +When the customer quota cannot accommodate another cached chunk, Caterva2 still +serves the remotely fetched result but skips retaining that chunk. A proxy with +cache policy `NONE` never retains misses. Already embedded valid chunks remain +readable without an outbound fetch. + +### Public HTTPS remains the network boundary + +Caterva2 keeps the security work already implemented: + +- remote resolution disabled by default +- exact administrator host allowlists +- HTTPS only +- public-address validation and DNS pinning +- redirects disabled +- URL credentials, queries, and fragments rejected +- descriptor inspection before generic Blosc2 object decoding +- carrier/source geometry validation + +These controls remain necessary because cache misses still cause outbound +requests selected by uploaded data. + +## Carrier Format + +The RemoteProxy format has not been released, so define version 1 directly as +the self-caching format. No upgrade path or compatibility contract is needed +for the earlier development-only reference format: + +```python +{ + "kind": "remote_proxy", + "version": 1, + "source": { + "kind": "fsspec", + "version": 1, + "urlpath": "https://datasets.example.org/array.b2nd", + }, + "cache_policy": "disk", + "max_cache_bytes": 268435456, +} +``` + +The ordinary B2ND chunk slots form the cache. Proxy-owned variable metadata +records the source stamp, fetched bitmap, cached sizes, and any persisted index +needed by the remote reader. + +Do not serialize: + +- server cache limits or server policy +- local paths +- credentials or request configuration +- locks, sessions, or live filesystem objects + +Do not add `proxy-source` metadata merely to trigger the legacy open path. The +`remote_proxy` B2 object kind remains the authoritative discriminator; its +decoder can construct a `Proxy` over the carrier after the source has been +resolved safely. + +## Python-Blosc2 Behavior + +### Creation + +Creating and saving a `NONE` RemoteProxy produces a metadata-sized carrier whose +data chunks are UNINIT: + +```python +proxy = blosc2.RemoteProxy("https://datasets.example.org/array.b2nd") +proxy.save("array-proxy.b2nd") +``` + +To create a self-caching carrier directly, select `DISK`, its path, and an +optional finite limit: + +```python +proxy = blosc2.RemoteProxy( + "https://datasets.example.org/array.b2nd", + cache_policy=blosc2.CachePolicy.DISK, + cache_path="array-proxy.b2nd", + max_cache_bytes=256 * 2**20, +) +``` + +Opening that carrier in append mode uses the carrier itself as the persistent +cache; it does not require a second `cache_path`. Read-only mode can consume +warm chunks but does not retain misses. + +### Reads + +For each requested chunk or block: + +1. Validate or refresh the remote source identity. +2. Serve a valid fetched entry from the carrier when available. +3. Fetch missing compressed bytes from the authorized source. +4. Return the requested logical result. +5. If writes are enabled, retain fetched chunks and enforce the configured cap. + +Eviction replaces complete least-recently-used chunks with UNINIT and updates +the fetched bitmap atomically enough that an interrupted write cannot cause an +unfetched chunk to be trusted. + +### Saving, CFrames, and downloads + +By default, `save()` and `to_cframe()` serialize the current physical carrier, +including valid warm cache data. Downloading the proxy `.b2nd` from Caterva2 +likewise returns the physical self-caching carrier with its warm chunks. A proxy +with no retained data remains metadata-sized. Credentials and runtime server +policy are never included. + +This is distinct from `api/fetch`: fetching returns the requested logical array +or slice, not the physical proxy/cache file. A concurrent physical download must +take the carrier lock or copy under that lock so the downloaded B2ND is +internally consistent. + +An explicit `include_cache=False` option produces a cold carrier without +mutating the warm source proxy: + +```python +proxy.save("cold-proxy.b2nd", include_cache=False) +frame = proxy.to_cframe(include_cache=False) +``` + +Caterva2 exposes the same choice as an optional `include_cache=false` parameter +on its existing physical download operation. Cold export preserves descriptor, +geometry, compression parameters, cache policy, and cache limit, while replacing +cached chunks with UNINIT and clearing fetched bitmaps, cached-size/LRU state, +stored remote indexes, and source stamps. It does not contact the remote source. + +### Source changes + +ETag or another stable source token is stored with the cache. Before serving a +cached entry after reopening, compare the current source token with the stored +token. A mismatch clears the fetched bitmap and accounting before reading data. +Geometry mismatch remains a hard error rather than an automatic rewrite of the +carrier's identity. + +If an HTTPS source supplies no stable validator, persistent reuse across +independent opens is unsafe. Such a source may be read without retention, or its +cache must be treated as empty on every new open. + +## Caterva2 Behavior + +### Existing API surface + +No new HTTP endpoint is needed: + +- upload stores the proxy through the existing upload path +- `api/info` reads local geometry and descriptor metadata without an outbound + request +- `api/fetch` resolves the source under policy and serves slices or indices +- `api/chunk` can serve a compressed chunk while applying the same cache and + quota rules +- physical download accepts `include_cache=false` to export a cold copy + +The physical carrier must not be mistaken for a complete materialized array +frame when serving a whole logical-array fetch. + +### Configuration and customer quota + +The existing remote-source security configuration remains: + +```toml +[server.remote_proxy] +enabled = true +allowed_hosts = ["datasets.example.org"] +timeout = 30 +max_concurrency = 8 +``` + +No Caterva2-specific cache limit is added. `max_cache_bytes` belongs to the +proxy payload and measures retained compressed chunk payload for that carrier; +small fixed metadata/index overhead is excluded. A `DISK` proxy must specify a +positive finite limit. + +Caterva2's existing server-wide `quota` is the aggregate storage bound for one +customer's virtual server. Automatic proxy fills must join the same disk-usage +checking and accounting path currently used by uploads and explicit chunk +writes. The check must cover concurrent fills rather than merely noticing the +larger file during a later state-directory scan. + +### Opening and resolution + +Caterva2 continues to inspect the carrier before `blosc2.open()` can resolve an +untrusted source. After HTTPS authorization and geometry validation, it opens +the carrier in the appropriate mode and constructs the existing Proxy cache +engine over the secure `FsspecNDSource`. + +- `NONE`: misses are not retained +- `DISK`: the carrier is opened append/write; misses populate it and its own LRU + eviction enforces `max_cache_bytes` +- customer quota exhausted: the result is served, but a miss is not retained + +### Concurrency + +Because reads may now write, Caterva2 must synchronize access per carrier. The +safe initial rule is one active cache-mutating operation per carrier, including +across server workers. Read-only metadata inspection need not take the write +lock. Locking must cover source-stamp validation, fetched-bitmap changes, chunk +writes, eviction, and bookkeeping persistence. + +The lock must follow Caterva2's existing dataset mutation/locking conventions +where possible. Process-local Python locks alone are insufficient when multiple +workers can open the same file. + +### HTTP metadata and ETags + +A cache fill changes the physical file mtime but not the logical remote-array +identity. API validators must not accidentally present cache churn as a user +dataset edit. The implementation must distinguish: + +- physical carrier identity, relevant when downloading or backing up the proxy +- logical array identity, derived from descriptor plus current source stamp + +This can initially be conservative by changing the API ETag after cache writes, +but clients must never receive stale logical data. A stable logical ETag is a +later optimization. `api/info` exposes the portable `b2o` descriptor but not +binary fetched bitmaps, source stamps, or other cache-engine bookkeeping. + +## Changes To The Current Caterva2 Slice + +Keep: + +- policy configuration and default deny +- raw pre-resolution carrier inspection +- HTTPS validation, public DNS pinning, and disabled redirects +- geometry/rank/chunk validation +- existing `api/info` and `api/fetch` dispatch +- rejection of embedded remote references + +Replace or revise: + +- replace operation-scoped `ServerRemoteProxy` assembly with a Proxy backed by + the uploaded carrier +- remove the requirement that persisted policy is `none` +- open data requests with controlled write access when caching is enabled +- replace byte-for-byte immutability assertions with bounded-cache assertions +- document physical mutation and source-stamp invalidation + +## Implementation Phases + +### Phase 0: Format and cache prototype + +- [x] Finalize the version-1 self-caching payload and metadata invariants. +- [x] Prototype `Proxy(src, _cache=carrier, _max_cache_bytes=...)` over a RemoteProxy + carrier. +- [x] Confirm eviction reclaims physical storage with bounded overhead. + +### Phase 1: Python self-caching carrier + +- [x] Make the decoded RemoteProxy retain its carrier. +- [x] Reuse the carrier as the Proxy cache for persistent mode. +- [x] Preserve and serialize fetched bitmap, stamp, sizes, and LRU state. +- [x] Make `save()` and `to_cframe()` include valid warm chunks. +- [x] Add non-destructive `include_cache=False` cold exports. + +### Phase 2: Caterva2 cache integration + +- [x] Replace no-retention resolution with the carrier-backed Proxy. +- [x] Integrate automatic cache fills with the existing customer-server quota + checks and accounting; skip retention when no quota remains. +- [x] Add per-carrier cross-worker locking. +- [x] Keep metadata inspection local and source resolution default-deny. +- [x] Validate persisted cache policy and require a finite positive per-proxy limit. + +### Phase 3: Tests and documentation + +- [x] Update Python and Caterva2 API documentation. +- [x] Add cold-read, warm-read, restart, eviction, and source-change tests. +- [x] Add warm and cold physical download/export tests. +- [x] Add concurrent-fill and interrupted-write tests. +- [x] Verify default-deny and HTTPS security tests still pass. +- [x] Verify existing legacy Proxy caches still reopen in the full suites. + +## Test Matrix + +- Empty carrier is metadata-sized and reports complete local geometry. +- First read fetches and embeds the required compressed chunks. +- Repeated covered read causes no remote data traffic. +- Reopening the same file reuses its embedded cache. +- A `NONE` proxy leaves carrier bytes and mtime unchanged. +- Bounded caching evicts whole LRU chunks and stays within payload limit plus + documented fixed overhead. +- `save()`/`to_cframe()` preserve valid warm chunks. +- `include_cache=False` exports a metadata-sized cold carrier without mutating + the warm proxy or contacting its source. +- Physical download preserves valid warm chunks by default and supports a cold + copy, while `api/fetch` returns logical array data. +- Changed source stamp invalidates all old cached entries before use. +- Changed source geometry raises without rewriting carrier identity. +- Missing source validator cannot silently reuse cache across opens. +- Concurrent reads cannot corrupt chunks, bitmaps, or accounting. +- Cache fills count against the customer virtual server's existing quota; when + it is exhausted, reads succeed without retaining additional chunks. +- Caterva2 default deny, destination allowlist, DNS pinning, redirect rejection, + and credential rejection remain effective. +- Existing info/fetch clients need no endpoint changes. + +## Acceptance Criteria + +1. A RemoteProxy `.b2nd` is both the portable descriptor and its persistent + bounded cache. +2. Caterva2 can host it through existing info/fetch APIs. +3. Public HTTPS cache misses are resolved only through server policy. +4. Warm chunks survive restart and avoid remote data traffic. +5. The proxy's compressed-payload cap is enforced by whole-chunk LRU + eviction. +6. Cache mutation never changes source descriptor or geometry. +7. Replaced sources cannot cause stale cached data to be served. +8. Concurrent requests cannot corrupt the carrier. +9. No credentials or server-specific runtime configuration are serialized. +10. Existing persistent Proxy caches retain documented compatibility. +11. RemoteProxy exposes no memory-cache policy; existing generic Proxy memory + behavior remains unchanged. +12. Users can download or export a cold proxy without altering its warm carrier. + +## Settled Product Decisions + +No RemoteProxy format has shipped, so payload version 1 is the self-caching +format and has no upgrade path. RemoteProxy accepts only `NONE` and `DISK`; +`MEMORY` is removed before release. Ordinary `save()`, `to_cframe()`, and +physical proxy downloads preserve valid warm cache data by default. Explicit +`include_cache=False` creates or downloads a cold copy without modifying the +hosted proxy. Logical `api/fetch` operations continue to return array data. diff --git a/plans/remote-proxy.md b/plans/remote-proxy.md index ea5b7631b..a75b7f8ad 100644 --- a/plans/remote-proxy.md +++ b/plans/remote-proxy.md @@ -1,5 +1,9 @@ # Plan: Persistable `RemoteProxy` +> **Superseded:** The self-caching carrier design in +> [`remote-proxy-v2.md`](remote-proxy-v2.md) is the authoritative implementation +> plan. This document records the earlier immutable-reference design. + ## Motivation Python-Blosc2 can already access remote B2ND arrays lazily: @@ -86,20 +90,105 @@ The following remain future work or deliberate follow-ups: the current explicit opt-in. - Cache-oriented `fetch()`/`afetch()` methods are not exposed on `RemoteProxy` under `NONE`; a separate materialization API can be designed later. -- Caterva2 credential selection, S3 support, cumulative fetched-byte budgets, - reference-chain resolution/cycle handling, and authenticated tenant-scoped - sessions remain to be implemented. Remote references embedded inside stored +- Caterva2 operation-wide request budgets and reference-chain resolution/cycle + handling remain to be implemented. Remote references embedded inside stored expressions are rejected until they can use the same secure resolver. - Pinned reference semantics, broader fsspec/server protocol allowlists, and any `C2Array.save(as_remote_proxy=True)` convenience are future decisions. +## Open Work Queue And Decisions + +The remaining server work should be tackled in the following order. The server +scope stays limited to public, credential-free HTTPS while resource controls +and recursive resolution are completed. + +### 1. Operation-wide resource budgets + +**Recommended next implementation.** The current Caterva2 policy bounds source +geometry, chunk count, concurrency, and each HTTP request's timeout. It does not +yet bound the aggregate work caused by one client operation. + +Add one request-scoped budget object shared by metadata discovery and all range +reads. It should account for: + +- total upstream response bytes, including metadata and unsuccessful responses +- range-request count and retry count +- elapsed wall-clock time, with the remaining deadline passed to each request +- live upstream tasks, with cancellation when the downstream request ends + +Exhaustion must abort the operation with a deterministic, non-sensitive error. +The budget should be enforced below `FsspecNDSource`, at the authorized +filesystem/transport boundary, so every request is charged independently of +the slice assembly strategy. Initial limits should be server-wide configuration +defaults; per-user limits can be added independently of remote credentials. + +Open implementation question: fsspec does not expose all transport accounting +through a stable public hook. Prefer a narrow Caterva2 HTTP filesystem wrapper +that charges opened responses and retries rather than adding server policy to +Python-Blosc2's general-purpose `Traffic` diagnostics. + +### 2. Public-only source boundary + +**Decision:** Caterva2 RemoteProxy resolution supports only remote arrays that +are publicly readable without credentials. The server does not select, store, +or forward credentials and does not accept credential profile names, provider +settings, endpoint overrides, headers, cookies, signed query parameters, or +arbitrary fsspec storage options from a carrier. + +The server protocol is HTTPS. Public S3 objects are supported through their +ordinary public HTTPS URLs. Native `s3://` resolution is deliberately out of +scope, even for anonymous buckets, so Caterva2 does not need provider-specific +filesystem configuration, endpoints, or bucket policy logic. + +Exact administrator host allowlists, public-address validation, DNS pinning, +disabled redirects, and rejection of descriptor-provided options remain +mandatory. Private sources, native object-store protocols, authenticated +upstream sessions, and tenant-scoped remote credentials are non-goals. Adding +any of them later requires a separate security design and must not weaken the +public HTTPS path by default. + +### 3. Nested references and stored expressions + +Direct `RemoteProxy` carriers are supported. Remote references inside persisted +`LazyExpr`/`LazyUDF` objects are intentionally rejected before generic Blosc2 +decoding, because decoding currently resolves operands without a Caterva2 +policy context. + +Supporting them requires a resolver/context injection point in Python-Blosc2, +then a Caterva2 graph traversal that carries one operation budget, records +normalized source identities, rejects cycles/self-reference, and enforces a +small maximum depth and hop count. Arbitrary nesting in other container types +must remain denied or unresolved until it goes through the same traversal. + +Open design question: make authorized resolution an explicit argument/context +to the B2 object decoder rather than installing a process-global resolver. A +process-global callback cannot safely represent concurrent requests or policy +contexts. + +### 4. Compatibility and release floor + +Caterva2 currently refuses to enable remote resolution when its installed +Python-Blosc2 lacks `RemoteProxy`. Before releasing the Caterva2 feature, raise +its declared minimum Python-Blosc2 version to the first release containing the +carrier format and secure filesystem-injection hooks. Keep the runtime check for +clear diagnostics in mixed deployments. + +### 5. End-to-end transport validation + +Add a controlled HTTPS integration fixture with real TLS, DNS resolution, and +range responses. It should exercise successful reads, byte/range budget +exhaustion, timeout/cancellation, changed ETag/geometry, and attempts to redirect +or resolve to a denied address. External public services should remain optional +network tests; the security assertions need a deterministic local harness whose +address classification is explicitly test-configurable. + ## Goals The first implementation should allow this workflow: ```python proxy = blosc2.RemoteProxy( - "s3://example/dataset.b2nd", + "https://datasets.example.org/dataset.b2nd", cache_policy=blosc2.CachePolicy.NONE, ) proxy.save("dataset-proxy.b2nd") @@ -358,6 +447,17 @@ Suggested descriptors: } ``` +```python +{ + "kind": "fsspec", + "version": 1, + "urlpath": "https://datasets.example.org/dataset.b2nd", +} +``` + +The Python client may still use a public S3 descriptor directly when its local +fsspec environment is configured for anonymous access: + ```python { "kind": "fsspec", @@ -366,6 +466,9 @@ Suggested descriptors: } ``` +This source form is not accepted by Caterva2; an uploaded reference must use +the public object's HTTPS URL. + The source reference must contain only location and format information. It must not include headers, bearer tokens, passwords, signed query parameters, filesystem objects, or arbitrary fsspec keyword arguments. @@ -391,7 +494,7 @@ Example payload: "source": { "kind": "fsspec", "version": 1, - "urlpath": "s3://public-bucket/dataset.b2nd", + "urlpath": "https://datasets.example.org/dataset.b2nd", }, "cache_policy": "none", } @@ -541,20 +644,19 @@ Caterva2 should: Support for fsspec protocol chaining such as archive-over-network URLs should be out of scope initially because every layer expands the policy surface. -### Credentials +### Credential-free boundary - Never serialize client credentials in the B2 object. - Strip or reject user-info, sensitive query parameters, custom headers, cookies, tokens, and arbitrary `storage_options` at creation and upload. -- Configure server credentials out of band. -- Scope credentials to the smallest allowed host, bucket, and prefix. -- Select credentials from the validated destination, never from untrusted - descriptor-provided provider names. +- Do not configure or select upstream credentials in the RemoteProxy resolver. +- Do not add authentication headers, cookies, signed queries, endpoint + overrides, or provider-specific storage options. - Avoid reflecting secrets or sensitive internal response bodies in errors. -Public Caterva2 references should continue to work without a persisted auth -token. Private references require credentials available to the resolving -server; client credentials cannot make an uploaded proxy portable safely. +Only publicly readable HTTPS references are supported by Caterva2. Private +references are rejected; client credentials cannot make an uploaded proxy +portable safely. ### Resource limits @@ -581,14 +683,13 @@ original object. The server must enforce: - rejection of direct or indirect self-references - one cumulative resource budget across the entire reference chain -### Tenant isolation and observability +### Request isolation and observability -- Never share authenticated sessions or memory caches across security - principals unless the cache key includes the full authorization context. +- Do not share operation-scoped assembly state across client requests. - Log descriptor identity, resolved destination, bytes, request count, timing, and policy decision without logging secrets. - Expose actionable but non-sensitive failures for denied destinations, - unavailable credentials, stale geometry, and exhausted limits. + unavailable sources, stale geometry, and exhausted limits. ## Compatibility And Migration @@ -697,11 +798,18 @@ and enum values are the compatibility surface, not its physical module. ### Phase 5: Caterva2 support -- Add `remote_proxy` discovery and read dispatch to Caterva2. -- Implement default-deny protocol and destination configuration. -- Add credential selection outside the descriptor. -- Add SSRF, redirect, DNS, resource-limit, cycle, and tenant-isolation tests. -- Confirm the uploaded carrier is never mutated. +- **Completed baseline:** direct `remote_proxy` discovery and read dispatch, + default-deny configuration, exact HTTPS host allowlists, public-address + validation, DNS pinning, disabled redirects, structural limits, and immutable + carrier handling. +- **5a:** add operation-wide byte, request, retry, deadline, and cancellation + budgets with enforcement at the authorized transport boundary. +- **5b:** add request-scoped observability and optional per-client rate limits + without introducing upstream credentials or retained cross-request state. +- **5c:** add policy-aware nested-reference decoding, graph depth/hop limits, + cycle detection, and one cumulative budget across the graph. +- Extend the security suite alongside each subphase; unsupported protocols and + embedded references remain denied until their corresponding subphase lands. This phase may live in the Caterva2 repository, but the feature should not be presented as safe for arbitrary uploads until both sides are complete. @@ -710,9 +818,9 @@ presented as safe for arbitrary uploads until both sides are complete. - Add a `RemoteProxy` API page and include it in the reference toctree. - Document the three cache policies and their lifetime guarantees. -- Add examples for a public Caterva2 source and an allowed fsspec HTTPS/S3 - source. -- Document that private sources use server-side credentials. +- Add examples for a public Caterva2 source and an allowed fsspec HTTPS source; + add anonymous S3 if its server adapter is included. +- Document that private and credential-bearing sources are unsupported. - Add a Caterva2 administrator guide for the security policy and operational limits. @@ -773,8 +881,8 @@ presented as safe for arbitrary uploads until both sides are complete. - Cycles and excessive reference depth are rejected. - Byte, range, concurrency, timeout, decompression, and allocation limits are enforced. -- One tenant cannot observe or reuse another tenant's authenticated cache or - session. +- One request cannot observe or reuse another request's temporary assembly + state. ## Acceptance Criteria @@ -791,17 +899,20 @@ The feature is ready when all of the following hold: 256 MiB for memory and unlimited for disk. 5. `RemoteProxy.save()` never serializes fetched data or credentials. 6. Legacy C2Array and persistent cache-proxy files keep their behavior. -7. Caterva2 rejects remote proxies by default and resolves them only through an - administrator-controlled destination and credential policy. -8. Geometry changes, unavailable credentials, denied destinations, and - resource-limit failures produce clear errors. +7. Caterva2 rejects remote proxies by default and resolves only public, + credential-free HTTPS sources allowed by administrator destination policy. +8. Geometry changes, denied destinations, and resource-limit failures produce + clear errors. 9. API and administrator documentation explain both caching semantics and the outbound-request security boundary. ## Future Considerations -- Which fsspec schemes should a Caterva2 server implementation support? A - narrow starting set such as HTTPS and S3 is preferable to arbitrary plugins. +- Native `s3://` support is out of scope for Caterva2; public S3 objects use + their HTTPS URLs. Reconsidering native object-store protocols requires a + separate proposal. +- Reconsidering authenticated sources requires a separate security plan; it is + not an incremental configuration switch. - When should pinned references be added, and what exact mismatch exception should they raise? - Should a future `C2Array.save(as_remote_proxy=True)` convenience exist, or is @@ -822,5 +933,6 @@ The smallest end-to-end path was implemented as follows: not reuse data. Caterva2-source support and memory/disk policy integration are also implemented -in the Python client. Caterva2 server resolution remains a separate follow-up -behind a default-deny configuration, as described in Phase 5. +in the Python client. Caterva2 implements the default-deny, public HTTPS server +baseline described in Phase 5. The remaining server work is ordered in +`Open Work Queue And Decisions` and the Phase 5 subphases above. diff --git a/src/blosc2/__init__.py b/src/blosc2/__init__.py index 5f4023811..8065079e9 100644 --- a/src/blosc2/__init__.py +++ b/src/blosc2/__init__.py @@ -234,7 +234,6 @@ class CachePolicy(Enum): """Retention policy for data read through a remote proxy.""" NONE = "none" - MEMORY = "memory" DISK = "disk" diff --git a/src/blosc2/lazyexpr.py b/src/blosc2/lazyexpr.py index bfc6efa10..31be7cf39 100644 --- a/src/blosc2/lazyexpr.py +++ b/src/blosc2/lazyexpr.py @@ -710,7 +710,7 @@ def save(self, **kwargs: Any) -> None: * If an operand is a :ref:`Proxy`, keep in mind that Python-Blosc2 will only be able to reopen it as such if its source is a :ref:`SChunk`, :ref:`NDArray` or a :ref:`C2Array` (see :func:`blosc2.open` notes section for more info). A :ref:`RemoteProxy` is persisted as its - reference-only source descriptor. + portable source descriptor rather than embedding its carrier cache. * This is currently only supported for :ref:`LazyExpr` and :ref:`LazyUDF` (including kernels decorated with :func:`blosc2.dsl_kernel`). * User metadata can be attached via :attr:`vlmeta`. For in-memory LazyArrays diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index cb1ea7034..becfc7c74 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -1120,7 +1120,9 @@ def __getitem__(self, item: slice | list[slice]) -> np.ndarray: except ValueError as exc: if getattr(self._schunk_cache, "mode", None) != "r" or "reading mode" not in str(exc): raise - return self.src[item] + # A range-backed source need not implement NumPy indexing itself. + # Assemble this one result in an ephemeral cache instead. + return blosc2.Proxy(self.src, _refresh_source=False)[item] result = self._cache[item] self._enforce_cache_limit(item) return result diff --git a/src/blosc2/remote_proxy.py b/src/blosc2/remote_proxy.py index d07e1f9d6..863c607f4 100644 --- a/src/blosc2/remote_proxy.py +++ b/src/blosc2/remote_proxy.py @@ -20,7 +20,7 @@ from blosc2.b2objects import make_b2object_carrier, write_b2object_payload from blosc2.info import InfoReporter, format_nbytes_info -DEFAULT_MEMORY_CACHE_BYTES = 256 * 2**20 +DEFAULT_DISK_CACHE_BYTES = 256 * 2**20 class _PolicyDefault: @@ -70,13 +70,11 @@ def _normalize_limit(policy, value): raise ValueError("max_cache_bytes is not applicable to CachePolicy.NONE") return None if value is _POLICY_DEFAULT: - return DEFAULT_MEMORY_CACHE_BYTES if policy is blosc2.CachePolicy.MEMORY else None - if value is None: - return None + return DEFAULT_DISK_CACHE_BYTES if isinstance(value, bool) or not isinstance(value, int): - raise TypeError("max_cache_bytes must be a positive integer or None") + raise TypeError("max_cache_bytes must be a positive integer") if value <= 0: - raise ValueError("max_cache_bytes must be a positive integer or None") + raise ValueError("max_cache_bytes must be a positive integer") return value @@ -91,12 +89,10 @@ def _validate_max_concurrency(value: int | None) -> int | None: class RemoteProxy(blosc2.Operand): - """A persistable reference to a Caterva2 or fsspec remote array. + """A persistable, optionally self-caching reference to a remote array. - Unlike :class:`Proxy`, the object serialized by :meth:`save` is never the - data cache. It is a metadata-only carrier which reopens with - :attr:`CachePolicy.NONE`, regardless of the live proxy's runtime policy. - Memory and disk caches are runtime choices and are not serialized. + With :attr:`CachePolicy.DISK`, the persisted B2ND carrier is itself the + bounded cache. With :attr:`CachePolicy.NONE`, reads retain no data. Parameters ---------- @@ -104,9 +100,8 @@ class RemoteProxy(blosc2.Operand): A single-file B2ND URL opened through fsspec, or a Caterva2 array reference. cache_policy: CachePolicy - ``NONE`` retains no array data between operations. ``MEMORY`` retains - compressed data in memory. ``DISK`` retains it in ``cache_path`` or - under ``cache_dir``. + ``NONE`` retains no array data. ``DISK`` retains compressed chunks in + the RemoteProxy carrier at ``cache_path`` or under ``cache_dir``. cache_path: str or path-like, optional Exact persistent cache filename. Only valid with ``DISK`` and mutually exclusive with ``cache_dir``. @@ -115,8 +110,7 @@ class RemoteProxy(blosc2.Operand): Only valid with ``DISK``. max_cache_bytes: int or None, optional Post-operation compressed-payload bound. It defaults to 256 MiB for - ``MEMORY`` and unlimited for ``DISK``; explicit ``None`` means - unlimited. It is not applicable to ``NONE``. + ``DISK`` and must always be finite. It is not applicable to ``NONE``. max_concurrency: int, optional Maximum number of independent remote fetches in flight. """ @@ -130,6 +124,7 @@ def __init__( cache_dir=None, max_cache_bytes=_POLICY_DEFAULT, max_concurrency: int | None = None, + _carrier=None, ): if not isinstance(cache_policy, blosc2.CachePolicy): raise TypeError("cache_policy must be a blosc2.CachePolicy instance") @@ -137,7 +132,12 @@ def __init__( raise ValueError("cache_dir and cache_path are mutually exclusive") if cache_policy is not blosc2.CachePolicy.DISK and (cache_dir is not None or cache_path is not None): raise ValueError("cache_dir and cache_path require CachePolicy.DISK") - if cache_policy is blosc2.CachePolicy.DISK and cache_dir is None and cache_path is None: + if ( + cache_policy is blosc2.CachePolicy.DISK + and cache_dir is None + and cache_path is None + and _carrier is None + ): raise ValueError("CachePolicy.DISK requires cache_dir or cache_path") self._cache_policy = cache_policy @@ -149,9 +149,13 @@ def __init__( self._expected_cparams = self.src.cparams self._refresh_lock = threading.Lock() self._proxy = None + self._carrier = _carrier + self._cache_status = None - if cache_policy is not blosc2.CachePolicy.NONE: - self._proxy = self._make_cache_proxy(cache_dir, cache_path) + if cache_policy is blosc2.CachePolicy.DISK: + if self._carrier is None: + self._carrier, self._cache_status = self._open_or_create_carrier(cache_dir, cache_path) + self._attach_carrier_cache() def _runtime_source(self, original): """Keep credentials in live process state, outside the descriptor.""" @@ -172,14 +176,40 @@ def _geometry(src): tuple(src.blocks), ) - def _make_cache_proxy(self, cache_dir, cache_path): - return blosc2.schunk._lazy_remote_proxy( + def _open_or_create_carrier(self, cache_dir, cache_path): + if cache_path is not None: + path = os.fspath(cache_path) + if os.path.isdir(path): + raise ValueError("cache_path must name a file, not a directory") + else: + path = blosc2.schunk.fsspec_cache_path(self._source_identity(), cache_dir, ".b2nd") + if os.path.exists(path): + kwargs = {"dparams": blosc2.DParams(nthreads=1)} + carrier = blosc2.blosc2_ext.open(path, "a", 0, **kwargs) + payload = carrier.schunk.vlmeta.get("b2o") + if payload != self._payload(): + raise ValueError(f"the RemoteProxy carrier at {path} has a different specification") + stored = carrier.schunk.vlmeta.get("proxy-stamp") + current = getattr(self.src, "stamp", None) + status = "invalidated/rebuilt" if stored is not None and current is not None and stored != current else "reused" + return carrier, status + carrier = self._to_b2object_carrier(urlpath=path, contiguous=True, mode="w") + return carrier, "created" + + def _attach_carrier_cache(self): + if self._carrier is None or self.cache_policy is not blosc2.CachePolicy.DISK: + self._proxy = None + return + if getattr(self.src, "stamp", None) is None: + # Without a stable validator, cached bytes cannot be trusted across + # independent opens. Reads still work, but misses are not retained. + self._proxy = None + return + self._proxy = blosc2.Proxy( self.src, - self._source_identity(), - cache_dir, - cache_path, - source_fresh=True, - max_cache_bytes=self._cache_limit, + _cache=self._carrier, + _refresh_source=False, + _max_cache_bytes=self._cache_limit, ) @staticmethod @@ -261,16 +291,6 @@ def _prepare_read(self): previous_stamp is None or current_stamp is None or current_stamp != previous_stamp ) if source_changed: - cache_path = self.cache_path - if ( - current_stamp is None - and self.cache_policy is blosc2.CachePolicy.DISK - and cache_path is not None - and os.path.exists(cache_path) - ): - # With no source identity, an on-disk cache cannot prove - # that its payload belongs to what the URL serves now. - blosc2.remove_urlpath(cache_path) fresh, _ = self._open_source( self._runtime_urlpath, self._max_concurrency, @@ -282,9 +302,7 @@ def _prepare_read(self): fresh.stamp = None self._validate_geometry(self._expected_geometry, src=fresh) self.src = fresh - if self.cache_policy is not blosc2.CachePolicy.NONE: - cache_dir = None - self._proxy = self._make_cache_proxy(cache_dir, cache_path) + self._attach_carrier_cache() return self.src if self._proxy is None else self._proxy @@ -311,12 +329,12 @@ def blocks(self): @property def cache_policy(self) -> blosc2.CachePolicy: - """The immutable runtime retention policy.""" + """The persisted retention policy.""" return self._cache_policy @property def max_cache_bytes(self) -> int | None: - """The immutable post-operation retained-cache bound.""" + """The persisted post-operation retained-cache bound.""" return self._cache_limit @property @@ -367,17 +385,15 @@ def urlpath(self): @property def cache_path(self): - """The runtime disk cache path, or ``None`` for other policies.""" - if self._proxy is None or self.cache_policy is not blosc2.CachePolicy.DISK: + """The self-caching carrier path, or ``None`` for other policies.""" + if self._carrier is None or self.cache_policy is not blosc2.CachePolicy.DISK: return None - return self._proxy.urlpath + return getattr(self._carrier.schunk, "urlpath", None) @property def cache_status(self): """How a persistent disk cache was handled, or ``None`` otherwise.""" - if self._proxy is None: - return None - return self._proxy.cache_status + return self._cache_status @property def cache_bytes(self) -> int: @@ -448,9 +464,8 @@ def _payload(self): "kind": "remote_proxy", "version": 1, "source": dict(self._source), - # Persistence is intentionally reference-only. Runtime cache state - # and local cache paths never cross this boundary. - "cache_policy": blosc2.CachePolicy.NONE.value, + "cache_policy": self.cache_policy.value, + "max_cache_bytes": self.max_cache_bytes, } def _to_b2object_carrier(self, **kwargs): @@ -466,23 +481,48 @@ def _to_b2object_carrier(self, **kwargs): write_b2object_payload(array, self._payload()) return array - def to_cframe(self) -> bytes: - """Serialize this reference, without cached array data, as a CFrame.""" - return self._to_b2object_carrier().to_cframe() + def _export_carrier(self, include_cache: bool): + if not isinstance(include_cache, bool): + raise TypeError("include_cache must be a boolean") + if include_cache and self._carrier is not None: + return self._carrier + return self._to_b2object_carrier() + + def to_cframe(self, *, include_cache: bool = True) -> bytes: + """Serialize the carrier, including valid cached chunks by default.""" + return self._export_carrier(include_cache).to_cframe() - def save(self, urlpath: str | os.PathLike, contiguous: bool = True, **kwargs) -> None: - """Persist this reference without its runtime cache or credentials.""" + def save( + self, + urlpath: str | os.PathLike, + contiguous: bool = True, + *, + include_cache: bool = True, + **kwargs, + ) -> None: + """Persist the carrier, including valid cached chunks by default.""" urlpath = os.fspath(urlpath) + carrier = self._export_carrier(include_cache) + source_path = getattr(carrier.schunk, "urlpath", None) + if source_path is not None and os.path.abspath(source_path) == os.path.abspath(urlpath): + return blosc2.blosc2_ext.check_access_mode(urlpath, "w") - kwargs.update(urlpath=urlpath, contiguous=contiguous, mode="w") - self._to_b2object_carrier(**kwargs) + carrier.save(urlpath, contiguous=contiguous, **kwargs) @classmethod def _from_payload(cls, payload, carrier): - if set(payload) != {"kind", "version", "source", "cache_policy"}: + if set(payload) != {"kind", "version", "source", "cache_policy", "max_cache_bytes"}: raise ValueError("persisted RemoteProxy payload contains unsupported fields") - if payload.get("cache_policy") != blosc2.CachePolicy.NONE.value: - raise ValueError("persisted RemoteProxy objects must use cache policy 'none'") + try: + policy = blosc2.CachePolicy(payload.get("cache_policy")) + except ValueError as exc: + raise ValueError("persisted RemoteProxy has an unsupported cache policy") from exc + limit = payload.get("max_cache_bytes") + if policy is blosc2.CachePolicy.NONE: + if limit is not None: + raise ValueError("persisted NONE RemoteProxy cannot have max_cache_bytes") + elif isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0: + raise ValueError("persisted DISK RemoteProxy requires positive max_cache_bytes") source = payload.get("source") if not isinstance(source, dict) or source.get("version") != 1: raise ValueError("unsupported RemoteProxy source descriptor") @@ -504,7 +544,8 @@ def _from_payload(cls, payload, carrier): else: raise ValueError(f"unsupported RemoteProxy source kind: {source_kind!r}") expected = (carrier.shape, carrier.dtype, carrier.chunks, carrier.blocks) - obj = cls(urlpath, cache_policy=blosc2.CachePolicy.NONE) + kwargs = {} if policy is blosc2.CachePolicy.NONE else {"max_cache_bytes": limit} + obj = cls(urlpath, cache_policy=policy, _carrier=carrier, **kwargs) obj._validate_geometry(expected) return obj diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 51a105e57..85fbb902a 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -2019,11 +2019,7 @@ def _remote_proxy_options(kwargs, cache_dir, cache_path, max_concurrency): policy = kwargs.pop("cache_policy", None) limit = kwargs.pop("max_cache_bytes", None) if not policy_present: - policy = ( - blosc2.CachePolicy.DISK - if cache_dir is not None or cache_path is not None - else blosc2.CachePolicy.MEMORY - ) + policy = blosc2.CachePolicy.DISK if cache_dir is not None or cache_path is not None else blosc2.CachePolicy.NONE options = { "cache_policy": policy, "cache_dir": cache_dir, @@ -2293,14 +2289,14 @@ def open( cache_policy: CachePolicy, optional With ``lazy=True`` on a remote source, return a :ref:`RemoteProxy` using the requested retention policy. ``NONE`` retains no data - between operations, ``MEMORY`` retains compressed data in memory, - and ``DISK`` requires ``cache_dir`` or ``cache_path``. When omitted, - the existing :ref:`Proxy` behavior is preserved. - max_cache_bytes: int or None, optional + and ``DISK`` retains compressed chunks in its carrier and requires + ``cache_dir`` or ``cache_path`` when creating one from a remote URL. + When omitted, the existing :ref:`Proxy` behavior is preserved. + max_cache_bytes: int, optional With ``lazy=True``, enable a :ref:`RemoteProxy` and bound retained - compressed cache payload after each operation. The memory-policy - default is 256 MiB; disk is unlimited unless this is explicitly set. - This does not bound the current operation's working set or result. + compressed cache payload after each operation. ``DISK`` defaults to + 256 MiB and always has a finite bound. This does not bound the + current operation's working set or result. mmap_mode: str, optional If set, the file will be memory-mapped instead of using the default I/O functions and the `mode` argument will be ignored. @@ -2358,11 +2354,13 @@ def open( formats do. ``cache_dir`` and ``lazy`` above lift that, each in its own way. - * Persistent data handling follows a strict no-hidden-writes rule: + * Persistent data handling follows a no-hidden-writes rule except for an + explicitly self-caching :ref:`RemoteProxy`: - ``mode='r'`` is observational only and never mutates the opened object. - - ``mode='a'`` / ``mode='w'`` only persist explicit mutations requested by the - caller; runtime caches are not serialized back to disk. + - ``mode='a'`` permits a ``DISK`` RemoteProxy to retain remote chunks in + its own carrier. Other execution caches are not serialized implicitly. + - ``mode='w'`` persists explicit mutations requested by the caller. * If the original object saved in :paramref:`urlpath` is a :ref:`Proxy`, this function reconstructs sources backed by a persistent local diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index a883550bf..9be98273e 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -651,7 +651,8 @@ def test_http_remote_proxy_checks_identity_without_refetching_cached_data(tmp_pa with _ranged_server(path) as (urlbase, requests): remote = blosc2.RemoteProxy( f"{urlbase}/stable.b2nd", - cache_policy=blosc2.CachePolicy.MEMORY, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=tmp_path / "stable-proxy.b2nd", ) np.testing.assert_array_equal(remote[3:5, 100:120], data[3:5, 100:120]) diff --git a/tests/test_remote_proxy.py b/tests/test_remote_proxy.py index eb57d269f..745c5682a 100644 --- a/tests/test_remote_proxy.py +++ b/tests/test_remote_proxy.py @@ -8,7 +8,6 @@ from __future__ import annotations import asyncio -import hashlib import fsspec import numpy as np @@ -34,20 +33,25 @@ def test_cache_policy_validation(tmp_path): assert none.cache_policy is blosc2.CachePolicy.NONE assert none.max_cache_bytes is None - memory = blosc2.RemoteProxy(url, cache_policy=blosc2.CachePolicy.MEMORY) - assert memory.max_cache_bytes == 256 * 2**20 - disk = blosc2.RemoteProxy( url, cache_policy=blosc2.CachePolicy.DISK, cache_path=tmp_path / "cache.b2nd", ) - assert disk.max_cache_bytes is None + assert disk.max_cache_bytes == 256 * 2**20 + assert disk.cache_path == str(tmp_path / "cache.b2nd") with pytest.raises(TypeError, match="CachePolicy"): blosc2.RemoteProxy(url, cache_policy="memory") with pytest.raises(ValueError, match="not applicable"): blosc2.RemoteProxy(url, max_cache_bytes=1) + with pytest.raises(TypeError, match="positive integer"): + blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=tmp_path / "unlimited.b2nd", + max_cache_bytes=None, + ) with pytest.raises(ValueError, match="requires cache_dir or cache_path"): blosc2.RemoteProxy(url, cache_policy=blosc2.CachePolicy.DISK) with pytest.raises(ValueError, match="max_concurrency"): @@ -79,10 +83,8 @@ def test_open_selects_remote_proxy_only_for_explicit_policy(tmp_path): assert none.cache_policy is blosc2.CachePolicy.NONE np.testing.assert_array_equal(none[:100_000], data[:100_000]) - memory = blosc2.open(url, lazy=True, max_cache_bytes=120_000) - assert isinstance(memory, blosc2.RemoteProxy) - assert memory.cache_policy is blosc2.CachePolicy.MEMORY - assert memory.max_cache_bytes == 120_000 + with pytest.raises(ValueError, match="not applicable"): + blosc2.open(url, lazy=True, max_cache_bytes=120_000) disk = blosc2.open( url, @@ -110,47 +112,7 @@ def test_none_does_not_retain_remote_data(): assert proxy.cache_bytes == 0 -def test_memory_bound_evicts_lru_chunk(): - url, data = _remote_array("memory-bound.b2nd") - proxy = blosc2.RemoteProxy( - url, - cache_policy=blosc2.CachePolicy.MEMORY, - max_cache_bytes=120_000, - ) - - np.testing.assert_array_equal(proxy[:100_000], data[:100_000]) - np.testing.assert_array_equal(proxy[100_000:200_000], data[100_000:200_000]) - assert proxy.cache_bytes <= 120_000 - - proxy.traffic.reset() - np.testing.assert_array_equal(proxy[:100_000], data[:100_000]) - assert proxy.traffic.requests > 0 - assert proxy.cache_bytes <= 120_000 - - -def test_memory_bound_refreshes_lru_on_cache_hit(): - url, data = _remote_array("memory-lru.b2nd") - proxy = blosc2.RemoteProxy( - url, - cache_policy=blosc2.CachePolicy.MEMORY, - max_cache_bytes=220_000, - ) - - proxy[:100_000] - proxy[100_000:200_000] - proxy[:100_000] # chunk 0 is now newer than chunk 1 - proxy[200_000:300_000] - - proxy.traffic.reset() - np.testing.assert_array_equal(proxy[:100_000], data[:100_000]) - assert proxy.traffic.requests == 0 - - proxy.traffic.reset() - np.testing.assert_array_equal(proxy[100_000:200_000], data[100_000:200_000]) - assert proxy.traffic.requests > 0 - - -def test_disk_bound_is_optional_and_shrinks_cache(tmp_path): +def test_disk_bound_shrinks_self_caching_carrier(tmp_path): url, data = _remote_array("disk-bound.b2nd") cache_path = tmp_path / "bounded-cache.b2nd" proxy = blosc2.RemoteProxy( @@ -176,11 +138,41 @@ def test_disk_bound_is_optional_and_shrinks_cache(tmp_path): assert reopened.cache_bytes <= 120_000 -def test_reference_roundtrip_is_none_and_does_not_mutate(tmp_path): +def test_interrupted_fetch_leaves_a_reusable_carrier(tmp_path): + url, data = _remote_array("interrupted.b2nd", nchunks=3, chunk_size=100_000) + cache_path = tmp_path / "interrupted-proxy.b2nd" + proxy = blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=cache_path, + max_cache_bytes=1_000_000, + ) + get_chunk = proxy.src.get_chunk + + def interrupted(nchunk): + if nchunk == 1: + raise RuntimeError("simulated interruption") + return get_chunk(nchunk) + + proxy.src.get_chunk = interrupted + with pytest.raises(RuntimeError, match="simulated interruption"): + proxy[:] + + carrier = blosc2.blosc2_ext.open(str(cache_path), "r", 0, dparams=blosc2.DParams(nthreads=1)) + assert carrier.schunk.vlmeta.get("proxy-fetched") + proxy.src.get_chunk = get_chunk + np.testing.assert_array_equal(proxy[:], data) + + reopened = blosc2.open(cache_path, mode="r") + np.testing.assert_array_equal(reopened[:], data) + + +def test_disk_roundtrip_preserves_warm_cache_and_cold_escape_hatch(tmp_path): url, data = _remote_array("roundtrip.b2nd") original = blosc2.RemoteProxy( url, - cache_policy=blosc2.CachePolicy.MEMORY, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=tmp_path / "live-proxy.b2nd", max_cache_bytes=120_000, ) original[:100_000] @@ -191,18 +183,39 @@ def test_reference_roundtrip_is_none_and_does_not_mutate(tmp_path): "kind": "remote_proxy", "version": 1, "source": {"kind": "fsspec", "version": 1, "urlpath": url}, - "cache_policy": "none", + "cache_policy": "disk", + "max_cache_bytes": 120_000, } + assert carrier.schunk.vlmeta.get("proxy-cache-sizes") - path = tmp_path / "reference.b2nd" - original.save(path) - before = hashlib.sha256(path.read_bytes()).digest(), path.stat().st_size, path.stat().st_mtime_ns - restored = blosc2.open(path, mode="r") + cold_carrier = blosc2.ndarray_from_cframe(original.to_cframe(include_cache=False)) + assert cold_carrier.schunk.vlmeta["b2o"] == carrier.schunk.vlmeta["b2o"] + assert not cold_carrier.schunk.vlmeta.get("proxy-cache-sizes", {}) + assert original.cache_bytes > 0 + + warm_path = tmp_path / "warm.b2nd" + original.save(warm_path) + restored = blosc2.open(warm_path, mode="r") assert isinstance(restored, blosc2.RemoteProxy) - assert restored.cache_policy is blosc2.CachePolicy.NONE + assert restored.cache_policy is blosc2.CachePolicy.DISK + restored.traffic.reset() np.testing.assert_array_equal(restored[:100_000], data[:100_000]) - after = hashlib.sha256(path.read_bytes()).digest(), path.stat().st_size, path.stat().st_mtime_ns - assert after == before + assert restored.traffic.requests == 0 + + cold_path = tmp_path / "cold.b2nd" + original.save(cold_path, include_cache=False) + cold = blosc2.open(cold_path, mode="r") + cold.traffic.reset() + np.testing.assert_array_equal(cold[:100_000], data[:100_000]) + assert cold.traffic.requests > 0 + assert cold_path.stat().st_size < warm_path.stat().st_size + + mutable = blosc2.open(cold_path, mode="a") + np.testing.assert_array_equal(mutable[:100_000], data[:100_000]) + reopened = blosc2.open(cold_path, mode="r") + reopened.traffic.reset() + np.testing.assert_array_equal(reopened[:100_000], data[:100_000]) + assert reopened.traffic.requests == 0 def test_reference_rejects_changed_source_geometry(tmp_path): @@ -229,19 +242,19 @@ def test_open_reference_rejects_geometry_changed_before_read(tmp_path): restored[:] -@pytest.mark.parametrize("policy", [blosc2.CachePolicy.MEMORY, blosc2.CachePolicy.DISK]) -def test_runtime_cache_is_invalidated_after_same_geometry_replacement(tmp_path, policy): - url, data = _remote_array(f"same-geometry-{policy.value}.b2nd", nchunks=1, chunk_size=100) - kwargs = ( - {"cache_path": tmp_path / f"{policy.value}-cache.b2nd"} if policy is blosc2.CachePolicy.DISK else {} +def test_runtime_cache_is_invalidated_after_same_geometry_replacement(tmp_path): + url, data = _remote_array("same-geometry-disk.b2nd", nchunks=1, chunk_size=100) + proxy = blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=tmp_path / "same-geometry-cache.b2nd", ) - proxy = blosc2.RemoteProxy(url, cache_policy=policy, **kwargs) traffic = proxy.traffic np.testing.assert_array_equal(proxy[:], data) replacement = np.arange(100, dtype=np.uint8) array = blosc2.asarray(replacement, chunks=(100,), blocks=(100,)) - fsspec.filesystem("memory").pipe_file(f"same-geometry-{policy.value}.b2nd", array.to_cframe()) + fsspec.filesystem("memory").pipe_file("same-geometry-disk.b2nd", array.to_cframe()) np.testing.assert_array_equal(proxy[:], replacement) assert proxy.traffic is traffic @@ -253,7 +266,7 @@ def test_reference_rejects_runtime_cache_policy_in_payload(): payload = dict(carrier.schunk.vlmeta["b2o"]) payload["cache_policy"] = "memory" - with pytest.raises(ValueError, match="must use cache policy 'none'"): + with pytest.raises(ValueError, match="unsupported cache policy"): decode_b2object_payload(payload, carrier=carrier) @@ -350,8 +363,7 @@ def fake_fetch_data(path, urlbase, params, auth_token=None, as_blosc2=False, tra assert calls == [{"slice_": "2:5"}] -@pytest.mark.parametrize("policy", [blosc2.CachePolicy.MEMORY, blosc2.CachePolicy.DISK]) -def test_caterva2_runtime_caches_reuse_chunks(monkeypatch, tmp_path, policy): +def test_caterva2_disk_cache_reuses_chunks(monkeypatch, tmp_path): data = np.arange(10, dtype=np.int32) local = blosc2.asarray(data, chunks=(5,), blocks=(5,)) compressed = [local.schunk.get_chunk(i) for i in range(2)] @@ -378,11 +390,10 @@ def fake_get_chunk(self, nchunk): monkeypatch.setattr(blosc2_c2array, "info", fake_info) monkeypatch.setattr(blosc2.C2Array, "get_chunk", fake_get_chunk) - kwargs = {"cache_path": tmp_path / "caterva2-cache.b2nd"} if policy is blosc2.CachePolicy.DISK else {} remote = blosc2.RemoteProxy( blosc2.URLPath("@public/cache.b2nd", urlbase="https://example.org/c2"), - cache_policy=policy, - **kwargs, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=tmp_path / "caterva2-cache.b2nd", ) np.testing.assert_array_equal(remote[:5], data[:5]) @@ -456,7 +467,7 @@ def test_disk_cache_survives_reopen_without_remote_data_traffic(tmp_path): assert reopened.traffic.requests == 0 -def test_save_after_disk_cache_use_remains_reference_only(tmp_path): +def test_save_after_disk_cache_use_preserves_or_strips_cache(tmp_path): url, data = _remote_array("save-after-disk.b2nd", nchunks=2, chunk_size=100_000) proxy = blosc2.RemoteProxy( url, @@ -465,11 +476,19 @@ def test_save_after_disk_cache_use_remains_reference_only(tmp_path): ) np.testing.assert_array_equal(proxy[:100_000], data[:100_000]) - reference_path = tmp_path / "saved-reference.b2nd" - proxy.save(reference_path) - carrier = blosc2.ndarray_from_cframe(reference_path.read_bytes()) - assert carrier.schunk.vlmeta["b2o"]["cache_policy"] == "none" - assert "proxy-source" not in carrier.schunk.meta + warm_path = tmp_path / "saved-warm.b2nd" + cold_path = tmp_path / "saved-cold.b2nd" + proxy.save(warm_path) + proxy.save(cold_path, include_cache=False) + warm = blosc2.open(warm_path, mode="r") + cold = blosc2.open(cold_path, mode="r") + warm.traffic.reset() + cold.traffic.reset() + np.testing.assert_array_equal(warm[:100_000], data[:100_000]) + np.testing.assert_array_equal(cold[:100_000], data[:100_000]) + assert warm.traffic.requests == 0 + assert cold.traffic.requests > 0 + assert "proxy-source" not in warm._carrier.schunk.meta def test_reference_size_is_independent_of_remote_payload(tmp_path): From dfb193e706f4f5801d8116cc4941aada8204cae9 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 5 Sep 2026 10:44:27 +0200 Subject: [PATCH 14/89] Unify lazy remote array access under RemoteProxy with in-memory caching --- doc/guides/remote_arrays.md | 171 ++++++++++++++++----------- doc/reference/remoteproxy.rst | 13 ++ plans/remote-proxy-v3.md | 88 ++++++++++++++ src/blosc2/__init__.py | 1 + src/blosc2/remote_proxy.py | 123 ++++++++++++++----- src/blosc2/schunk.py | 76 ++++++------ tests/ndarray/test_c2array_blocks.py | 25 ++-- tests/test_fsspec.py | 36 +++++- tests/test_remote_proxy.py | 106 +++++++++++++++-- 9 files changed, 478 insertions(+), 161 deletions(-) create mode 100644 plans/remote-proxy-v3.md diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index fa3ea3c92..0b3daaa3b 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -1,6 +1,8 @@ # Working with Remote Arrays -Blosc2 can open an array without downloading it first. Metadata is read at open time; array data is fetched only when a slice needs it and is then kept in a local cache. +Blosc2 can open remote arrays without downloading them first. Metadata is read instantly at open time; array data is fetched only when a slice needs it and is then kept in a local cache. + +All lazy remote array access in Python-Blosc2 is unified under {ref}`RemoteProxy`. ## Choose a remote route @@ -34,7 +36,7 @@ A `URLPath` always means Caterva2. If its `urlbase` is omitted, the server comes ### What each route supports -Both routes return a {ref}`Proxy` when opened with `lazy=True`, so slicing and caching work the same way. Their sources differ: +When opened with `lazy=True`, both routes return a {ref}`RemoteProxy`, providing an identical user interface for slicing, caching, and introspection. What differs is the types of remote objects each backend can open: | Remote object | fsspec URL | Caterva2 `URLPath` | |---|---|---| @@ -44,33 +46,46 @@ Both routes return a {ref}`Proxy` when opened with `lazy=True`, so slicing and c | Lazy or computed array | No | Yes | | Whole `.b2z` `TreeStore` or `DictStore` | No | No; open one array-like leaf | -fsspec supplies byte ranges. Python-Blosc2 parses the `.b2nd` frame to discover its geometry and chunk offsets, making this route direct and efficient for standalone arrays. +- **fsspec** supplies byte ranges. Python-Blosc2 parses the remote `.b2nd` frame to discover its geometry and chunk offsets, making this route direct and efficient for standalone arrays. +- **Caterva2** understands dataset paths, array metadata, and slicing. It can therefore expose array-like data that is not stored as a standalone Blosc2 frame, as well as apply authentication or server-side computation. Use Caterva2's navigation API to find a leaf in a remote hierarchy, then open that leaf with a `URLPath`. + +`lazy=True` changes *when* data is fetched; it does not expand the underlying storage formats supported by either route. -Caterva2 understands dataset paths, array metadata, and slicing. It can therefore expose array-like data that is not stored as a standalone Blosc2 frame, as well as apply authentication or server-side computation. Use Caterva2's navigation API to find a leaf in a remote hierarchy, then open that leaf with a `URLPath`. +## Cache policies and memory management -`lazy=True` changes when data is fetched; it does not expand the formats supported by either route. +Every lazy open uses a cache policy. By default, fetched data is cached in memory and bounded to prevent excessive RAM consumption. -## Choose a cache +### In-memory caching (`CachePolicy.MEMORY` — Default) -Every lazy open creates a cache. By default it lives in memory and disappears with the proxy: +When opened without disk options, `blosc2.open(..., lazy=True)` retains fetched chunks in RAM as a {ref}`RemoteProxy` with {attr}`CachePolicy.MEMORY `: ```python a = blosc2.open("s3://bucket/big.b2nd", lazy=True) -a[10:12, 500:600] # fetched and cached -a[10:12, 500:600] # served from memory +a[10:12, 500:600] # fetched and cached in RAM +a[10:12, 500:600] # served from memory cache (no network traffic) ``` -Set `cache_dir` to let Blosc2 manage a cache file inside a directory: +To prevent memory leaks or out-of-memory errors on massive datasets, in-memory caches are bounded by `max_cache_bytes` (defaults to 256 MiB) with automatic LRU eviction: + +```python +# Custom in-memory limit (e.g. 512 MiB): +a = blosc2.open("s3://bucket/big.b2nd", lazy=True, max_cache_bytes=512 * 2**20) +``` + +### Persistent disk caching (`CachePolicy.DISK`) + +Set `cache_dir` or `cache_path` to persist fetched data across sessions. Providing either option with `lazy=True` configures the {ref}`RemoteProxy` to use persistent disk caching ({attr}`CachePolicy.DISK `): ```python url = "s3://bucket/big.b2nd" +# Blosc2 manages a cache file inside a directory: a = blosc2.open(url, lazy=True, cache_dir="./b2cache") a[100:110, :50] # fetched and stored under ./b2cache -# A later process can reuse the same cache. +# A later process can reuse the same cache: a = blosc2.open(url, lazy=True, cache_dir="./b2cache") -a[100:110, :50] # no request +a[100:110, :50] # served from local disk (no network traffic) ``` Use `cache_path` instead when the cache should have an exact filename: @@ -79,14 +94,78 @@ Use `cache_path` instead when the cache should have an exact filename: a = blosc2.open(url, lazy=True, cache_path="big-cache.b2nd") ``` -In both cases, the cache is an ordinary `.b2nd` array that starts small and grows as regions are read. `cache_dir` and `cache_path` are mutually exclusive. +In both cases, the cache is an ordinary `.b2nd` carrier file managed as a {ref}`RemoteProxy` with {attr}`CachePolicy.DISK `. It starts small and retains compressed chunks up to a finite bound (256 MiB by default, or customized via `max_cache_bytes`). `cache_dir` and `cache_path` are mutually exclusive. Authenticated Caterva2 caches must be private to one user. Reopen them under an equivalent authenticated {func}`blosc2.c2context`; do not share a cache directory between users. +### Stateless streaming (`CachePolicy.NONE`) + +To stream data without retaining any chunks after each operation, specify {attr}`CachePolicy.NONE `: + +```python +stream = blosc2.open( + "s3://bucket/big.b2nd", + lazy=True, + cache_policy=blosc2.CachePolicy.NONE, +) +``` + +Each read pulls only the bytes required for the slice and retains no cache payload. + +> [!NOTE] +> `max_cache_bytes` is applied after each operation completes. It bounds the retained compressed cache payload; it does not limit the temporary working set or the decompressed NumPy array requested by the caller. + +## Only what a slice touches + +Blosc2 arrays are compressed in chunks, which are divided into smaller blocks. For a small slice, fetching only its blocks can avoid transferring most of a large chunk. + +![A proxy fetches missing regions from the remote array into its local cache. The fetch method returns the cache container, while indexing returns only the requested values.](../tutorials/images/remote_proxy.png) + +Purple regions are cached; red regions are still remote. The grid is schematic: where byte ranges are available, the fetched regions can be blocks within a chunk. `fetch()` fills and returns the cache container, whereas indexing returns only the requested values. + +The proxy chooses blocks or whole chunks automatically. It fetches a whole chunk when most of its blocks are needed or when the source cannot expose block ranges, as with computed Caterva2 datasets. Independent reads overlap, with up to eight concurrent requests by default; use `max_concurrency=1` when concurrency does not help. + +Stepped slices also use the block grid. For example, `a[::5]` can reduce transfers along an axis whose blocks do not already span that axis. + +### Explicit cache pre-fetching + +You can warm the cache proactively using `fetch()` or `afetch()`: + +```python +# Synchronously pre-fetch a region into the cache: +cached_container = a.fetch(slice(0, 10_000)) + +# Or asynchronously in an async event loop: +cached_container = await a.afetch(slice(10_000, 20_000)) +``` + +The underlying local cache container (an `NDArray`) can also be accessed directly via `a.cache`. + +## Measure network traffic + +{ref}`RemoteProxy`, {ref}`C2Array`, and {ref}`Proxy` objects expose cumulative request and byte counts through {ref}`Traffic`. The count starts when the remote source is opened, so it includes metadata as well as array data: + +```python +a = blosc2.open("s3://bucket/big.b2nd", lazy=True) + +a.traffic.reset() +corner = a[0, :100, :100] +print(a.traffic) # requests and bytes fetched + +a.traffic.reset() +corner = a[0, :100, :100] +print(a.traffic) # Traffic(requests=0, nbytes=0) -> cache hit! +``` + +Use `reset()` or subtract two readings to measure one operation. `traffic` is `None` for a local source because no network transport exists. + +`examples/c2array-traffic.py` compares block, chunk, and cached reads against a live Caterva2 dataset. + +## Persist and reopen remote references + ### Persist a self-caching remote proxy -Use {ref}`RemoteProxy` when the `.b2nd` file should carry a portable remote -descriptor and, optionally, its own bounded persistent cache: +Use {ref}`RemoteProxy` directly when a `.b2nd` file should carry a portable remote descriptor and, optionally, its own bounded persistent cache: ```python remote = blosc2.RemoteProxy( @@ -96,13 +175,9 @@ remote = blosc2.RemoteProxy( remote.save("big-reference.b2nd") ``` -The saved object contains source and geometry metadata but no credentials. With -`CachePolicy.NONE`, repeated reads contact the source and do not mutate the -carrier. +The saved object contains source and geometry metadata but no credentials. With `CachePolicy.NONE`, repeated reads contact the source and do not mutate the carrier. -`RemoteProxy` also supports bounded disk caching. The proxy carrier itself is -the cache, with a finite 256 MiB compressed-payload bound by default or an -explicit bound: +With `CachePolicy.DISK`, the proxy carrier itself is the cache: ```python remote = blosc2.RemoteProxy( @@ -113,13 +188,7 @@ remote = blosc2.RemoteProxy( ) ``` -Saving and serializing preserve valid warm chunks by default. Pass -`include_cache=False` to `save()` or `to_cframe()` to export a cold copy without -mutating the warm carrier. A saved disk proxy opened with `mode="a"` retains -misses in itself; `mode="r"` can use warm chunks but does not retain misses. - -The bound is applied after each operation. It does not limit the temporary -working set or a NumPy result requested by the caller. +Saving and serializing preserve valid warm chunks by default. Pass `include_cache=False` to `save()` or `to_cframe()` to export a cold reference copy without mutating the warm carrier. ### Reopen a cache file independently @@ -133,58 +202,22 @@ a[100:110, :50] # cached data stays local a[500:510, :50] # missing data is fetched from the recorded source and cached ``` -This cache is operational, but not necessarily self-contained. Regions not fetched previously still require the original source. Opening with `mode="a"` lets newly fetched regions extend the cache; the default `mode="r"` keeps the cache file unchanged. +Opening an on-disk carrier with `mode="a"` returns a {ref}`RemoteProxy` and lets newly fetched regions extend the cache; opening with `mode="r"` keeps the cache file unchanged. Legacy proxy caches created by older Blosc2 versions are also detected and reopened as a {ref}`Proxy`. Independent reopening works for fsspec URLs, Caterva2 datasets, and persistent local Blosc2 sources. The required runtime environment must still be available: fsspec backends and their configuration must be installed, local source paths must remain valid, and authenticated Caterva2 caches must be reopened inside an equivalent {func}`blosc2.c2context`. Caterva2 credentials are not stored in the cache file. An arbitrary custom {ref}`ProxyNDSource` cannot be reconstructed because its Python class and runtime state are not serialized. In that case, recreate the source explicitly and attach the existing cache with `blosc2.Proxy(source, urlpath="big-cache.b2nd", mode="a")`. -## Only what a slice touches - -Blosc2 arrays are compressed in chunks, which are divided into smaller blocks. For a small slice, fetching only its blocks can avoid transferring most of a large chunk. - -![A proxy fetches missing regions from the remote array into its local cache. The fetch method returns the cache container, while indexing returns only the requested values.](../tutorials/images/remote_proxy.png) - -Purple regions are cached; red regions are still remote. The grid is schematic: where byte ranges are available, the fetched regions can be blocks within a chunk. `fetch()` fills and returns the cache container, whereas indexing returns only the requested values. - -The proxy chooses blocks or whole chunks automatically. It fetches a whole chunk when most of its blocks are needed or when the source cannot expose block ranges, as with computed Caterva2 datasets. Independent reads overlap, with up to eight concurrent requests by default; use `max_concurrency=1` when concurrency does not help. - -Stepped slices also use the block grid. For example, `p[::5]` can reduce transfers along an axis whose blocks do not already span that axis. A bare {ref}`C2Array` does not accept stepped slices; its proxy does. - -### Measure network traffic - -{ref}`C2Array` and remote {ref}`Proxy` objects expose cumulative request and byte counts through {ref}`Traffic`. The count starts when the remote source is opened, so it includes metadata as well as array data: - -```python -source = blosc2.C2Array( - "@public/examples/kevlar-tomo.b2nd", - urlbase="https://cat2.cloud/demo", -) -p = blosc2.Proxy(source) - -p.traffic.reset() -corner = p[0, :100, :100] -print(p.traffic) # requests and bytes fetched - -p.traffic.reset() -p[0, :100, :100] -print(p.traffic) # Traffic(requests=0, nbytes=0) -``` - -Use `reset()` or subtract two readings to measure one operation. `Proxy.traffic` is `None` for a local source because no network transport exists. - -`examples/c2array-traffic.py` compares block, chunk, and cached reads against a live Caterva2 dataset. - ## Retrieve scattered points A proxy maps coordinate arrays and boolean masks to the blocks that contain their selected points: ```python -p[rows, :100] -p[mask] +a[rows, :100] +a[mask] ``` -For Caterva2, a bare {ref}`C2Array` can be substantially more efficient: it sends the coordinates to the server, which returns only the selected values. Prefer direct `C2Array` indexing for sparse, one-off point retrieval; prefer a proxy when reuse through a local cache matters. +For Caterva2, a bare {ref}`C2Array` can be substantially more efficient for one-off point queries: it sends coordinates to the server, which evaluates the selection and returns only the selected values. Prefer direct `C2Array` indexing for sparse, one-off point retrieval; prefer a proxy when reuse through a local cache matters. ## Handle remote changes @@ -286,4 +319,4 @@ For ordinary S3 access, use `blosc2.open("s3://bucket/big.b2nd", lazy=True)`; th - `examples/ndarray/rw-fsspec.py` — fsspec reading and writing examples. - `examples/fsspec-cat2-access.py` — one dataset and cache through fsspec and Caterva2. - `examples/c2array-traffic.py` — block, chunk, and cached transfer sizes. -- {ref}`C2Array`, {ref}`FsspecNDSource`, {ref}`ByteRangeNDSource`, {ref}`Proxy`, {ref}`RemoteProxy`, and {ref}`Traffic` — API reference pages. +- {ref}`RemoteProxy`, {ref}`C2Array`, {ref}`FsspecNDSource`, {ref}`ByteRangeNDSource`, {ref}`Proxy`, and {ref}`Traffic` — API reference pages. diff --git a/doc/reference/remoteproxy.rst b/doc/reference/remoteproxy.rst index cfef56412..0eaf1f5e5 100644 --- a/doc/reference/remoteproxy.rst +++ b/doc/reference/remoteproxy.rst @@ -36,6 +36,10 @@ source identity and verifies that shape, dtype, chunks, and blocks still match the captured geometry. A replacement with different geometry is rejected; cached disk data is invalidated when the source identity moves. +Ephemeral in-memory caching is available through :attr:`blosc2.CachePolicy.MEMORY`. +Fetched chunks are kept in RAM, bounded by a finite 256 MiB compressed-payload limit by default +(customizable via ``max_cache_bytes``) with automatic LRU eviction. + Persistent caching is available through :attr:`blosc2.CachePolicy.DISK`. Disk caches have a finite 256 MiB compressed-payload bound by default and can take an explicit ``max_cache_bytes`` bound. The bound is enforced after an @@ -51,6 +55,11 @@ returned NumPy array. max_cache_bytes=2 * 2**30, ) +When opening a remote array via :func:`blosc2.open` with ``lazy=True``, a :class:`RemoteProxy` +is always returned: specifying ``cache_dir`` or ``cache_path`` configures it with +:attr:`blosc2.CachePolicy.DISK`, while omitting them configures it with +:attr:`blosc2.CachePolicy.MEMORY`. + By default, :meth:`RemoteProxy.save ` and :meth:`RemoteProxy.to_cframe ` include valid warm chunks. Pass ``include_cache=False`` for a cold carrier without changing the @@ -81,6 +90,8 @@ file. Read-only mode can use warm chunks but does not retain misses: .. automethod:: __init__ .. automethod:: __getitem__ + .. automethod:: fetch + .. automethod:: afetch .. automethod:: get_chunk .. automethod:: aget_chunk .. automethod:: save @@ -93,11 +104,13 @@ file. Read-only mode can use warm chunks but does not retain misses: .. autoattribute:: cparams .. autoattribute:: nbytes .. autoattribute:: info + .. autoattribute:: cache .. autoattribute:: cache_bytes .. autoattribute:: cache_policy .. autoattribute:: max_cache_bytes .. autoattribute:: cache_path .. autoattribute:: cache_status + .. autoattribute:: schunk .. autoattribute:: source .. autoattribute:: traffic .. autoattribute:: urlpath diff --git a/plans/remote-proxy-v3.md b/plans/remote-proxy-v3.md new file mode 100644 index 000000000..bce06908e --- /dev/null +++ b/plans/remote-proxy-v3.md @@ -0,0 +1,88 @@ +# Plan: Unified RemoteProxy with Memory and Disk Caching (v3) + +## Status + +Planned for implementation in Python-Blosc2. + +## Purpose + +Unify all lazy remote dataset access under `RemoteProxy`. Previously, `blosc2.open(url, lazy=True)` returned a legacy `blosc2.Proxy` when opened without disk storage options, and a `blosc2.RemoteProxy` when `cache_dir` or `cache_path` was specified. + +In v3, `blosc2.open(url, lazy=True)` **always** returns a `RemoteProxy`. To achieve this cleanly while maintaining the fast, ephemeral in-memory caching behavior users expect, `CachePolicy.MEMORY` is reinstated as a first-class policy alongside `CachePolicy.DISK` and `CachePolicy.NONE`. + +## Core Decisions + +### 1. Unified Return Type: Always `RemoteProxy` + +`blosc2.open(url, lazy=True)` unconditionally returns an instance of `blosc2.RemoteProxy`: +- When neither `cache_dir` nor `cache_path` is specified: defaults to `CachePolicy.MEMORY`. +- When `cache_dir` or `cache_path` is specified: defaults to `CachePolicy.DISK`. +- When `cache_policy` is passed explicitly: respects the requested policy (`NONE`, `MEMORY`, or `DISK`). + +Users interact with a single, consistent API (`.source`, `.info`, `.traffic`, `.cache_policy`, `.cache_bytes`, `.max_cache_bytes`, `.fetch()`, `.save()`). + +### 2. First-Class Cache Policies + +`blosc2.CachePolicy` provides three explicit retention policies: + +1. **`NONE`**: + - Stateless floating reference. + - Fetches only data required for the current operation and retains no cached chunks. + - `max_cache_bytes` must be `None`. + +2. **`MEMORY`**: + - Ephemeral in-memory cache held in client RAM during the process lifetime. + - Bounded by `max_cache_bytes` (defaults to 256 MiB). + - Automatically applies LRU chunk eviction when retained payload exceeds `max_cache_bytes`. + - Requires no local files or directories (`cache_dir` and `cache_path` must be `None`). + +3. **`DISK`**: + - Persistent carrier cache on disk. + - Bounded by `max_cache_bytes` (defaults to 256 MiB) with LRU chunk eviction. + - Requires `cache_dir` or `cache_path` when creating from a remote URL. + +### 3. Server-Side Protection in Caterva2 + +Caterva2 maintains its strict server-side gate in `caterva2/services/remote_proxy.py`: +- Carriers uploaded to Caterva2 with `cache_policy` other than `"none"` or `"disk"` raise `RemoteProxyDenied` (HTTP 403). +- Caterva2 servers are therefore immune to memory-hogging exploits from crafted carriers, without requiring artificial restrictions on Python-Blosc2 client code. + +### 4. Direct Carrier Export & Deserialization + +- For `CachePolicy.MEMORY`: + - `save()` or `to_cframe()` exports the carrier structure and metadata with `cache_policy: "memory"`. + - Since in-memory chunks are process-local, only the cold descriptor is persisted. + - When reopened via `blosc2.open("saved.b2nd")`, it initializes as an in-memory `RemoteProxy` ready to cache misses in RAM. +- For `CachePolicy.DISK`: + - `save()` and `to_cframe()` include warm cached chunks by default unless `include_cache=False`. +- For `CachePolicy.NONE`: + - Persisted as a cold reference without data. + +### 5. `fetch()` and `afetch()` Support on `RemoteProxy` + +`RemoteProxy` exposes `fetch(item=None)` and `afetch(item=None)`: +- When caching is enabled (`MEMORY` or `DISK`), delegates to the internal cache engine and returns the cache container. +- When `cache_policy` is `NONE`, raises `NotImplementedError`. +- Exposes `cache` property returning the cache container or `None`. + +## Implementation Tasks + +1. **Enum & Policies (`src/blosc2/__init__.py`)**: + - Re-introduce `MEMORY = "memory"` in `CachePolicy`. + +2. **RemoteProxy Implementation (`src/blosc2/remote_proxy.py`)**: + - Update `_normalize_limit` to accept `_POLICY_DEFAULT` (256 MiB) for `CachePolicy.MEMORY`. + - Update `__init__` validation: ensure `cache_dir` and `cache_path` are only used with `DISK`. + - In `_attach_carrier_cache`: instantiate an in-memory `Proxy` with `_max_cache_bytes` when `cache_policy is CachePolicy.MEMORY`. + - In `_export_carrier`: return `_to_b2object_carrier()` for `MEMORY` (no disk carrier to export). + - In `_from_payload`: decode `CachePolicy.MEMORY` with positive `max_cache_bytes`, instantiating with in-memory cache. + - Add `fetch(item=None)`, `async afetch(item=None)`, and `@property def cache`. + +3. **Open Integration (`src/blosc2/schunk.py`)**: + - In `_remote_proxy_options`: when `lazy=True`, always return options (defaulting to `DISK` if disk options are present, else `MEMORY`). + - In `_open_fsspec_url` and `_open_c2_urlpath`: always return `RemoteProxy` for `lazy=True`. + +4. **Documentation & Tests**: + - Update `doc/guides/remote_arrays.md` and docstrings to describe `CachePolicy.MEMORY`. + - Update tests in `tests/test_remote_proxy.py`, `tests/test_fsspec.py`, and `tests/ndarray/test_c2array_blocks.py`. + - Run full pytest suite and lint checks. diff --git a/src/blosc2/__init__.py b/src/blosc2/__init__.py index 8065079e9..5f4023811 100644 --- a/src/blosc2/__init__.py +++ b/src/blosc2/__init__.py @@ -234,6 +234,7 @@ class CachePolicy(Enum): """Retention policy for data read through a remote proxy.""" NONE = "none" + MEMORY = "memory" DISK = "disk" diff --git a/src/blosc2/remote_proxy.py b/src/blosc2/remote_proxy.py index 863c607f4..72f079d80 100644 --- a/src/blosc2/remote_proxy.py +++ b/src/blosc2/remote_proxy.py @@ -92,7 +92,9 @@ class RemoteProxy(blosc2.Operand): """A persistable, optionally self-caching reference to a remote array. With :attr:`CachePolicy.DISK`, the persisted B2ND carrier is itself the - bounded cache. With :attr:`CachePolicy.NONE`, reads retain no data. + bounded cache. With :attr:`CachePolicy.MEMORY`, chunks are retained in + process memory up to a bounded size. With :attr:`CachePolicy.NONE`, reads + retain no data. Parameters ---------- @@ -100,7 +102,8 @@ class RemoteProxy(blosc2.Operand): A single-file B2ND URL opened through fsspec, or a Caterva2 array reference. cache_policy: CachePolicy - ``NONE`` retains no array data. ``DISK`` retains compressed chunks in + ``NONE`` retains no array data. ``MEMORY`` retains compressed chunks + in client process memory. ``DISK`` retains compressed chunks in the RemoteProxy carrier at ``cache_path`` or under ``cache_dir``. cache_path: str or path-like, optional Exact persistent cache filename. Only valid with ``DISK`` and mutually @@ -110,7 +113,7 @@ class RemoteProxy(blosc2.Operand): Only valid with ``DISK``. max_cache_bytes: int or None, optional Post-operation compressed-payload bound. It defaults to 256 MiB for - ``DISK`` and must always be finite. It is not applicable to ``NONE``. + ``DISK`` and ``MEMORY`` and must always be finite. It is not applicable to ``NONE``. max_concurrency: int, optional Maximum number of independent remote fetches in flight. """ @@ -156,6 +159,8 @@ def __init__( if self._carrier is None: self._carrier, self._cache_status = self._open_or_create_carrier(cache_dir, cache_path) self._attach_carrier_cache() + elif cache_policy is blosc2.CachePolicy.MEMORY: + self._attach_carrier_cache() def _runtime_source(self, original): """Keep credentials in live process state, outside the descriptor.""" @@ -185,32 +190,50 @@ def _open_or_create_carrier(self, cache_dir, cache_path): path = blosc2.schunk.fsspec_cache_path(self._source_identity(), cache_dir, ".b2nd") if os.path.exists(path): kwargs = {"dparams": blosc2.DParams(nthreads=1)} - carrier = blosc2.blosc2_ext.open(path, "a", 0, **kwargs) - payload = carrier.schunk.vlmeta.get("b2o") - if payload != self._payload(): - raise ValueError(f"the RemoteProxy carrier at {path} has a different specification") - stored = carrier.schunk.vlmeta.get("proxy-stamp") - current = getattr(self.src, "stamp", None) - status = "invalidated/rebuilt" if stored is not None and current is not None and stored != current else "reused" - return carrier, status + try: + carrier = blosc2.blosc2_ext.open(path, "a", 0, **kwargs) + payload = carrier.schunk.vlmeta.get("b2o") + if payload != self._payload(): + raise ValueError(f"the RemoteProxy carrier at {path} has a different specification") + stored = carrier.schunk.vlmeta.get("proxy-stamp") + current = getattr(self.src, "stamp", None) + status = ( + "invalidated/rebuilt" + if stored is not None and current is not None and stored != current + else "reused" + ) + return carrier, status + except Exception as exc: + if isinstance(exc, ValueError) and "has a different specification" in str(exc): + raise + blosc2.remove_urlpath(path) carrier = self._to_b2object_carrier(urlpath=path, contiguous=True, mode="w") return carrier, "created" def _attach_carrier_cache(self): - if self._carrier is None or self.cache_policy is not blosc2.CachePolicy.DISK: - self._proxy = None - return - if getattr(self.src, "stamp", None) is None: - # Without a stable validator, cached bytes cannot be trusted across - # independent opens. Reads still work, but misses are not retained. + if self.cache_policy is blosc2.CachePolicy.DISK: + if self._carrier is None: + self._proxy = None + return + if getattr(self.src, "stamp", None) is None: + # Without a stable validator, cached bytes cannot be trusted across + # independent opens. Reads still work, but misses are not retained. + self._proxy = None + return + self._proxy = blosc2.Proxy( + self.src, + _cache=self._carrier, + _refresh_source=False, + _max_cache_bytes=self._cache_limit, + ) + elif self.cache_policy is blosc2.CachePolicy.MEMORY: + self._proxy = blosc2.Proxy( + self.src, + _refresh_source=False, + _max_cache_bytes=self._cache_limit, + ) + else: self._proxy = None - return - self._proxy = blosc2.Proxy( - self.src, - _cache=self._carrier, - _refresh_source=False, - _max_cache_bytes=self._cache_limit, - ) @staticmethod def _open_source(urlpath, max_concurrency, *, traffic=None): @@ -280,10 +303,7 @@ def _prepare_read(self): if refresh is None: refresh = getattr(self.src, "refresh_stamp", None) if refresh is not None: - if isinstance(self.src, blosc2.C2Array): - refresh(force=True) - else: - refresh() + refresh() self._validate_geometry(self._expected_geometry) current_stamp = getattr(self.src, "stamp", None) @@ -376,6 +396,22 @@ def source(self) -> dict: """A copy of the credential-free source descriptor.""" return dict(self._source) + @property + def schunk(self): + """The underlying carrier's or cache's :class:`SChunk`, or None if unattached.""" + if self._carrier is not None: + return getattr(self._carrier, "schunk", self._carrier) + if self._proxy is not None: + return self._proxy.schunk + return None + + @property + def cache(self): + """The local container used as cache, or None if caching is disabled.""" + if self._proxy is not None: + return getattr(self._proxy, "cache", getattr(self._proxy, "_cache", None)) + return self._carrier + @property def urlpath(self): """The remote fsspec URL or credential-free Caterva2 URLPath.""" @@ -436,6 +472,32 @@ def _chunk_slice(self, nchunk: int): for coord, chunk, size in zip(coords, self.chunks, self.shape, strict=True) ) + def fetch(self, item=(), max_concurrency: int | None = None): + """Fetch remote data into the cache container. + + Only valid when caching is enabled (:attr:`CachePolicy.DISK` or + :attr:`CachePolicy.MEMORY`). + """ + backend = self._prepare_read() + if not isinstance(backend, blosc2.Proxy): + raise NotImplementedError("fetch requires CachePolicy.DISK or CachePolicy.MEMORY") + result = backend.fetch(item, max_concurrency=max_concurrency) + backend._enforce_cache_limit(item) + return result + + async def afetch(self, item=(), max_concurrency: int | None = None): + """Asynchronously fetch remote data into the cache container. + + Only valid when caching is enabled (:attr:`CachePolicy.DISK` or + :attr:`CachePolicy.MEMORY`). + """ + backend = self._prepare_read() + if not isinstance(backend, blosc2.Proxy): + raise NotImplementedError("afetch requires CachePolicy.DISK or CachePolicy.MEMORY") + result = await backend.afetch(item, max_concurrency=max_concurrency) + backend._enforce_cache_limit(item) + return result + def get_chunk(self, nchunk: int) -> bytes: backend = self._prepare_read() if not isinstance(backend, blosc2.Proxy): @@ -522,7 +584,7 @@ def _from_payload(cls, payload, carrier): if limit is not None: raise ValueError("persisted NONE RemoteProxy cannot have max_cache_bytes") elif isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0: - raise ValueError("persisted DISK RemoteProxy requires positive max_cache_bytes") + raise ValueError(f"persisted {policy.name} RemoteProxy requires positive max_cache_bytes") source = payload.get("source") if not isinstance(source, dict) or source.get("version") != 1: raise ValueError("unsupported RemoteProxy source descriptor") @@ -545,7 +607,8 @@ def _from_payload(cls, payload, carrier): raise ValueError(f"unsupported RemoteProxy source kind: {source_kind!r}") expected = (carrier.shape, carrier.dtype, carrier.chunks, carrier.blocks) kwargs = {} if policy is blosc2.CachePolicy.NONE else {"max_cache_bytes": limit} - obj = cls(urlpath, cache_policy=policy, _carrier=carrier, **kwargs) + carrier_arg = carrier if policy is blosc2.CachePolicy.DISK else None + obj = cls(urlpath, cache_policy=policy, _carrier=carrier_arg, **kwargs) obj._validate_geometry(expected) return obj diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 85fbb902a..cb027b3a4 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -2010,16 +2010,21 @@ def _remote_cache_options(kwargs: dict) -> tuple[str | pathlib.Path | None, str return (cache_dir if cache_dir is not None else cache_storage), cache_path -def _remote_proxy_options(kwargs, cache_dir, cache_path, max_concurrency): +def _remote_proxy_options(kwargs, cache_dir, cache_path, max_concurrency, *, lazy=False): """Return explicit RemoteProxy options, or None for the legacy lazy Proxy path.""" policy_present = "cache_policy" in kwargs limit_present = "max_cache_bytes" in kwargs - if not policy_present and not limit_present: + if not lazy and not policy_present and not limit_present: return None policy = kwargs.pop("cache_policy", None) limit = kwargs.pop("max_cache_bytes", None) if not policy_present: - policy = blosc2.CachePolicy.DISK if cache_dir is not None or cache_path is not None else blosc2.CachePolicy.NONE + if cache_dir is not None or cache_path is not None: + policy = blosc2.CachePolicy.DISK + elif lazy: + policy = blosc2.CachePolicy.MEMORY + else: + policy = blosc2.CachePolicy.NONE options = { "cache_policy": policy, "cache_dir": cache_dir, @@ -2105,7 +2110,7 @@ def _open_c2_urlpath(urlpath: blosc2.URLPath, mode: str, offset: int, kwargs: di cache_dir, cache_path = _remote_cache_options(kwargs) max_concurrency = kwargs.pop("max_concurrency", None) lazy = kwargs.pop("lazy", False) - remote_proxy_options = _remote_proxy_options(kwargs, cache_dir, cache_path, max_concurrency) + remote_proxy_options = _remote_proxy_options(kwargs, cache_dir, cache_path, max_concurrency, lazy=lazy) requested = [key for key, value in kwargs.items() if value is not None] if requested: raise NotImplementedError(f"{', '.join(requested)} is not supported for Caterva2 arrays") @@ -2162,7 +2167,7 @@ def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): cache_dir, cache_path = _remote_cache_options(kwargs) max_concurrency = kwargs.pop("max_concurrency", None) lazy = kwargs.pop("lazy", False) - remote_proxy_options = _remote_proxy_options(kwargs, cache_dir, cache_path, max_concurrency) + remote_proxy_options = _remote_proxy_options(kwargs, cache_dir, cache_path, max_concurrency, lazy=lazy) if lazy: if offset != 0: raise NotImplementedError("offset is not supported with lazy=True") @@ -2255,17 +2260,13 @@ def open( (e.g. in a file containing several such objects). kwargs: dict, optional lazy: bool, optional - For an fsspec URL, return a :ref:`Proxy` over a standalone, - contiguous :ref:`NDArray` frame and read the byte ranges a slice - touches. For a Caterva2 :ref:`URLPath`, return a :ref:`Proxy` over - one array-like dataset; stored ``.b2nd`` arrays use byte ranges when - available, while HDF5 datasets, ``.b2z`` leaves, and computed arrays - fall back to semantic chunk requests. Neither form opens a whole - remote store hierarchy. A slice landing in a small part of a large + For an fsspec URL or a Caterva2 :ref:`URLPath`, return a :ref:`RemoteProxy` over + the remote dataset and read the byte ranges a slice touches. Neither form opens + a whole remote store hierarchy. A slice landing in a small part of a large chunk costs only the *blocks* it touches when ranges are available; chunks small enough to be one cheap request are still fetched whole. - What arrives is kept in memory, under ``cache_dir``, or at the exact - ``cache_path`` when either is given. + What arrives is kept in memory (defaulting to :attr:`CachePolicy.MEMORY`), + under ``cache_dir``, or at the exact ``cache_path`` (as :attr:`CachePolicy.DISK`). max_concurrency: int, optional Only with ``lazy``: how many fetches to run at once, in a thread pool. A slice against an object store is almost entirely round-trip @@ -2274,29 +2275,28 @@ def open( hide, where the pool costs about 10 microseconds per chunk and saves nothing. cache_dir: str | pathlib.Path, optional - For fsspec URLs and lazy Caterva2 :ref:`URLPath` objects, a directory holding this container's local - copy — the whole thing, or just the chunks and blocks ``lazy`` has - fetched so far. Either way a later run starts from what is already - there, and the copy is discarded when the remote no longer matches - it. There is no default on purpose, so nothing writes to a disk you - did not name. + For fsspec URLs and lazy Caterva2 :ref:`URLPath` objects, a directory holding this container's + local copy — either the whole thing, or just the chunks and blocks ``lazy`` has fetched so far + (as a persistent :ref:`RemoteProxy` with :attr:`CachePolicy.DISK`). Either way a later run + starts from what is already there, and the copy is discarded when the remote no longer matches + it. There is no default on purpose, so nothing writes to a disk you did not name. cache_path: str | pathlib.Path, optional With ``lazy=True``, the exact file to use for the remote array's - persistent proxy cache. Mutually exclusive with ``cache_dir``. + persistent :ref:`RemoteProxy` cache (:attr:`CachePolicy.DISK`). Mutually exclusive with + ``cache_dir``. cache_storage: str | pathlib.Path, optional Deprecated alias for ``cache_dir``. Mutually exclusive with ``cache_dir`` and ``cache_path``. cache_policy: CachePolicy, optional With ``lazy=True`` on a remote source, return a :ref:`RemoteProxy` - using the requested retention policy. ``NONE`` retains no data - and ``DISK`` retains compressed chunks in its carrier and requires - ``cache_dir`` or ``cache_path`` when creating one from a remote URL. - When omitted, the existing :ref:`Proxy` behavior is preserved. + using the requested retention policy (``NONE``, ``MEMORY``, or ``DISK``). + When omitted, passing ``cache_dir`` or ``cache_path`` defaults to + ``CachePolicy.DISK``, while omitting them defaults to ``CachePolicy.MEMORY``. max_cache_bytes: int, optional - With ``lazy=True``, enable a :ref:`RemoteProxy` and bound retained - compressed cache payload after each operation. ``DISK`` defaults to - 256 MiB and always has a finite bound. This does not bound the - current operation's working set or result. + With ``lazy=True``, bound retained compressed cache payload for a + :ref:`RemoteProxy` after each operation. Defaults to 256 MiB for both + ``DISK`` and ``MEMORY``, and always has a finite bound. This does not bound + the current operation's working set or result. mmap_mode: str, optional If set, the file will be memory-mapped instead of using the default I/O functions and the `mode` argument will be ignored. @@ -2338,10 +2338,9 @@ def open( * If :paramref:`urlpath` is a :ref:`URLPath` instance, :paramref:`mode` must be 'r' and :paramref:`offset` must be 0. Without ``lazy=True`` it - returns a :ref:`C2Array`; with ``lazy=True`` it returns a :ref:`Proxy`, - optionally persisted under ``cache_dir`` or at ``cache_path``. Supplying - ``cache_policy`` or ``max_cache_bytes`` explicitly selects a - :ref:`RemoteProxy` instead. + returns a :ref:`C2Array`. With ``lazy=True``, it returns a :ref:`RemoteProxy` + (defaulting to ``CachePolicy.DISK`` when ``cache_dir`` or ``cache_path`` is + provided, and ``CachePolicy.MEMORY`` otherwise). Authenticated users sharing a machine must use separate caches. * fsspec URLs need the ``fsspec`` extra (``pip install "blosc2[fsspec]"``) and @@ -2351,8 +2350,9 @@ def open( A plain URL read rebuilds the object from a cframe held in memory, so it covers ``.b2nd``, ``.b2f`` and ``.b2e`` only -- a ``.b2z`` store is a zip archive rather than a cframe, and needs ``cache_dir`` like the directory - formats do. ``cache_dir`` and ``lazy`` above lift that, each - in its own way. + formats do. With ``lazy=True``, it returns a :ref:`RemoteProxy` (using + ``CachePolicy.DISK`` with ``cache_dir`` or ``cache_path``, and + ``CachePolicy.MEMORY`` otherwise). * Persistent data handling follows a no-hidden-writes rule except for an explicitly self-caching :ref:`RemoteProxy`: @@ -2362,9 +2362,9 @@ def open( its own carrier. Other execution caches are not serialized implicitly. - ``mode='w'`` persists explicit mutations requested by the caller. - * If the original object saved in :paramref:`urlpath` is a :ref:`Proxy`, - this function reconstructs sources backed by a persistent local - :ref:`SChunk` or :ref:`NDArray`, an fsspec URL, or a remote + * If the original object saved in :paramref:`urlpath` is a :ref:`Proxy` + or a :ref:`RemoteProxy`, this function reconstructs sources backed by a + persistent local :ref:`SChunk` or :ref:`NDArray`, an fsspec URL, or a remote :ref:`C2Array`. Custom proxy sources must be recreated explicitly because their Python class and runtime state are not stored in the cache. diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index 8511f5f20..ff870a57d 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -333,16 +333,17 @@ def test_open_urlpath_lazy_memory_cache(server, any_chunk_wants_blocks): proxy = blosc2.open(urlpath, lazy=True, max_concurrency=3) assert [endpoint for endpoint, _, _ in srv.log] == ["info"] - assert isinstance(proxy, blosc2.Proxy) + assert isinstance(proxy, blosc2.RemoteProxy) + assert proxy.cache_policy is blosc2.CachePolicy.MEMORY assert isinstance(proxy.src, blosc2.C2Array) assert proxy.src.max_concurrency == 3 - assert proxy.urlpath is None + assert proxy.cache_path is None result = proxy[0:5, 0:10] served = len(srv.log) assert np.array_equal(result, data[0:5, 0:10]) assert np.array_equal(proxy[0:5, 0:10], result) - assert len(srv.log) == served + assert [endpoint for endpoint, _, _ in srv.log[served:]] == ["info"] def test_open_urlpath_lazy_persistent_cache(tmp_path, server, any_chunk_wants_blocks): @@ -353,15 +354,17 @@ def test_open_urlpath_lazy_persistent_cache(tmp_path, server, any_chunk_wants_bl srv.log.clear() proxy = blosc2.open(urlpath, lazy=True, cache_dir=cache_dir) + assert isinstance(proxy, blosc2.RemoteProxy) assert [endpoint for endpoint, _, _ in srv.log] == ["info"] assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) del proxy srv.log.clear() proxy = blosc2.open(urlpath, lazy=True, cache_dir=cache_dir) + assert isinstance(proxy, blosc2.RemoteProxy) assert [endpoint for endpoint, _, _ in srv.log] == ["info"] assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) - assert [endpoint for endpoint, _, _ in srv.log] == ["info"] + assert [endpoint for endpoint, _, _ in srv.log] == ["info", "info"] assert len(list(cache_dir.glob("*.b2nd"))) == 1 @@ -372,19 +375,20 @@ def test_open_urlpath_lazy_exact_cache_path(tmp_path, server, any_chunk_wants_bl cache_path = tmp_path / "chosen.b2nd" proxy = blosc2.open(urlpath, lazy=True, cache_path=cache_path) + assert isinstance(proxy, blosc2.RemoteProxy) assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) - assert proxy.urlpath == str(cache_path) - assert proxy.schunk.meta["proxy-source"]["source_kind"] == "caterva2" + assert proxy.cache_path == str(cache_path) + assert proxy.source["kind"] == "caterva2" del proxy srv.log.clear() proxy = blosc2.open(cache_path, mode="a") - assert isinstance(proxy, blosc2.Proxy) + assert isinstance(proxy, blosc2.RemoteProxy) assert isinstance(proxy.src, blosc2.C2Array) assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) - assert [endpoint for endpoint, _, _ in srv.log] == ["info"] + assert [endpoint for endpoint, _, _ in srv.log] == ["info", "info"] assert np.array_equal(proxy[100:105, 0:10], data[100:105, 0:10]) - assert len(srv.log) > 1 + assert any(endpoint != "info" for endpoint, _, _ in srv.log) def test_open_urlpath_lazy_uses_c2context_without_persisting_token(tmp_path, server): @@ -397,7 +401,8 @@ def test_open_urlpath_lazy_uses_c2context_without_persisting_token(tmp_path, ser with blosc2.c2context(urlbase=array.urlbase, auth_token=token): proxy = blosc2.open(urlpath, lazy=True, cache_dir=cache_dir) assert np.array_equal(proxy[0:5, 0:5], data[0:5, 0:5]) - assert proxy.schunk.meta["proxy-source"]["urlpath"][2] is None + assert proxy.source["kind"] == "caterva2" + assert "auth_token" not in proxy.source cache = next(cache_dir.glob("*.b2nd")) reopened = blosc2.open(cache, mode="a") diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index 9be98273e..ad6df9da6 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -514,7 +514,7 @@ def test_lazy_with_exact_cache_path(tmp_path): p = blosc2.open(url, lazy=True, cache_path=cache_path) assert np.array_equal(p[0:100], a[0:100]) - assert p.urlpath == str(cache_path) + assert p.cache_path == str(cache_path) assert cache_path.is_file() q = blosc2.open(url, lazy=True, cache_path=cache_path) @@ -527,9 +527,35 @@ def test_exact_cache_path_reopens_as_lazy_fsspec_proxy(tmp_path, monkeypatch): cache_path = tmp_path / "independent.b2nd" p = blosc2.open(url, lazy=True, cache_path=cache_path) assert np.array_equal(p[0:100], a[0:100]) - source_meta = p.schunk.meta["proxy-source"] - assert source_meta["source_kind"] == "fsspec" - assert source_meta["urlpath"] == url + assert p.source["kind"] == "fsspec" + assert p.source["urlpath"] == url + del p + + fetched = [] + orig = blosc2.FsspecNDSource.get_chunk + monkeypatch.setattr( + blosc2.FsspecNDSource, + "get_chunk", + lambda self, nchunk: (fetched.append(nchunk), orig(self, nchunk))[1], + ) + + reopened = blosc2.open(cache_path, mode="a") + assert isinstance(reopened, blosc2.RemoteProxy) + assert isinstance(reopened.src, blosc2.FsspecNDSource) + assert np.array_equal(reopened[0:100], a[0:100]) + assert fetched == [] + assert np.array_equal(reopened[500:600], a[500:600]) + assert fetched == [5] + + +def test_legacy_proxy_cache_reopens_as_proxy(tmp_path, monkeypatch): + a = blosc2.arange(0, 1000, dtype="i4", chunks=(100,)) + url = _put("legacycache.b2nd", a) + cache_path = tmp_path / "legacy.b2nd" + src = blosc2.FsspecNDSource(url) + p = blosc2.Proxy(src, urlpath=cache_path, mode="a") + assert np.array_equal(p[0:100], a[0:100]) + assert "proxy-source" in p.schunk.meta del p fetched = [] @@ -613,7 +639,7 @@ def test_http_url_is_read_through_fsspec(tmp_path): requests.clear() lazy = blosc2.open(f"{urlbase}/big.b2nd", lazy=True, cache_dir=str(tmp_path / "cs")) - assert isinstance(lazy, blosc2.Proxy) + assert isinstance(lazy, blosc2.RemoteProxy) assert isinstance(lazy.src, blosc2.FsspecNDSource) assert ":etag:" in lazy.src.stamp assert requests == ["bytes=0-8191"] # metadata and identity, one round trip diff --git a/tests/test_remote_proxy.py b/tests/test_remote_proxy.py index 745c5682a..b623d2ace 100644 --- a/tests/test_remote_proxy.py +++ b/tests/test_remote_proxy.py @@ -71,30 +71,38 @@ def test_remote_proxy_array_operand_interface(): assert url in dict(expression.info_items)["operands"].values() -def test_open_selects_remote_proxy_only_for_explicit_policy(tmp_path): +def test_open_lazy_selects_remote_proxy(tmp_path): url, data = _remote_array("open-policy.b2nd") - legacy = blosc2.open(url, lazy=True) - assert isinstance(legacy, blosc2.Proxy) - assert not isinstance(legacy, blosc2.RemoteProxy) + mem = blosc2.open(url, lazy=True) + assert isinstance(mem, blosc2.RemoteProxy) + assert mem.cache_policy is blosc2.CachePolicy.MEMORY + assert mem.cache_path is None + assert mem.max_cache_bytes == 256 * 2**20 + np.testing.assert_array_equal(mem[:100_000], data[:100_000]) none = blosc2.open(url, lazy=True, cache_policy=blosc2.CachePolicy.NONE) assert isinstance(none, blosc2.RemoteProxy) assert none.cache_policy is blosc2.CachePolicy.NONE np.testing.assert_array_equal(none[:100_000], data[:100_000]) - with pytest.raises(ValueError, match="not applicable"): - blosc2.open(url, lazy=True, max_cache_bytes=120_000) + mem_bounded = blosc2.open(url, lazy=True, max_cache_bytes=120_000) + assert isinstance(mem_bounded, blosc2.RemoteProxy) + assert mem_bounded.cache_policy is blosc2.CachePolicy.MEMORY + assert mem_bounded.max_cache_bytes == 120_000 disk = blosc2.open( url, lazy=True, - cache_policy=blosc2.CachePolicy.DISK, cache_path=tmp_path / "open-cache.b2nd", max_cache_bytes=120_000, ) assert isinstance(disk, blosc2.RemoteProxy) assert disk.cache_policy is blosc2.CachePolicy.DISK + assert disk.max_cache_bytes == 120_000 + + with pytest.raises(NotImplementedError, match="require lazy=True"): + blosc2.open(url, lazy=False, max_cache_bytes=120_000) def test_none_does_not_retain_remote_data(): @@ -260,16 +268,27 @@ def test_runtime_cache_is_invalidated_after_same_geometry_replacement(tmp_path): assert proxy.traffic is traffic -def test_reference_rejects_runtime_cache_policy_in_payload(): +def test_reference_rejects_unknown_cache_policy_in_payload(): url, _ = _remote_array("bad-persisted-policy.b2nd", nchunks=1, chunk_size=100) carrier = blosc2.ndarray_from_cframe(blosc2.RemoteProxy(url).to_cframe()) payload = dict(carrier.schunk.vlmeta["b2o"]) - payload["cache_policy"] = "memory" + payload["cache_policy"] = "unknown_policy" with pytest.raises(ValueError, match="unsupported cache policy"): decode_b2object_payload(payload, carrier=carrier) +def test_reference_rejects_memory_policy_without_positive_limit_in_payload(): + url, _ = _remote_array("bad-memory-policy.b2nd", nchunks=1, chunk_size=100) + carrier = blosc2.ndarray_from_cframe(blosc2.RemoteProxy(url).to_cframe()) + payload = dict(carrier.schunk.vlmeta["b2o"]) + payload["cache_policy"] = "memory" + payload["max_cache_bytes"] = None + + with pytest.raises(ValueError, match="persisted MEMORY RemoteProxy requires positive max_cache_bytes"): + decode_b2object_payload(payload, carrier=carrier) + + @pytest.mark.parametrize( ("field", "value", "error"), [ @@ -521,3 +540,72 @@ def test_persistence_rejects_credentials_and_chained_urls(url): def test_fsspec_refs_reject_credentials(url): with pytest.raises(ValueError): blosc2.Ref.fsspec_ref(url) + + +def test_memory_cache_eviction_and_retention(): + url, data = _remote_array("mem-evict.b2nd", nchunks=5, chunk_size=20_000) + proxy = blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.MEMORY, + max_cache_bytes=50_000, + ) + assert proxy.cache_policy is blosc2.CachePolicy.MEMORY + assert proxy.max_cache_bytes == 50_000 + assert proxy.cache_path is None + assert proxy.cache is not None + + proxy.traffic.reset() + np.testing.assert_array_equal(proxy[:20_000], data[:20_000]) + assert proxy.traffic.requests > 0 + + proxy.traffic.reset() + np.testing.assert_array_equal(proxy[:20_000], data[:20_000]) + assert proxy.traffic.requests == 0 + + np.testing.assert_array_equal(proxy[:], data) + assert proxy.cache_bytes <= 50_000 + + +def test_memory_cache_fetch_and_afetch(): + url, _ = _remote_array("mem-fetch.b2nd", nchunks=3, chunk_size=10_000) + proxy = blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.MEMORY, + ) + cached_container = proxy.fetch(slice(0, 10_000)) + assert cached_container is proxy.cache + assert proxy.cache_bytes > 0 + + async_container = asyncio.run(proxy.afetch(slice(10_000, 20_000))) + assert async_container is proxy.cache + + +def test_memory_proxy_save_and_reopen(tmp_path): + url, data = _remote_array("mem-save.b2nd", nchunks=2, chunk_size=10_000) + proxy = blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.MEMORY, + max_cache_bytes=100_000, + ) + save_path = tmp_path / "saved_memory_proxy.b2nd" + proxy.save(save_path) + + reopened = blosc2.open(save_path, mode="r") + assert isinstance(reopened, blosc2.RemoteProxy) + assert reopened.cache_policy is blosc2.CachePolicy.MEMORY + assert reopened.max_cache_bytes == 100_000 + assert reopened.cache_path is None + np.testing.assert_array_equal(reopened[:], data) + + +def test_remote_proxy_fetch_rejects_none_policy(): + url, _ = _remote_array("none-fetch.b2nd", nchunks=1, chunk_size=100) + proxy = blosc2.RemoteProxy(url, cache_policy=blosc2.CachePolicy.NONE) + with pytest.raises( + NotImplementedError, match=r"fetch requires CachePolicy\.DISK or CachePolicy\.MEMORY" + ): + proxy.fetch() + with pytest.raises( + NotImplementedError, match=r"afetch requires CachePolicy\.DISK or CachePolicy\.MEMORY" + ): + asyncio.run(proxy.afetch()) From 8d4ae0d35171371113635b41ed785bc0a2414411 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 5 Sep 2026 14:37:44 +0200 Subject: [PATCH 15/89] Support unbounded cache for DISK remote proxies and update docs --- doc/guides/remote_arrays.md | 35 +++- doc/reference/remoteproxy.rst | 23 ++- plans/remote-proxy-v2.md | 2 + plans/remote-proxy-v3.md | 22 ++- plans/remote-proxy-v4.md | 361 ++++++++++++++++++++++++++++++++++ plans/remote-proxy.md | 2 +- src/blosc2/remote_proxy.py | 182 +++++++++++------ src/blosc2/schunk.py | 7 +- tests/test_fsspec.py | 11 +- tests/test_remote_proxy.py | 237 +++++++++++++++++++++- 10 files changed, 796 insertions(+), 86 deletions(-) create mode 100644 plans/remote-proxy-v4.md diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index 0b3daaa3b..33cc8789e 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -53,7 +53,7 @@ When opened with `lazy=True`, both routes return a {ref}`RemoteProxy`, providing ## Cache policies and memory management -Every lazy open uses a cache policy. By default, fetched data is cached in memory and bounded to prevent excessive RAM consumption. +Every lazy open uses a cache policy. By default, fetched data is cached in memory with a bound on retained compressed payload. ### In-memory caching (`CachePolicy.MEMORY` — Default) @@ -65,7 +65,7 @@ a[10:12, 500:600] # fetched and cached in RAM a[10:12, 500:600] # served from memory cache (no network traffic) ``` -To prevent memory leaks or out-of-memory errors on massive datasets, in-memory caches are bounded by `max_cache_bytes` (defaults to 256 MiB) with automatic LRU eviction: +In-memory caches use `max_cache_bytes` (defaults to 256 MiB) with automatic LRU eviction after operations, including failed fetches. This is not a peak RAM limit: metadata, in-flight transfers, decompression buffers, and results are excluded. Large operations can exceed it substantially. ```python # Custom in-memory limit (e.g. 512 MiB): @@ -94,7 +94,7 @@ Use `cache_path` instead when the cache should have an exact filename: a = blosc2.open(url, lazy=True, cache_path="big-cache.b2nd") ``` -In both cases, the cache is an ordinary `.b2nd` carrier file managed as a {ref}`RemoteProxy` with {attr}`CachePolicy.DISK `. It starts small and retains compressed chunks up to a finite bound (256 MiB by default, or customized via `max_cache_bytes`). `cache_dir` and `cache_path` are mutually exclusive. +In both cases, the cache is an ordinary `.b2nd` carrier file managed as a {ref}`RemoteProxy` with {attr}`CachePolicy.DISK `. It starts small and retains compressed chunks up to a finite bound (256 MiB by default, or customized via `max_cache_bytes`; pass `max_cache_bytes=None` for an unbounded disk cache that never evicts). `cache_dir` and `cache_path` are mutually exclusive. Authenticated Caterva2 caches must be private to one user. Reopen them under an equivalent authenticated {func}`blosc2.c2context`; do not share a cache directory between users. @@ -119,9 +119,9 @@ Each read pulls only the bytes required for the slice and retains no cache paylo Blosc2 arrays are compressed in chunks, which are divided into smaller blocks. For a small slice, fetching only its blocks can avoid transferring most of a large chunk. -![A proxy fetches missing regions from the remote array into its local cache. The fetch method returns the cache container, while indexing returns only the requested values.](../tutorials/images/remote_proxy.png) +![A proxy fetches missing regions from the remote array into its local cache. Indexing returns the requested values.](../tutorials/images/remote_proxy.png) -Purple regions are cached; red regions are still remote. The grid is schematic: where byte ranges are available, the fetched regions can be blocks within a chunk. `fetch()` fills and returns the cache container, whereas indexing returns only the requested values. +Purple regions are cached; red regions are still remote. The grid is schematic: where byte ranges are available, the fetched regions can be blocks within a chunk. `fetch()` warms the cache and returns the proxy, whereas indexing returns the requested values. The proxy chooses blocks or whole chunks automatically. It fetches a whole chunk when most of its blocks are needed or when the source cannot expose block ranges, as with computed Caterva2 datasets. Independent reads overlap, with up to eight concurrent requests by default; use `max_concurrency=1` when concurrency does not help. @@ -133,13 +133,17 @@ You can warm the cache proactively using `fetch()` or `afetch()`: ```python # Synchronously pre-fetch a region into the cache: -cached_container = a.fetch(slice(0, 10_000)) +a.fetch(slice(0, 10_000)) # Or asynchronously in an async event loop: -cached_container = await a.afetch(slice(10_000, 20_000)) +await a.afetch(slice(10_000, 20_000)) ``` -The underlying local cache container (an `NDArray`) can also be accessed directly via `a.cache`. +Both methods return `a`. Prefetched data may be evicted to satisfy the cache limit; later indexing fetches it again as needed. Use `a.materialize(item)` for an independent, complete `NDArray`. Its output and temporary buffer are outside the cache limit. + +`a.cache` exposes the underlying cache for inspection. It may contain missing or evicted chunks and must not be treated as a complete array or mutated by callers. + +Operations on a single handle are serialized through fetching, result assembly, eviction, and export. Async methods run synchronous operations in a worker thread; cancelling the await does not stop an already running fetch. Separate handles or processes sharing a disk carrier require external locking. ## Measure network traffic @@ -188,7 +192,20 @@ remote = blosc2.RemoteProxy( ) ``` -Saving and serializing preserve valid warm chunks by default. Pass `include_cache=False` to `save()` or `to_cframe()` to export a cold reference copy without mutating the warm carrier. +Disk proxies preserve warm chunks by default. Memory proxies always export cold carriers. Pass `include_cache=False` to export a cold copy without mutating the warm carrier. + +An explicit export policy produces a cold carrier with that policy, leaving the live proxy unchanged: + +```python +a = blosc2.open("https://datasets.example.org/big.b2nd", lazy=True) +a.save("portable.b2nd", cache_policy=blosc2.CachePolicy.NONE) +``` + +Caterva2 servers accept persisted `MEMORY` carriers (under opt-in policy) but execute them without retained caching (identical to `NONE`), repeatedly fetching required regions from the remote source while preserving the requested limit for downloads; older Caterva2 servers reject `MEMORY` resolution entirely. Use `DISK` if you want Caterva2 to retain compressed chunks on the server within its configured quota. Policy-changing or cold exports must use a different destination from the live disk cache. + +Memory-only access accepts runtime URLs such as signed URLs and fsspec chains. Such URLs cannot be exported or obtained as portable `.source` descriptors; disk caching and reference-only construction continue to require persistable URLs. + +Existing files are never automatically deleted when opening or validating a cache fails. Open legacy caches directly with `blosc2.open(cache_path)`, or choose a new cache path for `RemoteProxy`. Preserve or explicitly remove corrupt files before recreating their cache. ### Reopen a cache file independently diff --git a/doc/reference/remoteproxy.rst b/doc/reference/remoteproxy.rst index 0eaf1f5e5..3cbe94606 100644 --- a/doc/reference/remoteproxy.rst +++ b/doc/reference/remoteproxy.rst @@ -42,7 +42,8 @@ Fetched chunks are kept in RAM, bounded by a finite 256 MiB compressed-payload l Persistent caching is available through :attr:`blosc2.CachePolicy.DISK`. Disk caches have a finite 256 MiB compressed-payload bound by default and can -take an explicit ``max_cache_bytes`` bound. The bound is enforced after an +take an explicit ``max_cache_bytes`` bound, or ``max_cache_bytes=None`` for an +unbounded cache that never evicts chunks. When bounded, the limit is enforced after an operation completes and therefore does not limit its temporary working set or returned NumPy array. @@ -62,10 +63,27 @@ is always returned: specifying ``cache_dir`` or ``cache_path`` configures it wit By default, :meth:`RemoteProxy.save ` and :meth:`RemoteProxy.to_cframe ` include valid warm -chunks. Pass ``include_cache=False`` for a cold carrier without changing the +chunks for DISK proxies; MEMORY proxies always export cold carriers. +Pass ``include_cache=False`` for a cold carrier without changing the warm original. The cache policy and limit remain in both forms; local paths and authentication data are not serialized. +Pass ``cache_policy=blosc2.CachePolicy.NONE`` (or another policy) to either +export method to produce a cold carrier with an explicit policy, leaving the +live proxy unchanged. Caterva2 servers accept persisted MEMORY carriers under +opt-in policy but execute them without retained caching (identical to NONE); +older Caterva2 servers reject MEMORY resolution. Use DISK for retained carrier +caching on Caterva2. Cold exports must not overwrite the live disk carrier. + +``fetch()`` and ``afetch()`` prefetch and return the proxy. Eviction may discard +requested chunks; ``materialize(item)`` returns an independent complete NDArray. +The raw ``cache`` is incomplete storage for inspection, not a materialized array. + +Reads and exports on one handle are serialized. Async methods use worker threads; +cancelling an await does not stop a running operation. Separate handles and +processes sharing a carrier need external locking. Unreadable cache files are +preserved and their opening errors are propagated. + Authentication supplied to a live Caterva2 source is deliberately omitted from the carrier. Caterva2's first server implementation resolves public HTTPS sources only; client credentials never travel with the proxy. @@ -95,6 +113,7 @@ file. Read-only mode can use warm chunks but does not retain misses: .. automethod:: get_chunk .. automethod:: aget_chunk .. automethod:: save + .. automethod:: materialize .. automethod:: to_cframe .. autoattribute:: shape .. autoattribute:: dtype diff --git a/plans/remote-proxy-v2.md b/plans/remote-proxy-v2.md index 5fe1920bc..b84497b53 100644 --- a/plans/remote-proxy-v2.md +++ b/plans/remote-proxy-v2.md @@ -2,6 +2,8 @@ ## Status +Superseded by remote-proxy-v3.md for the Python client API and cache policies. + Implemented and validated in Python-Blosc2 and Caterva2. The product and persistence contracts are settled; the phase checklist and test matrix below describe the delivered version-1 behavior. diff --git a/plans/remote-proxy-v3.md b/plans/remote-proxy-v3.md index bce06908e..671819267 100644 --- a/plans/remote-proxy-v3.md +++ b/plans/remote-proxy-v3.md @@ -2,11 +2,21 @@ ## Status -Planned for implementation in Python-Blosc2. +Implemented in Python-Blosc2. This document supersedes v2 for the client API. + +Review follow-up: fetch/afetch return the proxy after prefetching; materialize +returns an independent NDArray. Reads and exports on one handle are serialized, +and async methods use worker threads (cancellation does not interrupt a running +fetch). Independent handles/processes require external carrier locking. +Floating Caterva2 references force identity refresh, including completed arrays. +Failed cache opens preserve existing files. Explicit export cache_policy selects +a cold NONE, MEMORY, or DISK carrier without changing the live policy. +Memory-only runtime URLs need not be portable, but export and source descriptor +access validate portability. Cache limits exclude peak working memory and results. ## Purpose -Unify all lazy remote dataset access under `RemoteProxy`. Previously, `blosc2.open(url, lazy=True)` returned a legacy `blosc2.Proxy` when opened without disk storage options, and a `blosc2.RemoteProxy` when `cache_dir` or `cache_path` was specified. +Unify all lazy remote dataset access under `RemoteProxy`. Previously, lazy opens returned a legacy `Proxy` unless `cache_policy` or `max_cache_bytes` explicitly selected `RemoteProxy`, including when disk storage options were supplied. In v3, `blosc2.open(url, lazy=True)` **always** returns a `RemoteProxy`. To achieve this cleanly while maintaining the fast, ephemeral in-memory caching behavior users expect, `CachePolicy.MEMORY` is reinstated as a first-class policy alongside `CachePolicy.DISK` and `CachePolicy.NONE`. @@ -42,10 +52,12 @@ Users interact with a single, consistent API (`.source`, `.info`, `.traffic`, `. - Requires `cache_dir` or `cache_path` when creating from a remote URL. ### 3. Server-Side Protection in Caterva2 +*(Updated in v4: Caterva2 accepts persisted `MEMORY` carriers under opt-in policy but executes them using the same no-retention path as `NONE`, avoiding unmanaged server RAM caching while preserving the requested limit for downloads; older Caterva2 servers still reject `MEMORY` resolution).* Caterva2 maintains its strict server-side gate in `caterva2/services/remote_proxy.py`: -- Carriers uploaded to Caterva2 with `cache_policy` other than `"none"` or `"disk"` raise `RemoteProxyDenied` (HTTP 403). -- Caterva2 servers are therefore immune to memory-hogging exploits from crafted carriers, without requiring artificial restrictions on Python-Blosc2 client code. +- Carriers uploaded to Caterva2 with `cache_policy` other than `"none"`, `"memory"`, or `"disk"` raise `RemoteProxyDenied` (HTTP 403). +- MEMORY carriers execute without retained array-data caching across operations. +- Server resource controls, default-deny policy, and secure filesystem restrictions remain in effect. ### 4. Direct Carrier Export & Deserialization @@ -61,7 +73,7 @@ Caterva2 maintains its strict server-side gate in `caterva2/services/remote_prox ### 5. `fetch()` and `afetch()` Support on `RemoteProxy` `RemoteProxy` exposes `fetch(item=None)` and `afetch(item=None)`: -- When caching is enabled (`MEMORY` or `DISK`), delegates to the internal cache engine and returns the cache container. +- When caching is enabled (`MEMORY` or `DISK`), prefetches through the cache engine and returns the proxy. Requested chunks may already have been evicted; materialize() returns an independent complete array when needed. - When `cache_policy` is `NONE`, raises `NotImplementedError`. - Exposes `cache` property returning the cache container or `None`. diff --git a/plans/remote-proxy-v4.md b/plans/remote-proxy-v4.md new file mode 100644 index 000000000..d778115e5 --- /dev/null +++ b/plans/remote-proxy-v4.md @@ -0,0 +1,361 @@ +# Plan: Caterva2 executes MEMORY RemoteProxy carriers without retained caching and supports unbounded DISK cache (v4) + +## Status and decision + +Implemented in Caterva2 and Python-Blosc2. This extends v3's client API, +replaces its server rejection of MEMORY carriers, and adds support for unbounded +persistent DISK caching (`max_cache_bytes=None`). It does not supersede the +existing HTTPS security boundary. + +1. Keep Python-Blosc2's lazy remote-open default at MEMORY with a 256 MiB retained + compressed-payload limit. Accept persisted MEMORY carriers in Caterva2, but + execute them using the same no-retention path as NONE. Preserve the original + MEMORY policy and limit in the uploaded file and its downloads. +2. Support `max_cache_bytes=None` for `CachePolicy.DISK` in both Python-Blosc2 and + Caterva2. Passing `None` disables LRU cache eviction (unbounded cache size). + On Caterva2, an unbounded DISK cache operates without eviction when no server + customer quota is set, and is clamped to remaining capacity (`retained + available`) + when a customer storage quota is configured. + +No server memory-cache registry, aggregate memory-cache quota, new configuration +knob, carrier format version, or Python-Blosc2 runtime-construction hook is needed. +This removes retained server data caching for MEMORY, not temporary memory use. + +## Inspected baseline + +Inspected on 2026-09-05: + +- Python-Blosc2 checkout: `/Users/faltet/blosc/python-blosc2`, HEAD + `dfb193e706f4f5801d8116cc4941aada8204cae9`, including the local v3 review changes. +- Caterva2 checkout: `/Users/faltet/ironArray/caterva2`, HEAD + `7478babd6e66dd32413ebe20691f931a8355215c`. No tracked diff was reported there + during inspection. + +Caterva2 file paths below are relative to that repository. Function names are +the primary implementation anchors; line numbers may move. + +### Current resolver and runtime construction + +`caterva2/services/remote_proxy.py` already separates secure resolution from +runtime execution: + +1. `inspect()` opens the physical carrier with `raw_carrier()`, inspects its B2 + marker and payload, and avoids generic B2-object decoding. Inspection uses + carrier locks; a lock sidecar may be created even for observational access. +2. `_validated_source()` checks default-deny configuration, exact payload/source + fields, format versions, cache policy, and URL restrictions. It currently + accepts only NONE and DISK. DISK requires a positive integer limit, excluding + booleans; NONE requires a null limit. +3. `resolve()` validates public DNS answers, creates a pinned HTTPS filesystem, + and passes that exact filesystem to `blosc2.FsspecNDSource(..., _filesystem=fs)`. + It validates source/carrier geometry and rank, logical-byte, and chunk limits. +4. It returns `ServerRemoteProxy(source, expected, carrier, payload)` directly. + It does **not** construct a Python `blosc2.RemoteProxy` from a URL or decode + its persisted policy through `RemoteProxy._from_payload()`. + +`ServerRemoteProxy` currently copies the payload's `cache_policy` and +`max_cache_bytes`. Its `_backend()` uses a temporary +`blosc2.Proxy(self.src, _refresh_source=False)` whenever policy is not DISK or +the operation's disk-cache allowance is zero. That temporary cache is local to +the operation. `read()` assembles the result through it, while `get_chunk()` +reads directly from the authorized source on this path. + +Only the DISK path opens the carrier in append mode and attaches it as `_cache`. +`current_cache_bytes()` already returns zero for non-DISK policies. + +**Consequence:** accepting MEMORY in validation would already reach the desired +non-disk branch. Nevertheless, explicitly normalizing the runtime policy makes +the contract visible and avoids relying on an accidental "anything but disk" +fallback. + +### API, quota, metadata, and download integration + +- `services/server.py::open_b2()` calls `inspect()` and `resolve()` before generic + `blosc2.open()`, translating resolution denial to HTTP 403. +- `remote_proxy_cache_limit()` returns zero for non-DISK runtimes. DISK remains + subject to the carrier cap and existing customer storage quota. +- `read_remote_proxy()` applies dataset locking and disk-growth accounting to + slice/index reads. The `api/chunk` branch similarly obtains a cache allowance + and invokes `ServerRemoteProxy.get_chunk()` in a worker thread. +- `services/srv_utils.py::read_metadata()` inspects the raw carrier without + resolving its source. `api/info` marks RemoteProxy data as + `accept_ranges="none"` and exposes only its portable `b2o` variable metadata, + removing cache-engine bookkeeping from the response. +- The physical download path inspects the carrier and calls `export_cframe()` + under locks. Warm export serializes the raw carrier; cold export constructs a + cold carrier with the original payload. Neither resolves the source. + +These paths already support the separation between logical reads and physical +downloads required by this proposal. + +## Behavior contract + +| Stored policy | Stored limit | Caterva2 effective policy | Retained server data | Python reopen | +| --- | --- | --- | --- | --- | +| NONE | null | NONE | None | NONE | +| MEMORY | positive integer; default 268435456 | NONE | None between operations | MEMORY with original limit | +| DISK | positive integer | DISK | Carrier cache within cap and quota | DISK with positive limit | +| DISK | null (unbounded) | DISK | Carrier cache within customer quota (unbounded if no quota) | DISK with unbounded cache (no eviction) | + +For a MEMORY carrier: + +- Each independent read fetches the required upstream data again; there is no + reusable server array-data cache across reads or requests. +- Temporary compressed assembly, metadata, transport state, decompression + buffers, and output may exist. Do not assert that literally no memory is used + or that every HTTP request is duplicated identically across operations. +- The stored positive limit is validated and preserved, but does not set a + runtime memory budget or allocate that amount on the server. +- Neither reads nor quota handling populate the physical carrier or change its + payload. Carrier bytes, size, and mtime remain unchanged by those reads; + lock-sidecar activity is outside that invariant. +- Physical downloads preserve the requested policy and limit. A cold download + also preserves them; `include_cache=false` is not a policy override. +- Embedded chunk data in a crafted MEMORY carrier is ignored for logical reads. + Physical warm download can preserve those original bytes; cold download drops + cache state. Do not trust them merely because the carrier is structurally valid. +- Actual source replacement between API requests is observed through fresh + resolution. V4 does not promise an atomic snapshot across multiple upstream + reads or add source refresh inside a long-lived ServerRemoteProxy instance. + +For an unbounded DISK carrier (`max_cache_bytes=None`): + +- In Python-Blosc2, chunks fetched into the carrier are retained without LRU + eviction (`proxy.cache_bytes` reflects `carrier.schunk.cbytes`). +- In Caterva2, `_validated_source()` accepts `max_cache_bytes: null` for `disk`. +- When server customer storage quota is enabled, `remote_proxy_cache_limit()` clamps + the effective limit to remaining capacity (`retained + available`), preventing + the unbounded carrier from exceeding server quota. When quota is disabled, the + proxy operates without a limit (`None`). +- `ServerRemoteProxy.current_cache_bytes()` falls back to `carrier.schunk.cbytes` + when `proxy-cache-sizes` is not present, since `blosc2.Proxy` does not maintain + an LRU size table for unbounded caches. +- Exporting a DISK proxy to DISK preserves `max_cache_bytes=None`; exporting to + MEMORY falls back to `DEFAULT_DISK_CACHE_BYTES` (since MEMORY requires a finite + positive integer limit); exporting to NONE sets `max_cache_bytes=None`. + +## Implementation steps + +### 1. Accept and validate MEMORY descriptors + +In `_validated_source()`: + +- Keep the exact field, version, source-kind, and default-deny checks. +- Validate both `"memory"` and `"disk"` using the same positive-integer rule. + Reject missing/null limits, booleans, floats, strings, zero, and negatives. +- Keep `"none"` restricted to a null limit and reject unknown policy strings. +- Update denial messages so MEMORY is recognized rather than described as an + unsupported policy. +- Preserve the input payload; do not rewrite it to NONE or remove its limit. + +All URL checks still apply to MEMORY: public credential-free HTTPS only, exact +host allowlist, no query or fragment, no user information, and no custom source +fields. Client support for runtime signed URLs does not extend server support. + +### 2. Make runtime policy explicit in ServerRemoteProxy + +Use the existing constructor with the already authorized source. Proposed +initialization, after validation by `resolve()`: + +```python +self.requested_cache_policy = payload["cache_policy"] +self.requested_max_cache_bytes = payload["max_cache_bytes"] +self.cache_policy = ( + "none" if self.requested_cache_policy == "memory" + else self.requested_cache_policy +) +self.max_cache_bytes = ( + self.requested_max_cache_bytes if self.cache_policy == "disk" else None +) +``` + +Here `cache_policy` remains the execution-facing attribute used by existing +backend and quota branches. Optionally provide a read-only +`effective_cache_policy` alias for diagnostics; do not introduce two mutable +execution-policy fields. Constructor documentation must distinguish requested +and effective values. If adding a mapping helper, reject unknown policies rather +than silently converting them to NONE, and reuse it for optional diagnostics. + +Do not mutate `payload`, `carrier.schunk.vlmeta`, or any process-global resolver. +Do not call `blosc2.RemoteProxy(url)` or `blosc2.open(url)` to perform conversion: +that could recreate the source outside the authorized filesystem boundary. + +Keep `_backend()`, `read()`, `get_chunk()`, and `current_cache_bytes()` on their +existing non-DISK paths. A temporary generic Proxy for slice assembly is correct +NONE behavior; no persistent MEMORY backend is ever attached to the runtime. + +### 3. Audit server consumers without adding cache infrastructure + +Confirm all branches consuming `ServerRemoteProxy.cache_policy` use the effective +value. Preserve `remote_proxy_cache_limit()`'s zero result for effective NONE. +Keep existing dataset and carrier locking; removing locks is not part of v4. +Exercise both full/sliced/fancy-index fetches and compressed chunk requests. + +No automatic cache-growth charge should occur for MEMORY reads. Ordinary upload +storage and existing lock files remain subject to current server accounting. +Keep DISK quota enforcement, invalidation, and serialization unchanged. + +### 4. Preserve metadata and exports + +Keep `api/info`'s stored `b2o` descriptor untouched and `accept_ranges="none"`. +Do not insert requested/effective fields inside `b2o`: Python-Blosc2 validates +that payload's exact field set. + +Recommended initial scope: expose requested/effective values on the internal +runtime and document their mapping; keep the public metadata schema unchanged. +Public diagnostics can be a follow-up. If included now, add a declared optional +response-model field outside the portable descriptor, with client compatibility +tests. Describe it as the configured execution mapping, not proof that resolution +is enabled or that a particular URL is authorized. Metadata must remain local +and usable when resolution is disabled. + +Physical downloads must continue using `export_cframe(carrier, original_payload)` +and must never serialize a normalized runtime object or normalized payload. + +### 5. Documentation and compatibility + +Update Caterva2 `doc/utilities/cat2-server.md` and comments in +`caterva2-server.sample.toml`: MEMORY carriers are accepted under the same source +policy but execute without retained caching. No new TOML option is introduced. + +Update Python-Blosc2 `doc/guides/remote_arrays.md`, +`doc/reference/remoteproxy.rst`, and the status of the v3 server-policy section +when the server change ships. Explain client/server performance differences and +the unchanged 256 MiB client default. Describe older Caterva2 servers as still +rejecting MEMORY resolution, rather than claiming universal support. + +No B2 object version bump is needed: MEMORY is already a Python-Blosc2 v3 policy. +Check Caterva2's declared Python-Blosc2 dependency and release floor before +shipping. Reopening downloaded MEMORY carriers requires a client version with +MEMORY support. Do not infer that version solely from `hasattr(RemoteProxy)`. + +### 6. Support unbounded DISK cache (`max_cache_bytes=None`) + +In Python-Blosc2: + +- Update `_normalize_limit()` in `blosc2/remote_proxy.py`: allow `value is None` + when `policy is CachePolicy.DISK`, returning `None`. Continue strictly requiring + a positive integer for `MEMORY`, and forbidding limits for `NONE`. +- Update `_export_carrier()`: when exporting with `cache_policy=CachePolicy.DISK`, + preserve `self.max_cache_bytes` (which can be `None`), rather than coercing it + to `DEFAULT_DISK_CACHE_BYTES`. When exporting to `MEMORY`, fall back to + `DEFAULT_DISK_CACHE_BYTES` (since MEMORY requires a finite positive integer limit). +- Add `_validate_payload_limit()` helper and update `_from_payload()`: allow + `max_cache_bytes: null` for persisted DISK payloads, while continuing to reject + booleans, strings, zero, and negative values. +- Document unbounded DISK caching in docstrings (`RemoteProxy`, `blosc2.open`) and + Sphinx/Myst documentation (`doc/reference/remoteproxy.rst`, `doc/guides/remote_arrays.md`). +- Add tests in `tests/test_remote_proxy.py`: test initialization, rejection of + `None` for MEMORY, rejection of non-integers/negatives for DISK, persistence round-trip + and payload validation, and verify that chunks are retained without eviction. + +In Caterva2: + +- In `caterva2/services/remote_proxy.py::_validated_source()`: accept + `max_cache_bytes: null` for `disk`, while rejecting invalid types (booleans, + strings, non-positive numbers). Keep `memory` strictly requiring positive integers. +- In `caterva2/services/remote_proxy.py::ServerRemoteProxy`: + - In `current_cache_bytes()`: check if `proxy-cache-sizes` is present in `vlmeta`; + if absent (as is the case when `blosc2.Proxy` runs with `max_cache_bytes=None`), + fall back to `carrier.schunk.cbytes`. + - In `_backend()`: handle `self.max_cache_bytes is None` when applying `cache_limit`, + avoiding `TypeError` in `min()`. +- In `caterva2/services/server.py::remote_proxy_cache_limit()`: when + `proxy.max_cache_bytes is None`, return `None` if no customer quota is configured; + if customer storage quota is enabled, clamp to available quota (`retained + available`). +- Update docs and sample config (`doc/utilities/cat2-server.md`, `caterva2-server.sample.toml`). +- Add tests in `caterva2/tests/test_remote_proxy.py` and `caterva2/tests/test_api.py`. + +## Verification plan + +Extend `caterva2/tests/test_remote_proxy.py` using its deterministic fixtures: + +1. Replace the old blanket MEMORY-rejection expectation in + `test_cache_specification_is_strict`. Add valid MEMORY cases with default and + custom limits, invalid-limit cases for MEMORY and DISK, and unknown policies. +2. Parameterize default-deny and unsafe-destination tests over all three policies + with valid corresponding limits. Preserve private-address, pinned resolver, + redirect-disabled, source-field, and embedded-reference rejection coverage. +3. Extend `test_allowed_source_is_resolved_with_the_secure_filesystem` to MEMORY. + Assert identity of the supplied `_filesystem`, requested MEMORY/effective NONE, + and no call to the ordinary RemoteProxy constructor or generic decoder. +4. Extend `_server_proxy` fixtures to create MEMORY carriers. Read the same slice + twice on one runtime and after reconstructing the runtime. Assert correct + values and upstream chunk/block data calls for each read. Repeat for chunks. + Instrument data operations, not just aggregate metadata/request counters. +5. Compare carrier bytes, size, and mtime before and after reads; allow sidecar + creation. Assert no attached reusable backend, current cache bytes zero, and + zero disk-cache allowance regardless of storage quota. +6. Exercise a MEMORY carrier containing synthetic warm cache bookkeeping/data; + logical reads must still use the authorized source and ignore those chunks. +7. Preserve the payload through both warm and cold `export_cframe()` calls and + reopen the exported MEMORY artifact in Python. Verify its original policy, + limit, correct data, and client cache reuse with a controlled source fixture. +8. Keep existing DISK warm-reopen, cold-export, zero-quota, concurrent-fill, and + secure-filesystem tests passing. Add source/geometry replacement cases across + resolutions for MEMORY without claiming stronger within-operation consistency. + +Extend `caterva2/tests/test_api.py`: + +- Parameterize existing discovery/default-deny and physical-download tests for + MEMORY; info/download must work without outbound resolution even when disabled. +- Add enabled-resolution fetch and chunk coverage using a controlled source and + server fixture. The current API tests use a running server; monkeypatching only + the client test process does not patch that server. Use an in-process fixture + with injection at the authorized filesystem boundary, or a controlled HTTPS + fixture with explicitly test-scoped DNS classification. +- Assert requested descriptor preservation, logical fetch results rather than + carrier placeholders, repeated upstream data reads, no carrier mutation, and + unchanged range-advertisement behavior. + +### Verification results + +The implementation is verified across both repositories in the `blosc2` conda environment: + +1. **Python-Blosc2**: + - `pytest tests/test_remote_proxy.py`: 62 passed in 1.27s. + - Tested MEMORY/DISK/NONE policy configurations, parameter defaults, strict descriptor validation, + unbounded DISK caching without LRU eviction (`test_unlimited_disk_cache_does_not_evict`), + persisted payload decode/reopen (`test_reference_accepts_disk_policy_with_none_limit_in_payload`), + and invalid payload limits (`test_reference_rejects_invalid_disk_limit_in_payload`). + - `ruff check` and `ruff format --check`: all checks passed cleanly. + +2. **Caterva2**: + - `pytest caterva2/tests/test_remote_proxy.py`: 58 passed in 1.01s. + - Covered default-deny across policies, allowed HTTPS destinations, strict cache specification, + secure filesystem pinning, ServerRemoteProxy MEMORY execution without retained caching, + carrier file immutability, synthetic cached chunk rejection, warm/cold cframe export and Python reopen, + geometry replacement detection, customer quota clamping for bounded and unbounded proxies, + unbounded DISK server caching without eviction (`test_unlimited_disk_server_proxy_caches_without_eviction`), + and concurrent fills. + - `pytest caterva2/tests/test_api.py`: 126 passed, 109 skipped in 5.05s. + Included discovery of NONE/MEMORY/DISK (bounded and unbounded), resolution denial before open, + and in-process ASGI resolution and fetch (`test_remote_proxy_memory_enabled_resolution_fetch_and_chunk`). + - `ruff check` and `ruff format --check`: all checks passed cleanly. + +## Limits and deferred work + +Existing rank, logical-byte, chunk-count, timeout, and concurrency controls remain +in effect. They are not an aggregate process RAM cap or an operation-wide network +budget. MEMORY-to-NONE does not solve large result allocation, many concurrent +requests, multi-worker peak memory, or a source changing during range assembly. +Do not describe it as immunity from memory exhaustion. + +Server MEMORY retention, shared cache registries, memory quotas, signed/private +server sources, nested reference resolution, and broader snapshot guarantees +remain separate future designs. The small policy translation proposed here +should not expand those boundaries. + +## Acceptance criteria + +- [x] Valid public HTTPS MEMORY carriers resolve under the existing opt-in policy. +- [x] Their runtime uses the already authorized source and the existing NONE path. +- [x] Repeated logical reads retain no array-data cache between operations and do not + modify the uploaded carrier. +- [x] Stored/downloaded policy remains MEMORY with its original positive limit, and + a compatible Python client restores MEMORY behavior. +- [x] `CachePolicy.DISK` supports `max_cache_bytes=None` (unbounded cache) in Python-Blosc2 + and Caterva2, bypassing LRU eviction while respecting Caterva2 customer storage quota if configured. +- [x] NONE and DISK behavior and all security gates remain intact. +- [x] No Python-Blosc2 construction hook, memory quota knob, or carrier format change + is introduced merely to implement this mapping. diff --git a/plans/remote-proxy.md b/plans/remote-proxy.md index a75b7f8ad..bc7a9923d 100644 --- a/plans/remote-proxy.md +++ b/plans/remote-proxy.md @@ -1,7 +1,7 @@ # Plan: Persistable `RemoteProxy` > **Superseded:** The self-caching carrier design in -> [`remote-proxy-v2.md`](remote-proxy-v2.md) is the authoritative implementation +> [`remote-proxy-v3.md`](remote-proxy-v3.md) is the authoritative client implementation > plan. This document records the earlier immutable-reference design. ## Motivation diff --git a/src/blosc2/remote_proxy.py b/src/blosc2/remote_proxy.py index 72f079d80..9af7c632a 100644 --- a/src/blosc2/remote_proxy.py +++ b/src/blosc2/remote_proxy.py @@ -9,9 +9,11 @@ from __future__ import annotations +import asyncio import math import os import threading +from functools import wraps from urllib.parse import parse_qsl, urlsplit import numpy as np @@ -42,6 +44,19 @@ def __repr__(self) -> str: ) +def _serialized_operation(method): + @wraps(method) + def locked(self, *args, **kwargs): + with self._operation_lock: + try: + return method(self, *args, **kwargs) + finally: + if self._proxy is not None: + self._proxy._enforce_cache_limit(tuple(slice(0, 0) for _ in self.shape)) + + return locked + + def _validate_persistable_url(url: str) -> None: """Reject URL features that would put credentials in a portable carrier.""" if "::" in url: @@ -71,6 +86,10 @@ def _normalize_limit(policy, value): return None if value is _POLICY_DEFAULT: return DEFAULT_DISK_CACHE_BYTES + if value is None: + if policy is blosc2.CachePolicy.DISK: + return None + raise TypeError("max_cache_bytes must be a positive integer") if isinstance(value, bool) or not isinstance(value, int): raise TypeError("max_cache_bytes must be a positive integer") if value <= 0: @@ -88,6 +107,17 @@ def _validate_max_concurrency(value: int | None) -> int | None: return value +def _validate_payload_limit(policy: blosc2.CachePolicy, limit) -> None: + if policy is blosc2.CachePolicy.NONE: + if limit is not None: + raise ValueError("persisted NONE RemoteProxy cannot have max_cache_bytes") + elif policy is blosc2.CachePolicy.DISK: + if limit is not None and (isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0): + raise ValueError("persisted DISK RemoteProxy requires positive max_cache_bytes or None") + elif isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0: + raise ValueError(f"persisted {policy.name} RemoteProxy requires positive max_cache_bytes") + + class RemoteProxy(blosc2.Operand): """A persistable, optionally self-caching reference to a remote array. @@ -113,7 +143,9 @@ class RemoteProxy(blosc2.Operand): Only valid with ``DISK``. max_cache_bytes: int or None, optional Post-operation compressed-payload bound. It defaults to 256 MiB for - ``DISK`` and ``MEMORY`` and must always be finite. It is not applicable to ``NONE``. + ``DISK`` and ``MEMORY``. Passing ``None`` with ``DISK`` disables cache + eviction (unbounded cache). ``MEMORY`` requires a finite positive integer. + It is not applicable to ``NONE``. max_concurrency: int, optional Maximum number of independent remote fetches in flight. """ @@ -146,11 +178,14 @@ def __init__( self._cache_policy = cache_policy self._cache_limit = _normalize_limit(cache_policy, max_cache_bytes) self._max_concurrency = _validate_max_concurrency(max_concurrency) - self.src, self._source = self._open_source(urlpath, self._max_concurrency) + self.src, self._source = self._open_source( + urlpath, self._max_concurrency, persistable=cache_policy is not blosc2.CachePolicy.MEMORY + ) self._runtime_urlpath = self._runtime_source(urlpath) self._expected_geometry = self._geometry(self.src) self._expected_cparams = self.src.cparams self._refresh_lock = threading.Lock() + self._operation_lock = threading.RLock() self._proxy = None self._carrier = _carrier self._cache_status = None @@ -190,23 +225,22 @@ def _open_or_create_carrier(self, cache_dir, cache_path): path = blosc2.schunk.fsspec_cache_path(self._source_identity(), cache_dir, ".b2nd") if os.path.exists(path): kwargs = {"dparams": blosc2.DParams(nthreads=1)} - try: - carrier = blosc2.blosc2_ext.open(path, "a", 0, **kwargs) - payload = carrier.schunk.vlmeta.get("b2o") - if payload != self._payload(): - raise ValueError(f"the RemoteProxy carrier at {path} has a different specification") - stored = carrier.schunk.vlmeta.get("proxy-stamp") - current = getattr(self.src, "stamp", None) - status = ( - "invalidated/rebuilt" - if stored is not None and current is not None and stored != current - else "reused" + carrier = blosc2.blosc2_ext.open(path, "a", 0, **kwargs) + payload = carrier.schunk.vlmeta.get("b2o") + if payload != self._payload(): + raise ValueError( + f"the RemoteProxy carrier at {path} has a different specification; " + "open legacy Proxy caches directly with blosc2.open(cache_path), " + "or choose a new cache_path" ) - return carrier, status - except Exception as exc: - if isinstance(exc, ValueError) and "has a different specification" in str(exc): - raise - blosc2.remove_urlpath(path) + stored = carrier.schunk.vlmeta.get("proxy-stamp") + current = getattr(self.src, "stamp", None) + status = ( + "invalidated/rebuilt" + if stored is not None and current is not None and stored != current + else "reused" + ) + return carrier, status carrier = self._to_b2object_carrier(urlpath=path, contiguous=True, mode="w") return carrier, "created" @@ -236,10 +270,10 @@ def _attach_carrier_cache(self): self._proxy = None @staticmethod - def _open_source(urlpath, max_concurrency, *, traffic=None): + def _open_source(urlpath, max_concurrency, *, traffic=None, persistable=True): if isinstance(urlpath, blosc2.C2Array): src = urlpath - if src.urlbase is not None: + if persistable and src.urlbase is not None: _validate_persistable_url(src.urlbase) source = { "kind": "caterva2", @@ -248,7 +282,7 @@ def _open_source(urlpath, max_concurrency, *, traffic=None): "urlbase": src.urlbase, } elif isinstance(urlpath, blosc2.URLPath): - if urlpath.urlbase is not None: + if persistable and urlpath.urlbase is not None: _validate_persistable_url(urlpath.urlbase) src = blosc2.C2Array( urlpath.path, @@ -263,7 +297,8 @@ def _open_source(urlpath, max_concurrency, *, traffic=None): "urlbase": src.urlbase, } elif isinstance(urlpath, str): - _validate_persistable_url(urlpath) + if persistable: + _validate_persistable_url(urlpath) kwargs = {} if max_concurrency is None else {"max_concurrency": max_concurrency} src = blosc2.FsspecNDSource(urlpath, _traffic=traffic, **kwargs) source = {"kind": "fsspec", "version": 1, "urlpath": urlpath} @@ -303,7 +338,10 @@ def _prepare_read(self): if refresh is None: refresh = getattr(self.src, "refresh_stamp", None) if refresh is not None: - refresh() + if isinstance(self.src, blosc2.C2Array): + refresh(force=True) + else: + refresh() self._validate_geometry(self._expected_geometry) current_stamp = getattr(self.src, "stamp", None) @@ -315,6 +353,7 @@ def _prepare_read(self): self._runtime_urlpath, self._max_concurrency, traffic=self.traffic, + persistable=self.cache_policy is not blosc2.CachePolicy.MEMORY, ) if current_stamp is None and not isinstance(fresh, blosc2.C2Array): # No stable validator means cached bytes cannot safely be @@ -394,6 +433,7 @@ def info_items(self) -> list[tuple[str, object]]: @property def source(self) -> dict: """A copy of the credential-free source descriptor.""" + self._payload() # Runtime-only URLs must not escape as portable descriptors. return dict(self._source) @property @@ -440,6 +480,7 @@ def cache_bytes(self) -> int: return self._proxy.schunk.cbytes return self._proxy._retained_cache_bytes() + @_serialized_operation def __getitem__(self, item): backend = self._prepare_read() if isinstance(backend, blosc2.Proxy): @@ -472,32 +513,40 @@ def _chunk_slice(self, nchunk: int): for coord, chunk, size in zip(coords, self.chunks, self.shape, strict=True) ) + @_serialized_operation def fetch(self, item=(), max_concurrency: int | None = None): """Fetch remote data into the cache container. - Only valid when caching is enabled (:attr:`CachePolicy.DISK` or - :attr:`CachePolicy.MEMORY`). + Return this proxy, not a materialized array. Eviction may discard + prefetched chunks. Use indexing for values or :meth:`materialize` + for an independent NDArray. Requires MEMORY or DISK caching. """ backend = self._prepare_read() if not isinstance(backend, blosc2.Proxy): raise NotImplementedError("fetch requires CachePolicy.DISK or CachePolicy.MEMORY") - result = backend.fetch(item, max_concurrency=max_concurrency) + backend.fetch(item, max_concurrency=max_concurrency) backend._enforce_cache_limit(item) - return result + return self async def afetch(self, item=(), max_concurrency: int | None = None): - """Asynchronously fetch remote data into the cache container. + """Prefetch in a worker thread and return this proxy, like :meth:`fetch`. - Only valid when caching is enabled (:attr:`CachePolicy.DISK` or - :attr:`CachePolicy.MEMORY`). + Requires MEMORY or DISK. Cancelling the await does not interrupt an + already running fetch, which retains the operation lock until done. """ - backend = self._prepare_read() - if not isinstance(backend, blosc2.Proxy): + if self.cache_policy is blosc2.CachePolicy.NONE: raise NotImplementedError("afetch requires CachePolicy.DISK or CachePolicy.MEMORY") - result = await backend.afetch(item, max_concurrency=max_concurrency) - backend._enforce_cache_limit(item) - return result + return await asyncio.to_thread(self.fetch, item, max_concurrency=max_concurrency) + + def materialize(self, item=(), **kwargs): + """Return an independent NDArray containing the requested values. + + The output and temporary NumPy buffer are not bounded by max_cache_bytes. + Keyword arguments are forwarded to blosc2.asarray. + """ + return blosc2.asarray(self[item], **kwargs) + @_serialized_operation def get_chunk(self, nchunk: int) -> bytes: backend = self._prepare_read() if not isinstance(backend, blosc2.Proxy): @@ -509,19 +558,12 @@ def get_chunk(self, nchunk: int) -> bytes: return chunk async def aget_chunk(self, nchunk: int) -> bytes: - backend = self._prepare_read() - if not isinstance(backend, blosc2.Proxy): - method = getattr(backend, "aget_chunk", None) - if method is None: - raise NotImplementedError("the remote source does not provide asynchronous chunk reads") - return await method(nchunk) - item = self._chunk_slice(nchunk) - await backend.afetch(item) - chunk = backend.schunk.get_chunk(nchunk) - backend._enforce_cache_limit(item) - return chunk + return await asyncio.to_thread(self.get_chunk, nchunk) def _payload(self): + url = self._source.get("urlpath", self._source.get("urlbase")) + if url is not None: + _validate_persistable_url(url) return { "kind": "remote_proxy", "version": 1, @@ -543,28 +585,58 @@ def _to_b2object_carrier(self, **kwargs): write_b2object_payload(array, self._payload()) return array - def _export_carrier(self, include_cache: bool): + def _export_carrier(self, include_cache: bool, cache_policy=None): if not isinstance(include_cache, bool): raise TypeError("include_cache must be a boolean") + if cache_policy is not None: + if not isinstance(cache_policy, blosc2.CachePolicy): + raise TypeError("cache_policy must be a blosc2.CachePolicy instance") + carrier = self._to_b2object_carrier() + payload = self._payload() + payload["cache_policy"] = cache_policy.value + if cache_policy is blosc2.CachePolicy.NONE: + payload["max_cache_bytes"] = None + elif cache_policy is blosc2.CachePolicy.DISK: + payload["max_cache_bytes"] = ( + self.max_cache_bytes + if self.cache_policy is not blosc2.CachePolicy.NONE + else DEFAULT_DISK_CACHE_BYTES + ) + else: + payload["max_cache_bytes"] = self.max_cache_bytes or DEFAULT_DISK_CACHE_BYTES + write_b2object_payload(carrier, payload) + return carrier if include_cache and self._carrier is not None: return self._carrier return self._to_b2object_carrier() - def to_cframe(self, *, include_cache: bool = True) -> bytes: - """Serialize the carrier, including valid cached chunks by default.""" - return self._export_carrier(include_cache).to_cframe() + @_serialized_operation + def to_cframe(self, *, include_cache: bool = True, cache_policy=None) -> bytes: + """Export a carrier. Only DISK preserves warm chunks by default. + An explicit cache_policy exports a cold carrier with that policy. + """ + return self._export_carrier(include_cache, cache_policy).to_cframe() + + @_serialized_operation def save( self, urlpath: str | os.PathLike, contiguous: bool = True, *, include_cache: bool = True, + cache_policy=None, **kwargs, ) -> None: - """Persist the carrier, including valid cached chunks by default.""" + """Save a carrier; MEMORY exports are cold. See :meth:`to_cframe`.""" urlpath = os.fspath(urlpath) - carrier = self._export_carrier(include_cache) + if ( + (cache_policy is not None or not include_cache) + and self.cache_path is not None + and os.path.abspath(self.cache_path) == os.path.abspath(urlpath) + ): + raise ValueError("cold or policy-changing export requires a different destination") + carrier = self._export_carrier(include_cache, cache_policy) source_path = getattr(carrier.schunk, "urlpath", None) if source_path is not None and os.path.abspath(source_path) == os.path.abspath(urlpath): return @@ -580,11 +652,7 @@ def _from_payload(cls, payload, carrier): except ValueError as exc: raise ValueError("persisted RemoteProxy has an unsupported cache policy") from exc limit = payload.get("max_cache_bytes") - if policy is blosc2.CachePolicy.NONE: - if limit is not None: - raise ValueError("persisted NONE RemoteProxy cannot have max_cache_bytes") - elif isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0: - raise ValueError(f"persisted {policy.name} RemoteProxy requires positive max_cache_bytes") + _validate_payload_limit(policy, limit) source = payload.get("source") if not isinstance(source, dict) or source.get("version") != 1: raise ValueError("unsupported RemoteProxy source descriptor") diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index cb027b3a4..2bf99e1e2 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -2292,11 +2292,12 @@ def open( using the requested retention policy (``NONE``, ``MEMORY``, or ``DISK``). When omitted, passing ``cache_dir`` or ``cache_path`` defaults to ``CachePolicy.DISK``, while omitting them defaults to ``CachePolicy.MEMORY``. - max_cache_bytes: int, optional + max_cache_bytes: int or None, optional With ``lazy=True``, bound retained compressed cache payload for a :ref:`RemoteProxy` after each operation. Defaults to 256 MiB for both - ``DISK`` and ``MEMORY``, and always has a finite bound. This does not bound - the current operation's working set or result. + ``DISK`` and ``MEMORY``. Passing ``None`` with ``DISK`` disables cache + eviction (unbounded cache). This does not bound the current operation's + working set or result. mmap_mode: str, optional If set, the file will be memory-mapped instead of using the default I/O functions and the `mode` argument will be ignored. diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index ad6df9da6..a20943749 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -859,9 +859,8 @@ def test_lazy_empty_array(tmp_path): assert np.array_equal(b[:], np.zeros((0,), dtype="i4")) -def test_lazy_cache_rebuilt_when_corrupt(tmp_path): - # An interrupted run can leave a half-written cache behind; the whole point of - # cache_dir is surviving across runs, so it has to be discarded, not fatal +def test_lazy_cache_preserved_when_corrupt(tmp_path): + # An unreadable file cannot safely be identified as a disposable cache. a = blosc2.arange(100, dtype="i4", chunks=(10,)) fsspec.filesystem("memory").pipe_file("/c.b2nd", a.to_cframe()) @@ -870,8 +869,10 @@ def test_lazy_cache_rebuilt_when_corrupt(tmp_path): cache = next(p for p in tmp_path.iterdir() if p.suffix == ".b2nd") cache.write_bytes(cache.read_bytes()[:50]) - with blosc2.open("memory://c.b2nd", lazy=True, cache_dir=tmp_path) as b: - assert np.array_equal(b[:], a[:]) + before = cache.read_bytes() + with pytest.raises(RuntimeError): + blosc2.open("memory://c.b2nd", lazy=True, cache_dir=tmp_path) + assert cache.read_bytes() == before def test_max_concurrency_needs_lazy(tmp_path): diff --git a/tests/test_remote_proxy.py b/tests/test_remote_proxy.py index b623d2ace..2c82b1e2d 100644 --- a/tests/test_remote_proxy.py +++ b/tests/test_remote_proxy.py @@ -45,13 +45,26 @@ def test_cache_policy_validation(tmp_path): blosc2.RemoteProxy(url, cache_policy="memory") with pytest.raises(ValueError, match="not applicable"): blosc2.RemoteProxy(url, max_cache_bytes=1) + disk_unlimited = blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=tmp_path / "unlimited.b2nd", + max_cache_bytes=None, + ) + assert disk_unlimited.max_cache_bytes is None with pytest.raises(TypeError, match="positive integer"): blosc2.RemoteProxy( url, - cache_policy=blosc2.CachePolicy.DISK, - cache_path=tmp_path / "unlimited.b2nd", + cache_policy=blosc2.CachePolicy.MEMORY, max_cache_bytes=None, ) + with pytest.raises(ValueError, match="positive integer"): + blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=tmp_path / "neg.b2nd", + max_cache_bytes=-1, + ) with pytest.raises(ValueError, match="requires cache_dir or cache_path"): blosc2.RemoteProxy(url, cache_policy=blosc2.CachePolicy.DISK) with pytest.raises(ValueError, match="max_concurrency"): @@ -289,6 +302,43 @@ def test_reference_rejects_memory_policy_without_positive_limit_in_payload(): decode_b2object_payload(payload, carrier=carrier) +def test_reference_accepts_disk_policy_with_none_limit_in_payload(tmp_path): + url, data = _remote_array("unlimited-payload.b2nd", nchunks=2, chunk_size=50) + cache_path = tmp_path / "unlimited-carrier.b2nd" + proxy = blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=cache_path, + max_cache_bytes=None, + ) + assert proxy.max_cache_bytes is None + carrier_raw = blosc2.ndarray_from_cframe(proxy.to_cframe()) + payload = dict(carrier_raw.schunk.vlmeta["b2o"]) + decoded_carrier = decode_b2object_payload(payload, carrier=carrier_raw) + assert payload["max_cache_bytes"] is None + assert decoded_carrier.max_cache_bytes is None + + reopened = blosc2.open(cache_path) + assert isinstance(reopened, blosc2.RemoteProxy) + assert reopened.cache_policy is blosc2.CachePolicy.DISK + assert reopened.max_cache_bytes is None + np.testing.assert_array_equal(reopened[:], data) + + +@pytest.mark.parametrize("invalid_limit", [False, True, 0, -10, "1000"]) +def test_reference_rejects_invalid_disk_limit_in_payload(invalid_limit): + url, _ = _remote_array("bad-disk-limit.b2nd", nchunks=1, chunk_size=100) + carrier = blosc2.ndarray_from_cframe(blosc2.RemoteProxy(url).to_cframe()) + payload = dict(carrier.schunk.vlmeta["b2o"]) + payload["cache_policy"] = "disk" + payload["max_cache_bytes"] = invalid_limit + + with pytest.raises( + ValueError, match="persisted DISK RemoteProxy requires positive max_cache_bytes or None" + ): + decode_b2object_payload(payload, carrier=carrier) + + @pytest.mark.parametrize( ("field", "value", "error"), [ @@ -573,11 +623,11 @@ def test_memory_cache_fetch_and_afetch(): cache_policy=blosc2.CachePolicy.MEMORY, ) cached_container = proxy.fetch(slice(0, 10_000)) - assert cached_container is proxy.cache + assert cached_container is proxy assert proxy.cache_bytes > 0 async_container = asyncio.run(proxy.afetch(slice(10_000, 20_000))) - assert async_container is proxy.cache + assert async_container is proxy def test_memory_proxy_save_and_reopen(tmp_path): @@ -609,3 +659,182 @@ def test_remote_proxy_fetch_rejects_none_policy(): NotImplementedError, match=r"afetch requires CachePolicy\.DISK or CachePolicy\.MEMORY" ): asyncio.run(proxy.afetch()) + + +def test_prefetch_over_limit_and_materialize(): + url, data = _remote_array("prefetch-limit.b2nd", nchunks=3, chunk_size=10_000) + proxy = blosc2.open(url, lazy=True, max_cache_bytes=12_000) + assert proxy.fetch() is proxy + assert proxy.cache_bytes <= 12_000 + assert asyncio.run(proxy.afetch()) is proxy + np.testing.assert_array_equal(proxy.materialize()[:], data) + assert proxy.cache_bytes <= 12_000 + + +def test_open_error_preserves_existing_file(tmp_path, monkeypatch): + url, _ = _remote_array("open-error.b2nd", nchunks=1, chunk_size=100) + path = tmp_path / "existing.b2nd" + blosc2.arange(100).save(path) + before = path.read_bytes() + + def fail(*args, **kwargs): + raise OSError("transient open failure") + + monkeypatch.setattr(blosc2.blosc2_ext, "open", fail) + with pytest.raises(OSError, match="transient"): + blosc2.open(url, lazy=True, cache_path=path) + assert path.read_bytes() == before + + +def test_completed_caterva2_reference_refreshes_identity(monkeypatch): + state = {"nonce": "old"} + + def info(*args, **kwargs): + return { + "shape": [10], + "chunks": [5], + "blocks": [5], + "dtype": " 0 + + +def test_runtime_signed_url_is_not_exportable(): + url, data = _remote_array("signed.b2nd?token=secret", nchunks=1, chunk_size=100) + proxy = blosc2.open(url, lazy=True) + np.testing.assert_array_equal(proxy[:], data) + with pytest.raises(ValueError, match="credential-like"): + proxy.to_cframe() + with pytest.raises(ValueError, match="credential-like"): + _ = proxy.source + + +def test_interrupted_memory_fetch_enforces_limit(monkeypatch): + url, _ = _remote_array("interrupted-memory.b2nd", nchunks=3, chunk_size=10_000) + proxy = blosc2.open(url, lazy=True, max_cache_bytes=100) + original = proxy.src.get_chunk + + def interrupted(nchunk): + if nchunk == 1: + raise RuntimeError("interrupted") + return original(nchunk) + + monkeypatch.setattr(proxy.src, "get_chunk", interrupted) + with pytest.raises(RuntimeError, match="interrupted"): + proxy.fetch(max_concurrency=1) + assert proxy.cache_bytes <= 100 + + +def test_concurrent_reads_with_eviction(): + from concurrent.futures import ThreadPoolExecutor + + url, data = _remote_array("concurrent-memory.b2nd", nchunks=3, chunk_size=10_000) + proxy = blosc2.open(url, lazy=True, max_cache_bytes=100) + slices = [slice(0, 20_000), slice(10_000, 30_000)] * 4 + with ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(proxy.__getitem__, slices)) + for item, result in zip(slices, results, strict=True): + np.testing.assert_array_equal(result, data[item]) + assert proxy.cache_bytes <= 100 + + +def test_legacy_cache_url_open_preserves_file(tmp_path): + url, data = _remote_array("legacy-preserve.b2nd", nchunks=1, chunk_size=100) + path = tmp_path / "legacy.b2nd" + legacy = blosc2.Proxy(blosc2.FsspecNDSource(url), urlpath=path, mode="a") + np.testing.assert_array_equal(legacy[:], data) + before = path.read_bytes() + with pytest.raises(ValueError, match="open legacy Proxy caches directly"): + blosc2.open(url, lazy=True, cache_path=path) + assert path.read_bytes() == before + np.testing.assert_array_equal(blosc2.open(path)[:], data) + + +def test_memory_warm_export_is_cold(): + url, _ = _remote_array("warm-memory.b2nd", nchunks=1, chunk_size=100) + proxy = blosc2.open(url, lazy=True) + proxy[:] + carrier = blosc2.ndarray_from_cframe(proxy.to_cframe()) + assert carrier.schunk.vlmeta["b2o"]["cache_policy"] == "memory" + assert not carrier.schunk.vlmeta.get("proxy-fetched") + assert proxy.cache_bytes > 0 + + +def test_cold_export_cannot_overwrite_live_carrier(tmp_path): + url, _ = _remote_array("same-export.b2nd", nchunks=1, chunk_size=100) + path = tmp_path / "live.b2nd" + proxy = blosc2.open(url, lazy=True, cache_path=path) + proxy[:] + before = path.read_bytes() + for kwargs in ({"include_cache": False}, {"cache_policy": blosc2.CachePolicy.NONE}): + with pytest.raises(ValueError, match="different destination"): + proxy.save(path, **kwargs) + assert path.read_bytes() == before + + +def test_unlimited_disk_cache_does_not_evict(tmp_path): + url, data = _remote_array("unlimited-disk.b2nd", nchunks=5, chunk_size=20) + carrier_path = tmp_path / "unlimited.b2nd" + proxy = blosc2.open(url, lazy=True, cache_path=carrier_path, max_cache_bytes=None) + assert proxy.max_cache_bytes is None + + # Read all chunks + np.testing.assert_array_equal(proxy[:], data) + assert proxy.cache_bytes > 0 + initial_cache_bytes = proxy.cache_bytes + + # Access individual chunks again, verify no eviction occurred + for i in range(5): + np.testing.assert_array_equal(proxy[i * 20 : (i + 1) * 20], data[i * 20 : (i + 1) * 20]) + assert proxy.cache_bytes == initial_cache_bytes + + # Carrier has all chunks warm + carrier = blosc2.ndarray_from_cframe(proxy.to_cframe()) + assert carrier.schunk.cbytes == initial_cache_bytes + assert carrier.schunk.vlmeta.get("proxy-fetched") == b"\x1f" + assert carrier.schunk.vlmeta["b2o"]["max_cache_bytes"] is None + + # Export tests + # 1. Exporting with cache_policy=DISK preserves None limit + disk_export = blosc2.ndarray_from_cframe(proxy.to_cframe(cache_policy=blosc2.CachePolicy.DISK)) + assert disk_export.schunk.vlmeta["b2o"]["cache_policy"] == "disk" + assert disk_export.schunk.vlmeta["b2o"]["max_cache_bytes"] is None + + # 2. Exporting with cache_policy=MEMORY falls back to default limit + mem_export = blosc2.ndarray_from_cframe(proxy.to_cframe(cache_policy=blosc2.CachePolicy.MEMORY)) + assert mem_export.schunk.vlmeta["b2o"]["cache_policy"] == "memory" + assert mem_export.schunk.vlmeta["b2o"]["max_cache_bytes"] == blosc2.remote_proxy.DEFAULT_DISK_CACHE_BYTES + + # 3. Exporting with cache_policy=NONE sets limit to None + none_export = blosc2.ndarray_from_cframe(proxy.to_cframe(cache_policy=blosc2.CachePolicy.NONE)) + assert none_export.schunk.vlmeta["b2o"]["cache_policy"] == "none" + assert none_export.schunk.vlmeta["b2o"]["max_cache_bytes"] is None From 7ff88423056e9bf92c4ee4c620d42caa31cc67ed Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 5 Sep 2026 14:58:34 +0200 Subject: [PATCH 16/89] Fix remote proxy cache accounting and document quota behavior --- doc/guides/remote_arrays.md | 2 +- plans/remote-proxy-v4.md | 46 +++++++++++++++++++++++-------------- src/blosc2/proxy.py | 5 ++++ tests/test_remote_proxy.py | 18 +++++++++++++++ 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index 33cc8789e..a56372671 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -201,7 +201,7 @@ a = blosc2.open("https://datasets.example.org/big.b2nd", lazy=True) a.save("portable.b2nd", cache_policy=blosc2.CachePolicy.NONE) ``` -Caterva2 servers accept persisted `MEMORY` carriers (under opt-in policy) but execute them without retained caching (identical to `NONE`), repeatedly fetching required regions from the remote source while preserving the requested limit for downloads; older Caterva2 servers reject `MEMORY` resolution entirely. Use `DISK` if you want Caterva2 to retain compressed chunks on the server within its configured quota. Policy-changing or cold exports must use a different destination from the live disk cache. +Caterva2 servers accept persisted `MEMORY` carriers (under opt-in policy) but execute them without retained caching (identical to `NONE`), repeatedly fetching required regions from the remote source while preserving the requested limit for downloads; older Caterva2 servers reject `MEMORY` resolution entirely. `DISK` retains compressed chunks when no customer quota is configured. On quota-enabled servers, valid warm disk chunks are reused but misses are served without retention; automatic fills await a shared physical-storage reservation mechanism. Policy-changing or cold exports must use a different destination from the live disk cache. Memory-only access accepts runtime URLs such as signed URLs and fsspec chains. Such URLs cannot be exported or obtained as portable `.source` descriptors; disk caching and reference-only construction continue to require persistable URLs. diff --git a/plans/remote-proxy-v4.md b/plans/remote-proxy-v4.md index d778115e5..dad5a807e 100644 --- a/plans/remote-proxy-v4.md +++ b/plans/remote-proxy-v4.md @@ -14,8 +14,8 @@ existing HTTPS security boundary. 2. Support `max_cache_bytes=None` for `CachePolicy.DISK` in both Python-Blosc2 and Caterva2. Passing `None` disables LRU cache eviction (unbounded cache size). On Caterva2, an unbounded DISK cache operates without eviction when no server - customer quota is set, and is clamped to remaining capacity (`retained + available`) - when a customer storage quota is configured. + customer quota is set. With a customer storage quota, carriers are read-only: + valid warm chunks are reused and misses are served without retention. No server memory-cache registry, aggregate memory-cache quota, new configuration knob, carrier format version, or Python-Blosc2 runtime-construction hook is needed. @@ -94,8 +94,8 @@ downloads required by this proposal. | --- | --- | --- | --- | --- | | NONE | null | NONE | None | NONE | | MEMORY | positive integer; default 268435456 | NONE | None between operations | MEMORY with original limit | -| DISK | positive integer | DISK | Carrier cache within cap and quota | DISK with positive limit | -| DISK | null (unbounded) | DISK | Carrier cache within customer quota (unbounded if no quota) | DISK with unbounded cache (no eviction) | +| DISK | positive integer | DISK | Bounded fills without quota; read-only cache with quota | DISK with positive limit | +| DISK | null (unbounded) | DISK | Unbounded fills without quota; read-only cache with quota | DISK with unbounded cache (no eviction) | For a MEMORY carrier: @@ -123,13 +123,13 @@ For an unbounded DISK carrier (`max_cache_bytes=None`): - In Python-Blosc2, chunks fetched into the carrier are retained without LRU eviction (`proxy.cache_bytes` reflects `carrier.schunk.cbytes`). - In Caterva2, `_validated_source()` accepts `max_cache_bytes: null` for `disk`. -- When server customer storage quota is enabled, `remote_proxy_cache_limit()` clamps - the effective limit to remaining capacity (`retained + available`), preventing - the unbounded carrier from exceeding server quota. When quota is disabled, the - proxy operates without a limit (`None`). -- `ServerRemoteProxy.current_cache_bytes()` falls back to `carrier.schunk.cbytes` - when `proxy-cache-sizes` is not present, since `blosc2.Proxy` does not maintain - an LRU size table for unbounded caches. +- When customer quota is enabled, `remote_proxy_cache_limit()` returns zero: + disk caches are consumed read-only and misses use temporary assembly. Payload + limits cannot reserve physical metadata growth across all writers and workers. + With quota disabled, the proxy operates without a limit (`None`). +- `ServerRemoteProxy.current_cache_bytes()` uses `carrier.schunk.cbytes`, not + potentially stale or user-supplied size tables. Unbounded Proxy writes remove + old `proxy-cache-sizes` metadata so later bounded readers rebuild accounting. - Exporting a DISK proxy to DISK preserves `max_cache_bytes=None`; exporting to MEMORY falls back to `DEFAULT_DISK_CACHE_BYTES` (since MEMORY requires a finite positive integer limit); exporting to NONE sets `max_cache_bytes=None`. @@ -141,8 +141,8 @@ For an unbounded DISK carrier (`max_cache_bytes=None`): In `_validated_source()`: - Keep the exact field, version, source-kind, and default-deny checks. -- Validate both `"memory"` and `"disk"` using the same positive-integer rule. - Reject missing/null limits, booleans, floats, strings, zero, and negatives. +- MEMORY requires a positive integer; DISK accepts a positive integer or null. + Reject missing fields, booleans, floats, strings, zero, and negatives. - Keep `"none"` restricted to a null limit and reject unknown policy strings. - Update denial messages so MEMORY is recognized rather than described as an unsupported policy. @@ -255,14 +255,13 @@ In Caterva2: `max_cache_bytes: null` for `disk`, while rejecting invalid types (booleans, strings, non-positive numbers). Keep `memory` strictly requiring positive integers. - In `caterva2/services/remote_proxy.py::ServerRemoteProxy`: - - In `current_cache_bytes()`: check if `proxy-cache-sizes` is present in `vlmeta`; - if absent (as is the case when `blosc2.Proxy` runs with `max_cache_bytes=None`), - fall back to `carrier.schunk.cbytes`. + - In `current_cache_bytes()`: use physical compressed payload (`schunk.cbytes`) + rather than trusting persisted accounting tables. - In `_backend()`: handle `self.max_cache_bytes is None` when applying `cache_limit`, avoiding `TypeError` in `min()`. - In `caterva2/services/server.py::remote_proxy_cache_limit()`: when `proxy.max_cache_bytes is None`, return `None` if no customer quota is configured; - if customer storage quota is enabled, clamp to available quota (`retained + available`). + if customer storage quota is enabled, return zero and reuse disk carriers read-only. - Update docs and sample config (`doc/utilities/cat2-server.md`, `caterva2-server.sample.toml`). - Add tests in `caterva2/tests/test_remote_proxy.py` and `caterva2/tests/test_api.py`. @@ -310,6 +309,19 @@ Extend `caterva2/tests/test_api.py`: ### Verification results +The results below record the original v4 implementation. Review follow-up replaces +approximate quota clamping with read-only disk caches on quota-enabled servers, +invalidates stale size tables during unbounded writes, and restores test globals +using monkeypatch. Strict automatic growth under quota remains deferred until all +writers share physical-storage reservations; it is not delivered by payload LRU +limits. Follow-up regressions cover transitions, stale tables, warm read-only hits, +misses, and concurrent reads of different carriers. + +Follow-up verification in the blosc2 environment: 133 passed, 1 skipped in the +combined targeted client/resolver/API run; 213 passed in the broader Python +proxy/fsspec/expression regression run. Ruff lint and formatting checks pass in +each repository using its own working directory and configuration. + The implementation is verified across both repositories in the `blosc2` conda environment: 1. **Python-Blosc2**: diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index becfc7c74..e3262ff1e 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -649,6 +649,11 @@ def _save_fetched(self) -> None: self._schunk_cache.vlmeta["proxy-cache-sizes"] = { str(nchunk): size for nchunk, size in self._cache_sizes.items() } + elif "proxy-cache-sizes" in self._schunk_cache.vlmeta: + # A cache may previously have been bounded (for example by a server + # quota). Unbounded writes do not maintain this table, so remove it + # before a future bounded reader can mistake old sizes for current ones. + del self._schunk_cache.vlmeta["proxy-cache-sizes"] # Where the source read things to be, so the next run over this cache need # not ask again. Only for a source that can name the bytes it read: an # unstamped one cannot tell a replaced frame from the one these positions diff --git a/tests/test_remote_proxy.py b/tests/test_remote_proxy.py index 2c82b1e2d..6cc8a61bf 100644 --- a/tests/test_remote_proxy.py +++ b/tests/test_remote_proxy.py @@ -18,6 +18,24 @@ from blosc2.b2objects import decode_b2object_payload +def test_bounded_unbounded_cache_accounting_transition(tmp_path): + url, data = _remote_array("accounting-transition.b2nd", nchunks=3, chunk_size=10_000) + path = tmp_path / "transition.b2nd" + creator = blosc2.RemoteProxy( + url, cache_policy=blosc2.CachePolicy.DISK, cache_path=path, max_cache_bytes=None + ) + bounded = blosc2.Proxy(creator.src, _cache=creator._carrier, _max_cache_bytes=20_000) + np.testing.assert_array_equal(bounded[:10_000], data[:10_000]) + assert creator.schunk.vlmeta.get("proxy-cache-sizes") + unbounded = blosc2.open(path, mode="a") + np.testing.assert_array_equal(unbounded[:], data) + assert "proxy-cache-sizes" not in unbounded.schunk.vlmeta + bounded = blosc2.Proxy(unbounded.src, _cache=unbounded._carrier, _max_cache_bytes=20_000) + assert bounded._retained_cache_bytes() == unbounded.schunk.cbytes + np.testing.assert_array_equal(bounded[:], data) + assert bounded._retained_cache_bytes() <= 20_000 + + def _remote_array(name="remote-proxy.b2nd", *, nchunks=4, chunk_size=100_000): data = np.random.default_rng(1).integers(0, 256, nchunks * chunk_size, dtype=np.uint8) array = blosc2.asarray(data, chunks=(chunk_size,), blocks=(chunk_size,)) From 1d193f636cc79469b7d15ce5bb2e088494b5f92f Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 5 Sep 2026 18:27:43 +0200 Subject: [PATCH 17/89] Skip remote proxy tests when fsspec is not installed --- tests/test_remote_proxy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_remote_proxy.py b/tests/test_remote_proxy.py index 6cc8a61bf..1e0868590 100644 --- a/tests/test_remote_proxy.py +++ b/tests/test_remote_proxy.py @@ -9,7 +9,6 @@ import asyncio -import fsspec import numpy as np import pytest @@ -17,6 +16,8 @@ import blosc2.c2array as blosc2_c2array from blosc2.b2objects import decode_b2object_payload +fsspec = pytest.importorskip("fsspec") + def test_bounded_unbounded_cache_accounting_transition(tmp_path): url, data = _remote_array("accounting-transition.b2nd", nchunks=3, chunk_size=10_000) From 14299106a50adcbc4f43deafe21c92a16434d5eb Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 6 Sep 2026 00:17:47 +0200 Subject: [PATCH 18/89] Document remote proxy v5 storage admission --- doc/guides/remote_arrays.md | 2 +- plans/remote-proxy-v5.md | 453 ++++++++++++++++++++++++++++++++++++ 2 files changed, 454 insertions(+), 1 deletion(-) create mode 100644 plans/remote-proxy-v5.md diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index a56372671..b1465780c 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -201,7 +201,7 @@ a = blosc2.open("https://datasets.example.org/big.b2nd", lazy=True) a.save("portable.b2nd", cache_policy=blosc2.CachePolicy.NONE) ``` -Caterva2 servers accept persisted `MEMORY` carriers (under opt-in policy) but execute them without retained caching (identical to `NONE`), repeatedly fetching required regions from the remote source while preserving the requested limit for downloads; older Caterva2 servers reject `MEMORY` resolution entirely. `DISK` retains compressed chunks when no customer quota is configured. On quota-enabled servers, valid warm disk chunks are reused but misses are served without retention; automatic fills await a shared physical-storage reservation mechanism. Policy-changing or cold exports must use a different destination from the live disk cache. +Caterva2 servers accept persisted `MEMORY` carriers (under opt-in policy) but execute them without retained caching (identical to `NONE`), repeatedly fetching required regions from the remote source while preserving the requested limit for downloads; older Caterva2 servers reject `MEMORY` resolution entirely. `DISK` retains compressed chunks up to its payload limit. Caterva2 servers with v5 storage admission can also retain misses under a configured customer quota, after reserving the full physical replacement size through their shared SQLite ledger. If admission fails, reads still return data without retaining misses. Older quota-enabled servers only reuse valid warm chunks. Policy-changing or cold exports must use a different destination from the live disk cache. Memory-only access accepts runtime URLs such as signed URLs and fsspec chains. Such URLs cannot be exported or obtained as portable `.source` descriptors; disk caching and reference-only construction continue to require persistable URLs. diff --git a/plans/remote-proxy-v5.md b/plans/remote-proxy-v5.md new file mode 100644 index 000000000..3096b79df --- /dev/null +++ b/plans/remote-proxy-v5.md @@ -0,0 +1,453 @@ +# Plan: Shared storage reservations and growable remote proxies under customer quota (v5) + +## Status and objective + +First staged-replacement implementation added to Caterva2. The design below +records the original proposal; the implementation decisions and remaining work +in the next section supersede its open choices. This is not a claim that every +benchmark, optimization, or observability item below is complete. + +## Implementation decisions and remaining work + +- `services/storage_quota.py` uses standard-library `sqlite3`, WAL, FULL + synchronous transactions, schema version 1, and separate `account`, `objects`, + and `operations` tables. Connections are short-lived and never shared between + threads/processes. Usage is summed transactionally rather than duplicated in + independently updated counters. +- Owned `public`, `shared`, and `personal` regular files are charged by `st_size`. + Peer storage retains its separate budget. Media, authentication files, + SQLite/WAL, directories, and lock sidecars are excluded operational storage. + This deliberately replaces the old whole-state-directory accounting boundary. +- Candidates are built in memory, then reserve positive final-size growth and + their entire staging-file size before disk writes. `quota_work_bytes` defaults + to `"1G"` and bounds aggregate coordinated disk staging. It does not bound RAM, + HTTP request spooling, operational metadata, or old inodes held by readers; + customer quota is not a guarantee against filling the underlying volume. +- Atomic replacement is the selected correctness reference. Stable per-path OS + locks fence publishers and recovery; generation checks reject stale snapshots. + No heartbeat or TTL frees a live writer's reservation. Recovery syncs the + surviving target, cleans operation-owned staging, then records actual size. + Admission pressure also attempts recovery of other dead workers' operations. +- A shared initialization barrier fences startup inventory against publishers. + Startup reconciles offline changes and reduced quotas. External online writers + are unsupported. All workers must use the same configuration. Local + `publish_root` cannot point into the server's state directory under quota. +- Quota-enabled uploads/imports, chunk writes, append, expression output, + notebooks, HDF5 proxy creation, copying/moving, deletion, publishing metadata, + and archive extraction use shared admission. Directory/archive operations are + file-by-file, not transactional batches; moves copy before deleting and need + capacity for both. Reserved `.b2lock` sidecars may keep removed directories + from disappearing. Arbitrary out-of-band writes from user code are not covered. +- Both logical slices and compressed chunk requests retain DISK misses only + after admission; denials, contention, and publication failure retain the + fetched result. Finite and unlimited payload caps remain distinct from quota. + MEMORY still executes as NONE; portable descriptors and client defaults do + not change. No new Python-Blosc2 runtime API was required. +- One bounded reclamation pass can cold-replace up to four previously validated + DISK carriers, oldest first, after recovery and an admission retry. Descriptor + and user metadata survive; cache bitmaps, indexes, and payload are cleared + together. Generation checks fence stale victims. Ordinary data is not evicted. +- `StorageQuota.usage()` reports committed, reserved, and working bytes. Public + admin endpoints, persistent denial/recovery counters, precise reclaimable-byte + reporting, finer victim fairness/hysteresis, and optimized partial pruning + remain follow-up work. + +### Validation and performance baseline + +New local tests cover exact admission, metadata overhead, separate staging +budgets, independent-process competition, live-owner recovery fencing, worker +death before/after replacement, offline reconciliation, open reader snapshots, +generation conflicts, source reads on denial/SQLite failure, cross-proxy pruning, +threaded uploads versus fills, and authenticated writer/fetch API integration. +The existing remote-proxy and API tests are also used as regression coverage. +Validation in the blosc2 environment: 89 quota/resolver tests passed with +warnings treated as errors. The authenticated API, chunk-write and HDF5 suites +passed with customer quota temporarily enabled in the isolated test server +(292 passed, 4 skipped); they also passed without quota (292 passed, 4 skipped). +The temporary configuration was restored and was not installed in the checkout. +The publication regression test now waits for its own fill nonce, not a stale +published file left by an earlier test. Basic partial-block-to-whole-chunk +transitions and legacy Python UDF persistence have dedicated quota tests. +This does not yet exercise every host-power-loss boundary, every supported OS, +or a production-volume parallel workload. + +`examples/benchmark_storage_quota.py` in Caterva2 compares staged admission to +the existing non-quota in-place path (not a quota-safe in-place implementation). +One local macOS run with an 8 MiB incompressible array and an in-memory upstream: + +| Chunk size | In-place cold ms/fill | Staged cold ms/fill | Staged warm ms/read | Candidate bytes written | +| --- | ---: | ---: | ---: | ---: | +| 256 KiB | 0.952 | 8.955 | 8.323 | 138,463,971 | +| 1 MiB | 1.712 | 7.918 | 6.958 | 37,756,992 | + +Whole-carrier copying also penalizes warm reads. These are illustrative +microbenchmark measurements, not production performance claims. Efficient warm +reads, a proven-bound in-place path, many-tiny-chunk/partial-block stress tests, +and the broader contention/peak-storage benchmark matrix remain follow-up work. +Large candidates exceeding the staging budget fall back to no retention. + +Restore automatic DISK RemoteProxy cache fills when a Caterva2 customer has a +configured storage quota. Coordinate growth across that customer's proxies, +uploads, and other storage writers, and eventually reclaim disposable cached +chunks across proxies when capacity is needed. + +Keep the v4 read-only behavior as the safe fallback until admission and recovery +are complete: reuse valid warm chunks, serve misses temporarily, and retain +nothing when permission to grow cannot be established. + +## Deployment and execution model + +Confirmed assumptions: + +- One customer's virtual Caterva2 server runs on one host with local storage. +- Its worker processes share one state directory and one customer storage quota. +- Different customers have independent state directories and quota accounts. +- Peer access reaches independent Caterva2 servers on other machines. The local + server accounts for locally retained peer data; no quota database is shared + with the upstream server. +- A proxy does not own a dedicated process or thread. Async requests dispatch + blocking operations to a shared thread pool, and multiple server processes may + handle requests for the same carrier. + +These assumptions permit SQLite coordination across local processes. Sharing the +database across hosts or a network filesystem is outside this design. + +## Existing implementation anchors + +Paths in this section are relative to the Caterva2 repository at +`/Users/faltet/ironArray/caterva2`, inspected during the v4/v5 discussion. + +- `caterva2/services/db.py`: authentication uses SQLAlchemy and aiosqlite with + `/db.sqlite`. The schema currently contains the user table. +- `caterva2/services/server.py::lifespan`: initializes/disposes that database + conditionally on authentication. Storage coordination must not inherit that + dependency on login being enabled. +- `caterva2/services/srv_utils.py::Database`: `/db.json` holds server + state as an in-memory model rewritten to JSON. It is not a transactional quota + store and must not be used for cross-worker reservations. +- `server.py::get_disk_usage`, `get_disk_usage_written`, and + `account_chunk_written`: current accounting uses directory scans and a + process-local counter. These are insufficient for shared admission. +- `server.py::remote_proxy_cache_limit`: currently returns zero under customer + quota. `read_remote_proxy` and the `api/chunk` branch use that allowance. +- `services/remote_proxy.py::ServerRemoteProxy`: receives an already authorized + source, uses the carrier for DISK caching, and supports read-only warm reuse + with temporary miss assembly when cache allowance is zero. +- `server.py::dataset_lock`, `dataset_thread_lock`, and + `remote_proxy.py::carrier_thread_lock`: existing process-local guards accompany + Blosc2 carrier file locks. Preserve their ordering and thread/GIL protections. +- `caterva2/c2cache/peercache.py`: existing peer-cache pruning provides useful + chunk-eviction and recency mechanisms, but its scans and post-growth eviction + do not implement cross-worker storage reservations. + +Python-Blosc2 anchors are `src/blosc2/proxy.py` for fetched bitmaps, physical +chunks, size accounting, and eviction, and `src/blosc2/remote_proxy.py` for the +portable carrier contract. A new physical-growth planning hook may be needed; +it is not assumed to exist today. + +## Proposed architecture + +Introduce a local storage coordinator, backed by `/storage.sqlite`. +Every quota-consuming server mutation asks it for admission. SQLite coordinates +ownership of capacity; per-carrier locks protect file contents and readers. + +```mermaid +flowchart TD + A["Concurrent requests: proxies, uploads, other writes"] --> C["Storage coordinator"] + C <--> DB[("storage.sqlite: usage, objects, reservations")] + C --> D{"Capacity available?"} + D -->|Yes| R["Reserve physical growth"] + R --> W["Mutate under target carrier lock"] + W --> F["Record actual size; release reservation"] + F --> DB + D -->|No| P["Claim disposable cache victim"] + P --> E["Prune under victim carrier lock"] + E --> U["Measure reclaimed space; update usage"] + U --> D + D -->|No eligible victim| N["Proxy read succeeds without retaining misses"] +``` + +Network fetching occurs outside SQLite transactions. Successful reservations +remain recorded while filesystem work runs; they do not require an open database +transaction. Pruning must be bounded so repeated admission retries cannot loop +indefinitely under pressure. + +Use a dedicated database rather than extending authentication tables. This +separates lifecycle, schema migration, and frequent accounting traffic from user +management while reusing the installed SQLite/aiosqlite infrastructure. + +## Accounting contract + +For one quota account, the admission invariant is: + +```text +committed chargeable storage + outstanding reserved growth <= customer quota +``` + +If existing data already exceeds a newly configured or reduced quota, preserve +user data, prohibit positive-growth admissions, and allow reads and safe pruning. +Do not pretend the invariant already holds during that reconciliation state. + +Distinguish two different limits: + +- The carrier's `max_cache_bytes` bounds retained compressed cache payload. A + null DISK limit disables its own LRU bound. +- Customer quota bounds chargeable physical file growth, including carrier + headers, indexes, bitmaps, and metadata. It cannot be implemented by passing + remaining customer capacity as a compressed-payload limit. + +An unlimited carrier has no private cache cap, but it never bypasses customer +admission. MEMORY continues to execute as NONE on Caterva2; no server memory +cache registry or memory quota is introduced. + +### Scope decisions required before implementation + +Define chargeable files once and reuse that definition for migration, scanning, +admission, and reconciliation. Inventory public, shared, personal, peer-cache, +temporary, and internal state files. Existing peer-cache budgets are separate +policies; do not accidentally exclude their physical files from customer quota +if they are currently counted. Either make their writers participate or explicitly +document and implement a different accounting boundary. + +Decide whether quota means apparent file length (matching current `st_size` +accounting) or allocated filesystem blocks. Proposed first version: retain +`st_size` semantics and describe this as chargeable stored bytes, not a guarantee +against exhausting the underlying volume. + +Database/WAL/SHM files, lock sidecars, and working storage need an explicit policy. +Recommended: distinguish managed data quota from bounded operational headroom, +and do not charge recursive growth of the quota ledger through its own ledger. +The operational budget must still be provisioned and bounded. This is a deliberate +accounting-policy decision, not permission to create unlimited temporary files. + +## Database model and lifecycle + +Suggested minimal tables (final SQL and migration strategy are implementation work): + +| Table | Core fields | Role | +| --- | --- | --- | +| `storage_usage` | account ID, quota, committed bytes, reserved bytes, reconciliation state | One account per customer state directory | +| `objects` | stable object ID, relative path, generation, measured bytes, type, cache eligibility, coarse last-use time | Identify storage and pruning candidates | +| `operations` | operation ID, object ID, owner token, expected generation, reserved bytes, state, heartbeat, recovery metadata | Durable reservations and mutation intent | +| `schema_version` | version | Explicit migrations independent of authentication | + +Avoid counting the same reservation both in the usage row and operation rows +without transactional updates and a reconciliation check. Add nonnegative-value +constraints and uniqueness rules for live object mutation claims. + +Initialize storage coordination for quota-enabled servers regardless of login. +Use one database engine/pool per process; do not share connections across process +forks. All workers must see the same configured quota. WAL and a bounded busy +timeout are candidates; configure them explicitly and verify their behavior in +multi-process tests. Use durable transaction settings suitable for reservations. + +Use short write transactions, such as `BEGIN IMMEDIATE`, to check capacity and +create/update the reservation atomically. Handle SQLite contention explicitly; +cache admission may fall back to no retention after bounded retries. + +Do not place network transfers, filesystem scans, carrier copying, or waits for +file locks inside a SQLite transaction. Do not update database recency per block +read; batch/coarsen touches so hot reads do not serialize on the SQLite writer. + +## Growth workflow + +1. Resolve the remote source through the existing authorized filesystem path. + Fetch missing data into operation-scoped buffers outside database transactions. + Apply existing request/resource controls and a bounded working-storage policy. +2. Prepare a candidate change or conservative physical-growth bound. Acquire the + target carrier's existing mutation guards and verify source identity, geometry, + and carrier generation. If preparation used an earlier generation, revalidate + or rebuild before admission. +3. In a short database transaction, verify object ownership/generation and reserve + positive physical growth if capacity permits. Persist mutation intent and an + operation token. Commit the database transaction before filesystem mutation. +4. Perform the admitted mutation under the carrier lock. Never exceed the reserved + physical-growth bound. If more space is required, acquire an additional + reservation before that growth or abandon the candidate safely. +5. Measure the result and finalize in a short transaction: update object size and + generation, adjust committed bytes, consume/release the reservation, and mark + the operation complete. +6. Return the logical result whether retention was admitted or not. Failure of + retention must not turn a successfully fetched result into incomplete data. + +Uploads and ordinary user writes differ at step 6: admission denial returns the +existing appropriate quota error rather than silently losing a requested write. +Replacement credits old storage only when it is actually replaced; two live +copies during staging must not be counted as one if working storage is included. + +## Determining physical growth: prototype before selecting the production path + +### Option A: staged carrier replacement + +Build a candidate carrier, measure its serialized file size, and publish it only +after admission and generation validation. This provides an exact final-size +baseline and keeps rejected candidates from modifying the live carrier. + +Costs and required checks: + +- Copying/rebuilding large carriers per small fill may be prohibitive. Batch + chunk admissions where practical and measure write amplification. +- Temporary space must be bounded before candidate creation; final-size + admission alone does not bound peak physical occupancy. +- Atomic rename does not by itself make SQLite and filesystem state atomic. +- Verify how carrier sidecar locks, already open handles, inode replacement, + downloads, and platform-specific rename behavior interact. All relevant readers + must obey a compatible lifetime/locking protocol. +- Persist enough intent to distinguish an unpublished candidate from a published + replacement whose database finalization was interrupted. Flush/fsync ordering + and parent-directory durability must be specified. + +Use this as the correctness reference, not an assumption that replacing files is +already safe with the existing open-handle behavior. + +### Option B: reserve a proven bound and mutate in place + +Preferable for frequent chunk fills if Python-Blosc2/C-Blosc2 can provide a +reliable upper bound on all physical growth, including metadata and any temporary +rewrite space. A cache update planning/admission hook may be necessary. + +Do not guess a fixed metadata allowance or reserve only compressed chunk bytes. +Specify interrupted-write behavior, accounting updates, and rollback/recovery +before enabling this path. If a reliable bound cannot be established, retain +the staged reference path or skip retention. + +Benchmark both paths on cold and warm carriers, many tiny chunks, large contiguous +frames, partial blocks, and batches. Select the production path from measured +costs and demonstrated correctness. + +## Pruning across proxies + +After shared admission works, permit reclaiming disposable DISK cache payload +from other proxies. Preserve descriptors, geometry, original data, and user +metadata. Cache eviction must use the cache engine's bookkeeping-aware mechanisms, +not raw chunk replacement that leaves fetched bits or indexes inconsistent. + +Start with coarse per-proxy recency and prune chunks in batches. Keep per-proxy +LRU for its own cap. Define priority so a hot proxy cannot repeatedly strip every +other proxy's working set; use hysteresis/bounded work to avoid fill-prune thrash. + +Pruning sequence: + +1. Select and claim an eligible victim in a short transaction. Do not credit the + space expected to be reclaimed. +2. End the transaction, acquire that carrier's guards, and revalidate the claim, + generation, and eligibility. Skip active or unavailable victims after bounded + waiting. +3. Evict a batch safely and measure the resulting physical file length. If pruning + itself needs temporary growth, account for it before starting. +4. Commit the measured reduction and release the claim. Only now can reservations + spend the freed capacity. + +Avoid holding the requesting carrier's lock while waiting for a victim's lock. +Release/revalidate the requester when necessary. Keep one consistent lock order: +carrier guard may enclose short SQLite work, but an open SQLite transaction must +never wait for a carrier. No operation should hold multiple carrier locks for a +routine pruning pass. + +## Crash recovery and reconciliation + +SQLite transactions do not include B2ND file writes. Model operations explicitly, +for example `reserved -> applying -> completed`, with an aborted/recovering path. +Finalize and abort operations idempotently using their unique tokens. + +Recovery must cover: + +- Reservation committed, filesystem work never started. +- Partial in-place write or partially built candidate. +- Replacement published, final size not recorded in SQLite. +- Pruning completed, reclaimed capacity not yet credited. +- Worker cancellation, process termination, host restart, or ledger unavailability. + +A heartbeat timeout is only a signal to investigate. Do not free a reservation +while its owner might still write. Recovery needs carrier-lock acquisition, +owner/generation fencing, and inspection of actual files. An old worker must +verify its token is still authorized before publishing or mutating after a claim +has been recovered. Prefer conservative over-accounting until reconciliation. + +Initial inventory and recovery scans must not race untracked writers. Establish +an initialization/reconciliation barrier, handle multiple workers starting at +once, and define how explicit out-of-band filesystem edits are detected. External +uncoordinated writers cannot be covered by a strict online quota guarantee. +If the ledger is unavailable or inconsistent, serve reads without new retention +and fail quota-controlled user mutations clearly rather than bypassing admission. + +## Implementation stages + +### Stage 1: shared accounting and reservations + +- Add `services/storage_quota.py` (proposed module), schema migration, startup, + shutdown, inventory, and recoverable reservation primitives. +- Audit every physical writer: uploads, URL imports, chunk writes, array/store + creation, replacements, transformations/unfolding, deletion/rename, and any + chargeable peer-cache or background writes. +- Route all quota-consuming writers through the coordinator. Preserve quota + error semantics for explicit user mutations. +- Test contention and recovery independently of remote cache filling. +- Keep quota-enabled remote caches read-only until the writer audit is complete. + +### Stage 2: admit DISK proxy growth + +- Prototype and benchmark physical-growth strategies, including working storage. +- Integrate the selected strategy into both slice/index and compressed-chunk + requests while retaining the authorized source and existing locks. +- Enforce both the private payload cap and shared physical quota. Support null + DISK limits and finite limits equally. +- Retain misses when reservation succeeds; otherwise return fetched results + without retaining them. No automatic cross-proxy pruning is required yet. +- Preserve warm/cold exports, source invalidation, and MEMORY-to-NONE behavior. + +### Stage 3: coordinated cross-proxy pruning + +- Add coarse recency, victim claims, safe batch eviction, fairness, and bounded + admission retry after actual reclamation. +- Permit pruning for explicit uploads as well as cache fills if that product + policy is accepted; never reclaim ordinary user data automatically. +- Add diagnostics for used/reserved/reclaimable bytes, denied cache admissions, + recovered operations, and pruning. Do not expose credentials or source secrets. + +## Test and benchmark matrix + +- Two processes reserve against one quota: e.g. with 100 MiB free, 70 MiB and + 50 MiB requests cannot both be admitted without intervening reclamation. +- Threads, separate event loops, and separate server workers; no dedicated worker + per proxy assumed. Database startup and schema initialization races included. +- Carrier metadata causes more growth than compressed payload; reservation covers + the full admitted change. Include partial-block and many-small-chunk cases. +- Upload competes with proxy fill, two distinct proxies fill concurrently, and + chunk writes compete with both. All use the same account. +- Quota full, quota reduced below existing usage, ledger busy/unavailable, and + no reclaimable chunks: reads still succeed without cache growth. +- A victim is in use, replaced, deleted, or warmed after candidate selection; + no stale decision corrupts data or frees fictitious capacity. +- Kill a worker at each filesystem/database boundary and recover exactly once; + a paused old worker cannot publish after its claim is fenced out. +- Warm cache reads, bounded/unbounded transitions, source replacement, logical + fetches, compressed chunks, and physical downloads retain v4 correctness. +- Working-storage exhaustion, replacement with live readers, pruning failure, + cancellation, and process restart leave valid carriers or recoverable state. +- Measure database contention, transactions per fill, latency, staging peak space, + bytes rewritten per admitted chunk, and pruning churn under parallel workloads. + +Use deterministic local upstream fixtures and multi-process tests on temporary +local state directories. Run Python and test/build commands in the blosc2 conda +environment. No runtime tests are claimed by this planning document. + +## Acceptance and non-goals + +V5 is complete when quota-enabled servers can safely grow DISK proxy caches, +different workers cannot spend the same capacity, physical growth is admitted +before it happens, and interrupted operations reconcile without under-accounting +or corrupted reads. Cross-proxy pruning only credits verified reclaimed space. + +No new carrier format or changed client MEMORY default is required. Distributed +quota coordination, shared network-filesystem SQLite, private/signed upstream +server access, server MEMORY retention, and hard process-RAM limits are out of +scope. SQLite is the local coordination mechanism, not a substitute for carrier +integrity locks or for a defined filesystem durability protocol. + +## SQLite references + +- [Transactions](https://www.sqlite.org/lang_transaction.html): write transaction + admission, BEGIN IMMEDIATE, and contention behavior. +- [Write-ahead logging](https://www.sqlite.org/wal.html): concurrent readers, + single-writer coordination, checkpoints, and the same-host constraint. From a7298d534e538c00dbe1f608f34fdeeaddf5980e Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 6 Sep 2026 09:18:53 +0200 Subject: [PATCH 19/89] New RemoteProxy.with_sparse_cache() for deploying a sparse cache --- src/blosc2/proxy.py | 81 +++++++++++++--- src/blosc2/remote_proxy.py | 191 +++++++++++++++++++++++++++++++++---- tests/test_remote_proxy.py | 137 ++++++++++++++++++++++++++ 3 files changed, 377 insertions(+), 32 deletions(-) diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index e3262ff1e..f3a49f7b4 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -47,6 +47,7 @@ "proxy-fetched", "proxy-fetched-blocks", "proxy-fetched-bpc", + "proxy-dirty", "proxy-stamp", "proxy-index", } @@ -151,6 +152,7 @@ def __init__( kwargs = {} self._cache = kwargs.pop("_cache", None) self._max_cache_bytes = _validate_max_cache_bytes(kwargs.pop("_max_cache_bytes", None)) + self._persistent_dirty = bool(kwargs.pop("_persistent_dirty", False)) vlmeta = kwargs.pop("vlmeta", None) caterva2_env = kwargs.pop("caterva2_env", False) # Before anything is built or emptied: a call that is going to be refused @@ -262,6 +264,7 @@ def __init__( self._specialized = getattr(self._schunk_cache, "nspecialized", 0) if self.urlpath is None: self.urlpath = getattr(self._schunk_cache, "urlpath", None) + self._recover_interrupted_mutation() self._fetched = self._adopt_cache(fresh, self._schunk_cache.nchunks) self._cache_sizes: dict[int, int] = {} self._cache_lru = OrderedDict() @@ -423,6 +426,45 @@ def _mark_fetched(self, nchunk: int, nblock: int | None = None) -> None: for n in blocks: self._fetched[(base + n) // 8] |= 1 << ((base + n) % 8) + def _recover_interrupted_mutation(self) -> None: + """Forget cache state left behind by a process that died while writing. + + Chunk payloads are disposable and may remain on disk, but clearing the + fetched maps makes every one of them unreachable until it has been + fetched and published again. This is deliberately conservative because + a dirty marker cannot identify the exact instruction at which its owner + stopped. + """ + vlmeta = self._schunk_cache.vlmeta + if not self._persistent_dirty or vlmeta.get("proxy-dirty") is None: + return + if getattr(self._schunk_cache, "mode", None) == "r": + return + self._forget_fetched(self._schunk_cache.nchunks) + if vlmeta.get("proxy-cache-sizes") is not None: + del vlmeta["proxy-cache-sizes"] + del vlmeta["proxy-dirty"] + + def _begin_persistent_mutation(self) -> None: + if self._persistent_dirty: + self._schunk_cache.vlmeta["proxy-dirty"] = {"pid": os.getpid(), "version": 1} + + def _end_persistent_mutation(self) -> None: + if self._persistent_dirty and self._schunk_cache.vlmeta.get("proxy-dirty") is not None: + del self._schunk_cache.vlmeta["proxy-dirty"] + + def _refresh_shared_cache(self) -> None: + """Reload proxy bookkeeping after taking a shared cache's frame lock.""" + if not self._persistent_dirty: + return + self._recover_interrupted_mutation() + self._fetched = self._load_fetched(self._schunk_cache.nchunks) + self._cache_sizes.clear() + self._cache_lru.clear() + self._hot_payloads.clear() + self._restore_cache_accounting() + self._specialized = getattr(self._schunk_cache, "nspecialized", 0) + def _is_fetched(self, nchunk: int, nblock: int = 0) -> bool: n = nchunk * self._blocks_per_chunk + nblock return bool(self._fetched[n // 8] >> (n % 8) & 1) @@ -493,18 +535,28 @@ def _enforce_cache_limit(self, item) -> None: self._cache_lru.move_to_end(nchunk) evicted = False - while self._retained_cache_bytes() > self._max_cache_bytes and self._cache_lru: - nchunk, _ = self._cache_lru.popitem(last=False) - self._cache_sizes.pop(nchunk, None) - self._hot_payloads.pop(nchunk, None) - self._schunk_cache.update_special(nchunk, blosc2.SpecialValue.UNINIT) - base = nchunk * self._blocks_per_chunk - for n in range(base, base + self._blocks_per_chunk): - self._fetched[n // 8] &= ~(1 << (n % 8)) - evicted = True - if evicted: - self._specialized = getattr(self._schunk_cache, "nspecialized", self._specialized) - self._save_fetched() + if self._retained_cache_bytes() > self._max_cache_bytes and self._cache_lru: + self._begin_persistent_mutation() + try: + while self._retained_cache_bytes() > self._max_cache_bytes and self._cache_lru: + nchunk, _ = self._cache_lru.popitem(last=False) + self._cache_sizes.pop(nchunk, None) + self._hot_payloads.pop(nchunk, None) + self._schunk_cache.update_special(nchunk, blosc2.SpecialValue.UNINIT) + base = nchunk * self._blocks_per_chunk + for n in range(base, base + self._blocks_per_chunk): + self._fetched[n // 8] &= ~(1 << (n % 8)) + evicted = True + if evicted: + self._specialized = getattr(self._schunk_cache, "nspecialized", self._specialized) + self._save_fetched() + except Exception: + # Keep the dirty marker: the next locked operation will discard the + # generation's fetched state before serving from it. + raise + else: + if evicted: + self._end_persistent_mutation() def _plan(self, item): """Where *item* lands on the cache's grid, read once for a fetch. @@ -800,12 +852,15 @@ def fetch( pass missing = self._missing_chunks(item) + if missing: + self._begin_persistent_mutation() try: for nchunk, chunk in self._get_chunks(missing, max_concurrency): self._store_chunk(nchunk, chunk) finally: if missing: self._save_fetched() + self._end_persistent_mutation() return self._cache @@ -859,6 +914,7 @@ def _fetch_by_block(self, item, max_concurrency: int | None): missing = self._missing_blocks(item) if not missing: return self._cache + self._begin_persistent_mutation() # A transport that batches ranges pays the block path's fixed cost once # for the whole fetch, so what it wants asked is the wave rather than the # chunk; see `ByteRangeNDSource._wave_saves`. @@ -914,6 +970,7 @@ def fetch(task): self._write_blocks(nchunk, pending.pop(nchunk), layouts[nchunk][0]) finally: self._save_fetched() + self._end_persistent_mutation() return self._cache diff --git a/src/blosc2/remote_proxy.py b/src/blosc2/remote_proxy.py index 9af7c632a..d6aabd76f 100644 --- a/src/blosc2/remote_proxy.py +++ b/src/blosc2/remote_proxy.py @@ -13,6 +13,7 @@ import math import os import threading +from contextlib import nullcontext from functools import wraps from urllib.parse import parse_qsl, urlsplit @@ -48,11 +49,19 @@ def _serialized_operation(method): @wraps(method) def locked(self, *args, **kwargs): with self._operation_lock: - try: - return method(self, *args, **kwargs) - finally: - if self._proxy is not None: - self._proxy._enforce_cache_limit(tuple(slice(0, 0) for _ in self.shape)) + cache_lock = ( + self._runtime_cache.holding_lock() + if self._shared_runtime_cache and self._runtime_cache is not None + else nullcontext() + ) + with cache_lock: + if self._shared_runtime_cache and self._proxy is not None: + self._proxy._refresh_shared_cache() + try: + return method(self, *args, **kwargs) + finally: + if self._proxy is not None: + self._proxy._enforce_cache_limit(tuple(slice(0, 0) for _ in self.shape)) return locked @@ -121,10 +130,12 @@ def _validate_payload_limit(policy: blosc2.CachePolicy, limit) -> None: class RemoteProxy(blosc2.Operand): """A persistable, optionally self-caching reference to a remote array. - With :attr:`CachePolicy.DISK`, the persisted B2ND carrier is itself the - bounded cache. With :attr:`CachePolicy.MEMORY`, chunks are retained in - process memory up to a bounded size. With :attr:`CachePolicy.NONE`, reads - retain no data. + With :attr:`CachePolicy.DISK`, the public constructor uses the persisted + B2ND carrier itself as the bounded cache. Server code can instead use + :meth:`with_sparse_cache` to keep a private directory-backed runtime cache + beside a portable carrier. With :attr:`CachePolicy.MEMORY`, chunks are + retained in process memory up to a bounded size. With + :attr:`CachePolicy.NONE`, reads retain no data. Parameters ---------- @@ -160,6 +171,7 @@ def __init__( max_cache_bytes=_POLICY_DEFAULT, max_concurrency: int | None = None, _carrier=None, + _runtime_cache_path=None, ): if not isinstance(cache_policy, blosc2.CachePolicy): raise TypeError("cache_policy must be a blosc2.CachePolicy instance") @@ -172,6 +184,7 @@ def __init__( and cache_dir is None and cache_path is None and _carrier is None + and _runtime_cache_path is None ): raise ValueError("CachePolicy.DISK requires cache_dir or cache_path") @@ -188,12 +201,23 @@ def __init__( self._operation_lock = threading.RLock() self._proxy = None self._carrier = _carrier + self._runtime_cache = _carrier if cache_policy is blosc2.CachePolicy.DISK else None + self._shared_runtime_cache = _runtime_cache_path is not None self._cache_status = None if cache_policy is blosc2.CachePolicy.DISK: - if self._carrier is None: + if _runtime_cache_path is not None: + self._runtime_cache, self._cache_status = self._open_or_create_sparse_cache( + _runtime_cache_path + ) + elif self._carrier is None: self._carrier, self._cache_status = self._open_or_create_carrier(cache_dir, cache_path) - self._attach_carrier_cache() + self._runtime_cache = self._carrier + cache_lock = ( + self._runtime_cache.holding_lock() if self._shared_runtime_cache else nullcontext() + ) + with cache_lock: + self._attach_carrier_cache() elif cache_policy is blosc2.CachePolicy.MEMORY: self._attach_carrier_cache() @@ -244,9 +268,124 @@ def _open_or_create_carrier(self, cache_dir, cache_path): carrier = self._to_b2object_carrier(urlpath=path, contiguous=True, mode="w") return carrier, "created" + def _open_or_create_sparse_cache(self, cache_path): + """Open a server-owned sparse runtime cache or create it cold. + + This is deliberately separate from ``cache_path`` in the public + constructor: portable RemoteProxy carriers remain contiguous files. + """ + path = os.fspath(cache_path) + if os.path.exists(path): + if not os.path.isdir(path): + raise ValueError("runtime_cache_path must name a sparse frame directory") + runtime = blosc2.blosc2_ext.open( + path, "a", 0, dparams=blosc2.DParams(nthreads=1), locking=True + ) + if runtime.schunk.vlmeta.get("b2o") != self._payload(): + raise ValueError(f"the sparse runtime cache at {path} has a different specification") + self._validate_geometry( + (runtime.shape, runtime.dtype, runtime.chunks, runtime.blocks), src=self.src + ) + stored = runtime.schunk.vlmeta.get("proxy-stamp") + current = getattr(self.src, "stamp", None) + status = ( + "invalidated/rebuilt" + if stored is not None and current is not None and stored != current + else "reused" + ) + return runtime, status + + if self._carrier is not None: + self._validate_warm_seed(self._carrier) + + runtime = self._to_b2object_carrier( + urlpath=path, contiguous=False, mode="w", locking=True + ) + if self._carrier is not None: + self._import_warm_seed(self._carrier, runtime) + return runtime, "created" + + def _import_warm_seed(self, seed, runtime) -> None: + """Migrate valid warm chunks once into a newly-created runtime cache.""" + seed = self._validate_warm_seed(seed) + seed_schunk = getattr(seed, "schunk", seed) + stamp = getattr(self.src, "stamp", None) + if stamp is None or seed_schunk.vlmeta.get("proxy-stamp") != stamp: + return + + bpc = seed_schunk.vlmeta.get("proxy-fetched-bpc", 1) + if not isinstance(bpc, int) or bpc <= 0: + return + key = "proxy-fetched-blocks" if bpc > 1 else "proxy-fetched" + fetched = seed_schunk.vlmeta.get(key) + expected_size = (seed_schunk.nchunks * bpc + 7) // 8 + if not isinstance(fetched, bytes) or len(fetched) != expected_size: + return + for nchunk in range(seed_schunk.nchunks): + start = nchunk * bpc + if any(fetched[n // 8] >> (n % 8) & 1 for n in range(start, start + bpc)): + runtime.schunk.update_chunk(nchunk, seed_schunk.get_chunk(nchunk)) + for name in ( + "proxy-cache-sizes", + "proxy-fetched", + "proxy-fetched-blocks", + "proxy-fetched-bpc", + "proxy-index", + "proxy-stamp", + ): + value = seed_schunk.vlmeta.get(name) + if value is not None: + runtime.schunk.vlmeta[name] = value + + def _validate_warm_seed(self, seed): + seed = getattr(seed, "cache", seed) + seed_schunk = getattr(seed, "schunk", seed) + self._validate_geometry((seed.shape, seed.dtype, seed.chunks, seed.blocks)) + seed_payload = seed_schunk.vlmeta.get("b2o") + if ( + not isinstance(seed_payload, dict) + or seed_payload.get("kind") != "remote_proxy" + or seed_payload.get("source") != self._source + ): + raise ValueError("the warm carrier belongs to a different remote source") + return seed + + @classmethod + def with_sparse_cache( + cls, + urlpath, + runtime_cache_path, + *, + carrier=None, + max_cache_bytes=_POLICY_DEFAULT, + max_concurrency: int | None = None, + ): + """Attach an authorized remote source to a private sparse disk cache. + + This server-facing constructor keeps the portable carrier separate from + the mutable directory-backed runtime cache. All processes using the + directory must construct it through this method so frame locking and + interrupted-mutation recovery remain enabled. + + ``carrier`` is the portable RemoteProxy carrier. If it contains valid + warm chunks when the sparse runtime cache is first created, those chunks + are copied into the runtime cache. Both copies continue to exist until + the server replaces the portable carrier with a cold descriptor. After + migration, only the runtime cache is consulted for cached data, so an + evicted chunk cannot be resurrected from the carrier. + """ + return cls( + urlpath, + cache_policy=blosc2.CachePolicy.DISK, + max_cache_bytes=max_cache_bytes, + max_concurrency=max_concurrency, + _carrier=carrier, + _runtime_cache_path=runtime_cache_path, + ) + def _attach_carrier_cache(self): if self.cache_policy is blosc2.CachePolicy.DISK: - if self._carrier is None: + if self._runtime_cache is None: self._proxy = None return if getattr(self.src, "stamp", None) is None: @@ -256,9 +395,10 @@ def _attach_carrier_cache(self): return self._proxy = blosc2.Proxy( self.src, - _cache=self._carrier, + _cache=self._runtime_cache, _refresh_source=False, _max_cache_bytes=self._cache_limit, + _persistent_dirty=self._shared_runtime_cache, ) elif self.cache_policy is blosc2.CachePolicy.MEMORY: self._proxy = blosc2.Proxy( @@ -439,10 +579,12 @@ def source(self) -> dict: @property def schunk(self): """The underlying carrier's or cache's :class:`SChunk`, or None if unattached.""" - if self._carrier is not None: - return getattr(self._carrier, "schunk", self._carrier) if self._proxy is not None: return self._proxy.schunk + if self._runtime_cache is not None: + return getattr(self._runtime_cache, "schunk", self._runtime_cache) + if self._carrier is not None: + return getattr(self._carrier, "schunk", self._carrier) return None @property @@ -450,7 +592,7 @@ def cache(self): """The local container used as cache, or None if caching is disabled.""" if self._proxy is not None: return getattr(self._proxy, "cache", getattr(self._proxy, "_cache", None)) - return self._carrier + return self._runtime_cache @property def urlpath(self): @@ -466,6 +608,13 @@ def cache_path(self): return None return getattr(self._carrier.schunk, "urlpath", None) + @property + def runtime_cache_path(self): + """The mutable sparse cache directory, when one is attached.""" + if not self._shared_runtime_cache or self._runtime_cache is None: + return None + return getattr(self._runtime_cache.schunk, "urlpath", None) + @property def cache_status(self): """How a persistent disk cache was handled, or ``None`` otherwise.""" @@ -606,8 +755,8 @@ def _export_carrier(self, include_cache: bool, cache_policy=None): payload["max_cache_bytes"] = self.max_cache_bytes or DEFAULT_DISK_CACHE_BYTES write_b2object_payload(carrier, payload) return carrier - if include_cache and self._carrier is not None: - return self._carrier + if include_cache and self._runtime_cache is not None: + return self._runtime_cache return self._to_b2object_carrier() @_serialized_operation @@ -632,8 +781,10 @@ def save( urlpath = os.fspath(urlpath) if ( (cache_policy is not None or not include_cache) - and self.cache_path is not None - and os.path.abspath(self.cache_path) == os.path.abspath(urlpath) + and any( + path is not None and os.path.abspath(path) == os.path.abspath(urlpath) + for path in (self.cache_path, self.runtime_cache_path) + ) ): raise ValueError("cold or policy-changing export requires a different destination") carrier = self._export_carrier(include_cache, cache_policy) diff --git a/tests/test_remote_proxy.py b/tests/test_remote_proxy.py index 1e0868590..9f2b5154c 100644 --- a/tests/test_remote_proxy.py +++ b/tests/test_remote_proxy.py @@ -178,6 +178,143 @@ def test_disk_bound_shrinks_self_caching_carrier(tmp_path): assert reopened.cache_bytes <= 120_000 +def test_server_sparse_cache_reopens_and_exports_portable_carriers(tmp_path): + url, data = _remote_array("server-sparse.b2nd", nchunks=3, chunk_size=100_000) + runtime_path = tmp_path / "private-runtime" + proxy = blosc2.RemoteProxy.with_sparse_cache( + url, runtime_path, max_cache_bytes=120_000 + ) + + assert runtime_path.is_dir() + assert proxy.runtime_cache_path == str(runtime_path) + assert proxy.cache_path is None + np.testing.assert_array_equal(proxy[:100_000], data[:100_000]) + + reopened = blosc2.RemoteProxy.with_sparse_cache( + url, runtime_path, max_cache_bytes=120_000 + ) + reopened.traffic.reset() + np.testing.assert_array_equal(reopened[:100_000], data[:100_000]) + assert reopened.traffic.requests == 0 + + warm = blosc2.ndarray_from_cframe(reopened.to_cframe()) + cold = blosc2.ndarray_from_cframe(reopened.to_cframe(include_cache=False)) + assert warm.schunk.vlmeta.get("proxy-fetched") + assert not cold.schunk.vlmeta.get("proxy-fetched") + assert warm.schunk.vlmeta["b2o"] == cold.schunk.vlmeta["b2o"] + + +def test_server_sparse_handles_refresh_shared_fetched_state(tmp_path): + url, data = _remote_array("server-shared.b2nd", nchunks=2, chunk_size=100) + runtime_path = tmp_path / "shared-runtime" + first = blosc2.RemoteProxy.with_sparse_cache(url, runtime_path) + second = blosc2.RemoteProxy.with_sparse_cache(url, runtime_path) + + np.testing.assert_array_equal(first[:100], data[:100]) + second.traffic.reset() + np.testing.assert_array_equal(second[:100], data[:100]) + assert second.traffic.requests == 0 + + np.testing.assert_array_equal(second[100:], data[100:]) + first.traffic.reset() + np.testing.assert_array_equal(first[100:], data[100:]) + assert first.traffic.requests == 0 + assert first.schunk.vlmeta["proxy-fetched"] == b"\x03" + + +def test_server_sparse_cache_recovers_a_dirty_generation(tmp_path): + url, data = _remote_array("server-dirty.b2nd", nchunks=2, chunk_size=100) + runtime_path = tmp_path / "dirty-runtime" + proxy = blosc2.RemoteProxy.with_sparse_cache(url, runtime_path) + np.testing.assert_array_equal(proxy[:100], data[:100]) + proxy.schunk.vlmeta["proxy-dirty"] = {"pid": -1, "version": 1} + del proxy + + recovered = blosc2.RemoteProxy.with_sparse_cache(url, runtime_path) + recovered.traffic.reset() + np.testing.assert_array_equal(recovered[:100], data[:100]) + assert recovered.traffic.requests > 0 + assert "proxy-dirty" not in recovered.schunk.vlmeta + + +def test_server_sparse_cache_reuses_partial_blocks(tmp_path): + data = np.random.default_rng(2).integers(0, 256, 200, dtype=np.uint8) + array = blosc2.asarray(data, chunks=(100,), blocks=(10,)) + url = "memory://server-partial-blocks.b2nd" + fsspec.filesystem("memory").pipe_file("server-partial-blocks.b2nd", array.to_cframe()) + runtime_path = tmp_path / "partial-runtime" + + proxy = blosc2.RemoteProxy.with_sparse_cache(url, runtime_path) + np.testing.assert_array_equal(proxy[:10], data[:10]) + assert proxy.schunk.vlmeta.get("proxy-fetched-blocks") + assert proxy.schunk.vlmeta["proxy-fetched-bpc"] == 10 + + reopened = blosc2.RemoteProxy.with_sparse_cache(url, runtime_path) + reopened.traffic.reset() + np.testing.assert_array_equal(reopened[:10], data[:10]) + assert reopened.traffic.requests == 0 + + +def test_server_sparse_cache_invalidates_same_geometry_replacement(tmp_path): + url, data = _remote_array("server-replaced.b2nd", nchunks=2, chunk_size=100) + runtime_path = tmp_path / "replaced-runtime" + proxy = blosc2.RemoteProxy.with_sparse_cache(url, runtime_path) + np.testing.assert_array_equal(proxy[:100], data[:100]) + + replacement = np.arange(200, dtype=np.uint8) + array = blosc2.asarray(replacement, chunks=(100,), blocks=(100,)) + fsspec.filesystem("memory").pipe_file("server-replaced.b2nd", array.to_cframe()) + + proxy.traffic.reset() + np.testing.assert_array_equal(proxy[:100], replacement[:100]) + assert proxy.traffic.requests > 0 + + +def test_server_sparse_warm_seed_is_migrated_only_once(tmp_path): + url, data = _remote_array("server-seed.b2nd", nchunks=2, chunk_size=100) + seed = blosc2.RemoteProxy( + url, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=tmp_path / "seed.b2nd", + max_cache_bytes=None, + ) + np.testing.assert_array_equal(seed[:100], data[:100]) + + runtime_path = tmp_path / "seed-runtime" + runtime = blosc2.RemoteProxy.with_sparse_cache( + url, runtime_path, carrier=seed._carrier, max_cache_bytes=1 + ) + runtime.traffic.reset() + np.testing.assert_array_equal(runtime[:100], data[:100]) + assert runtime.traffic.requests == 0 + assert runtime.cache_bytes == 0 + del runtime + + reopened = blosc2.RemoteProxy.with_sparse_cache( + url, runtime_path, carrier=seed._carrier, max_cache_bytes=1 + ) + reopened.traffic.reset() + np.testing.assert_array_equal(reopened[:100], data[:100]) + assert reopened.traffic.requests > 0 + + +def test_server_sparse_rejects_a_seed_from_another_source(tmp_path): + first_url, _ = _remote_array("server-first-seed.b2nd", nchunks=1, chunk_size=100) + second_url, _ = _remote_array("server-second-seed.b2nd", nchunks=1, chunk_size=100) + seed = blosc2.RemoteProxy( + first_url, + cache_policy=blosc2.CachePolicy.DISK, + cache_path=tmp_path / "other-seed.b2nd", + ) + + runtime_path = tmp_path / "wrong-seed-runtime" + with pytest.raises(ValueError, match="different remote source"): + blosc2.RemoteProxy.with_sparse_cache( + second_url, runtime_path, carrier=seed._carrier + ) + assert not runtime_path.exists() + + def test_interrupted_fetch_leaves_a_reusable_carrier(tmp_path): url, data = _remote_array("interrupted.b2nd", nchunks=3, chunk_size=100_000) cache_path = tmp_path / "interrupted-proxy.b2nd" From a77ac97b8ffe10dc43dfbed5fea21cddf405732a Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 6 Sep 2026 09:19:38 +0200 Subject: [PATCH 20/89] Update to latest C-Blosc2 source --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 38284e6cc..69b60a25c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,8 +16,8 @@ project(python-blosc2) # Update to the latest minimum C-Blosc2 version, and to the actual bundled one set(BLOSC2_MIN_VERSION 3.3.0) -set(BLOSC2_BUNDLED_VERSION v3.3.3) -# set(BLOSC2_BUNDLED_VERSION a7f9a5be3527e87a7cbce1c16977835361d27033) +#set(BLOSC2_BUNDLED_VERSION v3.3.3) +set(BLOSC2_BUNDLED_VERSION a54e259aeb1c0e62d6bbea28934369d4831e4046) if(WIN32 AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang") message(FATAL_ERROR "Windows builds require clang-cl. Set CC/CXX to clang-cl or configure CMake with -T ClangCL.") From 5fbd00a874c461af0fb6c382686d91947394eebd Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 6 Sep 2026 14:23:52 +0200 Subject: [PATCH 21/89] Add sparse remote proxy v7 support --- plans/remote-proxy-v7.md | 809 +++++++++++++++++++++++++++++++++++++ src/blosc2/proxy.py | 42 +- src/blosc2/remote_proxy.py | 138 ++++++- tests/test_remote_proxy.py | 57 ++- 4 files changed, 1011 insertions(+), 35 deletions(-) create mode 100644 plans/remote-proxy-v7.md diff --git a/plans/remote-proxy-v7.md b/plans/remote-proxy-v7.md new file mode 100644 index 000000000..0c919c26b --- /dev/null +++ b/plans/remote-proxy-v7.md @@ -0,0 +1,809 @@ +# Remote proxy v7: Caterva2 sparse-cache lifecycle and soft quota + +## Status and objective + +Implementation checkpoint: Caterva2 now uses the sparse backend as its runtime +default. The implementation includes authorized source attachment, shared quota admission, private +generations, coldify, retirement/cleanup, offline pruning, conservative recovery, +and warm export artifacts. Endpoint benchmarks +and their raw samples live in Caterva2's `examples/benchmarks/remote_proxy_v7.md` +and `remote_proxy_v7_results/`. + +This is the first implementation/measurement iteration, not completion of all +acceptance criteria below. It uses full-generation measurement/fsync after writes, +discards interrupted generations rather than resuming their transitions, and +reserves the full staging budget for one warm export. Incremental mutation +reports, bounded inventories, configurable maintenance tuning, large logical +chunk-count/RSS benchmarks, and power-loss validation remain follow-up work. +The rebuilt C-Blosc2 now removes payload-free sparse chunk files on successful +eviction; zero-length leftovers mentioned below apply to legacy/interrupted state. + +The public carrier remains contiguous and portable, while mutable DISK state lives +in private sparse generations. The implementation is applied to Caterva2's server, +quota coordinator, recovery paths, API tests, and benchmark harness. + +The server-boundary benchmark uses real ASGI routes and the rebuilt C-Blosc2. +For an 8 MiB source with a 1 MiB cache, v7 was 2.42x faster on warm hits and +1.98x faster on partial growth, but 0.57x on eviction-heavy churn. For a 64 MiB +source with a 32 MiB cache, v7 was 2.65x faster on cold fill, 8.82x on warm hits, +1.90x on churn, and 6.73x on partial growth. Raw samples and methodology live +in Caterva2's `examples/benchmarks/remote_proxy_v7.md`. + +## Decisions fixed by v7 + +- Runtime layout is `/.remote-cache///`. + Both identifiers are random UUID hex strings assigned by the server. User + paths and source URLs never appear in private filenames. +- The public path remains a contiguous RemoteProxy carrier. It is the portable + descriptor and never becomes a directory. +- An uploaded warm carrier is migrated once into a new sparse generation. Once + that generation is active, it is the only authoritative cache. Caterva2 then + cold-replaces the public carrier. Both copies remain charged until replacement + completes. +- `NONE` and `MEMORY` continue to execute without server retention. Only requested + `DISK` policy creates a runtime generation. +- Cache lifecycle exists even when customer quota is disabled. Quota controls + admission and pruning; it must not be the owner registry. +- The customer quota is soft for runtime-cache fills. Ordinary dataset writes + retain their existing staged admission and `quota_work_bytes` behavior. +- Charge regular dataset files by their existing `st_size` rule. Charge every + entry in an active, retired, or trash-resident sparse generation by allocated + size (`st_blocks * 512`), falling back to `st_size` where allocated size is + unavailable. Include `chunks.b2frame`, `.b2lock`, numbered chunk files, + directories, and zero-length entries' directory allocation. Stable lifecycle + locks under `.storage` remain operational and excluded. +- Cache pruning uses whole-chunk LRU batches. It never chooses ordinary datasets. +- Copy creates a new cache identity and exports a cold carrier, including when + the source is a legacy warm carrier. Move is implemented as + copy-plus-delete and also starts cold at the destination. This deliberately + gives up cached data to keep current non-atomic directory move semantics and + crash recovery simple. +- Removing or replacing a RemoteProxy retires its active generation. The API may + finish after the generation is atomically moved to private trash, but its bytes + remain charged until background or startup deletion completes. +- Rollback ignores private caches and serves public cold carriers safely. It does + not translate sparse generations back into a mutable public carrier. + +## Scope and non-goals + +The first deployment remains one customer state directory shared by local worker +processes on one host. SQLite and file locks on a network filesystem, shared state +across hosts, hard physical-volume guarantees, and conversion of ordinary writes +to in-place mutation are out of scope. + +Do not weaken the existing HTTPS allowlist, DNS pinning, redirect refusal, +credential-free descriptor validation, geometry limits, or embedded-reference +guard. Every request resolves and authorizes the source before consulting a +private cache, including a cache hit. + +Peer cache storage retains its independent quota and pruning implementation. +Media, SQLite/WAL files, staged candidates, authentication state, and lifecycle +locks keep their existing operational accounting classifications. + +## Python-Blosc2 contract required by Caterva2 + +The Caterva2 integration now uses the supported Python-Blosc2 sparse-cache +contract below rather than reaching through `_proxy` internals: + +1. Let `with_sparse_cache()` accept Caterva2's already-authorized + `FsspecNDSource` together with its credential-free source descriptor. It must + retain that source object and its pinned filesystem; it must not reopen the URL + through default fsspec transport. Validate the descriptor used for persistence + against the supplied source. +2. `cache_contains(item=(), *, nchunk=None) -> bool`, evaluated under the sparse + frame lock, so a pure hit avoids a SQLite reservation and filesystem scan. + Also expose a supported cache-operation context that keeps the frame lock held + across checking and reading, or an atomic `read_cached()` returning a hit/result + pair. A separate check followed by an ordinary mutating read is insufficient. +3. `trim_cache(target_bytes, *, max_chunks) -> CacheMutation`, which evicts no + more than `max_chunks` least-recently-used chunks, updates fetched/block/index + state through Proxy, and returns the affected chunk numbers plus payload sizes + before and after. Persist enough LRU ordering in the sparse metadata for a new + process to make the same coarse ordering decision; do not create one SQLite row + per chunk. +4. A mutation result for reads/fetches containing whether storage changed, + affected chunk numbers, payload bytes before/after, and the index/metadata + files that changed. Caterva2 uses this to stat only affected files. +5. A stable warm export to a destination path while holding one consistent cache + snapshot. Confirm that sparse-to-contiguous `save()` has bounded peak memory; + otherwise add a bounded serializer before enabling warm exports. +6. Offline sparse inspection and dirty recovery without constructing a remote + source. Startup must not call the URL-based constructor or perform outbound + requests. If a damaged frame cannot be recovered offline, retire it and let a + later authorized request create a fresh generation. + +Specify mutation results as a supported value type: affected chunk IDs (including +evictions), old/new payload bytes, changed metadata paths, and created/deleted +entries. Paths must be relative to the generation and validated by the server. +The result must include writes performed during attachment/recovery, not just +explicit fetches. Define `target_bytes` and `max_cache_bytes` as compressed cached +payload limits; customer charge additionally includes allocated metadata and +directory space. An exception may follow a partial mutation: retain durable +accounting intent and reconcile even when no mutation result was returned. + +The existing persistent `proxy-dirty` marker remains the payload-integrity fence. +All processes attach with `with_sparse_cache()`, which enables compatible frame +locking, refreshes shared fetched state, and recovers a marker only after obtaining +the frame lock. Caterva2 adds lifecycle and accounting intent around it. + +The current server scans one generation after a mutation when a precise mutation +report is unavailable, but never scans on a hit. This is the safe initial fallback; +replace it with mutation-result accounting after benchmarking millions of logical +chunks and long-running quota convergence. + +## Private paths and API isolation + +Add these settings-derived paths without adding them to `storage_quota.ROOTS` or +any provider/root registry: + +```text +/.remote-cache/ active and retired generation directories +/.remote-cache/.trash/ atomically detached generations awaiting deletion +/.storage/exports/ controlled warm-export artifacts +``` + +Create them mode `0700` where supported. Reject `.remote-cache` and `.storage` +explicitly in path resolution and writable-path helpers even though public APIs +currently accept only `@public`, `@shared`, and `@personal`. Static mounts, root +listing, dataset walking, providers, the web viewer, archive expansion, and HDF5 +unfolding must never traverse these paths. + +Only the remote-cache registry constructs private paths. Validate stored relative +paths as exactly `.remote-cache/<32 hex>/<32 hex>` or +`.remote-cache/.trash/<32 hex>` before opening, renaming, or deleting them. Reject +symlinks at every component and never follow links during measurement or cleanup. + +## SQLite schema version 2 + +Initialize `storage.sqlite` at schema version 2 inside the existing startup +initialization guard. Preserve `objects`, `operations`, and `account` for ordinary +file publication. The existing `objects.cache` column remains for compatibility; +sparse runtime bytes never use it. + +Add: + +```sql +CREATE TABLE remote_objects ( + object_id TEXT PRIMARY KEY, + path TEXT UNIQUE, + carrier_generation TEXT NOT NULL, + spec_hash TEXT NOT NULL, + source_stamp TEXT, + active_generation TEXT, + parent_charge_bytes INTEGER NOT NULL DEFAULT 0 CHECK(parent_charge_bytes >= 0), + updated REAL NOT NULL +); + +CREATE TABLE remote_generations ( + generation_id TEXT PRIMARY KEY, + object_id TEXT NOT NULL, + relpath TEXT UNIQUE NOT NULL, + state TEXT NOT NULL CHECK(state IN + ('building', 'active', 'retired', 'trash')), + spec_hash TEXT NOT NULL, + source_stamp TEXT NOT NULL, + max_cache_bytes INTEGER, + payload_bytes INTEGER NOT NULL CHECK(payload_bytes >= 0), + charge_bytes INTEGER NOT NULL CHECK(charge_bytes >= 0), + inode_count INTEGER NOT NULL CHECK(inode_count >= 0), + touched REAL NOT NULL, + created REAL NOT NULL +); + +CREATE UNIQUE INDEX one_active_remote_generation +ON remote_generations(object_id) WHERE state='active'; + +CREATE TABLE remote_operations ( + id TEXT PRIMARY KEY, + generation_id TEXT NOT NULL UNIQUE, + kind TEXT NOT NULL CHECK(kind IN + ('build', 'fill', 'prune', 'retire', 'delete', 'coldify')), + estimate INTEGER NOT NULL CHECK(estimate >= 0), + previous_charge INTEGER NOT NULL CHECK(previous_charge >= 0), + details BLOB NOT NULL, + started REAL NOT NULL +); + +CREATE TABLE remote_work ( + id TEXT PRIMARY KEY, + kind TEXT NOT NULL CHECK(kind IN ('export', 'rebuild')), + reserved INTEGER NOT NULL CHECK(reserved >= 0), + relpath TEXT NOT NULL, + started REAL NOT NULL +); + +CREATE TABLE remote_orphans ( + id TEXT PRIMARY KEY, + relpath TEXT UNIQUE NOT NULL, + charge_bytes INTEGER NOT NULL CHECK(charge_bytes >= 0), + inode_count INTEGER NOT NULL CHECK(inode_count >= 0), + updated REAL NOT NULL +); +``` + +Add `cache_fill_suspended INTEGER NOT NULL DEFAULT 0` to `account`. Use `NULL` +quota internally to mean unlimited if the coordinator is instantiated only for +lifecycle. Alternatively retain the public `settings.quota == 0` convention and +branch admission before reading the account limit; do not encode unlimited as an +arbitrarily large integer. + +Maintain the invariant that `remote_objects.active_generation` is either NULL or +names that object's sole `state='active'` row. Change both fields in the same +transaction whenever a generation activates or retires. + +`remote_objects.path` is the current binding, not permanent ownership history. +On deletion/replacement set the old object's path to NULL and retire its active +generation in the same transaction. Its UUID continues to own retired/trash rows +while a new object can claim the same public path. Keep the former path in the +operation details for recovery. Every lookup must revalidate its binding after +acquiring the object lock; acquiring a lock for an obsolete lookup is not enough. + +`remote_operations.details` is a versioned, credential-free msgpack record. For +build/coldify it records the expected public signature, expected cold artifact +digest, spec/stamp, and associated ordinary-publication operation ID; for +retire/delete it records old and intended trash paths. Insert intent before the +first filesystem mutation. Recovery must distinguish its own completed coldify +from an unrelated replacement with the same descriptor. + +`spec_hash` is SHA-256 over canonical msgpack or sorted JSON containing the +credential-free RemoteProxy payload, shape, dtype string, chunks, blocks, and +runtime cache-format version. `source_stamp` is the stable validator observed from +the already-authorized source. `carrier_generation` is the existing +`storage_quota.signature()` JSON for the public carrier. + +Do not enable SQLite foreign-key cascades for filesystem ownership. Filesystem +removal must be recoverable and charged until it succeeds; explicit state +transitions are clearer than a database cascade that forgets live bytes. + +Extend `StorageQuota.usage()` diagnostics with: + +```text +dataset_used sum(objects.size) +remote_cache_used generation charge + object-parent charge + orphan charge +used dataset_used + remote_cache_used +reserved existing publication growth + remote operation estimates +working existing candidates + remote_work reservations +cache_fill_suspended +``` + +Keep existing keys for API compatibility. + +Update every ordinary `StorageQuota.publish()` admission query to include +`remote_cache_used` and outstanding remote estimates. Likewise, remote fill +admission includes ordinary publication reservations. This is one shared account: +an upload and a cache fill cannot independently spend the same headroom. When no +quota is configured, the coordinator still provides compare-and-swap publication +for internal coldify and lifecycle operations but skips customer-capacity denial. + +## Locks and ordering + +Use stable lifecycle locks under `.storage`, keyed by object or generation UUID, +in addition to Python-Blosc2's sparse-frame `.b2lock`: + +- Dataset path locks protect public carrier compare-and-swap. +- Object locks protect path binding, active-generation replacement, and coldify. +- Generation locks protect registry state, retirement, trash rename, measurement, + and deletion. +- Sparse frame locks protect chunks and their index/bitmaps. + +Rules: + +1. Never acquire an OS lock from inside a SQLite transaction. +2. For multiple dataset paths, acquire path locks in lexical relative-path order. +3. Acquire dataset path, object, generation, then sparse frame lock in that order. +4. Begin any short SQLite transaction only after required OS locks are held; no + code elsewhere may hold SQLite and then wait for those locks. +5. Perform DNS, network, serialization, directory walking, fsync, rename, and + deletion outside SQLite transactions. +6. Never use elapsed time to steal ownership. Recovery attempts the stable OS lock + non-blocking; failure means a live worker may still own the operation. + +The initialization guard precedes path/object/generation locks. Acquire global +migration/prune guards before their subordinate locks and only non-blocking. +Refactor `publish()` into an outer lock-acquiring wrapper and an internal +`publish_locked()` with an explicit caller-held path-lock contract. Coldify and +CRUD hooks call the latter; reacquiring the same file lock through a second file +descriptor can deadlock. Do not trigger recursive global pruning from inside a +generation guard. Queue it after releasing request locks. + +For v7, hold an exclusive generation guard for every sparse operation, including +attachment, cached reads, fills, export, and recovery. Close cache handles before +releasing it; no process may retain an open sparse handle across retirement or +trash deletion. This intentionally serializes operations on one generation. +Different generations remain concurrent. Optimize reader leases only after +benchmarking, with a separate documented deletion protocol. + +The process-local `dataset_lock()` remains useful for avoiding event-loop thread +contention, but correctness must depend only on cross-process locks and SQLite. + +## Resolution and generation attachment + +Refactor `caterva2/services/remote_proxy.py` so `resolve()` receives the public +carrier path and storage coordinator. Preserve the current policy validation and +source creation before registry lookup. + +For every request: + +1. Inspect the public carrier without resolving embedded references. +2. Validate the descriptor and authorize the HTTPS destination. +3. Construct the pinned remote source, obtain its stable stamp, and validate + geometry and configured limits. +4. Compute `spec_hash` and snapshot the public carrier generation. +5. Acquire the path lock, recheck the signature against the authorized snapshot, + then find/create the binding under its object lock. If the snapshot changed, + release locks and restart resolution with a bounded retry count; never attach + storage authorized for the previous carrier to its replacement. +6. Reuse the active generation only when object ID, spec hash, source stamp, + requested DISK policy, and runtime format all match. +7. If the source stamp, descriptor, geometry, policy, or externally observed + carrier generation changed, build a new generation and retire the old one. +8. Attach with `blosc2.RemoteProxy.with_sparse_cache(authorized_source, + runtime_cache_path, source_descriptor=payload["source"], + carrier=warm_carrier, max_cache_bytes=requested_limit)`. + +An old `ServerRemoteProxy` retains its generation ID. Before publishing any cache +mutation it revalidates that the row is still active while holding the generation +lock. If it has been retired, it returns source data without retention. Existing +readers may finish; an old writer cannot mutate or reactivate a newer generation. + +A source with no stable stamp always executes without retention. Do not create a +runtime directory for it. + +## First attachment and warm-carrier migration + +Warm migration is lazy: it happens on the first authorized operation after upload, +not during upload, so uploading a descriptor never initiates an outbound request. +Only one process obtains a non-blocking global migration lock before duplicating a +warm carrier; concurrent requests serve its valid warm chunks read-only and fetch +misses without retention until migration finishes. + +Under path, object, and new-generation locks: + +1. Insert a `building` generation and `build` operation with a coarse estimate in + a short transaction. +2. Close the transaction and call `with_sparse_cache(authorized_source, ..., + source_descriptor=payload["source"], carrier=carrier)`. The helper copies only + fetched state validated against the authorized source stamp. +3. Fsync the sparse generation and its parent, measure its full charge, and mark it + active. Retire any previous active generation in the same transaction and + release the estimate. +4. Build a cold contiguous carrier from the immutable public snapshot, preserving + the RemoteProxy payload, fixed metadata, and every non-reserved user vlmeta. + Remove fetched, cache-size, proxy-index, source-stamp, and dirty bookkeeping. +5. Cold-replace the public carrier through `StorageQuota.publish()` using its + original signature. This is a shrinking ordinary write and retains existing crash + recovery. +6. Update `carrier_generation` to the resulting signature. If compare-and-swap + loses to a user replacement, keep the new sparse generation retired and never + bind it to the replacement. + +The active sparse cache is authoritative from step 3. A crash before coldify may +leave both warm copies, which is safe and fully charged. Startup recovery retries +coldify only when path, descriptor hash, source stamp, and recorded public +generation still match. It never reimports the warm carrier into an existing +generation. + +Warm migration is necessary lifecycle work rather than a discretionary cache fill, +so customer quota may be exceeded temporarily by the duplicate. Record its full +estimate so other workers do not interpret that headroom as free. Before copying, +check configured operational free-space headroom; on denial, ENOSPC, or another +failure, retire and clean the partial generation, leave the public warm carrier +unchanged, and continue serving it read-only. A later request may retry migration. + +Use `publish_locked()` for step 5 because the path guard is already held. Record +a `coldify` intent before publishing, with its ordinary-publication operation ID +durable before candidate rename. Reconciliation recovers this intent before +interpreting a changed public signature as an external replacement. A failed +coldify after successful activation leaves the active cache authoritative and +retries only coldify; it must not restart seed migration. The partial-generation +cleanup fallback above applies to failures before activation. + +Pending coldify is persistent work even after a successful build has released its +estimate. Atomically replace the build operation with coldify intent on activation; +the unique operation-per-generation constraint must never erase unfinished work. +Subsequent requests serve that generation read-only until coldify completes, so a +fill cannot overwrite its recovery record. Back off retries on staging failures. + +## Soft-quota fill protocol + +Route `ServerRemoteProxy.quota_read()` through the sparse generation for every +retained DISK cache. A denied or failed retention attempt falls back to a +no-retention read and still returns the logical result. + +For a logical slice or compressed chunk request: + +1. Authorize and resolve the source as described above. No cache hit skips this. +2. Acquire the generation guard, revalidate its active binding, and keep it until + handles are closed and accounting is finalized. Check and read a hit atomically + under the frame lock; update coarse recency at most once per ten seconds and + avoid SQLite admission. Recency persistence is best-effort and cannot turn a + successful cached read into an error. +3. For a miss, compute a conservative estimate from missing compressed blocks or + chunks and allocated metadata growth. Bound only the payload component by + remaining per-proxy `max_cache_bytes` where finite. Estimates + coordinate workers; they are not physical-growth guarantees. +4. In a short `BEGIN IMMEDIATE` transaction, sum dataset charge, generation + charge, existing reservations, and remote estimates. If fills are suspended or + projected usage exceeds quota, refuse retention. Otherwise insert a `fill` + operation. With no configured quota, insert intent with estimate zero for + recovery/accounting only. +5. Close the transaction, then call the supported combined fetch/read-and-mutate + operation under the generation and frame guards. Durable SQLite intent must + precede all writes; Python-Blosc2 brackets payload writes with its dirty marker. + V7 does not assume a separate fetch/publish API. Holding these guards across + network I/O is acceptable initially; benchmark its same-cache contention. +6. Enforce requested `max_cache_bytes` within that operation and obtain the logical + result plus mutation report, including any evictions. A request larger than the + per-proxy limit must still return its full result without retaining all of it. +7. Stat only affected payload and metadata entries, fsync as required, and update + `payload_bytes`, `charge_bytes`, `inode_count`, and `touched`; delete the + operation in one short transaction. +8. If actual aggregate usage exceeds quota, set `cache_fill_suspended=1` and queue + bounded pruning. Return the already assembled logical result. + +Any SQLite admission failure, busy timeout, stale generation, cache-write error, +ENOSPC, or retention denial falls back to a no-retention read. If data was already +assembled, return it directly; otherwise refetch without retention. Upstream +authorization, validation, or fetch errors still propagate normally. + +Use a shared low watermark of 90% of configured quota. Once suspended, admit no +new discretionary cache fill until pruning or reconciliation observes usage at or +below that watermark. If ordinary dataset bytes alone exceed the watermark, cache +fills remain suspended. Do not claim a fixed maximum overshoot. + +Admission estimates cover positive allocated growth, including index and directory +changes; a payload limit of zero does not imply zero filesystem overhead. Do not +subtract proposed evictions until they have actually freed charge. A finite +payload cap only bounds the payload component of the estimate. + +## Accounting and reconciliation + +Maintain charge from Python-Blosc2 mutation results where available. The current +release still performs a full-generation measurement after mutations as a safe +fallback; a cache hit does not stat every chunk. Coarse recency updates occur at +most every ten seconds. + +Run full reconciliation: + +- at coordinator startup under the initialization guard; +- after recovery of a dead operation; +- after an unexpected mutation/accounting error; +- periodically, default every five minutes, in bounded generation batches; +- on an administrator diagnostic request. + +Reconciliation walks `.remote-cache` without following symlinks, measures files +and directories, repairs charge/inode counts, discovers registered directories +missing on disk, and moves unregistered validly-named directories to trash before +deletion. It never adopts an orphan as active from pathname alone. + +It also checks every `remote_objects.path` against the current public file without +making outbound requests. A missing file, changed carrier signature, non-RemoteProxy +replacement, or changed descriptor/geometry retires the bound generation. A +matching descriptor whose remote source changed is detected later during an +authorized request, when obtaining the source stamp is permitted. + +Skip NULL path bindings. Reconcile under the corresponding generation guard and +recover pending lifecycle/publication intents before comparing signatures. Do +not classify a live `building` directory as an orphan. Record discovered orphan +charge and durable cleanup intent before trash movement; missing files release +charge only after guarded verification. Charge object-parent directories once +per object in a separate accounting total included in `remote_cache_used`, rather +than once per generation. Shared cache/trash roots remain operational overhead. + +Missing active storage retires the generation and makes subsequent reads cold. +Malformed paths, symlinks, descriptor mismatches, or unreadable sparse metadata +are quarantined in trash and remain charged until removed. A reduced quota sets +fill suspension and schedules pruning; startup does not delete ordinary datasets. + +## Batched pruning + +Trigger pruning after an overshooting fill, quota reduction, periodic +reconciliation above quota, or an admission refusal. Only one process obtains the +non-blocking global prune lock; others return their logical results. + +Select active DISK generations ordered by coarse `touched`, excluding the +generation serving the triggering result. For each candidate: + +1. Acquire its generation lock non-blocking and revalidate active state. +2. Compute bytes still needed to reach the aggregate low watermark and call + `trim_cache(max(0, candidate_payload_bytes - needed), max_chunks=64)`. +3. Finalize affected-file charge and recency in a short transaction. +4. Stop after 64 chunks total, four generations, 100 ms of mutation work, or + aggregate usage at/below the low watermark, whichever comes first. + +Schedule another batch if still over quota. Empty generations remain valid cold +caches; zero-length chunk files and directory/index overhead remain charged. +Whole-generation retirement is a fallback only for corrupt, stale, or deleted +objects, not normal quota pressure. + +The triggering generation is excluded only while its request guard is held. A +later background batch must include it; otherwise a customer with one cache can +remain suspended forever. Recompute actual allocated bytes after each batch: +payload bytes evicted are not necessarily physical bytes reclaimed. If no payload +can be reclaimed, stop rescheduling immediate batches, retain suspension, and +report irreducible metadata/dataset charge. Retry on periodic reconciliation or +a storage change. The 100 ms limit is checked between chunks and cannot bound a +single blocking filesystem operation. + +Record a `prune` intent before calling `trim_cache()` and leave it for recovery +if eviction or accounting fails. Exclude generations with pending coldify/build +work; the unique operation row is never replaced by a competing operation. + +## Export and download behavior + +`include_cache=false` reads the public cold carrier snapshot and returns it without +opening or mutating the sparse generation. Before first migration, when the public +carrier may still be warm, build a cold snapshot locally by stripping its cache +state without resolving the source. Preserve requested policy and limit, user +metadata, and existing credential-free descriptor semantics. + +Default warm export uses a controlled artifact: + +1. Authorize the source and bind the active generation. +2. Reserve estimated artifact bytes in `remote_work` against + `quota_work_bytes`. A cold export remains available if this reservation fails. +3. Create `.storage/exports/.b2nd` with `O_EXCL` and mode `0600`. +4. Under the generation/frame snapshot guard, serialize a contiguous carrier that + merges valid runtime state and preserves public user metadata. +5. Fsync and close the artifact, record its exact length and a strong ETag derived + from its bytes or immutable operation identity plus digest, then release the + generation guard. +6. Serve all full and range responses from that one artifact. Never range-read a + changing sparse directory. +7. Remove the artifact and `remote_work` row after response completion or + cancellation. Startup removes artifacts whose OS owner lock is obtainable. + +Acquire a stable per-export owner lock before inserting its reservation and hold +it through streaming and cleanup. Preserve actual artifact charge/reservation if +unlink fails. Cold snapshots that need serialization use the same work budget and +cleanup protocol. Reserve work jointly with ordinary publication candidates; +adjust an underestimated reservation before further growth or abort the export. +A staging failure returns a documented capacity error for a requested warm export; +do not silently substitute a cold result. Each HTTP request creates its own +snapshot: honor `If-Range` against that snapshot's ETag and return a full response +when it differs. Cross-request artifact reuse is outside v7 scope. + +Do not hold a SQLite transaction while serializing or streaming. Export artifacts +are operational working storage and not customer `used` bytes, but they consume +the separately configured work budget. Document that soft customer quota cannot +prevent ENOSPC and that warm export may fail when staging headroom is unavailable. + +## Dataset lifecycle + +Route every coordinated public/shared/personal mutation through remote-cache +lifecycle hooks in `server.py` and `storage_quota.py`: + +- **Upload or replacement:** after successful public publish, compare the prior + registry binding. Retire old generations even when the new carrier has the same + URL or descriptor. The replacement receives a new object ID on first resolve. +- **Delete:** acquire the dataset/object/generation guards, remove the public file + through ordinary publication, mark the generation retired, atomically rename its + directory to `.remote-cache/.trash/`, then delete outside the + locks. Charge remains until deletion is measured complete. +- **Directory delete:** retain current file-by-file semantics; each RemoteProxy + child executes the same retirement path. +- **Copy:** export a cold portable carrier locally, including from a legacy warm + source, preserving descriptor, policy, limit, and user metadata. Do not resolve + its source. Never share an object ID or sparse directory. +- **Move:** use current copy-plus-delete behavior. Destination starts with a new + identity; source cache retires after the source generation is successfully + removed. +- **Append/update/resize:** these already reject B2 object carriers where + applicable. Any generic replacement route must still invoke retirement. +- **Customer removal/state cleanup:** retire every `remote_objects` row, move all + generations to trash under their guards, and keep their charge until deletion. + +Centralize these hooks in the storage coordinator rather than adding endpoint-only +cleanup. CLI, web, API, HDF5 workflows, and future writers must receive identical +behavior. + +Remove `remote_objects` only after its public path is gone or no longer a matching +RemoteProxy and all of its generation rows have been deleted. A row with retired +or trash storage remains the ownership anchor during cleanup. + +## Recovery state machine + +At startup and on demand, inspect each `remote_operations` row. Attempt the +generation lock non-blocking; skip it if busy. + +- `build`: if the directory is absent, delete the generation and release estimate. + If present, use offline inspection/recovery to validate recorded spec/stamp and + measure it. Activate only a durably completed build whose public binding still + matches; retire an incomplete or ambiguous build. Source freshness is checked + again on the next authorized request. +- `fill` or `prune`: recover offline under the frame lock. Python-Blosc2 clears untrusted + fetched state left by a dirty owner. Measure the generation, finalize charge, + clear the reservation, and leave it active only if its registry binding matches. +- `retire`: complete the active-to-retired transition and trash rename. +- `delete`: finish trash deletion, then remove the charged generation row. +- `coldify`: recover the public-path publication first, then compare the resulting + carrier. Update its generation only if signature/digest and publication intent + identify the expected cold artifact; otherwise + retire the cache rather than binding it to unknown bytes. + +Recovery is idempotent. At every state, the public cold/warm carrier remains an +independent source descriptor, so discarding a private generation cannot lose user +data. Never trust a fetched bit merely because SQLite says a fill completed; the +sparse frame and its dirty marker are authoritative for payload integrity. + +## Configuration, diagnostics, and operations + +The runtime has no public cache-backend selector. Sparse v7 is always used for +retained DISK caches; `NONE` and `MEMORY` retain nothing on the server. Keep only +the operational tuning settings that are implemented: + +```toml +[server.remote_proxy] +cache_low_watermark = 0.90 +cache_reconcile_seconds = 300 +cache_prune_chunks = 64 +cache_prune_generations = 4 +cache_min_free_bytes = "1G" +``` + +Sparse mode requires schema v2 and the pinned Python-Blosc2 API. The public carrier +remains the rollback-safe cold descriptor: discarding a private generation never +requires translating it back into a mutable public carrier. + +The free-space check requires `free - estimated_operation_bytes` to remain above +`cache_min_free_bytes` before migration, rebuild, or export starts. It is a +best-effort ENOSPC guard rather than a reservation against unrelated processes. + +Expose authenticated diagnostics containing aggregate dataset/cache charge, +outstanding estimates, work reservations, suspended state, active/retired/trash +generation counts, inode count, oldest recency, overshoot bytes and age, recovery +count, prune work, and last reconciliation. Do not expose source URLs, opaque +filesystem paths, or cache contents in the web viewer. + +Log generation IDs and hashed object IDs, not credential-bearing URLs. Emit +metrics for fill hit/miss/refusal, actual-versus-estimated growth, dirty recovery, +pruned chunks/bytes, trash backlog, export size/time/RSS, quota overshoot magnitude, +and time to return below the low watermark. + +Start maintenance tasks in the server lifespan and cancel/join them on shutdown. +Each worker may wake a task, but non-blocking global guards elect one executor for +each batch. Run blocking cache/filesystem work in the existing worker-thread path, +never on the event loop. Keep a persistent cursor for bounded reconciliation; +resume failed trash deletion with capped exponential backoff. Cancellation of an +HTTP request must not release generation/owner locks while its worker thread still +mutates storage. Let it finish accounting or leave recoverable intent before +closing handles. Cleanup failures remain visible and charged. + +Define the durability boundary explicitly: after intent commit, write and fsync +payload/index state in the order required by Python-Blosc2, durably clear its dirty +marker, and only then finalize SQLite accounting. Fsync both parents of trash +renames. Validate this ordering against the actual helper implementation before +claiming power-loss safety; SIGKILL tests alone establish process-death behavior. + +## Implementation sequence + +1. Land and pin the supported Python-Blosc2 accounting, pruning, and export APIs. +2. Add schema-v2 lifecycle initialization and combined dataset/cache accounting. +3. Implement private path validation, locks, operation recovery, reconciliation, + generation creation, warm migration, coldify, retirement, and trash cleanup. +4. Route sparse hits/fills, logical slices, and `/api/chunk` through soft + admission with no-retention fallback. +5. Add bounded pruning, hysteresis, maintenance scheduling, diagnostics, and + warm export artifacts with ranges, ETags, cancellation, and work reservations. +6. Run fault-injection, multiprocess, endpoint, and v5 comparison benchmarks. + This sequence is complete for the current release candidate; the remaining + work is listed under follow-up items below. + +## Follow-up items after the current implementation + +These items are useful improvements, but do not block the current sparse-default +release candidate: + +- Replace full-generation post-mutation measurement with incremental accounting + from Python-Blosc2 mutation reports. The current scan is correct and bounded by + the generation size, but it adds latency to fills and pruning. +- Add bounded reconciliation cursors and benchmark millions of logical chunks, + sparse metadata, inode counts, peak RSS, and long-running quota convergence. +- Measure multiprocess throughput and same-generation contention separately from + the existing correctness tests. +- Add power-loss testing for filesystem and SQLite durability. Process-death + recovery is covered; the implementation does not claim crash atomicity after a + power failure. +- Tune staging estimates and warm-export reservations instead of reserving the + full work budget for one export. +- Remove the internal contiguous compatibility path after the v5 comparison + benchmark is no longer needed. It is not a supported Caterva2 deployment mode. + +## Caterva2 module map + +- `caterva2/services/remote_proxy.py`: authorized-source attachment, generation + binding, sparse read/chunk routing, warm migration, coldify, pruning adapter, + export snapshot, and no-retention fallback. +- `caterva2/services/storage_quota.py`: schema migration, always-available storage + coordinator, combined admission, private measurement, operation recovery, + reconciliation, retirement, trash deletion, and diagnostics. +- `caterva2/services/server.py`: initialize the coordinator, pass public paths into + resolution, replace `quota_read()` routing, and call centralized lifecycle hooks + from write/remove/copy/move paths and download endpoints. +- `caterva2/services/settings.py`, `caterva2-server.sample.toml`, and + `doc/utilities/cat2-server.md`: runtime tuning, hysteresis, reconciliation, work-budget, + accounting, cleanup, and operational-headroom configuration/documentation. +- `caterva2/tests/test_remote_proxy.py`: source policy, sparse behavior, generation + binding, migration, export, and fallback tests. +- `caterva2/tests/test_storage_quota.py` and + `caterva2/tests/test_storage_quota_api.py`: schema, multiprocess admission, + lifecycle, recovery, pruning, endpoint, and quota-accounting tests. +- `examples/benchmarks/remote_proxy_v7.py` and + `examples/benchmarks/remote_proxy_v7.md`: server-boundary v5/v7 comparison using + the same deterministic local range source and machine-readable raw results. + +## Required tests + +Use deterministic local range-capable sources before external HTTPS tests. + +Add targeted regressions for the review's implementation boundaries: + +- Replace/delete/recreate the same public path while old trash cannot be removed; + a new UUID can bind the path and both generations remain charged. +- Pause between hit detection and data extraction while another process attempts + eviction/deletion; no unreserved fill or access to removed handles occurs. +- Crash after cold publication but before binding finalization; recovery recognizes + the exact artifact, while a same-descriptor user replacement gets a new identity. +- Run startup recovery with all outbound transport constructors forbidden. +- Overshoot with one cache, and with only irreducible metadata remaining; pruning + either reaches the watermark or reports suspension without a busy loop. +- Cancel an in-flight read/export with a live worker thread; locks and reservations + remain owned until mutation/stream cleanup ends. +- Verify schema initialization and quota-disabled lifecycle. + +- Two and eight processes filling the same generation, different generations, + and overlapping partial blocks; readers overlap eviction without corrupt or + zero-filled results. +- Competing fills and ordinary uploads cannot all treat the same quota headroom as + free. Actual overshoot is charged and converges through bounded pruning. +- Worker death before/after every dirty marker, chunk truncate/write, index write, + fetched bitmap, accounting finalization, active-generation switch, cold publish, + trash rename, and deletion boundary. +- Recovery never steals a live worker's reservation or generation and is + idempotent across repeated restarts. +- Source replacement with identical geometry creates a new generation. Geometry, + descriptor, policy, and limit changes cannot attach stale storage. +- Authorization executes on cache hits; allowlist removal immediately prevents + use of already-warm data. +- Warm migration preserves valid uploaded chunks and user metadata, coldifies the + public carrier, counts both copies during the transition, and never resurrects + an evicted seed chunk. +- `include_cache=false` is cold and non-mutating. Warm full/range downloads use one + stable artifact and ETag; cancellation, timeout, restart, and ENOSPC remove or + recover reservations and artifacts. +- Delete, directory delete, replacement, move, copy, and customer removal produce + the lifecycle specified above. Private bytes remain charged until trash removal. +- Startup inventory includes registered and orphan private generations while API, + root listings, search, providers, and the web viewer never reveal them. +- Bounded and unlimited per-proxy limits remain distinct from customer quota. + MEMORY and NONE retain nothing. +- SQLite busy/error, malformed registry rows, missing directories, symlinks, + permission failures, and quota disabled/enabled transitions preserve logical + reads and produce reconciled accounting. + +## Benchmarks and acceptance + +Repeat the v6 cold fill, warm hit, LRU churn, and round-robin partial-block tests +through Caterva2 endpoints. Add concurrent same/different proxy fills, millions of +logical chunks with few resident chunks, pruning, startup reconciliation, cold and +warm export, range download, peak RSS, metadata operations, inode count, actual +allocated blocks, estimate error, and quota overshoot duration. + +Accept v7 when: + +- ordinary fills, partial growth, and eviction never rewrite unrelated cached + payload; +- operation memory does not scale with the remaining carrier tail; +- no stale or dirty fetched bit can serve incomplete data after process death; +- every retained private byte is eventually reflected in account usage, and + overshoot converges without failing logical reads; +- delete and replacement cannot leave an unregistered, uncharged active cache; +- public carriers remain portable and private directories remain unreachable from + API and web namespaces; +- warm/cold export semantics, authorization, validators, geometry checks, MEMORY + behavior, and client defaults remain compatible; +- discarding sparse state can serve the public carrier without migration. + +No hard RAM, fixed overshoot, or physical-volume guarantee is implied. Operational +monitoring and free-space headroom remain required even with correct soft-quota +accounting. diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index f3a49f7b4..e932015a4 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -12,6 +12,7 @@ import math import os import textwrap +import time from collections import OrderedDict from collections.abc import Sequence from concurrent.futures import ThreadPoolExecutor @@ -48,6 +49,8 @@ "proxy-fetched-blocks", "proxy-fetched-bpc", "proxy-dirty", + "proxy-lru", + "proxy-lru-time", "proxy-stamp", "proxy-index", } @@ -494,7 +497,7 @@ def _sync_evictions(self) -> None: def _restore_cache_accounting(self) -> None: """Restore compressed-byte accounting for a bounded cache.""" - if self._max_cache_bytes is None: + if self._max_cache_bytes is None and not self._persistent_dirty: return stored = self._schunk_cache.vlmeta.get("proxy-cache-sizes", {}) if not isinstance(stored, dict): @@ -512,10 +515,13 @@ def _restore_cache_accounting(self) -> None: size = len(self._schunk_cache.get_chunk(nchunk)) self._cache_sizes[nchunk] = size self._cache_lru[nchunk] = None + for nchunk in self._schunk_cache.vlmeta.get("proxy-lru", ()): + if nchunk in self._cache_lru: + self._cache_lru.move_to_end(nchunk) def _remember_cached(self, nchunk: int, size: int) -> None: """Record the current compressed size and recency of one cached chunk.""" - if self._max_cache_bytes is None: + if self._max_cache_bytes is None and not self._persistent_dirty: return self._cache_sizes[nchunk] = size self._cache_lru.pop(nchunk, None) @@ -528,17 +534,33 @@ def _retained_cache_bytes(self) -> int: def _enforce_cache_limit(self, item) -> None: """Touch *item* and evict whole LRU chunks after its result is assembled.""" - if self._max_cache_bytes is None: + if self._max_cache_bytes is None and not self._persistent_dirty: return for nchunk in self._wanted_chunks(item): if nchunk in self._cache_sizes: self._cache_lru.move_to_end(nchunk) - evicted = False - if self._retained_cache_bytes() > self._max_cache_bytes and self._cache_lru: + if self._max_cache_bytes is not None: + self._trim_cache(self._max_cache_bytes) + if self._persistent_dirty and self._cache_lru: + vlmeta = self._schunk_cache.vlmeta + now = time.time() + if now - vlmeta.get("proxy-lru-time", 0) >= 10: + # Recency is advisory: a lost touch cannot validate payload. + vlmeta["proxy-lru"] = list(self._cache_lru) + vlmeta["proxy-lru-time"] = now + + def _trim_cache(self, target_bytes, *, max_chunks=None): + """Caller holds the shared frame guard; return evicted logical IDs.""" + evicted = [] + if self._retained_cache_bytes() > target_bytes and self._cache_lru and max_chunks != 0: self._begin_persistent_mutation() try: - while self._retained_cache_bytes() > self._max_cache_bytes and self._cache_lru: + while ( + self._retained_cache_bytes() > target_bytes + and self._cache_lru + and (max_chunks is None or len(evicted) < max_chunks) + ): nchunk, _ = self._cache_lru.popitem(last=False) self._cache_sizes.pop(nchunk, None) self._hot_payloads.pop(nchunk, None) @@ -546,7 +568,7 @@ def _enforce_cache_limit(self, item) -> None: base = nchunk * self._blocks_per_chunk for n in range(base, base + self._blocks_per_chunk): self._fetched[n // 8] &= ~(1 << (n % 8)) - evicted = True + evicted.append(nchunk) if evicted: self._specialized = getattr(self._schunk_cache, "nspecialized", self._specialized) self._save_fetched() @@ -557,6 +579,7 @@ def _enforce_cache_limit(self, item) -> None: else: if evicted: self._end_persistent_mutation() + return tuple(evicted) def _plan(self, item): """Where *item* lands on the cache's grid, read once for a fetch. @@ -697,10 +720,13 @@ def _save_fetched(self) -> None: self._schunk_cache.vlmeta[self._fetched_key] = bytes(self._fetched) if self._blocks_per_chunk > 1: self._schunk_cache.vlmeta["proxy-fetched-bpc"] = self._blocks_per_chunk - if self._max_cache_bytes is not None: + if self._max_cache_bytes is not None or self._persistent_dirty: self._schunk_cache.vlmeta["proxy-cache-sizes"] = { str(nchunk): size for nchunk, size in self._cache_sizes.items() } + if self._persistent_dirty: + self._schunk_cache.vlmeta["proxy-lru"] = list(self._cache_lru) + self._schunk_cache.vlmeta["proxy-lru-time"] = time.time() elif "proxy-cache-sizes" in self._schunk_cache.vlmeta: # A cache may previously have been bounded (for example by a server # quota). Unbounded writes do not maintain this table, so remove it diff --git a/src/blosc2/remote_proxy.py b/src/blosc2/remote_proxy.py index d6aabd76f..331d2b4e1 100644 --- a/src/blosc2/remote_proxy.py +++ b/src/blosc2/remote_proxy.py @@ -15,6 +15,7 @@ import threading from contextlib import nullcontext from functools import wraps +from types import SimpleNamespace from urllib.parse import parse_qsl, urlsplit import numpy as np @@ -172,6 +173,7 @@ def __init__( max_concurrency: int | None = None, _carrier=None, _runtime_cache_path=None, + _source_descriptor=None, ): if not isinstance(cache_policy, blosc2.CachePolicy): raise TypeError("cache_policy must be a blosc2.CachePolicy instance") @@ -191,9 +193,19 @@ def __init__( self._cache_policy = cache_policy self._cache_limit = _normalize_limit(cache_policy, max_cache_bytes) self._max_concurrency = _validate_max_concurrency(max_concurrency) - self.src, self._source = self._open_source( - urlpath, self._max_concurrency, persistable=cache_policy is not blosc2.CachePolicy.MEMORY - ) + self._authorized_source = _source_descriptor is not None + if self._authorized_source: + if not isinstance(urlpath, blosc2.FsspecNDSource): + raise TypeError("source_descriptor requires an authorized FsspecNDSource") + expected = {"kind": "fsspec", "version": 1, "urlpath": urlpath.urlpath} + if _source_descriptor != expected: + raise ValueError("source_descriptor does not match the supplied source") + _validate_persistable_url(urlpath.urlpath) + self.src, self._source = urlpath, dict(expected) + else: + self.src, self._source = self._open_source( + urlpath, self._max_concurrency, persistable=cache_policy is not blosc2.CachePolicy.MEMORY + ) self._runtime_urlpath = self._runtime_source(urlpath) self._expected_geometry = self._geometry(self.src) self._expected_cparams = self.src.cparams @@ -213,9 +225,7 @@ def __init__( elif self._carrier is None: self._carrier, self._cache_status = self._open_or_create_carrier(cache_dir, cache_path) self._runtime_cache = self._carrier - cache_lock = ( - self._runtime_cache.holding_lock() if self._shared_runtime_cache else nullcontext() - ) + cache_lock = self._runtime_cache.holding_lock() if self._shared_runtime_cache else nullcontext() with cache_lock: self._attach_carrier_cache() elif cache_policy is blosc2.CachePolicy.MEMORY: @@ -278,9 +288,7 @@ def _open_or_create_sparse_cache(self, cache_path): if os.path.exists(path): if not os.path.isdir(path): raise ValueError("runtime_cache_path must name a sparse frame directory") - runtime = blosc2.blosc2_ext.open( - path, "a", 0, dparams=blosc2.DParams(nthreads=1), locking=True - ) + runtime = blosc2.blosc2_ext.open(path, "a", 0, dparams=blosc2.DParams(nthreads=1), locking=True) if runtime.schunk.vlmeta.get("b2o") != self._payload(): raise ValueError(f"the sparse runtime cache at {path} has a different specification") self._validate_geometry( @@ -298,9 +306,7 @@ def _open_or_create_sparse_cache(self, cache_path): if self._carrier is not None: self._validate_warm_seed(self._carrier) - runtime = self._to_b2object_carrier( - urlpath=path, contiguous=False, mode="w", locking=True - ) + runtime = self._to_b2object_carrier(urlpath=path, contiguous=False, mode="w", locking=True) if self._carrier is not None: self._import_warm_seed(self._carrier, runtime) return runtime, "created" @@ -309,6 +315,8 @@ def _import_warm_seed(self, seed, runtime) -> None: """Migrate valid warm chunks once into a newly-created runtime cache.""" seed = self._validate_warm_seed(seed) seed_schunk = getattr(seed, "schunk", seed) + if seed_schunk.vlmeta.get("proxy-dirty") is not None: + return # An interrupted seed has no trustworthy fetched bitmap. stamp = getattr(self.src, "stamp", None) if stamp is None or seed_schunk.vlmeta.get("proxy-stamp") != stamp: return @@ -357,6 +365,7 @@ def with_sparse_cache( runtime_cache_path, *, carrier=None, + source_descriptor=None, max_cache_bytes=_POLICY_DEFAULT, max_concurrency: int | None = None, ): @@ -373,6 +382,11 @@ def with_sparse_cache( the server replaces the portable carrier with a cold descriptor. After migration, only the runtime cache is consulted for cached data, so an evicted chunk cannot be resurrected from the carrier. + + Pass an already-authorized ``FsspecNDSource`` and its credential-free + ``source_descriptor`` to retain the caller's transport and source + snapshot. The caller must authorize and refresh that snapshot before + each attachment; this form never reopens its URL or refreshes its source. """ return cls( urlpath, @@ -381,8 +395,88 @@ def with_sparse_cache( max_concurrency=max_concurrency, _carrier=carrier, _runtime_cache_path=runtime_cache_path, + _source_descriptor=source_descriptor, ) + @_serialized_operation + def read_cached(self, item=(), *, nchunk=None): + """Return ``(hit, result)`` atomically, without fetching a missing block. + + The source must already have been authorized by the caller. A miss + returns ``(False, None)``. This operation does not refresh the source. + """ + if self._proxy is None: + return False, None + if nchunk is not None: + item = self._chunk_slice(nchunk) + if self._proxy._missing_blocks(item): + return False, None + result = self._proxy._cache[item] if nchunk is None else self.schunk.get_chunk(nchunk) + self._proxy._enforce_cache_limit(item) + return True, result + + @_serialized_operation + def cache_contains(self, item=(), *, nchunk=None): + """Check cached coverage; use ``read_cached`` for an atomic hit/read.""" + if nchunk is not None: + item = self._chunk_slice(nchunk) + return self._proxy is not None and not self._proxy._missing_blocks(item) + + @property + def cached_payload_bytes(self): + """Compressed resident payload accounting from the attached snapshot.""" + return 0 if self._proxy is None else sum(self._proxy._cache_sizes.values()) + + @_serialized_operation + def trim_cache(self, target_bytes, *, max_chunks=64): + """Evict at most ``max_chunks`` LRU chunks toward a payload-byte target. + + Return the evicted logical chunk numbers. Allocated filesystem charge + must be measured separately, including after a partially failed eviction. + """ + for name, value in (("target_bytes", target_bytes), ("max_chunks", max_chunks)): + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{name} must be a non-negative integer") + if self._proxy is None: + return () + return self._proxy._trim_cache(target_bytes, max_chunks=max_chunks) + + @staticmethod + def trim_sparse_cache(runtime_cache_path, target_bytes, *, max_chunks=64): + """Trim an offline private cache without constructing a remote source. + + The server must hold its generation lifecycle guard. Return + ``(evicted_chunk_numbers, remaining_payload_bytes)``. A dirty cache is + conservatively invalidated before trimming; filesystem charge still + needs measurement because unreachable payload can remain on disk. + """ + for value in (target_bytes, max_chunks): + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError("target_bytes and max_chunks must be non-negative integers") + cache = blosc2.blosc2_ext.open(os.fspath(runtime_cache_path), "a", 0, locking=True) + with cache.holding_lock(): + bpc = cache.schunk.vlmeta.get("proxy-fetched-bpc", 1) + + def unavailable(*args, **kwargs): + raise RuntimeError("offline cache maintenance cannot fetch remote data") + + source = SimpleNamespace( + shape=cache.shape, + dtype=cache.dtype, + chunks=cache.chunks, + blocks=cache.blocks, + cparams=cache.cparams, + stamp=cache.schunk.vlmeta.get("proxy-stamp"), + blocks_per_chunk=bpc, + wants_blocks=unavailable, + chunk_layout=unavailable, + block_plan=unavailable, + read_range=unavailable, + ) + backend = blosc2.Proxy(source, _cache=cache, _refresh_source=False, _persistent_dirty=True) + evicted = backend._trim_cache(target_bytes, max_chunks=max_chunks) + return evicted, sum(backend._cache_sizes.values()) + def _attach_carrier_cache(self): if self.cache_policy is blosc2.CachePolicy.DISK: if self._runtime_cache is None: @@ -472,6 +566,8 @@ def _validate_geometry(self, expected, *, src=None) -> None: def _prepare_read(self): """Refresh source identity and return the backend for one operation.""" + if self._authorized_source: + return self._proxy if self._proxy is not None else self.src with self._refresh_lock: previous_stamp = getattr(self.src, "stamp", None) refresh = getattr(self.src, "refresh_identity", None) @@ -722,6 +818,15 @@ def _payload(self): } def _to_b2object_carrier(self, **kwargs): + if self._carrier is not None: + kwargs.setdefault( + "meta", + { + name: self._carrier.schunk.meta[name] + for name in self._carrier.schunk.meta + if name not in {"b2nd", "b2o", "proxy"} + }, + ) array = make_b2object_carrier( "remote_proxy", self.shape, @@ -779,12 +884,9 @@ def save( ) -> None: """Save a carrier; MEMORY exports are cold. See :meth:`to_cframe`.""" urlpath = os.fspath(urlpath) - if ( - (cache_policy is not None or not include_cache) - and any( - path is not None and os.path.abspath(path) == os.path.abspath(urlpath) - for path in (self.cache_path, self.runtime_cache_path) - ) + if (cache_policy is not None or not include_cache) and any( + path is not None and os.path.abspath(path) == os.path.abspath(urlpath) + for path in (self.cache_path, self.runtime_cache_path) ): raise ValueError("cold or policy-changing export requires a different destination") carrier = self._export_carrier(include_cache, cache_policy) diff --git a/tests/test_remote_proxy.py b/tests/test_remote_proxy.py index 9f2b5154c..57df7bff7 100644 --- a/tests/test_remote_proxy.py +++ b/tests/test_remote_proxy.py @@ -181,18 +181,14 @@ def test_disk_bound_shrinks_self_caching_carrier(tmp_path): def test_server_sparse_cache_reopens_and_exports_portable_carriers(tmp_path): url, data = _remote_array("server-sparse.b2nd", nchunks=3, chunk_size=100_000) runtime_path = tmp_path / "private-runtime" - proxy = blosc2.RemoteProxy.with_sparse_cache( - url, runtime_path, max_cache_bytes=120_000 - ) + proxy = blosc2.RemoteProxy.with_sparse_cache(url, runtime_path, max_cache_bytes=120_000) assert runtime_path.is_dir() assert proxy.runtime_cache_path == str(runtime_path) assert proxy.cache_path is None np.testing.assert_array_equal(proxy[:100_000], data[:100_000]) - reopened = blosc2.RemoteProxy.with_sparse_cache( - url, runtime_path, max_cache_bytes=120_000 - ) + reopened = blosc2.RemoteProxy.with_sparse_cache(url, runtime_path, max_cache_bytes=120_000) reopened.traffic.reset() np.testing.assert_array_equal(reopened[:100_000], data[:100_000]) assert reopened.traffic.requests == 0 @@ -309,9 +305,7 @@ def test_server_sparse_rejects_a_seed_from_another_source(tmp_path): runtime_path = tmp_path / "wrong-seed-runtime" with pytest.raises(ValueError, match="different remote source"): - blosc2.RemoteProxy.with_sparse_cache( - second_url, runtime_path, carrier=seed._carrier - ) + blosc2.RemoteProxy.with_sparse_cache(second_url, runtime_path, carrier=seed._carrier) assert not runtime_path.exists() @@ -994,3 +988,48 @@ def test_unlimited_disk_cache_does_not_evict(tmp_path): none_export = blosc2.ndarray_from_cframe(proxy.to_cframe(cache_policy=blosc2.CachePolicy.NONE)) assert none_export.schunk.vlmeta["b2o"]["cache_policy"] == "none" assert none_export.schunk.vlmeta["b2o"]["max_cache_bytes"] is None + + +def test_authorized_sparse_snapshot_never_reopens(tmp_path, monkeypatch): + url, data = _remote_array("authorized-sparse.b2nd", nchunks=3, chunk_size=10000) + source = blosc2.FsspecNDSource(url) + descriptor = {"kind": "fsspec", "version": 1, "urlpath": url} + + def forbidden(*args, **kwargs): + raise AssertionError("authorized transport was reopened or refreshed") + + monkeypatch.setattr(blosc2.RemoteProxy, "_open_source", forbidden) + monkeypatch.setattr(source, "refresh_stamp", forbidden, raising=False) + monkeypatch.setattr(source, "refresh_identity", forbidden, raising=False) + path = tmp_path / "authorized" + proxy = blosc2.RemoteProxy.with_sparse_cache( + source, path, source_descriptor=descriptor, max_cache_bytes=None + ) + assert proxy.src is source + assert proxy.read_cached(slice(0, 10000)) == (False, None) + np.testing.assert_array_equal(proxy[:10000], data[:10000]) + hit, result = proxy.read_cached(slice(0, 10000)) + assert hit + np.testing.assert_array_equal(result, data[:10000]) + assert proxy.cached_payload_bytes >= 10000 + assert proxy.trim_cache(0, max_chunks=1) == (0,) + assert not proxy.cache_contains(nchunk=0) + np.testing.assert_array_equal(proxy[:], data) + del proxy + evicted, remaining = blosc2.RemoteProxy.trim_sparse_cache(path, 0, max_chunks=1) + assert len(evicted) == 1 + assert remaining >= 20000 + with pytest.raises(ValueError, match="does not match"): + blosc2.RemoteProxy.with_sparse_cache( + source, path, source_descriptor=dict(descriptor, urlpath="memory://other") + ) + + +def test_sparse_seed_with_dirty_marker_is_not_imported(tmp_path): + url, data = _remote_array("dirty-seed.b2nd", nchunks=2, chunk_size=10000) + seed = blosc2.RemoteProxy(url, cache_policy=blosc2.CachePolicy.DISK, cache_path=tmp_path / "seed.b2nd") + np.testing.assert_array_equal(seed[:10000], data[:10000]) + seed.schunk.vlmeta["proxy-dirty"] = {"version": 1} + runtime = blosc2.RemoteProxy.with_sparse_cache(url, tmp_path / "runtime", carrier=seed.cache) + assert not runtime.cache_contains(nchunk=0) + np.testing.assert_array_equal(runtime[:], data) From 3d2b5e295f60ffcf84e86b95ba24ede839868200 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 6 Sep 2026 14:33:43 +0200 Subject: [PATCH 22/89] Clarify contiguous compatibility mode --- plans/remote-proxy-v7.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plans/remote-proxy-v7.md b/plans/remote-proxy-v7.md index 0c919c26b..865da77fc 100644 --- a/plans/remote-proxy-v7.md +++ b/plans/remote-proxy-v7.md @@ -706,8 +706,8 @@ release candidate: power failure. - Tune staging estimates and warm-export reservations instead of reserving the full work budget for one export. -- Remove the internal contiguous compatibility path after the v5 comparison - benchmark is no longer needed. It is not a supported Caterva2 deployment mode. +- Retain the internal contiguous compatibility path for deterministic comparison + and explicit rollback tests; it is not a public Caterva2 deployment setting. ## Caterva2 module map From b8a38650a3895c1cb34b04582a6458c6c996d28b Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 6 Sep 2026 14:52:28 +0200 Subject: [PATCH 23/89] Set cache maintenance interval to one minute --- plans/remote-proxy-v7.md | 1 + 1 file changed, 1 insertion(+) diff --git a/plans/remote-proxy-v7.md b/plans/remote-proxy-v7.md index 865da77fc..238770e22 100644 --- a/plans/remote-proxy-v7.md +++ b/plans/remote-proxy-v7.md @@ -634,6 +634,7 @@ the operational tuning settings that are implemented: ```toml [server.remote_proxy] +cache_maintenance_seconds = 60 cache_low_watermark = 0.90 cache_reconcile_seconds = 300 cache_prune_chunks = 64 From d8d21edca5bdf0b630ca674b42f121ca3ad39f1d Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 7 Sep 2026 11:58:26 +0200 Subject: [PATCH 24/89] Support storage_options in blosc2.open and organize remote examples --- doc/getting_started/installation.rst | 4 +- doc/guides/remote_arrays.md | 57 +++- doc/reference/fsspecndsource.rst | 4 +- examples/{ => remote}/c2array-get-slice.py | 0 examples/{ => remote}/c2array-traffic.py | 0 examples/{ndarray => remote}/c2array_expr.py | 0 .../{ndarray => remote}/concurrent-fsspec.py | 0 examples/{ => remote}/fsspec-cat2-access.py | 0 examples/{ndarray => remote}/proxy-carray.py | 0 examples/{ndarray => remote}/rw-fsspec.py | 0 examples/remote/s3-access.py | 276 ++++++++++++++++++ src/blosc2/core.py | 36 ++- src/blosc2/ndarray.py | 7 +- src/blosc2/proxy_source.py | 6 +- src/blosc2/remote_proxy.py | 28 +- src/blosc2/schunk.py | 34 ++- tests/test_fsspec.py | 23 ++ tests/test_fsspec_s3.py | 48 +++ 18 files changed, 493 insertions(+), 30 deletions(-) rename examples/{ => remote}/c2array-get-slice.py (100%) rename examples/{ => remote}/c2array-traffic.py (100%) rename examples/{ndarray => remote}/c2array_expr.py (100%) rename examples/{ndarray => remote}/concurrent-fsspec.py (100%) rename examples/{ => remote}/fsspec-cat2-access.py (100%) rename examples/{ndarray => remote}/proxy-carray.py (100%) rename examples/{ndarray => remote}/rw-fsspec.py (100%) create mode 100755 examples/remote/s3-access.py diff --git a/doc/getting_started/installation.rst b/doc/getting_started/installation.rst index 36f3213c5..bb12a8319 100644 --- a/doc/getting_started/installation.rst +++ b/doc/getting_started/installation.rst @@ -63,8 +63,8 @@ With the ``fsspec`` extra, :func:`blosc2.open` accepts any fsspec URL, chained ones included, and reads it whole, through a local cache (``cache_dir=``), or by fetching only the chunks and blocks a slice touches (``lazy=True``); see :func:`blosc2.open` and :ref:`FsspecNDSource` for what each mode supports. -``examples/ndarray/rw-fsspec.py`` walks through all three plus the write side, -and ``examples/ndarray/concurrent-fsspec.py`` shows what overlapping the +``examples/remote/rw-fsspec.py`` walks through all three plus the write side, +and ``examples/remote/concurrent-fsspec.py`` shows what overlapping the fetches buys; both run with no network or credentials. Source code diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index b1465780c..f52dfbac5 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -51,6 +51,50 @@ When opened with `lazy=True`, both routes return a {ref}`RemoteProxy`, providing `lazy=True` changes *when* data is fetched; it does not expand the underlying storage formats supported by either route. +## Access S3 and cloud object stores + +For arrays stored on Amazon S3 or S3-compatible cloud object stores (Backblaze B2, MinIO, Cloudflare R2, Ceph, Wasabi, etc.), open the `s3://` URL with `lazy=True`: + +```python +import blosc2 + +# Using default credentials from environment or ~/.aws/credentials +a = blosc2.open("s3://bucket/big.b2nd", lazy=True) +``` + +### Storage options and authentication + +Pass a `storage_options` dictionary to configure credentials, AWS profiles, or custom S3 endpoint URLs: + +```python +storage_options = { + "profile": "blosc2", # named profile from ~/.aws/credentials + "endpoint_url": "https://s3.us-west-001.backblazeb2.com", # custom endpoint + # Or explicit keys: + # "key": "AWS_ACCESS_KEY_ID", + # "secret": "AWS_SECRET_ACCESS_KEY", + # Or anonymous public access: + # "anon": True, +} + +a = blosc2.open( + "s3://bucket/big.b2nd", + lazy=True, + storage_options=storage_options, +) +``` + +The options in `storage_options` are forwarded directly to `fsspec` (and `s3fs`). + +### S3 performance: latency, caching, and concurrency + +Object stores typically incur 20–100 ms of latency per HTTP range request. Python-Blosc2 addresses this in two ways: + +1. **Caching**: Chunks and blocks fetched for a slice are kept in the local cache (in RAM by default, or persisted to disk with `cache_dir=` or `cache_path=`). Re-fetching previously read regions requires zero network round trips and zero bytes transferred. +2. **Concurrent fetches**: Independent range requests for required chunks and blocks are issued concurrently in a thread pool (configured via `max_concurrency=`, default 8). + +The runnable script `examples/remote/s3-access.py` demonstrates opening `.b2nd` and `.zarr` datasets from S3, timing metadata discovery vs. slice fetching, measuring network traffic with {ref}`Traffic`, and showing the impact of chunk caching. + ## Cache policies and memory management Every lazy open uses a cache policy. By default, fetched data is cached in memory with a bound on retained compressed payload. @@ -163,7 +207,7 @@ print(a.traffic) # Traffic(requests=0, nbytes=0) -> cache hit! Use `reset()` or subtract two readings to measure one operation. `traffic` is `None` for a local source because no network transport exists. -`examples/c2array-traffic.py` compares block, chunk, and cached reads against a live Caterva2 dataset. +`examples/remote/c2array-traffic.py` compares block, chunk, and cached reads against a live Caterva2 dataset. ## Persist and reopen remote references @@ -333,7 +377,12 @@ For ordinary S3 access, use `blosc2.open("s3://bucket/big.b2nd", lazy=True)`; th ## See also - {doc}`Tutorial 6 <../tutorials/06.remote_proxy>` — a step-by-step introduction with output. -- `examples/ndarray/rw-fsspec.py` — fsspec reading and writing examples. -- `examples/fsspec-cat2-access.py` — one dataset and cache through fsspec and Caterva2. -- `examples/c2array-traffic.py` — block, chunk, and cached transfer sizes. +- `examples/remote/s3-access.py` — S3 access comparing Blosc2 and Zarr with timing and network traffic metering. +- `examples/remote/c2array-get-slice.py` — opening and reading remote Caterva2 arrays via URLPath. +- `examples/remote/c2array-traffic.py` — block, chunk, and cached transfer sizes against Caterva2. +- `examples/remote/c2array_expr.py` — lazy expression evaluation on remote Caterva2 arrays. +- `examples/remote/concurrent-fsspec.py` — concurrent chunk fetching (`max_concurrency`) on high-latency stores. +- `examples/remote/fsspec-cat2-access.py` — one dataset and cache through fsspec and Caterva2. +- `examples/remote/proxy-carray.py` — creating a persistent local disk proxy of a remote Caterva2 array. +- `examples/remote/rw-fsspec.py` — fsspec reading and writing examples. - {ref}`RemoteProxy`, {ref}`C2Array`, {ref}`FsspecNDSource`, {ref}`ByteRangeNDSource`, {ref}`Proxy`, and {ref}`Traffic` — API reference pages. diff --git a/doc/reference/fsspecndsource.rst b/doc/reference/fsspecndsource.rst index 8b53dfff0..d8bc84f3a 100644 --- a/doc/reference/fsspecndsource.rst +++ b/doc/reference/fsspecndsource.rst @@ -14,9 +14,9 @@ For the Caterva2 alternative and a capability comparison, see :doc:`Working with Remote Arrays <../guides/remote_arrays>`. For other sources, see :ref:`ProxyNDSource` and :ref:`ProxySource`. -``examples/ndarray/rw-fsspec.py`` is a runnable walkthrough of this and the +``examples/remote/rw-fsspec.py`` is a runnable walkthrough of this and the other two ways to read an fsspec URL, and of writing one back. -``examples/ndarray/concurrent-fsspec.py`` measures ``max_concurrency`` against a +``examples/remote/concurrent-fsspec.py`` measures ``max_concurrency`` against a filesystem with a simulated round trip, since no protocol that runs offline has latency for the thread pool to hide. diff --git a/examples/c2array-get-slice.py b/examples/remote/c2array-get-slice.py similarity index 100% rename from examples/c2array-get-slice.py rename to examples/remote/c2array-get-slice.py diff --git a/examples/c2array-traffic.py b/examples/remote/c2array-traffic.py similarity index 100% rename from examples/c2array-traffic.py rename to examples/remote/c2array-traffic.py diff --git a/examples/ndarray/c2array_expr.py b/examples/remote/c2array_expr.py similarity index 100% rename from examples/ndarray/c2array_expr.py rename to examples/remote/c2array_expr.py diff --git a/examples/ndarray/concurrent-fsspec.py b/examples/remote/concurrent-fsspec.py similarity index 100% rename from examples/ndarray/concurrent-fsspec.py rename to examples/remote/concurrent-fsspec.py diff --git a/examples/fsspec-cat2-access.py b/examples/remote/fsspec-cat2-access.py similarity index 100% rename from examples/fsspec-cat2-access.py rename to examples/remote/fsspec-cat2-access.py diff --git a/examples/ndarray/proxy-carray.py b/examples/remote/proxy-carray.py similarity index 100% rename from examples/ndarray/proxy-carray.py rename to examples/remote/proxy-carray.py diff --git a/examples/ndarray/rw-fsspec.py b/examples/remote/rw-fsspec.py similarity index 100% rename from examples/ndarray/rw-fsspec.py rename to examples/remote/rw-fsspec.py diff --git a/examples/remote/s3-access.py b/examples/remote/s3-access.py new file mode 100755 index 000000000..2fa26b370 --- /dev/null +++ b/examples/remote/s3-access.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Open a remote S3 array (Blosc2 .b2nd or Zarr .zarr) and print metadata and sample data. + +Usage: + python s3-access.py [--profile PROFILE] [--endpoint-url ENDPOINT_URL] + +Examples: + python s3-access.py s3://blosc2/cube-1k-1k-1k.b2nd + python s3-access.py s3://blosc2/cube-1k-1k-1k.zarr + python s3-access.py s3://blosc2/cube-1k-1k-1k-1shard.zarr +""" + +from __future__ import annotations + +import argparse +import sys +import time +from typing import Any + +import fsspec +import zarr + +import blosc2 + +DEFAULT_PROFILE = "blosc2" +DEFAULT_ENDPOINT_URL = "https://s3.us-west-001.backblazeb2.com" + + +def get_sample_slice(arr: Any) -> Any: + """Extract a representative small sample slice regardless of array dimensionality.""" + ndim = getattr(arr, "ndim", None) + if ndim is None: + ndim = len(arr.shape) if hasattr(arr, "shape") else 0 + if ndim == 0: + return arr[()] + if ndim == 1: + return arr[: min(10, arr.shape[0])] + if ndim == 2: + return arr[: min(10, arr.shape[0]), : min(5, arr.shape[1])] + + idx: list[Any] = [slice(0, min(10, arr.shape[0]))] + for _ in range(ndim - 2): + idx.append(0) + idx.append(slice(0, min(5, arr.shape[-1]))) + return arr[tuple(idx)] + + +class Traffic: + """Track data transferred over the network.""" + + def __init__(self, nbytes: int = 0) -> None: + self.nbytes = nbytes + + def charge(self, n: int) -> None: + self.nbytes += n + + +class TrackingFile: + """Wrapper around a file-like object to track bytes read.""" + + def __init__(self, f: Any, traffic: Traffic) -> None: + self._f = f + self._traffic = traffic + + def read(self, *args: Any, **kwargs: Any) -> Any: + data = self._f.read(*args, **kwargs) + if data: + self._traffic.charge(len(data)) + return data + + def readinto(self, b: Any) -> Any: + n = self._f.readinto(b) + if n: + self._traffic.charge(n) + return n + + def __getattr__(self, name: str) -> Any: + return getattr(self._f, name) + + +def _instrument_store(store: Any, traffic: Traffic) -> None: + """Instrument an async Zarr store to record network byte traffic.""" + orig_get = store.get + + async def tracked_get(*args: Any, **kwargs: Any) -> Any: + val = await orig_get(*args, **kwargs) + if val is not None: + traffic.charge(len(val)) + return val + + orig_get_partial = store.get_partial_values + + async def tracked_get_partial(*args: Any, **kwargs: Any) -> Any: + vals = await orig_get_partial(*args, **kwargs) + for buf in vals: + if buf is not None: + traffic.charge(len(buf)) + return vals + + store.get = tracked_get + store.get_partial_values = tracked_get_partial + + if hasattr(store, "get_sync"): + orig_get_sync = store.get_sync + + def tracked_get_sync(*args: Any, **kwargs: Any) -> Any: + val = orig_get_sync(*args, **kwargs) + if val is not None: + traffic.charge(len(val)) + return val + + store.get_sync = tracked_get_sync + + +def open_remote_array( + url: str, + profile: str = DEFAULT_PROFILE, + endpoint_url: str = DEFAULT_ENDPOINT_URL, +) -> tuple[str, Any]: + """Open remote array depending on extension (.b2nd vs .zarr/.zip). + + Returns (format_name, array_object). + """ + storage_options = { + "profile": profile, + "endpoint_url": endpoint_url, + } + clean_url = url.rstrip("/") + + if clean_url.endswith((".b2nd", ".b2frame")): + arr = blosc2.open(url, lazy=True, storage_options=storage_options) + return "Blosc2 (Lazy RemoteProxy)", arr + + if clean_url.endswith((".zarr.zip", ".zip")): + from zarr.storage import ZipStore + + traffic = Traffic() + raw_f = fsspec.open(url, "rb", **storage_options).open() + tf = TrackingFile(raw_f, traffic) + store = ZipStore(tf, mode="r") + arr = zarr.open(store=store) + arr.traffic = traffic + return "Zarr (Zip)", arr + + if clean_url.endswith(".zarr") or clean_url.endswith(".zarr/"): + from zarr.storage import FsspecStore + + traffic = Traffic() + store = FsspecStore.from_url(url, storage_options=storage_options, read_only=True) + _instrument_store(store, traffic) + arr = zarr.open(store=store, mode="r") + arr.traffic = traffic + return "Zarr", arr + + # Fallback / heuristic: try zarr first, then blosc2 + try: + from zarr.storage import FsspecStore + + traffic = Traffic() + store = FsspecStore.from_url(url, storage_options=storage_options, read_only=True) + _instrument_store(store, traffic) + arr = zarr.open(store=store, mode="r") + arr.traffic = traffic + return "Zarr", arr + except Exception: + arr = blosc2.open(url, lazy=True, storage_options=storage_options) + return "Blosc2 (Lazy RemoteProxy)", arr + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Open a remote array in S3 (.b2nd or .zarr) and print metadata and data.", + ) + parser.add_argument("url", help="Remote S3 URL (e.g. s3://blosc2/cube.b2nd or s3://blosc2/cube.zarr)") + parser.add_argument( + "--profile", + default=DEFAULT_PROFILE, + help=f"AWS CLI credential profile (default: '{DEFAULT_PROFILE}')", + ) + parser.add_argument( + "--endpoint-url", + default=DEFAULT_ENDPOINT_URL, + help=f"S3 endpoint URL (default: '{DEFAULT_ENDPOINT_URL}')", + ) + + args = parser.parse_args() + + print(f"Accessing: {args.url}") + t0 = time.perf_counter() + try: + fmt, arr = open_remote_array( + url=args.url, + profile=args.profile, + endpoint_url=args.endpoint_url, + ) + except Exception as exc: + print(f"Error opening remote array: {exc}", file=sys.stderr) + return 1 + t_open = time.perf_counter() - t0 + + def get_traffic_bytes() -> int | None: + traffic = getattr(arr, "traffic", None) + if traffic is None: + src = getattr(arr, "src", None) + traffic = getattr(src, "traffic", None) + if traffic is not None and hasattr(traffic, "nbytes"): + return int(traffic.nbytes) + return None + + b_open = get_traffic_bytes() + + print(f"\n[Format: {fmt}]") + if hasattr(arr, "info"): + print(arr.info) + print("Shape: ", arr.shape) + print("Dtype: ", arr.dtype) + chunks = getattr(arr, "chunks", None) + if chunks is not None: + print("Chunks:", chunks) + blocks = getattr(arr, "blocks", None) + if blocks is not None: + print("Blocks:", blocks) + + print("\nSample slice data (1st fetch):") + t0 = time.perf_counter() + sample = get_sample_slice(arr) + t_fetch1 = time.perf_counter() - t0 + b_after_fetch1 = get_traffic_bytes() + b_fetch1 = (b_after_fetch1 - b_open) if (b_after_fetch1 is not None and b_open is not None) else None + print(sample) + + # Re-fetch the same slice to test caching behavior + t0 = time.perf_counter() + _ = get_sample_slice(arr) + t_fetch2 = time.perf_counter() - t0 + b_after_fetch2 = get_traffic_bytes() + b_fetch2 = ( + (b_after_fetch2 - b_after_fetch1) + if (b_after_fetch2 is not None and b_after_fetch1 is not None) + else None + ) + + print("\nTiming & Network Traffic:") + if b_open is not None: + print(f" - Metadata open : {t_open * 1000:7.1f} ms ({b_open / 1024:8.2f} KB transferred)") + else: + print(f" - Metadata open : {t_open * 1000:7.1f} ms") + + if b_fetch1 is not None: + print(f" - 1st slice fetch: {t_fetch1 * 1000:7.1f} ms ({b_fetch1 / 1024:8.2f} KB transferred)") + else: + print(f" - 1st slice fetch: {t_fetch1 * 1000:7.1f} ms") + + if b_fetch2 is not None: + tag = " (cache hit!)" if b_fetch2 == 0 else "" + print( + f" - 2nd slice fetch: {t_fetch2 * 1000:7.1f} ms ({b_fetch2 / 1024:8.2f} KB transferred){tag}" + ) + else: + print(f" - 2nd slice fetch: {t_fetch2 * 1000:7.1f} ms") + + if b_after_fetch2 is not None: + print(f" - Total network : {b_after_fetch2 / 1024:8.2f} KB transferred from S3") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/blosc2/core.py b/src/blosc2/core.py index 0982adf6f..92869a60d 100644 --- a/src/blosc2/core.py +++ b/src/blosc2/core.py @@ -574,7 +574,9 @@ def save_array(arr: np.ndarray, urlpath: str, chunksize: int | None = None, **kw return pack_tensor(arr, chunksize=chunksize, urlpath=urlpath, **kwargs) -def load_array(urlpath: str, dparams: dict | None = None) -> np.ndarray: +def load_array( + urlpath: str, dparams: dict | None = None, *, storage_options: dict | None = None +) -> np.ndarray: """Load a serialized NumPy array from a file. Parameters @@ -584,6 +586,9 @@ def load_array(urlpath: str, dparams: dict | None = None) -> np.ndarray: dparams: dict, optional A dictionary with the decompression parameters, which can be used in the :func:`~blosc2.decompress2` function. + storage_options: dict, optional + Parameters passed to the underlying ``fsspec`` filesystem when opening + an fsspec URL. Returns ------- @@ -616,7 +621,7 @@ def load_array(urlpath: str, dparams: dict | None = None) -> np.ndarray: :func:`~blosc2.pack_tensor` """ # May we raise a DeprecationWarning here in the future? - return load_tensor(urlpath, dparams=dparams) + return load_tensor(urlpath, dparams=dparams, storage_options=storage_options) def normalize_urlpath(urlpath: object) -> object: @@ -676,9 +681,9 @@ def _import_fsspec(urlpath: str): return fsspec -def fsspec_open(urlpath: str, mode: str): +def fsspec_open(urlpath: str, mode: str, storage_options: dict | None = None): """`fsspec.open()`, but complaining properly when fsspec is missing.""" - return _import_fsspec(urlpath).open(urlpath, mode) + return _import_fsspec(urlpath).open(urlpath, mode, **(storage_options or {})) def fsspec_cache_path(urlpath: str, cache_storage: str | pathlib.Path, suffix: str = "") -> str: @@ -704,7 +709,9 @@ def __call__(self, path: str) -> str: return SuffixedCacheMapper() -def localize_fsspec_url(urlpath: str, cache_storage: str | pathlib.Path) -> str: +def localize_fsspec_url( + urlpath: str, cache_storage: str | pathlib.Path, storage_options: dict | None = None +) -> str: """Materialize the container at *urlpath* under *cache_storage*, return its local path. Single-file containers go through fsspec's ``filecache``, which downloads @@ -717,7 +724,7 @@ def localize_fsspec_url(urlpath: str, cache_storage: str | pathlib.Path) -> str: from fsspec.utils import tokenize cache_storage = str(cache_storage) - fs, path = fsspec.url_to_fs(urlpath) + fs, path = fsspec.url_to_fs(urlpath, **(storage_options or {})) if not fs.isdir(path): # check_files is off by default in fsspec, which would happily serve a @@ -728,7 +735,7 @@ def localize_fsspec_url(urlpath: str, cache_storage: str | pathlib.Path) -> str: "check_files": True, "cache_mapper": _suffixed_cache_mapper(), } - with fsspec.open(f"filecache::{urlpath}", "rb", filecache=opts) as f: + with fsspec.open(f"filecache::{urlpath}", "rb", filecache=opts, **(storage_options or {})) as f: return f.name localdir = fsspec_cache_path(urlpath, cache_storage) @@ -793,10 +800,13 @@ def pack_tensor( # Object stores cannot be written incrementally, so build the whole cframe in # memory and PUT it in one go. remote_urlpath = kwargs.get("urlpath") if is_fsspec_url(kwargs.get("urlpath")) else None + storage_options = kwargs.pop("storage_options", None) if remote_urlpath is not None: del kwargs["urlpath"] # A remote write always replaces, but reading mode still forbids one blosc2_ext.check_access_mode(remote_urlpath, kwargs.pop("mode", "a")) + elif storage_options is not None: + raise ValueError("storage_options is only supported for fsspec URLs") schunk = blosc2.SChunk(chunksize=chunksize, data=arr, **kwargs) @@ -818,7 +828,7 @@ def pack_tensor( if remote_urlpath is not None: cframe = schunk.to_cframe() - with fsspec_open(remote_urlpath, "wb") as f: + with fsspec_open(remote_urlpath, "wb", storage_options=storage_options) as f: f.write(cframe) return len(cframe) @@ -944,7 +954,9 @@ def save_tensor( return pack_tensor(tensor, chunksize=chunksize, urlpath=urlpath, **kwargs) -def load_tensor(urlpath: str, dparams: dict | None = None) -> tensorflow.Tensor | torch.Tensor | np.ndarray: +def load_tensor( + urlpath: str, dparams: dict | None = None, *, storage_options: dict | None = None +) -> tensorflow.Tensor | torch.Tensor | np.ndarray: """Load a serialized PyTorch or TensorFlow tensor or NumPy array from a file. Parameters @@ -956,6 +968,10 @@ def load_tensor(urlpath: str, dparams: dict | None = None) -> tensorflow.Tensor A dictionary with the decompression parameters, which are the same as those used in the :func:`~blosc2.decompress2` function. + storage_options: dict, optional + Parameters passed to the underlying ``fsspec`` filesystem when opening + an fsspec URL. + Returns ------- out: tensor or ndarray @@ -985,7 +1001,7 @@ def load_tensor(urlpath: str, dparams: dict | None = None) -> tensorflow.Tensor :func:`~blosc2.save_tensor` :func:`~blosc2.pack_tensor` """ - schunk = blosc2.open(urlpath, mode="r", dparams=dparams) + schunk = blosc2.open(urlpath, mode="r", dparams=dparams, storage_options=storage_options) return _unpack_tensor(schunk) diff --git a/src/blosc2/ndarray.py b/src/blosc2/ndarray.py index d2fb889a1..71eaf45a1 100644 --- a/src/blosc2/ndarray.py +++ b/src/blosc2/ndarray.py @@ -5084,14 +5084,19 @@ def save(self, urlpath: str, contiguous=True, **kwargs: Any) -> None: raise NotImplementedError( "a sparse frame is a directory, so it cannot be saved to an fsspec URL" ) + storage_options = kwargs.pop("storage_options", None) # An object store takes the whole thing at once, and always replaces, # but reading mode still forbids a write blosc2_ext.check_access_mode(urlpath, kwargs.pop("mode", "w")) array = self.copy(**kwargs) if kwargs else self - with fsspec_open(urlpath, "wb") as f: + with fsspec_open(urlpath, "wb", storage_options=storage_options) as f: f.write(array.to_cframe()) return + if "storage_options" in kwargs and kwargs["storage_options"] is not None: + raise ValueError("storage_options is only supported for fsspec URLs") + kwargs.pop("storage_options", None) + blosc2_ext.check_access_mode(urlpath, "w") # Add urlpath to kwargs kwargs["urlpath"] = urlpath diff --git a/src/blosc2/proxy_source.py b/src/blosc2/proxy_source.py index 411c49f7c..61ecfd322 100644 --- a/src/blosc2/proxy_source.py +++ b/src/blosc2/proxy_source.py @@ -1273,6 +1273,8 @@ class FsspecNDSource(ByteRangeNDSource): The fsspec URL of the frame. max_concurrency: int, optional As in :ref:`ByteRangeNDSource`. + storage_options: dict, optional + Parameters passed to the underlying ``fsspec`` filesystem. """ def __init__( @@ -1280,6 +1282,7 @@ def __init__( urlpath: str, max_concurrency: int = REMOTE_MAX_CONCURRENCY, *, + storage_options: dict | None = None, _filesystem=None, _traffic=None, ): @@ -1287,7 +1290,7 @@ def __init__( fsspec = _import_fsspec(urlpath) if _filesystem is None: - fs, path = fsspec.url_to_fs(urlpath) + fs, path = fsspec.url_to_fs(urlpath, **(storage_options or {})) else: fs = _filesystem path = fs._strip_protocol(urlpath) @@ -1299,6 +1302,7 @@ def __init__( "chunk by chunk; open it with cache_dir= instead" ) self._fs, self._path = fs, path + self.storage_options = storage_options or {} # Identifies the remote bytes, so a cache built against them can tell it # has gone stale -- and chunk offsets from a replaced frame are garbage. # fsspec's own token, rather than a tuple of the metadata fields we guess diff --git a/src/blosc2/remote_proxy.py b/src/blosc2/remote_proxy.py index 331d2b4e1..b3fb2acd7 100644 --- a/src/blosc2/remote_proxy.py +++ b/src/blosc2/remote_proxy.py @@ -160,6 +160,9 @@ class RemoteProxy(blosc2.Operand): It is not applicable to ``NONE``. max_concurrency: int, optional Maximum number of independent remote fetches in flight. + storage_options: dict, optional + Parameters passed to the underlying ``fsspec`` filesystem when opening + an fsspec URL. """ def __init__( @@ -171,6 +174,7 @@ def __init__( cache_dir=None, max_cache_bytes=_POLICY_DEFAULT, max_concurrency: int | None = None, + storage_options: dict | None = None, _carrier=None, _runtime_cache_path=None, _source_descriptor=None, @@ -195,6 +199,8 @@ def __init__( self._max_concurrency = _validate_max_concurrency(max_concurrency) self._authorized_source = _source_descriptor is not None if self._authorized_source: + if storage_options is not None: + raise ValueError("storage_options cannot be used with an authorized source") if not isinstance(urlpath, blosc2.FsspecNDSource): raise TypeError("source_descriptor requires an authorized FsspecNDSource") expected = {"kind": "fsspec", "version": 1, "urlpath": urlpath.urlpath} @@ -204,7 +210,10 @@ def __init__( self.src, self._source = urlpath, dict(expected) else: self.src, self._source = self._open_source( - urlpath, self._max_concurrency, persistable=cache_policy is not blosc2.CachePolicy.MEMORY + urlpath, + self._max_concurrency, + persistable=cache_policy is not blosc2.CachePolicy.MEMORY, + storage_options=storage_options, ) self._runtime_urlpath = self._runtime_source(urlpath) self._expected_geometry = self._geometry(self.src) @@ -504,8 +513,17 @@ def _attach_carrier_cache(self): self._proxy = None @staticmethod - def _open_source(urlpath, max_concurrency, *, traffic=None, persistable=True): + def _open_source( + urlpath, + max_concurrency, + *, + traffic=None, + persistable=True, + storage_options: dict | None = None, + ): if isinstance(urlpath, blosc2.C2Array): + if storage_options is not None: + raise ValueError("storage_options is only supported for fsspec URLs") src = urlpath if persistable and src.urlbase is not None: _validate_persistable_url(src.urlbase) @@ -516,8 +534,8 @@ def _open_source(urlpath, max_concurrency, *, traffic=None, persistable=True): "urlbase": src.urlbase, } elif isinstance(urlpath, blosc2.URLPath): - if persistable and urlpath.urlbase is not None: - _validate_persistable_url(urlpath.urlbase) + if storage_options is not None: + raise ValueError("storage_options is only supported for fsspec URLs") src = blosc2.C2Array( urlpath.path, urlbase=urlpath.urlbase, @@ -534,6 +552,8 @@ def _open_source(urlpath, max_concurrency, *, traffic=None, persistable=True): if persistable: _validate_persistable_url(urlpath) kwargs = {} if max_concurrency is None else {"max_concurrency": max_concurrency} + if storage_options is not None: + kwargs["storage_options"] = storage_options src = blosc2.FsspecNDSource(urlpath, _traffic=traffic, **kwargs) source = {"kind": "fsspec", "version": 1, "urlpath": urlpath} else: diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 2bf99e1e2..6ec86930e 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -2010,7 +2010,9 @@ def _remote_cache_options(kwargs: dict) -> tuple[str | pathlib.Path | None, str return (cache_dir if cache_dir is not None else cache_storage), cache_path -def _remote_proxy_options(kwargs, cache_dir, cache_path, max_concurrency, *, lazy=False): +def _remote_proxy_options( + kwargs, cache_dir, cache_path, max_concurrency, *, lazy=False, storage_options=None +): """Return explicit RemoteProxy options, or None for the legacy lazy Proxy path.""" policy_present = "cache_policy" in kwargs limit_present = "max_cache_bytes" in kwargs @@ -2033,6 +2035,8 @@ def _remote_proxy_options(kwargs, cache_dir, cache_path, max_concurrency, *, laz } if limit_present: options["max_cache_bytes"] = limit + if storage_options is not None: + options["storage_options"] = storage_options return options @@ -2041,6 +2045,7 @@ def _lazy_fsspec_proxy( cache_dir: str | pathlib.Path | None, cache_path: str | pathlib.Path | None, max_concurrency: int | None = None, + storage_options: dict | None = None, ): """Wrap a remote frame in a Proxy that fetches chunks on demand. @@ -2050,6 +2055,8 @@ def _lazy_fsspec_proxy( """ # None leaves the default where it belongs, on the source itself kwargs = {} if max_concurrency is None else {"max_concurrency": max_concurrency} + if storage_options is not None: + kwargs["storage_options"] = storage_options src = blosc2.FsspecNDSource(urlpath, **kwargs) return _lazy_remote_proxy(src, urlpath, cache_dir, cache_path) @@ -2165,9 +2172,12 @@ def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): raise NotImplementedError(f"fsspec URLs can only be opened with mode='r', not {mode!r}") cache_dir, cache_path = _remote_cache_options(kwargs) + storage_options = kwargs.pop("storage_options", None) max_concurrency = kwargs.pop("max_concurrency", None) lazy = kwargs.pop("lazy", False) - remote_proxy_options = _remote_proxy_options(kwargs, cache_dir, cache_path, max_concurrency, lazy=lazy) + remote_proxy_options = _remote_proxy_options( + kwargs, cache_dir, cache_path, max_concurrency, lazy=lazy, storage_options=storage_options + ) if lazy: if offset != 0: raise NotImplementedError("offset is not supported with lazy=True") @@ -2176,7 +2186,9 @@ def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): raise NotImplementedError(f"{', '.join(requested)} is not supported with lazy=True") if remote_proxy_options is not None: return blosc2.RemoteProxy(urlpath, **remote_proxy_options) - return _lazy_fsspec_proxy(urlpath, cache_dir, cache_path, max_concurrency) + return _lazy_fsspec_proxy( + urlpath, cache_dir, cache_path, max_concurrency, storage_options=storage_options + ) if remote_proxy_options is not None: raise NotImplementedError("cache_policy and max_cache_bytes require lazy=True") @@ -2189,7 +2201,9 @@ def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): raise NotImplementedError("max_concurrency is only supported with lazy=True") if cache_dir is not None: - return open(localize_fsspec_url(urlpath, cache_dir), mode, offset, **kwargs) + return open( + localize_fsspec_url(urlpath, cache_dir, storage_options=storage_options), mode, offset, **kwargs + ) if offset != 0: raise NotImplementedError("offset on an fsspec URL requires passing cache_dir=") @@ -2202,7 +2216,7 @@ def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): "directory containers (.b2d, sparse frames) on an fsspec URL require " "passing cache_dir= to fetch them locally first" ) - with fsspec_open(urlpath, "rb") as f: + with fsspec_open(urlpath, "rb", storage_options=storage_options) as f: return blosc2.from_cframe(f.read()) @@ -2321,6 +2335,9 @@ def open( dparams: dict A dictionary with the decompression parameters, which are the same that can be used in the :func:`~blosc2.decompress2` function. + storage_options: dict, optional + Parameters passed to the underlying ``fsspec`` filesystem when opening + an fsspec URL (for instance credentials, endpoint URL, token, client_kwargs, etc.). Returns ------- @@ -2346,7 +2363,8 @@ def open( * fsspec URLs need the ``fsspec`` extra (``pip install "blosc2[fsspec]"``) and the driver for the protocol (``s3fs``, ``gcsfs``...), which fsspec asks for - by name when it is missing; credentials are configured there, not here. + by name when it is missing. Driver and protocol parameters (credentials, + endpoint URL, region, etc.) can be passed directly via ``storage_options``. ``mode != 'r'`` always raises, as object stores have no rename and no locks. A plain URL read rebuilds the object from a cframe held in memory, so it covers ``.b2nd``, ``.b2f`` and ``.b2e`` only -- a ``.b2z`` store is a zip @@ -2418,6 +2436,10 @@ def open( if is_fsspec_url(urlpath): return _open_fsspec_url(urlpath, mode, offset, kwargs) + if "storage_options" in kwargs and kwargs["storage_options"] is not None: + raise ValueError("storage_options is only supported for fsspec URLs") + kwargs.pop("storage_options", None) + # Keep explicit store paths on the direct dispatch path. For regular # Blosc containers, try the standard open first and only fall back to the # more expensive store probing when that fails. diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index a20943749..447ddd4ae 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -1476,3 +1476,26 @@ def test_lazy_blocks_with_a_repeated_value_chunk(monkeypatch, any_chunk_wants_bl assert np.array_equal(p[0:5, 0:10], np.full((5, 10), 3.5)) assert len(chunks) == 1 assert np.array_equal(p[...], np.full((400, 500), 3.5)) + + +def test_open_memory_url_with_storage_options(): + a = blosc2.arange(10, dtype="i4") + a.save("memory://so_test.b2nd", storage_options={}) + b = blosc2.open("memory://so_test.b2nd", storage_options={}) + assert isinstance(b, blosc2.NDArray) + assert np.array_equal(b[:], a[:]) + + lazy_b = blosc2.open("memory://so_test.b2nd", lazy=True, storage_options={}) + assert isinstance(lazy_b, blosc2.RemoteProxy) + assert np.array_equal(lazy_b[:], a[:]) + + +def test_fsspec_ndsource_and_remote_proxy_storage_options(): + a = blosc2.arange(20, dtype="i4") + a.save("memory://so_source.b2nd") + + src = blosc2.FsspecNDSource("memory://so_source.b2nd", storage_options={}) + assert src.storage_options == {} + + proxy = blosc2.RemoteProxy("memory://so_source.b2nd", storage_options={}) + assert np.array_equal(proxy[:], a[:]) diff --git a/tests/test_fsspec_s3.py b/tests/test_fsspec_s3.py index 61b148d54..2d411e4c8 100644 --- a/tests/test_fsspec_s3.py +++ b/tests/test_fsspec_s3.py @@ -197,3 +197,51 @@ def test_a_kept_index_spares_a_later_run_the_reads(s3_endpoint, tmp_path): assert np.array_equal(p[60:62, 30:40], data[60:62, 30:40]) assert len(traffic) == 1 # one block, and nothing to say where it was assert np.array_equal(p[...], data) + + +def test_storage_options(s3_endpoint, tmp_path): + storage_options = { + "endpoint_url": s3_endpoint, + "key": "testing", + "secret": "testing", + "client_kwargs": {"region_name": "eu-west-1"}, + } + # Save with storage_options + url = f"s3://{BUCKET}/so_test.b2nd" + data = np.arange(500, dtype=np.int32) + a = blosc2.asarray(data, chunks=(100,)) + a.save(url, mode="w", storage_options=storage_options) + + # Open whole with storage_options + b = blosc2.open(url, storage_options=storage_options) + assert np.array_equal(b[:], data) + + # Open lazy with storage_options + lazy_b = blosc2.open(url, lazy=True, storage_options=storage_options) + assert np.array_equal(lazy_b[50:150], data[50:150]) + assert np.array_equal(lazy_b[:], data) + + # Open with cache_dir and storage_options + cached_b = blosc2.open(url, cache_dir=tmp_path, storage_options=storage_options) + assert np.array_equal(cached_b[:], data) + + # save_array and save_tensor with storage_options + url_arr = f"s3://{BUCKET}/so_save_array.b2nd" + blosc2.save_array(data, url_arr, storage_options=storage_options) + assert np.array_equal(blosc2.load_array(url_arr, storage_options=storage_options), data) + + url_tensor = f"s3://{BUCKET}/so_save_tensor.b2nd" + blosc2.save_tensor(data, url_tensor, storage_options=storage_options) + assert np.array_equal(blosc2.load_tensor(url_tensor, storage_options=storage_options), data) + + +def test_storage_options_invalid_path(tmp_path): + local_file = str(tmp_path / "local.b2nd") + a = blosc2.arange(10) + a.save(local_file) + with pytest.raises(ValueError, match="storage_options is only supported for fsspec URLs"): + blosc2.open(local_file, storage_options={"foo": "bar"}) + with pytest.raises(ValueError, match="storage_options is only supported for fsspec URLs"): + a.save(local_file, mode="w", storage_options={"foo": "bar"}) + with pytest.raises(ValueError, match="storage_options is only supported for fsspec URLs"): + blosc2.save_array(np.arange(10), local_file, storage_options={"foo": "bar"}) From f47e35dd61073885c6c440d4a9b52257d696ec06 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 7 Sep 2026 12:36:48 +0200 Subject: [PATCH 25/89] Add b2nd-to-zarr CLI utility --- doc/getting_started/installation.rst | 3 + pyproject.toml | 3 + src/blosc2/cli/__init__.py | 4 + src/blosc2/cli/b2nd_to_zarr.py | 677 +++++++++++++++++++++++++++ tests/test_b2nd_to_zarr.py | 141 ++++++ 5 files changed, 828 insertions(+) create mode 100644 src/blosc2/cli/b2nd_to_zarr.py create mode 100644 tests/test_b2nd_to_zarr.py diff --git a/doc/getting_started/installation.rst b/doc/getting_started/installation.rst index bb12a8319..9e6dd5dd4 100644 --- a/doc/getting_started/installation.rst +++ b/doc/getting_started/installation.rst @@ -40,6 +40,9 @@ grouped into *extras* that you opt into with the ``blosc2[extra]`` syntax: * - ``parquet`` - The ``parquet-to-blosc2`` converter (``pyarrow``); see :doc:`../guides/parquet_to_blosc2`. + * - ``zarr`` + - The ``b2nd-to-zarr`` (or ``blosc2-to-zarr``) converter (``zarr``); converts + Blosc2 NDArrays into Zarr arrays. * - ``fsspec`` - Reading and writing single-file containers through any `fsspec `_ URL. The HTTP(S) driver is diff --git a/pyproject.toml b/pyproject.toml index b5703539d..d3efcb6ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ documentation = "https://www.blosc.org/python-blosc2/python-blosc2.html" [project.optional-dependencies] parquet = ["pyarrow"] +zarr = ["zarr"] # The b2view terminal viewer (the `b2view` script) is opt-in: most users want # blosc2 only as a compression library, and the TUI stack has no use under # wasm32 (no TTY). Install with `pip install "blosc2[tui]"`. This also pulls @@ -67,6 +68,8 @@ fsspec = ["fsspec[http]"] [project.scripts] parquet-to-blosc2 = "blosc2.cli.parquet_to_blosc2:main" +b2nd-to-zarr = "blosc2.cli.b2nd_to_zarr:main" +blosc2-to-zarr = "blosc2.cli.b2nd_to_zarr:main" b2view = "blosc2.b2view.cli:main" [dependency-groups] diff --git a/src/blosc2/cli/__init__.py b/src/blosc2/cli/__init__.py index e74517cc8..a2f7e2317 100644 --- a/src/blosc2/cli/__init__.py +++ b/src/blosc2/cli/__init__.py @@ -1 +1,5 @@ """Command-line utilities for blosc2.""" + +from .b2nd_to_zarr import b2nd_to_zarr, blosc2_to_zarr + +__all__ = ["b2nd_to_zarr", "blosc2_to_zarr"] diff --git a/src/blosc2/cli/b2nd_to_zarr.py b/src/blosc2/cli/b2nd_to_zarr.py new file mode 100644 index 000000000..46e9884cb --- /dev/null +++ b/src/blosc2/cli/b2nd_to_zarr.py @@ -0,0 +1,677 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Convert a Blosc2 .b2nd file (NDArray) into a Zarr array.""" + +from __future__ import annotations + +import argparse +import contextlib +import itertools +import math +import os +import shutil +import sys +import time +from pathlib import Path +from typing import Any + +import numpy as np + +import blosc2 + +try: + from rich.console import Console + from rich.progress import ( + BarColumn, + Progress, + TaskProgressColumn, + TextColumn, + TimeElapsedColumn, + TimeRemainingColumn, + TransferSpeedColumn, + ) + from rich.table import Table + + HAVE_RICH = True +except ImportError: + HAVE_RICH = False + +DEFAULT_BUFFER_SIZE_MB = 128 + + +def require_zarr() -> Any: + """Ensure the zarr package is installed.""" + try: + import zarr + + return zarr + except ImportError as exc: + raise ImportError("b2nd-to-zarr requires zarr; install it with: pip install 'blosc2[zarr]'") from exc + + +def format_bytes(size: float) -> str: + """Format byte count into human-readable string.""" + units = ["B", "KB", "MB", "GB", "TB"] + unit_idx = 0 + while size >= 1024.0 and unit_idx < len(units) - 1: + size /= 1024.0 + unit_idx += 1 + return f"{size:,.2f} {units[unit_idx]}" if unit_idx > 0 else f"{int(size)} B" + + +def get_path_size(path: Path) -> int: + """Get total byte size of a file or directory tree.""" + if not path.exists(): + return 0 + if path.is_file(): + return path.stat().st_size + total = 0 + for root, _, files in os.walk(path): + for f in files: + fp = os.path.join(root, f) + with contextlib.suppress(OSError): + total += os.path.getsize(fp) + return total + + +def parse_shape(shape_str: str) -> tuple[int, ...]: + """Parse comma- or whitespace-separated integer dimensions.""" + cleaned = shape_str.replace("x", ",").replace(" ", ",").replace("(", "").replace(")", "") + dims = [int(p.strip()) for p in cleaned.split(",") if p.strip()] + if not dims or any(d <= 0 for d in dims): + raise ValueError(f"Invalid shape string '{shape_str}', dimensions must be positive integers.") + return tuple(dims) + + +def compute_copy_shape( + shape: tuple[int, ...], + target_chunks: tuple[int, ...], + itemsize: int, + max_buffer_bytes: int = DEFAULT_BUFFER_SIZE_MB * 1024 * 1024, +) -> tuple[int, ...]: + """Compute batch copy shape aligned with chunk boundaries to optimize memory and throughput.""" + copy_shape = list(target_chunks) + ndim = len(shape) + + # First attempt: expand inner dimensions to full shape (from right to left) + # to maximize contiguous memory slicing in C-order arrays. + for d in reversed(range(ndim)): + test_shape = list(copy_shape) + test_shape[d] = shape[d] + test_bytes = math.prod(test_shape) * itemsize + if test_bytes <= max_buffer_bytes: + copy_shape[d] = shape[d] + else: + current_bytes = math.prod(copy_shape) * itemsize + if current_bytes < max_buffer_bytes: + bytes_per_chunk = current_bytes // math.ceil(copy_shape[d] / target_chunks[d]) + max_chunks = max_buffer_bytes // bytes_per_chunk + num_chunks = math.ceil(shape[d] / target_chunks[d]) + mult = min(max(1, max_chunks), num_chunks) + copy_shape[d] = min(shape[d], mult * target_chunks[d]) + break + + # Second attempt: if memory headroom remains, expand leading dimensions + for d in range(ndim): + current_bytes = math.prod(copy_shape) * itemsize + if current_bytes >= max_buffer_bytes: + break + num_chunks = math.ceil(shape[d] / target_chunks[d]) + curr_chunks = math.ceil(copy_shape[d] / target_chunks[d]) + if curr_chunks < num_chunks: + bytes_per_chunk = current_bytes // curr_chunks + max_chunks = max_buffer_bytes // bytes_per_chunk + mult = min(max(1, max_chunks), num_chunks) + if mult > curr_chunks: + copy_shape[d] = min(shape[d], mult * target_chunks[d]) + + return tuple(copy_shape) + + +def resolve_shuffle(arr: blosc2.NDArray, shuffle_mode: str) -> str: + """Resolve shuffle filter mode from blosc2 cparams or explicit argument.""" + if shuffle_mode != "auto": + return shuffle_mode + cparams = getattr(arr, "cparams", None) + if cparams and hasattr(cparams, "filters"): + if blosc2.Filter.BITSHUFFLE in cparams.filters: + return "bitshuffle" + if blosc2.Filter.SHUFFLE in cparams.filters: + return "shuffle" + return "noshuffle" + + +def resolve_cname(arr: blosc2.NDArray, codec_name: str) -> str: + """Resolve compression algorithm name from blosc2 cparams or explicit argument.""" + valid_cnames = ("lz4", "lz4hc", "blosclz", "snappy", "zlib", "zstd") + if codec_name in valid_cnames: + return codec_name + if codec_name == "blosc": + return "zstd" + if codec_name == "auto": + cparams = getattr(arr, "cparams", None) + if cparams and hasattr(cparams, "codec"): + candidate = cparams.codec.name.lower() + if candidate in valid_cnames: + return candidate + return "zstd" + raise ValueError( + f"Unsupported codec '{codec_name}'. Supported: auto, blosc, zstd, lz4, lz4hc, blosclz, zlib, gzip, none" + ) + + +def determine_zarr_codec( + arr: blosc2.NDArray, + codec_name: str = "auto", + clevel: int | None = None, + shuffle_mode: str = "auto", + blocksize: int | None = None, +) -> Any: + """Determine the Zarr compressor codec to use based on Blosc2 cparams and user options.""" + if codec_name in ("none", "uncompressed"): + return None + + cparams = getattr(arr, "cparams", None) + actual_clevel = clevel if clevel is not None else getattr(cparams, "clevel", 5) + + if codec_name == "gzip": + from zarr.codecs import GzipCodec + + return GzipCodec(level=actual_clevel) + + actual_blocksize = 0 + if blocksize is not None: + actual_blocksize = blocksize + elif cparams and getattr(cparams, "blocksize", 0) > 0: + actual_blocksize = cparams.blocksize + + from zarr.codecs import BloscCodec + + return BloscCodec( + cname=resolve_cname(arr, codec_name), + clevel=actual_clevel, + shuffle=resolve_shuffle(arr, shuffle_mode), + typesize=arr.dtype.itemsize, + blocksize=actual_blocksize, + ) + + +def copy_metadata(arr: blosc2.NDArray, z_arr: Any) -> None: + """Copy non-internal metadata and vlmeta from Blosc2 NDArray to Zarr attrs.""" + schunk = getattr(arr, "schunk", None) + if schunk is None: + return + + user_meta: dict[str, Any] = {} + meta = getattr(schunk, "meta", None) + if meta is not None: + with contextlib.suppress(Exception): + for k in meta: + if k != "b2nd": + with contextlib.suppress(Exception): + user_meta[k] = meta[k] + + vlmeta = getattr(schunk, "vlmeta", None) + if vlmeta is not None: + with contextlib.suppress(Exception): + for k in vlmeta: + with contextlib.suppress(Exception): + user_meta[k] = vlmeta[k] + + for k, v in user_meta.items(): + with contextlib.suppress(Exception): + z_arr.attrs[k] = v + + +def copy_batches( + arr: blosc2.NDArray, + z_arr: Any, + batch_slices: list[tuple[slice, ...]], + total_bytes: int, + quiet: bool = False, + verbose: bool = False, +) -> float: + """Copy data slices from Blosc2 NDArray to Zarr array with progress tracking.""" + total_batches = len(batch_slices) + t_start = time.perf_counter() + + if HAVE_RICH and not quiet: + console = Console() + with Progress( + TextColumn("[bold blue]{task.description}"), + BarColumn(), + TaskProgressColumn(), + TransferSpeedColumn(), + TimeElapsedColumn(), + TimeRemainingColumn(), + console=console, + ) as progress: + task = progress.add_task("Converting b2nd -> zarr...", total=total_bytes) + for s in batch_slices: + data = arr[s] + z_arr[s] = data + batch_size = data.nbytes + progress.update(task, advance=batch_size) + del data + else: + for idx, s in enumerate(batch_slices): + data = arr[s] + z_arr[s] = data + del data + if verbose and not quiet: + pct = (idx + 1) / total_batches * 100 + print(f"Batch {idx + 1}/{total_batches} ({pct:.1f}%) written") + + return time.perf_counter() - t_start + + +def verify_arrays( + b2_arr: blosc2.NDArray, + z_arr: Any, + slices_list: list[tuple[slice, ...]], + full: bool = False, + quiet: bool = False, +) -> bool: + """Verify that data in Zarr array matches source Blosc2 NDArray.""" + if b2_arr.shape != z_arr.shape: + raise ValueError(f"Shape mismatch: Blosc2 {b2_arr.shape} vs Zarr {z_arr.shape}") + if b2_arr.dtype != z_arr.dtype: + raise ValueError(f"Dtype mismatch: Blosc2 {b2_arr.dtype} vs Zarr {z_arr.dtype}") + + if not full: + corner_indices = [ + tuple(0 for _ in b2_arr.shape), + tuple(-1 for _ in b2_arr.shape), + tuple(s // 2 for s in b2_arr.shape), + ] + for idx in corner_indices: + v_b2 = b2_arr[idx] + v_z = z_arr[idx] + if not np.array_equal(v_b2, v_z): + raise ValueError(f"Verification failed at index {idx}: Blosc2={v_b2} != Zarr={v_z}") + + if slices_list: + for s in (slices_list[0], slices_list[-1]): + b2_chunk = b2_arr[s] + z_chunk = z_arr[s] + if not np.array_equal(b2_chunk, z_chunk): + raise ValueError(f"Verification failed for slice {s}") + if not quiet: + print("✓ Fast sample verification passed!") + return True + + if not quiet: + print("Performing full verification across all slices...") + for idx, s in enumerate(slices_list): + b2_data = b2_arr[s] + z_data = z_arr[s] + if not np.array_equal(b2_data, z_data): + raise ValueError(f"Full verification failed at slice {s} (batch {idx})") + if not quiet: + print("✓ Full element verification passed!") + return True + + +def b2nd_to_zarr( + input_path: str | Path, + output_path: str | Path | None = None, + *, + chunks: tuple[int, ...] | None = None, + shards: tuple[int, ...] | None = None, + sharded: bool = False, + codec: str = "auto", + clevel: int | None = None, + shuffle: str = "auto", + blocksize: int | None = None, + zarr_format: int = 3, + buffer_size_mb: int = DEFAULT_BUFFER_SIZE_MB, + overwrite: bool = False, + verify: bool = False, + full_verify: bool = False, + quiet: bool = False, + verbose: bool = False, +) -> dict[str, Any]: + """Convert a Blosc2 .b2nd file containing an NDArray to a Zarr array.""" + zarr = require_zarr() + + src_path = Path(input_path).expanduser().resolve() + if not src_path.exists(): + raise FileNotFoundError(f"Input file not found: {src_path}") + + dst_path = ( + src_path.with_suffix(".zarr") if output_path is None else Path(output_path).expanduser().resolve() + ) + + if dst_path.exists(): + if not overwrite: + raise FileExistsError( + f"Destination path already exists: {dst_path}. Use -f/--force/--overwrite to replace it." + ) + if dst_path.is_dir(): + shutil.rmtree(dst_path) + else: + dst_path.unlink() + + t_start = time.perf_counter() + arr = blosc2.open(str(src_path)) + if not isinstance(arr, blosc2.NDArray): + raise TypeError( + f"Expected a blosc2.NDArray object in '{src_path}', but found {type(arr).__name__}. " + "Only NDArray objects are supported for now." + ) + + shape = arr.shape + dtype = arr.dtype + itemsize = dtype.itemsize + uncompressed_bytes = math.prod(shape) * itemsize + src_file_size = get_path_size(src_path) + + target_chunks = chunks if chunks is not None else arr.chunks + target_shards = shards + + if sharded and target_shards is None: + blocks = getattr(arr, "blocks", None) + if blocks is not None and all(c % b == 0 for c, b in zip(arr.chunks, blocks, strict=True)): + target_shards = arr.chunks + target_chunks = blocks + else: + target_shards = arr.chunks + + if zarr_format == 2 and target_shards is not None: + raise ValueError("Sharding is only supported in Zarr format 3.") + + compressor = determine_zarr_codec( + arr, codec_name=codec, clevel=clevel, shuffle_mode=shuffle, blocksize=blocksize + ) + unit_shape = target_shards if target_shards is not None else target_chunks + copy_shape = compute_copy_shape( + shape, unit_shape, itemsize, max_buffer_bytes=buffer_size_mb * 1024 * 1024 + ) + + slices_per_dim = [ + [slice(i, min(i + step, s)) for i in range(0, s, step)] + for s, step in zip(shape, copy_shape, strict=True) + ] + batch_slices = list(itertools.product(*slices_per_dim)) + + if verbose and not quiet: + print(f"Source: {src_path} ({format_bytes(src_file_size)})") + print(f"Destination: {dst_path}") + print(f"Array shape: {shape}, dtype: {dtype}") + print(f"Target chunks: {target_chunks}, shards: {target_shards}") + print(f"Batch copy shape: {copy_shape} ({len(batch_slices)} batches)") + print(f"Compressor: {compressor}") + + is_zip = dst_path.name.endswith(".zip") + zip_store = None + if is_zip: + from zarr.storage import ZipStore + + zip_store = ZipStore(str(dst_path), mode="w") + target_store = zip_store + else: + target_store = str(dst_path) + + z_arr = zarr.create_array( + store=target_store, + shape=shape, + chunks=target_chunks, + shards=target_shards, + dtype=dtype, + compressors=compressor, + zarr_format=zarr_format, + overwrite=overwrite, + ) + + copy_metadata(arr, z_arr) + copy_duration = copy_batches(arr, z_arr, batch_slices, uncompressed_bytes, quiet=quiet, verbose=verbose) + if zip_store is not None: + zip_store.close() + + throughput_mb_s = (uncompressed_bytes / (1024 * 1024)) / copy_duration if copy_duration > 0 else 0.0 + + if verify or full_verify: + if is_zip: + from zarr.storage import ZipStore + + read_store = ZipStore(str(dst_path), mode="r") + z_arr_read = zarr.open(store=read_store) + verify_arrays(arr, z_arr_read, batch_slices, full=full_verify, quiet=quiet) + read_store.close() + else: + verify_arrays(arr, z_arr, batch_slices, full=full_verify, quiet=quiet) + + dst_size = get_path_size(dst_path) + total_duration = time.perf_counter() - t_start + + return { + "src_path": src_path, + "dst_path": dst_path, + "shape": shape, + "dtype": dtype, + "chunks": target_chunks, + "shards": target_shards, + "codec": compressor, + "src_size_bytes": src_file_size, + "dst_size_bytes": dst_size, + "uncompressed_bytes": uncompressed_bytes, + "copy_duration_sec": copy_duration, + "total_duration_sec": total_duration, + "throughput_mb_s": throughput_mb_s, + "src_cratio": uncompressed_bytes / src_file_size if src_file_size > 0 else 1.0, + "dst_cratio": uncompressed_bytes / dst_size if dst_size > 0 else 1.0, + } + + +# Alias for symmetry with blosc2-to-zarr +blosc2_to_zarr = b2nd_to_zarr + + +def build_parser(prog: str = "b2nd-to-zarr") -> argparse.ArgumentParser: + """Build the command-line argument parser.""" + return argparse.ArgumentParser( + prog=prog, + description="Convert a Blosc2 .b2nd file (NDArray) into a Zarr array.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=f"""\ +Examples: + {prog} cube.b2nd + {prog} cube.b2nd cube.zarr --chunks 20,500,500 + {prog} cube.b2nd -f --verify --full-verify + {prog} cube.b2nd --codec zstd --clevel 3 + {prog} cube.b2nd --sharded +""", + ) + + +def main(argv: list[str] | None = None) -> int: + """Main CLI entry point.""" + prog = Path(sys.argv[0]).name if sys.argv and sys.argv[0] else "b2nd-to-zarr" + parser = build_parser(prog=prog) + + parser.add_argument("input", help="Path to input .b2nd file") + parser.add_argument( + "output", + nargs="?", + default=None, + help="Path to output .zarr array (default: .zarr)", + ) + parser.add_argument( + "-o", + "--output-path", + dest="output_opt", + default=None, + help="Optional explicit output path (overrides positional output)", + ) + parser.add_argument( + "-c", + "--chunks", + type=str, + default=None, + help="Zarr chunk shape, e.g. '10,1000,1000' (default: inherit from .b2nd)", + ) + parser.add_argument( + "--shards", + type=str, + default=None, + help="Zarr shard shape (for sharded arrays in Zarr v3)", + ) + parser.add_argument( + "--sharded", + action="store_true", + help="Enable sharding automatically: map .b2nd chunks to shards, and blocks to chunks", + ) + parser.add_argument( + "--codec", + type=str, + default="auto", + choices=["auto", "blosc", "zstd", "lz4", "lz4hc", "blosclz", "zlib", "gzip", "none"], + help="Compression codec for Zarr (default: 'auto', matching .b2nd cparams)", + ) + parser.add_argument( + "--clevel", + type=int, + default=None, + help="Compression level (0-9, default: inherit from .b2nd)", + ) + parser.add_argument( + "--shuffle", + type=str, + default="auto", + choices=["auto", "noshuffle", "shuffle", "bitshuffle"], + help="Shuffle filter (default: 'auto', matching .b2nd filters)", + ) + parser.add_argument( + "--blocksize", + type=int, + default=None, + help="Block size in bytes for Blosc codec (default: inherit from .b2nd cparams)", + ) + parser.add_argument( + "--zarr-format", + type=int, + default=3, + choices=[2, 3], + help="Zarr specification format (default: 3)", + ) + parser.add_argument( + "-b", + "--buffer-size", + type=int, + default=DEFAULT_BUFFER_SIZE_MB, + help=f"Max memory buffer size in MB for batch copying (default: {DEFAULT_BUFFER_SIZE_MB})", + ) + parser.add_argument( + "-f", + "--force", + "--overwrite", + dest="overwrite", + action="store_true", + help="Overwrite destination store if it already exists", + ) + parser.add_argument( + "--verify", + action="store_true", + help="Verify array integrity (corner and sample equality check)", + ) + parser.add_argument( + "--full-verify", + action="store_true", + help="Perform thorough element-by-element verification of all chunks", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Show detailed conversion information", + ) + parser.add_argument( + "-q", + "--quiet", + action="store_true", + help="Suppress all progress and informational output", + ) + + args = parser.parse_args(argv) + + out_path = args.output_opt if args.output_opt is not None else args.output + parsed_chunks = parse_shape(args.chunks) if args.chunks else None + parsed_shards = parse_shape(args.shards) if args.shards else None + + try: + summary = b2nd_to_zarr( + input_path=args.input, + output_path=out_path, + chunks=parsed_chunks, + shards=parsed_shards, + sharded=args.sharded, + codec=args.codec, + clevel=args.clevel, + shuffle=args.shuffle, + blocksize=args.blocksize, + zarr_format=args.zarr_format, + buffer_size_mb=args.buffer_size, + overwrite=args.overwrite, + verify=args.verify, + full_verify=args.full_verify, + quiet=args.quiet, + verbose=args.verbose, + ) + except Exception as exc: + print(f"Error: {exc}", file=sys.stderr) + if args.verbose: + import traceback + + traceback.print_exc() + return 1 + + if not args.quiet: + if HAVE_RICH: + console = Console() + table = Table(title="[bold green]Conversion Summary[/bold green]", box=None) + table.add_column("Property", style="bold cyan") + table.add_column("Value") + table.add_row("Source", f"{summary['src_path']} ({format_bytes(summary['src_size_bytes'])})") + table.add_row( + "Destination", f"{summary['dst_path']} ({format_bytes(summary['dst_size_bytes'])})" + ) + table.add_row("Shape / Dtype", f"{summary['shape']} / {summary['dtype']}") + table.add_row("Chunks", str(summary["chunks"])) + if summary["shards"] is not None: + table.add_row("Shards", str(summary["shards"])) + table.add_row("Uncompressed Size", format_bytes(summary["uncompressed_bytes"])) + table.add_row("Source Compression", f"{summary['src_cratio']:.2f}x") + table.add_row("Zarr Compression", f"{summary['dst_cratio']:.2f}x") + table.add_row( + "Conversion Speed", + f"{summary['throughput_mb_s']:.1f} MB/s ({summary['copy_duration_sec']:.2f} s)", + ) + table.add_row("Total Time", f"{summary['total_duration_sec']:.2f} s") + console.print() + console.print(table) + else: + print("\n--- Conversion Summary ---") + print(f"Source: {summary['src_path']} ({format_bytes(summary['src_size_bytes'])})") + print(f"Destination: {summary['dst_path']} ({format_bytes(summary['dst_size_bytes'])})") + print(f"Shape / Dtype: {summary['shape']} / {summary['dtype']}") + print(f"Chunks: {summary['chunks']}") + if summary["shards"] is not None: + print(f"Shards: {summary['shards']}") + print(f"Uncompressed: {format_bytes(summary['uncompressed_bytes'])}") + print(f"Source CRatio: {summary['src_cratio']:.2f}x") + print(f"Zarr CRatio: {summary['dst_cratio']:.2f}x") + print( + f"Conversion Speed: {summary['throughput_mb_s']:.1f} MB/s ({summary['copy_duration_sec']:.2f} s)" + ) + print(f"Total Time: {summary['total_duration_sec']:.2f} s") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_b2nd_to_zarr.py b/tests/test_b2nd_to_zarr.py new file mode 100644 index 000000000..cff1c599c --- /dev/null +++ b/tests/test_b2nd_to_zarr.py @@ -0,0 +1,141 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +from __future__ import annotations + +import numpy as np +import pytest + +import blosc2 +from blosc2.cli.b2nd_to_zarr import b2nd_to_zarr, blosc2_to_zarr, main + +zarr = pytest.importorskip("zarr") + + +def test_b2nd_to_zarr_basic(tmp_path): + src = tmp_path / "data.b2nd" + dst = tmp_path / "data.zarr" + + shape = (100, 50) + chunks = (20, 25) + data = np.arange(5000, dtype=np.float32).reshape(shape) + a = blosc2.asarray(data, chunks=chunks, urlpath=str(src), mode="w") + a.schunk.vlmeta["custom_key"] = "custom_value" + + summary = b2nd_to_zarr(src, dst, verify=True) + assert summary["shape"] == shape + assert summary["dtype"] == np.float32 + assert summary["chunks"] == chunks + assert dst.exists() + + z = zarr.open(store=str(dst), mode="r") + assert z.shape == shape + assert z.dtype == np.float32 + np.testing.assert_array_equal(z[:], data) + assert z.attrs.get("custom_key") == "custom_value" + + +def test_b2nd_to_zarr_custom_chunks_and_codec(tmp_path): + src = tmp_path / "cube.b2nd" + dst = tmp_path / "cube.zarr" + + shape = (30, 40, 50) + data = np.arange(np.prod(shape), dtype=np.int32).reshape(shape) + blosc2.asarray(data, chunks=(10, 10, 10), urlpath=str(src), mode="w") + + new_chunks = (15, 20, 25) + summary = b2nd_to_zarr( + src, + dst, + chunks=new_chunks, + codec="lz4", + clevel=3, + shuffle="shuffle", + full_verify=True, + ) + assert summary["chunks"] == new_chunks + + z = zarr.open(store=str(dst), mode="r") + assert z.chunks == new_chunks + np.testing.assert_array_equal(z[:], data) + + +def test_b2nd_to_zarr_sharded(tmp_path): + src = tmp_path / "sharded.b2nd" + dst = tmp_path / "sharded.zarr" + + shape = (40, 40) + data = np.arange(1600, dtype=np.int64).reshape(shape) + blosc2.asarray(data, chunks=(20, 20), blocks=(5, 5), urlpath=str(src), mode="w") + + summary = b2nd_to_zarr(src, dst, sharded=True, verify=True) + assert summary["shards"] == (20, 20) + assert summary["chunks"] == (5, 5) + + z = zarr.open(store=str(dst), mode="r") + assert z.shards == (20, 20) + assert z.chunks == (5, 5) + np.testing.assert_array_equal(z[:], data) + + +def test_b2nd_to_zarr_zip_store(tmp_path): + src = tmp_path / "arr.b2nd" + dst = tmp_path / "arr.zarr.zip" + + data = np.linspace(0, 1, 1000, dtype=np.float64) + blosc2.asarray(data, chunks=(100,), urlpath=str(src), mode="w") + + summary = blosc2_to_zarr(src, dst, verify=True) + assert dst.exists() + assert summary["dst_path"] == dst + + from zarr.storage import ZipStore + + store = ZipStore(str(dst), mode="r") + z = zarr.open(store=store) + np.testing.assert_allclose(z[:], data) + store.close() + + +def test_b2nd_to_zarr_overwrite_protection(tmp_path): + src = tmp_path / "data.b2nd" + dst = tmp_path / "data.zarr" + + data = np.arange(10, dtype=np.int32) + blosc2.asarray(data, urlpath=str(src), mode="w") + + b2nd_to_zarr(src, dst) + assert dst.exists() + + with pytest.raises(FileExistsError): + b2nd_to_zarr(src, dst, overwrite=False) + + b2nd_to_zarr(src, dst, overwrite=True) + assert dst.exists() + + +def test_cli_main(tmp_path, capsys): + src = tmp_path / "cli.b2nd" + dst = tmp_path / "cli.zarr" + + data = np.arange(50, dtype=np.int16) + blosc2.asarray(data, chunks=(10,), urlpath=str(src), mode="w") + + ret = main([str(src), str(dst), "--verify", "--chunks", "25", "--quiet"]) + assert ret == 0 + assert dst.exists() + + z = zarr.open(store=str(dst), mode="r") + assert z.chunks == (25,) + np.testing.assert_array_equal(z[:], data) + + +def test_cli_main_errors(tmp_path, capsys): + ret = main(["/nonexistent/path/here.b2nd"]) + assert ret == 1 + captured = capsys.readouterr() + assert "Error:" in captured.err From 389ae67d573465fd57dae83f27300729dcddda74 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 7 Sep 2026 14:28:57 +0200 Subject: [PATCH 26/89] Add immutable remote proxy sources --- doc/getting_started/installation.rst | 5 +- doc/guides/remote_arrays.md | 13 + doc/reference/classes.rst | 2 + doc/reference/msgpack_serialization.rst | 4 +- doc/reference/remoteproxy.rst | 30 +- examples/remote/s3-access.py | 66 +---- plans/remote-proxy-v8.md | 371 ++++++++++++++++++++++++ pyproject.toml | 2 +- src/blosc2/__init__.py | 2 + src/blosc2/proxy.py | 12 +- src/blosc2/ref.py | 16 +- src/blosc2/remote_proxy.py | 127 +++++++- src/blosc2/schunk.py | 60 +++- src/blosc2/zarr_source.py | 193 ++++++++++++ tests/ndarray/test_c2array_blocks.py | 6 +- tests/test_remote_proxy.py | 46 ++- tests/test_zarr_source.py | 227 +++++++++++++++ 17 files changed, 1080 insertions(+), 102 deletions(-) create mode 100644 plans/remote-proxy-v8.md create mode 100644 src/blosc2/zarr_source.py create mode 100644 tests/test_zarr_source.py diff --git a/doc/getting_started/installation.rst b/doc/getting_started/installation.rst index 9e6dd5dd4..32d68311a 100644 --- a/doc/getting_started/installation.rst +++ b/doc/getting_started/installation.rst @@ -41,8 +41,8 @@ grouped into *extras* that you opt into with the ``blosc2[extra]`` syntax: - The ``parquet-to-blosc2`` converter (``pyarrow``); see :doc:`../guides/parquet_to_blosc2`. * - ``zarr`` - - The ``b2nd-to-zarr`` (or ``blosc2-to-zarr``) converter (``zarr``); converts - Blosc2 NDArrays into Zarr arrays. + - Lazy Zarr sources and the ``b2nd-to-zarr`` (or ``blosc2-to-zarr``) + converter (``zarr``). * - ``fsspec`` - Reading and writing single-file containers through any `fsspec `_ URL. The HTTP(S) driver is @@ -60,6 +60,7 @@ argument in shells like ``zsh`` that treat brackets specially): pip install "blosc2[parquet]" # the Parquet converter pip install "blosc2[fsspec]" # fsspec URLs, including HTTP(S) pip install "blosc2[fsspec]" s3fs # fsspec URLs, plus the S3 driver + pip install "blosc2[zarr,fsspec]" s3fs # remote Zarr arrays on S3 pip install "blosc2[tui,parquet]" # several at once With the ``fsspec`` extra, :func:`blosc2.open` accepts any fsspec URL, chained diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index f52dfbac5..e118f3ee9 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -11,6 +11,7 @@ The argument passed to {func}`blosc2.open` selects the route: | Argument | Route | What it names | |---|---|---| | A URL string such as `s3://...` or `https://...` | fsspec | A byte-addressable, standalone `.b2nd` file | +| A URL containing a `.zarr` path component | Zarr | One immutable Zarr v2 or v3 array | | A {ref}`URLPath` | Caterva2 | One array-like dataset on a Caterva2 server | ```python @@ -32,6 +33,17 @@ a.shape, a.dtype # metadata is available immediately a[100:110, :50] # data is fetched now ``` +Remote Zarr needs `pip install "blosc2[zarr,fsspec]"` plus the protocol driver +(`s3fs` for S3). The URL names the array itself; nested array paths work, while +opening a group asks for an array path. For a suffix-free URL, pass +`source_format="zarr"`. Converted Blosc2 chunks are cached under an immutable +source contract, so publish changed data at a new URL or replace its cache. + +`RemoteProxy` assumes remote sources are immutable by default, avoiding a +metadata request before every read. For a replaceable `.b2nd` or Caterva2 +source, pass `assume_immutable=False` to refresh its identity and invalidate +stale cached chunks before each operation. + A `URLPath` always means Caterva2. If its `urlbase` is omitted, the server comes from {func}`blosc2.c2context` or `BLOSC_C2URLBASE`. Other transports can be added with a custom {ref}`ByteRangeNDSource`; see [Use your own transport](#use-your-own-transport). ### What each route supports @@ -41,6 +53,7 @@ When opened with `lazy=True`, both routes return a {ref}`RemoteProxy`, providing | Remote object | fsspec URL | Caterva2 `URLPath` | |---|---|---| | Standalone contiguous `.b2nd` | Yes | Yes | +| Zarr v2/v3 array | Yes, with `source_format="zarr"` | No | | HDF5 dataset | No | Yes | | NDArray leaf inside `.b2z` | No | Yes | | Lazy or computed array | No | Yes | diff --git a/doc/reference/classes.rst b/doc/reference/classes.rst index a09bd06f6..1d4c5a693 100644 --- a/doc/reference/classes.rst +++ b/doc/reference/classes.rst @@ -32,6 +32,7 @@ without chunk caching. Proxy ProxySource ProxyNDSource + ZarrNDSource SimpleProxy Traffic @@ -144,6 +145,7 @@ container APIs above. proxyndsource byterangendsource fsspecndsource + zarrndsource simpleproxy traffic embed_store diff --git a/doc/reference/msgpack_serialization.rst b/doc/reference/msgpack_serialization.rst index ee23905e9..c1b206f96 100644 --- a/doc/reference/msgpack_serialization.rst +++ b/doc/reference/msgpack_serialization.rst @@ -106,7 +106,7 @@ Only durable reference-style operands are supported: - persistent local Blosc2 operands reopenable from ``urlpath`` - remote ``C2Array`` operands -- ``RemoteProxy`` operands for fsspec or Caterva2 references +- ``RemoteProxy`` operands for fsspec, Zarr, or Caterva2 references - ``DictStore`` members reopenable from ``(.b2d|.b2z, key)`` Purely in-memory operands are intentionally rejected. This keeps msgpack @@ -133,7 +133,7 @@ Supported operands are the same durable reference-style operands used for - persistent local Blosc2 operands reopenable from ``urlpath`` - remote ``C2Array`` operands -- ``RemoteProxy`` operands for fsspec or Caterva2 references +- ``RemoteProxy`` operands for fsspec, Zarr, or Caterva2 references - ``DictStore`` members reopenable from ``(.b2d|.b2z, key)`` Plain Python ``LazyUDF`` callables are intentionally not serialized by diff --git a/doc/reference/remoteproxy.rst b/doc/reference/remoteproxy.rst index 3cbe94606..a0d0d1179 100644 --- a/doc/reference/remoteproxy.rst +++ b/doc/reference/remoteproxy.rst @@ -3,9 +3,10 @@ RemoteProxy =========== -``RemoteProxy`` is a persistable proxy for one remote B2ND array. It accepts an -fsspec URL or a Caterva2 :ref:`URLPath`. With disk caching enabled, its B2ND -carrier is both the portable descriptor and the bounded compressed-data cache. +``RemoteProxy`` is a persistable proxy for one remote B2ND or Zarr array. It +accepts an fsspec URL or a Caterva2 :ref:`URLPath`. With disk caching enabled, +its B2ND carrier is both the portable descriptor and the bounded compressed-data +cache. The default policy is :attr:`blosc2.CachePolicy.NONE`: each operation reads the remote data it needs and no fetched data is retained afterwards. Saving such an @@ -31,10 +32,25 @@ URL: ) ) -References are floating: before each data operation, ``RemoteProxy`` checks the -source identity and verifies that shape, dtype, chunks, and blocks still match -the captured geometry. A replacement with different geometry is rejected; -cached disk data is invalidated when the source identity moves. +By default, ``RemoteProxy`` assumes its source is immutable and skips remote +identity checks before reads. For a replaceable single-file or Caterva2 source, +pass ``assume_immutable=False`` to refresh its identity and invalidate stale +cached data before each operation. + +Zarr URLs use a different contract: a ``.zarr`` path component selects +:ref:`ZarrNDSource`, or pass ``source_format="zarr"`` for a suffix-free path. +The URL names one array, including its path inside a hierarchy. Zarr sources are +assumed immutable for the lifetime of every cache; replacing data beneath the +same URL may mix stale and new chunks. Use a new URL or replace the cache when +publishing a new dataset. Mutable Zarr stores are not supported. + +.. code-block:: python + + remote = blosc2.open( + "s3://public-bucket/hierarchy.zarr/d0/a1", + lazy=True, + storage_options={"anon": True}, + ) Ephemeral in-memory caching is available through :attr:`blosc2.CachePolicy.MEMORY`. Fetched chunks are kept in RAM, bounded by a finite 256 MiB compressed-payload limit by default diff --git a/examples/remote/s3-access.py b/examples/remote/s3-access.py index 2fa26b370..3d57dc072 100755 --- a/examples/remote/s3-access.py +++ b/examples/remote/s3-access.py @@ -15,6 +15,7 @@ python s3-access.py s3://blosc2/cube-1k-1k-1k.b2nd python s3-access.py s3://blosc2/cube-1k-1k-1k.zarr python s3-access.py s3://blosc2/cube-1k-1k-1k-1shard.zarr + python s3-access.py s3://blosc2/hierarchy.zarr/d0/d1/a2 """ from __future__ import annotations @@ -24,9 +25,6 @@ import time from typing import Any -import fsspec -import zarr - import blosc2 DEFAULT_PROFILE = "blosc2" @@ -85,40 +83,6 @@ def __getattr__(self, name: str) -> Any: return getattr(self._f, name) -def _instrument_store(store: Any, traffic: Traffic) -> None: - """Instrument an async Zarr store to record network byte traffic.""" - orig_get = store.get - - async def tracked_get(*args: Any, **kwargs: Any) -> Any: - val = await orig_get(*args, **kwargs) - if val is not None: - traffic.charge(len(val)) - return val - - orig_get_partial = store.get_partial_values - - async def tracked_get_partial(*args: Any, **kwargs: Any) -> Any: - vals = await orig_get_partial(*args, **kwargs) - for buf in vals: - if buf is not None: - traffic.charge(len(buf)) - return vals - - store.get = tracked_get - store.get_partial_values = tracked_get_partial - - if hasattr(store, "get_sync"): - orig_get_sync = store.get_sync - - def tracked_get_sync(*args: Any, **kwargs: Any) -> Any: - val = orig_get_sync(*args, **kwargs) - if val is not None: - traffic.charge(len(val)) - return val - - store.get_sync = tracked_get_sync - - def open_remote_array( url: str, profile: str = DEFAULT_PROFILE, @@ -139,6 +103,8 @@ def open_remote_array( return "Blosc2 (Lazy RemoteProxy)", arr if clean_url.endswith((".zarr.zip", ".zip")): + import fsspec + import zarr from zarr.storage import ZipStore traffic = Traffic() @@ -149,29 +115,9 @@ def open_remote_array( arr.traffic = traffic return "Zarr (Zip)", arr - if clean_url.endswith(".zarr") or clean_url.endswith(".zarr/"): - from zarr.storage import FsspecStore - - traffic = Traffic() - store = FsspecStore.from_url(url, storage_options=storage_options, read_only=True) - _instrument_store(store, traffic) - arr = zarr.open(store=store, mode="r") - arr.traffic = traffic - return "Zarr", arr - - # Fallback / heuristic: try zarr first, then blosc2 - try: - from zarr.storage import FsspecStore - - traffic = Traffic() - store = FsspecStore.from_url(url, storage_options=storage_options, read_only=True) - _instrument_store(store, traffic) - arr = zarr.open(store=store, mode="r") - arr.traffic = traffic - return "Zarr", arr - except Exception: - arr = blosc2.open(url, lazy=True, storage_options=storage_options) - return "Blosc2 (Lazy RemoteProxy)", arr + arr = blosc2.open(url, lazy=True, storage_options=storage_options) + label = "Zarr" if arr.source["kind"] == "zarr" else "Blosc2" + return f"{label} (Lazy RemoteProxy)", arr def main() -> int: diff --git a/plans/remote-proxy-v8.md b/plans/remote-proxy-v8.md new file mode 100644 index 000000000..8956327fc --- /dev/null +++ b/plans/remote-proxy-v8.md @@ -0,0 +1,371 @@ +# Remote proxy v8: remote arrays with immutable-by-default Blosc2 caching + +## Objective and status + +Status: implemented in this branch. + +Add `ZarrNDSource`, a `ProxyNDSource` adapter that reads remote Zarr arrays and +returns Blosc2 compressed chunks. Reuse the existing `Proxy` and `RemoteProxy` +cache implementations, including bounded memory caches, portable B2ND carriers, +and the sparse runtime caches introduced for Caterva2 in v7. + +RemoteProxy sources are assumed immutable by default. Callers following a mutable +single-file source can opt into identity checks with `assume_immutable=False`. +Zarr remains immutable-only in this version because changes to one object do not +provide an identity for the complete array. Zarr remains an optional dependency, +like fsspec. The adapter belongs in Python-Blosc2 and must be usable without a +Caterva2 server. + +## Fixed decisions + +- Read Zarr through Zarr-Python's public array/store APIs. Do not implement a + second metadata parser, chunk-key encoder, codec pipeline, or shard reader. +- Cache converted Blosc2 chunks in existing B2ND containers. Do not use Zarr's + `CacheStore` or maintain a second cache of Zarr objects. +- Fetch and convert whole logical Zarr chunks. A shard is a storage object, not + the cache chunk shape. Let Zarr resolve chunk reads within shards. +- Support Zarr format 2 and 3 through a supported Zarr-Python 3 release. This + does not require supporting the older Zarr-Python 2 API. +- Start with fixed-size numeric and boolean arrays representable by B2ND. + Reject unsupported dtypes explicitly before creating a cache. +- `RemoteProxy(..., assume_immutable=True)` is the default for every source and + skips metadata polling before reads. `False` retains identity refresh and cache + invalidation for mutable single-file and Caterva2 sources. +- A Zarr source identity names immutable contents for the lifetime of its caches. + No per-chunk ETag checks, TTL, or mutable Zarr-store support. +- Apply the same immutable default and persisted flag to Blosc2 and Caterva2 + source descriptors. +- Do not change C/Cython code unless implementation exposes a demonstrated + blocker that cannot be handled by existing Python APIs. + +## Existing code to reuse + +`src/blosc2/proxy_source.py` defines `ProxyNDSource`: shape, chunks, blocks, +dtype, compression parameters, and `get_chunk(nchunk)`. Its optional block-range +interface is unnecessary for this adapter. `Traffic` already provides +thread-safe counters for received bytes and data-bearing requests. + +`src/blosc2/proxy.py` creates a B2ND cache from that geometry, fetches missing +chunks, inserts compressed chunks, and tracks retention and eviction. These +mechanisms should remain the owners of cache state. + +`src/blosc2/remote_proxy.py` handles source descriptors, geometry validation, +source stamps, carriers, and sparse attachment. Its URL-string branch currently +always creates `FsspecNDSource`. Source identity, payload reconstruction, and +authorized attachment currently distinguish only fsspec and Caterva2 sources. + +`src/blosc2/schunk.py` owns the public `blosc2.open()` remote dispatch and cache +options. Changes must reach this shared path, not just the S3 example. + +`pyproject.toml` already defines `zarr = ["zarr"]` and a separate `fsspec` extra. +Reuse these extras. The existing example already opens Zarr stores and measures +their reads, but its Zarr branch returns an uncached Zarr array. + +## Proposed public opening API + +Add `source_format=None` to `RemoteProxy` and accept it through +`blosc2.open(..., lazy=True)`. Initially allow `None`, `"blosc2"`, and `"zarr"`. +Add `assume_immutable=True` to both entry points and persist it in every source +descriptor. Users following replaceable sources must set it to `False`. + +```python +arr = blosc2.open( + "s3://blosc2/cube-1k-1k-1k.zarr", + lazy=True, + source_format="zarr", + cache_policy=blosc2.CachePolicy.MEMORY, + storage_options={"anon": True}, +) +values = arr[:10, 0, :5] + +arr = blosc2.RemoteProxy( + "s3://bucket/hierarchy.zarr/d0/d1/a2", + source_format="zarr", + cache_policy=blosc2.CachePolicy.DISK, + cache_path="array-cache.b2nd", +) +``` + +An explicit format supports URLs without a `.zarr` suffix. With no explicit +format, recognize a `.zarr` path component, including a trailing slash and nested +array paths; retain the existing Blosc2 default for other remote URLs. Inspect +the parsed URL path, not query-string text. Explicit selection wins over the +heuristic. Do not catch arbitrary opening errors and retry another format. + +The URL names the array itself, including its path within any hierarchy. Opening +a group raises an actionable error asking for an array path. Do not recursively +discover or select an array. ZIP-backed Zarr is outside this first version and +must not be misclassified as ordinary directory/object-store Zarr. + +Keep existing cache-policy defaults: direct `RemoteProxy` and `blosc2.open` +already have their own defaults. Adding Zarr must not silently change them. +Reject incompatible format options on Caterva2 `URLPath` inputs and reject Zarr +opening without `lazy=True` in this release. Existing local Blosc2 opening stays +unchanged; a direct adapter can accept a local store for testing. + +## Adapter implementation + +Create `src/blosc2/zarr_source.py` and export `ZarrNDSource` from `blosc2`. +The module may import NumPy and existing Blosc2 helpers at module scope, but must +not import Zarr or fsspec until construction needs them. + +### Construction and metadata + +1. Import optional dependencies with actionable errors. +2. Open a read-only store and use `zarr.open_array(..., mode="r")`. +3. Read metadata only: shape, logical chunks, dtype, and the metadata required + to validate compatibility. Construction must not fetch array payloads. +4. Normalize geometry to Python tuples and dtype to a NumPy dtype. Validate + dimensions, positive chunk extents, dtype support, and Blosc2 size limits + before allocating conversion buffers or cache containers. +5. Set `serves_blocks = False`. Expose the existing concurrency and traffic + conventions so `Proxy` can fetch chunks through its current executor. + +Do not treat Zarr's `arr.blocks` as a block shape: it is an indexing interface. +Choose cache blocks with the existing Blosc2 partition helper, supplying the +chosen chunk shape. Use normal Blosc2 compression defaults initially. These +parameters describe the converted cache, not the original Zarr compressor. + +Cache partitioning must be reproducible across reopening. Persisted carrier +chunks and blocks are authoritative when restoring a cache; pass that geometry +through the adapter reconstruction path and validate it against the source. +Do not accidentally reject old caches because an automatic block-size heuristic +or a default compression setting changed in a later library release. + +### `get_chunk(nchunk)` + +1. Validate the chunk number and map it to C-order logical chunk coordinates. + Reuse an existing coordinate helper if one fits; otherwise use NumPy's + unraveling with the grid obtained by ceiling-dividing shape by chunks. +2. Compute the clipped array slice for that chunk and read it through Zarr. +3. Normalize the decoded result to the advertised dtype and contiguous order. + Preserve numeric values, including endian conversion where needed; never + reinterpret foreign-endian bytes as native values. +4. For edge chunks, place valid values in an initialized full-chunk buffer. + Padding lies outside the logical array and must never contain uninitialized + memory. Missing Zarr chunks inside the logical array are filled by Zarr using + the declared fill value, not by substituting zero in the adapter. +5. Encode a single-chunk B2ND temporary with the cache's exact chunks, blocks, + dtype, and compression parameters, then return `get_chunk(0)` from it. + This delegates multidimensional block layout and padding to Blosc2. + +Keep each conversion buffer and temporary container local to the call. Reuse +`Proxy`'s fetch scheduling; do not add another thread pool or a shared mutable +scratch array. Investigate Zarr's internal concurrency when measuring peak +memory: the outer limit bounds adapter chunk calls, not necessarily every +internal store request. Add an asynchronous adapter method only if an existing +public async path requires it, and use Zarr's supported scheduling mechanism. + +Document the initial memory ceiling: concurrently decoded whole chunks plus +conversion/compression buffers. `max_cache_bytes` limits retained compressed +payload, not transient decoded memory or process RSS. + +### Supported representations + +Cover bool, integer, floating-point, and complex dtypes supported by B2ND, both +Zarr formats, alternative codec pipelines, nonzero fill values, edge chunks, +and Zarr v3 sharding. Let Zarr decode storage order and transpose codecs. + +Reject object, variable-length, string, structured, and other unsupported dtype +representations with a clear `TypeError` in this first version. Test scalar and +zero-length arrays against existing B2ND geometry constraints; support them if +the normal path works, otherwise reject explicitly at construction and document +the limitation. Never defer such failures until after a partially written cache. + +## Immutable identity and persistence + +Use a distinct source descriptor, keeping the existing outer RemoteProxy payload +version if its schema remains compatible: + +```json +{ + "kind": "zarr", + "version": 1, + "urlpath": "s3://bucket/hierarchy.zarr/d0/d1/a2", + "assume_immutable": true +} +``` + +Version 1 records `assume_immutable`, which defaults to `true`. Zarr requires it +to remain true in this version. Persisting the option preserves read behavior +after reopening and makes the assumption visible to clients and Caterva2. +Unknown fields, versions, and unsupported values fail closed. + +Generate a stable non-null source stamp from a canonical, credential-free +descriptor plus the adapter encoding version and normalized cache geometry. +Use a deterministic digest, not Python's randomized `hash()`. This stamp denotes +an identity under the immutable contract; it is not a remotely verified content +hash. Include interpretation metadata where practical to detect changed metadata +on reopening, without claiming to detect payload-only mutations. + +Do not implement a remote refresh method that fetches metadata on every read. +Ensure `_prepare_read()` keeps the stable source and cache instead of treating +it as an unstamped source. Existing geometry validation still runs on reopening. +Document that replacing data under the same identity violates the contract and +may serve stale data or mix old cached chunks with newly fetched chunks. A new +dataset should have a new immutable URL; manual cache replacement is the escape +hatch for intentionally reused URLs. + +Update every relevant branch together: `_open_source`, `_source_identity`, +`urlpath`, payload validation/reconstruction, `with_sparse_cache`, carrier +exports, and source-spec comparisons. Preserve live storage options on any +reconstruction within a process, but never persist them. A fresh process must +resolve credentials from its own environment, as with existing sources. + +Check direct `Proxy(ZarrNDSource(...))` metadata too: it must not be labeled as +a local Blosc2 source merely because it exposes `urlpath`. If direct persistent +Proxy caches are supported, add their source reconstruction alongside the +RemoteProxy path; do not produce a cache that saves successfully but reopens +through the wrong backend. + +## Optional dependencies + +- Reuse `blosc2[zarr]`; do not add Zarr to mandatory runtime dependencies. +- Select the minimum Zarr-Python 3 version actually required by the public APIs + used and verify it. Do not require local version 3.3 merely because it is + installed. Adjust the existing extra's version floor only with that evidence. +- Keep fsspec independently optional. Document remote installs as + `pip install "blosc2[zarr,fsspec]"`, adding `s3fs` for S3 as today. +- Import Zarr inside construction or a file-local helper, following the + actionable-error pattern of `core._import_fsspec`. A missing Zarr error should + name `blosc2[zarr]`; missing protocol backends retain their useful errors. +- Importing `blosc2`, using local B2ND arrays, and opening remote B2ND sources + must work without Zarr installed. Local Zarr adapter use should not require + fsspec when the chosen store does not need it. +- Move the S3 example's unconditional Zarr import into the branch that needs + it. Preserve any existing user edits when implementing the example update. +- Optional dependency tests use `pytest.importorskip`; also test the actual + missing-dependency error path separately so skipping cannot hide it. + +## Traffic, example, and performance expectations + +Count encoded bytes received beneath the Zarr decoder. Reuse the example's +instrumentation approach as a reference, but implement the adapter's tracking +through supported store/transport extension points. Cover ordinary reads and +partial/shard reads; avoid double-counting delegated batched operations. +The counter measures received payload bytes, not headers or decoded array size. +Place it below any local cache so a Blosc2 hit charges no remote payload. + +Update `examples/remote/s3-access.py` so directory/object-store Zarr arrays open +through `blosc2.open(..., lazy=True)` and print RemoteProxy cache details. +Display the cache's actual block shape. Keep ZIP behavior explicitly separate. +Metadata, first slice, second slice, and transferred-byte reporting should remain +comparable to the current example. + +The first miss pays Zarr decoding plus Blosc2 encoding. A hit reads the Blosc2 +cache without repeating either remote reads or Zarr decoding. No timing speedup +is promised for a cold read. Warm-cache tests assert zero payload reads, not +a machine-dependent millisecond threshold. Measure cold/warm latency, encoded +network bytes, retained Blosc2 bytes, and transient memory separately. + +## Caterva2 integration boundary + +The Python-Blosc2 implementation comes first. Actual Caterva2 changes belong in +the Caterva2 repository as a subsequent integration phase; v7 remains the cache +lifecycle and quota design. + +Prepare the adapter to accept a server-supplied, already-authorized store or +filesystem internally. Extend authorized sparse attachment to recognize the +concrete Zarr source and validate the exact descriptor against it. Do not accept +arbitrary source objects as implicitly authorized. + +An authorized attachment must retain the supplied transport for metadata, +chunks, shard indexes, and all subsequent reads. It must never reopen the URL +using an unrestricted default filesystem. Keep authorization before cache access, +including hits, as required by the existing Caterva2 contract. + +Caterva2 must authorize an array prefix and validate all derived object paths; +authorizing one metadata URL alone is insufficient for a multi-object source. +Preserve HTTPS allowlists, DNS pinning, redirect refusal, credential isolation, +geometry limits, and embedded-reference restrictions. Confirm the Zarr pipeline +cannot escape the authorized transport/prefix through an unsupported store or +codec; restrict server support explicitly where necessary. + +Once those boundaries are validated, reuse the existing sparse generation, +locking, dirty recovery, eviction, quota, and warm-export paths. Downstream +Caterva2 clients receive ordinary Blosc2 chunks and need no Zarr dependency. +The Caterva2 server reading the external Zarr source needs the optional extra. +An immutable-source stamp does not itself grant authorization. + +## Implementation sequence and checks + +### 1. Adapter and dependency isolation + +Implement the adapter, deferred import, metadata validation, and chunk conversion. +Add local temporary Zarr fixtures in `tests/test_zarr_source.py` and exercise the +adapter through `Proxy`, not only by calling `get_chunk()` directly. + +Verify format 2/3, one and multiple dimensions, exact and edge chunks, missing +chunks/nonzero fill, endian handling, representative supported dtypes, unsupported +dtypes, and sharded format 3. Check scalar/empty geometry explicitly. Include a +codec other than Blosc to establish that conversion does not assume Blosc bytes. + +### 2. RemoteProxy opening and immutable caching + +Implement dispatch, immutable descriptors/stamps, and policy integration. +Use counted local or memory-backed stores for deterministic tests without public +network access. Ensure metadata opening does not fetch chunk data. + +Test explicit/automatic selection, nested arrays, suffix-free explicit URLs, +trailing slashes, group errors, and useful authentication/opening failures. +Test NONE, MEMORY, bounded DISK, and unbounded DISK using existing conventions. +Repeated and overlapping reads must fetch only absent chunks; eviction must +cause a later read to fetch again. A repeated hit must not poll metadata either. +An oversized chunk must obey the existing post-operation retention bound. + +### 3. Persistence and server attachment + +Test cold and warm carrier reopening, MEMORY exports remaining cold, geometry +mismatch rejection, changed encoding identity, source-spec mismatch, and credential +exclusion. Exercise `save()`, `to_cframe()`, materialization, and a simple lazy +expression round trip. Check direct persistent Proxy reconstruction if exposed. + +Exercise sparse attachment and warm seeding with an authorized fake transport. +Patch unrestricted source opening to raise and verify metadata reads, misses, +and hits still follow the authorized path. Preserve existing sparse recovery +and eviction behavior rather than duplicating its tests for every Zarr dtype. + +### 4. Documentation and example + +Document the adapter API, supported arrays, optional installation, cache +representation, immutable contract, and memory/performance limits. Add the new +source to the existing proxy API documentation and example descriptions. +Run the S3 example manually against the supplied unsharded and sharded datasets +when credentials/network access are available; keep public S3 tests marked +`network` and outside the default suite. + +### 5. Validation and handoff + +Use the `blosc2` conda environment for all Python, installation, and tests. +Run focused adapter/RemoteProxy/Proxy/fsspec tests first, then the default suite +and repository lint checks after integration. Use the established pytest fixtures +and parametrization rather than adding a new test framework. + +Validate optional imports in a subprocess with Zarr imports blocked, and with +fsspec blocked for unrelated functionality. Verify the minimum supported Zarr +version in a suitable test environment before setting its requirement floor. +Record any dependency-version or network checks that could not run. + +## Completion criteria + +- A remote Zarr array opens as a RemoteProxy and produces correct slice values. +- Retained payloads are usable Blosc2 chunks with correct B2ND block layout. +- A warm hit performs no remote payload or metadata reads under the immutable + contract; eviction and cache limits retain their existing semantics. +- DISK carriers and sparse caches reopen safely with the correct source and + geometry; credentials are absent from persisted metadata. +- Zarr remains optional and missing dependencies produce actionable errors. +- An authorized server source cannot fall back to unrestricted transport. +- Existing Blosc2/Caterva2 source tests continue to pass. + +## Deferred work + +Mutable-store validation, per-object versions, refresh policies, and TTL are +future changes requiring a new explicit consistency contract. Direct reuse of +compatible compressed Zarr chunks, block-range conversion, shard batching, +custom codec optimizations, variable-length dtypes, ZIP sources, and hierarchy +browsing are deferred until a concrete workload needs them. + +The ponytail choice is one source adapter using existing Zarr decoding and +Blosc2 caching. No cache engine, codec framework, or generalized plugin registry +is needed for this feature. diff --git a/pyproject.toml b/pyproject.toml index d3efcb6ba..8d9efbb31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ documentation = "https://www.blosc.org/python-blosc2/python-blosc2.html" [project.optional-dependencies] parquet = ["pyarrow"] -zarr = ["zarr"] +zarr = ["zarr>=3.0.9"] # The b2view terminal viewer (the `b2view` script) is opt-in: most users want # blosc2 only as a compression library, and the TUI stack has no use under # wasm32 (no TTY). Install with `pip install "blosc2[tui]"`. This also pulls diff --git a/src/blosc2/__init__.py b/src/blosc2/__init__.py index 5f4023811..35d351fa2 100644 --- a/src/blosc2/__init__.py +++ b/src/blosc2/__init__.py @@ -603,6 +603,7 @@ def _raise(exc): FsspecNDSource, Traffic, ) +from .zarr_source import ZarrNDSource from .indexing import Index from .schunk import SChunk, load, open @@ -901,6 +902,7 @@ def _raise(exc): "ByteRangeNDSource", "FsspecNDSource", "Traffic", + "ZarrNDSource", "Proxy", "ProxyNDField", "ProxyNDSource", diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index e932015a4..518b780fe 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -64,6 +64,12 @@ _JIT_EXECUTION_TUNING_KWARGS = frozenset({"jit", "jit_backend", "fp_accuracy"}) +def _source_urlpath(src): + if src.urlpath is None: + raise ValueError("persistent Proxy caches require a path-backed source") + return src.urlpath + + def _validate_max_cache_bytes(value: int | None) -> int | None: if value is None: return None @@ -203,9 +209,9 @@ def __init__( "caterva2_env": caterva2_env, } container = getattr(self.src, "schunk", self.src) - if isinstance(self.src, blosc2.FsspecNDSource): - meta_val["source_kind"] = "fsspec" - meta_val["urlpath"] = self.src.urlpath + if isinstance(self.src, (blosc2.FsspecNDSource, blosc2.ZarrNDSource)): + meta_val["source_kind"] = "zarr" if isinstance(self.src, blosc2.ZarrNDSource) else "fsspec" + meta_val["urlpath"] = _source_urlpath(self.src) # Keep the legacy field populated so older readers still # reopen this cache, albeit through their eager URL path. meta_val["local_abspath"] = self.src.urlpath diff --git a/src/blosc2/ref.py b/src/blosc2/ref.py index 292f611bc..31daa9816 100644 --- a/src/blosc2/ref.py +++ b/src/blosc2/ref.py @@ -20,7 +20,7 @@ class Ref: - a persistent local Blosc2 object reopenable from ``urlpath`` - a member inside a :class:`blosc2.DictStore` - a remote :class:`blosc2.C2Array` - - an fsspec URL used by a :class:`blosc2.RemoteProxy` + - an fsspec or Zarr URL used by a :class:`blosc2.RemoteProxy` Instances can be created directly, from dictionaries via :meth:`from_dict`, or from supported objects via :meth:`from_object`. Use :meth:`open` to @@ -34,12 +34,12 @@ class Ref: urlbase: str | None = None def __post_init__(self) -> None: - if self.kind in {"urlpath", "fsspec"}: + if self.kind in {"urlpath", "fsspec", "zarr"}: if not isinstance(self.urlpath, str): raise TypeError(f"Ref(kind={self.kind!r}) requires a string 'urlpath'") if self.key is not None or self.path is not None or self.urlbase is not None: raise ValueError(f"Ref(kind={self.kind!r}) only supports the 'urlpath' field") - if self.kind == "fsspec": + if self.kind in {"fsspec", "zarr"}: # Keep structured references subject to the same credential and # portability checks as an explicit RemoteProxy. The import is # local because Ref is imported before the public proxy module. @@ -81,6 +81,10 @@ def c2array_ref(cls, path: str, urlbase: str | None = None) -> Ref: def fsspec_ref(cls, urlpath: str) -> Ref: return cls(kind="fsspec", urlpath=urlpath) + @classmethod + def zarr_ref(cls, urlpath: str) -> Ref: + return cls(kind="zarr", urlpath=urlpath) + @classmethod def from_dict(cls, payload: dict[str, Any]) -> Ref: if not isinstance(payload, dict): @@ -106,6 +110,8 @@ def from_object(cls, obj: Any) -> Ref: source = obj.source if source["kind"] == "caterva2": return cls.c2array_ref(source["path"], source["urlbase"]) + if source["kind"] == "zarr": + return cls.zarr_ref(source["urlpath"]) return cls.fsspec_ref(source["urlpath"]) if isinstance(obj, blosc2.Proxy): obj = obj._cache @@ -121,7 +127,7 @@ def from_object(cls, obj: Any) -> Ref: def to_dict(self) -> dict[str, Any]: payload = {"kind": self.kind, "version": 1} - if self.kind in {"urlpath", "fsspec"}: + if self.kind in {"urlpath", "fsspec", "zarr"}: payload["urlpath"] = self.urlpath elif self.kind == "dictstore_key": payload["urlpath"] = self.urlpath @@ -144,4 +150,6 @@ def open(self): return blosc2.C2Array(self.path, urlbase=self.urlbase) if self.kind == "fsspec": return blosc2.RemoteProxy(self.urlpath) + if self.kind == "zarr": + return blosc2.RemoteProxy(self.urlpath, source_format="zarr") raise ValueError(f"Unsupported Ref kind: {self.kind!r}") diff --git a/src/blosc2/remote_proxy.py b/src/blosc2/remote_proxy.py index b3fb2acd7..b19261abf 100644 --- a/src/blosc2/remote_proxy.py +++ b/src/blosc2/remote_proxy.py @@ -46,6 +46,28 @@ def __repr__(self) -> str: ) +def _normalize_source_format(urlpath, source_format): + if source_format not in {None, "blosc2", "zarr"}: + raise ValueError("source_format must be None, 'blosc2', or 'zarr'") + if source_format is not None: + return source_format + if isinstance(urlpath, blosc2.ZarrNDSource): + return "zarr" + if ( + isinstance(urlpath, str) + and "::" not in urlpath + and any(part.endswith(".zarr") for part in urlsplit(urlpath).path.split("/")) + ): + return "zarr" + return "blosc2" + + +def _validate_assume_immutable(value, name="assume_immutable"): + if not isinstance(value, bool): + raise TypeError(f"{name} must be a bool") + return value + + def _serialized_operation(method): @wraps(method) def locked(self, *args, **kwargs): @@ -163,6 +185,12 @@ class RemoteProxy(blosc2.Operand): storage_options: dict, optional Parameters passed to the underlying ``fsspec`` filesystem when opening an fsspec URL. + source_format: {None, "blosc2", "zarr"}, optional + Format of a URL source. A ``.zarr`` path component selects Zarr when + omitted. + assume_immutable: bool, optional + Skip remote identity checks before reads. Defaults to ``True``. Set to + ``False`` when the object at the URL may be replaced. """ def __init__( @@ -175,12 +203,17 @@ def __init__( max_cache_bytes=_POLICY_DEFAULT, max_concurrency: int | None = None, storage_options: dict | None = None, + source_format: str | None = None, + assume_immutable: bool = True, _carrier=None, _runtime_cache_path=None, _source_descriptor=None, + _source_blocks=None, + _source_cparams=None, ): if not isinstance(cache_policy, blosc2.CachePolicy): raise TypeError("cache_policy must be a blosc2.CachePolicy instance") + assume_immutable = _validate_assume_immutable(assume_immutable) if cache_dir is not None and cache_path is not None: raise ValueError("cache_dir and cache_path are mutually exclusive") if cache_policy is not blosc2.CachePolicy.DISK and (cache_dir is not None or cache_path is not None): @@ -197,13 +230,24 @@ def __init__( self._cache_policy = cache_policy self._cache_limit = _normalize_limit(cache_policy, max_cache_bytes) self._max_concurrency = _validate_max_concurrency(max_concurrency) + if isinstance(urlpath, (blosc2.URLPath, blosc2.C2Array)) and source_format is not None: + raise ValueError("source_format is not supported for Caterva2 inputs") + self._source_format = _normalize_source_format(urlpath, source_format) self._authorized_source = _source_descriptor is not None if self._authorized_source: if storage_options is not None: raise ValueError("storage_options cannot be used with an authorized source") - if not isinstance(urlpath, blosc2.FsspecNDSource): - raise TypeError("source_descriptor requires an authorized FsspecNDSource") - expected = {"kind": "fsspec", "version": 1, "urlpath": urlpath.urlpath} + if not isinstance(urlpath, (blosc2.FsspecNDSource, blosc2.ZarrNDSource)): + raise TypeError("source_descriptor requires an authorized FsspecNDSource or ZarrNDSource") + assume_immutable = _validate_assume_immutable( + _source_descriptor.get("assume_immutable"), "source_descriptor assume_immutable" + ) + expected = { + "kind": "zarr" if isinstance(urlpath, blosc2.ZarrNDSource) else "fsspec", + "version": 1, + "urlpath": urlpath.urlpath, + "assume_immutable": assume_immutable, + } if _source_descriptor != expected: raise ValueError("source_descriptor does not match the supplied source") _validate_persistable_url(urlpath.urlpath) @@ -214,7 +258,13 @@ def __init__( self._max_concurrency, persistable=cache_policy is not blosc2.CachePolicy.MEMORY, storage_options=storage_options, + source_format=self._source_format, + assume_immutable=assume_immutable, + blocks=_source_blocks, + cparams=_source_cparams, ) + self._assume_immutable = assume_immutable + self._storage_options = storage_options self._runtime_urlpath = self._runtime_source(urlpath) self._expected_geometry = self._geometry(self.src) self._expected_cparams = self.src.cparams @@ -377,6 +427,7 @@ def with_sparse_cache( source_descriptor=None, max_cache_bytes=_POLICY_DEFAULT, max_concurrency: int | None = None, + assume_immutable: bool = True, ): """Attach an authorized remote source to a private sparse disk cache. @@ -402,6 +453,7 @@ def with_sparse_cache( cache_policy=blosc2.CachePolicy.DISK, max_cache_bytes=max_cache_bytes, max_concurrency=max_concurrency, + assume_immutable=assume_immutable, _carrier=carrier, _runtime_cache_path=runtime_cache_path, _source_descriptor=source_descriptor, @@ -520,8 +572,14 @@ def _open_source( traffic=None, persistable=True, storage_options: dict | None = None, + source_format: str | None = None, + assume_immutable: bool = True, + blocks=None, + cparams=None, ): if isinstance(urlpath, blosc2.C2Array): + if source_format not in {None, "blosc2"}: + raise ValueError("source_format is not supported for Caterva2 inputs") if storage_options is not None: raise ValueError("storage_options is only supported for fsspec URLs") src = urlpath @@ -532,8 +590,11 @@ def _open_source( "version": 1, "path": src.path, "urlbase": src.urlbase, + "assume_immutable": assume_immutable, } elif isinstance(urlpath, blosc2.URLPath): + if source_format not in {None, "blosc2"}: + raise ValueError("source_format is not supported for Caterva2 URLPath inputs") if storage_options is not None: raise ValueError("storage_options is only supported for fsspec URLs") src = blosc2.C2Array( @@ -547,6 +608,7 @@ def _open_source( "version": 1, "path": src.path, "urlbase": src.urlbase, + "assume_immutable": assume_immutable, } elif isinstance(urlpath, str): if persistable: @@ -554,8 +616,27 @@ def _open_source( kwargs = {} if max_concurrency is None else {"max_concurrency": max_concurrency} if storage_options is not None: kwargs["storage_options"] = storage_options - src = blosc2.FsspecNDSource(urlpath, _traffic=traffic, **kwargs) - source = {"kind": "fsspec", "version": 1, "urlpath": urlpath} + source_format = _normalize_source_format(urlpath, source_format) + if source_format == "zarr": + if not assume_immutable: + raise NotImplementedError("mutable Zarr sources are not supported") + src = blosc2.ZarrNDSource( + urlpath, _traffic=traffic, blocks=blocks, cparams=cparams, **kwargs + ) + source = { + "kind": "zarr", + "version": 1, + "urlpath": urlpath, + "assume_immutable": assume_immutable, + } + else: + src = blosc2.FsspecNDSource(urlpath, _traffic=traffic, **kwargs) + source = { + "kind": "fsspec", + "version": 1, + "urlpath": urlpath, + "assume_immutable": assume_immutable, + } else: raise TypeError("RemoteProxy requires a URL string, URLPath, or C2Array") @@ -564,7 +645,7 @@ def _open_source( return src, source def _source_identity(self) -> str: - if self._source["kind"] == "fsspec": + if self._source["kind"] in {"fsspec", "zarr"}: return self._source["urlpath"] return f"caterva2:{blosc2.c2array._server_url(self.src.urlbase, self.src.path)}" @@ -586,7 +667,7 @@ def _validate_geometry(self, expected, *, src=None) -> None: def _prepare_read(self): """Refresh source identity and return the backend for one operation.""" - if self._authorized_source: + if self._authorized_source or self._assume_immutable: return self._proxy if self._proxy is not None else self.src with self._refresh_lock: previous_stamp = getattr(self.src, "stamp", None) @@ -610,6 +691,9 @@ def _prepare_read(self): self._max_concurrency, traffic=self.traffic, persistable=self.cache_policy is not blosc2.CachePolicy.MEMORY, + storage_options=self._storage_options, + source_format=self._source_format, + assume_immutable=self._assume_immutable, ) if current_stamp is None and not isinstance(fresh, blosc2.C2Array): # No stable validator means cached bytes cannot safely be @@ -692,6 +776,11 @@ def source(self) -> dict: self._payload() # Runtime-only URLs must not escape as portable descriptors. return dict(self._source) + @property + def assume_immutable(self) -> bool: + """Whether reads skip remote identity checks.""" + return self._assume_immutable + @property def schunk(self): """The underlying carrier's or cache's :class:`SChunk`, or None if unattached.""" @@ -713,7 +802,7 @@ def cache(self): @property def urlpath(self): """The remote fsspec URL or credential-free Caterva2 URLPath.""" - if self._source["kind"] == "fsspec": + if self._source["kind"] in {"fsspec", "zarr"}: return self._source["urlpath"] return blosc2.URLPath(self._source["path"], urlbase=self._source["urlbase"]) @@ -931,25 +1020,41 @@ def _from_payload(cls, payload, carrier): raise ValueError("unsupported RemoteProxy source descriptor") source_kind = source.get("kind") if source_kind == "fsspec": - if set(source) != {"kind", "version", "urlpath"}: + if set(source) != {"kind", "version", "urlpath", "assume_immutable"}: raise ValueError("fsspec RemoteProxy source descriptors contain unsupported fields") urlpath = source.get("urlpath") if not isinstance(urlpath, str): raise TypeError("fsspec RemoteProxy sources require a string 'urlpath'") elif source_kind == "caterva2": - if set(source) != {"kind", "version", "path", "urlbase"}: + if set(source) != {"kind", "version", "path", "urlbase", "assume_immutable"}: raise ValueError("Caterva2 RemoteProxy source descriptors contain unsupported fields") path = source.get("path") urlbase = source.get("urlbase") if not isinstance(path, str) or (urlbase is not None and not isinstance(urlbase, str)): raise TypeError("Caterva2 RemoteProxy sources require string 'path' and 'urlbase' fields") urlpath = blosc2.URLPath(path, urlbase=urlbase) + elif source_kind == "zarr": + if set(source) != {"kind", "version", "urlpath", "assume_immutable"}: + raise ValueError("Zarr RemoteProxy source descriptors contain unsupported fields") + urlpath = source.get("urlpath") + if not isinstance(urlpath, str): + raise TypeError("Zarr RemoteProxy sources require a string 'urlpath'") else: raise ValueError(f"unsupported RemoteProxy source kind: {source_kind!r}") + _validate_assume_immutable(source.get("assume_immutable"), "source assume_immutable") expected = (carrier.shape, carrier.dtype, carrier.chunks, carrier.blocks) kwargs = {} if policy is blosc2.CachePolicy.NONE else {"max_cache_bytes": limit} carrier_arg = carrier if policy is blosc2.CachePolicy.DISK else None - obj = cls(urlpath, cache_policy=policy, _carrier=carrier_arg, **kwargs) + obj = cls( + urlpath, + cache_policy=policy, + source_format="zarr" if source_kind == "zarr" else None, + assume_immutable=source["assume_immutable"], + _carrier=carrier_arg, + _source_blocks=carrier.blocks if source_kind == "zarr" else None, + _source_cparams=carrier.cparams if source_kind == "zarr" else None, + **kwargs, + ) obj._validate_geometry(expected) return obj diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 6ec86930e..5669da195 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -1895,6 +1895,11 @@ def process_opened_object(res): if source_kind == "fsspec": src = blosc2.FsspecNDSource(proxy_src["urlpath"]) return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) + if source_kind == "zarr": + src = blosc2.ZarrNDSource( + proxy_src["urlpath"], blocks=proxy_cache.blocks, cparams=proxy_cache.cparams + ) + return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) if source_kind == "caterva2": src = blosc2.C2Array(proxy_src["urlpath"][0], proxy_src["urlpath"][1], proxy_src["urlpath"][2]) return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) @@ -2010,8 +2015,23 @@ def _remote_cache_options(kwargs: dict) -> tuple[str | pathlib.Path | None, str return (cache_dir if cache_dir is not None else cache_storage), cache_path +def _validate_fsspec_source_format(source_format, lazy): + if source_format not in {None, "blosc2", "zarr"}: + raise ValueError("source_format must be None, 'blosc2', or 'zarr'") + if source_format == "zarr" and not lazy: + raise NotImplementedError("Zarr sources require lazy=True") + + def _remote_proxy_options( - kwargs, cache_dir, cache_path, max_concurrency, *, lazy=False, storage_options=None + kwargs, + cache_dir, + cache_path, + max_concurrency, + *, + lazy=False, + storage_options=None, + source_format=None, + assume_immutable=True, ): """Return explicit RemoteProxy options, or None for the legacy lazy Proxy path.""" policy_present = "cache_policy" in kwargs @@ -2032,11 +2052,14 @@ def _remote_proxy_options( "cache_dir": cache_dir, "cache_path": cache_path, "max_concurrency": max_concurrency, + "assume_immutable": assume_immutable, } if limit_present: options["max_cache_bytes"] = limit if storage_options is not None: options["storage_options"] = storage_options + if source_format is not None: + options["source_format"] = source_format return options @@ -2116,13 +2139,24 @@ def _open_c2_urlpath(urlpath: blosc2.URLPath, mode: str, offset: int, kwargs: di cache_dir, cache_path = _remote_cache_options(kwargs) max_concurrency = kwargs.pop("max_concurrency", None) + immutable_present = "assume_immutable" in kwargs + assume_immutable = kwargs.pop("assume_immutable", True) + source_format = kwargs.pop("source_format", None) + if source_format not in {None, "blosc2", "zarr"}: + raise ValueError("source_format must be None, 'blosc2', or 'zarr'") + if source_format is not None: + raise ValueError("source_format is not supported for Caterva2 URLPath inputs") lazy = kwargs.pop("lazy", False) - remote_proxy_options = _remote_proxy_options(kwargs, cache_dir, cache_path, max_concurrency, lazy=lazy) + remote_proxy_options = _remote_proxy_options( + kwargs, cache_dir, cache_path, max_concurrency, lazy=lazy, assume_immutable=assume_immutable + ) requested = [key for key, value in kwargs.items() if value is not None] if requested: raise NotImplementedError(f"{', '.join(requested)} is not supported for Caterva2 arrays") if not lazy: + if immutable_present: + raise NotImplementedError("assume_immutable requires lazy=True") if remote_proxy_options is not None: raise NotImplementedError("cache_policy and max_cache_bytes require lazy=True") if cache_dir is not None or cache_path is not None: @@ -2173,10 +2207,21 @@ def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): cache_dir, cache_path = _remote_cache_options(kwargs) storage_options = kwargs.pop("storage_options", None) + source_format = kwargs.pop("source_format", None) max_concurrency = kwargs.pop("max_concurrency", None) + immutable_present = "assume_immutable" in kwargs + assume_immutable = kwargs.pop("assume_immutable", True) lazy = kwargs.pop("lazy", False) + _validate_fsspec_source_format(source_format, lazy) remote_proxy_options = _remote_proxy_options( - kwargs, cache_dir, cache_path, max_concurrency, lazy=lazy, storage_options=storage_options + kwargs, + cache_dir, + cache_path, + max_concurrency, + lazy=lazy, + storage_options=storage_options, + source_format=source_format, + assume_immutable=assume_immutable, ) if lazy: if offset != 0: @@ -2190,6 +2235,9 @@ def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): urlpath, cache_dir, cache_path, max_concurrency, storage_options=storage_options ) + if immutable_present: + raise NotImplementedError("assume_immutable requires lazy=True") + if remote_proxy_options is not None: raise NotImplementedError("cache_policy and max_cache_bytes require lazy=True") @@ -2338,6 +2386,12 @@ def open( storage_options: dict, optional Parameters passed to the underlying ``fsspec`` filesystem when opening an fsspec URL (for instance credentials, endpoint URL, token, client_kwargs, etc.). + source_format: {None, "blosc2", "zarr"}, optional + Format of a lazy remote source. A ``.zarr`` URL path component selects + Zarr automatically; an explicit value supports suffix-free array paths. + assume_immutable: bool, optional + With ``lazy=True``, skip remote identity checks before reads. Defaults + to ``True``; set to ``False`` when the remote object may be replaced. Returns ------- diff --git a/src/blosc2/zarr_source.py b/src/blosc2/zarr_source.py new file mode 100644 index 000000000..b32a34ae7 --- /dev/null +++ b/src/blosc2/zarr_source.py @@ -0,0 +1,193 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""A :class:`ProxyNDSource` backed by an immutable Zarr array.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +from urllib.parse import urlsplit + +import numpy as np + +import blosc2 +from blosc2.proxy_source import REMOTE_MAX_CONCURRENCY, ProxyNDSource, Traffic + + +def _counting_store(zarr, store, traffic): + class CountingStore(zarr.storage.WrapperStore): + async def get(self, key, prototype, byte_range=None): + value = await super().get(key, prototype, byte_range) + if value is not None: + traffic.charge(len(value)) + return value + + async def get_partial_values(self, prototype, key_ranges): + values = await super().get_partial_values(prototype, key_ranges) + received = sum(len(value) for value in values if value is not None) + if received: + traffic.charge(received) + return values + + return CountingStore(store) + + +class ZarrNDSource(ProxyNDSource): + """Read an immutable Zarr array as Blosc2-compressed logical chunks. + + Replacing data beneath the same store identity violates this adapter's + contract and may leave previously converted chunks stale. Peak working + memory includes concurrently decoded Zarr chunks and their Blosc2 + conversion buffers; ``max_cache_bytes`` only limits retained compressed + chunks. + """ + + serves_blocks = False + encoding_version = 1 + + def __init__( + self, + store, + *, + storage_options: dict | None = None, + max_concurrency: int = REMOTE_MAX_CONCURRENCY, + blocks=None, + cparams=None, + _traffic: Traffic | None = None, + _urlpath: str | None = None, + ): + try: + import zarr + except ImportError as exc: + raise ImportError( + "ZarrNDSource requires Zarr-Python; install it with 'pip install blosc2[zarr]'" + ) from exc + + if isinstance(store, os.PathLike): + store = os.fspath(store) + self.urlpath = _urlpath if _urlpath is not None else store if isinstance(store, str) else None + self.max_concurrency = max_concurrency + remote = isinstance(store, str) and bool(urlsplit(store).scheme) + self.traffic = _traffic if _traffic is not None else Traffic() if remote else None + open_store = store + if remote: + try: + import fsspec + except ImportError as exc: + raise ImportError( + "remote Zarr sources require fsspec; install with 'pip install blosc2[zarr,fsspec]'" + ) from exc + + source = zarr.storage.FsspecStore.from_mapper( + fsspec.get_mapper(store, **(storage_options or {})), read_only=True + ) + open_store = source + if self.traffic is not None: + open_store = _counting_store(zarr, open_store, self.traffic) + try: + self.array = zarr.open_array( + store=open_store, + mode="r", + ) + except Exception as exc: + if type(exc).__name__ in {"ContainsGroupError", "NodeTypeValidationError"}: + raise ValueError(f"{store!r} is a Zarr group; pass the path of an array") from exc + raise + + self._shape = tuple(int(value) for value in self.array.shape) + self._chunks = tuple(int(value) for value in self.array.chunks) + try: + self._dtype = np.dtype(self.array.dtype) + except TypeError as exc: + raise TypeError( + f"ZarrNDSource only supports fixed-size boolean and numeric dtypes, got {self.array.dtype}" + ) from exc + self._validate_metadata() + _, computed_blocks = blosc2.compute_chunks_blocks( + self._shape, chunks=self._chunks, blocks=blocks, dtype=self._dtype, cparams=cparams + ) + self._blocks = tuple(computed_blocks) + self._cparams = ( + blosc2.CParams(typesize=self._dtype.itemsize) + if cparams is None + else blosc2.CParams(**cparams) + if isinstance(cparams, dict) + else cparams + ) + identity = { + "encoding_version": self.encoding_version, + "urlpath": self.urlpath, + "shape": self._shape, + "chunks": self._chunks, + "blocks": self._blocks, + "dtype": self._dtype.str, + } + self.stamp = hashlib.sha256( + json.dumps(identity, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + def _validate_metadata(self) -> None: + if not self._shape: + raise ValueError("ZarrNDSource does not support scalar arrays") + if len(self._shape) > blosc2.MAX_DIM: + raise ValueError(f"Zarr arrays may have at most {blosc2.MAX_DIM} dimensions") + if any(size == 0 for size in self._shape): + raise ValueError("ZarrNDSource does not support zero-length dimensions") + if len(self._chunks) != len(self._shape) or any(size <= 0 for size in self._chunks): + raise ValueError("Zarr chunk extents must be positive and match the array dimensions") + if ( + self._dtype.fields is not None + or self._dtype.subdtype is not None + or self._dtype.kind not in "buifc" + ): + raise TypeError( + f"ZarrNDSource only supports fixed-size boolean and numeric dtypes, got {self._dtype}" + ) + chunk_nbytes = math.prod(self._chunks) * self._dtype.itemsize + if chunk_nbytes > blosc2.MAX_BUFFERSIZE: + raise ValueError( + f"Zarr chunks must be at most {blosc2.MAX_BUFFERSIZE} bytes, got {chunk_nbytes}" + ) + + @property + def shape(self) -> tuple: + return self._shape + + @property + def chunks(self) -> tuple: + return self._chunks + + @property + def blocks(self) -> tuple: + return self._blocks + + @property + def dtype(self) -> np.dtype: + return self._dtype + + @property + def cparams(self): + return self._cparams + + def get_chunk(self, nchunk: int) -> bytes: + grid = tuple(math.ceil(size / chunk) for size, chunk in zip(self.shape, self.chunks, strict=True)) + total = math.prod(grid) + if isinstance(nchunk, bool) or not isinstance(nchunk, int) or nchunk < 0 or nchunk >= total: + raise IndexError(f"nchunk must be in range [0, {total}), got {nchunk}") + coords = np.unravel_index(nchunk, grid) + selection = tuple( + slice(int(coord) * chunk, min((int(coord) + 1) * chunk, size)) + for coord, chunk, size in zip(coords, self.chunks, self.shape, strict=True) + ) + values = np.ascontiguousarray(self.array[selection], dtype=self.dtype) + buffer = np.zeros(self.chunks, dtype=self.dtype) + buffer[tuple(slice(0, size) for size in values.shape)] = values + converted = blosc2.asarray(buffer, chunks=self.chunks, blocks=self.blocks, cparams=self.cparams) + return converted.schunk.get_chunk(0) diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index ff870a57d..dd4a8c91e 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -343,7 +343,7 @@ def test_open_urlpath_lazy_memory_cache(server, any_chunk_wants_blocks): served = len(srv.log) assert np.array_equal(result, data[0:5, 0:10]) assert np.array_equal(proxy[0:5, 0:10], result) - assert [endpoint for endpoint, _, _ in srv.log[served:]] == ["info"] + assert [endpoint for endpoint, _, _ in srv.log[served:]] == [] def test_open_urlpath_lazy_persistent_cache(tmp_path, server, any_chunk_wants_blocks): @@ -364,7 +364,7 @@ def test_open_urlpath_lazy_persistent_cache(tmp_path, server, any_chunk_wants_bl assert isinstance(proxy, blosc2.RemoteProxy) assert [endpoint for endpoint, _, _ in srv.log] == ["info"] assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) - assert [endpoint for endpoint, _, _ in srv.log] == ["info", "info"] + assert [endpoint for endpoint, _, _ in srv.log] == ["info"] assert len(list(cache_dir.glob("*.b2nd"))) == 1 @@ -386,7 +386,7 @@ def test_open_urlpath_lazy_exact_cache_path(tmp_path, server, any_chunk_wants_bl assert isinstance(proxy, blosc2.RemoteProxy) assert isinstance(proxy.src, blosc2.C2Array) assert np.array_equal(proxy[0:5, 0:10], data[0:5, 0:10]) - assert [endpoint for endpoint, _, _ in srv.log] == ["info", "info"] + assert [endpoint for endpoint, _, _ in srv.log] == ["info"] assert np.array_equal(proxy[100:105, 0:10], data[100:105, 0:10]) assert any(endpoint != "info" for endpoint, _, _ in srv.log) diff --git a/tests/test_remote_proxy.py b/tests/test_remote_proxy.py index 57df7bff7..e51b89f98 100644 --- a/tests/test_remote_proxy.py +++ b/tests/test_remote_proxy.py @@ -88,6 +88,31 @@ def test_cache_policy_validation(tmp_path): blosc2.RemoteProxy(url, cache_policy=blosc2.CachePolicy.DISK) with pytest.raises(ValueError, match="max_concurrency"): blosc2.RemoteProxy(url, max_concurrency=0) + with pytest.raises(TypeError, match="assume_immutable"): + blosc2.RemoteProxy(url, assume_immutable=1) + + +def test_assume_immutable_controls_identity_refresh(monkeypatch): + url, _ = _remote_array("immutable-option.b2nd", nchunks=1, chunk_size=100) + immutable = blosc2.RemoteProxy(url) + mutable = blosc2.open(url, lazy=True, assume_immutable=False) + refreshed = [] + + monkeypatch.setattr( + immutable.src, + "refresh_identity", + lambda: (_ for _ in ()).throw(AssertionError("immutable source was refreshed")), + ) + monkeypatch.setattr(mutable.src, "refresh_identity", lambda: refreshed.append(True)) + + immutable._prepare_read() + mutable._prepare_read() + + assert immutable.assume_immutable is True + assert mutable.assume_immutable is False + assert immutable.source["assume_immutable"] is True + assert mutable.source["assume_immutable"] is False + assert refreshed == [True] def test_remote_proxy_array_operand_interface(): @@ -254,7 +279,7 @@ def test_server_sparse_cache_reuses_partial_blocks(tmp_path): def test_server_sparse_cache_invalidates_same_geometry_replacement(tmp_path): url, data = _remote_array("server-replaced.b2nd", nchunks=2, chunk_size=100) runtime_path = tmp_path / "replaced-runtime" - proxy = blosc2.RemoteProxy.with_sparse_cache(url, runtime_path) + proxy = blosc2.RemoteProxy.with_sparse_cache(url, runtime_path, assume_immutable=False) np.testing.assert_array_equal(proxy[:100], data[:100]) replacement = np.arange(200, dtype=np.uint8) @@ -353,7 +378,12 @@ def test_disk_roundtrip_preserves_warm_cache_and_cold_escape_hatch(tmp_path): assert carrier.schunk.vlmeta["b2o"] == { "kind": "remote_proxy", "version": 1, - "source": {"kind": "fsspec", "version": 1, "urlpath": url}, + "source": { + "kind": "fsspec", + "version": 1, + "urlpath": url, + "assume_immutable": True, + }, "cache_policy": "disk", "max_cache_bytes": 120_000, } @@ -392,7 +422,7 @@ def test_disk_roundtrip_preserves_warm_cache_and_cold_escape_hatch(tmp_path): def test_reference_rejects_changed_source_geometry(tmp_path): url, _ = _remote_array("changed-geometry.b2nd", nchunks=1, chunk_size=100) path = tmp_path / "changed-reference.b2nd" - blosc2.RemoteProxy(url).save(path) + blosc2.RemoteProxy(url, assume_immutable=False).save(path) replacement = blosc2.arange(200, dtype=np.uint8, chunks=(100,), blocks=(100,)) fsspec.filesystem("memory").pipe_file("changed-geometry.b2nd", replacement.to_cframe()) @@ -403,7 +433,7 @@ def test_reference_rejects_changed_source_geometry(tmp_path): def test_open_reference_rejects_geometry_changed_before_read(tmp_path): url, _ = _remote_array("changed-after-open.b2nd", nchunks=1, chunk_size=100) path = tmp_path / "changed-after-open-reference.b2nd" - blosc2.RemoteProxy(url).save(path) + blosc2.RemoteProxy(url, assume_immutable=False).save(path) restored = blosc2.open(path, mode="r") replacement = blosc2.arange(200, dtype=np.uint8, chunks=(100,), blocks=(100,)) @@ -419,6 +449,7 @@ def test_runtime_cache_is_invalidated_after_same_geometry_replacement(tmp_path): url, cache_policy=blosc2.CachePolicy.DISK, cache_path=tmp_path / "same-geometry-cache.b2nd", + assume_immutable=False, ) traffic = proxy.traffic np.testing.assert_array_equal(proxy[:], data) @@ -548,6 +579,7 @@ def fake_info(path, urlbase, params=None, headers=None, model=None, auth_token=N "version": 1, "path": "@personal/private.b2nd", "urlbase": "https://example.org/caterva2/", + "assume_immutable": True, } restored = blosc2.from_cframe(remote.to_cframe()) @@ -853,7 +885,9 @@ def info(*args, **kwargs): } monkeypatch.setattr(blosc2_c2array, "info", info) - proxy = blosc2.RemoteProxy(blosc2.URLPath("@public/a.b2nd", urlbase="https://example.org")) + proxy = blosc2.RemoteProxy( + blosc2.URLPath("@public/a.b2nd", urlbase="https://example.org"), assume_immutable=False + ) old = proxy.src.stamp state["nonce"] = "replacement" proxy._prepare_read() @@ -993,7 +1027,7 @@ def test_unlimited_disk_cache_does_not_evict(tmp_path): def test_authorized_sparse_snapshot_never_reopens(tmp_path, monkeypatch): url, data = _remote_array("authorized-sparse.b2nd", nchunks=3, chunk_size=10000) source = blosc2.FsspecNDSource(url) - descriptor = {"kind": "fsspec", "version": 1, "urlpath": url} + descriptor = {"kind": "fsspec", "version": 1, "urlpath": url, "assume_immutable": True} def forbidden(*args, **kwargs): raise AssertionError("authorized transport was reopened or refreshed") diff --git a/tests/test_zarr_source.py b/tests/test_zarr_source.py new file mode 100644 index 000000000..026338d77 --- /dev/null +++ b/tests/test_zarr_source.py @@ -0,0 +1,227 @@ +import builtins + +import numpy as np +import pytest + +import blosc2 + + +@pytest.fixture(scope="module") +def zarr(): + return pytest.importorskip("zarr") + + +@pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.parametrize("dtype", [np.bool_, np.int32, np.float64, np.complex64, ">i4"]) +def test_zarr_source_through_proxy(tmp_path, zarr, zarr_format, dtype): + path = tmp_path / f"array-{zarr_format}-{np.dtype(dtype).str}.zarr" + data = np.arange(35).reshape(5, 7).astype(dtype) + array = zarr.create_array(path, shape=data.shape, chunks=(3, 4), dtype=dtype, zarr_format=zarr_format) + array[:] = data + + source = blosc2.ZarrNDSource(path) + proxy = blosc2.Proxy(source) + + assert source.serves_blocks is False + assert source.shape == data.shape + assert source.chunks == (3, 4) + np.testing.assert_array_equal(proxy[:], data) + with pytest.raises(IndexError, match="nchunk"): + source.get_chunk(4) + + +def test_zarr_source_preserves_fill_and_edge_chunks(tmp_path, zarr): + path = tmp_path / "fill.zarr" + array = zarr.create_array( + path, shape=(5, 7), chunks=(3, 4), dtype=np.int16, fill_value=17, zarr_format=3 + ) + array[:2, :2] = 3 + + proxy = blosc2.Proxy(blosc2.ZarrNDSource(path)) + expected = np.full((5, 7), 17, dtype=np.int16) + expected[:2, :2] = 3 + np.testing.assert_array_equal(proxy[:], expected) + + +@pytest.mark.parametrize("dtype", ["U4", "S4", [("field", "i4")]]) +def test_zarr_source_rejects_unsupported_dtype(tmp_path, zarr, dtype): + path = tmp_path / "unsupported.zarr" + zarr.create_array(path, shape=(2,), chunks=(2,), dtype=dtype, zarr_format=2) + + with pytest.raises(TypeError, match="fixed-size boolean and numeric"): + blosc2.ZarrNDSource(path) + + +def test_zarr_source_rejects_variable_string_dtype(tmp_path, zarr): + path = tmp_path / "variable-string.zarr" + zarr.create_array(path, shape=(2,), chunks=(2,), dtype="str") + + with pytest.raises(TypeError, match="fixed-size boolean and numeric"): + blosc2.ZarrNDSource(path) + + +def test_zarr_source_rejects_scalar_and_empty(tmp_path, zarr): + scalar = tmp_path / "scalar.zarr" + empty = tmp_path / "empty.zarr" + zarr.create_array(scalar, shape=(), dtype="i4") + zarr.create_array(empty, shape=(0,), chunks=(1,), dtype="i4") + + with pytest.raises(ValueError, match="scalar"): + blosc2.ZarrNDSource(scalar) + with pytest.raises(ValueError, match="zero-length"): + blosc2.ZarrNDSource(empty) + + +def test_zarr_source_group_error(tmp_path, zarr): + path = tmp_path / "group.zarr" + zarr.create_group(path) + + with pytest.raises(ValueError, match="pass the path of an array"): + blosc2.ZarrNDSource(path) + + +@pytest.mark.parametrize( + ("url", "source_format"), + [ + ("memory://zarr-tests/trailing.zarr/", None), + ("memory://zarr-tests/hierarchy.zarr/d0/a", None), + ("memory://zarr-tests/suffix-free-open", "zarr"), + ], +) +def test_open_remote_zarr_as_remote_proxy(zarr, url, source_format): + data = np.arange(35, dtype=np.int32).reshape(5, 7) + array = zarr.create_array(url, shape=data.shape, chunks=(3, 4), dtype=data.dtype) + array[:] = data + + kwargs = {} if source_format is None else {"source_format": source_format} + proxy = blosc2.open(url, lazy=True, **kwargs) + + assert isinstance(proxy, blosc2.RemoteProxy) + assert proxy.source == { + "kind": "zarr", + "version": 1, + "urlpath": url, + "assume_immutable": True, + } + proxy.traffic.reset() + np.testing.assert_array_equal(proxy[1:5, 2:6], data[1:5, 2:6]) + assert proxy.traffic.nbytes > 0 + proxy.traffic.reset() + np.testing.assert_array_equal(proxy[1:5, 2:6], data[1:5, 2:6]) + assert proxy.traffic.requests == 0 + + +def test_remote_zarr_disk_carrier_reopens_warm(tmp_path, zarr): + url = "memory://zarr-tests/persistent.zarr" + data = np.arange(35, dtype=np.int32).reshape(5, 7) + array = zarr.create_array(url, shape=data.shape, chunks=(3, 4), dtype=data.dtype) + array[:] = data + path = tmp_path / "zarr-proxy.b2nd" + proxy = blosc2.RemoteProxy( + url, cache_policy=blosc2.CachePolicy.DISK, cache_path=path, source_format="zarr" + ) + np.testing.assert_array_equal(proxy[:3, :4], data[:3, :4]) + + reopened = blosc2.open(path) + assert isinstance(reopened, blosc2.RemoteProxy) + reopened.src.get_chunk = lambda nchunk: (_ for _ in ()).throw(AssertionError("cache miss")) + np.testing.assert_array_equal(reopened[:3, :4], data[:3, :4]) + + +def test_zarr_requires_lazy_open(): + with pytest.raises(NotImplementedError, match="lazy=True"): + blosc2.open("memory://zarr-tests/not-opened.zarr", source_format="zarr") + + +def test_zarr_rejects_mutable_source_mode(): + with pytest.raises(NotImplementedError, match="mutable Zarr"): + blosc2.RemoteProxy( + "memory://zarr-tests/not-opened.zarr", + source_format="zarr", + assume_immutable=False, + ) + + +def test_explicit_blosc2_format_overrides_zarr_suffix(): + fsspec = pytest.importorskip("fsspec") + url = "memory://zarr-tests/blosc2-array.zarr" + data = np.arange(8, dtype=np.int16) + array = blosc2.asarray(data, chunks=(4,), blocks=(4,)) + fsspec.filesystem("memory").pipe_file("zarr-tests/blosc2-array.zarr", array.to_cframe()) + + proxy = blosc2.open(url, lazy=True, source_format="blosc2") + assert proxy.source["kind"] == "fsspec" + np.testing.assert_array_equal(proxy[:], data) + + +def test_zarr_source_missing_dependency_error(monkeypatch): + real_import = builtins.__import__ + + def blocked_import(name, *args, **kwargs): + if name == "zarr": + raise ImportError("blocked for test") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", blocked_import) + with pytest.raises(ImportError, match=r"blosc2\[zarr\]"): + blosc2.ZarrNDSource("unused.zarr") + + +def test_direct_proxy_zarr_cache_reopens(tmp_path, zarr): + source_path = tmp_path / "source.zarr" + cache_path = tmp_path / "cache.b2nd" + data = np.arange(35, dtype=np.int32).reshape(5, 7) + array = zarr.create_array(source_path, shape=data.shape, chunks=(3, 4), dtype=data.dtype) + array[:] = data + proxy = blosc2.Proxy(blosc2.ZarrNDSource(source_path), urlpath=cache_path, mode="w") + np.testing.assert_array_equal(proxy[:3, :4], data[:3, :4]) + + reopened = blosc2.open(cache_path) + assert isinstance(reopened.src, blosc2.ZarrNDSource) + reopened.src.get_chunk = lambda nchunk: (_ for _ in ()).throw(AssertionError("cache miss")) + np.testing.assert_array_equal(reopened[:3, :4], data[:3, :4]) + + +def test_zarr_v3_shards_are_decoded_as_logical_chunks(tmp_path, zarr): + path = tmp_path / "sharded.zarr" + data = np.arange(64, dtype=np.float32).reshape(8, 8) + array = zarr.create_array(path, shape=data.shape, chunks=(2, 2), shards=(4, 4), dtype=data.dtype) + array[:] = data + + proxy = blosc2.Proxy(blosc2.ZarrNDSource(path)) + assert proxy.chunks == (2, 2) + np.testing.assert_array_equal(proxy[1:7, 1:7], data[1:7, 1:7]) + + +def test_zarr_ref_preserves_explicit_suffix_free_format(zarr): + url = "memory://zarr-tests/suffix-free" + data = np.arange(8, dtype=np.int16) + array = zarr.create_array(url, shape=data.shape, chunks=(4,), dtype=data.dtype) + array[:] = data + proxy = blosc2.RemoteProxy(url, source_format="zarr") + ref = blosc2.Ref.from_object(proxy) + + assert ref.kind == "zarr" + np.testing.assert_array_equal(ref.open()[:], data) + restored = blosc2.from_cframe(blosc2.lazyexpr("a + 1", operands={"a": proxy}).to_cframe()) + np.testing.assert_array_equal(restored[:], data + 1) + + +def test_authorized_zarr_store_is_retained_for_sparse_cache(tmp_path, monkeypatch, zarr): + store = zarr.storage.MemoryStore() + data = np.arange(8, dtype=np.int16) + array = zarr.create_array(store, shape=data.shape, chunks=(4,), dtype=data.dtype) + array[:] = data + url = "https://example.org/data.zarr" + source = blosc2.ZarrNDSource(store, _urlpath=url, _traffic=blosc2.Traffic()) + descriptor = {"kind": "zarr", "version": 1, "urlpath": url, "assume_immutable": True} + + monkeypatch.setattr( + blosc2.RemoteProxy, + "_open_source", + lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("unrestricted reopen")), + ) + proxy = blosc2.RemoteProxy.with_sparse_cache( + source, tmp_path / "runtime-cache", source_descriptor=descriptor + ) + np.testing.assert_array_equal(proxy[:], data) From aa6b219fa64250f8daf0ac71087a9d7d01a9434f Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 7 Sep 2026 18:53:12 +0200 Subject: [PATCH 27/89] Support fixed-size Zarr dtypes --- src/blosc2/zarr_source.py | 14 +++----------- tests/test_zarr_source.py | 39 ++++++++++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/src/blosc2/zarr_source.py b/src/blosc2/zarr_source.py index b32a34ae7..f56dedb82 100644 --- a/src/blosc2/zarr_source.py +++ b/src/blosc2/zarr_source.py @@ -106,9 +106,7 @@ def __init__( try: self._dtype = np.dtype(self.array.dtype) except TypeError as exc: - raise TypeError( - f"ZarrNDSource only supports fixed-size boolean and numeric dtypes, got {self.array.dtype}" - ) from exc + raise TypeError(f"ZarrNDSource only supports fixed-size dtypes, got {self.array.dtype}") from exc self._validate_metadata() _, computed_blocks = blosc2.compute_chunks_blocks( self._shape, chunks=self._chunks, blocks=blocks, dtype=self._dtype, cparams=cparams @@ -142,14 +140,8 @@ def _validate_metadata(self) -> None: raise ValueError("ZarrNDSource does not support zero-length dimensions") if len(self._chunks) != len(self._shape) or any(size <= 0 for size in self._chunks): raise ValueError("Zarr chunk extents must be positive and match the array dimensions") - if ( - self._dtype.fields is not None - or self._dtype.subdtype is not None - or self._dtype.kind not in "buifc" - ): - raise TypeError( - f"ZarrNDSource only supports fixed-size boolean and numeric dtypes, got {self._dtype}" - ) + if self._dtype.hasobject or self._dtype.itemsize == 0: + raise TypeError(f"ZarrNDSource only supports fixed-size dtypes, got {self._dtype}") chunk_nbytes = math.prod(self._chunks) * self._dtype.itemsize if chunk_nbytes > blosc2.MAX_BUFFERSIZE: raise ValueError( diff --git a/tests/test_zarr_source.py b/tests/test_zarr_source.py index 026338d77..f2dfe2884 100644 --- a/tests/test_zarr_source.py +++ b/tests/test_zarr_source.py @@ -43,20 +43,45 @@ def test_zarr_source_preserves_fill_and_edge_chunks(tmp_path, zarr): np.testing.assert_array_equal(proxy[:], expected) -@pytest.mark.parametrize("dtype", ["U4", "S4", [("field", "i4")]]) -def test_zarr_source_rejects_unsupported_dtype(tmp_path, zarr, dtype): - path = tmp_path / "unsupported.zarr" - zarr.create_array(path, shape=(2,), chunks=(2,), dtype=dtype, zarr_format=2) +@pytest.mark.parametrize( + "data", + [ + np.array([[b"one", b"two"], [b"three", b""]], dtype="S6"), + np.array([["one", "two"], ["three", ""]], dtype="U6"), + np.array([["2024-01-01", "2024-01-02"], ["2024-01-03", "2024-01-04"]], dtype="M8[ns]"), + np.array([[1, 2], [3, 4]], dtype="m8[ns]"), + ], +) +@pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") +def test_zarr_source_supports_fixed_size_dtypes(tmp_path, zarr, data, zarr_format): + path = tmp_path / "fixed-size.zarr" + array = zarr.create_array(path, data=data, chunks=data.shape, zarr_format=zarr_format) - with pytest.raises(TypeError, match="fixed-size boolean and numeric"): - blosc2.ZarrNDSource(path) + source = blosc2.ZarrNDSource(path) + proxy = blosc2.Proxy(source) + + assert source.dtype == data.dtype + np.testing.assert_array_equal(proxy[:], data) + + +def test_zarr_source_supports_structured_dtype(tmp_path, zarr): + data = np.array([(1, 1.5), (2, 2.5)], dtype=[("id", "i4"), ("value", "f4")]) + path = tmp_path / "structured.zarr" + zarr.create_array(path, data=data, chunks=data.shape, zarr_format=2, fill_value=(0, 0)) + + source = blosc2.ZarrNDSource(path) + proxy = blosc2.Proxy(source) + + assert source.dtype == data.dtype + np.testing.assert_array_equal(proxy[:], data) def test_zarr_source_rejects_variable_string_dtype(tmp_path, zarr): path = tmp_path / "variable-string.zarr" zarr.create_array(path, shape=(2,), chunks=(2,), dtype="str") - with pytest.raises(TypeError, match="fixed-size boolean and numeric"): + with pytest.raises(TypeError, match="fixed-size dtypes"): blosc2.ZarrNDSource(path) From 397388128b865eb51a3af03b527f8bc54a64a31f Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 7 Sep 2026 20:13:57 +0200 Subject: [PATCH 28/89] Support scalar and empty Zarr arrays --- plans/remote-proxy-v8.md | 20 ++++++++++---------- src/blosc2/zarr_source.py | 12 ++++++------ tests/test_zarr_source.py | 29 ++++++++++++++++++++++++----- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/plans/remote-proxy-v8.md b/plans/remote-proxy-v8.md index 8956327fc..b187cbe2a 100644 --- a/plans/remote-proxy-v8.md +++ b/plans/remote-proxy-v8.md @@ -26,8 +26,9 @@ Caterva2 server. the cache chunk shape. Let Zarr resolve chunk reads within shards. - Support Zarr format 2 and 3 through a supported Zarr-Python 3 release. This does not require supporting the older Zarr-Python 2 API. -- Start with fixed-size numeric and boolean arrays representable by B2ND. - Reject unsupported dtypes explicitly before creating a cache. +- Support fixed-size NumPy dtypes representable by B2ND, including fixed-width + strings, temporal values, and structured records. Reject object-bearing and + variable-length dtypes explicitly before creating a cache. - `RemoteProxy(..., assume_immutable=True)` is the default for every source and skips metadata polling before reads. `False` retains identity refresh and cache invalidation for mutable single-file and Caterva2 sources. @@ -162,15 +163,14 @@ payload, not transient decoded memory or process RSS. ### Supported representations -Cover bool, integer, floating-point, and complex dtypes supported by B2ND, both -Zarr formats, alternative codec pipelines, nonzero fill values, edge chunks, -and Zarr v3 sharding. Let Zarr decode storage order and transpose codecs. +Cover all fixed-size dtypes supported by B2ND, both Zarr formats, alternative +codec pipelines, nonzero fill values, edge chunks, scalar arrays, zero-length +arrays, and Zarr v3 sharding. Let Zarr decode storage order and transpose codecs. -Reject object, variable-length, string, structured, and other unsupported dtype -representations with a clear `TypeError` in this first version. Test scalar and -zero-length arrays against existing B2ND geometry constraints; support them if -the normal path works, otherwise reject explicitly at construction and document -the limitation. Never defer such failures until after a partially written cache. +Reject object-bearing and variable-length dtype representations with a clear +`TypeError`. Scalar arrays cache their one value; zero-length arrays have no +payload chunks and retain zero cache bytes. Never defer failures until after a +partially written cache. ## Immutable identity and persistence diff --git a/src/blosc2/zarr_source.py b/src/blosc2/zarr_source.py index f56dedb82..17b4e2641 100644 --- a/src/blosc2/zarr_source.py +++ b/src/blosc2/zarr_source.py @@ -132,12 +132,8 @@ def __init__( ).hexdigest() def _validate_metadata(self) -> None: - if not self._shape: - raise ValueError("ZarrNDSource does not support scalar arrays") if len(self._shape) > blosc2.MAX_DIM: raise ValueError(f"Zarr arrays may have at most {blosc2.MAX_DIM} dimensions") - if any(size == 0 for size in self._shape): - raise ValueError("ZarrNDSource does not support zero-length dimensions") if len(self._chunks) != len(self._shape) or any(size <= 0 for size in self._chunks): raise ValueError("Zarr chunk extents must be positive and match the array dimensions") if self._dtype.hasobject or self._dtype.itemsize == 0: @@ -178,8 +174,12 @@ def get_chunk(self, nchunk: int) -> bytes: slice(int(coord) * chunk, min((int(coord) + 1) * chunk, size)) for coord, chunk, size in zip(coords, self.chunks, self.shape, strict=True) ) - values = np.ascontiguousarray(self.array[selection], dtype=self.dtype) + values = np.asarray(self.array[selection], dtype=self.dtype) buffer = np.zeros(self.chunks, dtype=self.dtype) - buffer[tuple(slice(0, size) for size in values.shape)] = values + if self.shape: + values = np.ascontiguousarray(values) + buffer[tuple(slice(0, size) for size in values.shape)] = values + else: + buffer[()] = values converted = blosc2.asarray(buffer, chunks=self.chunks, blocks=self.blocks, cparams=self.cparams) return converted.schunk.get_chunk(0) diff --git a/tests/test_zarr_source.py b/tests/test_zarr_source.py index f2dfe2884..2503823b8 100644 --- a/tests/test_zarr_source.py +++ b/tests/test_zarr_source.py @@ -85,16 +85,35 @@ def test_zarr_source_rejects_variable_string_dtype(tmp_path, zarr): blosc2.ZarrNDSource(path) -def test_zarr_source_rejects_scalar_and_empty(tmp_path, zarr): +def test_zarr_source_supports_scalar_and_empty(tmp_path, zarr): scalar = tmp_path / "scalar.zarr" empty = tmp_path / "empty.zarr" zarr.create_array(scalar, shape=(), dtype="i4") + zarr.open_array(scalar, mode="r+")[()] = 42 zarr.create_array(empty, shape=(0,), chunks=(1,), dtype="i4") - with pytest.raises(ValueError, match="scalar"): - blosc2.ZarrNDSource(scalar) - with pytest.raises(ValueError, match="zero-length"): - blosc2.ZarrNDSource(empty) + scalar_proxy = blosc2.Proxy(blosc2.ZarrNDSource(scalar)) + empty_proxy = blosc2.Proxy(blosc2.ZarrNDSource(empty)) + + assert scalar_proxy[()] == 42 + np.testing.assert_array_equal(empty_proxy[:], np.empty(0, dtype=np.int32)) + + +def test_open_remote_zarr_scalar_and_empty(zarr): + scalar_url = "memory://zarr-tests/scalar.zarr" + empty_url = "memory://zarr-tests/empty.zarr" + scalar = zarr.create_array(scalar_url, shape=(), dtype="i4") + scalar[()] = 42 + zarr.create_array(empty_url, shape=(0,), chunks=(1,), dtype="i4") + + scalar_proxy = blosc2.open(scalar_url, lazy=True) + empty_proxy = blosc2.open(empty_url, lazy=True) + + assert isinstance(scalar_proxy, blosc2.RemoteProxy) + assert scalar_proxy[()] == 42 + assert isinstance(empty_proxy, blosc2.RemoteProxy) + np.testing.assert_array_equal(empty_proxy[:], np.empty(0, dtype=np.int32)) + assert empty_proxy.cache_bytes == 0 def test_zarr_source_group_error(tmp_path, zarr): From 4b1af615a5a8b3cf7fc52a2ace5f760d7ef5874d Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 8 Sep 2026 07:32:15 +0200 Subject: [PATCH 29/89] Support remote HDF5 arrays via kerchunk and unify dataset syntax --- doc/getting_started/installation.rst | 4 + doc/guides/remote_arrays.md | 32 +- doc/reference/classes.rst | 2 + doc/reference/hdf5ndsource.rst | 24 ++ doc/reference/remoteproxy.rst | 22 +- doc/reference/zarrndsource.rst | 22 + examples/remote/s3-access.py | 30 +- plans/remote-proxy-v9.md | 523 +++++++++++++++++++++++ pyproject.toml | 2 + src/blosc2/__init__.py | 3 + src/blosc2/core.py | 58 +++ src/blosc2/hdf5_source.py | 412 ++++++++++++++++++ src/blosc2/remote_proxy.py | 366 +++++++++++----- src/blosc2/schunk.py | 277 ++++++++---- src/blosc2/zarr_source.py | 63 ++- tests/test_hdf5_source.py | 602 +++++++++++++++++++++++++++ tests/test_zarr_source.py | 17 + 17 files changed, 2256 insertions(+), 203 deletions(-) create mode 100644 doc/reference/hdf5ndsource.rst create mode 100644 doc/reference/zarrndsource.rst create mode 100644 plans/remote-proxy-v9.md create mode 100644 src/blosc2/hdf5_source.py create mode 100644 tests/test_hdf5_source.py diff --git a/doc/getting_started/installation.rst b/doc/getting_started/installation.rst index 32d68311a..224f81f78 100644 --- a/doc/getting_started/installation.rst +++ b/doc/getting_started/installation.rst @@ -43,6 +43,9 @@ grouped into *extras* that you opt into with the ``blosc2[extra]`` syntax: * - ``zarr`` - Lazy Zarr sources and the ``b2nd-to-zarr`` (or ``blosc2-to-zarr``) converter (``zarr``). + * - ``hdf5`` + - Reading HDF5 datasets lazily as virtual arrays via kerchunk (``kerchunk``, + ``h5py``, ``hdf5plugin``). * - ``fsspec`` - Reading and writing single-file containers through any `fsspec `_ URL. The HTTP(S) driver is @@ -61,6 +64,7 @@ argument in shells like ``zsh`` that treat brackets specially): pip install "blosc2[fsspec]" # fsspec URLs, including HTTP(S) pip install "blosc2[fsspec]" s3fs # fsspec URLs, plus the S3 driver pip install "blosc2[zarr,fsspec]" s3fs # remote Zarr arrays on S3 + pip install "blosc2[hdf5,fsspec]" s3fs # remote HDF5 datasets on S3 pip install "blosc2[tui,parquet]" # several at once With the ``fsspec`` extra, :func:`blosc2.open` accepts any fsspec URL, chained diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index e118f3ee9..abd07f966 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -12,6 +12,7 @@ The argument passed to {func}`blosc2.open` selects the route: |---|---|---| | A URL string such as `s3://...` or `https://...` | fsspec | A byte-addressable, standalone `.b2nd` file | | A URL containing a `.zarr` path component | Zarr | One immutable Zarr v2 or v3 array | +| A URL containing a `.h5` or `.hdf5` path component, or `source_format="hdf5"` | HDF5 | One immutable HDF5 dataset via kerchunk | | A {ref}`URLPath` | Caterva2 | One array-like dataset on a Caterva2 server | ```python @@ -29,20 +30,39 @@ b = blosc2.open( lazy=True, ) +# Zarr and HDF5: addressing datasets inside containers +# Both formats support standard slashes (/), container separators (::), or the dataset= parameter: +c1 = blosc2.open("s3://bucket/hierarchy.zarr/d0/d1/a2", lazy=True) +c2 = blosc2.open("s3://bucket/hierarchy.zarr::d0/d1/a2", lazy=True) +c3 = blosc2.open("s3://bucket/hierarchy.zarr", lazy=True, dataset="d0/d1/a2") + +h1 = blosc2.open("s3://bucket/hierarchy.h5/d0/d1/a2", lazy=True) +h2 = blosc2.open("s3://bucket/hierarchy.h5::d0/d1/a2", lazy=True) +h3 = blosc2.open("s3://bucket/hierarchy.h5", lazy=True, dataset="d0/d1/a2") + a.shape, a.dtype # metadata is available immediately a[100:110, :50] # data is fetched now ``` Remote Zarr needs `pip install "blosc2[zarr,fsspec]"` plus the protocol driver -(`s3fs` for S3). The URL names the array itself; nested array paths work, while -opening a group asks for an array path. For a suffix-free URL, pass -`source_format="zarr"`. Converted Blosc2 chunks are cached under an immutable -source contract, so publish changed data at a new URL or replace its cache. +(`s3fs` for S3). Datasets can be named directly by path (`/sub/arr`), with the `::sub/arr` +separator, or via `dataset="sub/arr"`. For a suffix-free URL, pass `source_format="zarr"`. +Converted Blosc2 chunks are cached under an immutable source contract, so publish changed +data at a new URL or replace its cache. + +Remote HDF5 needs `pip install "blosc2[hdf5,fsspec]"` plus the protocol driver +(`s3fs` for S3). Datasets can be specified using standard slash syntax (`file.h5/d0/d1/a2`), +the double-colon separator (`file.h5::d0/d1/a2`), or the `dataset="d0/d1/a2"` parameter. +Pre-indexing is performed via `kerchunk`, and the resulting reference map is cached inside +the RemoteProxy carrier (`schunk.vlmeta["hdf5-refs"]`) so reopening the carrier requires +no network re-indexing. Use `blosc2.available_datasets(url)` to inspect datasets in an HDF5 +container. `RemoteProxy` assumes remote sources are immutable by default, avoiding a metadata request before every read. For a replaceable `.b2nd` or Caterva2 source, pass `assume_immutable=False` to refresh its identity and invalidate -stale cached chunks before each operation. +stale cached chunks before each operation. Mutable Zarr and HDF5 sources are +not supported. A `URLPath` always means Caterva2. If its `urlbase` is omitted, the server comes from {func}`blosc2.c2context` or `BLOSC_C2URLBASE`. Other transports can be added with a custom {ref}`ByteRangeNDSource`; see [Use your own transport](#use-your-own-transport). @@ -54,7 +74,7 @@ When opened with `lazy=True`, both routes return a {ref}`RemoteProxy`, providing |---|---|---| | Standalone contiguous `.b2nd` | Yes | Yes | | Zarr v2/v3 array | Yes, with `source_format="zarr"` | No | -| HDF5 dataset | No | Yes | +| HDF5 dataset | Yes, with `dataset="..."` | Yes | | NDArray leaf inside `.b2z` | No | Yes | | Lazy or computed array | No | Yes | | Whole `.b2z` `TreeStore` or `DictStore` | No | No; open one array-like leaf | diff --git a/doc/reference/classes.rst b/doc/reference/classes.rst index 1d4c5a693..91384774f 100644 --- a/doc/reference/classes.rst +++ b/doc/reference/classes.rst @@ -33,6 +33,7 @@ without chunk caching. ProxySource ProxyNDSource ZarrNDSource + HDF5NDSource SimpleProxy Traffic @@ -146,6 +147,7 @@ container APIs above. byterangendsource fsspecndsource zarrndsource + hdf5ndsource simpleproxy traffic embed_store diff --git a/doc/reference/hdf5ndsource.rst b/doc/reference/hdf5ndsource.rst new file mode 100644 index 000000000..e7d70b67a --- /dev/null +++ b/doc/reference/hdf5ndsource.rst @@ -0,0 +1,24 @@ +.. _HDF5NDSource: + +HDF5NDSource +============ + +``HDF5NDSource`` exposes an HDF5 dataset through :ref:`ProxyNDSource` using +``kerchunk`` metadata pre-indexing. Individual chunks are fetched on demand +and converted to Blosc2-compressed chunks stored in the surrounding +:ref:`Proxy` or :ref:`RemoteProxy` cache. + +The source is assumed immutable (``assume_immutable=True``). It supports fixed-size +boolean, integer, floating-point, complex, and fixed-length string arrays. +HDF5 filters such as Blosc2 (via ``hdf5plugin``), gzip, and uncompressed datasets +are supported. + +Install local support with ``pip install "blosc2[hdf5]"``. Remote datasets also +need ``blosc2[fsspec]`` and the protocol driver, such as ``s3fs`` for S3. + +.. autofunction:: blosc2.available_datasets + +.. autoclass:: blosc2.HDF5NDSource + + .. automethod:: __init__ + .. automethod:: get_chunk diff --git a/doc/reference/remoteproxy.rst b/doc/reference/remoteproxy.rst index a0d0d1179..0d846ecf0 100644 --- a/doc/reference/remoteproxy.rst +++ b/doc/reference/remoteproxy.rst @@ -3,7 +3,7 @@ RemoteProxy =========== -``RemoteProxy`` is a persistable proxy for one remote B2ND or Zarr array. It +``RemoteProxy`` is a persistable proxy for one remote B2ND, Zarr, or HDF5 array. It accepts an fsspec URL or a Caterva2 :ref:`URLPath`. With disk caching enabled, its B2ND carrier is both the portable descriptor and the bounded compressed-data cache. @@ -52,6 +52,25 @@ publishing a new dataset. Mutable Zarr stores are not supported. storage_options={"anon": True}, ) +HDF5 URLs (``.h5``, ``.hdf5``, or ``source_format="hdf5"``) select :ref:`HDF5NDSource`. +Datasets within an HDF5 container can be specified via standard slash syntax (``.../file.h5/dataset``), +the double-colon separator (``.../file.h5::dataset``), or the ``dataset="dataset"`` argument. +Zarr containers similarly accept all three forms (``.../file.zarr/dataset``, ``.../file.zarr::dataset``, +or ``dataset="dataset"``). +HDF5 datasets are read through ``kerchunk`` metadata pre-indexing. Like Zarr, HDF5 sources +are assumed immutable (``assume_immutable=True``); mutable HDF5 sources are not supported. +Pre-computed kerchunk references can be supplied via ``refs`` to avoid remote scanning. + +.. code-block:: python + + remote = blosc2.open( + "s3://public-bucket/hierarchy.h5/d0/d1/a2", + lazy=True, + storage_options={"profile": "blosc2"}, + ) + # Equivalent to "s3://public-bucket/hierarchy.h5::d0/d1/a2" + # or blosc2.open("s3://public-bucket/hierarchy.h5", lazy=True, dataset="d0/d1/a2", ...) + Ephemeral in-memory caching is available through :attr:`blosc2.CachePolicy.MEMORY`. Fetched chunks are kept in RAM, bounded by a finite 256 MiB compressed-payload limit by default (customizable via ``max_cache_bytes``) with automatic LRU eviction. @@ -149,6 +168,7 @@ file. Read-only mode can use warm chunks but does not retain misses: .. autoattribute:: source .. autoattribute:: traffic .. autoattribute:: urlpath + .. autoattribute:: dataset CachePolicy ----------- diff --git a/doc/reference/zarrndsource.rst b/doc/reference/zarrndsource.rst new file mode 100644 index 000000000..97037f5ac --- /dev/null +++ b/doc/reference/zarrndsource.rst @@ -0,0 +1,22 @@ +.. _ZarrNDSource: + +ZarrNDSource +============ + +``ZarrNDSource`` exposes a Zarr v2 or v3 array through :ref:`ProxyNDSource`. +Zarr decodes each logical chunk and Blosc2 stores the converted compressed +chunk in the surrounding :ref:`Proxy` or :ref:`RemoteProxy` cache. + +The source is assumed immutable for the cache lifetime. It supports fixed-size +boolean, integer, floating-point, and complex arrays. Scalar arrays, empty +dimensions, strings, objects, structured dtypes, and ZIP stores are not +supported. Concurrent reads temporarily hold decoded chunks and conversion +buffers in addition to the compressed cache. + +Install local support with ``pip install "blosc2[zarr]"``. Remote stores also +need ``blosc2[fsspec]`` and the protocol driver, such as ``s3fs``. + +.. autoclass:: blosc2.ZarrNDSource + + .. automethod:: __init__ + .. automethod:: get_chunk diff --git a/examples/remote/s3-access.py b/examples/remote/s3-access.py index 3d57dc072..8a978404c 100755 --- a/examples/remote/s3-access.py +++ b/examples/remote/s3-access.py @@ -6,7 +6,7 @@ # SPDX-License-Identifier: BSD-3-Clause ####################################################################### -"""Open a remote S3 array (Blosc2 .b2nd or Zarr .zarr) and print metadata and sample data. +"""Open a remote S3 array (Blosc2 .b2nd, Zarr .zarr, or HDF5 .h5) and print metadata and sample data. Usage: python s3-access.py [--profile PROFILE] [--endpoint-url ENDPOINT_URL] @@ -16,6 +16,9 @@ python s3-access.py s3://blosc2/cube-1k-1k-1k.zarr python s3-access.py s3://blosc2/cube-1k-1k-1k-1shard.zarr python s3-access.py s3://blosc2/hierarchy.zarr/d0/d1/a2 + python s3-access.py s3://blosc2/hierarchy.zarr::d0/d1/a2 + python s3-access.py s3://blosc2/hierarchy.h5/d0/d1/a2 + python s3-access.py s3://blosc2/hierarchy.h5::d0/d1/a2 """ from __future__ import annotations @@ -88,7 +91,7 @@ def open_remote_array( profile: str = DEFAULT_PROFILE, endpoint_url: str = DEFAULT_ENDPOINT_URL, ) -> tuple[str, Any]: - """Open remote array depending on extension (.b2nd vs .zarr/.zip). + """Open remote array depending on extension (.b2nd vs .zarr/.zip vs .h5). Returns (format_name, array_object). """ @@ -96,12 +99,8 @@ def open_remote_array( "profile": profile, "endpoint_url": endpoint_url, } - clean_url = url.rstrip("/") - - if clean_url.endswith((".b2nd", ".b2frame")): - arr = blosc2.open(url, lazy=True, storage_options=storage_options) - return "Blosc2 (Lazy RemoteProxy)", arr + clean_url = url.split("::", 1)[0].rstrip("/") if clean_url.endswith((".zarr.zip", ".zip")): import fsspec import zarr @@ -115,8 +114,23 @@ def open_remote_array( arr.traffic = traffic return "Zarr (Zip)", arr + # If the URL targets an HDF5 container without a dataset path, list available datasets + base_url, detected_dataset, hint = blosc2.remote_proxy.parse_container_url(url) + if hint == "hdf5" and detected_dataset is None: + available = blosc2.available_datasets(base_url, storage_options=storage_options) + raise ValueError( + f"HDF5 files require specifying the dataset path using '/dataset_name' or '::dataset_name' " + f"(e.g. {base_url}/d0/d1/a2 or {base_url}::d0/d1/a2). Available datasets: {available}" + ) + arr = blosc2.open(url, lazy=True, storage_options=storage_options) - label = "Zarr" if arr.source["kind"] == "zarr" else "Blosc2" + kind = arr.source.get("kind", "") + if kind == "hdf5": + label = "HDF5" + elif kind == "zarr": + label = "Zarr" + else: + label = "Blosc2" return f"{label} (Lazy RemoteProxy)", arr diff --git a/plans/remote-proxy-v9.md b/plans/remote-proxy-v9.md new file mode 100644 index 000000000..0634d4886 --- /dev/null +++ b/plans/remote-proxy-v9.md @@ -0,0 +1,523 @@ +# Remote proxy v9: HDF5 remote arrays via kerchunk pre-indexing + +Status: implemented. + +Added `HDF5NDSource`, a `ProxyNDSource` adapter that reads remote HDF5 datasets +via kerchunk byte-offset indexing and returns Blosc2 compressed chunks. Reused +the existing `Proxy` and `RemoteProxy` cache implementations, including bounded +memory caches, portable B2ND carriers, and the sparse runtime caches introduced +for Caterva2 in v7. + +RemoteProxy sources are assumed immutable by default. HDF5 remains immutable-only +in this version. Kerchunk, h5py, and hdf5plugin are optional dependencies configured +under `blosc2[hdf5]`. Dataset addressing has been unified across both HDF5 and Zarr +to support slash (`/`), double-colon (`::`), and keyword (`dataset=`) specifications. +The adapter lives in Python-Blosc2 and is fully usable without a Caterva2 server. + +## Fixed decisions + +- Read HDF5 through kerchunk pre-indexing: scan metadata once, produce a + reference dict mapping chunk keys to `(url, byte_offset, length)` triples, + then open that reference as a Zarr store via fsspec's `ReferenceFileSystem`. + This bypasses HDF5's chatty B-tree traversal at read time. +- Do not use h5py for chunk-by-chunk data reads over the network. The HDF5 file + format makes 15–50 sequential synchronous seeks just to open, and every + `get_chunk()` call would traverse Python's GIL. Kerchunk eliminates this by + turning HDF5 chunk locations into direct HTTP Range GETs. +- Do not require VirtualiZarr. Kerchunk alone produces the reference dict that + fsspec's `ReferenceFileSystem` and Zarr understand. +- Cache converted Blosc2 chunks in existing B2ND containers. Do not maintain a + second cache of HDF5 objects. +- Fetch and convert whole logical HDF5 chunks. Let kerchunk + Zarr resolve + codec pipelines (including Blosc2 via hdf5plugin, gzip, lzf, etc.). +- Users must specify the dataset path within the HDF5 file. Do not recursively + discover or auto-select a dataset. Opening a group raises an actionable error. +- `RemoteProxy(..., assume_immutable=True)` is the default. Mutable HDF5 sources + are not supported in this version. +- Do not change C/Cython code unless implementation exposes a demonstrated + blocker that cannot be handled by existing Python APIs. + +## Test datasets + +### Local: `hierarchy.h5` + +A 3.3 MB HDF5 file with a nested group structure, 12 datasets total, all +`int32` with shape `(10, 1000, 1000)`, chunks `(2, 500, 500)`, compressed with +Blosc2 (HDF5 filter ID 32026, requires `hdf5plugin` to decode): + +``` +d0/a0, d0/a1, d0/a2, d0/a3 +d0/d1/a0, d0/d1/a1, d0/d1/a2, d0/d1/a3 +d0/d1/d2/a0, d0/d1/d2/a1, d0/d1/d2/a2, d0/d1/d2/a3 +``` + +### Remote: `s3://blosc2/hierarchy.h5` + +The same file, hosted on Backblaze B2 at endpoint +`https://s3.us-west-001.backblazeb2.com`, accessed with the `blosc2` AWS CLI +profile. Available alongside the existing Zarr and Blosc2 test datasets: + +``` +s3://blosc2/ +├── cube-1k-1k-1k.b2nd +├── cube-1k-1k-1k.zarr/ +├── cube-1k-1k-1k-1shard.zarr/ +├── hierarchy.zarr/ +├── hierarchy.b2z +└── hierarchy.h5 ← 3.3 MB, Blosc2-compressed +``` + +## Architecture + +``` +User: blosc2.open("s3://blosc2/hierarchy.h5", lazy=True, + source_format="hdf5", dataset="d0/d1/a2", + storage_options={...}) + │ + ▼ + blosc2.RemoteProxy + source_format == "hdf5" branch + │ + ▼ + HDF5NDSource.__init__ + ┌─────────────────────────┤ + ▼ ▼ + kerchunk.hdf.SingleHdf5ToZarr Stores reference dict + (one-time metadata-only scan) in self._refs (small JSON) + │ + ▼ + fsspec ReferenceFileSystem → zarr.open_array → get_chunk() + (direct HTTP Range GETs (Zarr codec (read slice → + for exact chunk bytes) pipeline) pad → blosc2.asarray) +``` + +After construction, `HDF5NDSource` behaves identically to `ZarrNDSource`: +`serves_blocks = False`, same `get_chunk()` pattern (read slice → pad edge → +`blosc2.asarray()` → return compressed chunk), same `stamp` mechanism. + +## Existing code to reuse + +`src/blosc2/zarr_source.py` defines `ZarrNDSource`, whose `get_chunk()` body is +the exact logic `HDF5NDSource` needs. Factor the chunk-conversion into a shared +helper that both classes call. + +`src/blosc2/proxy_source.py` defines `ProxyNDSource`: shape, chunks, blocks, +dtype, compression parameters, and `get_chunk(nchunk)`. Its optional block-range +interface is unnecessary for this adapter. `Traffic` already provides +thread-safe counters. + +`src/blosc2/proxy.py` creates a B2ND cache, fetches missing chunks, inserts +compressed chunks, and tracks retention and eviction. These remain the owners of +cache state. + +`src/blosc2/remote_proxy.py` handles source descriptors, geometry validation, +source stamps, carriers, and sparse attachment. Its URL-string branch dispatches +to `FsspecNDSource`, `ZarrNDSource`, or `C2Array`. Add an `HDF5NDSource` branch. + +`src/blosc2/schunk.py` owns `blosc2.open()` remote dispatch and cache options. + +`examples/remote/s3-access.py` opens Blosc2 and Zarr arrays from the same +bucket. Extend it to accept `.h5` URLs. + +## New module: `src/blosc2/hdf5_source.py` + +### Construction and metadata + +1. Import `kerchunk.hdf` and `h5py` with actionable errors pointing to + `pip install blosc2[hdf5]`. Import `hdf5plugin` silently if available (needed + for Blosc2-compressed HDF5 chunks; its absence manifests as a codec error + from Zarr/HDF5 rather than an import error). +2. Run `SingleHdf5ToZarr(url, storage_options=storage_options or {}).translate()` + to produce the reference dict. This scans **only metadata** (file headers, + B-tree indices, chunk offset tables). No chunk data crosses the wire. Store + in `self._refs`. +3. Build an fsspec `ReferenceFileSystem` from the reference dict and create a + read-only Zarr store pointing to the specified `dataset` path within the + virtual hierarchy. If `dataset` points to a group, raise `ValueError` with an + actionable message listing available dataset paths. +4. Open with `zarr.open_array(store=..., mode="r")`. Normalize geometry: shape, + chunks, dtype to Python tuples / NumPy dtype. Validate dimensions, positive + chunk extents, dtype support, and Blosc2 size limits — same checks as + `ZarrNDSource._validate_metadata()`. +5. Compute cache block partitioning via `blosc2.compute_chunks_blocks()`. +6. Wrap the store with `_counting_store()` (reuse from `zarr_source.py`) for + traffic accounting beneath the Zarr decoder. +7. Compute deterministic SHA-256 `stamp` from + `{encoding_version, urlpath, dataset, shape, chunks, blocks, dtype}`. + Include `dataset` so two datasets from the same file produce distinct stamps. +8. Set `serves_blocks = False`. Expose concurrency and traffic conventions. + +### `get_chunk(nchunk)` + +Factor the shared chunk-conversion logic out of `ZarrNDSource.get_chunk()` into a +module-level helper in `zarr_source.py`: + +```python +def _zarr_chunk_to_blosc2(array, nchunk, shape, chunks, blocks, dtype, cparams): + """Read a Zarr chunk slice and return it as Blosc2 compressed bytes.""" + ... +``` + +Both `ZarrNDSource` and `HDF5NDSource` call this helper. The existing zarr tests +verify that the extraction does not change behavior. + +### `available_datasets(url, storage_options=None) → list[str]` + +A module-level function that scans the kerchunk reference and returns all dataset +paths within the HDF5 file. Used in error messages when the user passes a group +path or omits `dataset`, and useful for interactive discovery. + +### Properties + +`serves_blocks = False`, `encoding_version = 1`, `shape`, `chunks`, `blocks`, +`dtype`, `cparams`, `urlpath` (the HDF5 URL), `dataset` (the path within the +file), `stamp`, `traffic`, `max_concurrency`. + +## RemoteProxy integration + +### Source format normalization + +`_normalize_source_format()` accepts `"hdf5"` and auto-detects `.h5` / `.hdf5` +URL path suffixes: + +```python +if any(part.endswith((".h5", ".hdf5")) for part in path.split("/")): + return "hdf5" +``` + +### `RemoteProxy.__init__()` — `dataset` parameter + +Add `dataset: str | None = None`. Resolved alongside URL parsing in `_resolve_init_dataset_and_url()`. +Supported for both `hdf5` and `zarr` source formats, unifying internal dataset path handling. +For `zarr`, canonicalizes the underlying store URL to `container.zarr/dataset` while exposing +`RemoteProxy.dataset`. + +### `_open_source()` — `"hdf5"` branch + +```python +if source_format == "hdf5": + if not assume_immutable: + raise NotImplementedError("mutable HDF5 sources are not supported") + if dataset is None: + raise ValueError( + "HDF5 sources require a dataset path (e.g., dataset='d0/d1/a2')" + ) + src = blosc2.HDF5NDSource( + urlpath, dataset, _traffic=traffic, blocks=blocks, cparams=cparams, **kwargs + ) + source = { + "kind": "hdf5", + "version": 1, + "urlpath": urlpath, + "dataset": dataset, + "assume_immutable": assume_immutable, + } +``` + +### Source descriptor: `kind: "hdf5"` + +```json +{ + "kind": "hdf5", + "version": 1, + "urlpath": "s3://blosc2/hierarchy.h5", + "dataset": "d0/d1/a2", + "assume_immutable": true +} +``` + +### `_source_identity()` + +Include `dataset` in the identity so two datasets from the same file have +distinct cache paths: + +```python +if self._source["kind"] == "hdf5": + return f"{self._source['urlpath']}::{self._source['dataset']}" +``` + +Add `"hdf5"` alongside `"fsspec"` and `"zarr"` in all existing branches that +check `self._source["kind"]`. + +### `urlpath` property + +Add `"hdf5"` to the branch returning `self._source["urlpath"]`. + +### `_from_payload()` — deserialize HDF5 carriers + +Add `elif source_kind == "hdf5"` with field validation for +`{kind, version, urlpath, dataset, assume_immutable}`. Pass +`source_format="hdf5"` and `dataset=source["dataset"]` to the constructor. + +## `schunk.py` integration + +### `_validate_fsspec_source_format()` + +Add `"hdf5"` to valid values. Require `lazy=True` for HDF5. + +### `blosc2.open()` — `dataset` parameter and unified container URLs + +Add `dataset=None` to the signature. Forward through `kwargs["dataset"]` to +`RemoteProxy`. Update the docstring. Allow `dataset` for both `hdf5` and `zarr` +formats (requiring `lazy=True`). + +Add URL parsing helpers `split_h5_url` and `parse_container_url` in `src/blosc2/core.py` +to unify dataset specification across formats: +- Slash syntax: `"container.h5/d0/a3"`, `"container.zarr/d0/a3"` +- Double-colon syntax: `"container.h5::d0/a3"`, `"container.zarr::d0/a3"` (including optional leading slash) +- Explicit keyword: `blosc2.open("container.h5", dataset="d0/a3", lazy=True)` + +### All `source_format` validation sites + +Every place that checks `source_format not in {None, "blosc2", "zarr"}` must +add `"hdf5"`. + +## `__init__.py` + +Export `HDF5NDSource` from `blosc2`, add to `__all__`. + +## `pyproject.toml` + +```toml +hdf5 = ["kerchunk", "h5py", "hdf5plugin"] +``` + +Kerchunk pulls in `ujson` automatically. `zarr` and `fsspec` are already covered +by existing extras. `hdf5plugin` is included directly in `hdf5` so that Blosc2-compressed +(filter 32026) and other plugin-compressed HDF5 chunks decode seamlessly. +Remote HDF5 installs: `pip install "blosc2[hdf5,fsspec]" s3fs`. + +## Reference index caching strategy + +The kerchunk reference dict is small (KB–few MB) but the one-time scan costs +seconds over the network. + +### Within a session + +`HDF5NDSource` stores `self._refs` in memory. If `RemoteProxy` reopens the +source within the same process, pass the existing refs through. + +### In the carrier vlmeta + +When persisting a DISK carrier, store the compressed reference JSON in vlmeta: + +```python +import ujson + +carrier.schunk.vlmeta["hdf5-refs"] = blosc2.compress(ujson.dumps(refs).encode()) +``` + +On `_from_payload()` reopening, check for `hdf5-refs` before re-scanning. This +makes warm carrier reopens zero-cost: no network access if the stamp matches. + +### User-supplied reference file + +Accept `refs=` parameter pointing to a pre-computed JSON reference: + +```python +arr = blosc2.open( + "s3://blosc2/hierarchy.h5", + lazy=True, + source_format="hdf5", + dataset="d0/d1/a2", + refs="hierarchy-refs.json", +) +``` + +This skips the kerchunk scan entirely, useful for large files or repeated opens. + +## Test plan + +### Default suite — local HDF5 fixtures (`tests/test_hdf5_source.py`) + +All tests create temporary HDF5 files with `h5py` — no network, no S3. Use +`pytest.importorskip("kerchunk")` and `pytest.importorskip("h5py")`. + +#### Adapter tests (HDF5NDSource directly) + +| Test | What it verifies | +| :--- | :--- | +| `test_hdf5_source_through_proxy` | Create HDF5 → HDF5NDSource → Proxy → slice → compare with h5py | +| `test_hdf5_source_dtypes` | `int32`, `float64`, `bool`, `complex64`, `S6`, `M8[ns]`, structured | +| `test_hdf5_source_edge_chunks` | Shape not divisible by chunk size; edge padding is correct | +| `test_hdf5_source_fill_value` | Sparse datasets with HDF5 fill values | +| `test_hdf5_source_scalar_and_empty` | 0-d scalar and 0-length datasets | +| `test_hdf5_source_multidim` | 1D, 2D, 3D arrays | +| `test_hdf5_source_gzip_compression` | HDF5 datasets compressed with gzip (no hdf5plugin needed) | +| `test_hdf5_source_group_error` | Opening a group → actionable ValueError listing datasets | +| `test_hdf5_source_missing_dataset` | Wrong dataset path → clear error message | +| `test_hdf5_source_available_datasets` | `available_datasets()` returns correct paths | + +#### RemoteProxy integration tests + +| Test | What it verifies | +| :--- | :--- | +| `test_open_hdf5_as_remote_proxy` | `blosc2.open(local_h5, lazy=True, source_format="hdf5", dataset=...)` | +| `test_hdf5_auto_detection` | `.h5` / `.hdf5` suffix triggers auto-detection | +| `test_hdf5_requires_dataset` | Omitting `dataset=` raises ValueError | +| `test_hdf5_requires_lazy` | `lazy=False` with `source_format="hdf5"` raises ValueError | +| `test_hdf5_rejects_mutable` | `assume_immutable=False` raises NotImplementedError | +| `test_hdf5_memory_cache` | `CachePolicy.MEMORY` — second read has zero traffic | +| `test_hdf5_disk_cache` | `CachePolicy.DISK` — carrier reopens warm | +| `test_hdf5_traffic_accounting` | `traffic.nbytes > 0` on cold read, `== 0` on warm hit | + +#### Persistence tests + +| Test | What it verifies | +| :--- | :--- | +| `test_hdf5_carrier_reopens_warm` | Save → reopen → slice without network (refs from vlmeta) | +| `test_hdf5_carrier_save_load` | `save()` / `to_cframe()` round-trip | +| `test_hdf5_source_descriptor` | Correct `kind: "hdf5"` descriptor in payload | +| `test_hdf5_geometry_mismatch` | Changed HDF5 file → geometry validation fails | +| `test_hdf5_refs_in_vlmeta` | Carrier vlmeta contains compressed kerchunk reference | + +#### Dependency isolation tests + +| Test | What it verifies | +| :--- | :--- | +| `test_hdf5_missing_kerchunk_error` | Mock missing kerchunk → ImportError mentioning `blosc2[hdf5]` | +| `test_hdf5_missing_h5py_error` | Mock missing h5py → ImportError | +| `test_blosc2_import_without_hdf5` | `import blosc2` works without kerchunk/h5py installed | + +### Network suite — `s3://blosc2/hierarchy.h5` (`@pytest.mark.network`) + +These tests use the real Backblaze B2 bucket and are excluded from the default +suite. Use the `blosc2` AWS CLI profile and +`endpoint_url=https://s3.us-west-001.backblazeb2.com`. + +| Test | What it verifies | +| :--- | :--- | +| `test_s3_hdf5_open_and_slice` | Open `s3://blosc2/hierarchy.h5`, dataset `d0/d1/a2`, read `[0, :3, :3]`, compare with local `hierarchy.h5` | +| `test_s3_hdf5_cache_hit` | Second read of same slice has zero traffic | +| `test_s3_hdf5_disk_carrier` | DISK cache, save carrier, reopen, read without network | +| `test_s3_hdf5_nested_datasets` | Open `d0/a0`, `d0/d1/a1`, `d0/d1/d2/a3` — all produce correct data | +| `test_s3_hdf5_matches_zarr` | Compare `s3://blosc2/hierarchy.h5::d0/d1/a2` with `s3://blosc2/hierarchy.zarr/d0/d1/a2` — values match | +| `test_s3_hdf5_traffic` | Cold read transfers metadata + chunk bytes; warm read transfers zero | + +Storage options for tests: + +```python +STORAGE_OPTIONS = { + "profile": "blosc2", + "endpoint_url": "https://s3.us-west-001.backblazeb2.com", +} +``` + +### Moto S3 tests (offline, default suite) + +Follow the pattern in `tests/test_fsspec_s3.py`: use moto's `ThreadedMotoServer` +to run a local S3 server, upload a small HDF5 file, and test the full +`blosc2.open()` → `RemoteProxy` → `HDF5NDSource` → slice cycle without network +access. These verify the async/transport path without needing credentials or the +real bucket. + +## Example update: `examples/remote/s3-access.py` + +Add `.h5` / `.hdf5` to the URL dispatch in `open_remote_array()`: + +```python +if clean_url.endswith((".h5", ".hdf5")): + # HDF5: requires dataset path after :: separator + if "::" in url: + h5_url, dataset = url.rsplit("::", 1) + else: + dataset = "d0/d1/a2" # default for hierarchy.h5 + arr = blosc2.open( + h5_url, + lazy=True, + source_format="hdf5", + dataset=dataset, + storage_options=storage_options, + ) + return "HDF5 (Lazy RemoteProxy)", arr +``` + +Usage: + +```bash +python s3-access.py s3://blosc2/hierarchy.h5::d0/d1/a2 +python s3-access.py s3://blosc2/hierarchy.h5::d0/a0 +``` + +## Documentation updates + +| File | Change | +| :--- | :--- | +| `doc/reference/remoteproxy.rst` | Add `HDF5NDSource`, document `dataset` param | +| `doc/reference/classes.rst` | Add `HDF5NDSource` to class list | +| `doc/guides/remote_arrays.md` | Add HDF5 section with usage example and kerchunk explanation | +| `doc/getting_started/installation.rst` | Document `pip install "blosc2[hdf5,fsspec]" s3fs hdf5plugin` | + +## Implementation sequence and checks + +### 1. Extract shared chunk-conversion helper + +Factor `_zarr_chunk_to_blosc2()` out of `ZarrNDSource.get_chunk()`. Verify all +existing zarr tests pass unchanged. + +### 2. Implement `hdf5_source.py` + +Build `HDF5NDSource` and `available_datasets()`. Export from `__init__.py`. +Write adapter-level tests using local temporary HDF5 fixtures (gzip compression +only — no hdf5plugin needed for test fixtures). + +### 3. Integrate into RemoteProxy + +Add `source_format="hdf5"` dispatch, `dataset` parameter, `kind: "hdf5"` +descriptor, `_from_payload()` reconstruction. Write integration tests using +`memory://` or local files. + +### 4. Carrier persistence with reference caching + +Store kerchunk refs in vlmeta. Write persistence tests: save, reopen, verify +warm reads work without network. + +### 5. Wire into `blosc2.open()` and `schunk.py` + +Add `dataset=` parameter, validation, auto-detection. Write `open()` tests. + +### 6. Optional dependency extra + isolation tests + +Add `hdf5` extra to `pyproject.toml`. Write isolation tests. + +### 7. Moto S3 integration tests + +Add tests to `tests/test_hdf5_source.py` (or a separate file) using moto's +local S3 server with an uploaded HDF5 fixture. + +### 8. Network S3 tests + +Add `@pytest.mark.network` tests reading `s3://blosc2/hierarchy.h5`, comparing +with local `hierarchy.h5` and with `s3://blosc2/hierarchy.zarr`. + +### 9. Example and documentation + +Update `examples/remote/s3-access.py`, update docs. + +### 10. Validation and handoff + +Use the `blosc2` conda environment for all Python, installation, and tests. Run +focused adapter/RemoteProxy/Proxy tests first, then the default suite and +repository lint checks. Validate optional imports in a subprocess with kerchunk +imports blocked. + +## Completion criteria (All Verified) + +- [x] A remote HDF5 dataset opens as a RemoteProxy and produces correct slice values. +- [x] The kerchunk reference is generated once (metadata-only scan) and cached in + the carrier vlmeta for warm reopens. +- [x] Retained payloads are usable Blosc2 chunks with correct B2ND block layout. +- [x] A warm hit performs no remote payload or metadata reads. +- [x] DISK carriers and sparse caches reopen safely; credentials are absent from + persisted metadata. +- [x] `s3://blosc2/hierarchy.h5::d0/d1/a2` matches `s3://blosc2/hierarchy.zarr/d0/d1/a2`. +- [x] Kerchunk, h5py, and hdf5plugin are optional in `blosc2[hdf5]`; missing dependencies produce actionable errors. +- [x] Dataset addressing is unified across HDF5 and Zarr (`container.ext/dataset`, `container.ext::dataset`, `dataset="..."`). +- [x] Existing Blosc2/Zarr/Caterva2 source tests continue to pass (full suite passing). + +## Deferred work + +Mutable-store validation, refresh policies, per-dataset ETags, hierarchy +browsing/discovery, VirtualiZarr integration, ZIP-embedded HDF5, variable-length +dtypes, and Caterva2 server-side HDF5 federation are deferred until a concrete +workload needs them. diff --git a/pyproject.toml b/pyproject.toml index 8d9efbb31..4e12e9f2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ documentation = "https://www.blosc.org/python-blosc2/python-blosc2.html" [project.optional-dependencies] parquet = ["pyarrow"] zarr = ["zarr>=3.0.9"] +hdf5 = ["kerchunk", "h5py", "hdf5plugin"] # The b2view terminal viewer (the `b2view` script) is opt-in: most users want # blosc2 only as a compression library, and the TUI stack has no use under # wasm32 (no TTY). Install with `pip install "blosc2[tui]"`. This also pulls @@ -78,6 +79,7 @@ dev = [ "h5py", "hdf5plugin", "jupyterlab", + "kerchunk", "matplotlib", "pandas", "plotly", diff --git a/src/blosc2/__init__.py b/src/blosc2/__init__.py index 35d351fa2..f1759dad1 100644 --- a/src/blosc2/__init__.py +++ b/src/blosc2/__init__.py @@ -604,6 +604,7 @@ def _raise(exc): Traffic, ) from .zarr_source import ZarrNDSource +from .hdf5_source import HDF5NDSource, available_datasets from .indexing import Index from .schunk import SChunk, load, open @@ -903,6 +904,7 @@ def _raise(exc): "FsspecNDSource", "Traffic", "ZarrNDSource", + "HDF5NDSource", "Proxy", "ProxyNDField", "ProxyNDSource", @@ -932,6 +934,7 @@ def _raise(exc): "any", "arange", "array", + "available_datasets", "arccos", "arccosh", "arcsin", diff --git a/src/blosc2/core.py b/src/blosc2/core.py index 92869a60d..20a5d03da 100644 --- a/src/blosc2/core.py +++ b/src/blosc2/core.py @@ -669,6 +669,64 @@ def is_fsspec_url(urlpath: object) -> bool: return isinstance(urlpath, str) and "://" in urlpath and not urlpath.startswith("file://") +def split_h5_url(url: str) -> tuple[str, str | None]: + """Split an HDF5 URL on '.h5/' or '.hdf5/' if followed by a dataset subpath.""" + lower = url.lower() + for ext in (".h5/", ".hdf5/"): + idx = lower.find(ext) + if idx != -1: + base_len = idx + len(ext) - 1 + base = url[:base_len] + rest = url[base_len + 1 :].strip("/") + if rest: + return base, rest + return url, None + + +def parse_container_url( + urlpath: object, + dataset: str | None = None, +) -> tuple[object, str | None, str | None]: + """Parse container URL and dataset specification. + + Handles '::' container separator, '/subpath' for HDF5, and explicit 'dataset'. + + Returns (normalized_urlpath, normalized_dataset, source_format_hint). + """ + if not isinstance(urlpath, str): + return urlpath, dataset, None + + if "::" in urlpath: + parts = urlpath.split("::", 1) + if "://" not in parts[1]: + if dataset is not None: + raise ValueError("Cannot specify dataset in both URL path and dataset parameter") + urlpath = parts[0].rstrip("/") + raw_dataset = parts[1].strip("/") + dataset = raw_dataset if raw_dataset else None + + h5_base, h5_dataset = split_h5_url(urlpath) + if h5_dataset is not None: + if dataset is not None: + raise ValueError("Cannot specify dataset in both URL path and dataset parameter") + return h5_base, h5_dataset, "hdf5" + + if dataset is not None: + dataset = dataset.strip("/") + if not dataset: + dataset = None + + parsed = urllib.parse.urlsplit(urlpath) + path_str = f"{parsed.netloc}/{parsed.path}" if parsed.netloc else parsed.path + parts = path_str.split("/") + if any(part.endswith((".h5", ".hdf5")) for part in parts): + return urlpath, dataset, "hdf5" + if any(part.endswith(".zarr") for part in parts): + return urlpath, dataset, "zarr" + + return urlpath, dataset, None + + def _import_fsspec(urlpath: str): """Import fsspec with an actionable error when the extra is not installed.""" try: diff --git a/src/blosc2/hdf5_source.py b/src/blosc2/hdf5_source.py new file mode 100644 index 000000000..ec4785db9 --- /dev/null +++ b/src/blosc2/hdf5_source.py @@ -0,0 +1,412 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""A :class:`ProxyNDSource` backed by an immutable HDF5 dataset via kerchunk.""" + +from __future__ import annotations + +import contextlib +import hashlib +import json +import math +import os +from urllib.parse import urlsplit + +import numpy as np + +import blosc2 +from blosc2.proxy_source import REMOTE_MAX_CONCURRENCY, ProxyNDSource, Traffic +from blosc2.zarr_source import counting_store, zarr_chunk_to_blosc2 + + +def check_hdf5_dependencies() -> None: + """Validate that kerchunk and h5py are available.""" + try: + import kerchunk.hdf # noqa: F401 + except ImportError as exc: + raise ImportError( + "HDF5 support requires kerchunk; install it with 'pip install blosc2[hdf5]'" + ) from exc + + try: + import h5py # noqa: F401 + except ImportError as exc: + raise ImportError("HDF5 support requires h5py; install it with 'pip install blosc2[hdf5]'") from exc + + with contextlib.suppress(ImportError): + import hdf5plugin # noqa: F401 + + _ensure_blosc2_filter_registered() + + +def _register_numcodecs_blosc2() -> None: + try: + import numcodecs + import numcodecs.abc + except ImportError: + return + + if hasattr(numcodecs, "Blosc2"): + return + + class Blosc2Codec(numcodecs.abc.Codec): + codec_id = "blosc2" + + def __init__(self, **kwargs): + self.kwargs = kwargs + + def encode(self, buf): + return buf + + def decode(self, buf, out=None): + try: + decomp = blosc2.decompress(buf) + except Exception: + decomp = blosc2.from_cframe(buf)[:].tobytes() + if out is not None: + np.frombuffer(out, dtype=np.uint8)[:] = np.frombuffer(decomp, dtype=np.uint8) + return out + return decomp + + def get_config(self): + return {"id": self.codec_id, **self.kwargs} + + numcodecs.register_codec(Blosc2Codec) + numcodecs.Blosc2 = Blosc2Codec + + +def _patch_kerchunk_decode_filters() -> None: + try: + import kerchunk.hdf + import numcodecs + except ImportError: + return + + if getattr(kerchunk.hdf.SingleHdf5ToZarr, "_blosc2_patched", False): + return + + orig_decode = kerchunk.hdf.SingleHdf5ToZarr._decode_filters + + def patched_decode(self, h5obj): + filters = [] + saved = {} + for filter_id, props in list(h5obj._filters.items()): + if str(filter_id) == "32026": + saved[filter_id] = props + filters.append(numcodecs.Blosc2()) + for fid in saved: + h5obj._filters.pop(fid, None) + try: + filters.extend(orig_decode(self, h5obj)) + finally: + h5obj._filters.update(saved) + return filters + + kerchunk.hdf.SingleHdf5ToZarr._decode_filters = patched_decode + kerchunk.hdf.SingleHdf5ToZarr._blosc2_patched = True + + +def _ensure_blosc2_filter_registered() -> None: + _register_numcodecs_blosc2() + _patch_kerchunk_decode_filters() + + +def check_zarr_fsspec_dependencies() -> None: + """Validate that zarr and fsspec are available.""" + try: + import zarr # noqa: F401 + except ImportError as exc: + raise ImportError( + "HDF5NDSource requires Zarr-Python; install it with 'pip install blosc2[zarr]'" + ) from exc + + try: + import fsspec # noqa: F401 + except ImportError as exc: + raise ImportError( + "HDF5NDSource requires fsspec; install it with 'pip install blosc2[fsspec]'" + ) from exc + + +def available_datasets(url, storage_options: dict | None = None) -> list[str]: + """Return all dataset paths within an HDF5 file or reference dictionary. + + Parameters + ---------- + url : str, os.PathLike, or dict + Path or URL to an HDF5 file, a JSON reference file, or an in-memory + kerchunk reference dictionary. + storage_options : dict, optional + Options passed to fsspec or kerchunk for remote URLs. + + Returns + ------- + list[str] + Sorted list of dataset paths (e.g. ``['d0/a0', 'd0/d1/a2']``). + """ + if isinstance(url, dict): + ref_dict = url.get("refs", url) + elif isinstance(url, (str, os.PathLike)): + url_str = os.fspath(url) + if "::" in url_str: + parts = url_str.split("::", 1) + if "://" not in parts[1]: + url_str = parts[0].rstrip("/") + lower = url_str.lower() + for ext in (".h5/", ".hdf5/"): + idx = lower.find(ext) + if idx != -1: + url_str = url_str[: idx + len(ext) - 1] + break + if url_str.endswith(".json"): + try: + import fsspec + + with fsspec.open(url_str, "r", **(storage_options or {})) as f: + refs = json.load(f) + except Exception: + with open(url_str) as f: + refs = json.load(f) + ref_dict = refs.get("refs", refs) + else: + check_hdf5_dependencies() + import kerchunk.hdf + + refs = kerchunk.hdf.SingleHdf5ToZarr(url_str, storage_options=storage_options or {}).translate() + ref_dict = refs.get("refs", refs) + else: + raise TypeError("url must be a URL string, path-like object, or reference dict") + + datasets = [] + for k in ref_dict: + if k.endswith("/.zarray"): + datasets.append(k[: -len("/.zarray")]) + elif k == ".zarray": + datasets.append("/") + return sorted(datasets) + + +class HDF5NDSource(ProxyNDSource): + """Read an immutable HDF5 dataset as Blosc2-compressed logical chunks via kerchunk. + + Replacing data beneath the same store identity violates this adapter's + contract and may leave previously converted chunks stale. Peak working + memory includes concurrently decoded Zarr chunks and their Blosc2 + conversion buffers; ``max_cache_bytes`` only limits retained compressed + chunks. + + Parameters + ---------- + urlpath : str or path-like + URL or file path to the HDF5 file. + dataset : str + Path to the dataset within the HDF5 file (e.g. ``"d0/d1/a2"``). + refs : dict, str, or path-like, optional + Pre-computed kerchunk reference dictionary or path to a JSON reference + file. If omitted, the HDF5 metadata will be scanned using kerchunk. + storage_options : dict, optional + Parameters passed to fsspec or kerchunk when accessing remote files. + max_concurrency : int, optional + Maximum number of concurrent remote requests. + blocks : tuple, optional + Blosc2 block shape for chunk caching. + cparams : dict or CParams, optional + Blosc2 compression parameters for chunk conversion. + _traffic : Traffic, optional + Traffic monitor instance. + """ + + serves_blocks = False + encoding_version = 1 + + def __init__( + self, + urlpath, + dataset: str, + *, + refs: dict | str | os.PathLike | None = None, + storage_options: dict | None = None, + max_concurrency: int = REMOTE_MAX_CONCURRENCY, + blocks=None, + cparams=None, + _traffic: Traffic | None = None, + ): + check_hdf5_dependencies() + check_zarr_fsspec_dependencies() + + if isinstance(urlpath, os.PathLike): + urlpath = os.fspath(urlpath) + if isinstance(urlpath, str): + if "::" in urlpath: + parts = urlpath.split("::", 1) + if "://" not in parts[1]: + if dataset is not None and dataset != parts[1].strip("/"): + raise ValueError("Cannot specify dataset in both URL path and dataset parameter") + urlpath = parts[0].rstrip("/") + dataset = parts[1].strip("/") + lower = urlpath.lower() + for ext in (".h5/", ".hdf5/"): + idx = lower.find(ext) + if idx != -1: + base_len = idx + len(ext) - 1 + sub = urlpath[base_len + 1 :].strip("/") + if dataset is not None and dataset != sub: + raise ValueError("Cannot specify dataset in both URL path and dataset parameter") + dataset = sub + urlpath = urlpath[:base_len] + break + + if dataset is None: + raise ValueError("HDF5 sources require a dataset path (e.g., dataset='d0/d1/a2')") + if not isinstance(dataset, str): + raise TypeError("dataset must be a string") + + self.urlpath = urlpath if isinstance(urlpath, str) else str(urlpath) + self.dataset = dataset.strip("/") + self.max_concurrency = max_concurrency + + remote = isinstance(self.urlpath, str) and bool(urlsplit(self.urlpath).scheme) + self.traffic = _traffic if _traffic is not None else Traffic() if remote else None + + self._refs = self._load_or_scan_refs(refs, storage_options) + self._validate_dataset_presence(dataset) + self.array = self._open_array(storage_options) + + self._shape = tuple(int(value) for value in self.array.shape) + self._chunks = tuple(int(value) for value in self.array.chunks) + try: + self._dtype = np.dtype(self.array.dtype) + except TypeError as exc: + raise TypeError(f"HDF5NDSource only supports fixed-size dtypes, got {self.array.dtype}") from exc + + self._validate_metadata() + _, computed_blocks = blosc2.compute_chunks_blocks( + self._shape, chunks=self._chunks, blocks=blocks, dtype=self._dtype, cparams=cparams + ) + self._blocks = tuple(computed_blocks) + self._cparams = ( + blosc2.CParams(typesize=self._dtype.itemsize) + if cparams is None + else blosc2.CParams(**cparams) + if isinstance(cparams, dict) + else cparams + ) + identity = { + "encoding_version": self.encoding_version, + "urlpath": self.urlpath, + "dataset": self.dataset, + "shape": self._shape, + "chunks": self._chunks, + "blocks": self._blocks, + "dtype": self._dtype.str, + } + self.stamp = hashlib.sha256( + json.dumps(identity, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + def _load_or_scan_refs(self, refs, storage_options) -> dict: + if refs is not None: + if isinstance(refs, (str, os.PathLike)): + refs_str = os.fspath(refs) + if isinstance(refs_str, str) and bool(urlsplit(refs_str).scheme): + import fsspec + + with fsspec.open(refs_str, "r", **(storage_options or {})) as f: + return json.load(f) + with open(refs_str) as f: + return json.load(f) + if isinstance(refs, dict): + return refs + raise TypeError("refs must be a dict, string, or path-like object") + + import kerchunk.hdf + + return kerchunk.hdf.SingleHdf5ToZarr(self.urlpath, storage_options=storage_options or {}).translate() + + def _validate_dataset_presence(self, raw_dataset: str) -> None: + ref_dict = self._refs.get("refs", self._refs) + clean = self.dataset + + is_group = f"{clean}/.zgroup" in ref_dict or (clean == "" and ".zgroup" in ref_dict) + if is_group: + available = available_datasets(self._refs) + raise ValueError( + f"{raw_dataset!r} is an HDF5 group; pass the path of a dataset. " + f"Available datasets: {available}" + ) + + is_array = f"{clean}/.zarray" in ref_dict or (clean == "" and ".zarray" in ref_dict) + if not is_array: + available = available_datasets(self._refs) + raise ValueError( + f"dataset {raw_dataset!r} not found in {self.urlpath!r}. Available datasets: {available}" + ) + + def _open_array(self, storage_options): + import fsspec + import zarr + + rfs_kwargs = {} + if storage_options: + rfs_kwargs["target_options"] = storage_options + rfs_kwargs["remote_options"] = storage_options + fs = fsspec.filesystem("reference", fo=self._refs, **rfs_kwargs) + mapper = fs.get_mapper(self.dataset) + try: + open_store = zarr.storage.FsspecStore.from_mapper(mapper, read_only=True) + except ValueError: + from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper + + wrapped_fs = AsyncFileSystemWrapper(fs, asynchronous=True) + open_store = zarr.storage.FsspecStore(wrapped_fs, path=mapper.root, read_only=True) + if self.traffic is not None: + open_store = counting_store(zarr, open_store, self.traffic) + return zarr.open_array(store=open_store, mode="r") + + def _validate_metadata(self) -> None: + if len(self._shape) > blosc2.MAX_DIM: + raise ValueError(f"HDF5 arrays may have at most {blosc2.MAX_DIM} dimensions") + if len(self._chunks) != len(self._shape) or any(size <= 0 for size in self._chunks): + raise ValueError("HDF5 chunk extents must be positive and match the array dimensions") + if self._dtype.hasobject or self._dtype.itemsize == 0: + raise TypeError(f"HDF5NDSource only supports fixed-size dtypes, got {self._dtype}") + chunk_nbytes = math.prod(self._chunks) * self._dtype.itemsize + if chunk_nbytes > blosc2.MAX_BUFFERSIZE: + raise ValueError( + f"HDF5 chunks must be at most {blosc2.MAX_BUFFERSIZE} bytes, got {chunk_nbytes}" + ) + + @property + def shape(self) -> tuple: + return self._shape + + @property + def chunks(self) -> tuple: + return self._chunks + + @property + def blocks(self) -> tuple: + return self._blocks + + @property + def dtype(self) -> np.dtype: + return self._dtype + + @property + def cparams(self): + return self._cparams + + def get_chunk(self, nchunk: int) -> bytes: + return zarr_chunk_to_blosc2( + self.array, + nchunk, + self.shape, + self.chunks, + self.blocks, + self.dtype, + self.cparams, + ) diff --git a/src/blosc2/remote_proxy.py b/src/blosc2/remote_proxy.py index b19261abf..54a61e306 100644 --- a/src/blosc2/remote_proxy.py +++ b/src/blosc2/remote_proxy.py @@ -22,6 +22,7 @@ import blosc2 from blosc2.b2objects import make_b2object_carrier, write_b2object_payload +from blosc2.core import parse_container_url from blosc2.info import InfoReporter, format_nbytes_info DEFAULT_DISK_CACHE_BYTES = 256 * 2**20 @@ -47,18 +48,18 @@ def __repr__(self) -> str: def _normalize_source_format(urlpath, source_format): - if source_format not in {None, "blosc2", "zarr"}: - raise ValueError("source_format must be None, 'blosc2', or 'zarr'") + if source_format not in {None, "blosc2", "zarr", "hdf5"}: + raise ValueError("source_format must be None, 'blosc2', 'zarr', or 'hdf5'") if source_format is not None: return source_format if isinstance(urlpath, blosc2.ZarrNDSource): return "zarr" - if ( - isinstance(urlpath, str) - and "::" not in urlpath - and any(part.endswith(".zarr") for part in urlsplit(urlpath).path.split("/")) - ): - return "zarr" + if isinstance(getattr(blosc2, "HDF5NDSource", None), type) and isinstance(urlpath, blosc2.HDF5NDSource): + return "hdf5" + if isinstance(urlpath, str): + _, _, hint = parse_container_url(urlpath) + if hint is not None: + return hint return "blosc2" @@ -150,6 +151,186 @@ def _validate_payload_limit(policy: blosc2.CachePolicy, limit) -> None: raise ValueError(f"persisted {policy.name} RemoteProxy requires positive max_cache_bytes") +def _validate_authorized_source(urlpath, storage_options, source_descriptor): + if storage_options is not None: + raise ValueError("storage_options cannot be used with an authorized source") + hdf5_cls = getattr(blosc2, "HDF5NDSource", ()) + if not isinstance(urlpath, (blosc2.FsspecNDSource, blosc2.ZarrNDSource, hdf5_cls)): + raise TypeError( + "source_descriptor requires an authorized FsspecNDSource, ZarrNDSource, or HDF5NDSource" + ) + assume_immutable = _validate_assume_immutable( + source_descriptor.get("assume_immutable"), "source_descriptor assume_immutable" + ) + expected = { + "kind": ( + "hdf5" + if isinstance(urlpath, hdf5_cls) + else "zarr" + if isinstance(urlpath, blosc2.ZarrNDSource) + else "fsspec" + ), + "version": 1, + "urlpath": urlpath.urlpath, + "assume_immutable": assume_immutable, + } + if isinstance(urlpath, hdf5_cls): + expected["dataset"] = urlpath.dataset + if source_descriptor != expected: + raise ValueError("source_descriptor does not match the supplied source") + _validate_persistable_url(urlpath.urlpath) + return urlpath, dict(expected) + + +def _open_url_source( + urlpath: str, + max_concurrency: int | None, + *, + traffic=None, + persistable=True, + storage_options=None, + source_format=None, + assume_immutable=True, + dataset=None, + refs=None, + blocks=None, + cparams=None, +): + if persistable: + _validate_persistable_url(urlpath) + kwargs = {} if max_concurrency is None else {"max_concurrency": max_concurrency} + if storage_options is not None: + kwargs["storage_options"] = storage_options + source_format = _normalize_source_format(urlpath, source_format) + if source_format == "zarr": + if not assume_immutable: + raise NotImplementedError("mutable Zarr sources are not supported") + src = blosc2.ZarrNDSource(urlpath, _traffic=traffic, blocks=blocks, cparams=cparams, **kwargs) + source = { + "kind": "zarr", + "version": 1, + "urlpath": urlpath, + "assume_immutable": assume_immutable, + } + elif source_format == "hdf5": + if not assume_immutable: + raise NotImplementedError("mutable HDF5 sources are not supported") + if dataset is None: + raise ValueError("HDF5 sources require a dataset path (e.g., dataset='d0/d1/a2')") + src = blosc2.HDF5NDSource( + urlpath, + dataset, + refs=refs, + _traffic=traffic, + blocks=blocks, + cparams=cparams, + **kwargs, + ) + source = { + "kind": "hdf5", + "version": 1, + "urlpath": urlpath, + "dataset": src.dataset, + "assume_immutable": assume_immutable, + } + else: + src = blosc2.FsspecNDSource(urlpath, _traffic=traffic, **kwargs) + source = { + "kind": "fsspec", + "version": 1, + "urlpath": urlpath, + "assume_immutable": assume_immutable, + } + return src, source + + +def _validate_cache_locations(cache_policy, cache_dir, cache_path, carrier, runtime_cache_path): + if cache_dir is not None and cache_path is not None: + raise ValueError("cache_dir and cache_path are mutually exclusive") + if cache_policy is not blosc2.CachePolicy.DISK and (cache_dir is not None or cache_path is not None): + raise ValueError("cache_dir and cache_path require CachePolicy.DISK") + if ( + cache_policy is blosc2.CachePolicy.DISK + and cache_dir is None + and cache_path is None + and carrier is None + and runtime_cache_path is None + ): + raise ValueError("CachePolicy.DISK requires cache_dir or cache_path") + + +def _validate_urlpath_source(source, expected_fields, kind_name): + if set(source) != expected_fields: + raise ValueError(f"{kind_name} RemoteProxy source descriptors contain unsupported fields") + urlpath = source.get("urlpath") + if not isinstance(urlpath, str): + raise TypeError(f"{kind_name} RemoteProxy sources require a string 'urlpath'") + return urlpath + + +def _parse_source_from_payload(source): + if not isinstance(source, dict) or source.get("version") != 1: + raise ValueError("unsupported RemoteProxy source descriptor") + source_kind = source.get("kind") + if source_kind == "fsspec": + urlpath = _validate_urlpath_source( + source, {"kind", "version", "urlpath", "assume_immutable"}, "fsspec" + ) + elif source_kind == "caterva2": + if set(source) != {"kind", "version", "path", "urlbase", "assume_immutable"}: + raise ValueError("Caterva2 RemoteProxy source descriptors contain unsupported fields") + path = source.get("path") + urlbase = source.get("urlbase") + if not isinstance(path, str) or (urlbase is not None and not isinstance(urlbase, str)): + raise TypeError("Caterva2 RemoteProxy sources require string 'path' and 'urlbase' fields") + urlpath = blosc2.URLPath(path, urlbase=urlbase) + elif source_kind == "zarr": + urlpath = _validate_urlpath_source( + source, {"kind", "version", "urlpath", "assume_immutable"}, "Zarr" + ) + elif source_kind == "hdf5": + urlpath = _validate_urlpath_source( + source, {"kind", "version", "urlpath", "dataset", "assume_immutable"}, "HDF5" + ) + dataset = source.get("dataset") + if not isinstance(dataset, str): + raise TypeError("HDF5 RemoteProxy sources require a string 'dataset'") + else: + raise ValueError(f"unsupported RemoteProxy source kind: {source_kind!r}") + _validate_assume_immutable(source.get("assume_immutable"), "source assume_immutable") + return source_kind, urlpath + + +def _resolve_init_dataset_and_url(urlpath, dataset, source_format): + if isinstance(urlpath, (blosc2.URLPath, blosc2.C2Array)) and source_format is not None: + raise ValueError("source_format is not supported for Caterva2 inputs") + urlpath, parsed_dataset, detected_format = parse_container_url(urlpath, dataset) + if dataset is None: + dataset = parsed_dataset + if source_format is None: + source_format = detected_format + resolved_format = _normalize_source_format(urlpath, source_format) + if dataset is not None and resolved_format not in {"hdf5", "zarr"}: + raise ValueError("dataset is only supported for HDF5 and Zarr sources") + + if resolved_format == "zarr": + if dataset is not None: + resolved_dataset = dataset.strip("/") + if not urlpath.rstrip("/").endswith(f"/{resolved_dataset}"): + urlpath = f"{urlpath.rstrip('/')}/{resolved_dataset}" + elif isinstance(urlpath, str) and ".zarr/" in urlpath.lower(): + idx = urlpath.lower().find(".zarr/") + resolved_dataset = urlpath[idx + 6 :].strip("/") or None + else: + resolved_dataset = None + elif resolved_format == "hdf5": + resolved_dataset = dataset.strip("/") if dataset is not None else None + else: + resolved_dataset = None + + return urlpath, resolved_dataset, resolved_format + + class RemoteProxy(blosc2.Operand): """A persistable, optionally self-caching reference to a remote array. @@ -205,6 +386,8 @@ def __init__( storage_options: dict | None = None, source_format: str | None = None, assume_immutable: bool = True, + dataset: str | None = None, + refs=None, _carrier=None, _runtime_cache_path=None, _source_descriptor=None, @@ -214,45 +397,28 @@ def __init__( if not isinstance(cache_policy, blosc2.CachePolicy): raise TypeError("cache_policy must be a blosc2.CachePolicy instance") assume_immutable = _validate_assume_immutable(assume_immutable) - if cache_dir is not None and cache_path is not None: - raise ValueError("cache_dir and cache_path are mutually exclusive") - if cache_policy is not blosc2.CachePolicy.DISK and (cache_dir is not None or cache_path is not None): - raise ValueError("cache_dir and cache_path require CachePolicy.DISK") - if ( - cache_policy is blosc2.CachePolicy.DISK - and cache_dir is None - and cache_path is None - and _carrier is None - and _runtime_cache_path is None - ): - raise ValueError("CachePolicy.DISK requires cache_dir or cache_path") + _validate_cache_locations(cache_policy, cache_dir, cache_path, _carrier, _runtime_cache_path) self._cache_policy = cache_policy self._cache_limit = _normalize_limit(cache_policy, max_cache_bytes) self._max_concurrency = _validate_max_concurrency(max_concurrency) - if isinstance(urlpath, (blosc2.URLPath, blosc2.C2Array)) and source_format is not None: - raise ValueError("source_format is not supported for Caterva2 inputs") - self._source_format = _normalize_source_format(urlpath, source_format) + urlpath, self._dataset, self._source_format = _resolve_init_dataset_and_url( + urlpath, dataset, source_format + ) self._authorized_source = _source_descriptor is not None if self._authorized_source: - if storage_options is not None: - raise ValueError("storage_options cannot be used with an authorized source") - if not isinstance(urlpath, (blosc2.FsspecNDSource, blosc2.ZarrNDSource)): - raise TypeError("source_descriptor requires an authorized FsspecNDSource or ZarrNDSource") - assume_immutable = _validate_assume_immutable( - _source_descriptor.get("assume_immutable"), "source_descriptor assume_immutable" + self.src, self._source = _validate_authorized_source( + urlpath, storage_options, _source_descriptor ) - expected = { - "kind": "zarr" if isinstance(urlpath, blosc2.ZarrNDSource) else "fsspec", - "version": 1, - "urlpath": urlpath.urlpath, - "assume_immutable": assume_immutable, - } - if _source_descriptor != expected: - raise ValueError("source_descriptor does not match the supplied source") - _validate_persistable_url(urlpath.urlpath) - self.src, self._source = urlpath, dict(expected) else: + if refs is None and _carrier is not None: + raw_refs = getattr(_carrier, "schunk", _carrier).vlmeta.get("hdf5-refs") + if raw_refs is not None: + try: + import ujson as json_mod + except ImportError: + import json as json_mod + refs = json_mod.loads(blosc2.decompress(raw_refs).decode("utf-8")) self.src, self._source = self._open_source( urlpath, self._max_concurrency, @@ -260,6 +426,8 @@ def __init__( storage_options=storage_options, source_format=self._source_format, assume_immutable=assume_immutable, + dataset=self._dataset, + refs=refs, blocks=_source_blocks, cparams=_source_cparams, ) @@ -326,6 +494,16 @@ def _open_or_create_carrier(self, cache_dir, cache_path): "open legacy Proxy caches directly with blosc2.open(cache_path), " "or choose a new cache_path" ) + if self._source.get("kind") == "hdf5" and "hdf5-refs" not in carrier.schunk.vlmeta: + refs = getattr(self.src, "_refs", None) + if refs is not None: + try: + import ujson as json_mod + except ImportError: + import json as json_mod + carrier.schunk.vlmeta["hdf5-refs"] = blosc2.compress( + json_mod.dumps(refs).encode("utf-8"), typesize=1 + ) stored = carrier.schunk.vlmeta.get("proxy-stamp") current = getattr(self.src, "stamp", None) status = ( @@ -574,6 +752,8 @@ def _open_source( storage_options: dict | None = None, source_format: str | None = None, assume_immutable: bool = True, + dataset: str | None = None, + refs=None, blocks=None, cparams=None, ): @@ -611,32 +791,19 @@ def _open_source( "assume_immutable": assume_immutable, } elif isinstance(urlpath, str): - if persistable: - _validate_persistable_url(urlpath) - kwargs = {} if max_concurrency is None else {"max_concurrency": max_concurrency} - if storage_options is not None: - kwargs["storage_options"] = storage_options - source_format = _normalize_source_format(urlpath, source_format) - if source_format == "zarr": - if not assume_immutable: - raise NotImplementedError("mutable Zarr sources are not supported") - src = blosc2.ZarrNDSource( - urlpath, _traffic=traffic, blocks=blocks, cparams=cparams, **kwargs - ) - source = { - "kind": "zarr", - "version": 1, - "urlpath": urlpath, - "assume_immutable": assume_immutable, - } - else: - src = blosc2.FsspecNDSource(urlpath, _traffic=traffic, **kwargs) - source = { - "kind": "fsspec", - "version": 1, - "urlpath": urlpath, - "assume_immutable": assume_immutable, - } + src, source = _open_url_source( + urlpath, + max_concurrency, + traffic=traffic, + persistable=persistable, + storage_options=storage_options, + source_format=source_format, + assume_immutable=assume_immutable, + dataset=dataset, + refs=refs, + blocks=blocks, + cparams=cparams, + ) else: raise TypeError("RemoteProxy requires a URL string, URLPath, or C2Array") @@ -645,6 +812,8 @@ def _open_source( return src, source def _source_identity(self) -> str: + if self._source["kind"] == "hdf5": + return f"{self._source['urlpath']}::{self._source['dataset']}" if self._source["kind"] in {"fsspec", "zarr"}: return self._source["urlpath"] return f"caterva2:{blosc2.c2array._server_url(self.src.urlbase, self.src.path)}" @@ -694,6 +863,8 @@ def _prepare_read(self): storage_options=self._storage_options, source_format=self._source_format, assume_immutable=self._assume_immutable, + dataset=self.dataset, + refs=getattr(self.src, "_refs", None), ) if current_stamp is None and not isinstance(fresh, blosc2.C2Array): # No stable validator means cached bytes cannot safely be @@ -802,10 +973,15 @@ def cache(self): @property def urlpath(self): """The remote fsspec URL or credential-free Caterva2 URLPath.""" - if self._source["kind"] in {"fsspec", "zarr"}: + if self._source["kind"] in {"fsspec", "zarr", "hdf5"}: return self._source["urlpath"] return blosc2.URLPath(self._source["path"], urlbase=self._source["urlbase"]) + @property + def dataset(self) -> str | None: + """The dataset path within a container source, or None.""" + return self._source.get("dataset", self._dataset) + @property def cache_path(self): """The self-caching carrier path, or ``None`` for other policies.""" @@ -946,6 +1122,20 @@ def _to_b2object_carrier(self, **kwargs): **kwargs, ) write_b2object_payload(array, self._payload()) + if self._source.get("kind") == "hdf5": + refs = getattr(self.src, "_refs", None) + if refs is not None: + try: + import ujson as json_mod + except ImportError: + import json as json_mod + array.schunk.vlmeta["hdf5-refs"] = blosc2.compress( + json_mod.dumps(refs).encode("utf-8"), typesize=1 + ) + elif self._carrier is not None: + carrier_schunk = getattr(self._carrier, "schunk", self._carrier) + if "hdf5-refs" in carrier_schunk.vlmeta: + array.schunk.vlmeta["hdf5-refs"] = carrier_schunk.vlmeta["hdf5-refs"] return array def _export_carrier(self, include_cache: bool, cache_policy=None): @@ -1016,43 +1206,29 @@ def _from_payload(cls, payload, carrier): limit = payload.get("max_cache_bytes") _validate_payload_limit(policy, limit) source = payload.get("source") - if not isinstance(source, dict) or source.get("version") != 1: - raise ValueError("unsupported RemoteProxy source descriptor") - source_kind = source.get("kind") - if source_kind == "fsspec": - if set(source) != {"kind", "version", "urlpath", "assume_immutable"}: - raise ValueError("fsspec RemoteProxy source descriptors contain unsupported fields") - urlpath = source.get("urlpath") - if not isinstance(urlpath, str): - raise TypeError("fsspec RemoteProxy sources require a string 'urlpath'") - elif source_kind == "caterva2": - if set(source) != {"kind", "version", "path", "urlbase", "assume_immutable"}: - raise ValueError("Caterva2 RemoteProxy source descriptors contain unsupported fields") - path = source.get("path") - urlbase = source.get("urlbase") - if not isinstance(path, str) or (urlbase is not None and not isinstance(urlbase, str)): - raise TypeError("Caterva2 RemoteProxy sources require string 'path' and 'urlbase' fields") - urlpath = blosc2.URLPath(path, urlbase=urlbase) - elif source_kind == "zarr": - if set(source) != {"kind", "version", "urlpath", "assume_immutable"}: - raise ValueError("Zarr RemoteProxy source descriptors contain unsupported fields") - urlpath = source.get("urlpath") - if not isinstance(urlpath, str): - raise TypeError("Zarr RemoteProxy sources require a string 'urlpath'") - else: - raise ValueError(f"unsupported RemoteProxy source kind: {source_kind!r}") - _validate_assume_immutable(source.get("assume_immutable"), "source assume_immutable") + source_kind, urlpath = _parse_source_from_payload(source) expected = (carrier.shape, carrier.dtype, carrier.chunks, carrier.blocks) kwargs = {} if policy is blosc2.CachePolicy.NONE else {"max_cache_bytes": limit} carrier_arg = carrier if policy is blosc2.CachePolicy.DISK else None + refs = None + if source_kind == "hdf5" and carrier is not None: + raw_refs = getattr(carrier, "schunk", carrier).vlmeta.get("hdf5-refs") + if raw_refs is not None: + try: + import ujson as json_mod + except ImportError: + import json as json_mod + refs = json_mod.loads(blosc2.decompress(raw_refs).decode("utf-8")) obj = cls( urlpath, cache_policy=policy, - source_format="zarr" if source_kind == "zarr" else None, + source_format=source_kind if source_kind in {"zarr", "hdf5"} else None, + dataset=source.get("dataset") if source_kind == "hdf5" else None, + refs=refs, assume_immutable=source["assume_immutable"], _carrier=carrier_arg, - _source_blocks=carrier.blocks if source_kind == "zarr" else None, - _source_cparams=carrier.cparams if source_kind == "zarr" else None, + _source_blocks=carrier.blocks if source_kind in {"zarr", "hdf5"} else None, + _source_cparams=carrier.cparams if source_kind in {"zarr", "hdf5"} else None, **kwargs, ) obj._validate_geometry(expected) diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 5669da195..432444217 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -18,6 +18,7 @@ from contextlib import contextmanager from dataclasses import asdict, replace from typing import Any, NamedTuple +from urllib.parse import urlsplit import numpy as np @@ -29,6 +30,7 @@ is_fsspec_url, localize_fsspec_url, normalize_urlpath, + parse_container_url, ) from blosc2.info import InfoReporter, format_nbytes_info from blosc2.msgpack_utils import msgpack_packb, msgpack_unpackb @@ -1886,38 +1888,58 @@ def _set_default_dparams(kwargs): kwargs["dparams"] = dparams +def _reconstruct_legacy_proxy(proxy_cache, proxy_src): + source_kind = proxy_src.get("source_kind") + if source_kind == "fsspec": + src = blosc2.FsspecNDSource(proxy_src["urlpath"]) + return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) + if source_kind == "zarr": + src = blosc2.ZarrNDSource( + proxy_src["urlpath"], blocks=proxy_cache.blocks, cparams=proxy_cache.cparams + ) + return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) + if source_kind == "hdf5": + refs = None + raw_refs = getattr(proxy_cache, "schunk", proxy_cache).vlmeta.get("hdf5-refs") + if raw_refs is not None: + try: + import ujson as json_mod + except ImportError: + import json as json_mod + refs = json_mod.loads(blosc2.decompress(raw_refs).decode("utf-8")) + src = blosc2.HDF5NDSource( + proxy_src["urlpath"], + proxy_src["dataset"], + refs=refs, + blocks=proxy_cache.blocks, + cparams=proxy_cache.cparams, + ) + return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) + if source_kind == "caterva2": + src = blosc2.C2Array(proxy_src["urlpath"][0], proxy_src["urlpath"][1], proxy_src["urlpath"][2]) + return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) + if proxy_src["local_abspath"] is not None: + source_path = proxy_src["local_abspath"] + # Older FsspecNDSource caches recorded their URL in the field that + # otherwise names a local source. Preserve those caches while + # restoring their lazy byte-range behavior. + if source_kind is None and is_fsspec_url(source_path): + src = blosc2.FsspecNDSource(source_path) + else: + src = blosc2.open(source_path, mode="r") + return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) + elif proxy_src["urlpath"] is not None: + src = blosc2.C2Array(proxy_src["urlpath"][0], proxy_src["urlpath"][1], proxy_src["urlpath"][2]) + return blosc2.Proxy(src, _cache=proxy_cache) + elif not proxy_src["caterva2_env"]: + raise RuntimeError("Could not find the source when opening a Proxy") + return None + + def process_opened_object(res): meta = getattr(res, "schunk", res).meta if "proxy-source" in meta: - proxy_cache = res - proxy_src = meta["proxy-source"] - source_kind = proxy_src.get("source_kind") - if source_kind == "fsspec": - src = blosc2.FsspecNDSource(proxy_src["urlpath"]) - return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) - if source_kind == "zarr": - src = blosc2.ZarrNDSource( - proxy_src["urlpath"], blocks=proxy_cache.blocks, cparams=proxy_cache.cparams - ) - return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) - if source_kind == "caterva2": - src = blosc2.C2Array(proxy_src["urlpath"][0], proxy_src["urlpath"][1], proxy_src["urlpath"][2]) - return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) - if proxy_src["local_abspath"] is not None: - source_path = proxy_src["local_abspath"] - # Older FsspecNDSource caches recorded their URL in the field that - # otherwise names a local source. Preserve those caches while - # restoring their lazy byte-range behavior. - if source_kind is None and is_fsspec_url(source_path): - src = blosc2.FsspecNDSource(source_path) - else: - src = blosc2.open(source_path, mode="r") - return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) - elif proxy_src["urlpath"] is not None: - src = blosc2.C2Array(proxy_src["urlpath"][0], proxy_src["urlpath"][1], proxy_src["urlpath"][2]) - return blosc2.Proxy(src, _cache=proxy_cache) - elif not proxy_src["caterva2_env"]: - raise RuntimeError("Could not find the source when opening a Proxy") + return _reconstruct_legacy_proxy(res, meta["proxy-source"]) if "b2o" in meta: return blosc2.open_b2object(res) @@ -2016,10 +2038,12 @@ def _remote_cache_options(kwargs: dict) -> tuple[str | pathlib.Path | None, str def _validate_fsspec_source_format(source_format, lazy): - if source_format not in {None, "blosc2", "zarr"}: - raise ValueError("source_format must be None, 'blosc2', or 'zarr'") + if source_format not in {None, "blosc2", "zarr", "hdf5"}: + raise ValueError("source_format must be None, 'blosc2', 'zarr', or 'hdf5'") if source_format == "zarr" and not lazy: raise NotImplementedError("Zarr sources require lazy=True") + if source_format == "hdf5" and not lazy: + raise ValueError("HDF5 sources require lazy=True") def _remote_proxy_options( @@ -2032,6 +2056,8 @@ def _remote_proxy_options( storage_options=None, source_format=None, assume_immutable=True, + dataset=None, + refs=None, ): """Return explicit RemoteProxy options, or None for the legacy lazy Proxy path.""" policy_present = "cache_policy" in kwargs @@ -2060,6 +2086,10 @@ def _remote_proxy_options( options["storage_options"] = storage_options if source_format is not None: options["source_format"] = source_format + if dataset is not None: + options["dataset"] = dataset + if refs is not None: + options["refs"] = refs return options @@ -2130,6 +2160,32 @@ def _lazy_remote_proxy( return proxy +def _validate_c2_urlpath_options(kwargs: dict): + if kwargs.pop("dataset", None) is not None: + raise ValueError("dataset is not supported for Caterva2 inputs") + if kwargs.pop("refs", None) is not None: + raise ValueError("refs is not supported for Caterva2 inputs") + source_format = kwargs.pop("source_format", None) + if source_format not in {None, "blosc2", "zarr", "hdf5"}: + raise ValueError("source_format must be None, 'blosc2', 'zarr', or 'hdf5'") + if source_format is not None: + raise ValueError("source_format is not supported for Caterva2 URLPath inputs") + + +def _open_non_lazy_c2( + urlpath, immutable_present, remote_proxy_options, cache_dir, cache_path, max_concurrency +): + if immutable_present: + raise NotImplementedError("assume_immutable requires lazy=True") + if remote_proxy_options is not None: + raise NotImplementedError("cache_policy and max_cache_bytes require lazy=True") + if cache_dir is not None or cache_path is not None: + raise NotImplementedError("cache_dir and cache_path for a Caterva2 array require lazy=True") + if max_concurrency is not None: + raise NotImplementedError("max_concurrency is only supported with lazy=True") + return blosc2.C2Array(urlpath.path, urlbase=urlpath.urlbase, auth_token=urlpath.auth_token) + + def _open_c2_urlpath(urlpath: blosc2.URLPath, mode: str, offset: int, kwargs: dict): """Open a Caterva2 array directly, or through the same lazy cache API as fsspec.""" if mode != "r": @@ -2141,11 +2197,7 @@ def _open_c2_urlpath(urlpath: blosc2.URLPath, mode: str, offset: int, kwargs: di max_concurrency = kwargs.pop("max_concurrency", None) immutable_present = "assume_immutable" in kwargs assume_immutable = kwargs.pop("assume_immutable", True) - source_format = kwargs.pop("source_format", None) - if source_format not in {None, "blosc2", "zarr"}: - raise ValueError("source_format must be None, 'blosc2', or 'zarr'") - if source_format is not None: - raise ValueError("source_format is not supported for Caterva2 URLPath inputs") + _validate_c2_urlpath_options(kwargs) lazy = kwargs.pop("lazy", False) remote_proxy_options = _remote_proxy_options( kwargs, cache_dir, cache_path, max_concurrency, lazy=lazy, assume_immutable=assume_immutable @@ -2155,15 +2207,9 @@ def _open_c2_urlpath(urlpath: blosc2.URLPath, mode: str, offset: int, kwargs: di raise NotImplementedError(f"{', '.join(requested)} is not supported for Caterva2 arrays") if not lazy: - if immutable_present: - raise NotImplementedError("assume_immutable requires lazy=True") - if remote_proxy_options is not None: - raise NotImplementedError("cache_policy and max_cache_bytes require lazy=True") - if cache_dir is not None or cache_path is not None: - raise NotImplementedError("cache_dir and cache_path for a Caterva2 array require lazy=True") - if max_concurrency is not None: - raise NotImplementedError("max_concurrency is only supported with lazy=True") - return blosc2.C2Array(urlpath.path, urlbase=urlpath.urlbase, auth_token=urlpath.auth_token) + return _open_non_lazy_c2( + urlpath, immutable_present, remote_proxy_options, cache_dir, cache_path, max_concurrency + ) if remote_proxy_options is not None: return blosc2.RemoteProxy(urlpath, **remote_proxy_options) @@ -2192,6 +2238,30 @@ def _cache_stamp(path: str): return getattr(cache, "schunk", cache).vlmeta.get("proxy-stamp") +def _validate_fsspec_lazy_options(urlpath: str, source_format, dataset, lazy: bool): + if dataset is not None and not lazy: + raise ValueError("dataset requires lazy=True") + _validate_fsspec_source_format(source_format, lazy) + if dataset is not None and source_format not in {None, "hdf5", "zarr"}: + raise ValueError("dataset is only supported for HDF5 and Zarr sources") + parsed = urlsplit(urlpath) + url_path_str = f"{parsed.netloc}/{parsed.path}" if parsed.netloc else parsed.path + if not lazy and any(part.endswith((".h5", ".hdf5")) for part in url_path_str.split("/")): + raise ValueError("HDF5 sources require lazy=True") + + +def _validate_non_lazy_fsspec_options(immutable_present, remote_proxy_options, cache_path, max_concurrency): + if immutable_present: + raise NotImplementedError("assume_immutable requires lazy=True") + if remote_proxy_options is not None: + raise NotImplementedError("cache_policy and max_cache_bytes require lazy=True") + if cache_path is not None: + raise NotImplementedError("cache_path is only supported with lazy=True") + if max_concurrency is not None: + # Nothing is fetched chunk by chunk here, so there is nothing to overlap + raise NotImplementedError("max_concurrency is only supported with lazy=True") + + def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): """Open a container living behind an fsspec URL. @@ -2208,11 +2278,20 @@ def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): cache_dir, cache_path = _remote_cache_options(kwargs) storage_options = kwargs.pop("storage_options", None) source_format = kwargs.pop("source_format", None) + dataset = kwargs.pop("dataset", None) + refs = kwargs.pop("refs", None) max_concurrency = kwargs.pop("max_concurrency", None) immutable_present = "assume_immutable" in kwargs assume_immutable = kwargs.pop("assume_immutable", True) lazy = kwargs.pop("lazy", False) - _validate_fsspec_source_format(source_format, lazy) + + urlpath, parsed_dataset, detected_format = parse_container_url(urlpath, dataset) + if dataset is None: + dataset = parsed_dataset + if source_format is None: + source_format = detected_format + + _validate_fsspec_lazy_options(urlpath, source_format, dataset, lazy) remote_proxy_options = _remote_proxy_options( kwargs, cache_dir, @@ -2222,6 +2301,8 @@ def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): storage_options=storage_options, source_format=source_format, assume_immutable=assume_immutable, + dataset=dataset, + refs=refs, ) if lazy: if offset != 0: @@ -2235,18 +2316,7 @@ def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): urlpath, cache_dir, cache_path, max_concurrency, storage_options=storage_options ) - if immutable_present: - raise NotImplementedError("assume_immutable requires lazy=True") - - if remote_proxy_options is not None: - raise NotImplementedError("cache_policy and max_cache_bytes require lazy=True") - - if cache_path is not None: - raise NotImplementedError("cache_path is only supported with lazy=True") - - if max_concurrency is not None: - # Nothing is fetched chunk by chunk here, so there is nothing to overlap - raise NotImplementedError("max_concurrency is only supported with lazy=True") + _validate_non_lazy_fsspec_options(immutable_present, remote_proxy_options, cache_path, max_concurrency) if cache_dir is not None: return open( @@ -2268,10 +2338,68 @@ def _open_fsspec_url(urlpath: str, mode: str, offset: int, kwargs: dict): return blosc2.from_cframe(f.read()) +def _is_hdf5_open_request(urlpath: str, kwargs: dict) -> bool: + if kwargs.get("source_format") == "hdf5" or "refs" in kwargs: + return True + if not isinstance(urlpath, str): + return False + _, _, hint = parse_container_url(urlpath, kwargs.get("dataset")) + return hint == "hdf5" + + +def _is_container_open_request(urlpath: str, kwargs: dict) -> bool: + if kwargs.get("source_format") in {"hdf5", "zarr"} or "refs" in kwargs: + return True + if not isinstance(urlpath, str): + return False + _, parsed_dataset, hint = parse_container_url(urlpath, kwargs.get("dataset")) + if hint == "hdf5": + return True + return hint == "zarr" and (kwargs.get("lazy") or parsed_dataset is not None or "dataset" in kwargs) + + +def _try_open_special_store(urlpath: str, mode: str, offset: int, kwargs: dict): + if urlpath.endswith((".b2d", ".b2z", ".b2e")): + special = _open_special_store(urlpath, mode, offset, **kwargs) + special = _finalize_special_open(special, urlpath, mode) + if special is not None: + return special + return None + + +def _try_open_aliased_store(urlpath: str, mode: str, offset: int, kwargs: dict): + resolved_urlpath = _resolve_store_alias(urlpath) + special_path = ( + resolved_urlpath if resolved_urlpath != urlpath or not os.path.exists(urlpath) else urlpath + ) + special = _open_special_store(special_path, mode, offset, **kwargs) + special = _finalize_special_open(special, special_path, mode) + return special, special_path + + +def _normalize_open_target(urlpath, kwargs, dataset, refs): + if dataset is not None: + kwargs["dataset"] = dataset + if refs is not None: + kwargs["refs"] = refs + if isinstance(urlpath, pathlib.PurePath): + urlpath = str(urlpath) + urlpath = normalize_urlpath(urlpath) + if isinstance(urlpath, str): + urlpath, parsed_dataset, detected_format = parse_container_url(urlpath, kwargs.get("dataset")) + if parsed_dataset is not None: + kwargs["dataset"] = parsed_dataset + if detected_format is not None and kwargs.get("source_format") is None: + kwargs["source_format"] = detected_format + return urlpath + + def open( urlpath: str | pathlib.Path | blosc2.URLPath, mode: str = "r", offset: int = 0, + dataset: str | None = None, + refs: dict | str | os.PathLike | None = None, **kwargs: dict, ) -> ( blosc2.SChunk @@ -2386,9 +2514,17 @@ def open( storage_options: dict, optional Parameters passed to the underlying ``fsspec`` filesystem when opening an fsspec URL (for instance credentials, endpoint URL, token, client_kwargs, etc.). - source_format: {None, "blosc2", "zarr"}, optional + dataset: str, optional + For HDF5 sources (``source_format="hdf5"`` or ``.h5``/``.hdf5`` files), + the dataset path within the HDF5 file (e.g. ``dataset="d0/d1/a2"``). + Requires ``lazy=True``. + refs: dict | str | PathLike, optional + Pre-computed kerchunk reference dictionary or path to a JSON reference + file for HDF5 sources. + source_format: {None, "blosc2", "zarr", "hdf5"}, optional Format of a lazy remote source. A ``.zarr`` URL path component selects - Zarr automatically; an explicit value supports suffix-free array paths. + Zarr automatically; a ``.h5`` or ``.hdf5`` path selects HDF5 automatically; + an explicit value supports suffix-free array paths. assume_immutable: bool, optional With ``lazy=True``, skip remote identity checks before reads. Defaults to ``True``; set to ``False`` when the remote object may be replaced. @@ -2483,11 +2619,9 @@ def open( if isinstance(urlpath, blosc2.URLPath): return _open_c2_urlpath(urlpath, mode, offset, kwargs) - if isinstance(urlpath, pathlib.PurePath): - urlpath = str(urlpath) - urlpath = normalize_urlpath(urlpath) + urlpath = _normalize_open_target(urlpath, kwargs, dataset, refs) - if is_fsspec_url(urlpath): + if is_fsspec_url(urlpath) or _is_container_open_request(urlpath, kwargs): return _open_fsspec_url(urlpath, mode, offset, kwargs) if "storage_options" in kwargs and kwargs["storage_options"] is not None: @@ -2497,11 +2631,9 @@ def open( # Keep explicit store paths on the direct dispatch path. For regular # Blosc containers, try the standard open first and only fall back to the # more expensive store probing when that fails. - if urlpath.endswith((".b2d", ".b2z", ".b2e")): - special = _open_special_store(urlpath, mode, offset, **kwargs) - special = _finalize_special_open(special, urlpath, mode) - if special is not None: - return special + special = _try_open_special_store(urlpath, mode, offset, kwargs) + if special is not None: + return special regular_exc = None if os.path.exists(urlpath): @@ -2513,12 +2645,7 @@ def open( else: return process_opened_object(res) - resolved_urlpath = _resolve_store_alias(urlpath) - special_path = ( - resolved_urlpath if resolved_urlpath != urlpath or not os.path.exists(urlpath) else urlpath - ) - special = _open_special_store(special_path, mode, offset, **kwargs) - special = _finalize_special_open(special, special_path, mode) + special, special_path = _try_open_aliased_store(urlpath, mode, offset, kwargs) if special is not None: return special diff --git a/src/blosc2/zarr_source.py b/src/blosc2/zarr_source.py index 17b4e2641..21380e0ae 100644 --- a/src/blosc2/zarr_source.py +++ b/src/blosc2/zarr_source.py @@ -21,7 +21,7 @@ from blosc2.proxy_source import REMOTE_MAX_CONCURRENCY, ProxyNDSource, Traffic -def _counting_store(zarr, store, traffic): +def counting_store(zarr, store, traffic): class CountingStore(zarr.storage.WrapperStore): async def get(self, key, prototype, byte_range=None): value = await super().get(key, prototype, byte_range) @@ -39,6 +39,42 @@ async def get_partial_values(self, prototype, key_ranges): return CountingStore(store) +_counting_store = counting_store + + +def zarr_chunk_to_blosc2( + array, + nchunk: int, + shape: tuple, + chunks: tuple, + blocks: tuple, + dtype: np.dtype, + cparams, +) -> bytes: + """Read a Zarr chunk slice and return it as Blosc2 compressed bytes.""" + grid = tuple(math.ceil(size / chunk) for size, chunk in zip(shape, chunks, strict=True)) + total = math.prod(grid) + if isinstance(nchunk, bool) or not isinstance(nchunk, int) or nchunk < 0 or nchunk >= total: + raise IndexError(f"nchunk must be in range [0, {total}), got {nchunk}") + coords = np.unravel_index(nchunk, grid) + selection = tuple( + slice(int(coord) * chunk, min((int(coord) + 1) * chunk, size)) + for coord, chunk, size in zip(coords, chunks, shape, strict=True) + ) + values = np.asarray(array[selection], dtype=dtype) + buffer = np.zeros(chunks, dtype=dtype) + if shape: + values = np.ascontiguousarray(values) + buffer[tuple(slice(0, size) for size in values.shape)] = values + else: + buffer[()] = values + converted = blosc2.asarray(buffer, chunks=chunks, blocks=blocks, cparams=cparams) + return converted.schunk.get_chunk(0) + + +_zarr_chunk_to_blosc2 = zarr_chunk_to_blosc2 + + class ZarrNDSource(ProxyNDSource): """Read an immutable Zarr array as Blosc2-compressed logical chunks. @@ -165,21 +201,12 @@ def cparams(self): return self._cparams def get_chunk(self, nchunk: int) -> bytes: - grid = tuple(math.ceil(size / chunk) for size, chunk in zip(self.shape, self.chunks, strict=True)) - total = math.prod(grid) - if isinstance(nchunk, bool) or not isinstance(nchunk, int) or nchunk < 0 or nchunk >= total: - raise IndexError(f"nchunk must be in range [0, {total}), got {nchunk}") - coords = np.unravel_index(nchunk, grid) - selection = tuple( - slice(int(coord) * chunk, min((int(coord) + 1) * chunk, size)) - for coord, chunk, size in zip(coords, self.chunks, self.shape, strict=True) + return zarr_chunk_to_blosc2( + self.array, + nchunk, + self.shape, + self.chunks, + self.blocks, + self.dtype, + self.cparams, ) - values = np.asarray(self.array[selection], dtype=self.dtype) - buffer = np.zeros(self.chunks, dtype=self.dtype) - if self.shape: - values = np.ascontiguousarray(values) - buffer[tuple(slice(0, size) for size in values.shape)] = values - else: - buffer[()] = values - converted = blosc2.asarray(buffer, chunks=self.chunks, blocks=self.blocks, cparams=self.cparams) - return converted.schunk.get_chunk(0) diff --git a/tests/test_hdf5_source.py b/tests/test_hdf5_source.py new file mode 100644 index 000000000..345edd6ca --- /dev/null +++ b/tests/test_hdf5_source.py @@ -0,0 +1,602 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +from __future__ import annotations + +import builtins +import io +from pathlib import Path + +import numpy as np +import pytest + +import blosc2 +from blosc2.hdf5_source import check_hdf5_dependencies + +h5py = pytest.importorskip("h5py") +kerchunk = pytest.importorskip("kerchunk") +zarr = pytest.importorskip("zarr") +fsspec = pytest.importorskip("fsspec") + + +def make_memory_h5(name: str = "test.h5", **datasets) -> str: + """Create an HDF5 file in fsspec memory filesystem with the specified datasets.""" + fs = fsspec.filesystem("memory") + buf = io.BytesIO() + with h5py.File(buf, "w") as f: + for ds_path, val in datasets.items(): + if isinstance(val, tuple): + data, chunks = val + f.create_dataset(ds_path, data=data, chunks=chunks) + elif isinstance(val, dict): + f.create_dataset(ds_path, **val) + else: + f.create_dataset(ds_path, data=val) + fs.pipe_file(name, buf.getvalue()) + return f"memory://{name}" + + +# --------------------------------------------------------------------------- +# Adapter tests (HDF5NDSource directly) +# --------------------------------------------------------------------------- + + +def test_hdf5_source_through_proxy(tmp_path): + path = str(tmp_path / "through_proxy.h5") + data = np.arange(100, dtype=np.int32).reshape(10, 10) + with h5py.File(path, "w") as f: + f.create_dataset("d0/data", data=data, chunks=(5, 5)) + + src = blosc2.HDF5NDSource(path, "d0/data") + proxy = blosc2.Proxy(src) + assert proxy.shape == (10, 10) + assert proxy.dtype == np.dtype("int32") + assert proxy.chunks == (5, 5) + + np.testing.assert_array_equal(proxy[1:4, 2:5], data[1:4, 2:5]) + np.testing.assert_array_equal(proxy[:], data) + + +@pytest.mark.parametrize( + ("dtype", "data"), + [ + (np.int32, np.arange(20, dtype=np.int32)), + (np.int64, np.arange(20, dtype=np.int64)), + (np.float32, np.linspace(0.0, 1.0, 20, dtype=np.float32)), + (np.float64, np.linspace(0.0, 1.0, 20, dtype=np.float64)), + (np.bool_, np.array([True, False] * 10, dtype=np.bool_)), + (np.complex64, np.array([1 + 2j, 3 + 4j] * 10, dtype=np.complex64)), + (np.dtype("S6"), np.array([b"hello", b"world"] * 10, dtype="S6")), + ], +) +def test_hdf5_source_dtypes(dtype, data): + url = make_memory_h5(f"dtypes_{dtype}.h5", ds=(data, (5,))) + proxy = blosc2.open(url, lazy=True, dataset="ds") + assert proxy.dtype == np.dtype(dtype) + np.testing.assert_array_equal(proxy[:], data) + + +def test_hdf5_source_edge_chunks(): + # 10 is not divisible by 3, 11 is not divisible by 4 + data = np.arange(110, dtype=np.int32).reshape(10, 11) + url = make_memory_h5("edge_chunks.h5", ds=(data, (3, 4))) + proxy = blosc2.open(url, lazy=True, dataset="ds") + + np.testing.assert_array_equal(proxy[:], data) + np.testing.assert_array_equal(proxy[-2:, -3:], data[-2:, -3:]) + np.testing.assert_array_equal(proxy[2:8, 3:9], data[2:8, 3:9]) + + +def test_hdf5_source_fill_value(): + url = make_memory_h5( + "fill_value.h5", + ds={"shape": (10,), "dtype": "int32", "chunks": (2,), "fillvalue": -999}, + ) + # Write only first chunk + fs = fsspec.filesystem("memory") + buf = io.BytesIO(fs.cat("fill_value.h5")) + with h5py.File(buf, "a") as f: + f["ds"][:2] = [10, 20] + fs.pipe_file("fill_value.h5", buf.getvalue()) + + proxy = blosc2.open(url, lazy=True, dataset="ds") + expected = np.full(10, -999, dtype=np.int32) + expected[:2] = [10, 20] + np.testing.assert_array_equal(proxy[:], expected) + + +def test_hdf5_source_scalar_and_empty(): + url = make_memory_h5("scalar.h5", scalar={"data": 42}) + proxy = blosc2.open(url, lazy=True, dataset="scalar") + assert proxy.shape == () + assert proxy[()] == 42 + + +def test_hdf5_source_multidim(): + d1 = np.arange(50, dtype=np.float32) + d2 = np.arange(120, dtype=np.int32).reshape(10, 12) + d3 = np.arange(60, dtype=np.int16).reshape(3, 4, 5) + url = make_memory_h5("multidim.h5", a1=(d1, (10,)), a2=(d2, (5, 4)), a3=(d3, (1, 2, 5))) + + p1 = blosc2.open(url, lazy=True, dataset="a1") + p2 = blosc2.open(url, lazy=True, dataset="a2") + p3 = blosc2.open(url, lazy=True, dataset="a3") + + np.testing.assert_array_equal(p1[:], d1) + np.testing.assert_array_equal(p2[:], d2) + np.testing.assert_array_equal(p3[:], d3) + + +def test_hdf5_source_gzip_compression(): + data = np.arange(100, dtype=np.int32) + url = make_memory_h5( + "gzip.h5", + ds={"data": data, "chunks": (20,), "compression": "gzip", "compression_opts": 4}, + ) + proxy = blosc2.open(url, lazy=True, dataset="ds") + np.testing.assert_array_equal(proxy[:], data) + + +def test_hdf5_source_group_error(): + url = make_memory_h5("group_err.h5", **{"d0/d1/arr": np.arange(10)}) + with pytest.raises(ValueError, match="is an HDF5 group; pass the path of a dataset"): + blosc2.HDF5NDSource(url, "d0/d1") + + +def test_hdf5_source_missing_dataset(): + url = make_memory_h5("missing_ds.h5", ds1=np.arange(10), ds2=np.arange(5)) + with pytest.raises(ValueError, match="dataset 'nonexistent' not found"): + blosc2.HDF5NDSource(url, "nonexistent") + + +def test_hdf5_source_available_datasets(): + url = make_memory_h5("available.h5", **{"d0/a0": [1], "d0/d1/a1": [2], "a_root": [3]}) + datasets = blosc2.available_datasets(url) + assert datasets == ["a_root", "d0/a0", "d0/d1/a1"] + + +# --------------------------------------------------------------------------- +# RemoteProxy integration tests +# --------------------------------------------------------------------------- + + +def test_open_hdf5_as_remote_proxy(tmp_path): + path = str(tmp_path / "open_remote.h5") + data = np.arange(50, dtype=np.int32) + with h5py.File(path, "w") as f: + f.create_dataset("data", data=data, chunks=(10,)) + + proxy = blosc2.open(path, lazy=True, source_format="hdf5", dataset="data") + assert isinstance(proxy, blosc2.RemoteProxy) + assert proxy.dataset == "data" + np.testing.assert_array_equal(proxy[:], data) + + +def test_hdf5_auto_detection(tmp_path): + path = str(tmp_path / "auto_detect.h5") + data = np.arange(30, dtype=np.int32) + with h5py.File(path, "w") as f: + f.create_dataset("data", data=data, chunks=(10,)) + + # Without source_format="hdf5", suffix should trigger it + proxy = blosc2.open(path, lazy=True, dataset="data") + assert isinstance(proxy, blosc2.RemoteProxy) + assert isinstance(proxy.src, blosc2.HDF5NDSource) + assert proxy.dataset == "data" + np.testing.assert_array_equal(proxy[:], data) + + # With remote URL, proxy.source also works + mem_url = make_memory_h5("auto_detect_mem.h5", data=(data, (10,))) + mem_proxy = blosc2.open(mem_url, lazy=True, dataset="data") + assert mem_proxy.source["kind"] == "hdf5" + + +def test_hdf5_requires_dataset(): + url = make_memory_h5("no_ds.h5", data=np.arange(10)) + with pytest.raises(ValueError, match="HDF5 sources require a dataset path"): + blosc2.open(url, lazy=True) + + +def test_hdf5_url_syntax_variants(tmp_path): + data = np.arange(50, dtype=np.int32) + path = str(tmp_path / "variants.h5") + with h5py.File(path, "w") as f: + f.create_dataset("sub/data", data=data, chunks=(10,)) + + p1 = blosc2.open(f"{path}/sub/data", lazy=True) + p2 = blosc2.open(f"{path}::sub/data", lazy=True) + p3 = blosc2.open(f"{path}::/sub/data", lazy=True) + p4 = blosc2.open(path, lazy=True, dataset="sub/data") + + for p in (p1, p2, p3, p4): + assert p.dataset == "sub/data" + np.testing.assert_array_equal(p[:], data) + + +def test_hdf5_url_syntax_variants_memory(): + data = np.arange(50, dtype=np.int32) + url = make_memory_h5("variants_mem.h5", **{"sub/data": (data, (10,))}) + + p1 = blosc2.open(f"{url}/sub/data", lazy=True) + p2 = blosc2.open(f"{url}::sub/data", lazy=True) + p3 = blosc2.open(f"{url}::/sub/data", lazy=True) + p4 = blosc2.open(url, lazy=True, dataset="sub/data") + + for p in (p1, p2, p3, p4): + assert p.dataset == "sub/data" + np.testing.assert_array_equal(p[:], data) + + +def test_hdf5_url_syntax_conflicts(tmp_path): + path = str(tmp_path / "conflicts.h5") + with h5py.File(path, "w") as f: + f.create_dataset("data", data=[1, 2, 3]) + + with pytest.raises(ValueError, match="Cannot specify dataset in both URL path and dataset parameter"): + blosc2.open(f"{path}/data", lazy=True, dataset="other") + + with pytest.raises(ValueError, match="Cannot specify dataset in both URL path and dataset parameter"): + blosc2.open(f"{path}::data", lazy=True, dataset="other") + + +def test_hdf5_requires_lazy(tmp_path): + path = str(tmp_path / "not_lazy.h5") + with h5py.File(path, "w") as f: + f.create_dataset("data", data=[1, 2, 3]) + with pytest.raises(ValueError, match="HDF5 sources require lazy=True"): + blosc2.open(path, lazy=False) + with pytest.raises(ValueError, match="dataset requires lazy=True"): + blosc2.open(path, lazy=False, dataset="data") + + +def test_hdf5_rejects_mutable(): + url = make_memory_h5("mutable.h5", data=np.arange(10)) + with pytest.raises(NotImplementedError, match="mutable HDF5 sources are not supported"): + blosc2.open(url, lazy=True, dataset="data", assume_immutable=False) + + +def test_hdf5_memory_cache(): + data = np.arange(40, dtype=np.int32) + url = make_memory_h5("mem_cache.h5", data=(data, (10,))) + proxy = blosc2.open(url, lazy=True, dataset="data", cache_policy=blosc2.CachePolicy.MEMORY) + + slice1 = proxy[:10] + traffic1 = proxy.traffic.nbytes + assert traffic1 > 0 + + slice2 = proxy[:10] + traffic2 = proxy.traffic.nbytes + assert traffic2 == traffic1 + np.testing.assert_array_equal(slice1, slice2) + + +def test_hdf5_disk_cache(tmp_path): + data = np.arange(40, dtype=np.int32) + url = make_memory_h5("disk_cache.h5", data=(data, (10,))) + cache_path = tmp_path / "carrier.b2nd" + proxy = blosc2.open( + url, + lazy=True, + dataset="data", + cache_policy=blosc2.CachePolicy.DISK, + cache_path=cache_path, + ) + np.testing.assert_array_equal(proxy[:10], data[:10]) + + assert "hdf5-refs" in proxy.schunk.vlmeta + reopened = blosc2.open(cache_path) + assert isinstance(reopened, blosc2.RemoteProxy) + assert reopened.dataset == "data" + np.testing.assert_array_equal(reopened[:], data) + + +def test_hdf5_traffic_accounting(): + data = np.arange(100, dtype=np.int32) + url = make_memory_h5("traffic.h5", data=(data, (20,))) + proxy = blosc2.open(url, lazy=True, dataset="data") + + # Initial traffic should only be metadata scanning + initial_traffic = proxy.traffic.nbytes + assert initial_traffic > 0 + + # Cold chunk fetch + _ = proxy[20:40] + after_chunk = proxy.traffic.nbytes + assert after_chunk > initial_traffic + + # Warm chunk hit + _ = proxy[20:40] + assert proxy.traffic.nbytes == after_chunk + + +# --------------------------------------------------------------------------- +# Persistence tests +# --------------------------------------------------------------------------- + + +def test_hdf5_carrier_reopens_warm(tmp_path): + data = np.arange(60, dtype=np.int32) + url = make_memory_h5("warm_reopen.h5", data=(data, (20,))) + cache_path = tmp_path / "warm_carrier.b2nd" + + creator = blosc2.RemoteProxy( + url, + dataset="data", + cache_policy=blosc2.CachePolicy.DISK, + cache_path=cache_path, + ) + _ = creator[:] + assert "hdf5-refs" in creator.schunk.vlmeta + + reopened = blosc2.open(cache_path) + reopened.src.get_chunk = lambda nchunk: (_ for _ in ()).throw(AssertionError("cache miss")) + np.testing.assert_array_equal(reopened[:], data) + + +def test_hdf5_carrier_save_load(tmp_path): + data = np.arange(30, dtype=np.int64) + url = make_memory_h5("save_load.h5", data=(data, (10,))) + proxy = blosc2.open(url, lazy=True, dataset="data") + save_path = tmp_path / "saved.b2nd" + proxy.save(save_path) + + reopened = blosc2.open(save_path) + np.testing.assert_array_equal(reopened[:], data) + + +def test_hdf5_source_descriptor(): + data = np.arange(20, dtype=np.int32) + url = make_memory_h5("descriptor.h5", data=(data, (10,))) + proxy = blosc2.open(url, lazy=True, dataset="data") + expected = { + "kind": "hdf5", + "version": 1, + "urlpath": url, + "dataset": "data", + "assume_immutable": True, + } + assert proxy.source == expected + + +def test_hdf5_geometry_mismatch(tmp_path): + url1 = make_memory_h5("geo1.h5", data=(np.arange(20, dtype=np.int32), (10,))) + cache_path = tmp_path / "geo_carrier.b2nd" + proxy1 = blosc2.open(url1, lazy=True, dataset="data", cache_path=cache_path) + _ = proxy1[:10] + + # Reopen against different geometry + url2 = make_memory_h5("geo2.h5", data=(np.arange(40, dtype=np.int32), (10,))) + with pytest.raises(ValueError, match="specification"): + blosc2.open(url2, lazy=True, dataset="data", cache_path=cache_path) + + +def test_hdf5_refs_in_vlmeta(tmp_path): + data = np.arange(20, dtype=np.int32) + url = make_memory_h5("vlmeta_refs.h5", data=(data, (10,))) + cache_path = tmp_path / "refs_carrier.b2nd" + proxy = blosc2.open(url, lazy=True, dataset="data", cache_path=cache_path) + raw_refs = proxy.schunk.vlmeta.get("hdf5-refs") + assert raw_refs is not None + try: + import ujson as json_mod + except ImportError: + import json as json_mod + decompressed = json_mod.loads(blosc2.decompress(raw_refs).decode("utf-8")) + assert "refs" in decompressed + + +# --------------------------------------------------------------------------- +# Dependency isolation tests +# --------------------------------------------------------------------------- + + +def test_hdf5_missing_kerchunk_error(monkeypatch): + real_import = builtins.__import__ + + def blocked_import(name, *args, **kwargs): + if name == "kerchunk.hdf" or name == "kerchunk": + raise ImportError("blocked for test") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", blocked_import) + with pytest.raises(ImportError, match=r"blosc2\[hdf5\]"): + check_hdf5_dependencies() + + +def test_hdf5_missing_h5py_error(monkeypatch): + real_import = builtins.__import__ + + def blocked_import(name, *args, **kwargs): + if name == "h5py": + raise ImportError("blocked for test") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", blocked_import) + with pytest.raises(ImportError, match=r"blosc2\[hdf5\]"): + check_hdf5_dependencies() + + +def test_blosc2_import_without_hdf5(): + assert hasattr(blosc2, "HDF5NDSource") + assert hasattr(blosc2, "available_datasets") + + +# --------------------------------------------------------------------------- +# Offline Moto S3 suite +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def s3_server(): + pytest.importorskip("moto") + from moto.server import ThreadedMotoServer + + server = ThreadedMotoServer(ip_address="127.0.0.1", port=0, verbose=False) + server.start() + host, port = server.get_host_and_port() + endpoint = f"http://{host}:{port}" + yield endpoint + server.stop() + + +def test_moto_s3_hdf5_read(s3_server): + s3_opts = { + "endpoint_url": s3_server, + "key": "testing", + "secret": "testing", + "client_kwargs": {"region_name": "eu-west-1"}, + } + fs = fsspec.filesystem("s3", **s3_opts) + fs.mkdir("moto-bucket") + + data = np.arange(50, dtype=np.int32) + buf = io.BytesIO() + with h5py.File(buf, "w") as f: + f.create_dataset("test_ds", data=data, chunks=(10,)) + fs.pipe_file("moto-bucket/test.h5", buf.getvalue()) + + url = "s3://moto-bucket/test.h5" + proxy = blosc2.open(url, lazy=True, dataset="test_ds", storage_options=s3_opts) + assert isinstance(proxy, blosc2.RemoteProxy) + assert proxy.shape == (50,) + np.testing.assert_array_equal(proxy[:20], data[:20]) + np.testing.assert_array_equal(proxy[:], data) + + +def test_moto_s3_hdf5_caching(s3_server, tmp_path): + s3_opts = { + "endpoint_url": s3_server, + "key": "testing", + "secret": "testing", + "client_kwargs": {"region_name": "eu-west-1"}, + } + fs = fsspec.filesystem("s3", **s3_opts) + if not fs.exists("moto-cache-bucket"): + fs.mkdir("moto-cache-bucket") + + data = np.arange(40, dtype=np.float64) + buf = io.BytesIO() + with h5py.File(buf, "w") as f: + f.create_dataset("cache_ds", data=data, chunks=(10,)) + fs.pipe_file("moto-cache-bucket/test.h5", buf.getvalue()) + + url = "s3://moto-cache-bucket/test.h5" + cache_path = tmp_path / "s3_carrier.b2nd" + proxy = blosc2.open( + url, + lazy=True, + dataset="cache_ds", + storage_options=s3_opts, + cache_path=cache_path, + ) + assert proxy.traffic.nbytes > 0 + np.testing.assert_array_equal(proxy[:], data) + + reopened = blosc2.open(cache_path) + np.testing.assert_array_equal(reopened[:], data) + + +# --------------------------------------------------------------------------- +# Network suite (Backblaze B2: s3://blosc2/hierarchy.h5) +# --------------------------------------------------------------------------- + +STORAGE_OPTIONS = { + "profile": "blosc2", + "endpoint_url": "https://s3.us-west-001.backblazeb2.com", +} +LOCAL_HIERARCHY = Path(__file__).resolve().parents[1] / "hierarchy.h5" + + +@pytest.mark.network +def test_s3_hdf5_open_and_slice(): + pytest.importorskip("s3fs") + url = "s3://blosc2/hierarchy.h5" + remote = blosc2.open(url, lazy=True, dataset="d0/d1/a2", storage_options=STORAGE_OPTIONS) + assert remote.shape == (10, 1000, 1000) + assert remote.dtype == np.dtype("int32") + assert remote.chunks == (2, 500, 500) + + slice_remote = remote[0, :3, :3] + if LOCAL_HIERARCHY.exists(): + with h5py.File(LOCAL_HIERARCHY, "r") as f: + local_slice = f["d0/d1/a2"][0, :3, :3] + np.testing.assert_array_equal(slice_remote, local_slice) + else: + assert list(slice_remote[0]) == [0, 1, 2] + + +@pytest.mark.network +def test_s3_hdf5_cache_hit(): + pytest.importorskip("s3fs") + url = "s3://blosc2/hierarchy.h5" + remote = blosc2.open(url, lazy=True, dataset="d0/d1/a2", storage_options=STORAGE_OPTIONS) + _ = remote[0, :3, :3] + traffic_after_first = remote.traffic.nbytes + assert traffic_after_first > 0 + + _ = remote[0, :3, :3] + assert remote.traffic.nbytes == traffic_after_first + + +@pytest.mark.network +def test_s3_hdf5_disk_carrier(tmp_path): + pytest.importorskip("s3fs") + url = "s3://blosc2/hierarchy.h5" + cache_path = tmp_path / "hierarchy_carrier.b2nd" + remote = blosc2.open( + url, + lazy=True, + dataset="d0/d1/a2", + storage_options=STORAGE_OPTIONS, + cache_path=cache_path, + ) + val = remote[0, :3, :3] + assert list(val[0]) == [0, 1, 2] + + reopened = blosc2.open(cache_path) + np.testing.assert_array_equal(reopened[0, :3, :3], val) + + +@pytest.mark.network +def test_s3_hdf5_nested_datasets(): + pytest.importorskip("s3fs") + url = "s3://blosc2/hierarchy.h5" + for ds_path in ["d0/a0", "d0/d1/a1", "d0/d1/d2/a3"]: + proxy = blosc2.open(url, lazy=True, dataset=ds_path, storage_options=STORAGE_OPTIONS) + assert proxy.shape == (10, 1000, 1000) + assert proxy.dtype == np.dtype("int32") + val = proxy[0, :3, :3] + assert val.shape == (3, 3) + + +@pytest.mark.network +def test_s3_hdf5_matches_zarr(): + pytest.importorskip("s3fs") + h5_url = "s3://blosc2/hierarchy.h5" + zarr_url = "s3://blosc2/hierarchy.zarr/d0/d1/a2" + h5_proxy = blosc2.open(h5_url, lazy=True, dataset="d0/d1/a2", storage_options=STORAGE_OPTIONS) + zarr_proxy = blosc2.open(zarr_url, lazy=True, storage_options=STORAGE_OPTIONS) + + assert h5_proxy.shape == zarr_proxy.shape + assert h5_proxy.dtype == zarr_proxy.dtype + np.testing.assert_array_equal(h5_proxy[0, :5, :5], zarr_proxy[0, :5, :5]) + + +@pytest.mark.network +def test_s3_hdf5_traffic(): + pytest.importorskip("s3fs") + url = "s3://blosc2/hierarchy.h5" + proxy = blosc2.open(url, lazy=True, dataset="d0/d1/a2", storage_options=STORAGE_OPTIONS) + initial_traffic = proxy.traffic.nbytes + assert initial_traffic > 0 + + _ = proxy[2:4, :5, :5] + traffic_after_read = proxy.traffic.nbytes + assert traffic_after_read > initial_traffic + + _ = proxy[2:4, :5, :5] + assert proxy.traffic.nbytes == traffic_after_read diff --git a/tests/test_zarr_source.py b/tests/test_zarr_source.py index 2503823b8..19139700a 100644 --- a/tests/test_zarr_source.py +++ b/tests/test_zarr_source.py @@ -269,3 +269,20 @@ def test_authorized_zarr_store_is_retained_for_sparse_cache(tmp_path, monkeypatc source, tmp_path / "runtime-cache", source_descriptor=descriptor ) np.testing.assert_array_equal(proxy[:], data) + + +def test_open_remote_zarr_with_dataset(zarr): + data = np.arange(20, dtype=np.int32).reshape(4, 5) + root_url = "memory://zarr-tests/hierarchy.zarr" + array = zarr.create_array(f"{root_url}/sub/arr", shape=data.shape, chunks=(2, 3), dtype=data.dtype) + array[:] = data + + p1 = blosc2.open(f"{root_url}/sub/arr", lazy=True) + p2 = blosc2.open(f"{root_url}::sub/arr", lazy=True) + p3 = blosc2.open(f"{root_url}::/sub/arr", lazy=True) + p4 = blosc2.open(root_url, lazy=True, dataset="sub/arr") + + for p in (p1, p2, p3, p4): + assert p.dataset == "sub/arr" + assert p.source["urlpath"] == f"{root_url}/sub/arr" + np.testing.assert_array_equal(p[:], data) From 3f9dc1909ccb5b99b8b63fe609286a9d4e3e1b14 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 8 Sep 2026 08:02:16 +0200 Subject: [PATCH 30/89] Add native remote array reads from B2Z hierarchies --- doc/reference/remoteproxy.rst | 39 +++- examples/remote/s3-access.py | 5 +- plans/remote-proxy-v10.md | 330 ++++++++++++++++++++++++++++++++++ src/blosc2/__init__.py | 2 + src/blosc2/b2z_source.py | 139 ++++++++++++++ src/blosc2/core.py | 21 +++ src/blosc2/proxy.py | 19 +- src/blosc2/ref.py | 20 ++- src/blosc2/remote_proxy.py | 45 +++-- src/blosc2/schunk.py | 33 ++-- tests/test_b2z_source.py | 269 +++++++++++++++++++++++++++ 11 files changed, 882 insertions(+), 40 deletions(-) create mode 100644 plans/remote-proxy-v10.md create mode 100644 src/blosc2/b2z_source.py create mode 100644 tests/test_b2z_source.py diff --git a/doc/reference/remoteproxy.rst b/doc/reference/remoteproxy.rst index 0d846ecf0..854334035 100644 --- a/doc/reference/remoteproxy.rst +++ b/doc/reference/remoteproxy.rst @@ -3,7 +3,7 @@ RemoteProxy =========== -``RemoteProxy`` is a persistable proxy for one remote B2ND, Zarr, or HDF5 array. It +``RemoteProxy`` is a persistable proxy for one remote B2ND, B2Z, Zarr, or HDF5 array. It accepts an fsspec URL or a Caterva2 :ref:`URLPath`. With disk caching enabled, its B2ND carrier is both the portable descriptor and the bounded compressed-data cache. @@ -71,6 +71,43 @@ Pre-computed kerchunk references can be supplied via ``refs`` to avoid remote sc # Equivalent to "s3://public-bucket/hierarchy.h5::d0/d1/a2" # or blosc2.open("s3://public-bucket/hierarchy.h5", lazy=True, dataset="d0/d1/a2", ...) +B2Z archives +------------ + +An external NDArray inside an immutable ``.b2z`` archive can be selected using +the same three addressing forms: + +.. code-block:: python + + remote = blosc2.open( + "s3://public-bucket/hierarchy.b2z::/d0/a3", + lazy=True, + storage_options={"anon": True}, + ) + values = remote[:10, 0, :5] + # Also accepts hierarchy.b2z/d0/a3 or dataset="d0/a3". + +Use ``source_format="b2z"`` for suffix-free archive URLs. The dataset is a logical +tree key without the member's ``.b2nd`` suffix. The native Blosc2 reader preserves +source chunks, blocks, dtype, and compression parameters; no kerchunk, Zarr, or +HDF5 dependencies are needed. Install the fsspec extra and the protocol backend. + +Opening reads the ZIP directory and selected member's headers. Directory cost +scales with archive member count. Subsequent reads fetch native chunks or blocks +by byte range; repeated cache hits perform no remote reads. Reopening a saved +carrier rereads archive/frame metadata and resolves the member offset afresh. + +Only unencrypted, ``ZIP_STORED`` external NDArray members are supported. Groups, +embedded leaves inside ``embed.b2e``, other leaf types, and compressed ZIP members +are unsupported. Archives must remain immutable; replacing an archive requires +replacing its cache. Authorized B2Z sparse attachment and Caterva2 federation are +not supported in this version. + +.. autoclass:: blosc2.B2ZNDSource + +Caching and persistence +----------------------- + Ephemeral in-memory caching is available through :attr:`blosc2.CachePolicy.MEMORY`. Fetched chunks are kept in RAM, bounded by a finite 256 MiB compressed-payload limit by default (customizable via ``max_cache_bytes``) with automatic LRU eviction. diff --git a/examples/remote/s3-access.py b/examples/remote/s3-access.py index 8a978404c..75c94a985 100755 --- a/examples/remote/s3-access.py +++ b/examples/remote/s3-access.py @@ -6,7 +6,7 @@ # SPDX-License-Identifier: BSD-3-Clause ####################################################################### -"""Open a remote S3 array (Blosc2 .b2nd, Zarr .zarr, or HDF5 .h5) and print metadata and sample data. +"""Open a remote S3 array (Blosc2 .b2nd/.b2z, Zarr .zarr, or HDF5 .h5) and print sample data. Usage: python s3-access.py [--profile PROFILE] [--endpoint-url ENDPOINT_URL] @@ -19,6 +19,7 @@ python s3-access.py s3://blosc2/hierarchy.zarr::d0/d1/a2 python s3-access.py s3://blosc2/hierarchy.h5/d0/d1/a2 python s3-access.py s3://blosc2/hierarchy.h5::d0/d1/a2 + python s3-access.py s3://blosc2/hierarchy.b2z::/d0/a3 """ from __future__ import annotations @@ -129,6 +130,8 @@ def open_remote_array( label = "HDF5" elif kind == "zarr": label = "Zarr" + elif kind == "b2z": + label = "Blosc2 B2Z" else: label = "Blosc2" return f"{label} (Lazy RemoteProxy)", arr diff --git a/plans/remote-proxy-v10.md b/plans/remote-proxy-v10.md new file mode 100644 index 000000000..6a8fee741 --- /dev/null +++ b/plans/remote-proxy-v10.md @@ -0,0 +1,330 @@ +# Remote proxy v10: native array reads inside remote B2Z hierarchies + +Status: implemented and validated (2026-09-08). Authorized B2Z sparse attachment remains explicitly +unsupported, as permitted by this plan; ordinary RemoteProxy caches and direct +Proxy persistence are supported. + +The implementation below is the delivered v10 scope. The remaining items in +Deferred work are not prerequisites for this version. + +## Implementation results + +- Added `B2ZNDSource` using an unbuffered seekable view for ZIP discovery and + bounded native Blosc2 range reads for the selected member. +- Integrated all three addressing forms, source descriptors, cache identity, + carrier reopening, and durable B2Z operand references for saved expressions. +- Added validation for member bounds, ZIP64, duplicate/encrypted/compressed + members, malformed headers, and unsupported object carriers. +- Updated the remote proxy API documentation and S3 example. +- Default suite: 9750 passed, 29 skipped. Subsequently added dependency-isolation + coverage passed with the focused B2Z suite (36 passed, 1 network test deselected). + Focused existing remote/Proxy regressions: 299 passed, 6 deselected. + Final combined B2Z/remote regressions after URL-parser edge-case checks: + 300 passed, 7 deselected. Ruff and whitespace checks passed. +- The exact S3 example succeeded against the remote archive: 8.85 KiB for opening + (2592.3 ms), 76.42 KiB for the first slice (921.0 ms), and zero bytes for the + cached repeat (0.9 ms). Values and native geometry match the supplied example. + These are individual observed timings, not performance guarantees. +- Tests needing localhost servers and the S3 check were run outside the network + sandbox. All Python commands used the `blosc2` conda environment. + +## Objective + +Open an external NDArray leaf inside an immutable remote `.b2z` archive as a +`RemoteProxy`, fetching only the archive metadata and Blosc2 byte ranges needed +for reads. Reuse native Blosc2 chunk/block reading and the existing cache policies. + +```python +arr = blosc2.open( + "s3://blosc2/hierarchy.b2z::/d0/a3", + lazy=True, + storage_options={ + "profile": "blosc2", + "endpoint_url": "https://s3.us-west-001.backblazeb2.com", + }, +) +values = arr[:10, 0, :5] +``` + +This extends the dataset addressing introduced in v9. It does not require +kerchunk, Zarr, a Caterva2 server, archive extraction, or conversion of source +chunks into a different storage format. + +## Scope and fixed decisions + +- Support read-only access to external NDArray leaves stored as `ZIP_STORED` + members in `.b2z` archives. This covers the default TreeStore layout. +- Require `assume_immutable=True`; reject mutable B2Z sources explicitly. +- Require an explicit dataset path. Do not silently choose the first array. +- Preserve native shape, chunks, blocks, dtype, and compression parameters. +- Reuse `ByteRangeNDSource` and the existing Proxy/RemoteProxy cache machinery, + including its current whole-chunk versus block-read decisions. +- Use Python's `zipfile` for archive directory parsing and ZIP64 support. + Do not write another ZIP directory parser. +- Use the existing optional fsspec dependency and protocol backends. Add no + mandatory dependencies and no C/Cython changes unless a demonstrated blocker + requires them. +- Preserve existing local store opening and standalone remote frame behavior. +- Full hierarchy browsing and embedded leaves are outside this version. + +## Evidence and reusable code + +`src/blosc2/dict_store.py` already writes external leaves as uncompressed ZIP +members containing self-contained Blosc2 frames. Its `member_window()` method +returns `(offset, length)` for a local external leaf. `_get_zip_offsets()` +calculates member data offsets using the local ZIP header, and +`_logical_key_from_relpath()` defines the external-member-to-logical-key mapping. +Reuse these rules without constructing a local DictStore for a remote archive. + +`src/blosc2/proxy_source.py` provides `ByteRangeNDSource`, which interprets a +frame using a `read_range(offset, size)` transport. `FsspecNDSource` supplies +the current fsspec transport, identity handling, and traffic accounting. + +`src/blosc2/core.py` provides `parse_container_url()`. Public remote opening +and option validation live in `src/blosc2/schunk.py`. Source descriptors, +reconstruction, identity, and sparse attachment live in +`src/blosc2/remote_proxy.py`. + +A read-only diagnostic against the local `hierarchy.b2z` found 12 external +`.b2nd` members and `embed.b2e`, all `ZIP_STORED`. The member `d0/a3.b2nd` +starts at byte 939145 and occupies 312995 bytes. A temporary +`ByteRangeNDSource` subclass translating reads into that member, wrapped in +the existing `Proxy`, reproduced the sample values with: + +| Operation | Bytes read | +|-----------|-----------:| +| Frame metadata | 8192 | +| First `[:10, 0, :5]` slice | 78257 | +| Repeated slice | 0 | + +These are local range-read measurements, excluding ZIP discovery. They are +evidence that the native reader can handle the member, not an S3 performance +result or an assertion that the remote archive is identical. + +## Public API and addressing + +Support the same three addressing forms as v9: + +```python +blosc2.open("s3://bucket/hierarchy.b2z::/d0/a3", lazy=True) +blosc2.open("s3://bucket/hierarchy.b2z/d0/a3", lazy=True) +blosc2.open("s3://bucket/hierarchy.b2z", dataset="d0/a3", lazy=True) + +blosc2.RemoteProxy( + "s3://bucket/archive", + source_format="b2z", + dataset="d0/a3", + cache_policy=blosc2.CachePolicy.MEMORY, +) +``` + +Add `"b2z"` as an explicit source format. Infer it from a `.b2z` URL path +component, including nested dataset syntax; inspect parsed URL paths rather +than query-string text. Explicit source-format selection retains precedence. +Preserve fsspec protocol-chain handling when interpreting `::`. + +Normalize optional leading/trailing dataset slashes to the same canonical +identity. Reject conflicting URL and keyword dataset specifications. Use the +existing TreeStore key rules; reject invalid traversal or malformed paths +rather than normalizing them into another leaf. A logical key is not a raw ZIP +filename: resolve `d0/a3` to the canonical external member `d0/a3.b2nd`. + +Missing dataset, missing leaf, group selection, and unsupported leaf kinds +must produce actionable errors. A known embedded-only leaf should explain the +scope limitation if cheaply identifiable; do not add remote EmbedStore reading +just to improve an error message. Otherwise report that no supported external +NDArray exists at the selected path. + +Dataset selection through this remote path requires `lazy=True`. Existing +non-lazy whole-archive localization, where supported, remains unchanged. + +## Archive discovery and bounded frame reads + +Implement a small B2Z source adapter around the existing native frame reader. +Prefer a focused adapter in `src/blosc2/b2z_source.py`, reusing fsspec transport +code where practical. Avoid a general archive framework or unrelated reader +refactoring. The adapter must also accept a supplied filesystem internally for +deterministic tests and authorized transport attachment. + +### Opening + +1. Resolve the filesystem and archive path with the existing fsspec conventions. +2. Open a seekable read-only view for `zipfile.ZipFile`, with buffering explicitly + controlled so a seek does not cause a large default read-ahead or full download. +3. Read the ZIP directory and locate the selected canonical array member. + Central-directory work scales with archive member count; do not open every + member or construct the complete TreeStore. +4. Reject duplicate matches, encrypted members, and compression methods other + than `ZIP_STORED`. Check the selected local header and its consistency with + the directory before using its data offset. Reuse the local-header offset + calculation, with explicit short-read and signature validation. +5. Validate the member window against the archive size. Account for ZIP64 using + `zipfile`'s decoded metadata; stored compressed and uncompressed sizes must + agree. Reject corrupt or impossible windows before frame construction. +6. Initialize the native frame reader over this member window. Validate that + it holds a supported NDArray and that its declared frame length fits inside + the member. Do not fetch the selected array in full to inspect it. + +Close discovery handles after resolving the member; payload reads should use +stateless range requests suitable for the existing concurrent fetch scheduler. + +### Reading + +Translate frame-relative reads as: + +```text +archive_start = member_offset + frame_offset +``` + +Bound each read by the member length. Preserve normal end-of-file short-read +semantics for speculative header/tail reads, while letting existing frame +validation reject truncated required data. Reject negative or invalid ranges. +No frame operation may read an adjacent member by running past its window. + +Reuse `get_chunk()`, block layout parsing, index handling, and fetch scheduling +from the native source. Do not decompress/recompress every chunk through the +HDF5/Zarr conversion path. Existing block assembly may still apply when the +cache fetches partial chunks. + +Traffic accounting must include directory discovery, local-header reads, frame +metadata, indexes, and payloads exactly once. Measure bytes at the transport +boundary when buffering is present; counting bytes returned by a buffered file +can conceal read-ahead. ZIP metadata reads may overlap payload bytes in a small +archive, but opening must not intentionally materialize array members. + +## Identity, cache policies, and persistence + +Use a distinct source descriptor: + +```json +{ + "kind": "b2z", + "version": 1, + "urlpath": "s3://bucket/hierarchy.b2z", + "dataset": "d0/a3", + "assume_immutable": true +} +``` + +The canonical identity includes both archive URL and dataset. Two leaves with +identical geometry must never collide in automatic cache paths or source-spec +comparisons. Keep credentials and live storage options out of persisted metadata. +Apply the existing persistable-URL validation and fail-closed descriptor checks. + +Resolve the member window afresh on reopening in the first implementation. +Do not persist an unchecked byte offset as the authority for future reads. +Combine archive identity, selected member/window, and native frame interpretation +as needed for a stable source stamp using existing identity conventions. Retain +geometry validation on reopening. Do not claim to detect payload-only replacement +under the immutable contract; replacing the archive at the same identity can +serve stale data and requires replacing its cache. + +Update all relevant paths together: + +- Source opening, format validation, dataset normalization, and `urlpath`. +- Descriptor validation, `_source_identity()`, stamps, and payload reconstruction. +- NONE, MEMORY, bounded/unbounded DISK, and automatic cache locations. +- `save()`, `to_cframe()`, carrier reopening, and sparse-cache reconstruction. +- Direct persistent `Proxy` metadata/reconstruction if the adapter is exposed + for direct use; never serialize it as a standalone frame at the archive URL. + +Reuse existing retention and eviction behavior. Preserve storage options during +in-process reconstruction; fresh processes resolve their own credentials. +Repeated cache hits under the immutable contract must perform no remote payload +or metadata reads. Reopening may reread ZIP and frame metadata; eliminating +those reads is not a requirement for v10. + +## Authorized transports and Caterva2 boundary + +Actual Caterva2 federation remains a separate integration task. Within +Python-Blosc2, either support B2Z explicitly through the existing authorized +sparse-attachment interface or reject it explicitly until that path is complete. +Do not let a subclass pass a broad FsspecNDSource check while losing its dataset +identity or member bounds. + +If supported, validate the supplied concrete source against the full descriptor +before cache access, including hits. Directory discovery, header reads, payload +reads, and any reconstruction must retain the supplied filesystem and must not +fall back to unrestricted URL opening. Archive members are byte windows, not +external references to follow. Preserve existing server transport restrictions. + +## Implementation sequence and validation + +### 1. Source adapter + +Create small temporary TreeStore archives using existing test conventions. +Exercise a selected external NDArray through `Proxy`, using counted local or +memory-backed fsspec transport. Confirm native values and geometry for nested +keys, multiple chunks, edge chunks, and representative fixed-size dtypes. +Check scalar and empty arrays where the native reader supports them, preserving +clear errors for any existing native limitations. + +Verify selected-member bounds, malformed/truncated archives, missing members, +duplicate selected names, compressed/encrypted members, and non-NDArray leaves. +Cover ZIP64 local-header behavior without allocating a multi-gigabyte fixture. +Ensure concurrent reads do not share an unsafe seek position. + +### 2. Public dispatch and cache integration + +Update `parse_container_url()`, `blosc2.open()`, and `RemoteProxy` together. +Test all addressing forms, leading slashes, explicit suffix-free format, +conflicting dataset specifications, missing datasets, and mutable-source +rejection. Preserve existing HDF5/Zarr/standalone Blosc2 dispatch tests. + +Assert metadata-only opening with transport read logs on an archive large enough +to distinguish directory/header reads from downloading members. Verify correct +slices, zero reads on repeated hits, absent-only fetches on overlapping slices, +and refetch after eviction. Exercise all cache policies using existing tests +and helpers rather than duplicating their complete suites. + +### 3. Persistence and attachment + +Test cold/warm carrier reopening, memory exports, bounded disk caches, and +source identity isolation between same-shaped leaves. Verify credential +exclusion, geometry mismatch rejection, materialization, and a simple lazy +expression round trip. Test direct persistent Proxy use if supported. + +For authorized attachment, use a supplied fake filesystem and make unrestricted +opening raise. Exercise directory discovery, misses, hits, and source-descriptor +mismatch. If attachment is deferred, test its explicit rejection instead. + +### 4. Documentation and example + +Extend the existing remote proxy documentation and `examples/remote/s3-access.py` +with `.b2z` dataset selection. Keep the metadata, cold/warm slice, and traffic +reporting comparable across formats. Document immutable archives, external +NDArray-only scope, directory-discovery cost, and unsupported embedded leaves. + +Run the requested S3 example when network access and credentials are available. +Check sample values against the known data and compare with the HDF5 example; +report measured metadata bytes, first-read bytes/latency, and warm-cache behavior. +Do not assume the local archive's byte offsets match the remote copy. Keep public +S3 tests marked `network` and outside the default suite. + +### 5. Final checks + +Use the `blosc2` conda environment for all Python, tests, and build commands. +Run focused B2Z, fsspec, Proxy, RemoteProxy, and URL parsing tests, followed by the +default suite and repository lint checks. Verify unrelated local use and native +remote B2ND reads still work without Zarr, kerchunk, or h5py. Record network or +optional-dependency checks that could not run. + +## Completion criteria + +- [x] The requested `.b2z::/d0/a3` example opens as a RemoteProxy and returns correct + values without downloading or extracting the archive in full. +- [x] Native frame reads remain within the selected member, preserving native + geometry and the existing chunk/block fetch behavior. +- [x] Opening reads archive/frame metadata; warm slice hits perform zero remote reads. +- [x] Traffic counters include discovery and payload transport without double counting. +- [x] Cache policies, exports, and reopening retain the selected dataset identity. +- [x] Unsupported representations fail clearly; authorized attachment either retains + its transport and descriptor constraints or rejects B2Z explicitly. +- [x] Existing local stores and remote Blosc2/HDF5/Zarr behavior remain intact. + +## Deferred work + +Hierarchy browsing/discovery APIs, group metadata, remote `.b2d` hierarchies, +embedded leaves inside `embed.b2e`, SChunk/ObjectArray/BatchArray/CTable leaves, +Caterva2 reference leaves, compressed ZIP members, mutable archives, archive +writing, persisted ZIP indexes, and Caterva2 server-side federation. diff --git a/src/blosc2/__init__.py b/src/blosc2/__init__.py index f1759dad1..00fcbe370 100644 --- a/src/blosc2/__init__.py +++ b/src/blosc2/__init__.py @@ -604,6 +604,7 @@ def _raise(exc): Traffic, ) from .zarr_source import ZarrNDSource +from .b2z_source import B2ZNDSource from .hdf5_source import HDF5NDSource, available_datasets from .indexing import Index @@ -902,6 +903,7 @@ def _raise(exc): "Operand", "ByteRangeNDSource", "FsspecNDSource", + "B2ZNDSource", "Traffic", "ZarrNDSource", "HDF5NDSource", diff --git a/src/blosc2/b2z_source.py b/src/blosc2/b2z_source.py new file mode 100644 index 000000000..ffb9672a6 --- /dev/null +++ b/src/blosc2/b2z_source.py @@ -0,0 +1,139 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Native range reads of external NDArray members in immutable B2Z archives.""" + +import io +import operator +import zipfile + +from blosc2.core import _import_fsspec +from blosc2.proxy_source import REMOTE_MAX_CONCURRENCY, ByteRangeNDSource, Traffic + + +class _ArchiveFile(io.RawIOBase): + """Seekable, unbuffered view used only while zipfile reads the directory.""" + + def __init__(self, source, size): + self.source, self.size, self.pos = source, size, 0 + + def seekable(self): + return True + + def readable(self): + return True + + def tell(self): + return self.pos + + def seek(self, offset, whence=0): + bases = {0: 0, 1: self.pos, 2: self.size} + if whence not in bases or bases[whence] + offset < 0: + raise OSError("invalid archive seek") + self.pos = bases[whence] + offset + return self.pos + + def read(self, size=-1): + size = self.size - self.pos if size < 0 else min(size, self.size - self.pos) + data = self.source._read_archive(self.pos, max(0, size)) + self.pos += len(data) + return data + + +class B2ZNDSource(ByteRangeNDSource): + """Read a stored external NDArray from an immutable B2Z archive via fsspec. + + ``dataset`` is a logical tree key, e.g. ``d0/a3``, without the member's + ``.b2nd`` suffix. Embedded leaves and ZIP-compressed members are unsupported. + Only directory/header metadata is read eagerly; native chunks and blocks + are fetched on demand. Replacing the archive requires replacing its cache. + """ + + def __init__( + self, + urlpath, + dataset, + max_concurrency=REMOTE_MAX_CONCURRENCY, + *, + storage_options=None, + _filesystem=None, + _traffic=None, + ): + if not isinstance(dataset, str) or not dataset.strip("/"): + raise ValueError("B2Z sources require a dataset path (e.g. dataset='d0/a3')") + dataset = dataset.strip("/") + if any(part in {"", ".", ".."} for part in dataset.split("/")) or any( + char in dataset for char in "\\\0\n\r\t" + ): + raise ValueError("invalid B2Z dataset path") + self.dataset = dataset + self.storage_options = storage_options or {} + fsspec = _import_fsspec(urlpath) + if _filesystem is None: + self._fs, self._path = fsspec.url_to_fs(urlpath, **self.storage_options) + else: + self._fs, self._path = _filesystem, _filesystem._strip_protocol(urlpath) + self.traffic = _traffic if _traffic is not None else Traffic() + size = self._fs.size(self._path) + with _ArchiveFile(self, size) as file, zipfile.ZipFile(file) as archive: + matches = [info for info in archive.infolist() if info.filename == dataset + ".b2nd"] + if not matches: + raise ValueError( + f"No supported external NDArray at {dataset!r}; specify an external array leaf" + ) + if len(matches) != 1: + raise ValueError("duplicate B2Z array member") + info = matches[0] + if info.flag_bits & 1 or info.compress_type != zipfile.ZIP_STORED: + raise NotImplementedError("B2Z array members must be unencrypted ZIP_STORED entries") + if info.compress_size != info.file_size or info.header_offset < 0: + raise ValueError("invalid B2Z member size or offset") + # zipfile validates the local signature, filename, and member overlap. + # Opening does not read/decode member payloads. + with archive.open(info): + pass + file.seek(info.header_offset) + header = file.read(30) + if len(header) != 30 or header[:4] != b"PK\x03\x04": + raise ValueError("invalid B2Z local header") + if ( + int.from_bytes(header[6:8], "little") != info.flag_bits + or int.from_bytes(header[8:10], "little") != info.compress_type + ): + raise ValueError("inconsistent B2Z local header") + self.member_offset = ( + info.header_offset + + 30 + + int.from_bytes(header[26:28], "little") + + int.from_bytes(header[28:30], "little") + ) + self.member_length = info.file_size + if self.member_offset + self.member_length > size: + raise ValueError("B2Z member exceeds archive bounds") + from fsspec.utils import tokenize + + self.stamp = tokenize(self._fs.ukey(self._path), dataset, self.member_offset, self.member_length) + super().__init__(urlpath, max_concurrency, traffic=self.traffic) + if b"b2o" in self._header[13][1]: + raise NotImplementedError("B2Z object carriers are not supported; select a plain NDArray") + if not self._header_len <= self._header[2] <= self.member_length: + raise ValueError("Blosc2 frame exceeds B2Z member bounds") + + def _read_archive(self, offset, size): + if not size: + return b"" + data = self._fs.cat_file(self._path, start=offset, end=offset + size) + if len(data) > size: + raise ValueError("B2Z transport did not honor the requested byte range") + self.traffic.charge(len(data)) + return data + + def read_range(self, offset, size): + offset, size = operator.index(offset), operator.index(size) + if offset < 0 or size < 0: + raise ValueError("invalid B2Z frame range") + size = max(0, min(size, self.member_length - offset)) + return self._read_archive(self.member_offset + offset, size) diff --git a/src/blosc2/core.py b/src/blosc2/core.py index 20a5d03da..657f46ea4 100644 --- a/src/blosc2/core.py +++ b/src/blosc2/core.py @@ -683,6 +683,25 @@ def split_h5_url(url: str) -> tuple[str, str | None]: return url, None +def _parse_b2z_url(urlpath, dataset): + if "::" in urlpath: + return None # A remaining separator belongs to an fsspec protocol chain. + parsed = urllib.parse.urlsplit(urlpath) + path_str = f"{parsed.netloc}/{parsed.path}" if parsed.netloc else parsed.path + parts = path_str.split("/") + for index, part in enumerate(parts): + if part.endswith(".b2z"): + subpath = "/".join(parts[index + 1 :]).strip("/") + if subpath: + if dataset is not None: + raise ValueError("Cannot specify dataset in both URL path and dataset parameter") + base_path = parsed.path[: -len("/".join(parts[index + 1 :]))].rstrip("/") + urlpath = urllib.parse.urlunsplit(parsed._replace(path=base_path)) + dataset = subpath + return urlpath, dataset, "b2z" + return None + + def parse_container_url( urlpath: object, dataset: str | None = None, @@ -705,6 +724,8 @@ def parse_container_url( raw_dataset = parts[1].strip("/") dataset = raw_dataset if raw_dataset else None + if b2z := _parse_b2z_url(urlpath, dataset): + return b2z h5_base, h5_dataset = split_h5_url(urlpath) if h5_dataset is not None: if dataset is not None: diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index 518b780fe..d661f4d16 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -70,6 +70,17 @@ def _source_urlpath(src): return src.urlpath +def _remote_array_metadata(src): + if isinstance(src, blosc2.B2ZNDSource): + return {"source_kind": "b2z", "urlpath": src.urlpath, "dataset": src.dataset} + return { + "source_kind": "zarr" if isinstance(src, blosc2.ZarrNDSource) else "fsspec", + "urlpath": _source_urlpath(src), + # Preserve the legacy field for older readers of standalone sources. + "local_abspath": src.urlpath, + } + + def _validate_max_cache_bytes(value: int | None) -> int | None: if value is None: return None @@ -209,12 +220,8 @@ def __init__( "caterva2_env": caterva2_env, } container = getattr(self.src, "schunk", self.src) - if isinstance(self.src, (blosc2.FsspecNDSource, blosc2.ZarrNDSource)): - meta_val["source_kind"] = "zarr" if isinstance(self.src, blosc2.ZarrNDSource) else "fsspec" - meta_val["urlpath"] = _source_urlpath(self.src) - # Keep the legacy field populated so older readers still - # reopen this cache, albeit through their eager URL path. - meta_val["local_abspath"] = self.src.urlpath + if isinstance(self.src, (blosc2.FsspecNDSource, blosc2.ZarrNDSource, blosc2.B2ZNDSource)): + meta_val.update(_remote_array_metadata(self.src)) elif isinstance(self.src, blosc2.C2Array): meta_val["source_kind"] = "caterva2" # Authentication belongs to the reopening process, not to a diff --git a/src/blosc2/ref.py b/src/blosc2/ref.py index 31daa9816..e93eb73cf 100644 --- a/src/blosc2/ref.py +++ b/src/blosc2/ref.py @@ -20,7 +20,7 @@ class Ref: - a persistent local Blosc2 object reopenable from ``urlpath`` - a member inside a :class:`blosc2.DictStore` - a remote :class:`blosc2.C2Array` - - an fsspec or Zarr URL used by a :class:`blosc2.RemoteProxy` + - an fsspec or Zarr URL, or B2Z array key, used by a :class:`blosc2.RemoteProxy` Instances can be created directly, from dictionaries via :meth:`from_dict`, or from supported objects via :meth:`from_object`. Use :meth:`open` to @@ -47,13 +47,17 @@ def __post_init__(self) -> None: _validate_persistable_url(self.urlpath) return - if self.kind == "dictstore_key": + if self.kind in {"dictstore_key", "b2z"}: if not isinstance(self.urlpath, str): - raise TypeError("Ref(kind='dictstore_key') requires a string 'urlpath'") + raise TypeError(f"Ref(kind={self.kind!r}) requires a string 'urlpath'") if not isinstance(self.key, str): - raise TypeError("Ref(kind='dictstore_key') requires a string 'key'") + raise TypeError(f"Ref(kind={self.kind!r}) requires a string 'key'") if self.path is not None or self.urlbase is not None: - raise ValueError("Ref(kind='dictstore_key') only supports 'urlpath' and 'key'") + raise ValueError(f"Ref(kind={self.kind!r}) only supports 'urlpath' and 'key'") + if self.kind == "b2z": + from blosc2.remote_proxy import _validate_persistable_url + + _validate_persistable_url(self.urlpath) return if self.kind == "c2array": if not isinstance(self.path, str): @@ -112,6 +116,8 @@ def from_object(cls, obj: Any) -> Ref: return cls.c2array_ref(source["path"], source["urlbase"]) if source["kind"] == "zarr": return cls.zarr_ref(source["urlpath"]) + if source["kind"] == "b2z": + return cls(kind="b2z", urlpath=source["urlpath"], key=source["dataset"]) return cls.fsspec_ref(source["urlpath"]) if isinstance(obj, blosc2.Proxy): obj = obj._cache @@ -129,7 +135,7 @@ def to_dict(self) -> dict[str, Any]: payload = {"kind": self.kind, "version": 1} if self.kind in {"urlpath", "fsspec", "zarr"}: payload["urlpath"] = self.urlpath - elif self.kind == "dictstore_key": + elif self.kind in {"dictstore_key", "b2z"}: payload["urlpath"] = self.urlpath payload["key"] = self.key elif self.kind == "c2array": @@ -146,6 +152,8 @@ def open(self): return blosc2.open(self.urlpath, mode="r") if self.kind == "dictstore_key": return blosc2.DictStore(self.urlpath, mode="r")[self.key] + if self.kind == "b2z": + return blosc2.RemoteProxy(self.urlpath, source_format="b2z", dataset=self.key) if self.kind == "c2array": return blosc2.C2Array(self.path, urlbase=self.urlbase) if self.kind == "fsspec": diff --git a/src/blosc2/remote_proxy.py b/src/blosc2/remote_proxy.py index 54a61e306..e5828cb1a 100644 --- a/src/blosc2/remote_proxy.py +++ b/src/blosc2/remote_proxy.py @@ -48,8 +48,8 @@ def __repr__(self) -> str: def _normalize_source_format(urlpath, source_format): - if source_format not in {None, "blosc2", "zarr", "hdf5"}: - raise ValueError("source_format must be None, 'blosc2', 'zarr', or 'hdf5'") + if source_format not in {None, "blosc2", "zarr", "hdf5", "b2z"}: + raise ValueError("source_format must be None, 'blosc2', 'zarr', 'hdf5', or 'b2z'") if source_format is not None: return source_format if isinstance(urlpath, blosc2.ZarrNDSource): @@ -152,6 +152,8 @@ def _validate_payload_limit(policy: blosc2.CachePolicy, limit) -> None: def _validate_authorized_source(urlpath, storage_options, source_descriptor): + if isinstance(urlpath, blosc2.B2ZNDSource): + raise NotImplementedError("authorized B2Z sparse attachment is not supported yet") if storage_options is not None: raise ValueError("storage_options cannot be used with an authorized source") hdf5_cls = getattr(blosc2, "HDF5NDSource", ()) @@ -233,6 +235,17 @@ def _open_url_source( "dataset": src.dataset, "assume_immutable": assume_immutable, } + elif source_format == "b2z": + if not assume_immutable: + raise NotImplementedError("mutable B2Z sources are not supported") + src = blosc2.B2ZNDSource(urlpath, dataset, _traffic=traffic, **kwargs) + source = { + "kind": "b2z", + "version": 1, + "urlpath": urlpath, + "dataset": src.dataset, + "assume_immutable": True, + } else: src = blosc2.FsspecNDSource(urlpath, _traffic=traffic, **kwargs) source = { @@ -288,13 +301,13 @@ def _parse_source_from_payload(source): urlpath = _validate_urlpath_source( source, {"kind", "version", "urlpath", "assume_immutable"}, "Zarr" ) - elif source_kind == "hdf5": + elif source_kind in {"hdf5", "b2z"}: urlpath = _validate_urlpath_source( - source, {"kind", "version", "urlpath", "dataset", "assume_immutable"}, "HDF5" + source, {"kind", "version", "urlpath", "dataset", "assume_immutable"}, source_kind.upper() ) dataset = source.get("dataset") if not isinstance(dataset, str): - raise TypeError("HDF5 RemoteProxy sources require a string 'dataset'") + raise TypeError(f"{source_kind.upper()} RemoteProxy sources require a string 'dataset'") else: raise ValueError(f"unsupported RemoteProxy source kind: {source_kind!r}") _validate_assume_immutable(source.get("assume_immutable"), "source assume_immutable") @@ -310,8 +323,8 @@ def _resolve_init_dataset_and_url(urlpath, dataset, source_format): if source_format is None: source_format = detected_format resolved_format = _normalize_source_format(urlpath, source_format) - if dataset is not None and resolved_format not in {"hdf5", "zarr"}: - raise ValueError("dataset is only supported for HDF5 and Zarr sources") + if dataset is not None and resolved_format not in {"hdf5", "zarr", "b2z"}: + raise ValueError("dataset is only supported for HDF5 and Zarr sources or B2Z archives") if resolved_format == "zarr": if dataset is not None: @@ -323,7 +336,7 @@ def _resolve_init_dataset_and_url(urlpath, dataset, source_format): resolved_dataset = urlpath[idx + 6 :].strip("/") or None else: resolved_dataset = None - elif resolved_format == "hdf5": + elif resolved_format in {"hdf5", "b2z"}: resolved_dataset = dataset.strip("/") if dataset is not None else None else: resolved_dataset = None @@ -366,9 +379,11 @@ class RemoteProxy(blosc2.Operand): storage_options: dict, optional Parameters passed to the underlying ``fsspec`` filesystem when opening an fsspec URL. - source_format: {None, "blosc2", "zarr"}, optional - Format of a URL source. A ``.zarr`` path component selects Zarr when - omitted. + source_format: {None, "blosc2", "zarr", "hdf5", "b2z"}, optional + Format of a URL source, inferred from its container suffix when omitted. + dataset: str, optional + Array path within an HDF5, Zarr, or B2Z container. B2Z supports external + NDArray leaves in immutable archives, e.g. ``dataset="d0/a3"``. assume_immutable: bool, optional Skip remote identity checks before reads. Defaults to ``True``. Set to ``False`` when the object at the URL may be replaced. @@ -812,7 +827,7 @@ def _open_source( return src, source def _source_identity(self) -> str: - if self._source["kind"] == "hdf5": + if self._source["kind"] in {"hdf5", "b2z"}: return f"{self._source['urlpath']}::{self._source['dataset']}" if self._source["kind"] in {"fsspec", "zarr"}: return self._source["urlpath"] @@ -973,7 +988,7 @@ def cache(self): @property def urlpath(self): """The remote fsspec URL or credential-free Caterva2 URLPath.""" - if self._source["kind"] in {"fsspec", "zarr", "hdf5"}: + if self._source["kind"] in {"fsspec", "zarr", "hdf5", "b2z"}: return self._source["urlpath"] return blosc2.URLPath(self._source["path"], urlbase=self._source["urlbase"]) @@ -1222,8 +1237,8 @@ def _from_payload(cls, payload, carrier): obj = cls( urlpath, cache_policy=policy, - source_format=source_kind if source_kind in {"zarr", "hdf5"} else None, - dataset=source.get("dataset") if source_kind == "hdf5" else None, + source_format=source_kind if source_kind in {"zarr", "hdf5", "b2z"} else None, + dataset=source.get("dataset") if source_kind in {"hdf5", "b2z"} else None, refs=refs, assume_immutable=source["assume_immutable"], _carrier=carrier_arg, diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 432444217..4bb5b617b 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -1890,6 +1890,9 @@ def _set_default_dparams(kwargs): def _reconstruct_legacy_proxy(proxy_cache, proxy_src): source_kind = proxy_src.get("source_kind") + if source_kind == "b2z": + src = blosc2.B2ZNDSource(proxy_src["urlpath"], proxy_src["dataset"]) + return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) if source_kind == "fsspec": src = blosc2.FsspecNDSource(proxy_src["urlpath"]) return blosc2.Proxy(src, _cache=proxy_cache, _refresh_source=False) @@ -2038,8 +2041,8 @@ def _remote_cache_options(kwargs: dict) -> tuple[str | pathlib.Path | None, str def _validate_fsspec_source_format(source_format, lazy): - if source_format not in {None, "blosc2", "zarr", "hdf5"}: - raise ValueError("source_format must be None, 'blosc2', 'zarr', or 'hdf5'") + if source_format not in {None, "blosc2", "zarr", "hdf5", "b2z"}: + raise ValueError("source_format must be None, 'blosc2', 'zarr', 'hdf5', or 'b2z'") if source_format == "zarr" and not lazy: raise NotImplementedError("Zarr sources require lazy=True") if source_format == "hdf5" and not lazy: @@ -2242,8 +2245,8 @@ def _validate_fsspec_lazy_options(urlpath: str, source_format, dataset, lazy: bo if dataset is not None and not lazy: raise ValueError("dataset requires lazy=True") _validate_fsspec_source_format(source_format, lazy) - if dataset is not None and source_format not in {None, "hdf5", "zarr"}: - raise ValueError("dataset is only supported for HDF5 and Zarr sources") + if dataset is not None and source_format not in {None, "hdf5", "zarr", "b2z"}: + raise ValueError("dataset is only supported for HDF5 and Zarr sources or B2Z archives") parsed = urlsplit(urlpath) url_path_str = f"{parsed.netloc}/{parsed.path}" if parsed.netloc else parsed.path if not lazy and any(part.endswith((".h5", ".hdf5")) for part in url_path_str.split("/")): @@ -2355,7 +2358,9 @@ def _is_container_open_request(urlpath: str, kwargs: dict) -> bool: _, parsed_dataset, hint = parse_container_url(urlpath, kwargs.get("dataset")) if hint == "hdf5": return True - return hint == "zarr" and (kwargs.get("lazy") or parsed_dataset is not None or "dataset" in kwargs) + return (hint in {"zarr", "b2z"} or kwargs.get("source_format") == "b2z") and ( + kwargs.get("lazy") or parsed_dataset is not None or "dataset" in kwargs + ) def _try_open_special_store(urlpath: str, mode: str, offset: int, kwargs: dict): @@ -2389,7 +2394,11 @@ def _normalize_open_target(urlpath, kwargs, dataset, refs): urlpath, parsed_dataset, detected_format = parse_container_url(urlpath, kwargs.get("dataset")) if parsed_dataset is not None: kwargs["dataset"] = parsed_dataset - if detected_format is not None and kwargs.get("source_format") is None: + if ( + detected_format is not None + and kwargs.get("source_format") is None + and (detected_format != "b2z" or kwargs.get("lazy") or parsed_dataset is not None) + ): kwargs["source_format"] = detected_format return urlpath @@ -2515,16 +2524,17 @@ def open( Parameters passed to the underlying ``fsspec`` filesystem when opening an fsspec URL (for instance credentials, endpoint URL, token, client_kwargs, etc.). dataset: str, optional - For HDF5 sources (``source_format="hdf5"`` or ``.h5``/``.hdf5`` files), - the dataset path within the HDF5 file (e.g. ``dataset="d0/d1/a2"``). + Array path within HDF5, Zarr, or B2Z containers (e.g. ``dataset="d0/d1/a2"``). + B2Z supports external NDArray leaves in immutable archives. Requires ``lazy=True``. refs: dict | str | PathLike, optional Pre-computed kerchunk reference dictionary or path to a JSON reference file for HDF5 sources. - source_format: {None, "blosc2", "zarr", "hdf5"}, optional + source_format: {None, "blosc2", "zarr", "hdf5", "b2z"}, optional Format of a lazy remote source. A ``.zarr`` URL path component selects Zarr automatically; a ``.h5`` or ``.hdf5`` path selects HDF5 automatically; - an explicit value supports suffix-free array paths. + a ``.b2z`` path selects B2Z automatically. An explicit value supports + suffix-free array paths. assume_immutable: bool, optional With ``lazy=True``, skip remote identity checks before reads. Defaults to ``True``; set to ``False`` when the remote object may be replaced. @@ -2559,7 +2569,8 @@ def open( A plain URL read rebuilds the object from a cframe held in memory, so it covers ``.b2nd``, ``.b2f`` and ``.b2e`` only -- a ``.b2z`` store is a zip archive rather than a cframe, and needs ``cache_dir`` like the directory - formats do. With ``lazy=True``, it returns a :ref:`RemoteProxy` (using + formats do. With ``lazy=True`` and a dataset path, a B2Z archive serves + its selected external NDArray by byte range. Lazy opening returns a :ref:`RemoteProxy` (using ``CachePolicy.DISK`` with ``cache_dir`` or ``cache_path``, and ``CachePolicy.MEMORY`` otherwise). diff --git a/tests/test_b2z_source.py b/tests/test_b2z_source.py new file mode 100644 index 000000000..b9dc0ed00 --- /dev/null +++ b/tests/test_b2z_source.py @@ -0,0 +1,269 @@ +"""Native remote reads inside B2Z archives.""" + +import io +import subprocess +import sys +import zipfile + +import numpy as np +import pytest + +import blosc2 +from blosc2.core import parse_container_url + +fsspec = pytest.importorskip("fsspec") + + +def memory_archive(data=None, *, compression=zipfile.ZIP_STORED, zip64=False): + if data is None: + data = np.random.default_rng(42).integers(0, 256, (200, 1000), dtype="uint8") + array = blosc2.asarray(data, chunks=(40, 250), blocks=(10, 50)) + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", compression=compression) as archive: + with archive.open("d0/a.b2nd", "w", force_zip64=zip64) as member: + member.write(array.to_cframe()) + archive.writestr("d0/b.b2nd", blosc2.asarray(data[::-1].copy()).to_cframe()) + fs = fsspec.filesystem("memory") + fs.pipe_file("v10.b2z", buffer.getvalue()) + return "memory://v10.b2z", data + + +@pytest.mark.parametrize("address", ["::/d0/a", "/d0/a", "keyword"]) +def test_addressing_and_hits(address, monkeypatch): + url, data = memory_archive() + fs = fsspec.filesystem("memory") + reads = [] + original = type(fs).cat_file + + def counted(self, path, start=None, end=None, **kwargs): + reads.append((start, end)) + return original(self, path, start=start, end=end, **kwargs) + + monkeypatch.setattr(type(fs), "cat_file", counted) + arr = ( + blosc2.open(url, lazy=True, dataset="/d0/a/") + if address == "keyword" + else blosc2.open(url + address, lazy=True) + ) + assert arr.dataset == "d0/a" + assert arr.source["kind"] == "b2z" + assert arr.chunks == (40, 250) + assert arr.blocks == (10, 50) + assert arr.traffic.nbytes == sum(end - start for start, end in reads) + assert arr.traffic.nbytes < data.nbytes // 4 + reads.clear() + np.testing.assert_array_equal(arr[1:6, :5], data[1:6, :5]) + assert reads + lo, hi = arr.src.member_offset, arr.src.member_offset + arr.src.member_length + assert all(lo <= start < end <= hi for start, end in reads) + reads.clear() + np.testing.assert_array_equal(arr[1:6, :5], data[1:6, :5]) + assert not reads + np.testing.assert_array_equal(arr[-3:, -4:], data[-3:, -4:]) + + +@pytest.mark.parametrize("limit", [None, 1000, 300_000]) +def test_disk_persistence_and_eviction(tmp_path, limit): + url, data = memory_archive() + path = tmp_path / "cache.b2nd" + arr = blosc2.open(url, dataset="d0/a", lazy=True, cache_path=path, max_cache_bytes=limit) + np.testing.assert_array_equal(arr[:40], data[:40]) + if limit is not None: + assert arr.cache_bytes <= limit + reopened = blosc2.open(path, mode="a") + before = reopened.traffic.nbytes + np.testing.assert_array_equal(reopened[:40], data[:40]) + assert (reopened.traffic.nbytes == before) == (limit != 1000) + restored = blosc2.from_cframe(arr.to_cframe()) + np.testing.assert_array_equal(restored[-2:], data[-2:]) + assert restored.dataset == "d0/a" + + +def test_policies_exports_and_identity(tmp_path): + url, data = memory_archive() + none = blosc2.RemoteProxy(url, dataset="d0/a") + np.testing.assert_array_equal(none[:2], data[:2]) + before = none.traffic.nbytes + np.testing.assert_array_equal(none[:2], data[:2]) + assert none.traffic.nbytes > before + memory = blosc2.open(url, dataset="d0/a", lazy=True) + memory[:2] + memory.save(tmp_path / "cold.b2nd") + cold = blosc2.open(tmp_path / "cold.b2nd") + assert cold.cache_bytes == 0 + np.testing.assert_array_equal(cold.materialize((slice(0, 2),))[:], data[:2]) + a = blosc2.open(url, dataset="d0/a", lazy=True, cache_dir=tmp_path / "caches") + b = blosc2.open(url, dataset="d0/b", lazy=True, cache_dir=tmp_path / "caches") + assert a.cache_path != b.cache_path + np.testing.assert_array_equal(b[:2], data[::-1][:2]) + src = blosc2.B2ZNDSource(url, "d0/a") + proxy = blosc2.Proxy(src, urlpath=str(tmp_path / "legacy.b2nd"), mode="w") + proxy[:2] + np.testing.assert_array_equal(blosc2.open(tmp_path / "legacy.b2nd")[-2:], data[-2:]) + with pytest.raises(NotImplementedError, match="authorized B2Z"): + blosc2.RemoteProxy.with_sparse_cache(src, tmp_path / "sparse", source_descriptor=a.source) + + +def test_zip64_and_range_bounds(): + url, data = memory_archive(zip64=True) + src = blosc2.B2ZNDSource(url, "d0/a") + np.testing.assert_array_equal(blosc2.Proxy(src)[:], data) + assert src.read_range(src.member_length - 2, 100) == src.read_range(src.member_length - 2, 2) + assert src.read_range(src.member_length + 10, 100) == b"" + with pytest.raises(ValueError, match="range"): + src.read_range(-1, 10) + + +@pytest.mark.parametrize("dataset", [None, "", "/", "d0", "missing", "../d0/a", "d0//a", "d0/./a", "d0/\na"]) +def test_bad_datasets(dataset): + url, _ = memory_archive() + with pytest.raises(ValueError): + blosc2.open(url, lazy=True, dataset=dataset) + + +def test_options_and_bad_archives(): + url, _ = memory_archive() + with pytest.raises(NotImplementedError, match="mutable B2Z"): + blosc2.open(url, lazy=True, dataset="d0/a", assume_immutable=False) + with pytest.raises(ValueError, match="both"): + blosc2.open(url + "::d0/a", dataset="d0/a", lazy=True) + with pytest.raises(ValueError, match="lazy=True"): + blosc2.open(url, dataset="d0/a") + fs = fsspec.filesystem("memory") + fs.pipe_file("suffix-free", fs.cat_file("v10.b2z")) + assert ( + blosc2.open("memory://suffix-free", source_format="b2z", dataset="d0/a", lazy=True).dataset == "d0/a" + ) + url, _ = memory_archive(compression=zipfile.ZIP_DEFLATED) + with pytest.raises(NotImplementedError, match="ZIP_STORED"): + blosc2.open(url, dataset="d0/a", lazy=True) + fs.pipe_file("v10.b2z", b"not a zip") + with pytest.raises(zipfile.BadZipFile): + blosc2.open(url, dataset="d0/a", lazy=True) + + +def test_parser_query_and_local_store(tmp_path): + assert parse_container_url("zip://a.b2nd::memory://h.b2z")[2] is None + assert parse_container_url("memory://h.b2z/group.h5/a") == ("memory://h.b2z", "group.h5/a", "b2z") + assert parse_container_url("https://host/h.b2z/d0/a?x=1") == ("https://host/h.b2z?x=1", "d0/a", "b2z") + assert parse_container_url("https://host/frame?x=h.b2z/d0/a")[2] is None + path = tmp_path / "local.b2z" + with blosc2.TreeStore(path, mode="w") as store: + store["/a"] = np.arange(10) + with blosc2.open(path) as store: + np.testing.assert_array_equal(store["/a"][:], np.arange(10)) + + +@pytest.mark.parametrize("dtype", ["int32", "float64", "complex64", "S8", "datetime64[s]", ">i4"]) +def test_dtypes_and_edges(dtype): + data = np.arange(41 * 253).reshape(41, 253).astype(dtype) + url, _ = memory_archive(data) + arr = blosc2.open(url, dataset="d0/a", lazy=True) + np.testing.assert_array_equal(arr[:], data) + + +@pytest.mark.parametrize("shape", [(), (0,), (0, 5)]) +def test_scalar_and_empty(shape): + data = np.zeros(shape, dtype="int32") + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("a.b2nd", blosc2.asarray(data).to_cframe()) + fsspec.filesystem("memory").pipe_file("empty.b2z", buffer.getvalue()) + arr = blosc2.open("memory://empty.b2z", dataset="a", lazy=True) + np.testing.assert_array_equal(arr[()], data) + + +@pytest.mark.parametrize( + "damage", ["duplicate", "encrypted", "header", "frame_length", "truncated", "object"] +) +def test_invalid_members(damage): + url, _ = memory_archive() + fs = fsspec.filesystem("memory") + raw = bytearray(fs.cat_file("v10.b2z")) + if damage == "duplicate": + buffer = io.BytesIO(raw) + with zipfile.ZipFile(buffer, "a") as archive, pytest.warns(UserWarning, match="Duplicate"): + archive.writestr("d0/a.b2nd", b"duplicate") + raw = buffer.getvalue() + elif damage == "encrypted": + central = raw.index(b"PK\x01\x02") + raw[central + 8] |= 1 + raw[6] |= 1 + elif damage == "header": + raw[0] = 0 + elif damage == "frame_length": + offset = 30 + len("d0/a.b2nd") + raw[offset + 16 : offset + 24] = (len(raw) * 2).to_bytes(8, "big") + elif damage == "truncated": + raw = raw[:-100] + else: + buffer = io.BytesIO() + from blosc2.b2objects import make_b2object_carrier + + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("d0/a.b2nd", make_b2object_carrier("lazyexpr", (10,), "int32").to_cframe()) + raw = buffer.getvalue() + fs.pipe_file("v10.b2z", raw) + with pytest.raises((ValueError, NotImplementedError, zipfile.BadZipFile)): + blosc2.open(url, dataset="d0/a", lazy=True) + + +def test_geometry_change_and_expression(tmp_path): + url, data = memory_archive() + path = tmp_path / "geometry.b2nd" + arr = blosc2.open(url, dataset="d0/a", lazy=True, cache_path=path) + expr = arr + 2 + expr.save(tmp_path / "expr.b2nd") + np.testing.assert_array_equal(blosc2.open(tmp_path / "expr.b2nd")[:2], data[:2] + 2) + memory_archive(np.zeros((201, 1000), dtype="uint8")) + with pytest.raises(ValueError, match="geometry"): + blosc2.open(path) + + +def test_optional_dependencies(): + subprocess.run( + [ + sys.executable, + "-c", + """ +import builtins +original = builtins.__import__ +def blocked(name, *args, **kwargs): + if name.split('.')[0] in {'zarr', 'kerchunk', 'h5py', 'hdf5plugin'}: + raise ImportError('blocked optional dependency') + return original(name, *args, **kwargs) +builtins.__import__ = blocked +import blosc2 +import fsspec +import io +import zipfile +a = blosc2.arange(10) +fs = fsspec.filesystem('memory') +fs.pipe_file('plain.b2nd', a.to_cframe()) +assert blosc2.open('memory://plain.b2nd', lazy=True)[3] == 3 +buffer = io.BytesIO() +with zipfile.ZipFile(buffer, 'w') as archive: + archive.writestr('a.b2nd', a.to_cframe()) +fs.pipe_file('optional.b2z', buffer.getvalue()) +assert blosc2.open('memory://optional.b2z', lazy=True, dataset='a')[3] == 3 +""", + ], + check=True, + capture_output=True, + text=True, + ) + + +@pytest.mark.network +def test_s3_b2z_slice(): + pytest.importorskip("s3fs") + arr = blosc2.open( + "s3://blosc2/hierarchy.b2z::/d0/a3", + lazy=True, + storage_options={"profile": "blosc2", "endpoint_url": "https://s3.us-west-001.backblazeb2.com"}, + ) + expected = np.arange(10)[:, None] * 1_000_000 + np.arange(5) + np.testing.assert_array_equal(arr[:10, 0, :5], expected) + before = arr.traffic.nbytes + np.testing.assert_array_equal(arr[:10, 0, :5], expected) + assert arr.traffic.nbytes == before From 8fc6fb5880c9b21b35cfbb8cad7a54e9eb1bb927 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 8 Sep 2026 08:16:33 +0200 Subject: [PATCH 31/89] Optimize remote B2Z metadata reads --- doc/reference/remoteproxy.rst | 8 +++++++- plans/remote-proxy-v10.md | 27 +++++++++++++++++++++++++-- src/blosc2/b2z_source.py | 25 +++++++++++++++++++------ tests/test_b2z_source.py | 27 +++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 9 deletions(-) diff --git a/doc/reference/remoteproxy.rst b/doc/reference/remoteproxy.rst index 854334035..a5df7f662 100644 --- a/doc/reference/remoteproxy.rst +++ b/doc/reference/remoteproxy.rst @@ -93,9 +93,15 @@ source chunks, blocks, dtype, and compression parameters; no kerchunk, Zarr, or HDF5 dependencies are needed. Install the fsspec extra and the protocol backend. Opening reads the ZIP directory and selected member's headers. Directory cost -scales with archive member count. Subsequent reads fetch native chunks or blocks +scales with archive member count. An 8 KiB archive tail and 16 KiB member prefix +are prefetched to combine small metadata requests; larger directories or headers +fall back to exact reads. These temporary buffers are released after opening. +Subsequent reads fetch native chunks or blocks by byte range; repeated cache hits perform no remote reads. Reopening a saved carrier rereads archive/frame metadata and resolves the member offset afresh. +The optimized reader derives its source stamp from the same metadata response +used to obtain archive size. Caches from the initial v10 reader may therefore +refetch their contents once after upgrading. Only unencrypted, ``ZIP_STORED`` external NDArray members are supported. Groups, embedded leaves inside ``embed.b2e``, other leaf types, and compressed ZIP members diff --git a/plans/remote-proxy-v10.md b/plans/remote-proxy-v10.md index 6a8fee741..dc3ce96c7 100644 --- a/plans/remote-proxy-v10.md +++ b/plans/remote-proxy-v10.md @@ -9,8 +9,8 @@ Deferred work are not prerequisites for this version. ## Implementation results -- Added `B2ZNDSource` using an unbuffered seekable view for ZIP discovery and - bounded native Blosc2 range reads for the selected member. +- Added `B2ZNDSource` using a seekable view with bounded opening buffers for ZIP + discovery and bounded native Blosc2 range reads for the selected member. - Integrated all three addressing forms, source descriptors, cache identity, carrier reopening, and durable B2Z operand references for saved expressions. - Added validation for member bounds, ZIP64, duplicate/encrypted/compressed @@ -28,6 +28,29 @@ Deferred work are not prerequisites for this version. - Tests needing localhost servers and the S3 check were run outside the network sandbox. All Python commands used the `blosc2` conda environment. +## Opening optimization measurements + +Three fresh-process S3 runs per version against `hierarchy.b2z::/d0/a3`: + +| Metric | Initial reader | Buffered reader | +|--------|---------------:|----------------:| +| Median opening time | 2268.9 ms | 1197.3 ms | +| Opening requests | 2 HEAD + 7 GET | 1 HEAD + 2 GET | +| Opening payload | 9067 bytes | 24576 bytes | +| First-slice payload | 78257 bytes | 78257 bytes | +| Repeated-slice payload | 0 bytes | 0 bytes | + +Opening is approximately 47% faster in these measurements. An 8 KiB tail and +16 KiB member prefix combine ZIP and native-frame header reads, while larger +directories, comments, and extra fields retain exact-read fallbacks. Buffers +are released after source construction; payload cache ownership is unchanged. +Source stamps now use the existing archive-info response instead of a second +identity lookup. This changes stamps from the initial reader, so older warm +caches may refetch once. Values, geometry, and cached-repeat behavior were +verified on every measurement. These timings do not promise a first-slice speedup. +Validation: 39 focused B2Z tests passed; the default suite passed with 9754 tests +and 29 skips. Ruff and whitespace checks passed. + ## Objective Open an external NDArray leaf inside an immutable remote `.b2z` archive as a diff --git a/src/blosc2/b2z_source.py b/src/blosc2/b2z_source.py index ffb9672a6..3f50720d0 100644 --- a/src/blosc2/b2z_source.py +++ b/src/blosc2/b2z_source.py @@ -15,7 +15,7 @@ class _ArchiveFile(io.RawIOBase): - """Seekable, unbuffered view used only while zipfile reads the directory.""" + """Seekable view using the source's bounded opening buffers.""" def __init__(self, source, size): self.source, self.size, self.pos = source, size, 0 @@ -48,8 +48,8 @@ class B2ZNDSource(ByteRangeNDSource): ``dataset`` is a logical tree key, e.g. ``d0/a3``, without the member's ``.b2nd`` suffix. Embedded leaves and ZIP-compressed members are unsupported. - Only directory/header metadata is read eagerly; native chunks and blocks - are fetched on demand. Replacing the archive requires replacing its cache. + Opening uses bounded metadata prefetch; native chunks and blocks are fetched + on demand. Replacing the archive requires replacing its cache. """ def __init__( @@ -77,7 +77,12 @@ def __init__( else: self._fs, self._path = _filesystem, _filesystem._strip_protocol(urlpath) self.traffic = _traffic if _traffic is not None else Traffic() - size = self._fs.size(self._path) + object_info = self._fs.info(self._path) + size = object_info["size"] + self._opening_ranges = [] + # ponytail: small directories fit in 8 KiB; larger ones use exact reads. + tail_start = max(0, size - 8192) + self._opening_ranges.append((tail_start, self._read_archive(tail_start, size - tail_start))) with _ArchiveFile(self, size) as file, zipfile.ZipFile(file) as archive: matches = [info for info in archive.infolist() if info.filename == dataset + ".b2nd"] if not matches: @@ -89,8 +94,12 @@ def __init__( info = matches[0] if info.flag_bits & 1 or info.compress_type != zipfile.ZIP_STORED: raise NotImplementedError("B2Z array members must be unencrypted ZIP_STORED entries") - if info.compress_size != info.file_size or info.header_offset < 0: + if info.compress_size != info.file_size or not 0 <= info.header_offset <= size - 30: raise ValueError("invalid B2Z member size or offset") + # Cover the local header and the native reader's 8 KiB frame prefix + # together. Unusually long ZIP headers fall back to exact reads. + prefix = self._read_archive(info.header_offset, min(16384, size - info.header_offset)) + self._opening_ranges.append((info.header_offset, prefix)) # zipfile validates the local signature, filename, and member overlap. # Opening does not read/decode member payloads. with archive.open(info): @@ -115,8 +124,9 @@ def __init__( raise ValueError("B2Z member exceeds archive bounds") from fsspec.utils import tokenize - self.stamp = tokenize(self._fs.ukey(self._path), dataset, self.member_offset, self.member_length) + self.stamp = tokenize(urlpath, object_info, dataset, self.member_offset, self.member_length) super().__init__(urlpath, max_concurrency, traffic=self.traffic) + self._opening_ranges.clear() if b"b2o" in self._header[13][1]: raise NotImplementedError("B2Z object carriers are not supported; select a plain NDArray") if not self._header_len <= self._header[2] <= self.member_length: @@ -125,6 +135,9 @@ def __init__( def _read_archive(self, offset, size): if not size: return b"" + for start, data in self._opening_ranges: + if start <= offset and offset + size <= start + len(data): + return data[offset - start : offset - start + size] data = self._fs.cat_file(self._path, start=offset, end=offset + size) if len(data) > size: raise ValueError("B2Z transport did not honor the requested byte range") diff --git a/tests/test_b2z_source.py b/tests/test_b2z_source.py index b9dc0ed00..52c7c94d0 100644 --- a/tests/test_b2z_source.py +++ b/tests/test_b2z_source.py @@ -51,6 +51,8 @@ def counted(self, path, start=None, end=None, **kwargs): assert arr.blocks == (10, 50) assert arr.traffic.nbytes == sum(end - start for start, end in reads) assert arr.traffic.nbytes < data.nbytes // 4 + assert len(reads) == 2 # ZIP tail, then local header plus native frame prefix. + assert not arr.src._opening_ranges # Discovery buffers do not become a second payload cache. reads.clear() np.testing.assert_array_equal(arr[1:6, :5], data[1:6, :5]) assert reads @@ -114,6 +116,31 @@ def test_zip64_and_range_bounds(): src.read_range(-1, 10) +@pytest.mark.parametrize("variant", ["directory", "comment", "extra"]) +def test_opening_buffer_fallbacks(variant): + url, data = memory_archive() + fs = fsspec.filesystem("memory") + buffer = io.BytesIO(fs.cat_file("v10.b2z")) + if variant == "extra": + with zipfile.ZipFile(buffer) as archive: + frame = archive.read("d0/a.b2nd") + buffer = io.BytesIO() + info = zipfile.ZipInfo("d0/a.b2nd") + info.extra = b"\x00\xf0" + (20000).to_bytes(2, "little") + b"x" * 20000 + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr(info, frame) + else: + with zipfile.ZipFile(buffer, "a") as archive: + if variant == "comment": + archive.comment = b"x" * 60000 + else: + for index in range(300): + archive.writestr(f"directory/padding-{index}", b"") + fs.pipe_file("v10.b2z", buffer.getvalue()) + arr = blosc2.open(url, dataset="d0/a", lazy=True) + np.testing.assert_array_equal(arr[:], data) + + @pytest.mark.parametrize("dataset", [None, "", "/", "d0", "missing", "../d0/a", "d0//a", "d0/./a", "d0/\na"]) def test_bad_datasets(dataset): url, _ = memory_archive() From 480a41345610607b3e54335befb0c339770add15 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 8 Sep 2026 12:20:14 +0200 Subject: [PATCH 32/89] Support meta and vlmeta access and caching in RemoteProxy --- examples/remote/s3-access.py | 25 ++-- src/blosc2/__init__.py | 3 +- src/blosc2/hdf5_source.py | 7 + src/blosc2/proxy.py | 14 +- src/blosc2/proxy_source.py | 116 ++++++++++++++++ src/blosc2/remote_proxy.py | 159 +++++++++++++++++++++- src/blosc2/schunk.py | 4 +- src/blosc2/zarr_source.py | 7 + tests/ndarray/test_proxy.py | 12 ++ tests/test_b2z_source.py | 69 ++++++++++ tests/test_hdf5_source.py | 13 ++ tests/test_remote_proxy.py | 247 +++++++++++++++++++++++++++++++++++ tests/test_vlmeta.py | 5 + 13 files changed, 666 insertions(+), 15 deletions(-) diff --git a/examples/remote/s3-access.py b/examples/remote/s3-access.py index 75c94a985..cc7dca333 100755 --- a/examples/remote/s3-access.py +++ b/examples/remote/s3-access.py @@ -181,15 +181,22 @@ def get_traffic_bytes() -> int | None: print(f"\n[Format: {fmt}]") if hasattr(arr, "info"): - print(arr.info) - print("Shape: ", arr.shape) - print("Dtype: ", arr.dtype) - chunks = getattr(arr, "chunks", None) - if chunks is not None: - print("Chunks:", chunks) - blocks = getattr(arr, "blocks", None) - if blocks is not None: - print("Blocks:", blocks) + print(arr.info, end="") + else: + print(f"{'shape':<12} : {arr.shape}") + print(f"{'dtype':<12} : {arr.dtype}") + chunks = getattr(arr, "chunks", None) + if chunks is not None: + print(f"{'chunks':<12} : {chunks}") + blocks = getattr(arr, "blocks", None) + if blocks is not None: + print(f"{'blocks':<12} : {blocks}") + meta = getattr(arr, "meta", None) + if meta: + print(f"{'meta':<12} : {dict(meta)}") + vlmeta = getattr(arr, "vlmeta", None) + if vlmeta is not None: + print(f"{'vlmeta':<12} : {dict(vlmeta) if vlmeta else {}}") print("\nSample slice data (1st fetch):") t0 = time.perf_counter() diff --git a/src/blosc2/__init__.py b/src/blosc2/__init__.py index 00fcbe370..2ef2abe6b 100644 --- a/src/blosc2/__init__.py +++ b/src/blosc2/__init__.py @@ -619,7 +619,7 @@ def _raise(exc): jit, as_simpleproxy, ) -from .remote_proxy import RemoteProxy +from .remote_proxy import RemoteMetadataMapping, RemoteProxy from . import linalg from .linalg import tensordot, vecdot, permute_dims, matrix_transpose, matmul, transpose, diagonal, outer from .utils import linalg_funcs as linalg_funcs_list @@ -912,6 +912,7 @@ def _raise(exc): "ProxyNDSource", "ProxySource", "Ref", + "RemoteMetadataMapping", "RemoteProxy", "SChunk", "SimpleProxy", diff --git a/src/blosc2/hdf5_source.py b/src/blosc2/hdf5_source.py index ec4785db9..ffdd62bce 100644 --- a/src/blosc2/hdf5_source.py +++ b/src/blosc2/hdf5_source.py @@ -400,6 +400,13 @@ def dtype(self) -> np.dtype: def cparams(self): return self._cparams + @property + def vlmeta(self) -> dict: + try: + return dict(self.array.attrs) + except Exception: + return {} + def get_chunk(self, nchunk: int) -> bytes: return zarr_chunk_to_blosc2( self.array, diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index d661f4d16..587343501 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -174,6 +174,7 @@ def __init__( self._max_cache_bytes = _validate_max_cache_bytes(kwargs.pop("_max_cache_bytes", None)) self._persistent_dirty = bool(kwargs.pop("_persistent_dirty", False)) vlmeta = kwargs.pop("vlmeta", None) + meta_kw = kwargs.pop("meta", None) caterva2_env = kwargs.pop("caterva2_env", False) # Before anything is built or emptied: a call that is going to be refused # must leave the cache at `urlpath` exactly as it found it, and adopting @@ -230,7 +231,7 @@ def __init__( elif hasattr(container, "urlpath"): meta_val["source_kind"] = "local" meta_val["local_abspath"] = container.urlpath - meta = {"proxy-source": meta_val} + meta = {"proxy-source": meta_val, **(meta_kw or {})} if hasattr(self.src, "shape"): self._cache = blosc2.empty( self.src.shape, @@ -1268,6 +1269,17 @@ def info(self) -> str: def __str__(self): return f"Proxy({self.src}, urlpath={self.urlpath})" + @property + def meta(self) -> blosc2.schunk.meta: + """ + Get the fixed-length metalayers of the cache. + + See Also + -------- + :py:attr:`blosc2.schunk.SChunk.meta` + """ + return self._schunk_cache.meta + @property def vlmeta(self) -> blosc2.schunk.vlmeta: """ diff --git a/src/blosc2/proxy_source.py b/src/blosc2/proxy_source.py index 61ecfd322..5da5ddeb5 100644 --- a/src/blosc2/proxy_source.py +++ b/src/blosc2/proxy_source.py @@ -20,11 +20,13 @@ import ast import asyncio +import contextlib import math import struct import threading from abc import ABC, abstractmethod from collections.abc import Mapping, Sequence +from typing import Any try: from itertools import batched @@ -81,6 +83,8 @@ def batched(iterable, n): # has 4 KB of compressed offsets. _FRAME_PREFETCH = 8192 _INDEX_PREFETCH = 1 << 16 +_TRAILER_MINLEN = 25 +_TRAILER_PREFETCH = 4096 # How many partly filled chunks keep their blocks in memory as well as in the # cache. Adding a block to a chunk rewrites that chunk, and the blocks already @@ -342,6 +346,16 @@ async def aget_chunk(self, nchunk: int) -> bytes: "aget_chunk is only available if the source has an async aget_chunk method" ) + @property + def meta(self) -> dict: + """The fixed-length metadata of the source.""" + return {} + + @property + def vlmeta(self) -> dict: + """The variable-length metadata of the source.""" + return {} + class ProxySource(ABC): """ @@ -608,6 +622,58 @@ def _frame_metalayer(raw: bytes, header: list, name: str): return msgpack.unpackb(raw[offset + 5 : offset + 5 + nbytes], raw=False) +def _read_frame_metalayers(raw: bytes, header: list) -> dict[str, Any]: + """Decode all metalayers out of an already-read frame header.""" + if len(header) <= 13 or not header[13] or len(header[13]) < 2: + return {} + names_map = header[13][1] + if not isinstance(names_map, dict): + return {} + res = {} + for name_bytes in names_map: + name = name_bytes.decode("utf-8") if isinstance(name_bytes, bytes) else str(name_bytes) + with contextlib.suppress(Exception): + res[name] = _frame_metalayer(raw, header, name) + return res + + +def _parse_trailer_vlmeta(trailer_bytes: bytes) -> dict[str, Any]: + """Decode vlmetalayers mapping from trailer bytes.""" + import msgpack + + try: + trailer = msgpack.unpackb(trailer_bytes, raw=True, strict_map_key=False) + except Exception: + return {} + if not isinstance(trailer, list) or len(trailer) < 2: + return {} + vlmeta_section = trailer[1] + if not isinstance(vlmeta_section, list) or len(vlmeta_section) < 2: + return {} + names_map = vlmeta_section[1] + if not isinstance(names_map, dict): + return {} + res = {} + for name_bytes, offset in names_map.items(): + name = name_bytes.decode("utf-8") if isinstance(name_bytes, bytes) else str(name_bytes) + if offset + 5 > len(trailer_bytes) or trailer_bytes[offset] != 0xC6: + continue + content_len = int.from_bytes(trailer_bytes[offset + 1 : offset + 5], "big") + if offset + 5 + content_len > len(trailer_bytes): + continue + blob = trailer_bytes[offset + 5 : offset + 5 + content_len] + try: + decomp = blosc2.decompress(blob) + except Exception: + continue + try: + val = msgpack.unpackb(decomp, raw=False) + except Exception: + val = decomp + res[name] = val + return res + + def _chunk_extents(offsets: np.ndarray, header: list) -> np.ndarray: """How many bytes to read at each chunk offset to be sure of covering it. @@ -714,8 +780,11 @@ def __init__( # open. Chunk reads are stateless, so the index below is the only state a # thread pool shares, and the only thing here that needs a lock raw, self._header, self._head = _read_frame_header(self.read_range) + self._raw_header = raw self._header_len = len(raw) self._chunksize = self._header[8] + self._meta = None + self._vlmeta = None # Where the chunks are is read on the first one touched, not here: a # `Proxy` over a cache that already holds the slice asked for fetches # nothing, and then the offsets are a request spent on nothing at all. @@ -863,9 +932,12 @@ def _frame_index(self) -> tuple[np.ndarray, np.ndarray]: # offsets are found through both, so the header is read first, and # the offsets it locates are read again after it raw, self._header, self._head = _read_frame_header(self.read_range) + self._raw_header = raw self._header_len = len(raw) self._chunksize = self._header[8] self._index = None + self._meta = None + self._vlmeta = None self._stale = False if self._index is None: offsets = _read_frame_offsets(self.read_range, self._header, self._head, self._header_len) @@ -934,6 +1006,50 @@ def invalidate_index(self) -> None: # then presents as whole. A layout costs one header read to rebuild # and only the partly fetched chunks have one at all self._layouts.clear() + self._meta = None + self._vlmeta = None + + @property + def has_vlmetalayers(self) -> bool: + """Whether the underlying frame has variable-length metalayers.""" + return bool(len(self._header) > 11 and self._header[11]) + + @property + def meta(self) -> dict: + """Fixed-length metadata of the remote frame.""" + if self._meta is None: + self._meta = _read_frame_metalayers(self._raw_header, self._header) + return dict(self._meta) + + @property + def vlmeta(self) -> dict: + """Variable-length metadata of the remote frame.""" + if self._vlmeta is None: + self._vlmeta = self._read_frame_vlmeta() + return dict(self._vlmeta) + + def _read_frame_vlmeta(self) -> dict[str, Any]: + """Decode all vlmetalayers from the frame trailer, if present.""" + if not self.has_vlmetalayers: + return {} + frame_len = self._header[2] + if frame_len < _TRAILER_MINLEN: + return {} + prefetch_size = min(frame_len, _TRAILER_PREFETCH) + tail_start = frame_len - prefetch_size + tail = self.read_range(tail_start, prefetch_size) + if len(tail) < _TRAILER_MINLEN or tail[-23] != 0xCE: + return {} + trailer_len = int.from_bytes(tail[-22:-18], "big") + if trailer_len <= 0 or trailer_len > frame_len: + return {} + if trailer_len <= len(tail): + trailer_bytes = tail[-trailer_len:] + else: + trailer_bytes = self.read_range(frame_len - trailer_len, trailer_len) + if len(trailer_bytes) != trailer_len: + return {} + return _parse_trailer_vlmeta(trailer_bytes) @property def shape(self) -> tuple: diff --git a/src/blosc2/remote_proxy.py b/src/blosc2/remote_proxy.py index e5828cb1a..f9ca96b94 100644 --- a/src/blosc2/remote_proxy.py +++ b/src/blosc2/remote_proxy.py @@ -10,30 +10,84 @@ from __future__ import annotations import asyncio +import contextlib import math import os import threading +from collections.abc import Mapping from contextlib import nullcontext from functools import wraps from types import SimpleNamespace +from typing import Any from urllib.parse import parse_qsl, urlsplit import numpy as np import blosc2 -from blosc2.b2objects import make_b2object_carrier, write_b2object_payload +from blosc2.b2objects import ( + _B2OBJECT_USER_VLMETA_KEY, + make_b2object_carrier, + read_b2object_user_vlmeta, + write_b2object_payload, + write_b2object_user_vlmeta, +) from blosc2.core import parse_container_url from blosc2.info import InfoReporter, format_nbytes_info DEFAULT_DISK_CACHE_BYTES = 256 * 2**20 +class RemoteMetadataMapping(Mapping): + """Read-only dictionary-like mapping of remote array metadata.""" + + def __init__(self, data: Mapping | None = None): + self._data = dict(data) if data is not None else {} + + def __getitem__(self, key: str | slice) -> Any: + if isinstance(key, slice): + if key.start is None and key.stop is None and key.step is None: + return self.getall() + raise NotImplementedError("Slicing is not supported, unless [:]") + return self._data[key] + + def __iter__(self): + return iter(self._data) + + def __len__(self) -> int: + return len(self._data) + + def __contains__(self, key: object) -> bool: + return key in self._data + + def get(self, key: str, default: Any = None) -> Any: + return self._data.get(key, default) + + def getall(self) -> dict[str, Any]: + return self._data.copy() + + def copy(self) -> dict[str, Any]: + return self._data.copy() + + def __repr__(self) -> str: + return repr(self._data) + + def __str__(self) -> str: + return str(self._data) + + def __eq__(self, other: object) -> bool: + if isinstance(other, Mapping): + return self._data == dict(other) + return False + + class _PolicyDefault: def __repr__(self) -> str: return "" _POLICY_DEFAULT = _PolicyDefault() +_INTERNAL_CARRIER_METALAYERS = frozenset({"b2o", "proxy", "proxy-source"}) +_C2_INTERNAL_VLMETA_KEYS = frozenset({"fill_nonce", "fill_state"}) _SENSITIVE_QUERY_PARTS = ( "credential", "signature", @@ -458,6 +512,10 @@ def __init__( self._runtime_cache = _carrier if cache_policy is blosc2.CachePolicy.DISK else None self._shared_runtime_cache = _runtime_cache_path is not None self._cache_status = None + self._cached_meta = None + self._cached_vlmeta = None + self._meta_mapping = None + self._vlmeta_mapping = None if cache_policy is blosc2.CachePolicy.DISK: if _runtime_cache_path is not None: @@ -473,6 +531,12 @@ def __init__( elif cache_policy is blosc2.CachePolicy.MEMORY: self._attach_carrier_cache() + if self._carrier is not None: + if self._cached_meta is None: + self._cached_meta = self._meta_from_carrier(self._carrier) + if self._cached_vlmeta is None: + self._cached_vlmeta = read_b2object_user_vlmeta(self._carrier) + def _runtime_source(self, original): """Keep credentials in live process state, outside the descriptor.""" if isinstance(self.src, blosc2.C2Array): @@ -526,6 +590,11 @@ def _open_or_create_carrier(self, cache_dir, cache_path): if stored is not None and current is not None and stored != current else "reused" ) + if status == "reused": + if self._cached_meta is None: + self._cached_meta = self._meta_from_carrier(carrier) + if self._cached_vlmeta is None: + self._cached_vlmeta = read_b2object_user_vlmeta(carrier) return carrier, status carrier = self._to_b2object_carrier(urlpath=path, contiguous=True, mode="w") return carrier, "created" @@ -553,6 +622,11 @@ def _open_or_create_sparse_cache(self, cache_path): if stored is not None and current is not None and stored != current else "reused" ) + if status == "reused": + if self._cached_meta is None: + self._cached_meta = self._meta_from_carrier(runtime) + if self._cached_vlmeta is None: + self._cached_vlmeta = read_b2object_user_vlmeta(runtime) return runtime, status if self._carrier is not None: @@ -592,6 +666,7 @@ def _import_warm_seed(self, seed, runtime) -> None: "proxy-fetched-bpc", "proxy-index", "proxy-stamp", + _B2OBJECT_USER_VLMETA_KEY, ): value = seed_schunk.vlmeta.get(name) if value is not None: @@ -888,6 +963,10 @@ def _prepare_read(self): self._validate_geometry(self._expected_geometry, src=fresh) self.src = fresh self._attach_carrier_cache() + self._cached_meta = None + self._cached_vlmeta = None + self._meta_mapping = None + self._vlmeta_mapping = None return self.src if self._proxy is None else self._proxy @@ -967,6 +1046,68 @@ def assume_immutable(self) -> bool: """Whether reads skip remote identity checks.""" return self._assume_immutable + @property + def meta(self) -> RemoteMetadataMapping: + """The fixed-length metalayers of the remote array.""" + self._prepare_read() + if self._cached_meta is None: + self._cached_meta = self._fetch_meta() + self._meta_mapping = None + if self._meta_mapping is None: + self._meta_mapping = RemoteMetadataMapping(self._cached_meta) + return self._meta_mapping + + @property + def vlmeta(self) -> RemoteMetadataMapping: + """The variable-length metadata of the remote array.""" + self._prepare_read() + if self._cached_vlmeta is None: + self._cached_vlmeta = self._fetch_vlmeta() + self._vlmeta_mapping = None + if self._vlmeta_mapping is None: + self._vlmeta_mapping = RemoteMetadataMapping(self._cached_vlmeta) + return self._vlmeta_mapping + + @staticmethod + def _meta_from_carrier(carrier) -> dict[str, Any]: + carrier_schunk = getattr(carrier, "schunk", carrier) + return { + name: carrier_schunk.meta[name] + for name in carrier_schunk.meta + if name not in _INTERNAL_CARRIER_METALAYERS + } + + def _fetch_meta(self) -> dict[str, Any]: + if isinstance(self.src, blosc2.C2Array): + # The Caterva2 REST API (api/info) does not carry Blosc2 fixed metalayers. + return {} + meta = getattr(self.src, "meta", None) + if meta is not None and isinstance(meta, Mapping): + return {name: meta[name] for name in meta if name not in _INTERNAL_CARRIER_METALAYERS} + return {} + + def _fetch_vlmeta(self) -> dict[str, Any]: + vlmeta = getattr(self.src, "vlmeta", None) + if vlmeta is not None and isinstance(vlmeta, Mapping): + if isinstance(self.src, blosc2.C2Array): + res = {k: v for k, v in vlmeta.items() if k not in _C2_INTERNAL_VLMETA_KEYS} + elif isinstance(vlmeta, dict): + res = vlmeta.copy() + else: + res = dict(vlmeta) + else: + res = {} + if self._runtime_cache is not None: + cache_lock = ( + self._runtime_cache.holding_lock() + if self._shared_runtime_cache and hasattr(self._runtime_cache, "holding_lock") + else nullcontext() + ) + with self._operation_lock, cache_lock: + with contextlib.suppress(Exception): + write_b2object_user_vlmeta(self._runtime_cache, res) + return res + @property def schunk(self): """The underlying carrier's or cache's :class:`SChunk`, or None if unattached.""" @@ -1118,15 +1259,21 @@ def _payload(self): } def _to_b2object_carrier(self, **kwargs): + carrier_excluded = _INTERNAL_CARRIER_METALAYERS | {"b2nd"} if self._carrier is not None: kwargs.setdefault( "meta", { name: self._carrier.schunk.meta[name] for name in self._carrier.schunk.meta - if name not in {"b2nd", "b2o", "proxy"} + if name not in carrier_excluded }, ) + else: + kwargs.setdefault( + "meta", + {name: self.meta[name] for name in self.meta if name not in carrier_excluded}, + ) array = make_b2object_carrier( "remote_proxy", self.shape, @@ -1137,6 +1284,9 @@ def _to_b2object_carrier(self, **kwargs): **kwargs, ) write_b2object_payload(array, self._payload()) + user_vlmeta = dict(self.vlmeta) + if user_vlmeta: + write_b2object_user_vlmeta(array, user_vlmeta) if self._source.get("kind") == "hdf5": refs = getattr(self.src, "_refs", None) if refs is not None: @@ -1247,6 +1397,11 @@ def _from_payload(cls, payload, carrier): **kwargs, ) obj._validate_geometry(expected) + if carrier is not None: + if obj._cached_meta is None: + obj._cached_meta = obj._meta_from_carrier(carrier) + if obj._cached_vlmeta is None: + obj._cached_vlmeta = read_b2object_user_vlmeta(carrier) return obj def __enter__(self): diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 4bb5b617b..5c648ef19 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -123,7 +123,7 @@ def __setitem__(self, name, content): def __getitem__(self, name): _ = self._owner # dead-owner check: the raw C schunk pointer below dangles otherwise if isinstance(name, slice): - if name.start is None and name.stop is None: + if name.start is None and name.stop is None and name.step is None: # Return all the vlmetalayers return self.getall() raise NotImplementedError("Slicing is not supported, unless [:]") @@ -212,7 +212,7 @@ def __getitem__(self, item: str | slice) -> bytes | dict[str, bytes]: a dictionary with all the metalayers is returned. """ if isinstance(item, slice): - if item.start is None and item.stop is None: + if item.start is None and item.stop is None and item.step is None: return self.getall() raise NotImplementedError("Slicing is not supported, unless [:]") if self.__contains__(item): diff --git a/src/blosc2/zarr_source.py b/src/blosc2/zarr_source.py index 21380e0ae..868069e1d 100644 --- a/src/blosc2/zarr_source.py +++ b/src/blosc2/zarr_source.py @@ -200,6 +200,13 @@ def dtype(self) -> np.dtype: def cparams(self): return self._cparams + @property + def vlmeta(self) -> dict: + try: + return dict(self.array.attrs) + except Exception: + return {} + def get_chunk(self, nchunk: int) -> bytes: return zarr_chunk_to_blosc2( self.array, diff --git a/tests/ndarray/test_proxy.py b/tests/ndarray/test_proxy.py index 8e6d88657..3f3582880 100644 --- a/tests/ndarray/test_proxy.py +++ b/tests/ndarray/test_proxy.py @@ -377,6 +377,18 @@ def test_vlmeta_cannot_overwrite_proxy_state(): assert proxy.vlmeta["mine"] == "ok" +def test_proxy_meta_and_vlmeta_access(): + source = blosc2.asarray( + np.arange(20).reshape(4, 5), + chunks=(2, 5), + blocks=(1, 5), + ) + proxy = blosc2.Proxy(source, meta={"extra": 123}, vlmeta={"mine": "ok"}) + assert "b2nd" in proxy.meta + assert proxy.meta["extra"] == 123 + assert proxy.vlmeta["mine"] == "ok" + + def test_the_proxy_module_still_answers_for_the_source_names(): # They live in `blosc2.proxy_source` now, so that the modules bound early in # `blosc2/__init__` can reach them without dragging `proxy.py` in ahead of diff --git a/tests/test_b2z_source.py b/tests/test_b2z_source.py index 52c7c94d0..c58c1649c 100644 --- a/tests/test_b2z_source.py +++ b/tests/test_b2z_source.py @@ -247,6 +247,75 @@ def test_geometry_change_and_expression(tmp_path): blosc2.open(path) +def test_b2z_metadata_and_caching(): + data = np.arange(100, dtype=np.int32) + array = blosc2.asarray( + data, + chunks=(20,), + blocks=(10,), + meta={"sensor_info": {"model": "X1", "rate": 100}}, + ) + array.vlmeta["experiment_notes"] = {"comment": "test run", "valid": True} + plain_array = blosc2.asarray(data, chunks=(20,), blocks=(10,)) + + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_STORED) as archive: + archive.writestr("d0/with_meta.b2nd", array.to_cframe()) + archive.writestr("d0/plain.b2nd", plain_array.to_cframe()) + + fs = fsspec.filesystem("memory") + fs.pipe_file("meta_test.b2z", buffer.getvalue()) + + # 1. Test B2ZNDSource directly + src = blosc2.B2ZNDSource("memory://meta_test.b2z", "d0/with_meta") + assert src.has_vlmetalayers + assert "b2nd" in src.meta + assert src.meta["sensor_info"] == {"model": "X1", "rate": 100} + assert src.vlmeta["experiment_notes"] == {"comment": "test run", "valid": True} + + plain_src = blosc2.B2ZNDSource("memory://meta_test.b2z", "d0/plain") + assert not plain_src.has_vlmetalayers + assert "b2nd" in plain_src.meta + assert "sensor_info" not in plain_src.meta + plain_src.traffic.reset() + assert plain_src.vlmeta == {} + assert plain_src.traffic.requests == 0 + + # 2. Test RemoteProxy over B2Z + proxy = blosc2.open("memory://meta_test.b2z", dataset="d0/with_meta", lazy=True) + assert isinstance(proxy.meta, blosc2.RemoteMetadataMapping) + assert isinstance(proxy.vlmeta, blosc2.RemoteMetadataMapping) + assert proxy.meta["sensor_info"] == {"model": "X1", "rate": 100} + assert proxy.meta.get("sensor_info") == {"model": "X1", "rate": 100} + assert proxy.meta.get("nonexistent", "fallback") == "fallback" + assert "b2nd" in proxy.meta + assert "sensor_info" in proxy.meta + assert len(proxy.meta) >= 2 + assert proxy.meta[:] == proxy.meta.getall() + with pytest.raises(TypeError): + proxy.meta["new_meta"] = 123 + with pytest.raises(TypeError): + del proxy.meta["sensor_info"] + + assert proxy.vlmeta["experiment_notes"] == {"comment": "test run", "valid": True} + assert proxy.vlmeta.get("experiment_notes") == {"comment": "test run", "valid": True} + assert "experiment_notes" in proxy.vlmeta + assert len(proxy.vlmeta) == 1 + assert proxy.vlmeta[:] == {"experiment_notes": {"comment": "test run", "valid": True}} + with pytest.raises(TypeError): + proxy.vlmeta["new_vlmeta"] = 123 + with pytest.raises(TypeError): + del proxy.vlmeta["experiment_notes"] + + # In-memory caching: subsequent accesses issue 0 network traffic + proxy.traffic.reset() + _ = proxy.meta["sensor_info"] + _ = proxy.vlmeta["experiment_notes"] + _ = proxy.meta[:] + _ = proxy.vlmeta[:] + assert proxy.traffic.requests == 0 + + def test_optional_dependencies(): subprocess.run( [ diff --git a/tests/test_hdf5_source.py b/tests/test_hdf5_source.py index 345edd6ca..691171d38 100644 --- a/tests/test_hdf5_source.py +++ b/tests/test_hdf5_source.py @@ -573,6 +573,19 @@ def test_s3_hdf5_nested_datasets(): assert val.shape == (3, 3) +def test_hdf5_vlmeta(tmp_path): + path = str(tmp_path / "test_attrs.h5") + data = np.arange(100, dtype=np.int32).reshape(10, 10) + with h5py.File(path, "w") as f: + ds = f.create_dataset("d0/data", data=data, chunks=(5, 5)) + ds.attrs["description"] = "hdf5 dataset" + ds.attrs["sampling_rate"] = 250 + + src = blosc2.HDF5NDSource(path, "d0/data") + assert src.vlmeta["description"] == "hdf5 dataset" + assert src.vlmeta["sampling_rate"] == 250 + + @pytest.mark.network def test_s3_hdf5_matches_zarr(): pytest.importorskip("s3fs") diff --git a/tests/test_remote_proxy.py b/tests/test_remote_proxy.py index e51b89f98..d624004d9 100644 --- a/tests/test_remote_proxy.py +++ b/tests/test_remote_proxy.py @@ -1067,3 +1067,250 @@ def test_sparse_seed_with_dirty_marker_is_not_imported(tmp_path): runtime = blosc2.RemoteProxy.with_sparse_cache(url, tmp_path / "runtime", carrier=seed.cache) assert not runtime.cache_contains(nchunk=0) np.testing.assert_array_equal(runtime[:], data) + + +def test_remote_proxy_metadata_access_and_caching(): + data = np.arange(20_000, dtype=np.int32) + array = blosc2.asarray( + data, + chunks=(10_000,), + blocks=(5_000,), + meta={"experiment": {"id": 42, "user": "alice"}}, + ) + array.vlmeta["notes"] = {"status": "calibrated", "tags": ["optical", "v2"]} + url = "memory://metadata-test.b2nd" + fsspec.filesystem("memory").pipe_file("metadata-test.b2nd", array.to_cframe()) + + proxy = blosc2.RemoteProxy(url, cache_policy=blosc2.CachePolicy.MEMORY) + + # Fixed-length metadata + meta = proxy.meta + assert isinstance(meta, blosc2.RemoteMetadataMapping) + assert "b2nd" in meta + assert "experiment" in meta + assert meta["experiment"] == {"id": 42, "user": "alice"} + assert meta.get("experiment") == {"id": 42, "user": "alice"} + assert meta.get("missing", 999) == 999 + assert len(meta) >= 2 + assert meta[:] == meta.getall() + assert meta.copy() == meta.getall() + assert meta == meta.getall() + assert repr(meta) == repr(meta.getall()) + assert str(meta) == str(meta.getall()) + with pytest.raises(TypeError): + meta["new_key"] = 1 + with pytest.raises(TypeError): + del meta["experiment"] + with pytest.raises(NotImplementedError, match="Slicing is not supported"): + _ = meta[0:1] + + # Variable-length metadata + vlmeta = proxy.vlmeta + assert isinstance(vlmeta, blosc2.RemoteMetadataMapping) + assert "notes" in vlmeta + assert vlmeta["notes"] == {"status": "calibrated", "tags": ["optical", "v2"]} + assert vlmeta.get("notes") == {"status": "calibrated", "tags": ["optical", "v2"]} + assert len(vlmeta) == 1 + assert vlmeta[:] == {"notes": {"status": "calibrated", "tags": ["optical", "v2"]}} + with pytest.raises(TypeError): + vlmeta["new_key"] = 1 + with pytest.raises(TypeError): + del vlmeta["notes"] + + # In-memory caching: subsequent accesses issue 0 network traffic + proxy.traffic.reset() + _ = proxy.meta["experiment"] + _ = proxy.vlmeta["notes"] + _ = proxy.meta[:] + _ = proxy.vlmeta[:] + assert proxy.traffic.requests == 0 + + +def test_remote_proxy_disk_cache_persists_metadata_and_offline_reopen(tmp_path): + data = np.arange(20_000, dtype=np.int32) + array = blosc2.asarray( + data, + chunks=(10_000,), + blocks=(5_000,), + meta={"exp_header": "test_disk_meta"}, + ) + array.vlmeta["exp_trailer"] = [1, 2, 3] + url = "memory://disk-meta-test.b2nd" + fsspec.filesystem("memory").pipe_file("disk-meta-test.b2nd", array.to_cframe()) + + cache_path = tmp_path / "disk-meta-cache.b2nd" + first = blosc2.RemoteProxy(url, cache_policy=blosc2.CachePolicy.DISK, cache_path=cache_path) + assert first.meta["exp_header"] == "test_disk_meta" + assert first.vlmeta["exp_trailer"] == [1, 2, 3] + + # Reopen existing DISK cache + reopened = blosc2.RemoteProxy(url, cache_policy=blosc2.CachePolicy.DISK, cache_path=cache_path) + reopened.traffic.reset() + assert reopened.meta["exp_header"] == "test_disk_meta" + assert reopened.vlmeta["exp_trailer"] == [1, 2, 3] + # Reading metadata from existing carrier issues 0 remote requests! + assert reopened.traffic.requests == 0 + + # Open carrier directly using blosc2.open() + opened = blosc2.open(cache_path) + assert isinstance(opened, blosc2.RemoteProxy) + opened.traffic.reset() + assert opened.meta["exp_header"] == "test_disk_meta" + assert opened.vlmeta["exp_trailer"] == [1, 2, 3] + assert opened.traffic.requests == 0 + + +def test_remote_proxy_export_preserves_metadata(tmp_path): + data = np.arange(10_000, dtype=np.int32) + array = blosc2.asarray( + data, + chunks=(5_000,), + blocks=(2_500,), + meta={"export_meta": {"model": "sensor_a"}}, + ) + array.vlmeta["export_vlmeta"] = {"quality": "high"} + url = "memory://export-meta-test.b2nd" + fsspec.filesystem("memory").pipe_file("export-meta-test.b2nd", array.to_cframe()) + + proxy = blosc2.RemoteProxy(url, cache_policy=blosc2.CachePolicy.MEMORY) + + # 1. Export via save() + save_path = tmp_path / "saved_carrier.b2nd" + proxy.save(save_path) + saved_proxy = blosc2.open(save_path) + assert isinstance(saved_proxy, blosc2.RemoteProxy) + assert saved_proxy.meta["export_meta"] == {"model": "sensor_a"} + assert saved_proxy.vlmeta["export_vlmeta"] == {"quality": "high"} + + # 2. Export via to_cframe() + cframe = proxy.to_cframe() + restored_proxy = blosc2.from_cframe(cframe) + assert isinstance(restored_proxy, blosc2.RemoteProxy) + assert restored_proxy.meta["export_meta"] == {"model": "sensor_a"} + assert restored_proxy.vlmeta["export_vlmeta"] == {"quality": "high"} + + +def test_remote_proxy_invalidation_refetches_metadata(): + data1 = np.arange(10_000, dtype=np.int32) + array1 = blosc2.asarray(data1, chunks=(5_000,), blocks=(2_500,), meta={"v": 1}) + array1.vlmeta["note"] = "version 1" + url = "memory://mutable-meta.b2nd" + fsspec.filesystem("memory").pipe_file("mutable-meta.b2nd", array1.to_cframe()) + + proxy = blosc2.RemoteProxy(url, assume_immutable=False) + assert proxy.meta["v"] == 1 + assert proxy.vlmeta["note"] == "version 1" + + # Replace remote array with new version + data2 = np.arange(10_000, dtype=np.int32) + 100 + array2 = blosc2.asarray(data2, chunks=(5_000,), blocks=(2_500,), meta={"v": 2}) + array2.vlmeta["note"] = "version 2" + fsspec.filesystem("memory").pipe_file("mutable-meta.b2nd", array2.to_cframe()) + + # proxy detects the change and returns updated metadata + assert proxy.meta["v"] == 2 + assert proxy.vlmeta["note"] == "version 2" + + +def test_zarr_source_vlmeta(): + zarr = pytest.importorskip("zarr") + zarr_url = "memory://test_attrs.zarr" + mapper = fsspec.get_mapper(zarr_url) + z_arr = zarr.open_array( + store=mapper, + mode="w", + shape=(100,), + chunks=(50,), + dtype="i4", + ) + z_arr[:] = np.arange(100, dtype=np.int32) + z_arr.attrs["author"] = "researcher" + z_arr.attrs["dataset_id"] = 12345 + + src = blosc2.ZarrNDSource(zarr_url) + assert src.vlmeta["author"] == "researcher" + assert src.vlmeta["dataset_id"] == 12345 + + proxy = blosc2.RemoteProxy(zarr_url, source_format="zarr") + assert proxy.vlmeta["author"] == "researcher" + assert proxy.vlmeta["dataset_id"] == 12345 + + +def test_caterva2_vlmeta_filters_internal_keys(monkeypatch): + def fake_info(path, urlbase, params=None, headers=None, model=None, auth_token=None, traffic=None): + return { + "shape": [10], + "chunks": [5], + "blocks": [5], + "dtype": np.dtype(np.int32).str, + "schunk": { + "cparams": dict(blosc2.cparams_dflts), + "vlmeta": { + "fill_nonce": "secret_nonce_123", + "fill_state": "complete", + "user_tag": "public_data", + }, + }, + } + + monkeypatch.setattr(blosc2_c2array, "info", fake_info) + remote = blosc2.RemoteProxy(blosc2.URLPath("@public/test-vlmeta.b2nd", urlbase="https://example.org/c2")) + + # Caterva2 fixed metalayers are not supported by api/info, returns empty + assert remote.meta == {} + # Internal keys fill_nonce and fill_state are filtered out + assert remote.vlmeta == {"user_tag": "public_data"} + + # Export to carrier and verify roundtrip doesn't persist internal keys + cframe = remote.to_cframe() + restored = blosc2.from_cframe(cframe) + assert restored.vlmeta == {"user_tag": "public_data"} + + +def test_remote_proxy_filters_carrier_internal_metalayers(): + data = np.arange(10, dtype=np.int32) + meta = { + "user_key": "val1", + "b2o": {"kind": "carrier"}, + "proxy": {"foo": "bar"}, + "proxy-source": {"bar": "baz"}, + } + carrier = blosc2.asarray(data, chunks=(5,), blocks=(5,), meta=meta) + + url = "memory://carrier-leak-test.b2nd" + fsspec.filesystem("memory").pipe_file("carrier-leak-test.b2nd", carrier.to_cframe()) + + proxy = blosc2.RemoteProxy(url) + assert "user_key" in proxy.meta + assert "b2nd" in proxy.meta + assert "b2o" not in proxy.meta + assert "proxy" not in proxy.meta + assert "proxy-source" not in proxy.meta + + +def test_remote_proxy_metadata_slicing_and_caching(): + data = np.arange(10, dtype=np.int32) + arr = blosc2.asarray(data, chunks=(5,), blocks=(5,), meta={"foo": "bar"}) + arr.vlmeta["desc"] = "test" + url = "memory://slice-test.b2nd" + fsspec.filesystem("memory").pipe_file("slice-test.b2nd", arr.to_cframe()) + + proxy = blosc2.RemoteProxy(url) + # [:] full slice returns copy of dict + assert proxy.meta[:] == dict(proxy.meta) + assert proxy.vlmeta[:] == dict(proxy.vlmeta) + + # [::2] stepped slice raises NotImplementedError + with pytest.raises(NotImplementedError, match="Slicing is not supported, unless"): + _ = proxy.meta[::2] + with pytest.raises(NotImplementedError, match="Slicing is not supported, unless"): + _ = proxy.vlmeta[::2] + + # Repeated access returns the cached mapping instance without re-wrapping + meta1 = proxy.meta + meta2 = proxy.meta + assert meta1 is meta2 + + vlmeta1 = proxy.vlmeta + vlmeta2 = proxy.vlmeta + assert vlmeta1 is vlmeta2 diff --git a/tests/test_vlmeta.py b/tests/test_vlmeta.py index 2f729d1a8..51bc5eac1 100644 --- a/tests/test_vlmeta.py +++ b/tests/test_vlmeta.py @@ -107,6 +107,11 @@ def test_vlmeta_supports_ref_roundtrip(tmp_path): assert bulk["ref"] == ref np.testing.assert_array_equal(bulk["ref"].open()[:], array[:]) + with pytest.raises(NotImplementedError, match="Slicing is not supported, unless"): + _ = schunk.vlmeta[::2] + with pytest.raises(NotImplementedError, match="Slicing is not supported, unless"): + _ = schunk.meta[::2] + def delete(schunk): # Remove one of them From 00a1ab8edfea32efb69d2db605699cff7931290e Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 8 Sep 2026 12:52:24 +0200 Subject: [PATCH 33/89] Add RemoteProxy attrs alias and filter HDF5 dimension metadata --- doc/guides/remote_arrays.md | 5 +++++ doc/reference/remoteproxy.rst | 3 +++ src/blosc2/hdf5_source.py | 3 ++- src/blosc2/remote_proxy.py | 5 +++++ tests/test_hdf5_source.py | 8 ++++++++ tests/test_remote_proxy.py | 4 ++++ 6 files changed, 27 insertions(+), 1 deletion(-) diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index abd07f966..65f429f15 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -4,6 +4,11 @@ Blosc2 can open remote arrays without downloading them first. Metadata is read i All lazy remote array access in Python-Blosc2 is unified under {ref}`RemoteProxy`. +Read user attributes with `a.attrs["name"]` or get them all with `a.attrs[:]`. +`RemoteProxy.attrs` is a read-only alias for `RemoteProxy.vlmeta` and shares its +metadata cache across all source formats. HDF5 attributes exclude the +`_ARRAY_DIMENSIONS` metadata added by Kerchunk during translation. + ## Choose a remote route The argument passed to {func}`blosc2.open` selects the route: diff --git a/doc/reference/remoteproxy.rst b/doc/reference/remoteproxy.rst index a5df7f662..6cae2c474 100644 --- a/doc/reference/remoteproxy.rst +++ b/doc/reference/remoteproxy.rst @@ -200,6 +200,9 @@ file. Read-only mode can use warm chunks but does not retain misses: .. autoattribute:: blocks .. autoattribute:: cparams .. autoattribute:: nbytes + .. autoattribute:: meta + .. autoattribute:: vlmeta + .. autoattribute:: attrs .. autoattribute:: info .. autoattribute:: cache .. autoattribute:: cache_bytes diff --git a/src/blosc2/hdf5_source.py b/src/blosc2/hdf5_source.py index ffdd62bce..e81606532 100644 --- a/src/blosc2/hdf5_source.py +++ b/src/blosc2/hdf5_source.py @@ -403,7 +403,8 @@ def cparams(self): @property def vlmeta(self) -> dict: try: - return dict(self.array.attrs) + # Kerchunk adds dimension metadata to the translated Zarr attributes. + return {key: value for key, value in self.array.attrs.items() if key != "_ARRAY_DIMENSIONS"} except Exception: return {} diff --git a/src/blosc2/remote_proxy.py b/src/blosc2/remote_proxy.py index f9ca96b94..bb56813bd 100644 --- a/src/blosc2/remote_proxy.py +++ b/src/blosc2/remote_proxy.py @@ -1068,6 +1068,11 @@ def vlmeta(self) -> RemoteMetadataMapping: self._vlmeta_mapping = RemoteMetadataMapping(self._cached_vlmeta) return self._vlmeta_mapping + @property + def attrs(self) -> RemoteMetadataMapping: + """Read-only alias for :attr:`vlmeta`, sharing its metadata cache.""" + return self.vlmeta + @staticmethod def _meta_from_carrier(carrier) -> dict[str, Any]: carrier_schunk = getattr(carrier, "schunk", carrier) diff --git a/tests/test_hdf5_source.py b/tests/test_hdf5_source.py index 691171d38..18f97046c 100644 --- a/tests/test_hdf5_source.py +++ b/tests/test_hdf5_source.py @@ -584,6 +584,14 @@ def test_hdf5_vlmeta(tmp_path): src = blosc2.HDF5NDSource(path, "d0/data") assert src.vlmeta["description"] == "hdf5 dataset" assert src.vlmeta["sampling_rate"] == 250 + assert "_ARRAY_DIMENSIONS" not in src.vlmeta + assert "_ARRAY_DIMENSIONS" in src.array.attrs + + url = "memory://test_attrs.h5" + fsspec.filesystem("memory").pipe_file(url, Path(path).read_bytes()) + proxy = blosc2.RemoteProxy(url, source_format="hdf5", dataset="d0/data") + assert proxy.attrs is proxy.vlmeta + assert proxy.attrs[:] == {"description": "hdf5 dataset", "sampling_rate": 250} @pytest.mark.network diff --git a/tests/test_remote_proxy.py b/tests/test_remote_proxy.py index d624004d9..a40147e28 100644 --- a/tests/test_remote_proxy.py +++ b/tests/test_remote_proxy.py @@ -1106,6 +1106,7 @@ def test_remote_proxy_metadata_access_and_caching(): # Variable-length metadata vlmeta = proxy.vlmeta + assert proxy.attrs is vlmeta assert isinstance(vlmeta, blosc2.RemoteMetadataMapping) assert "notes" in vlmeta assert vlmeta["notes"] == {"status": "calibrated", "tags": ["optical", "v2"]} @@ -1123,6 +1124,7 @@ def test_remote_proxy_metadata_access_and_caching(): _ = proxy.vlmeta["notes"] _ = proxy.meta[:] _ = proxy.vlmeta[:] + _ = proxy.attrs["notes"] assert proxy.traffic.requests == 0 @@ -1210,6 +1212,7 @@ def test_remote_proxy_invalidation_refetches_metadata(): # proxy detects the change and returns updated metadata assert proxy.meta["v"] == 2 assert proxy.vlmeta["note"] == "version 2" + assert proxy.attrs["note"] == "version 2" def test_zarr_source_vlmeta(): @@ -1234,6 +1237,7 @@ def test_zarr_source_vlmeta(): proxy = blosc2.RemoteProxy(zarr_url, source_format="zarr") assert proxy.vlmeta["author"] == "researcher" assert proxy.vlmeta["dataset_id"] == 12345 + assert proxy.attrs is proxy.vlmeta def test_caterva2_vlmeta_filters_internal_keys(monkeypatch): From ee7d4fd1aa92a35f76bb603442403a049be877b7 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 8 Sep 2026 13:12:02 +0200 Subject: [PATCH 34/89] Read public Caterva2 attributes in remote arrays --- doc/guides/remote_arrays.md | 6 ++ plans/remote-proxy-v11.md | 117 ++++++++++++++++++++++++++++++++++++ src/blosc2/c2array.py | 11 ++++ src/blosc2/remote_proxy.py | 6 +- tests/test_remote_proxy.py | 32 ++++++++-- 5 files changed, 165 insertions(+), 7 deletions(-) create mode 100644 plans/remote-proxy-v11.md diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index 65f429f15..cf6f1aed1 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -9,6 +9,12 @@ Read user attributes with `a.attrs["name"]` or get them all with `a.attrs[:]`. metadata cache across all source formats. HDF5 attributes exclude the `_ARRAY_DIMENSIONS` metadata added by Kerchunk during translation. +For Caterva2 sources, user attributes come from the `attrs` field in `/api/info`. +`C2Array.attrs` uses the same field, with a fallback to variable metadata for +older servers. Its `vlmeta` property retains the original protocol metadata. +These properties do not write attributes to the server. For a saved remote +proxy served by Caterva2, attributes reflect the snapshot stored in its carrier. + ## Choose a remote route The argument passed to {func}`blosc2.open` selects the route: diff --git a/plans/remote-proxy-v11.md b/plans/remote-proxy-v11.md new file mode 100644 index 000000000..16ab0fe88 --- /dev/null +++ b/plans/remote-proxy-v11.md @@ -0,0 +1,117 @@ +# RemoteProxy v11: Caterva2 user attributes + +## Goal + +Expose consistent user attributes for arrays served through Caterva2, including +ordinary Blosc2 arrays, B2Z leaves, HDF5 leaves, and saved RemoteProxy arrays. +Reuse `/api/info` so reading attributes requires no additional endpoint or request. + +The preceding Python-Blosc2 change adds `RemoteProxy.attrs` as a read-only alias +for `RemoteProxy.vlmeta` and filters Kerchunk's `_ARRAY_DIMENSIONS` from +`HDF5NDSource.vlmeta`. + +## Current behavior and gap + +Caterva2 already exposes variable metadata through `schunk.vlmeta` in array +metadata responses and through `File.vlmeta` in its Python client. HDF5 adapters +already copy dataset attributes into their backing Blosc2 array's variable +metadata. The web metadata panel displays this mapping directly. + +Saved RemoteProxy arrays are different: `services/server.py:get_info` replaces +their variable metadata with only the `b2o` descriptor to avoid exposing binary +cache bookkeeping. This also discards the saved user attributes, which reside +under `_b2o_user_vlmeta` in the carrier. Python-Blosc2 already provides +`blosc2.b2objects.read_b2object_user_vlmeta()` to retrieve them. + +## Proposed changes + +### 1. Add public attributes to the existing response + +Add an `attrs` mapping to the relevant `/api/info` metadata models. Populate it +in Caterva2's shared metadata builder (`services/srv_utils.py:read_metadata`) +so standalone arrays and container leaves use the same behavior: + +- Ordinary Blosc2 arrays: expose user variable metadata. +- HDF5 leaves and legacy HDF5 proxies: expose dataset attributes, excluding + adapter bookkeeping such as `_ftype` and `_dsetname`. +- Saved RemoteProxy arrays: use `read_b2object_user_vlmeta()` on the raw carrier, + without resolving or fetching the remote array. + +Preserve the existing `schunk.vlmeta` response contract, including control +information used by existing clients. Keep public attributes separate from +proxy descriptors, cache bookkeeping, and fill protocol fields. Identify +internal fields explicitly; do not remove every underscore-prefixed user key. + +Keep the new field optional during compatibility handling: distinguish an +absent field from an explicitly empty mapping. Ensure peer responses and their +model conversion preserve the field. + +### 2. Expose attributes in the clients + +Add `.attrs` to Caterva2's `File` class (inherited by its dataset objects) and +Python-Blosc2's `C2Array`. Prefer the new response field; fall back to the +existing variable metadata when talking to an older server. Reuse each client's +existing metadata cache and refresh behavior. + +Update `RemoteProxy`'s Caterva2 metadata path to consume these public attributes. +Its `.attrs` and `.vlmeta` must continue to return the same read-only mapping. +Preserve existing `.vlmeta` compatibility in the lower-level clients, since +their callers may depend on protocol fields there. + +### 3. Update the web metadata panel and documentation + +Have `services/templates/includes/info_metadata.html` display `attrs`, with a +fallback for older metadata responses. Label the section "Attributes" and avoid +showing proxy descriptors or cache bookkeeping as user attributes. + +Document the new response field, client access, read-only remote behavior, and +older-server fallback. Explain that saved proxy attributes reflect the metadata +stored in the carrier; this change does not introduce remote metadata refreshes +on the server. + +### 4. Verify the complete route + +Extend existing tests rather than introducing a new test harness: + +- `/api/info` preserves scalar and nested user attributes for ordinary Blosc2 + arrays, B2Z leaves, HDF5 leaves, and saved RemoteProxy arrays. +- Saved proxies expose their stored user attributes without resolving their + source or exposing cache internals. +- Existing descriptor and fill protocol metadata remain available to callers + that use the existing response fields. +- Caterva2 `.attrs`, C2Array `.attrs`, and RemoteProxy `.attrs` return the + expected attributes; RemoteProxy `.attrs` and `.vlmeta` share their cache. +- Missing `attrs` falls back for older servers; explicit empty `attrs` stays + empty. Peer forwarding and the web metadata panel preserve the result. + +Run the affected API, container, HDF5, remote-proxy, and client tests in the +repositories' prescribed environments, together with their lint checks. + +## Scope + +Implementation spans `/Users/faltet/ironArray/caterva2` and +`/Users/faltet/blosc/python-blosc2`. The implementation is complete in both +working trees. + +Attribute writes, a separate metadata endpoint, new serialization formats, and +server-side refresh of saved remote attributes are outside this change. + +## Implementation and validation + +Implemented the public `attrs` field, both client properties, RemoteProxy's +Caterva2 integration, and the web metadata panel. The file-less HDF5 leaf +adapter also needed to copy attributes using the existing HDF5 conversion; +its previous implementation copied only array geometry. + +Validation in the `blosc2` conda environment: + +- Python-Blosc2 RemoteProxy, open-C2Array, and HDF5 source modules: 121 passed. +- Focused Caterva2 metadata, client, HDF5, B2Z, table, saved-proxy, panel, and + peer checks: 21 passed. +- Broader Caterva2 API, HDF5, TreeStore, CTable, and RemoteProxy modules, plus + attribute checks: 296 passed, 110 skipped, two failures. +- Both failures reproduced against an unchanged Caterva2 HEAD snapshot: + `test_remote_proxy_download_can_omit_cache[CachePolicy.DISK]` encounters the + disabled-resolution policy on warm download, and + `test_dir_named_like_container` encounters `dataset requires lazy=True`. +- Ruff lint, formatting, and diff whitespace checks pass for the changed files. diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index f37a33e2f..ab63f79a3 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -1639,6 +1639,17 @@ def vlmeta(self) -> dict: self._refresh_meta() return self.meta["schunk"]["vlmeta"] + @property + def attrs(self) -> dict: + """User attributes; changing this mapping does not update the server. + + Uses the cached metadata, refreshing after writes as :attr:`vlmeta` + does. Older servers without public attributes fall back to ``vlmeta``. + """ + self._refresh_meta() + attrs = self.meta.get("attrs") + return self.vlmeta if attrs is None else attrs + @property def info(self) -> InfoReporter: """ diff --git a/src/blosc2/remote_proxy.py b/src/blosc2/remote_proxy.py index bb56813bd..ed91f6196 100644 --- a/src/blosc2/remote_proxy.py +++ b/src/blosc2/remote_proxy.py @@ -1092,9 +1092,11 @@ def _fetch_meta(self) -> dict[str, Any]: return {} def _fetch_vlmeta(self) -> dict[str, Any]: - vlmeta = getattr(self.src, "vlmeta", None) + vlmeta = ( + self.src.attrs if isinstance(self.src, blosc2.C2Array) else getattr(self.src, "vlmeta", None) + ) if vlmeta is not None and isinstance(vlmeta, Mapping): - if isinstance(self.src, blosc2.C2Array): + if isinstance(self.src, blosc2.C2Array) and self.src.meta.get("attrs") is None: res = {k: v for k, v in vlmeta.items() if k not in _C2_INTERNAL_VLMETA_KEYS} elif isinstance(vlmeta, dict): res = vlmeta.copy() diff --git a/tests/test_remote_proxy.py b/tests/test_remote_proxy.py index a40147e28..8deee7cd4 100644 --- a/tests/test_remote_proxy.py +++ b/tests/test_remote_proxy.py @@ -1240,9 +1240,16 @@ def test_zarr_source_vlmeta(): assert proxy.attrs is proxy.vlmeta -def test_caterva2_vlmeta_filters_internal_keys(monkeypatch): +@pytest.mark.parametrize( + "attrs", [None, {}, {"user_tag": {"nested": [1, True]}, "fill_state": "user value"}] +) +def test_caterva2_vlmeta_filters_internal_keys(monkeypatch, attrs): + requests = 0 + def fake_info(path, urlbase, params=None, headers=None, model=None, auth_token=None, traffic=None): - return { + nonlocal requests + requests += 1 + result = { "shape": [10], "chunks": [5], "blocks": [5], @@ -1256,19 +1263,34 @@ def fake_info(path, urlbase, params=None, headers=None, model=None, auth_token=N }, }, } + if attrs is not None: + result["attrs"] = attrs + return result monkeypatch.setattr(blosc2_c2array, "info", fake_info) remote = blosc2.RemoteProxy(blosc2.URLPath("@public/test-vlmeta.b2nd", urlbase="https://example.org/c2")) # Caterva2 fixed metalayers are not supported by api/info, returns empty assert remote.meta == {} - # Internal keys fill_nonce and fill_state are filtered out - assert remote.vlmeta == {"user_tag": "public_data"} + # Only legacy responses need client-side filtering of internal keys. + expected = {"user_tag": "public_data"} if attrs is None else attrs + assert remote.vlmeta == expected + assert remote.attrs is remote.vlmeta + assert remote.src.attrs == (remote.src.vlmeta if attrs is None else attrs) + assert remote.src.vlmeta["fill_nonce"] == "secret_nonce_123" # Export to carrier and verify roundtrip doesn't persist internal keys cframe = remote.to_cframe() restored = blosc2.from_cframe(cframe) - assert restored.vlmeta == {"user_tag": "public_data"} + assert restored.vlmeta == expected + + previous_requests = requests + _ = remote.src.attrs + assert requests == previous_requests + attrs = {"refreshed": True} + remote.src._forget_index() + assert remote.src.attrs == attrs + assert requests == previous_requests + 1 def test_remote_proxy_filters_carrier_internal_metalayers(): From 27df9e5bac20ccb04c719a36bca5b89780cab852 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 8 Sep 2026 13:26:09 +0200 Subject: [PATCH 35/89] Add remote array support to b2view --- doc/guides/b2view.rst | 16 ++++++++++++++++ src/blosc2/b2view/app.py | 4 +++- src/blosc2/b2view/cli.py | 12 ++++++++++-- src/blosc2/b2view/model.py | 17 +++++++++++++---- tests/b2view/test_basics.py | 20 ++++++++++++++++++++ tests/b2view/test_cli.py | 31 +++++++++++++++++++++++++++++++ tests/test_b2view_model.py | 31 +++++++++++++++++++++++++++++++ 7 files changed, 124 insertions(+), 7 deletions(-) diff --git a/doc/guides/b2view.rst b/doc/guides/b2view.rst index c58a772cd..e96ab3000 100644 --- a/doc/guides/b2view.rst +++ b/doc/guides/b2view.rst @@ -56,6 +56,22 @@ You can also jump straight to a node and panel: b2view sample.b2z /dense/a --panel data +Remote arrays +~~~~~~~~~~~~~ + +Remote array URLs open lazily, fetching data as you browse and caching reads +in memory. A dataset inside a B2Z archive opens as a single array at the +viewer root; include its path in the URL using ``/`` or ``::``: + +.. code-block:: console + + b2view s3://blosc2/hierarchy.b2z/d0/d1/a2 --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com --panel data + +``--profile`` and ``--endpoint-url`` are optional; when omitted, the S3 backend +uses its normal credential and endpoint configuration. Install ``s3fs`` for +S3 access. Remote ``.b2nd`` arrays and supported HDF5/Zarr dataset URLs use +the same viewer (with their corresponding backend dependencies installed). + Step 3 — Navigate the data panel -------------------------------- diff --git a/src/blosc2/b2view/app.py b/src/blosc2/b2view/app.py index f86426f82..f38948097 100644 --- a/src/blosc2/b2view/app.py +++ b/src/blosc2/b2view/app.py @@ -2021,10 +2021,12 @@ def __init__( preview_cols: int = 10, download_url: str | None = None, info_url: str | None = None, + storage_options: dict[str, Any] | None = None, ): super().__init__() self.sub_title = f"Python-Blosc2 {blosc2.__version__}" # shown beside the title in the header self.urlpath = urlpath + self.storage_options = storage_options self.download_url = download_url # when set, fetch urlpath before browsing self.info_url = info_url # optional: metadata endpoint giving the size # Header label: the path as given on the CLI, or the @public-relative @@ -2126,7 +2128,7 @@ def _after_download(self, result: bool | str) -> None: def _start_browsing(self) -> None: """Open the bundle and populate the tree (the normal startup path).""" - self.browser = StoreBrowser(self.urlpath) + self.browser = StoreBrowser(self.urlpath, storage_options=self.storage_options) self.query_one(B2ViewHeader).set_filename(self._header_label) tree = self.query_one("#tree", Tree) tree.root.data = "/" diff --git a/src/blosc2/b2view/cli.py b/src/blosc2/b2view/cli.py index e485347f2..f0e0ecee0 100644 --- a/src/blosc2/b2view/cli.py +++ b/src/blosc2/b2view/cli.py @@ -39,9 +39,11 @@ def resolve_source( def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Browse a Blosc2 TreeStore bundle in the terminal.") - parser.add_argument("urlpath", nargs="?", default=None, help="Path to a .b2d directory or .b2z file") + parser = argparse.ArgumentParser(description="Browse a Blosc2 bundle or array in the terminal.") + parser.add_argument("urlpath", nargs="?", default=None, help="Local path or remote array URL") parser.add_argument("path", nargs="?", default="/", help="Optional starting path inside the bundle") + parser.add_argument("--profile", help="S3 credential profile") + parser.add_argument("--endpoint-url", help="S3 endpoint URL") parser.add_argument( "--download", nargs="?", @@ -114,6 +116,12 @@ def main(argv: list[str] | None = None) -> int: preview_cols=args.preview_cols, download_url=download_url, info_url=info_url, + storage_options={ + key: value + for key, value in {"profile": args.profile, "endpoint_url": args.endpoint_url}.items() + if value is not None + } + or None, ) app.run(mouse=args.mouse) return 0 diff --git a/src/blosc2/b2view/model.py b/src/blosc2/b2view/model.py index 840445c8e..65c99784a 100644 --- a/src/blosc2/b2view/model.py +++ b/src/blosc2/b2view/model.py @@ -9,6 +9,7 @@ import numpy as np import blosc2 +from blosc2.core import is_fsspec_url, parse_container_url # Above this uncompressed size, plot_series does not read the whole series at # once for an exact min/max envelope. Local objects are instead streamed in @@ -251,9 +252,17 @@ class StoreBrowser: slices. """ - def __init__(self, urlpath: str): + def __init__(self, urlpath: str, *, storage_options: dict[str, Any] | None = None): self.urlpath = urlpath - self.store = blosc2.open(urlpath, mode="r") + open_options = {} + if is_fsspec_url(urlpath): + _, dataset, source_format = parse_container_url(urlpath) + # Whole B2Z bundles keep their existing TreeStore opening path. + if source_format != "b2z" or dataset is not None: + open_options["lazy"] = True + if storage_options is not None: + open_options["storage_options"] = storage_options + self.store = blosc2.open(urlpath, mode="r", **open_options) self.is_tree = isinstance(self.store, blosc2.TreeStore) # Per-path row filters for CTable nodes (path -> expr / where() view) self._filters: dict[str, str] = {} @@ -523,7 +532,7 @@ def _row_index(row_slice): n, np.dtype(obj.dtype).itemsize, chunks[row_dim] if chunks else None, - remote=(kind == "c2array"), + remote=(kind == "c2array" or isinstance(obj, blosc2.RemoteProxy)), max_points=max_points, ) @@ -1203,7 +1212,7 @@ def object_kind(obj: Any) -> str: """Return a stable b2view kind string for *obj*.""" if isinstance(obj, blosc2.TreeStore): return "group" - if isinstance(obj, blosc2.NDArray): + if isinstance(obj, (blosc2.NDArray, blosc2.RemoteProxy)): return "ndarray" if isinstance(obj, blosc2.CTable): return "ctable" diff --git a/tests/b2view/test_basics.py b/tests/b2view/test_basics.py index 2e8c340e0..c4318e909 100644 --- a/tests/b2view/test_basics.py +++ b/tests/b2view/test_basics.py @@ -144,6 +144,26 @@ def _assert_ctable_window_values(page, expected): # ── Tree and panel focus navigation ────────────────────────────────────── +async def test_remote_array_startup(tmp_path): + fsspec = pytest.importorskip("fsspec") + data = np.arange(4 * 60 * 80, dtype=np.int32).reshape(4, 60, 80) + path = tmp_path / "remote.b2z" + with blosc2.TreeStore(str(path), mode="w") as store: + store["/d0/d1/a2"] = data + fsspec.filesystem("memory").pipe_file("b2view-tui.b2z", path.read_bytes()) + app = B2ViewApp("memory://b2view-tui.b2z/d0/d1/a2", start_panel="data") + async with app.run_test(size=TERM_SIZE) as pilot: + await wait_for_table(pilot) + assert isinstance(app.browser.store, blosc2.RemoteProxy) + assert app._data_layout.shape == data.shape + page = app.table_page + for column in page["columns"]: + np.testing.assert_array_equal( + page["data"][column], data[0, page["start"] : page["stop"], int(column)] + ) + assert app.focused is app.query_one("#data-table", DataTable) + + async def _wait_focus(pilot, expected_id: str) -> str | None: """Pause until the focused widget is *expected_id* (or give up).""" for _ in range(30): diff --git a/tests/b2view/test_cli.py b/tests/b2view/test_cli.py index 9b7dd7f5f..ab9d32041 100644 --- a/tests/b2view/test_cli.py +++ b/tests/b2view/test_cli.py @@ -15,6 +15,7 @@ DEFAULT_DOWNLOAD_PATH, DOWNLOAD_BASE_URL, INFO_BASE_URL, + main, resolve_source, ) @@ -54,3 +55,33 @@ def test_download_and_positional_exclusive(): def test_no_source_is_an_error(): with pytest.raises(ValueError, match="provide a path"): resolve_source(None, None) + + +@pytest.mark.parametrize("options", [{}, {"profile": "blosc2", "endpoint_url": "https://s3.example.com"}]) +def test_remote_options_reach_open(monkeypatch, options): + pytest.importorskip("textual") + import blosc2 + from blosc2.b2view.app import B2ViewApp + + url = "s3://blosc2/hierarchy.b2z/d0/d1/a2" + opened = [] + monkeypatch.setattr(blosc2, "open", lambda path, **kwargs: opened.append((path, kwargs))) + + def run(app, **kwargs): + # Stop after the real startup opens its browser, before widget lookup. + def no_widgets(*args): + raise LookupError("No widgets in CLI test") + + monkeypatch.setattr(app, "query_one", no_widgets) + with pytest.raises(LookupError, match="No widgets"): + app._start_browsing() + + monkeypatch.setattr(B2ViewApp, "run", run) + argv = [url] + for key, value in options.items(): + argv.extend(["--" + key.replace("_", "-"), value]) + assert main(argv) == 0 + expected = {"mode": "r", "lazy": True} + if options: + expected["storage_options"] = options + assert opened == [(url, expected)] diff --git a/tests/test_b2view_model.py b/tests/test_b2view_model.py index 582f26340..9320616cb 100644 --- a/tests/test_b2view_model.py +++ b/tests/test_b2view_model.py @@ -74,6 +74,37 @@ def test_store_browser_metadata_and_previews(tmp_path): np.testing.assert_array_equal(preview["data"]["x"], np.array([0, 1, 2])) +@pytest.mark.parametrize("suffix", [".b2nd", ".b2z/d0/d1/a2", ".b2z::d0/d1/a2"]) +def test_store_browser_remote_array(tmp_path, suffix): + fsspec = pytest.importorskip("fsspec") + data = np.arange(4 * 60 * 80, dtype=np.int32).reshape(4, 60, 80) + array = blosc2.asarray(data, chunks=(1, 20, 20), blocks=(1, 10, 10)) + array.vlmeta["description"] = "remote preview" + fs = fsspec.filesystem("memory") + if suffix == ".b2nd": + fs.pipe_file("b2view-array.b2nd", array.to_cframe()) + else: + path = tmp_path / "bundle.b2z" + with blosc2.TreeStore(str(path), mode="w") as store: + store["/d0/d1/a2"] = array + fs.pipe_file("b2view-array.b2z", path.read_bytes()) + + with StoreBrowser("memory://b2view-array" + suffix) as browser: + assert isinstance(browser.store, blosc2.RemoteProxy) + assert browser.list_children("/") == [] + info = browser.get_info("/") + assert info.kind == "ndarray" + assert info.metadata["shape"] == data.shape + assert info.user_attrs["description"] == "remote preview" + assert browser.store.cache_bytes == 0 + for _ in range(2): + before = browser.store.traffic.nbytes + preview = browser.preview("/", slice_indices=[2], start=3, stop=6, max_cols=5) + np.testing.assert_array_equal(np.column_stack(list(preview["data"].values())), data[2, 3:6, :5]) + if _ == 1: + assert browser.store.traffic.nbytes == before + + def test_store_browser_supports_standalone_ctable(tmp_path): path = tmp_path / "table.b2z" table = make_ctable(4) From 6248887cb2f6db3eb8b43007cbe297bfbb109ac0 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 8 Sep 2026 18:15:10 +0200 Subject: [PATCH 36/89] Add attrs as the recommended user metadata interface --- RELEASE_NOTES.md | 6 ++++ doc/guides/b2view.rst | 4 +-- doc/guides/remote_arrays.md | 6 +++- doc/reference/lazyarray.rst | 4 +-- doc/reference/ndarray.rst | 3 ++ doc/reference/ref.rst | 4 +-- doc/reference/remoteproxy.rst | 1 - doc/reference/schunk.rst | 30 ++++++++++++++++-- doc/reference/tree_store.rst | 2 +- examples/vlmeta.py | 18 +++++------ src/blosc2/batch_array.py | 8 +++++ src/blosc2/ctable.py | 24 ++++++++++----- src/blosc2/lazyexpr.py | 8 +++++ src/blosc2/list_array.py | 8 +++++ src/blosc2/ndarray.py | 8 +++++ src/blosc2/objectarray.py | 8 +++++ src/blosc2/proxy.py | 8 +++++ src/blosc2/proxy_source.py | 8 +++++ src/blosc2/schunk.py | 8 +++++ src/blosc2/tree_store.py | 8 +++++ tests/test_attrs.py | 57 +++++++++++++++++++++++++++++++++++ 21 files changed, 202 insertions(+), 29 deletions(-) create mode 100644 tests/test_attrs.py diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index fdc705ec4..90671d200 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -4,6 +4,12 @@ XXX version-specific blurb XXX +* Use `.attrs` as the recommended interface for user-defined metadata. Arrays, + containers and proxy sources now expose it as an alias for `.vlmeta`, preserving + existing storage and access rules. `C2Array.attrs` continues to select the + server's user attributes while `C2Array.vlmeta` retains protocol metadata. + `.vlmeta` remains supported and is not deprecated. + ## Changes from 4.11.0 to 4.12.0 This release focuses on efficient remote arrays. Blosc2 containers can now be diff --git a/doc/guides/b2view.rst b/doc/guides/b2view.rst index e96ab3000..39ec81f57 100644 --- a/doc/guides/b2view.rst +++ b/doc/guides/b2view.rst @@ -24,9 +24,9 @@ arrays and some metadata: import blosc2 with blosc2.TreeStore("sample.b2z", mode="w") as tstore: - tstore.vlmeta["author"] = "me" + tstore.attrs["author"] = "me" a = blosc2.linspace(0, 1, num=1_000_000, shape=(1000, 1000)) - a.vlmeta["description"] = "a 2-D linspace" + a.attrs["description"] = "a 2-D linspace" tstore["/dense/a"] = a tstore["/dense/b"] = blosc2.arange(10_000, shape=(10, 100, 10)) diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index cf6f1aed1..af56dd431 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -4,6 +4,7 @@ Blosc2 can open remote arrays without downloading them first. Metadata is read i All lazy remote array access in Python-Blosc2 is unified under {ref}`RemoteProxy`. +Use `.attrs` as the recommended interface for user-defined metadata. Read user attributes with `a.attrs["name"]` or get them all with `a.attrs[:]`. `RemoteProxy.attrs` is a read-only alias for `RemoteProxy.vlmeta` and shares its metadata cache across all source formats. HDF5 attributes exclude the @@ -11,7 +12,10 @@ metadata cache across all source formats. HDF5 attributes exclude the For Caterva2 sources, user attributes come from the `attrs` field in `/api/info`. `C2Array.attrs` uses the same field, with a fallback to variable metadata for -older servers. Its `vlmeta` property retains the original protocol metadata. +older servers. Unlike the aliases on other Blosc2 objects, `C2Array.attrs` +can differ from `C2Array.vlmeta`: the latter retains raw protocol metadata, +including operational entries such as `fill_nonce` and `fill_state`. +`vlmeta` remains supported for compatibility and is not deprecated. These properties do not write attributes to the server. For a saved remote proxy served by Caterva2, attributes reflect the snapshot stored in its carrier. diff --git a/doc/reference/lazyarray.rst b/doc/reference/lazyarray.rst index 9bcce3839..d77844c59 100644 --- a/doc/reference/lazyarray.rst +++ b/doc/reference/lazyarray.rst @@ -14,7 +14,7 @@ You can get an object following the LazyArray API in any of the following ways: The LazyArray object is a thin wrapper around the expression or user-defined function that allows for lazy computation. This means that the expression is not computed until the ``compute`` or ``__getitem__`` methods are called. The ``compute`` method will return a new NDArray object with the result of the expression evaluation. The ``__getitem__`` method will return a NumPy object instead. -LazyArray objects also support user metadata via :attr:`LazyArray.vlmeta`. For +LazyArray objects also support user metadata via :attr:`LazyArray.attrs`. For in-memory objects, this metadata lives on the Python object itself. For persisted LazyArrays reopened from disk, metadata is synchronized with the underlying carrier and survives reopening. @@ -40,7 +40,7 @@ See the `LazyExpr`_ and `LazyUDF`_ sections for more information. Attributes ---------- - .. autoattribute:: vlmeta + .. autoattribute:: attrs .. _LazyExpr: diff --git a/doc/reference/ndarray.rst b/doc/reference/ndarray.rst index de44ad4d9..555678217 100644 --- a/doc/reference/ndarray.rst +++ b/doc/reference/ndarray.rst @@ -6,6 +6,9 @@ NDArray The multidimensional data array class. Instances may be constructed using the constructor functions in the list below `NDArrayConstructors`_. In addition, all the functions from the :ref:`LazyArray` section can be used with NDArray instances. +Use ``array.attrs`` for user-defined metadata (see :ref:`MsgpackSerialization`). +It is a direct alias for ``array.vlmeta`` and uses the same persistent storage. + .. currentmodule:: blosc2 .. autoclass:: NDArray diff --git a/doc/reference/ref.rst b/doc/reference/ref.rst index 6a3489765..2cca03415 100644 --- a/doc/reference/ref.rst +++ b/doc/reference/ref.rst @@ -39,11 +39,11 @@ Example # A Ref can itself be persisted, for example as variable-length metadata # in another persistent Blosc2 object. catalog = blosc2.zeros(1, urlpath=catalog_path, mode="w") - catalog.schunk.vlmeta["array_ref"] = ref + catalog.schunk.attrs["array_ref"] = ref # Reopen the metadata holder and resolve the persisted reference. catalog = blosc2.open(catalog_path, mode="r") - restored_ref = catalog.schunk.vlmeta["array_ref"] + restored_ref = catalog.schunk.attrs["array_ref"] reopened = restored_ref.open() print(reopened[:]) # [0 1 2 3 4] diff --git a/doc/reference/remoteproxy.rst b/doc/reference/remoteproxy.rst index 6cae2c474..69e0ef81c 100644 --- a/doc/reference/remoteproxy.rst +++ b/doc/reference/remoteproxy.rst @@ -201,7 +201,6 @@ file. Read-only mode can use warm chunks but does not retain misses: .. autoattribute:: cparams .. autoattribute:: nbytes .. autoattribute:: meta - .. autoattribute:: vlmeta .. autoattribute:: attrs .. autoattribute:: info .. autoattribute:: cache diff --git a/doc/reference/schunk.rst b/doc/reference/schunk.rst index 856463691..c33d03dd2 100644 --- a/doc/reference/schunk.rst +++ b/doc/reference/schunk.rst @@ -10,7 +10,31 @@ The basic compressed data container (aka super-chunk). This class consists of a Metadata support ---------------- -``SChunk.vlmeta`` uses the general Blosc2 msgpack extensions. This means +Use ``obj.attrs`` as the recommended interface for user-defined metadata:: + + array = blosc2.zeros(10) + array.attrs["units"] = "kelvin" + print(array.attrs["units"]) + print(array.attrs[:]) + +``attrs`` delegates to ``vlmeta`` on ``NDArray``, ``SChunk``, ``ObjectArray``, +``BatchArray``, ``ListArray``, ``CTable``, ``TreeStore``, ``LazyArray``, ``Proxy`` +and ``RemoteProxy``, and on ``ProxyNDSource`` implementations. It preserves +existing persistence, serialization and access rules; it does not introduce +another metadata store or filter operational entries from ordinary containers. +Some objects return a fresh mapping wrapper on each access. + +``vlmeta`` remains supported with its existing behavior and is not deprecated. +Prefer ``attrs`` in new user-facing code. Constructor arguments named ``vlmeta`` +retain their existing names. + +``C2Array`` is the exception: ``attrs`` selects user attributes from the server, +whereas ``vlmeta`` retains raw protocol metadata. Older servers without an +``attrs`` field fall back to raw variable metadata. Neither property writes +changes to the server. ``RemoteProxy.attrs`` is read-only. See the +:doc:`remote array guide <../guides/remote_arrays>` for details. + +``SChunk.attrs`` uses the general Blosc2 msgpack extensions. This means variable-length metadata can store not only ordinary msgpack-safe Python values, but also the currently supported Blosc2 objects and references, including: @@ -21,8 +45,8 @@ including: - ``LazyExpr`` - ``LazyUDF`` backed by ``@blosc2.dsl_kernel`` -Both single-key access (``schunk.vlmeta["name"]``) and bulk access -(``schunk.vlmeta[:]``) use this serializer. +Both single-key access (``schunk.attrs["name"]``) and bulk access +(``schunk.attrs[:]``) use this serializer. Lazy expressions and supported lazy UDFs still require durable operand references only; purely in-memory operands are intentionally rejected. diff --git a/doc/reference/tree_store.rst b/doc/reference/tree_store.rst index d093809e2..7af844d96 100644 --- a/doc/reference/tree_store.rst +++ b/doc/reference/tree_store.rst @@ -105,7 +105,7 @@ Quick example Properties ---------- - .. autoattribute:: vlmeta + .. autoattribute:: attrs Public Members -------------- diff --git a/examples/vlmeta.py b/examples/vlmeta.py index af90f763a..91eb2d5c4 100644 --- a/examples/vlmeta.py +++ b/examples/vlmeta.py @@ -17,18 +17,18 @@ assert nchunks_ == (i + 1) # Initially the vlmeta is empty -print(len(schunk.vlmeta)) +print(len(schunk.attrs)) # Add a vlmeta -schunk.vlmeta["meta1"] = "first vlmetalayer" -print(schunk.vlmeta.getall()) +schunk.attrs["meta1"] = "first vlmetalayer" +print(schunk.attrs.getall()) # Update the vlmeta -schunk.vlmeta["meta1"] = "new vlmetalayer" -print(schunk.vlmeta.getall()) +schunk.attrs["meta1"] = "new vlmetalayer" +print(schunk.attrs.getall()) # Add another vlmeta -schunk.vlmeta["vlmeta2"] = "second vlmeta" +schunk.attrs["vlmeta2"] = "second vlmeta" # Check that it has been added -assert "vlmeta2" in schunk.vlmeta +assert "vlmeta2" in schunk.attrs # Delete a vlmeta -del schunk.vlmeta["vlmeta2"] -assert "vlmeta2" not in schunk.vlmeta +del schunk.attrs["vlmeta2"] +assert "vlmeta2" not in schunk.attrs diff --git a/src/blosc2/batch_array.py b/src/blosc2/batch_array.py index ddf9c793c..22a840379 100644 --- a/src/blosc2/batch_array.py +++ b/src/blosc2/batch_array.py @@ -873,6 +873,14 @@ def __iter__(self) -> Iterator[Batch]: def meta(self): return self.schunk.meta + @property + def attrs(self): + """User attributes; the recommended alias for :attr:`vlmeta`. + + Shares the existing metadata storage and access rules without filtering keys. + """ + return self.vlmeta + @property def vlmeta(self): return self.schunk.vlmeta diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 3a7fe84fe..2ffaa67bd 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -14569,6 +14569,14 @@ def schema(self) -> CompiledSchema: """The compiled schema that drives this table's columns and validation.""" return self._schema + @property + def attrs(self): + """User attributes; the recommended alias for :attr:`vlmeta`. + + Shares the existing metadata storage and access rules without filtering keys. + """ + return self.vlmeta + @property def vlmeta(self): """Variable-length metadata attached to this table. @@ -14587,16 +14595,16 @@ def vlmeta(self): ... class Row: ... x: int = 0 >>> t = blosc2.CTable(Row) - >>> t.vlmeta["author"] = "Alice" - >>> t.vlmeta["tags"] = ["alpha", "beta"] - >>> t.vlmeta["count"] = 42 - >>> print(t.vlmeta["author"]) + >>> t.attrs["author"] = "Alice" + >>> t.attrs["tags"] = ["alpha", "beta"] + >>> t.attrs["count"] = 42 + >>> print(t.attrs["author"]) Alice - >>> print(t.vlmeta[:]) + >>> print(t.attrs[:]) {'author': 'Alice', 'tags': ['alpha', 'beta'], 'count': 42} - >>> del t.vlmeta["count"] - >>> for name in t.vlmeta: - ... print(name, t.vlmeta[name]) + >>> del t.attrs["count"] + >>> for name in t.attrs: + ... print(name, t.attrs[name]) ... author Alice tags ['alpha', 'beta'] diff --git a/src/blosc2/lazyexpr.py b/src/blosc2/lazyexpr.py index 31be7cf39..5d6dfcb6d 100644 --- a/src/blosc2/lazyexpr.py +++ b/src/blosc2/lazyexpr.py @@ -488,6 +488,14 @@ def _sync_user_vlmeta(self) -> None: if array is not None: write_b2object_user_vlmeta(array, self._get_user_vlmeta()) + @property + def attrs(self): + """User attributes; the recommended alias for :attr:`vlmeta`. + + Shares the existing metadata storage and access rules without filtering keys. + """ + return self.vlmeta + @property def vlmeta(self) -> LazyArrayVLMeta: """User variable-length metadata for this LazyArray.""" diff --git a/src/blosc2/list_array.py b/src/blosc2/list_array.py index 288ae9d07..25143aea4 100644 --- a/src/blosc2/list_array.py +++ b/src/blosc2/list_array.py @@ -634,6 +634,14 @@ def meta(self): """Fixed-length metadata mapping for the underlying container.""" return self._backend.meta + @property + def attrs(self): + """User attributes; the recommended alias for :attr:`vlmeta`. + + Shares the existing metadata storage and access rules without filtering keys. + """ + return self.vlmeta + @property def vlmeta(self): """Variable-length metadata mapping for the underlying container.""" diff --git a/src/blosc2/ndarray.py b/src/blosc2/ndarray.py index 71eaf45a1..c66216684 100644 --- a/src/blosc2/ndarray.py +++ b/src/blosc2/ndarray.py @@ -3875,6 +3875,14 @@ def meta(self) -> dict: """The metadata of the array.""" return self.schunk.meta + @property + def attrs(self): + """User attributes; the recommended alias for :attr:`vlmeta`. + + Shares the existing metadata storage and access rules without filtering keys. + """ + return self.vlmeta + @property def vlmeta(self) -> dict: """The variable-length metadata of the array.""" diff --git a/src/blosc2/objectarray.py b/src/blosc2/objectarray.py index 326e5c748..41ca3d5d1 100644 --- a/src/blosc2/objectarray.py +++ b/src/blosc2/objectarray.py @@ -327,6 +327,14 @@ def __iter__(self) -> Iterator[Any]: def meta(self): return self.schunk.meta + @property + def attrs(self): + """User attributes; the recommended alias for :attr:`vlmeta`. + + Shares the existing metadata storage and access rules without filtering keys. + """ + return self.vlmeta + @property def vlmeta(self): return self.schunk.vlmeta diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index 587343501..25c7d0966 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -1280,6 +1280,14 @@ def meta(self) -> blosc2.schunk.meta: """ return self._schunk_cache.meta + @property + def attrs(self): + """User attributes; the recommended alias for :attr:`vlmeta`. + + Shares the existing metadata storage and access rules without filtering keys. + """ + return self.vlmeta + @property def vlmeta(self) -> blosc2.schunk.vlmeta: """ diff --git a/src/blosc2/proxy_source.py b/src/blosc2/proxy_source.py index 5da5ddeb5..ba2f42291 100644 --- a/src/blosc2/proxy_source.py +++ b/src/blosc2/proxy_source.py @@ -351,6 +351,14 @@ def meta(self) -> dict: """The fixed-length metadata of the source.""" return {} + @property + def attrs(self): + """User attributes; the recommended alias for :attr:`vlmeta`. + + Shares the existing metadata storage and access rules without filtering keys. + """ + return self.vlmeta + @property def vlmeta(self) -> dict: """The variable-length metadata of the source.""" diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 5c648ef19..8aa632f46 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -510,6 +510,14 @@ def meta(self) -> Meta: """ return Meta(self) + @property + def attrs(self): + """User attributes; the recommended alias for :attr:`vlmeta`. + + Shares the existing metadata storage and access rules without filtering keys. + """ + return self.vlmeta + @property def vlmeta(self) -> vlmeta: """ diff --git a/src/blosc2/tree_store.py b/src/blosc2/tree_store.py index da55f6737..fe6cffb5d 100644 --- a/src/blosc2/tree_store.py +++ b/src/blosc2/tree_store.py @@ -957,6 +957,14 @@ def get_subtree(self, path: str) -> TreeStore: return subtree + @property + def attrs(self): + """User attributes; the recommended alias for :attr:`vlmeta`. + + Shares the existing metadata storage and access rules without filtering keys. + """ + return self.vlmeta + @property def vlmeta(self) -> MutableMapping: """Access variable-length metadata for the TreeStore or current subtree. diff --git a/tests/test_attrs.py b/tests/test_attrs.py new file mode 100644 index 000000000..d5bc8557f --- /dev/null +++ b/tests/test_attrs.py @@ -0,0 +1,57 @@ +"""The user-facing metadata name shares the existing storage and access rules.""" + +from dataclasses import dataclass + +import pytest + +import blosc2 + + +@dataclass +class Row: + value: int = 0 + + +@pytest.mark.parametrize( + "factory", + [ + lambda: blosc2.zeros(4), + blosc2.SChunk, + blosc2.ObjectArray, + lambda: blosc2.BatchArray(items_per_block=2), + lambda: blosc2.ListArray(item_spec=blosc2.int32()), + lambda: blosc2.CTable(Row), + lambda: blosc2.zeros(4) + 1, + lambda: blosc2.Proxy(blosc2.zeros(4)), + ], +) +def test_attrs_alias(factory): + obj = factory() + assert obj.attrs is obj.vlmeta + obj.attrs["units"] = "kelvin" + assert obj.vlmeta["units"] == "kelvin" + obj.vlmeta["units"] = "celsius" + assert obj.attrs["units"] == "celsius" + del obj.attrs["units"] + assert "units" not in obj.vlmeta + + +def test_attrs_persistence_and_read_only(tmp_path): + path = tmp_path / "array.b2nd" + array = blosc2.zeros(4, urlpath=path) + array.attrs["units"] = "kelvin" + reopened = blosc2.open(path, mode="r") + assert reopened.attrs["units"] == "kelvin" + with pytest.raises(ValueError): + reopened.attrs["units"] = "celsius" + + path = tmp_path / "tree.b2z" + with blosc2.TreeStore(path, mode="w") as tree: + tree.attrs["units"] = "kelvin" + assert tree.vlmeta["units"] == "kelvin" + tree.vlmeta["units"] = "celsius" + assert tree.attrs["units"] == "celsius" + with blosc2.TreeStore(path, mode="r") as tree: + assert tree.attrs["units"] == "celsius" + with pytest.raises(ValueError): + tree.attrs["units"] = "kelvin" From c26ece6075684c6372ae8fb4d900577c80b3596a Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 8 Sep 2026 18:29:09 +0200 Subject: [PATCH 37/89] Hide hierarchy details for standalone b2view objects --- doc/guides/b2view.rst | 4 ++++ src/blosc2/b2view/app.py | 11 +++++---- src/blosc2/b2view/render.py | 5 ++-- tests/b2view/test_basics.py | 46 +++++++++++++++++++++++++++++++++++++ tests/b2view/test_render.py | 18 ++++++++++++++- 5 files changed, 77 insertions(+), 7 deletions(-) diff --git a/doc/guides/b2view.rst b/doc/guides/b2view.rst index 39ec81f57..cd05acdf1 100644 --- a/doc/guides/b2view.rst +++ b/doc/guides/b2view.rst @@ -45,6 +45,10 @@ left, and **meta**, **vlmeta** and **data** panels for the node selected in the tree. Move between panels with ``tab`` / ``shift+tab``, maximize the focused one with ``m`` (``r`` restores it), and quit with ``q``. +For standalone objects such as an NDArray or CTable, the tree panel is hidden, +the remaining panels use the full width, and focus starts in the data panel by default. +The metadata omits the internal root path; the header shows the source path. + By default the mouse is left to the terminal, so selecting and copying text works as in any other command line program. Pass ``--mouse`` to let b2view capture it instead: panels become clickable and the wheel scrolls the data diff --git a/src/blosc2/b2view/app.py b/src/blosc2/b2view/app.py index f38948097..fe7bd0996 100644 --- a/src/blosc2/b2view/app.py +++ b/src/blosc2/b2view/app.py @@ -1968,7 +1968,7 @@ class B2ViewApp(App): CSS = """ #main { height: 1fr; } #tree-pane { width: 35%; border: solid $primary; } - #right-pane { width: 65%; } + #right-pane { width: 1fr; } #top-row { height: 40%; } #meta-pane, #vlmeta-pane { width: 50%; border: solid $secondary; } #data-pane { height: 60%; border: solid $secondary; } @@ -2129,6 +2129,7 @@ def _after_download(self, result: bool | str) -> None: def _start_browsing(self) -> None: """Open the bundle and populate the tree (the normal startup path).""" self.browser = StoreBrowser(self.urlpath, storage_options=self.storage_options) + self.query_one("#tree-pane").display = self.browser.is_tree self.query_one(B2ViewHeader).set_filename(self._header_label) tree = self.query_one("#tree", Tree) tree.root.data = "/" @@ -2157,6 +2158,8 @@ def _apply_start_focus(self) -> None: def _focus_panel_by_name(self, name: str) -> None: """Focus a panel by its user-facing name.""" + if name == "tree" and not self.query_one("#tree-pane").display: + name = "data" panel_map = { "tree": lambda: self.query_one("#tree", Tree), "meta": lambda: self.query_one("#meta-scroll", VerticalScroll), @@ -2238,7 +2241,7 @@ def update_panels(self, path: str) -> None: vlmeta_widget = self.query_one("#vlmetadata", Static) try: info = self.browser.get_info(path) - metadata.update(make_metadata_renderable(info)) + metadata.update(make_metadata_renderable(info, show_path=self.browser.is_tree)) self.table_buffer = None self.grid_col_start = 0 self._data_layout = None @@ -2917,8 +2920,8 @@ def _focusable_panels(self): if data_table_row.display else self.query_one("#data-scroll", VerticalScroll) ) - return [ - self.query_one("#tree", Tree), + tree_panels = [self.query_one("#tree", Tree)] if self.query_one("#tree-pane").display else [] + return tree_panels + [ self.query_one("#meta-scroll", VerticalScroll), self.query_one("#vlmeta-scroll", VerticalScroll), data_panel, diff --git a/src/blosc2/b2view/render.py b/src/blosc2/b2view/render.py index 82df2de37..e304aa6e1 100644 --- a/src/blosc2/b2view/render.py +++ b/src/blosc2/b2view/render.py @@ -9,14 +9,15 @@ import numpy as np -def make_metadata_renderable(info): +def make_metadata_renderable(info, *, show_path=True): """Return a Rich renderable for ObjectInfo metadata.""" from rich.table import Table table = Table(show_header=False, box=None, expand=True) table.add_column("key", style="bold cyan", no_wrap=True) table.add_column("value") - table.add_row("path", info.path) + if show_path: + table.add_row("path", info.path) table.add_row("kind", info.kind) for key, value in info.metadata.items(): table.add_row(str(key), _format_metadata_value(value)) diff --git a/tests/b2view/test_basics.py b/tests/b2view/test_basics.py index c4318e909..438b86ce1 100644 --- a/tests/b2view/test_basics.py +++ b/tests/b2view/test_basics.py @@ -31,6 +31,7 @@ from __future__ import annotations import importlib.util +from dataclasses import dataclass import numpy as np import pytest @@ -155,6 +156,7 @@ async def test_remote_array_startup(tmp_path): async with app.run_test(size=TERM_SIZE) as pilot: await wait_for_table(pilot) assert isinstance(app.browser.store, blosc2.RemoteProxy) + assert not app.query_one("#tree-pane").display assert app._data_layout.shape == data.shape page = app.table_page for column in page["columns"]: @@ -173,6 +175,49 @@ async def _wait_focus(pilot, expected_id: str) -> str | None: return getattr(pilot.app.focused, "id", None) +@pytest.mark.parametrize("kind", ["ndarray", "ctable"]) +async def test_standalone_panel_layout(tmp_path, kind): + path = str(tmp_path / ("array.b2nd" if kind == "ndarray" else "table.b2z")) + if kind == "ndarray": + blosc2.asarray(np.arange(100), urlpath=path) + else: + + @dataclass + class Row: + x: int = 0 + + table = blosc2.CTable(Row, urlpath=path, mode="w") + table.extend({"x": np.arange(100)}) + table.close() + + app = B2ViewApp(path) + async with app.run_test(size=TERM_SIZE) as pilot: + await wait_for_table(pilot) + assert await _wait_focus(pilot, "data-table") == "data-table" + assert not app.query_one("#tree-pane").display + assert app.query_one("#right-pane").size.width == app.query_one("#main").size.width + for key, expected in [ + ("tab", "meta-scroll"), + ("tab", "vlmeta-scroll"), + ("tab", "data-table"), + ("shift+tab", "vlmeta-scroll"), + ("shift+tab", "meta-scroll"), + ("shift+tab", "data-table"), + ]: + await pilot.press(key) + assert await _wait_focus(pilot, expected) == expected + await pilot.press("m") + await pilot.pause() + assert app.screen.maximized is app.query_one("#data-pane") + await pilot.press("r") + await pilot.pause() + assert app.screen.maximized is None + assert not app.query_one("#tree-pane").display + await pilot.press("r") + await wait_for_table(pilot) + assert app.table_page["nrows"] == 100 + + async def test_start_panel_focus_with_path(store_path): """``--panel`` focuses the right widget on startup, even with a ``--path``. @@ -200,6 +245,7 @@ async def test_tree_and_panel_focus(store_path): async with app.run_test(size=TERM_SIZE) as pilot: await pilot.pause() assert isinstance(app.focused, Tree) + assert app.query_one("#tree-pane").display # Tab: tree -> meta -> vlmeta -> data and wraps back to the tree for expected in ["meta-scroll", "vlmeta-scroll", "data-scroll", "tree"]: diff --git a/tests/b2view/test_render.py b/tests/b2view/test_render.py index 102fdba95..d858a825d 100644 --- a/tests/b2view/test_render.py +++ b/tests/b2view/test_render.py @@ -8,8 +8,24 @@ """Unit tests for b2view cell formatting (no app session needed).""" import numpy as np +import pytest -from blosc2.b2view.render import column_float_decimals, format_cell +from blosc2.b2view.model import ObjectInfo +from blosc2.b2view.render import column_float_decimals, format_cell, make_metadata_renderable + + +@pytest.mark.parametrize("show_path", [False, True]) +def test_metadata_path_visibility(show_path): + pytest.importorskip("rich") + from rich.console import Console + + info = ObjectInfo(path="/", kind="ndarray", metadata={"shape": (10,)}) + console = Console(record=True) + console.print(make_metadata_renderable(info, show_path=show_path)) + rendered = console.export_text() + assert ("path" in rendered) is show_path + assert "ndarray" in rendered + assert "(10,)" in rendered def test_column_decimals_follow_max_magnitude(): From a7070c14abe6d0cbb489eb304aff7a2160722651 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 8 Sep 2026 18:37:25 +0200 Subject: [PATCH 38/89] Fix Zarr decoder startup in b2view --- src/blosc2/b2view/app.py | 8 ++++++ tests/b2view/test_basics.py | 51 +++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/src/blosc2/b2view/app.py b/src/blosc2/b2view/app.py index fe7bd0996..685eaafad 100644 --- a/src/blosc2/b2view/app.py +++ b/src/blosc2/b2view/app.py @@ -53,6 +53,7 @@ make_metadata_renderable, make_preview_renderables, ) +from blosc2.core import parse_container_url if TYPE_CHECKING: from textual import events @@ -2025,6 +2026,13 @@ def __init__( ): super().__init__() self.sub_title = f"Python-Blosc2 {blosc2.__version__}" # shown beside the title in the header + if parse_container_url(urlpath)[2] == "zarr": + # Initialize before Textual captures stderr (fileno=-1), which + # prevents multiprocessing's resource tracker from starting. + with contextlib.suppress(ImportError): + from numcodecs.blosc import get_mutex + + get_mutex() self.urlpath = urlpath self.storage_options = storage_options self.download_url = download_url # when set, fetch urlpath before browsing diff --git a/tests/b2view/test_basics.py b/tests/b2view/test_basics.py index 438b86ce1..16849679f 100644 --- a/tests/b2view/test_basics.py +++ b/tests/b2view/test_basics.py @@ -31,6 +31,8 @@ from __future__ import annotations import importlib.util +import subprocess +import sys from dataclasses import dataclass import numpy as np @@ -166,6 +168,55 @@ async def test_remote_array_startup(tmp_path): assert app.focused is app.query_one("#data-table", DataTable) +async def test_zarr_startup_fresh_process(tmp_path): + zarr = pytest.importorskip("zarr") + pytest.importorskip("fsspec") + path = tmp_path / "array.zarr" + zarr.create_array( + path, + data=np.arange(100, dtype=np.int32), + chunks=(100,), + compressors=[zarr.codecs.BloscCodec()], + zarr_format=3, + ) + # A fresh interpreter is essential: writing the fixture initializes the + # codec lock, masking failures when its first use is inside Textual. + result = subprocess.run( + [ + sys.executable, + "-c", + """ +import asyncio +import sys +from pathlib import Path +import fsspec +from blosc2.b2view.app import B2ViewApp + +async def main(): + path = Path(sys.argv[1]) + fs = fsspec.filesystem('memory') + for file in path.rglob('*'): + if file.is_file(): + fs.pipe_file('/startup.zarr/' + file.relative_to(path).as_posix(), file.read_bytes()) + app = B2ViewApp('memory://startup.zarr') + async with app.run_test(size=(120, 40)) as pilot: + await pilot.pause() + assert app.table_page is not None, 'Zarr preview failed to load' + assert app.table_page['nrows'] == 100 + values = next(iter(app.table_page['data'].values())) + assert list(values) == list(range(len(values))) + +asyncio.run(main()) +""", + str(path), + ], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stdout + result.stderr + + async def _wait_focus(pilot, expected_id: str) -> str | None: """Pause until the focused widget is *expected_id* (or give up).""" for _ in range(30): From 89548d9aa8c51c239d11b855ae478f176762feb3 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 9 Sep 2026 12:49:31 +0200 Subject: [PATCH 39/89] Add remote hierarchy browsing --- doc/guides/b2view.rst | 74 ++++- doc/reference/remoteproxy.rst | 10 + plans/remote-proxy-v12.md | 353 +++++++++++++++++++++ src/blosc2/b2view/app.py | 326 +++++++++++++++++-- src/blosc2/b2view/cli.py | 4 +- src/blosc2/b2view/hierarchy.py | 343 ++++++++++++++++++++ src/blosc2/b2view/model.py | 68 +++- src/blosc2/b2z_source.py | 243 ++++++++++---- src/blosc2/hdf5_source.py | 70 +++- src/blosc2/msgpack_utils.py | 20 ++ src/blosc2/proxy_source.py | 19 +- src/blosc2/remote_proxy.py | 2 +- src/blosc2/schunk.py | 6 + src/blosc2/zarr_source.py | 12 + tests/b2view/test_basics.py | 10 +- tests/b2view/test_cli.py | 26 +- tests/b2view/test_hierarchy.py | 528 +++++++++++++++++++++++++++++++ tests/b2view/tree_store_gen.py | 18 +- tests/ctable/test_object_spec.py | 2 +- tests/test_b2view_model.py | 2 +- tests/test_python_blosc.py | 15 +- tests/test_remote_proxy.py | 185 +++++++++++ 22 files changed, 2162 insertions(+), 174 deletions(-) create mode 100644 plans/remote-proxy-v12.md create mode 100644 src/blosc2/b2view/hierarchy.py create mode 100644 tests/b2view/test_hierarchy.py diff --git a/doc/guides/b2view.rst b/doc/guides/b2view.rst index cd05acdf1..97f49db54 100644 --- a/doc/guides/b2view.rst +++ b/doc/guides/b2view.rst @@ -4,7 +4,7 @@ b2view: Browse TreeStore Bundles in the Terminal The ``b2view`` CLI opens an interactive terminal browser (TUI) for Blosc2 TreeStore bundles, either sparse directories (``.b2d``) or compact zip-backed files (``.b2z``). It shows the tree of groups and nodes, the -metadata and vlmeta of the selected node, and a paged view of the data +metadata and attrs of the selected node, and a paged view of the data itself — NDArrays of any dimensionality as well as CTables. ``b2view`` is opt-in: install it with the ``tui`` extra — @@ -41,7 +41,7 @@ Step 2 — Open it b2view sample.b2z The screen is split into four panels: the **tree** of the bundle on the -left, and **meta**, **vlmeta** and **data** panels for the node selected +left, and **meta**, **attrs** and **data** panels for the node selected in the tree. Move between panels with ``tab`` / ``shift+tab``, maximize the focused one with ``m`` (``r`` restores it), and quit with ``q``. @@ -60,21 +60,71 @@ You can also jump straight to a node and panel: b2view sample.b2z /dense/a --panel data -Remote arrays -~~~~~~~~~~~~~ +Remote containers and arrays +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Remote array URLs open lazily, fetching data as you browse and caching reads -in memory. A dataset inside a B2Z archive opens as a single array at the -viewer root; include its path in the URL using ``/`` or ``::``: +Browse remote B2Z, Zarr, and HDF5 containers directly from their root: .. code-block:: console - b2view s3://blosc2/hierarchy.b2z/d0/d1/a2 --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com --panel data + b2view --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com s3://blosc2/hierarchy.b2z + b2view --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com s3://blosc2/hierarchy.zarr + b2view --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com s3://blosc2/hierarchy.h5 + +Append a group path to browse a subtree, or an array path to open a standalone +array without a tree panel. Both ``/`` and ``::`` dataset addressing work: + +.. code-block:: console + + b2view s3://blosc2/hierarchy.b2z/d0/d1 --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com + b2view s3://blosc2/hierarchy.zarr::d0/d1/a2 --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com --panel data + +The supplied URL stays in the header. Within a subtree, ``/`` refers to the +requested group. Group attributes belong to the selected group. Opening, +expansion, metadata reads, and array pages run in background workers; navigation +and quit remain available during a slow request. Refresh opens a new discovery +session, discards cached pages, and restores the selected path when it still +exists. Failed listings can be retried by selecting or expanding the group again. + +Browsing is read-only. Only the selected array retains a payload cache, bounded +to 64 MiB in hierarchy views; selecting another node releases it. Discovery reads +metadata, not every array's data. Metadata cost can grow with the number of +objects and chunks. Small objects may fit entirely within a bounded opening read. ``--profile`` and ``--endpoint-url`` are optional; when omitted, the S3 backend -uses its normal credential and endpoint configuration. Install ``s3fs`` for -S3 access. Remote ``.b2nd`` arrays and supported HDF5/Zarr dataset URLs use -the same viewer (with their corresponding backend dependencies installed). +uses its normal credential and endpoint configuration. Install ``blosc2[tui]`` +and ``blosc2[fsspec]`` plus ``s3fs`` for S3 access. Zarr requires +``blosc2[zarr]``; HDF5 requires ``blosc2[hdf5]`` (Kerchunk, h5py, and Zarr). +B2Z browsing does not require Zarr or HDF5 dependencies. + +Format details and limits: + +* **B2Z:** discovery shares the ZIP directory with the native array reader. + External arrays must be unencrypted, ZIP_STORED plain NDArrays. The embedded + index identifies embedded leaves and CTable boundaries; their payload previews + remain unavailable. Group attributes are read from external frame trailers or + the bounded native chunks containing embedded attribute frames. Embedded + attribute layouts with chunks larger than 1 MiB show a partial-metadata notice + instead of fetching large payloads. TreeStore has no separate empty-group + marker: an empty group is visible when its attribute frame records it. +* **Zarr:** v2 and v3 groups use consolidated metadata when available and normal + discovery otherwise. Unconsolidated groups need backend directory-listing + support and LIST permission. Direct arrays do not require listing their parent. + Empty groups and attributes are preserved. Unknown codecs and unsupported + dtypes remain visible; preview support follows the existing Zarr array reader. +* **HDF5:** Kerchunk translates metadata once per session and all selected leaves + reuse those references. Translation can enumerate many chunk references and + inline small values; it avoids full-file localization, but is not a constant-cost + operation. Empty groups and attributes are preserved. Failed dataset translations + become unavailable nodes without hiding supported siblings. The view covers + Kerchunk's representation: hard-link aliases may be omitted, and soft/external + links and group cycles are not followed. + +These internal browser adapters do not change the array-only contract of +``blosc2.open(..., lazy=True)`` or add a persisted RemoteProxy hierarchy descriptor. +Opening an entire remote B2Z through ``blosc2.open`` requires an explicit +``cache_dir`` for localization; use ``b2view`` for range-based hierarchy browsing. +Standalone remote ``.b2nd`` arrays retain their existing lazy viewer behavior. Step 3 — Navigate the data panel -------------------------------- @@ -144,4 +194,4 @@ CLI options ``--preview-rows N`` and ``--preview-cols N`` bound the size of each data page (20 rows by 10 columns by default), and ``--panel`` chooses the panel -focused on startup (``tree``, ``meta``, ``vlmeta`` or ``data``). +focused on startup (``tree``, ``meta``, ``attrs`` or ``data``). diff --git a/doc/reference/remoteproxy.rst b/doc/reference/remoteproxy.rst index 69e0ef81c..72a93b9d2 100644 --- a/doc/reference/remoteproxy.rst +++ b/doc/reference/remoteproxy.rst @@ -215,6 +215,16 @@ file. Read-only mode can use warm chunks but does not retain misses: .. autoattribute:: urlpath .. autoattribute:: dataset +RemoteMetadataMapping +--------------------- + +``RemoteProxy.attrs`` returns a read-only mapping that fetches array attributes +only when they are accessed. Use it like a dictionary, or use ``attrs[:]`` to +fetch all attributes at once. + +.. autoclass:: blosc2.RemoteMetadataMapping + :members: + CachePolicy ----------- diff --git a/plans/remote-proxy-v12.md b/plans/remote-proxy-v12.md new file mode 100644 index 000000000..2bebb9066 --- /dev/null +++ b/plans/remote-proxy-v12.md @@ -0,0 +1,353 @@ +# RemoteProxy v12: remote hierarchy browsing in b2view + +Status: implemented and validated on 2026-09-09. + +## Goal + +Browse a remote container from its root, expand groups, inspect attributes, and +preview selected arrays without downloading the complete container first: + +```sh +b2view --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com s3://blosc2/hierarchy.b2z +b2view --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com s3://blosc2/hierarchy.zarr +b2view --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com s3://blosc2/hierarchy.h5 +``` + +Support a URL pointing at a subgroup as well as the container root. Preserve the +existing standalone presentation when the URL selects an array: no tree panel, +no redundant internal root path, and the original source in the header. + +All three formats are feasible using existing dependencies and leaf readers. +B2Z needs archive discovery; Zarr needs group discovery; HDF5 needs its existing +Kerchunk references retained and reused across dataset selections. The main +shared work is connecting discovery to the browser without opening every leaf. + +## Current behavior and reusable pieces + +- `b2view/model.py:StoreBrowser` calls `blosc2.open()` and treats only a + `TreeStore` as a hierarchy. Its previews already handle remote array objects. +- `StoreBrowser.list_children()` queries descendants and opens terminal nodes + to classify them. That is unsuitable for remote discovery and also cannot + use the absence of descendants to distinguish an empty group from an array. +- Group metadata currently reads the store's attributes, rather than necessarily + the selected group's attributes. A remote adapter must return attributes for + the actual selected node. +- `b2view/app.py` already displays trees, expands nodes on demand, navigates to + an initial path, and hides the tree for standalone objects. Opening, listing, + and panel updates currently include synchronous calls on the UI thread. +- `core.py:parse_container_url()` handles container and dataset addressing. + Reuse it, including its distinction between dataset syntax and chained fsspec + URLs; do not add another suffix parser in the application. +- A whole remote B2Z currently bypasses lazy opening in `StoreBrowser`. Without + a local cache, the generic fsspec path in `schunk.py` reads the entire object + and passes ZIP bytes to `from_cframe()`. This is the wrong opening path. +- `b2z_source.py:B2ZNDSource` already reads the ZIP directory through a bounded, + seekable range view and opens external NDArray members lazily. It validates + member windows and uses native Blosc2 chunk/block reads. It requires a leaf + path and rejects embedded leaves and object carriers. +- `dict_store.py` defines canonical external member names and logical key + mapping. TreeStore also uses `embed.b2e` for information that cannot be + reconstructed from external member names alone. +- `zarr_source.py:ZarrNDSource` already opens an fsspec-backed Zarr array and + exposes its attributes. It explicitly rejects groups. +- `hdf5_source.py:available_datasets()` and `HDF5NDSource` already translate HDF5 + into Kerchunk references. The source accepts an existing reference dictionary, + recognizes `.zgroup` and `.zarray`, and opens arrays through a reference store. + Repeating translation for each selected dataset would waste substantial work. +- The recent b2view Zarr fix initializes the Numcodecs Blosc mutex before + Textual captures stderr. Group-root startup must preserve this behavior, and + HDF5 leaves using that decoder need the same fresh-process check. + +## Scope and decisions + +1. Deliver read-only b2view browsing first. Keep `RemoteProxy` an array operand; + do not make it represent groups or implement mutable remote TreeStore. +2. Use a small internal hierarchy adapter, shared by the three remote formats. + Do not subclass `TreeStore`: its local storage and mutation contracts do not + describe these sources. Do not introduce a plugin registry or public store + API for this release. +3. Keep existing `blosc2.open(..., lazy=True, dataset=...)` array behavior. + b2view explicitly selects the hierarchy adapter for container/group targets. + The erroneous generic B2Z root path also needs a shared dispatch guard so + other callers receive a useful error instead of a full download followed by + a frame-decoding failure. Existing explicit `cache_dir` localization remains + available; this plan does not make it b2view's default. +4. Opening and expanding a hierarchy may read metadata and list object keys. + It must not materialize every dataset or fetch array payloads merely to + classify children. Metadata cost can scale with container size; it is not + necessarily a constant number of bytes or requests. +5. Preview support matches the existing leaf readers. Unsupported datasets + remain visible with an explanatory message; one unsupported leaf must not + prevent browsing its siblings. +6. Keep remote sources immutable for a browsing session. Explicit refresh + rebuilds discovery state and invalidates cached node metadata and leaf + objects. Live mutation detection and persistent hierarchy caches are deferred. +7. Use the existing optional fsspec, Zarr, and HDF5/Kerchunk dependency groups. + B2Z browsing must work without Zarr, h5py, or Kerchunk installed. + +## Browser integration + +Introduce one internal module, tentatively `b2view/hierarchy.py`, for discovery +and format adapters. Keep format-specific range/translation helpers in their +existing source modules when also used by array readers. + +The browser needs only these operations: + +- Resolve whether the requested node is a group, supported array, or unsupported + object, independently of whether it has children. +- List direct children with their logical path, display name, kind, and expansion + capability. Reuse `NodeInfo` where practical; do not load data for classification. +- Read one node's metadata and user attributes. +- Open a selected supported leaf using the existing array source/proxy machinery. +- Close resources and rebuild the session on refresh. + +Keep these operations internal and concrete; a base class is unnecessary unless +implementation reveals meaningful shared behavior. `StoreBrowser` remains the +UI-facing adapter and routes hierarchy operations through this small surface. +Audit all uses of `self.store` and `is_tree`, especially `_get_object()`, `kind()`, +`get_info()`, `list_children()`, and `close()`. Existing local TreeStore and CTable +query behavior must remain intact. + +Separate node kind from expansion state. An empty group is still a group and +shows its own attributes. For a remote group whose children have not been +listed, allow expansion without recursively probing its descendants. Omit the +descendant count when obtaining it would require walking the whole hierarchy; +show direct-child counts after listing, rather than inventing a zero count. + +Use browser-relative paths for subtree views: `/` is the requested group, and +leaf resolution joins that view to the actual container path. The header keeps +the supplied source URL; the metadata path identifies the selected node within +the view. Handle root, trailing slash, subgroup, and direct-array URLs explicitly. +Preserve query strings, credentials, and storage options when resolving leaves; +never build child URLs by appending strings after a URL query or fragment. + +Retain discovery metadata for the session. Reuse the existing current leaf proxy +on repeated panel reads; avoid retaining an unbounded number of payload caches +while traversing a large tree. Start with the selected leaf and release it when +selection changes, unless existing cache ownership already provides a bound. + +## B2Z discovery + +### First milestone: external arrays and inferred groups + +Extract only the archive-opening/index functionality needed by both discovery +and `B2ZNDSource`. Preserve bounded range reads, ZIP64 support, malformed-header +checks, member validation, traffic accounting, and existing leaf cache identity. +Use `zipfile`, not a new ZIP parser. Share archive identity and directory results +within the browser session rather than rediscovering them on every click. + +Map canonical member names to logical keys using DictStore's existing rules. +Construct parent groups from the paths, hide storage implementation members, +and list direct children without opening the corresponding frames. Resolve a +selected external NDArray through the existing B2Z leaf reader. Validate the +selected frame before preview; a `.b2nd` suffix is not proof of a plain NDArray. + +Reject ambiguous duplicate logical keys, unsafe paths, and leaf/group collisions +with an actionable archive error. Preserve the existing reader's rejection of +encrypted or ZIP-compressed array members. Surface unsupported ordinary object +types as unavailable leaves where they can be identified reliably. + +This milestone is sufficient for the supplied hierarchy if it contains ordinary +external arrays. Verify its actual directory before claiming that coverage. + +### Complete the hierarchy's metadata + +Inspect the actual `embed.b2e` layout and TreeStore subtree metadata conventions +before implementation. External filenames alone do not establish empty groups, +embedded leaves, group attributes, or logical CTable object boundaries. + +Reuse EmbedStore decoding to read the index and group metadata required for +discovery. Determine whether these can be read through a bounded member window +without loading the entire embedded payload. Do not assume `embed.b2e` is small +or silently download it wholesale. Record the measured access pattern in tests. + +Show explicit empty groups and the correct root/subgroup attributes when the +stored format records them. If the format does not persist empty groups, document +that limitation rather than synthesizing them. Identify embedded objects and +CTable roots from their metadata and show an unsupported-preview message; do not +expose CTable column carriers as if they were an ordinary user group. + +If bounded embedded-index access needs a larger storage-layer change, deliver the +external-array milestone with a clear partial-discovery indication and document +the missing metadata. Do not call that full B2Z hierarchy support. Remote embedded +payload and CTable preview support are separate follow-up work. + +## Zarr discovery + +Open the requested node through Zarr's existing read-only fsspec store path and +distinguish arrays from groups using node metadata. Reuse that store and its +filesystem across navigation; keep leaf reads in `ZarrNDSource` and the existing +proxy implementation. + +Use Zarr's supported group/member APIs for immediate children and group attributes. +Verify the exact installed API and supported dependency versions during +implementation. Support both Zarr v2 and v3 storage layouts already covered by +the array reader. Use consolidated metadata when available and supported; fall +back to normal metadata discovery without requiring users to consolidate stores. + +Avoid recursive enumeration of chunk objects. An unconsolidated remote store may +require LIST requests and metadata requests for child nodes, and some backends +cannot list at all. Report missing listing capability/permission clearly. Direct +array URLs should keep working when the user can read an array but cannot list +its parent. Preserve empty groups and per-group attributes. + +List datasets with unsupported dtypes/codecs, but defer decoding to selection and +show the source reader's specific limitation. Do not create another Zarr decoder +or promise finer data-fetch granularity than the current reader provides. + +## HDF5 discovery + +Translate the file to Kerchunk references once per browsing session, reusing the +existing translation and filter-registration path. Factor the shared operation +out of `available_datasets()` and `_load_or_scan_refs()` only as needed; do not +add an independent HDF5 traversal implementation inside b2view. + +Build the hierarchy from `.zgroup`, `.zarray`, and `.zattrs` entries, including +explicit empty groups. Reuse those references when opening the selected dataset +through `HDF5NDSource` or the existing proxy path that accepts `refs`. Passing a +fresh URL alone must not trigger another translation for each leaf. Preserve +dataset identity and original source information when using the shared refs. + +Display root and group attributes from their reference metadata, filtering only +known adapter bookkeeping, consistently with existing leaf attributes. Measure +the initial scan: HDF5 translation can visit metadata across the file, enumerate +many chunk references, and inline some small values. Promise no full-file +localization, not zero payload bytes or metadata cost independent of file size. + +Check how the installed translator handles unsupported datasets, hard-link +aliases, soft/external links, and cycles. Do not recursively follow arbitrary +links or contact additional external sources as part of listing. Prefer visible +unsupported nodes where the translator supplies them. If translation omits an +object or aborts the whole scan, expose/document that limitation; isolating +unsupported siblings is a completion requirement for the supported test matrix. +Do not describe the resulting view as covering every HDF5 object type. + +## UI responsiveness, failures, and lifecycle + +Run remote opening, expansion, metadata loading, and array previews through +Textual workers, following existing worker patterns in the app. Keep local-only +operations simple. Capture selection/slice state before dispatch and apply results +on the UI thread only if they still match the active request and browser session. + +Provide loading state for startup and expansion, keep navigation and quit usable, +and make failed group listings retryable. Do not add paths to `loaded_paths` until +listing succeeds. A stale worker must not repaint an old selection or repopulate +a tree after refresh. Cancellation may not interrupt an underlying blocking read; +discard stale results and close their resources after that read completes. + +Refresh must replace the discovery session, clear loaded paths, discard old leaf +caches, and attempt to restore the selected logical path, falling back to the +view root if it vanished. Ensure normal shutdown, failed startup, and refresh +close owned handles without closing a filesystem still in use by another worker. +Keep credentials in runtime storage options and out of errors or serialized state. + +## Implementation order and checkpoints + +1. **Reproduce and characterize.** Add a small valid in-memory B2Z root case that + demonstrates the bad dispatch, and inspect representative fixtures for all + three formats. Confirm embedded metadata and HDF5 translation behavior. +2. **B2Z external-array vertical slice.** Implement shared archive discovery, + minimal browser routing, root/subgroup navigation, and selected-array preview. + Guard the generic non-lazy B2Z path and preserve explicit local caching. +3. **Finish browser behavior.** Add remote workers, stale-result handling, refresh, + failures, correct node attributes, and empty-group behavior. Complete the B2Z + metadata milestone or explicitly mark its bounded-index blocker. +4. **Zarr groups.** Add discovery through native group APIs and reuse leaf readers. + Exercise v2/v3, consolidated/unconsolidated metadata, and restricted listing. +5. **HDF5 groups.** Reuse one translation and reference dictionary across selections; + verify attributes, empty groups, supported filters, and unsupported objects. +6. **Validate and document.** Run the targeted suite and real remote commands, + record traffic and startup observations, and update user-facing examples. + +The earlier one-to-two-day estimate applies only to the narrow B2Z external-array +browser. It does not cover all three formats, complete embedded metadata support, +or the UI lifecycle work above. Re-estimate after the first checkpoint resolves +the embedded-index and translator limitations. + +## Validation + +Extend existing pytest modules and use local temporary containers plus fsspec's +memory filesystem for deterministic checks. Use a counting filesystem/store to +test actual reads, not merely calls to adapter methods. + +- Root and subgroup discovery returns stable direct children with correct kinds; + empty groups remain groups. Selected group attributes differ from root attrs. +- Equivalent small B2Z, Zarr, and HDF5 arrays produce identical bounded previews + through root navigation and direct leaf URLs, including slicing and paging. +- B2Z startup does not request the complete archive or read all array frames. + Selecting one array does not fetch sibling payloads. Cover large ZIP directories, + malformed/duplicate entries, object carriers, and embedded metadata boundaries. +- Zarr discovery does not decode chunks; cover v2/v3, consolidated metadata, + unconsolidated metadata, listing failure, empty groups, and unsupported leaves. +- HDF5 translation runs once per session, not per leaf; test multiple selections, + group attributes, empty groups, unsupported/link cases, and refresh translation. +- Root-open dispatch errors occur before a bulk download. Explicit cached B2Z + opening and existing direct remote leaf opening retain their behavior. +- Fresh-process TUI tests cover compressed Zarr and HDF5 leaves reached from a + group root, preserving the decoder-mutex regression check. +- Headless TUI tests verify tree visibility, keyboard focus, expansion, initial + path navigation, selected path display, refresh, quit, and retry after failure. + Use controlled slow reads to verify that stale results cannot repaint a newer + selection or session and that shutdown does not race resource cleanup. +- Existing local TreeStore, standalone NDArray/CTable, and remote leaf tests pass. + Verify optional dependency isolation so B2Z does not acquire Zarr/HDF5 imports. + +Run Python and all tests in the `blosc2` conda environment. A starting focused run: + +```sh +conda run --no-capture-output -n blosc2 pytest tests/b2view tests/test_b2z_source.py tests/test_zarr_source.py tests/test_hdf5_source.py -m 'not network and not heavy' -q +``` + +Include new discovery tests and affected generic-open/remote-proxy tests in that +run. Explicitly include TUI markers, which the default configuration excludes. +Run Ruff and whitespace checks on the changed files; broaden testing when shared +open/source code changes warrant it. + +Finally run the three commands in Goal against available remote fixtures with +the supplied profile/endpoint. For each, verify root, nested group, array values, +attributes, refresh, and direct leaf behavior. Record first-open metadata bytes, +request counts, first-preview bytes, and repeated-preview behavior separately. +Keep credentials out of fixtures and logs. Do not claim support based only on a +mocked tree or one successful array read. + +## Documentation and completion criteria + +Update `doc/guides/b2view.rst` with remote root/subgroup examples, dependency +requirements, listing permissions, read-only behavior, and format limitations. +Clarify that b2view's hierarchy adapter does not change the array-only contract +of lazy `blosc2.open()` or introduce a persisted RemoteProxy hierarchy descriptor. + +The full v12 plan is complete when all three supported hierarchy views navigate +correctly, selected arrays reuse existing lazy readers, metadata and unsupported +nodes are represented honestly, UI operations remain responsive, and the tests +above pass. A delivered B2Z external-array milestone should be labeled as such +until the remaining metadata work is complete. + +Deferred: remote writes, full remote TreeStore/CTable semantics, embedded B2Z +payload previews, cross-container links, recursive search, persistent hierarchy +indexes, live watching, and support for additional codecs/dtypes beyond the leaf +readers. These are not prerequisites for the three-format read-only browser. + +## Implementation record + +The remote fixtures in the goal were exercised through the headless b2view CLI +from the root and a subgroup, and direct array URLs were exercised through +``StoreBrowser``. Nested navigation, attributes, array values, refresh, and quit +passed for B2Z, Zarr, and HDF5. The measured root opens were: + +| Format | Initial metadata bytes | Initial S3 operations | First preview bytes | Repeated preview bytes | +| --- | ---: | --- | ---: | ---: | +| B2Z | 8,192 | 1 HEAD, 1 GET | 32,225 | 0 | +| Zarr | 132 | 7 HEAD, 5 LIST, 2 GET | 12,420 | 0 | +| HDF5 | 52,976 | 1 HEAD, 54 GET | 13,662 | 0 | + +Zarr discovery grew to 6,515 bytes after expanding through ``/d0/d1``. HDF5 +translation remained at 52,976 bytes through hierarchy expansion because its +references were built once at session start. The request counts reflect the +installed s3fs/Zarr/Kerchunk versions and the fixture's current metadata layout; +they are observations rather than API guarantees. + +B2Z embedded arrays and remote CTable previews remain unavailable as planned. +Embedded group attributes are read with bounded native chunk access; layouts +whose metadata lives in native chunks larger than 1 MiB show an explicit partial +metadata notice. Soft and external HDF5 links and group cycles are not followed. diff --git a/src/blosc2/b2view/app.py b/src/blosc2/b2view/app.py index 685eaafad..74610012b 100644 --- a/src/blosc2/b2view/app.py +++ b/src/blosc2/b2view/app.py @@ -3,8 +3,10 @@ from __future__ import annotations import contextlib +import copy import io import os +import threading from typing import TYPE_CHECKING, Any, ClassVar import numpy as np @@ -53,7 +55,7 @@ make_metadata_renderable, make_preview_renderables, ) -from blosc2.core import parse_container_url +from blosc2.core import is_fsspec_url, parse_container_url if TYPE_CHECKING: from textual import events @@ -1971,7 +1973,7 @@ class B2ViewApp(App): #tree-pane { width: 35%; border: solid $primary; } #right-pane { width: 1fr; } #top-row { height: 40%; } - #meta-pane, #vlmeta-pane { width: 50%; border: solid $secondary; } + #meta-pane, #attrs-pane { width: 50%; border: solid $secondary; } #data-pane { height: 60%; border: solid $secondary; } #tree { height: 1fr; } #data-header { height: auto; padding: 0 1; } @@ -1979,11 +1981,12 @@ class B2ViewApp(App): #data-table { width: 1fr; height: 1fr; } #row-scrollbar { width: 1; height: 1fr; color: $primary; } #col-scrollbar { height: 1; width: 1fr; color: $primary; } - #meta-scroll, #vlmeta-scroll, #data-scroll { height: 1fr; padding: 0 1; } - #tree-pane:focus-within, #meta-pane:focus-within, #vlmeta-pane:focus-within, #data-pane:focus-within { border: heavy $accent; } + #meta-scroll, #attrs-scroll, #data-scroll { height: 1fr; padding: 0 1; } + #tree-pane:focus-within, #meta-pane:focus-within, #attrs-pane:focus-within, #data-pane:focus-within { border: heavy $accent; } B2ViewPanel.-maximized, #tree-pane.-maximized, #meta-pane.-maximized, + #attrs-pane.-maximized, #data-pane.-maximized { width: 1fr; height: 1fr; } """ @@ -2026,7 +2029,7 @@ def __init__( ): super().__init__() self.sub_title = f"Python-Blosc2 {blosc2.__version__}" # shown beside the title in the header - if parse_container_url(urlpath)[2] == "zarr": + if parse_container_url(urlpath)[2] in {"zarr", "hdf5"}: # Initialize before Textual captures stderr (fileno=-1), which # prevents multiprocessing's resource tracker from starting. with contextlib.suppress(ImportError): @@ -2047,6 +2050,16 @@ def __init__( self.preview_cols = preview_cols self.browser: StoreBrowser | None = None self.loaded_paths: set[str] = set() + self._remote = is_fsspec_url(urlpath) + self._remote_session = 0 + self._remote_request = 0 + self._remote_page_request = 0 + self._remote_page_pending = False + self._remote_col_end = None + self._remote_children = {} + self._listing_paths = set() + self._selected_info = None + self._closing = False self.selected_path = "/" self.table_page: dict | None = None self.table_buffer: dict | None = None @@ -2079,10 +2092,10 @@ def compose(self) -> ComposeResult: meta_pane.border_title = "meta" with VerticalScroll(id="meta-scroll", can_focus=True): yield Static("Select a node", id="metadata") - with B2ViewPanel(id="vlmeta-pane") as vlmeta_pane: - vlmeta_pane.border_title = "vlmeta" - with VerticalScroll(id="vlmeta-scroll", can_focus=True): - yield Static("", id="vlmetadata") + with B2ViewPanel(id="attrs-pane") as attrs_pane: + attrs_pane.border_title = "attrs" + with VerticalScroll(id="attrs-scroll", can_focus=True): + yield Static("", id="attrs-data") with B2ViewPanel(id="data-pane") as data_pane: data_pane.border_title = "data" data_pane.border_subtitle = ( @@ -2136,7 +2149,14 @@ def _after_download(self, result: bool | str) -> None: def _start_browsing(self) -> None: """Open the bundle and populate the tree (the normal startup path).""" + if self._remote: + self.query_one("#metadata", Static).update("Loading remote container…") + self._open_remote(self._remote_session, self.start_path) + return self.browser = StoreBrowser(self.urlpath, storage_options=self.storage_options) + self._populate_browser() + + def _populate_browser(self) -> None: self.query_one("#tree-pane").display = self.browser.is_tree self.query_one(B2ViewHeader).set_filename(self._header_label) tree = self.query_one("#tree", Tree) @@ -2171,7 +2191,8 @@ def _focus_panel_by_name(self, name: str) -> None: panel_map = { "tree": lambda: self.query_one("#tree", Tree), "meta": lambda: self.query_one("#meta-scroll", VerticalScroll), - "vlmeta": lambda: self.query_one("#vlmeta-scroll", VerticalScroll), + "attrs": lambda: self.query_one("#attrs-scroll", VerticalScroll), + "vlmeta": lambda: self.query_one("#attrs-scroll", VerticalScroll), "data": lambda: ( self.query_one("#data-table", DataTable) if self.query_one("#data-table-row", Horizontal).display @@ -2214,19 +2235,122 @@ def _do_select(): self.call_after_refresh(_do_select) + @staticmethod + def _close_browser(browser): + with browser.io_lock: + browser.close() + def on_unmount(self) -> None: + self._closing = True + self._remote_session += 1 if self.browser is not None: - self.browser.close() + if self._remote: + threading.Thread(target=self._close_browser, args=(self.browser,), daemon=True).start() + else: + self.browser.close() + + def _deliver_remote(self, session, callback, *args): + """A blocking read may finish after cancellation, refresh, or shutdown.""" + if self._closing or session != self._remote_session: + return False + + def deliver(): + if self._closing or session != self._remote_session: + return False + callback(*args) + return True + + try: + return self.call_from_thread(deliver) + except RuntimeError: + return False + + @work(thread=True, exit_on_error=False) + def _open_remote(self, session, start_path): + browser = None + try: + browser = StoreBrowser(self.urlpath, storage_options=self.storage_options) + children = {} + if browser.is_tree: + parent = "/" + children[parent] = browser.list_children(parent) + for part in start_path.strip("/").split("/"): + target = parent.rstrip("/") + "/" + part + found = next((c for c in children[parent] if c.path == target), None) + if found is None or found.kind != "group": + break + parent = target + children[parent] = browser.list_children(parent) + if not self._deliver_remote(session, self._finish_remote_open, browser, children): + browser.close() + except Exception as exc: + if browser is not None: + browser.close() + self._deliver_remote(session, self._remote_error, exc) + + def _finish_remote_open(self, browser, children): + self.browser = browser + self._remote_children = children + self._populate_browser() + + def _remote_error(self, exc): + # Transport exceptions can include signed URLs or credentials. Keep + # source-specific limitations, but remove runtime URLs and option values. + import re + + message = re.sub(r"[a-zA-Z][a-zA-Z0-9+.-]*://[^\s'\"]+", "", str(exc)) + + def redact(options): + nonlocal message + for value in options.values(): + if isinstance(value, dict): + redact(value) + elif isinstance(value, str) and value: + message = message.replace(value, "