diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b3f199db1..868b01763 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -88,15 +88,45 @@ jobs: - name: Test (Windows) if: runner.os == 'Windows' - run: python -m pytest -m "not heavy and not network" --durations=25 -# env: + run: python -m pytest -m "not heavy and not network and not tui" --durations=25 + env: + # A test over this many seconds is a deadlock, not a slow test (the + # slowest is ~7 s): dump stacks and kill the worker so xdist names it + # instead of the job hanging until the 6-hour cap. The session budget + # catches the same hang in teardown or interpreter exit, where no test + # is running to be named. + PYTEST_DEADMAN_SECONDS: "300" + PYTEST_DEADMAN_SESSION_SECONDS: "1500" # BLOSC_NTHREADS: "1" # NUMEXPR_NUM_THREADS: "1" # OMP_NUM_THREADS: "1" - name: Test (non-Windows) if: runner.os != 'Windows' - run: python -m pytest -m "not heavy and not network" --durations=25 + run: python -m pytest -m "not heavy and not network and not tui" --durations=25 + env: + PYTEST_DEADMAN_SECONDS: "300" + PYTEST_DEADMAN_SESSION_SECONDS: "1500" + + # TUIs are timing-sensitive and only boot a headless terminal: one OS is + # enough, and a flake here is feedback, not a release blocker. + - name: Test TUI (non-blocking) + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' && !matrix.numpy-version + continue-on-error: true + run: python -m pytest -m "tui and not heavy and not network" -n 4 --dist loadfile --durations=10 + env: + PYTEST_DEADMAN_SECONDS: "300" + PYTEST_DEADMAN_SESSION_SECONDS: "900" + + - name: Deadlock stacks, if a worker had to be killed + if: always() + shell: bash + run: | + if [ -s deadman-stacks.log ]; then + echo "::group::Deadlock stacks" + cat deadman-stacks.log + echo "::endgroup::" + fi # The network tests talk to a live Caterva2 server and were 24 of the 25 # slowest tests in *both* the Windows and Linux jobs, with near-identical diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index daf86511b..7933e2ade 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,6 +21,9 @@ repos: - id: ruff-check args: ["--fix", "--show-fixes"] - id: ruff-format + # Ruff 0.16+ can claim Markdown code blocks; blacken-docs owns those + # blocks here, so keep Ruff on Python sources and notebooks. + types_or: [python, pyi, jupyter] exclude: ^bench/ - repo: https://github.com/adamchainz/blacken-docs diff --git a/CMakeLists.txt b/CMakeLists.txt index 38284e6cc..b65e96565 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,8 +16,8 @@ project(python-blosc2) # Update to the latest minimum C-Blosc2 version, and to the actual bundled one set(BLOSC2_MIN_VERSION 3.3.0) -set(BLOSC2_BUNDLED_VERSION v3.3.3) -# set(BLOSC2_BUNDLED_VERSION a7f9a5be3527e87a7cbce1c16977835361d27033) +set(BLOSC2_BUNDLED_VERSION v3.3.4) +#set(BLOSC2_BUNDLED_VERSION a54e259aeb1c0e62d6bbea28934369d4831e4046) if(WIN32 AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang") message(FATAL_ERROR "Windows builds require clang-cl. Set CC/CXX to clang-cl or configure CMake with -T ClangCL.") diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index fdc705ec4..4849966ef 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -2,7 +2,24 @@ ## Changes from 4.12.0 to 4.13.0 -XXX version-specific blurb XXX +This release adds portable remote array references, bounded caches, and remote +hierarchy browsing for B2Z, Zarr, and HDF5 containers. + +* `blosc2.open(..., lazy=True)` now returns a `RemoteArray` with a bounded + memory cache by default. `CachePolicy.NONE` disables retention; `cache_dir` + or `cache_path` selects persistent disk caching. `cache_storage` is deprecated + in favor of `cache_dir`. +* `RemoteStore` discovers remote hierarchies and shares a cache budget across + their arrays. Stores and arrays can export portable references with optional + cached data. `b2view` supports browsing these remote sources. +* New `ZarrNDSource`, `HDF5NDSource`, and `B2ZNDSource` adapters read immutable + arrays on demand. The `b2nd-to-zarr` command converts local NDArrays to Zarr. + +* Use `.attrs` as the recommended interface for user-defined metadata. Arrays, + containers and proxy sources now expose it as an alias for `.vlmeta`, preserving + existing storage and access rules. `C2Array.attrs` continues to select the + server's user attributes while `C2Array.vlmeta` retains protocol metadata. + `.vlmeta` remains supported and is not deprecated. ## Changes from 4.11.0 to 4.12.0 diff --git a/bench/remote_array_traffic.jsonl b/bench/remote_array_traffic.jsonl new file mode 100644 index 000000000..bdd1d1f03 --- /dev/null +++ b/bench/remote_array_traffic.jsonl @@ -0,0 +1,18 @@ +{"backend": "b2z", "transport": "s3", "url": "s3://blosc2/hierarchy.b2z", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 1.2488116670865566, "requests": 3, "methods": {"GET": 2, "HEAD": 1}, "connections": 1, "body_bytes": 24576, "traffic_requests": 2, "traffic_bytes": 24576}, "cold_slice": {"seconds": 0.49289220792707056, "requests": 2, "methods": {"GET": 2}, "connections": 0, "body_bytes": 16241, "traffic_requests": 2, "traffic_bytes": 16241}, "warm_slice": {"seconds": 0.0009889589855447412, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 1} +{"backend": "b2z", "transport": "https", "url": "https://f001.backblazeb2.com/file/blosc2/hierarchy.b2z", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 1.043098833062686, "requests": 3, "methods": {"GET": 2, "HEAD": 1}, "connections": 1, "body_bytes": 24576, "traffic_requests": 2, "traffic_bytes": 24576}, "cold_slice": {"seconds": 0.48495695809833705, "requests": 2, "methods": {"GET": 2}, "connections": 0, "body_bytes": 16241, "traffic_requests": 2, "traffic_bytes": 16241}, "warm_slice": {"seconds": 0.0010930829448625445, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 1} +{"backend": "zarr", "transport": "s3", "url": "s3://blosc2/hierarchy.zarr/d0/a3", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 7.477702042087913, "requests": 6, "methods": {"GET": 3, "HEAD": 3}, "connections": 5, "body_bytes": 1788, "traffic_requests": 1, "traffic_bytes": 1042}, "cold_slice": {"seconds": 0.4852743750670925, "requests": 2, "methods": {"GET": 1, "HEAD": 1}, "connections": 0, "body_bytes": 11645, "traffic_requests": 1, "traffic_bytes": 11645}, "warm_slice": {"seconds": 0.00036441697739064693, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 1} +{"backend": "zarr", "transport": "https", "url": "https://f001.backblazeb2.com/file/blosc2/hierarchy.zarr/d0/a3", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 0.6389846670208499, "requests": 3, "methods": {"GET": 3}, "connections": 3, "body_bytes": 1230, "traffic_requests": 1, "traffic_bytes": 1042}, "cold_slice": {"seconds": 0.265499584027566, "requests": 1, "methods": {"GET": 1}, "connections": 0, "body_bytes": 11645, "traffic_requests": 1, "traffic_bytes": 11645}, "warm_slice": {"seconds": 0.00035462493542581797, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 1} +{"backend": "h5", "transport": "s3", "url": "s3://blosc2/hierarchy.h5", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 11.35246516589541, "requests": 57, "methods": {"GET": 56, "HEAD": 1}, "connections": 1, "body_bytes": 47332, "traffic_requests": 2, "traffic_bytes": 392}, "cold_slice": {"seconds": 0.2362309170421213, "requests": 1, "methods": {"GET": 1}, "connections": 0, "body_bytes": 13399, "traffic_requests": 1, "traffic_bytes": 13399}, "warm_slice": {"seconds": 0.0005953341023996472, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 1} +{"backend": "h5", "transport": "https", "url": "https://f001.backblazeb2.com/file/blosc2/hierarchy.h5", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 11.591361916973256, "requests": 57, "methods": {"GET": 56, "HEAD": 1}, "connections": 1, "body_bytes": 47332, "traffic_requests": 2, "traffic_bytes": 392}, "cold_slice": {"seconds": 0.2375075420131907, "requests": 1, "methods": {"GET": 1}, "connections": 0, "body_bytes": 13399, "traffic_requests": 1, "traffic_bytes": 13399}, "warm_slice": {"seconds": 0.0005853750044479966, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 1} +{"backend": "b2z", "transport": "s3", "url": "s3://blosc2/hierarchy.b2z", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 1.2223038750234991, "requests": 3, "methods": {"GET": 2, "HEAD": 1}, "connections": 1, "body_bytes": 24576, "traffic_requests": 2, "traffic_bytes": 24576}, "cold_slice": {"seconds": 0.4608378750272095, "requests": 2, "methods": {"GET": 2}, "connections": 0, "body_bytes": 16241, "traffic_requests": 2, "traffic_bytes": 16241}, "warm_slice": {"seconds": 0.0009959590388461947, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 2} +{"backend": "b2z", "transport": "https", "url": "https://f001.backblazeb2.com/file/blosc2/hierarchy.b2z", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 1.0011600840371102, "requests": 3, "methods": {"GET": 2, "HEAD": 1}, "connections": 1, "body_bytes": 24576, "traffic_requests": 2, "traffic_bytes": 24576}, "cold_slice": {"seconds": 0.4179259999655187, "requests": 2, "methods": {"GET": 2}, "connections": 0, "body_bytes": 16241, "traffic_requests": 2, "traffic_bytes": 16241}, "warm_slice": {"seconds": 0.0010035419836640358, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 2} +{"backend": "zarr", "transport": "s3", "url": "s3://blosc2/hierarchy.zarr/d0/a3", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 1.3998966669896618, "requests": 6, "methods": {"GET": 3, "HEAD": 3}, "connections": 5, "body_bytes": 1788, "traffic_requests": 1, "traffic_bytes": 1042}, "cold_slice": {"seconds": 0.4981873750220984, "requests": 2, "methods": {"GET": 1, "HEAD": 1}, "connections": 0, "body_bytes": 11645, "traffic_requests": 1, "traffic_bytes": 11645}, "warm_slice": {"seconds": 0.00038591690827161074, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 2} +{"backend": "zarr", "transport": "https", "url": "https://f001.backblazeb2.com/file/blosc2/hierarchy.zarr/d0/a3", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 0.6370132910087705, "requests": 3, "methods": {"GET": 3}, "connections": 3, "body_bytes": 1230, "traffic_requests": 1, "traffic_bytes": 1042}, "cold_slice": {"seconds": 0.2670587080065161, "requests": 1, "methods": {"GET": 1}, "connections": 0, "body_bytes": 11645, "traffic_requests": 1, "traffic_bytes": 11645}, "warm_slice": {"seconds": 0.0004256248939782381, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 2} +{"backend": "h5", "transport": "s3", "url": "s3://blosc2/hierarchy.h5", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 11.579921333002858, "requests": 57, "methods": {"GET": 56, "HEAD": 1}, "connections": 1, "body_bytes": 47332, "traffic_requests": 2, "traffic_bytes": 392}, "cold_slice": {"seconds": 0.25883508403785527, "requests": 1, "methods": {"GET": 1}, "connections": 0, "body_bytes": 13399, "traffic_requests": 1, "traffic_bytes": 13399}, "warm_slice": {"seconds": 0.0006202919175848365, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 2} +{"backend": "h5", "transport": "https", "url": "https://f001.backblazeb2.com/file/blosc2/hierarchy.h5", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 11.457834250060841, "requests": 57, "methods": {"GET": 56, "HEAD": 1}, "connections": 1, "body_bytes": 47332, "traffic_requests": 2, "traffic_bytes": 392}, "cold_slice": {"seconds": 0.2313700410304591, "requests": 1, "methods": {"GET": 1}, "connections": 0, "body_bytes": 13399, "traffic_requests": 1, "traffic_bytes": 13399}, "warm_slice": {"seconds": 0.0005405419506132603, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 2} +{"backend": "b2z", "transport": "s3", "url": "s3://blosc2/hierarchy.b2z", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 1.2231810830999166, "requests": 3, "methods": {"GET": 2, "HEAD": 1}, "connections": 1, "body_bytes": 24576, "traffic_requests": 2, "traffic_bytes": 24576}, "cold_slice": {"seconds": 0.4398518749512732, "requests": 2, "methods": {"GET": 2}, "connections": 0, "body_bytes": 16241, "traffic_requests": 2, "traffic_bytes": 16241}, "warm_slice": {"seconds": 0.0009655829053372145, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 3} +{"backend": "b2z", "transport": "https", "url": "https://f001.backblazeb2.com/file/blosc2/hierarchy.b2z", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 1.0051324159139767, "requests": 3, "methods": {"GET": 2, "HEAD": 1}, "connections": 1, "body_bytes": 24576, "traffic_requests": 2, "traffic_bytes": 24576}, "cold_slice": {"seconds": 0.41061416699085385, "requests": 2, "methods": {"GET": 2}, "connections": 0, "body_bytes": 16241, "traffic_requests": 2, "traffic_bytes": 16241}, "warm_slice": {"seconds": 0.0004735830007120967, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 3} +{"backend": "zarr", "transport": "s3", "url": "s3://blosc2/hierarchy.zarr/d0/a3", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 1.4775527090532705, "requests": 6, "methods": {"GET": 3, "HEAD": 3}, "connections": 5, "body_bytes": 1788, "traffic_requests": 1, "traffic_bytes": 1042}, "cold_slice": {"seconds": 0.4406578750349581, "requests": 2, "methods": {"GET": 1, "HEAD": 1}, "connections": 0, "body_bytes": 11645, "traffic_requests": 1, "traffic_bytes": 11645}, "warm_slice": {"seconds": 0.00023274996783584356, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 3} +{"backend": "zarr", "transport": "https", "url": "https://f001.backblazeb2.com/file/blosc2/hierarchy.zarr/d0/a3", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 6.490244832937606, "requests": 3, "methods": {"GET": 3}, "connections": 3, "body_bytes": 1230, "traffic_requests": 1, "traffic_bytes": 1042}, "cold_slice": {"seconds": 0.34011666697915643, "requests": 2, "methods": {"GET": 2}, "connections": 0, "body_bytes": 11645, "traffic_requests": 1, "traffic_bytes": 11645}, "warm_slice": {"seconds": 0.00039166701026260853, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 3} +{"backend": "h5", "transport": "s3", "url": "s3://blosc2/hierarchy.h5", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 11.601201582932845, "requests": 57, "methods": {"GET": 56, "HEAD": 1}, "connections": 1, "body_bytes": 47332, "traffic_requests": 2, "traffic_bytes": 392}, "cold_slice": {"seconds": 0.2506593340076506, "requests": 1, "methods": {"GET": 1}, "connections": 0, "body_bytes": 13399, "traffic_requests": 1, "traffic_bytes": 13399}, "warm_slice": {"seconds": 0.0004546659765765071, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 3} +{"backend": "h5", "transport": "https", "url": "https://f001.backblazeb2.com/file/blosc2/hierarchy.h5", "dataset": "d0/a3", "shape": [10, 1000, 1000], "dtype": "int32", "chunks": [2, 500, 500], "blocks": [1, 50, 500], "slice": "[0, :100, :100]", "result_bytes": 40000, "cache_bytes": 15598, "blosc2_version": "4.13.0.dev0", "phases": {"open": {"seconds": 13.41276800003834, "requests": 57, "methods": {"GET": 56, "HEAD": 1}, "connections": 1, "body_bytes": 47332, "traffic_requests": 2, "traffic_bytes": 392}, "cold_slice": {"seconds": 0.23232333292253315, "requests": 1, "methods": {"GET": 1}, "connections": 0, "body_bytes": 13399, "traffic_requests": 1, "traffic_bytes": 13399}, "warm_slice": {"seconds": 0.0005118330009281635, "requests": 0, "methods": {}, "connections": 0, "body_bytes": 0, "traffic_requests": 0, "traffic_bytes": 0}}, "repetition": 3} diff --git a/bench/remote_array_traffic.md b/bench/remote_array_traffic.md new file mode 100644 index 000000000..75b591603 --- /dev/null +++ b/bench/remote_array_traffic.md @@ -0,0 +1,74 @@ +# RemoteArray cold/warm network measurements + +Measured 2026-09-09 on macOS arm64 in the `blosc2` conda environment. + +Three fresh-process trials per format/transport. Each opens `d0/a3`, reads `[0, :100, :100]`, then repeats that slice on the same handle. All six sources have shape `(10, 1000, 1000)`, dtype `int32`, chunks `(2, 500, 500)` and blocks `(1, 50, 500)`. The result is 40,000 bytes from a 40,000,000-byte logical array. Every result was checked against `np.arange(100)[:, None] * 1000 + np.arange(100)`. + +Policy: MEMORY, 64 MiB allowance, default concurrency and immutable-source behavior. Every handle retained 15,598 compressed cache bytes. Cold means a fresh process without an application payload cache or client connection pool; service/CDN caches and OS DNS caches were not cleared. Warm means a same-handle memory hit. Disk-cache reopening and the future RemoteStore manifests are not measured here. + +Sources: `s3://blosc2/hierarchy.{b2z,zarr,h5}` through `https://s3.us-west-001.backblazeb2.com` (runtime `blosc2` profile), and `https://f001.backblazeb2.com/file/blosc2/hierarchy.{b2z,zarr,h5}`. Zarr appends `/d0/a3`; B2Z/HDF5 use `dataset="d0/a3"`. + +## Cold total: opening plus first slice + +Times are medians; variable counts are shown as ranges. + +| Format | Transport | HTTP requests | New connections | Downloaded body bytes | Time (s) | Warm time (ms) | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| .b2z | S3 | 5 | 1 | 40,817 | 1.683 | 0.989 | +| .b2z | HTTPS | 5 | 1 | 40,817 | 1.419 | 1.004 | +| .zarr | S3 | 8 | 5 | 13,433 | 1.918 | 0.364 | +| .zarr | HTTPS | 4–5 | 3 | 12,875 | 0.904 | 0.392 | +| .h5 | S3 | 58 | 1 | 60,731 | 11.839 | 0.595 | +| .h5 | HTTPS | 58 | 1 | 60,731 | 11.829 | 0.541 | + +**Every warm read: 0 requests, 0 new connections and 0 downloaded bytes.** Byte and connection totals were identical across repetitions. Zarr HTTPS used 4 requests in two trials and 5 in one. + +## Opening versus first data read + +| Format | Transport | Phase | HTTP requests | New connections | Body bytes | Time (ms) | +| --- | --- | --- | ---: | ---: | ---: | ---: | +| .b2z | S3 | open | 3 | 1 | 24,576 | 1223.181 | +| .b2z | S3 | cold_slice | 2 | 0 | 16,241 | 460.838 | +| .b2z | S3 | warm_slice | 0 | 0 | 0 | 0.989 | +| .b2z | HTTPS | open | 3 | 1 | 24,576 | 1005.132 | +| .b2z | HTTPS | cold_slice | 2 | 0 | 16,241 | 417.926 | +| .b2z | HTTPS | warm_slice | 0 | 0 | 0 | 1.004 | +| .zarr | S3 | open | 6 | 5 | 1,788 | 1477.553 | +| .zarr | S3 | cold_slice | 2 | 0 | 11,645 | 485.274 | +| .zarr | S3 | warm_slice | 0 | 0 | 0 | 0.364 | +| .zarr | HTTPS | open | 3 | 3 | 1,230 | 638.985 | +| .zarr | HTTPS | cold_slice | 1–2 | 0 | 11,645 | 267.059 | +| .zarr | HTTPS | warm_slice | 0 | 0 | 0 | 0.392 | +| .h5 | S3 | open | 57 | 1 | 47,332 | 11579.921 | +| .h5 | S3 | cold_slice | 1 | 0 | 13,399 | 250.659 | +| .h5 | S3 | warm_slice | 0 | 0 | 0 | 0.595 | +| .h5 | HTTPS | open | 57 | 1 | 47,332 | 11591.362 | +| .h5 | HTTPS | cold_slice | 1 | 0 | 13,399 | 232.323 | +| .h5 | HTTPS | warm_slice | 0 | 0 | 0 | 0.541 | + +## Interpretation and accounting + +- HDF5 cold opening translates the file with Kerchunk: 57 requests and 47,332 downloaded bytes, followed by one data request (13,399 bytes). Discovery dominates this small slice. This supports the planned persistent discovery manifest, which is not implemented yet. +- B2Z uses five requests and 40,817 bytes on both transports. Opening reads a ZIP tail and member prefix; the first slice adds native frame/chunk reads. +- Zarr HTTPS uses 4–5 requests overall; S3 uses eight, including four HEAD requests. Data-slice bytes are identical (11,645), but metadata probing/error bodies differ. S3's additional round trips contribute to its higher median latency here. S3 and direct HTTPS both use HTTPS on the wire, but use different endpoints and client paths. +- Latency varies: one Zarr S3 open took 7.478 s (others 1.400/1.478 s); one Zarr HTTPS open took 6.490 s (others 0.637/0.639 s). The latter trial also issued a second GET for the slice, with unchanged total body bytes. The instrumentation does not establish why that extra send occurred. These trials remain in the raw results; no outliers were discarded. +- Every format returns the same values, but native compressed representations differ. This regular arange fixture and one small slice do not establish general compression or throughput rankings. +- Requests are counted at `aiohttp.ClientRequest.send`; successful connection creations at `TCPConnector._create_connection`. Multiple requests reuse connections. These are client send attempts/new connections, not server-side access-log counts. +- Bytes count response bodies received by `aiohttp.StreamReader.feed_data`, including discovery, missing-key responses and array data. They exclude HTTP headers, TLS/TCP overhead and outgoing bytes. These are body-traffic measurements, not packet-level link usage. +- Built-in `array.traffic` counters are retained in the raw results but are not used as total network traffic. HDF5 opening reports only 392 bytes there versus 47,332 response-body bytes: standalone Kerchunk scanning is currently outside that counter. HEAD requests and failed Zarr metadata probes are also absent from the built-in tally. + +## Fix and validation + +The initial HTTPS HDF5 trial failed with `ValueError: Cannot seek streaming HTTP file`. The shared `scan_hdf5_refs` helper used `block_size=0`, selecting a streaming HTTP file. It now uses `block_size=1, cache_type="none"`, preserving seekability without read-ahead for HTTP and S3. Results above were collected after this fix. + +Existing fsspec/HDF5 tests: 155 passed. The new HTTP metadata/range/warm-cache regression test also passed after correcting its fixture directory. All 18 real-network trials passed value and warm-cache assertions. Ruff check/format and diff whitespace checks passed for changed code. The full repository suite was not run for this measurement task. + +Environment: `blosc2 4.13.0.dev0`, `fsspec 2026.7.0`, `s3fs 2026.7.0`, `aiohttp 3.14.3`, `zarr 3.3.0`, `kerchunk 0.2.10`, `h5py 3.16.0`. + +## Reproduce + +```sh +conda run --no-capture-output -n blosc2 python bench/remote_array_traffic.py --repeats 3 > bench/remote_array_traffic.jsonl +``` + +Requires network access and the existing runtime `blosc2` S3 profile. Only remote reads are performed. See [benchmark script](remote_array_traffic.py) and [all trial results](remote_array_traffic.jsonl). diff --git a/bench/remote_array_traffic.py b/bench/remote_array_traffic.py new file mode 100644 index 000000000..a7c6bf983 --- /dev/null +++ b/bench/remote_array_traffic.py @@ -0,0 +1,139 @@ +"""Measure real remote cold/warm reads; run in the blosc2 conda environment. + +python bench/remote_array_traffic.py > bench/remote_array_traffic.jsonl + +Each trial runs in a fresh process. Transport bytes are aiohttp response bodies +(including metadata/errors), excluding headers and TLS/TCP overhead. Connections +are successful aiohttp connection creations, distinct from HTTP requests. +Uses the existing read-only Backblaze fixtures and the runtime blosc2 S3 profile. +""" + +import argparse +import json +import subprocess +import sys +from collections import Counter +from pathlib import Path +from time import perf_counter +from unittest.mock import patch + +import aiohttp +import numpy as np + +import blosc2 + + +def trial(backend, transport): + counts = Counter() + send = aiohttp.ClientRequest.send + connect = aiohttp.TCPConnector._create_connection + feed = aiohttp.StreamReader.feed_data + + async def counted_send(request, connection): + counts[request.method] += 1 + return await send(request, connection) + + async def counted_connect(connector, *args, **kwargs): + result = await connect(connector, *args, **kwargs) + counts["connections"] += 1 + return result + + def counted_feed(reader, data, *args, **kwargs): + counts["body_bytes"] += len(data) + return feed(reader, data, *args, **kwargs) + + root = "s3://blosc2" if transport == "s3" else "https://f001.backblazeb2.com/file/blosc2" + url = f"{root}/hierarchy.{backend}" + options = {} + if transport == "s3": + options = {"profile": "blosc2", "endpoint_url": "https://s3.us-west-001.backblazeb2.com"} + kwargs = {"dataset": "d0/a3"} if backend != "zarr" else {} + if backend == "zarr": + url += "/d0/a3" + phases = {} + + def measure(name, operation, array=None): + before = counts.copy() + traffic_before = (array.traffic.requests, array.traffic.nbytes) if array is not None else (0, 0) + start = perf_counter() + result = operation() + elapsed = perf_counter() - start + delta = counts - before + current = result if array is None else array + phases[name] = { + "seconds": elapsed, + "requests": sum(delta[method] for method in ("GET", "HEAD", "POST", "PUT", "DELETE")), + "methods": {method: delta[method] for method in ("GET", "HEAD") if delta[method]}, + "connections": delta["connections"], + "body_bytes": delta["body_bytes"], + "traffic_requests": current.traffic.requests - traffic_before[0], + "traffic_bytes": current.traffic.nbytes - traffic_before[1], + } + return result + + with ( + patch.object(aiohttp.ClientRequest, "send", counted_send), + patch.object(aiohttp.TCPConnector, "_create_connection", counted_connect), + patch.object(aiohttp.StreamReader, "feed_data", counted_feed), + ): + array = measure( + "open", + lambda: blosc2.open( + url, + lazy=True, + cache_policy=blosc2.CachePolicy.MEMORY, + max_cache_bytes=64 << 20, + storage_options=options, + **kwargs, + ), + ) + cold = measure("cold_slice", lambda: array[0, :100, :100], array) + warm = measure("warm_slice", lambda: array[0, :100, :100], array) + expected = np.arange(100)[:, None] * 1000 + np.arange(100) + np.testing.assert_array_equal(cold, expected) + np.testing.assert_array_equal(warm, cold) + assert phases["cold_slice"]["body_bytes"] > 0 + assert phases["warm_slice"]["requests"] == phases["warm_slice"]["body_bytes"] == 0 + assert array.cache_bytes <= 64 << 20 + return { + "backend": backend, + "transport": transport, + "url": url, + "dataset": "d0/a3", + "shape": array.shape, + "dtype": str(array.dtype), + "chunks": array.chunks, + "blocks": array.blocks, + "slice": "[0, :100, :100]", + "result_bytes": cold.nbytes, + "cache_bytes": array.cache_bytes, + "blosc2_version": blosc2.__version__, + "phases": phases, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--case", nargs=2, metavar=("BACKEND", "TRANSPORT")) + parser.add_argument("--repeats", type=int, default=3) + args = parser.parse_args() + if args.case: + print(json.dumps(trial(*args.case)), flush=True) + return + for repetition in range(args.repeats): + for backend in ("b2z", "zarr", "h5"): + for transport in ("s3", "https"): + result = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), "--case", backend, transport], + check=True, + stdout=subprocess.PIPE, + text=True, + timeout=240, + ) + row = json.loads(result.stdout) + row["repetition"] = repetition + 1 + print(json.dumps(row), flush=True) + + +if __name__ == "__main__": + main() diff --git a/doc/getting_started/installation.rst b/doc/getting_started/installation.rst index b0b086a04..224f81f78 100644 --- a/doc/getting_started/installation.rst +++ b/doc/getting_started/installation.rst @@ -40,12 +40,18 @@ grouped into *extras* that you opt into with the ``blosc2[extra]`` syntax: * - ``parquet`` - The ``parquet-to-blosc2`` converter (``pyarrow``); see :doc:`../guides/parquet_to_blosc2`. + * - ``zarr`` + - Lazy Zarr sources and the ``b2nd-to-zarr`` (or ``blosc2-to-zarr``) + converter (``zarr``). + * - ``hdf5`` + - Reading HDF5 datasets lazily as virtual arrays via kerchunk (``kerchunk``, + ``h5py``, ``hdf5plugin``). * - ``fsspec`` - Reading and writing single-file containers through any `fsspec - `_ URL. The driver for each - protocol is a separate install (``s3fs`` for ``s3://``, ``gcsfs`` for - ``gs://``, ``adlfs`` for ``abfs://``...), and credentials are configured - through the driver, not through blosc2. + `_ URL. The HTTP(S) driver is + included. Other protocol drivers are separate installs (``s3fs`` for + ``s3://``, ``gcsfs`` for ``gs://``, ``adlfs`` for ``abfs://``...), and + credentials are configured through the driver, not through blosc2. Install one or more extras by listing them in brackets (quote the argument in shells like ``zsh`` that treat brackets specially): @@ -55,15 +61,18 @@ argument in shells like ``zsh`` that treat brackets specially): pip install "blosc2[tui]" # the b2view terminal browser pip install "blosc2[hires]" # b2view + its high-res view (h key) pip install "blosc2[parquet]" # the Parquet converter + pip install "blosc2[fsspec]" # fsspec URLs, including HTTP(S) pip install "blosc2[fsspec]" s3fs # fsspec URLs, plus the S3 driver + pip install "blosc2[zarr,fsspec]" s3fs # remote Zarr arrays on S3 + pip install "blosc2[hdf5,fsspec]" s3fs # remote HDF5 datasets on S3 pip install "blosc2[tui,parquet]" # several at once With the ``fsspec`` extra, :func:`blosc2.open` accepts any fsspec URL, chained -ones included, and reads it whole, through a local cache (``cache_storage=``), or +ones included, and reads it whole, through a local cache (``cache_dir=``), or by fetching only the chunks and blocks a slice touches (``lazy=True``); see :func:`blosc2.open` and :ref:`FsspecNDSource` for what each mode supports. -``examples/ndarray/rw-fsspec.py`` walks through all three plus the write side, -and ``examples/ndarray/concurrent-fsspec.py`` shows what overlapping the +``examples/remote/rw-fsspec.py`` walks through all three plus the write side, +and ``examples/remote/concurrent-fsspec.py`` shows what overlapping the fetches buys; both run with no network or credentials. Source code diff --git a/doc/guides/b2view.rst b/doc/guides/b2view.rst index c58a772cd..f21763346 100644 --- a/doc/guides/b2view.rst +++ b/doc/guides/b2view.rst @@ -4,7 +4,7 @@ b2view: Browse TreeStore Bundles in the Terminal The ``b2view`` CLI opens an interactive terminal browser (TUI) for Blosc2 TreeStore bundles, either sparse directories (``.b2d``) or compact zip-backed files (``.b2z``). It shows the tree of groups and nodes, the -metadata and vlmeta of the selected node, and a paged view of the data +metadata and attrs of the selected node, and a paged view of the data itself — NDArrays of any dimensionality as well as CTables. ``b2view`` is opt-in: install it with the ``tui`` extra — @@ -24,9 +24,9 @@ arrays and some metadata: import blosc2 with blosc2.TreeStore("sample.b2z", mode="w") as tstore: - tstore.vlmeta["author"] = "me" + tstore.attrs["author"] = "me" a = blosc2.linspace(0, 1, num=1_000_000, shape=(1000, 1000)) - a.vlmeta["description"] = "a 2-D linspace" + a.attrs["description"] = "a 2-D linspace" tstore["/dense/a"] = a tstore["/dense/b"] = blosc2.arange(10_000, shape=(10, 100, 10)) @@ -41,10 +41,14 @@ Step 2 — Open it b2view sample.b2z The screen is split into four panels: the **tree** of the bundle on the -left, and **meta**, **vlmeta** and **data** panels for the node selected +left, and **meta**, **attrs** and **data** panels for the node selected in the tree. Move between panels with ``tab`` / ``shift+tab``, maximize the focused one with ``m`` (``r`` restores it), and quit with ``q``. +For standalone objects such as an NDArray or CTable, the tree panel is hidden, +the remaining panels use the full width, and focus starts in the data panel by default. +The metadata omits the internal root path; the header shows the source path. + By default the mouse is left to the terminal, so selecting and copying text works as in any other command line program. Pass ``--mouse`` to let b2view capture it instead: panels become clickable and the wheel scrolls the data @@ -56,6 +60,75 @@ You can also jump straight to a node and panel: b2view sample.b2z /dense/a --panel data +Remote containers and arrays +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Browse remote B2Z, Zarr, and HDF5 containers directly from their root: + +.. code-block:: console + + b2view --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com s3://blosc2/hierarchy.b2z + b2view --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com s3://blosc2/hierarchy.zarr + b2view --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com s3://blosc2/hierarchy.h5 + +Append a group path to browse a subtree, or an array path to open a standalone +array without a tree panel. Both ``/`` and ``::`` dataset addressing work: + +.. code-block:: console + + b2view s3://blosc2/hierarchy.b2z/d0/d1 --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com + b2view s3://blosc2/hierarchy.zarr::d0/d1/a2 --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com --panel data + +The supplied URL stays in the header. Within a subtree, ``/`` refers to the +requested group. Group attributes belong to the selected group. Opening, +expansion, metadata reads, and array pages run in background workers; navigation +and quit remain available during a slow request. Refresh opens a new discovery +session, discards cached pages, and restores the selected path when it still +exists. Failed listings can be retried by selecting or expanding the group again. + +Browsing is read-only. Remote roots and groups use ``RemoteStore`` with one +64 MiB MEMORY allowance shared across arrays. Selected leaves are ``RemoteArray`` +handles. Switching arrays or selecting a group releases the selected handle but +keeps its warm payload in the store until eviction or browser close. Revisited +chunks within the allowance need no further payload download. Discovery reads +metadata, not every array's data. Metadata cost can grow with the number of +objects and chunks. Small objects may fit entirely within a bounded opening read. + +``--profile`` and ``--endpoint-url`` are optional; when omitted, the S3 backend +uses its normal credential and endpoint configuration. Install ``blosc2[tui]`` +and ``blosc2[fsspec]`` plus ``s3fs`` for S3 access. Zarr requires +``blosc2[zarr]``; HDF5 requires ``blosc2[hdf5]`` (Kerchunk, h5py, and Zarr). +B2Z browsing does not require Zarr or HDF5 dependencies. + +Format details and limits: + +* **B2Z:** discovery shares the ZIP directory with the native array reader. + External arrays must be unencrypted, ZIP_STORED plain NDArrays. The embedded + index identifies embedded leaves and CTable boundaries; their payload previews + remain unavailable. Group attributes are read from external frame trailers or + the bounded native chunks containing embedded attribute frames. Embedded + attribute layouts with chunks larger than 1 MiB show a partial-metadata notice + instead of fetching large payloads. TreeStore has no separate empty-group + marker: an empty group is visible when its attribute frame records it. +* **Zarr:** v2 and v3 groups use consolidated metadata when available and normal + discovery otherwise. Unconsolidated groups need backend directory-listing + support and LIST permission. Direct arrays do not require listing their parent. + Empty groups and attributes are preserved. Unknown codecs and unsupported + dtypes remain visible; preview support follows the existing Zarr array reader. +* **HDF5:** Kerchunk translates metadata once per session and all selected leaves + reuse those references. Translation can enumerate many chunk references and + inline small values; it avoids full-file localization, but is not a constant-cost + operation. Empty groups and attributes are preserved. Failed dataset translations + become unavailable nodes without hiding supported siblings. The view covers + Kerchunk's representation: hard-link aliases may be omitted, and soft/external + links and group cycles are not followed. + +These internal browser adapters do not change the array-only contract of +``blosc2.open(..., lazy=True)`` or add a persisted RemoteArray hierarchy descriptor. +Opening an entire remote B2Z through ``blosc2.open`` requires an explicit +``cache_dir`` for localization; use ``b2view`` for range-based hierarchy browsing. +Standalone remote ``.b2nd`` arrays retain their existing lazy viewer behavior. + Step 3 — Navigate the data panel -------------------------------- @@ -124,4 +197,4 @@ CLI options ``--preview-rows N`` and ``--preview-cols N`` bound the size of each data page (20 rows by 10 columns by default), and ``--panel`` chooses the panel -focused on startup (``tree``, ``meta``, ``vlmeta`` or ``data``). +focused on startup (``tree``, ``meta``, ``attrs`` or ``data``). diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index 2d4a03a3f..87c8f63d5 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -1,135 +1,559 @@ -# Working with Remote Arrays +# Working with Remote Arrays and Stores -A Blosc2 array that lives on a server does not have to be downloaded to be used. Blosc2 opens it where it is, fetches only the pieces a slice touches, and keeps those in a local cache so the next run starts from them. +Blosc2 can open remote arrays and stores (arrays on a hierarchical container) without downloading them first. +Source metadata is read at open time; array data is fetched when a slice needs it and retained according to the cache policy. -## Three ways in +Python-Blosc2 provides two primary entry points for remote data: +- {ref}`RemoteArray`: Access and slice an individual remote array (a standalone `.b2nd` file or a specific dataset in a container). +- {ref}`RemoteStore`: Discover, navigate, and access multi-dataset hierarchies in B2Z, Zarr, or HDF5 containers, sharing a single cache budget across all leaves. -| Where the array lives | How to open it | -|---|---| -| Any URL fsspec reaches — `s3://`, `gs://`, `https://`, `zip://`… | `blosc2.open(url, lazy=True)` | -| A [Caterva2](https://ironarray.io/caterva2) server | `blosc2.C2Array(path, urlbase=...)` | -| Anything else | A `read_range()` of your own — see [Your own transport](#your-own-transport) | +```python +import blosc2 + +# Discover and read from a remote container (B2Z, Zarr, or HDF5) +with blosc2.RemoteStore("https://datasets.example.org/data.h5") as store: + print(store.keys()) # discover groups and datasets + group = store["experiment"] + array = group["temperature"] # yields a RemoteArray leaf + values = array[:100] # fetches only the requested slice + attrs = array.attrs[:] # fetches user metadata + +# Or open a single remote array directly +a = blosc2.open("s3://bucket/big.b2nd", lazy=True) +values = a[:100] +attrs = a.attrs[:] # fetches user metadata +``` + +The `b2view` terminal browser uses these public types with a 64 MiB allowance by default to let you explore remote containers interactively. +For a script showing hierarchy discovery, leaf previews, and persistent caching, see `examples/remote/store-browse.py`. + +## Choose a remote route + +The argument passed to {func}`blosc2.open` selects the route: + +| Argument | Route | What it names | +| ----------------------------------------------------------------------------- | -------- | ------------------------------------------------------- | +| A URL string such as `s3://...` or `https://...` | fsspec | A byte-addressable, standalone `.b2nd` file | +| A URL containing a `.b2z` path component, or `source_format="b2z"` | B2Z | One immutable external NDArray leaf in a `.b2z` archive | +| A URL containing a `.zarr` path component | Zarr | One immutable Zarr v2 or v3 array | +| A URL containing a `.h5` or `.hdf5` path component, or `source_format="hdf5"` | HDF5 | One immutable HDF5 dataset via kerchunk | +| A {ref}`URLPath` | Caterva2 | One array-like dataset on a Caterva2 server | +| An exported `.b2z` reference archive | B2Z | A restored {ref}`RemoteStore` reference hierarchy | + +```python +import blosc2 + +# fsspec: a plain web server, CDN, or cloud object store +a1 = blosc2.open("https://datasets.example.org/big.b2nd", lazy=True) +a2 = blosc2.open("s3://bucket/big.b2nd", lazy=True) + +# Caterva2: a dataset identified by root and path +b = blosc2.open( + blosc2.URLPath( + "@public/examples/lung-jpeg2000_10x.b2nd", + urlbase="https://cat2.cloud/demo", + ), + lazy=True, +) + +# Open individual datasets inside containers (B2Z, Zarr, or HDF5) +# Datasets can be named using slashes (/), container separators (::), or dataset=: +b1 = blosc2.open("s3://bucket/hierarchy.b2z/d0/d1/a2", lazy=True) +c1 = blosc2.open("https://datasets.example.org/hierarchy.zarr::d0/d1/a2", lazy=True) +h1 = blosc2.open( + "https://datasets.example.org/hierarchy.h5", lazy=True, dataset="d0/d1/a2" +) + +# Open whole hierarchies with RemoteStore to discover and navigate containers: +store_b2z = blosc2.RemoteStore("s3://bucket/hierarchy.b2z") +store_h5 = blosc2.RemoteStore("https://datasets.example.org/hierarchy.h5") + +# Reopen an exported reference snapshot (.b2z): +store_snap = blosc2.open("snapshot.b2z") + +a1.shape, a1.dtype # metadata is available immediately +a1[100:110, :50] # data is fetched now +``` + +Remote B2Z needs `pip install "blosc2[fsspec]"`. +HTTP and HTTPS URLs work out of the box; cloud object stores need their respective protocol driver (such as `s3fs` for S3, `gcsfs` for GCS, or `adlfs` for Azure). +It accesses external `ZIP_STORED` NDArray members in `.b2z` archives using native Blosc2 chunk and block range reads without decompressing or downloading the archive. +For a suffix-free URL, pass `source_format="b2z"`. +Embedded leaves and CTable columns are not supported as lazy NDArrays. + +Remote Zarr needs `pip install "blosc2[zarr,fsspec]"`. +HTTP/HTTPS works directly; cloud stores require their protocol driver (`s3fs` for S3, etc.). +Datasets can be named directly by path (`/sub/arr`), with the `::sub/arr` separator, or via `dataset="sub/arr"`. +For a suffix-free URL, pass `source_format="zarr"`. +Converted Blosc2 chunks are cached under an immutable source contract, so publish changed data at a new URL or replace its cache. + +Remote HDF5 needs `pip install "blosc2[hdf5,fsspec]"`. +HTTP/HTTPS works directly; cloud stores require their protocol driver (`s3fs` for S3, etc.). +Datasets can be specified using standard slash syntax (`file.h5/d0/d1/a2`), the double-colon separator (`file.h5::d0/d1/a2`), or the `dataset="d0/d1/a2"` parameter. +Pre-indexing is performed via `kerchunk`. When opening a single {ref}`RemoteArray`, the resulting reference map is cached inside the array carrier (`schunk.vlmeta["hdf5-refs"]`). When using {ref}`RemoteStore`, indexing is performed once for the entire container and shared across all leaves and sessions. +Use `blosc2.available_datasets(url)` to inspect datasets in an HDF5 container. + +`RemoteArray` assumes remote sources are immutable by default, avoiding a metadata request before every read. +For a replaceable `.b2nd` or Caterva2 source, pass `assume_immutable=False` to refresh its identity and invalidate stale cached chunks before each operation. +Mutable B2Z, Zarr, and HDF5 sources are not supported. + +A `URLPath` always means Caterva2. +If its `urlbase` is omitted, the server comes from {func}`blosc2.c2context` or `BLOSC_C2URLBASE`. +Other transports can be added with a custom {ref}`ByteRangeNDSource`; see [Use your own transport](#use-your-own-transport). + +### Choosing between fsspec and Caterva2 + +When opening an individual dataset with `lazy=True`, both fsspec URLs and Caterva2 `URLPath`s return a {ref}`RemoteArray`, providing an identical user interface for slicing, caching, and introspection. +For multi-dataset containers (B2Z, Zarr, and HDF5), a {ref}`RemoteStore` is returned instead, providing container-level discovery and shared caching across fsspec protocols. + +What differs between the transports is the types of remote objects each can open: + +| Remote object | fsspec URL / RemoteStore | Caterva2 `URLPath` | +| --------------------------------------- | -------------------------------- | ---------------------------- | +| Standalone contiguous `.b2nd` | Yes (`blosc2.open` / `RemoteArray`) | Yes | +| NDArray leaf inside `.b2z` | Yes (`blosc2.open` / `RemoteArray`) | Yes | +| Zarr v2/v3 array | Yes (`blosc2.open` / `RemoteArray`) | No | +| HDF5 dataset | Yes (`blosc2.open` / `RemoteArray`) | Yes | +| Whole container (.b2z, .zarr, .h5) | Yes (`blosc2.open` / `RemoteArray`) | Yes; zarr not yet | +| Lazy or computed array | No | Yes | + +- **fsspec** supplies byte ranges. + Python-Blosc2 parses the remote frame to discover its geometry and chunk offsets, making this route direct and efficient for standalone arrays. +- **Caterva2** understands dataset paths, array metadata, and slicing. + It can therefore expose array-like data that is not stored as a standalone Blosc2 frame, as well as apply authentication or server-side computation. + Use Caterva2's navigation API to find a leaf in a remote hierarchy, then open that leaf with a `URLPath`. + +`lazy=True` changes *when* data is fetched; it does not expand the underlying storage formats supported by either route. + +> [!TIP] +> **Browse remote hierarchies**: To explore groups, inspect attributes, or preview arrays in remote `.b2z`, `.zarr`, or `.h5` containers interactively in the terminal without downloading the complete container, use {doc}`b2view ` (e.g. `b2view s3://bucket/hierarchy.b2z`). To navigate containers programmatically in Python, use {ref}`RemoteStore`. + +## Explore remote hierarchies with RemoteStore + +When working with containers that hold multiple groups and datasets—such as `.b2z`, `.zarr`, or `.h5` files—use {ref}`RemoteStore` to discover, navigate, and access the hierarchy: ```python import blosc2 -# An object store, a web server, a zip on either of them +with blosc2.RemoteStore("https://datasets.example.org/data.h5") as store: + # 1. Discover immediate child groups and datasets (metadata only, no array data downloaded) + print(store.keys()) + + # 2. Inspect a node's kind and attributes + info = store.get_info("experiment") + print(info.kind) # "group", "ndarray", or "unsupported" + + # 3. Read user metadata on groups or arrays + print(store["experiment"].attrs[:]) + + # 4. Access an array leaf and slice it + temp = store["experiment/temperature"] + values = temp[:100] # fetches and caches only the requested slice +``` + +### Hierarchy navigation and inspection + +- **Child enumeration**: `store.keys()` and `for name in store:` list immediate children of the current store or group level without fetching array data. +- **Relative paths**: Lookups can use slash paths or chained indexing interchangeably (`store["experiment/temperature"]` is equivalent to `store["experiment"]["temperature"]`). Both return a {ref}`RemoteArray` leaf. +- **Node inspection with {ref}`RemoteNode`**: Call `store.get_info(name)` to inspect a node without creating leaf readers or allocating cache memory. A `RemoteNode` provides: + - `path`: relative dataset path. + - `kind`: `"group"`, `"ndarray"`, or `"unsupported"`. + - `attrs`: user metadata mapping (or `None` if array attributes require opening the leaf). + - `diagnostic`: explanation for unsupported nodes (e.g. non-array objects or unsupported codecs). +- **Graceful degradation**: Unsupported nodes remain visible during discovery and raise an informative `NotImplementedError` only when selected as arrays, allowing you to browse mixed containers without errors. + +### Shared caching across the hierarchy + +Unlike opening independent arrays with `blosc2.open(..., lazy=True)`, all leaves accessed through a `RemoteStore` share a single cache coordinator: + +- **Single shared budget**: The store defaults to {attr}`CachePolicy.MEMORY ` with a shared 256 MiB allowance across all arrays. You can customize this with `max_cache_bytes`. +- **Cross-leaf LRU eviction**: When total retained chunks reach the budget, the least-recently used chunks across *any* leaf in the store are evicted automatically. +- **Warm retention on close**: Closing an individual leaf handle (`array.close()`) does not discard its cached chunks from the store session. Re-accessing that dataset reuses the warm cache without re-downloading. +- **Accounting**: + - `store.cache_bytes`: total retained compressed payload across all leaves in the store. + - `array.cache_bytes`: payload retained specifically for that leaf. + - `store.traffic`: cumulative network requests and response bytes for the entire store, including both metadata discovery and chunk fetches. + +### Persistent disk caching with `cache_dir` + +Specify `cache_dir` when creating a `RemoteStore` to persist discovery metadata and downloaded chunks to local disk: + +```python +with blosc2.RemoteStore( + "https://datasets.example.org/data.h5", + cache_dir="./b2store_cache", + max_cache_bytes=512 * 2**20, # 512 MiB shared disk limit +) as store: + temp = store["experiment/temperature"] + values = temp[:100] +``` + +When reopening the same store later with the same `cache_dir`: +- Discovery metadata (such as B2Z member offsets or HDF5 Kerchunk reference maps) is restored from local disk, avoiding repeated remote translation scans. `store.metadata_bytes` reports the encoded manifest size. +- Retained leaf chunks are available immediately from disk without network transfers. +- Single-owner locks ensure that concurrent processes do not corrupt the shared cache. + +### Lifetime and clean shutdown + +- Use context managers (`with blosc2.RemoteStore(...) as store:`) for clean lifecycle management. +- Child handles (`RemoteArray` leaves or group views) remain usable even after the parent `store` handle closes. +- Transport sessions, HTTP connections, and disk cache locks are released automatically once the last dependent handle is closed or garbage collected. + +## Access HTTP/HTTPS, S3, and cloud storage + +Because Python-Blosc2 uses [fsspec](https://filesystem-spec.readthedocs.io/) under the hood, any remote protocol supported by fsspec can be used to open arrays lazily. + +### HTTP and HTTPS + +Publicly accessible arrays on any web server, CDN, or object store URL can be opened directly over HTTP or HTTPS without requiring cloud-specific libraries or credentials: + +```python +import blosc2 + +# Standalone array over HTTPS: +a = blosc2.open("https://datasets.example.org/big.b2nd", lazy=True) + +# Container dataset over HTTPS: +b = blosc2.open( + "https://f001.backblazeb2.com/file/blosc2/hierarchy.b2z::/d0/a3", + lazy=True, +) +``` + +### S3 and cloud object stores + +For arrays stored on Amazon S3 or S3-compatible cloud object stores (Backblaze B2, MinIO, Cloudflare R2, Ceph, Wasabi, etc.), install `s3fs` and open the `s3://` URL: + +```python +# Using default credentials from environment or ~/.aws/credentials a = blosc2.open("s3://bucket/big.b2nd", lazy=True) +``` + +Other cloud stores work similarly by installing their respective fsspec driver (e.g. `gcsfs` for Google Cloud `gs://` or `adlfs` for Azure `abfs://`). + +### Storage options and authentication + +Pass a `storage_options` dictionary to configure headers, credentials, or custom endpoints. +Options are forwarded directly to the underlying `fsspec` filesystem: -# A Caterva2 server -b = blosc2.C2Array( - "@public/examples/lung-jpeg2000_10x.b2nd", urlbase="https://cat2.cloud/demo" +```python +# For HTTP/HTTPS: custom headers or authentication tokens +a = blosc2.open( + "https://datasets.example.org/private.b2nd", + lazy=True, + storage_options={"headers": {"Authorization": "Bearer "}}, ) -a.shape, a.dtype # metadata only; nothing was downloaded -a[100:110, :50] # a NumPy array, fetched now +# For S3: AWS profiles, credentials, or custom endpoints +storage_options = { + "profile": "blosc2", # named profile from ~/.aws/credentials + "endpoint_url": "https://s3.us-west-001.backblazeb2.com", # custom endpoint + # Or explicit keys: + # "key": "AWS_ACCESS_KEY_ID", + # "secret": "AWS_SECRET_ACCESS_KEY", + # Or anonymous public access: + # "anon": True, +} +a = blosc2.open("s3://bucket/big.b2nd", lazy=True, storage_options=storage_options) ``` -`https://` means a plain web server — nginx, a CDN, an S3 website endpoint — anything that answers a `Range` request. A Caterva2 server is *not* reached that way: it names its datasets by root and path, so use {ref}`C2Array`. +### Remote performance: latency, caching, and concurrency + +Remote requests over HTTP or object stores typically incur 20–100 ms of latency per range request. +Python-Blosc2 addresses this in two ways: + +1. **Caching**: Chunks and blocks fetched for a slice are kept in the local cache (in RAM by default, or persisted to disk with `cache_dir=` or `cache_path=`). + Re-fetching previously read regions requires zero network round trips and zero bytes transferred. +2. **Concurrent fetches**: Independent range requests for required chunks and blocks are issued concurrently in a thread pool (configured via `max_concurrency=`, default 8). + +The runnable script `examples/remote/s3-access.py` demonstrates opening `.b2nd`, `.b2z`, `.zarr`, and `.h5` datasets over remote URLs (both S3 and HTTPS), timing metadata discovery vs. slice fetching, measuring network traffic with {ref}`Traffic`, and showing the impact of chunk caching. -## The cache +## Cache policies and memory management -Wrap either of those in a {ref}`Proxy` and what you read is kept: +Every lazy open uses a cache policy. +By default, fetched data is cached in memory with a bound on retained compressed payload. + +### In-memory caching (`CachePolicy.MEMORY` — Default) + +When opened without disk options, `blosc2.open(..., lazy=True)` retains fetched chunks in RAM as a {ref}`RemoteArray` with {attr}`CachePolicy.MEMORY `: ```python -p = blosc2.Proxy(b) # cache in memory, gone when the proxy is -p[10:12, 500:600] # fetched from the server, and kept -p[10:12, 500:600] # read from the cache, no request at all +a = blosc2.open("s3://bucket/big.b2nd", lazy=True) +a[10:12, 500:600] # fetched and cached in RAM +a[10:12, 500:600] # served from memory cache (no network traffic) ``` -Where that cache lives is yours to choose, and it is the one decision to make here. Say nothing and it is memory: fast, and it dies with the proxy, which is all a single process reading a slice twice needs. Name a file with `urlpath=` and the cache outlives the run: +In-memory caches use `max_cache_bytes` (defaults to 256 MiB) with automatic LRU eviction after operations, including failed fetches. +This is not a peak RAM limit: metadata, in-flight transfers, decompression buffers, and results are excluded. +Large operations can exceed it substantially. ```python -p = blosc2.Proxy(b, urlpath="lung-cache.b2nd", mode="a") -p[10:12, 500:600] # fetched from the server, and written to lung-cache.b2nd +# Custom in-memory limit (e.g. 512 MiB): +a = blosc2.open("s3://bucket/big.b2nd", lazy=True, max_cache_bytes=512 * 2**20) ``` -That file is an ordinary Blosc2 array holding only the pieces you touched — a few hundred bytes for a freshly opened proxy over a 64 MB dataset, growing as you read. It is a normal `.b2nd`: copy it, ship it, open it with {func}`blosc2.open`. With `mode="a"` a later run picks up where the last one left off. +### Persistent disk caching (`CachePolicy.DISK`) -{func}`blosc2.open` builds the proxy for you and offers the same choice under another name — `cache_storage=` a directory for a cache on disk, nothing for one in memory: +Set `cache_dir` or `cache_path` to persist fetched data across sessions ({attr}`CachePolicy.DISK `): ```python url = "s3://bucket/big.b2nd" -# First run: the slice is fetched, and lands under ./b2cache dir -a = blosc2.open(url, lazy=True, cache_storage="./b2cache") -a[100:110, :50] +# Blosc2 manages a cache file inside a directory: +a = blosc2.open(url, lazy=True, cache_dir="./b2cache") +a[100:110, :50] # fetched and stored under ./b2cache -# A later run, a different process: same call, served from ./b2cache -a = blosc2.open(url, lazy=True, cache_storage="./b2cache") -a[100:110, :50] # no request +# A later process can reuse the same cache: +a = blosc2.open(url, lazy=True, cache_dir="./b2cache") +a[100:110, :50] # served from local disk (no network traffic) ``` +- For an individual {ref}`RemoteArray`, pass `cache_dir` (Blosc2 creates the cache carrier inside that directory) or `cache_path` (to specify an exact carrier filename). +- For a {ref}`RemoteStore`, pass `cache_dir` to store discovered hierarchy metadata and all leaf caches together under that directory. +- In both cases, compressed chunks are retained up to `max_cache_bytes` (defaults to 256 MiB; pass `max_cache_bytes=None` for an unbounded disk cache that never evicts). + +Authenticated Caterva2 caches must be private to one user. +Reopen them under an equivalent authenticated {func}`blosc2.c2context`; do not share a cache directory between users. + +### Stateless streaming (`CachePolicy.NONE`) + +To stream data without retaining any chunks after each operation, specify {attr}`CachePolicy.NONE `: + +```python +stream = blosc2.open( + "s3://bucket/big.b2nd", + lazy=True, + cache_policy=blosc2.CachePolicy.NONE, +) +``` + +Each read pulls only the bytes required for the slice and retains no cache payload. + +> [!NOTE] +> `max_cache_bytes` is applied after each operation completes. +> It bounds the retained compressed cache payload; it does not limit the temporary working set or the decompressed NumPy array requested by the caller. + ## Only what a slice touches -A chunk is the unit a container is compressed in, and it can be several megabytes. Fetching a whole one to read a corner of it is most of the cost of a remote read, so Blosc2 fetches **blocks** — the smaller pieces a chunk is built from — whenever a slice lands in a small part of a large chunk. +Blosc2 arrays are compressed in chunks, which are divided into smaller blocks. +For a small slice, fetching only its blocks can avoid transferring most of a large chunk. + +![A remote array fetches missing regions from the remote array into its local cache. +Indexing returns the requested values.](../tutorials/images/remote_proxy.png) + +Purple regions are cached; red regions are still remote. +The grid is schematic: where byte ranges are available, the fetched regions can be blocks within a chunk. +`fetch()` warms the cache and returns the remote array, whereas indexing returns the requested values. -You do not ask for this; it happens when it pays. For example: +The remote array chooses blocks or whole chunks automatically. +It fetches a whole chunk when most of its blocks are needed or when the source cannot expose block ranges, as with computed Caterva2 datasets. +Independent reads overlap, with up to eight concurrent requests by default; use `max_concurrency=1` when concurrency does not help. -- On S3, block reads are **5–17x faster** on arrays with multi-megabyte chunks, and **2–5x** on 1 MB ones. -- On cat2.cloud's `kevlar-tomo.b2nd`, a corner slice costs **0.031 MB instead of 2.723 MB**, and a slice touching ten chunks takes **0.14 s against 1.01 s**. +Stepped slices also use the block grid. +For example, `a[::5]` can reduce transfers along an axis whose blocks do not already span that axis. -It is never a loss. A slice wanting more than half a chunk's blocks is wanting the chunk, and a fetch that would skip too little to pay for the extra round trip is made whole — both answered from metadata already in hand, before anything is read. Where blocks are not available the read falls back to whole chunks by itself: that happens for a dataset a Caterva2 server *computes* rather than stores (a lazy expression, an HDF5 leaf, a `.b2z` member), and for a server that stops honouring ranges. +### Explicit cache pre-fetching -Fetches also overlap: a lazy proxy runs 8 at a time by default. Pass `max_concurrency=1` for a local protocol with no latency to hide. +You can warm the cache proactively using `fetch()` or `afetch()`: + +```python +# Synchronously pre-fetch a region into the cache: +a.fetch(slice(0, 10_000)) + +# Or asynchronously in an async event loop: +await a.afetch(slice(10_000, 20_000)) +``` -A step other than 1 needs a proxy — a bare {ref}`C2Array` refuses one. Through a proxy it is placed on the block grid like any other key: `p[::2]` reads the blocks holding the coordinates it selects and no others, and `[::-1]` costs what its forward twin does. What that saves is `min(step, block extent along that axis)`, so it is nothing where blocks already span the axis whole — a step along the last dimension, usually — and the step's own factor where they do not. On `kevlar-tomo.b2nd`, whose blocks are one row deep, `[::2]` halves the read and `[::5]` cuts it fivefold. +Both methods return `a`. +Prefetched data may be evicted to satisfy the cache limit; later indexing fetches it again as needed. +Use `a.materialize(item)` for an independent, complete `NDArray`. +Its output and temporary buffer are outside the cache limit. -### Seeing byte savings +> [!NOTE] +> `fetch()` and `afetch()` require a writable cache. On an immutable cache snapshot (such as an archive opened with `mutable=False`), pre-fetching raises an error. -Wall time will not show you any of this: on a fast link a block read and a whole-chunk read take about as long and differ by the compression ratio in *bytes*. Bytes are also what a metered link and a shared server uplink run out of, so they are counted for you. {ref}`C2Array` and {ref}`Proxy` each carry a {ref}`Traffic` under `traffic` — cumulative requests and bytes, tallied at the transport, so the frame index and block offsets are in it too: +`a.cache` exposes the underlying cache for inspection. +It may contain missing or evicted chunks and must not be treated as a complete array or mutated by callers. + +Operations on a single handle are serialized through fetching, result assembly, eviction, and export. +Async methods run synchronous operations in a worker thread; cancelling the await does not stop an already running fetch. +Separate handles or processes sharing a disk carrier require external locking. + +## Measure network traffic + +{ref}`RemoteArray`, {ref}`RemoteStore`, {ref}`C2Array`, and {ref}`Proxy` objects expose cumulative request and byte counts through {ref}`Traffic`. +The count starts when the remote source is opened, so it includes metadata as well as array data: + +```python +a = blosc2.open("s3://bucket/big.b2nd", lazy=True) + +a.traffic.reset() +corner = a[0, :100, :100] +print(a.traffic) # requests and bytes fetched + +a.traffic.reset() +corner = a[0, :100, :100] +print(a.traffic) # Traffic(requests=0, nbytes=0) -> cache hit! +``` + +Use `reset()` or subtract two readings to measure one operation. +`traffic` is `None` for a local source because no network transport exists. +For a `RemoteStore`, `store.traffic` reports cumulative traffic across discovery and all leaf accesses in the session. + +`examples/remote/c2array-traffic.py` compares block, chunk, and cached reads against a live Caterva2 dataset. + +## Persist and reopen remote references + +Python-Blosc2 allows you to save remote references and their cached data to disk as portable files, and reopen them later without needing the original remote URL. + +### Persist a remote array reference (.b2nd) + +Use {ref}`RemoteArray` directly when a `.b2nd` file should carry a portable remote descriptor and, optionally, its own bounded persistent cache: ```python -b = blosc2.C2Array( - "@public/examples/kevlar-tomo.b2nd", urlbase="https://cat2.cloud/demo" +remote = blosc2.RemoteArray( + "s3://bucket/big.b2nd", + cache_policy=blosc2.CachePolicy.NONE, ) -p = blosc2.Proxy(b) +remote.save("big-reference.b2nd") +``` + +The saved object contains source and geometry metadata but no credentials. +With `CachePolicy.NONE`, repeated reads contact the source and do not mutate the carrier. +With `CachePolicy.DISK`, the carrier file itself is the cache and retains compressed chunks up to its payload limit. +Disk arrays preserve warm chunks by default; memory arrays export cold carriers. +Pass `include_cache=False` to export a cold copy without mutating the warm carrier. + +### Export a remote store snapshot (.b2z) -p.traffic.reset() -corner = p[0, :100, :100] -print(p.traffic) # Traffic(requests=4, nbytes=57767) +To export an entire remote hierarchy—including discovered groups, array geometry, source locators, and optional cached chunks—call `save()` on a {ref}`RemoteStore`: -p.traffic.reset() -p[0, :100, :100] # the same slice, from the cache -print(p.traffic) # Traffic(requests=0, nbytes=0) +```python +with blosc2.RemoteStore("https://datasets.example.org/data.h5") as store: + temp = store["experiment/temperature"] + temp[:100] # warms the cache for this slice + + # Save a portable .b2z reference archive containing discovery and warm chunks + store.save("snapshot.b2z") + + # Or export a cold reference containing only metadata and locators (no chunks) + store.save("cold_ref.b2z", include_cache=False) + + # Or export only a specific subtree + store["experiment"].save("experiment_sub.b2z") ``` -Take two readings and subtract, or `reset()` between them. `Proxy.traffic` is `None` over a local array — nothing crosses a wire there, and a zero would say the traffic was free rather than that it was never measured. `examples/c2array-traffic.py` runs the whole comparison against cat2.cloud's `kevlar-tomo.b2nd`: a 100x100 corner costs 0.055 MB against 1.296 MB for the chunk holding it — 23.5x — and nothing at all on the second read. +- **Portable reference**: The `.b2z` archive contains the discovered hierarchy, attributes, and source locators (such as the HDF5 Kerchunk reference map or B2Z member offsets), but no secrets or credentials. +- **`include_cache=True` (default)**: Bundles warm cached chunks along with metadata so reading previously fetched slices requires zero network traffic. +- **`include_cache=False`**: Omits cached payload chunks, producing a minimal reference archive for remote streaming. +- **Subtree export**: Calling `save()` on a group view exports that subtree with relative child keys and the appropriate source root. -## Scattered points +### Reopen reference files with `blosc2.open()` -A list of coordinates, or a boolean mask, is not a box — but every point it picks still lives in exactly one block, so it is placed on the block grid as exactly as a slice is: +Both `.b2nd` array carriers and `.b2z` store snapshots can be reopened directly with `blosc2.open()`: ```python -p[rows, :100] # rows is an array of three indices: three blocks, not three chunks -p[mask] # a mask picks coordinates too, and costs the same +# 1. Reopen an exported RemoteStore hierarchy: +with blosc2.open("snapshot.b2z") as restored: + print(restored.keys()) + temp = restored["experiment/temperature"] + values = temp[:100] # served from archive if cached; fetched remotely if missing + +# 2. Reopen a standalone RemoteArray carrier: +arr = blosc2.open("big-cache.b2nd", mode="a") +values = arr[:100] ``` -Nine scattered points of a 900³ array cost **236 KB in 19 requests** through a proxy, against 1.81 MB for the chunks holding them. +Opening an exported `.b2z` archive automatically recognizes the remote store marker and constructs a {ref}`RemoteStore`. All leaves opened from it share one cache coordinator and budget. +Opening an on-disk carrier with `mode="a"` returns a {ref}`RemoteArray` and lets newly fetched regions extend the cache; opening with `mode="r"` keeps the cache file unchanged. +Legacy proxy caches created by older Blosc2 versions are also detected and reopened as a {ref}`Proxy`. + +Independent reopening works for fsspec URLs, Caterva2 datasets, and persistent local Blosc2 sources. +The required runtime environment must still be available: fsspec backends and their configuration must be installed, local source paths must remain valid, and authenticated Caterva2 caches must be reopened inside an equivalent {func}`blosc2.c2context`. +Caterva2 credentials are not stored in the cache file. -However, a {ref}`C2Array` does better with no proxy at all: the coordinates go to the server, which gathers the points and sends back those alone — **271 bytes in one request** for the same nine. When you need efficient scattered retrievals, C2Array+Caterva2 is your best friend. +An arbitrary custom {ref}`ProxyNDSource` cannot be reconstructed because its Python class and runtime state are not serialized. +In that case, recreate the source explicitly and attach the existing cache with `blosc2.Proxy(source, urlpath="big-cache.b2nd", mode="a")`. -## When the remote changes underneath +### Cache mutability: immutable vs. mutable snapshots -A cache is only good while the bytes it was filled from are still there. Sources that can name their bytes — an fsspec URL by its token, a Caterva2 array by an identifier the server keeps — are checked against what the cache recorded: +When saving an export, you can configure whether the resulting snapshot operates in **immutable** or **mutable** mode via the `mutable` argument (or the `.mutable` property on `RemoteStore` / `RemoteArray`): ```python -p = blosc2.Proxy(src, urlpath="cache.b2nd", mode="a") -# ValueError: the cache at cache.b2nd was built against different remote bytes; -# pass mode='w' to fetch them anew +# Default is mutable=False (immutable snapshot) +store.save("read_only.b2z", mutable=False) + +# Or export a mutable snapshot +store.save("writable.b2z", mutable=True) ``` -`mode="w"` starts the cache empty and refetches. For a source that cannot name its bytes, the cache is adopted on geometry alone — same shape, dtype and partitioning — so an array rewritten in place while its geometry stayed the same is served from the cache as it was. Use `mode="w"` when that is a possibility. +| Mode | Behavior when reopened | Cache misses | Modifying operations | +| --- | --- | --- | --- | +| **Immutable** (`mutable=False`, default) | Reads directly in-place from `.b2z` without disk writes. Safe on read-only media (`chmod 0o444`). | Fetched transiently into RAM to satisfy the read; never written to disk or the archive. | `fetch()`, `afetch()`, `trim_cache()`, and `refresh()` are disallowed. | +| **Mutable** (`mutable=True`) | Staged into an independent writable runtime cache directory. Original `.b2z` stays untouched. | Fetched and cached to disk under standard LRU eviction rules. | Fully supported. Can be opened with a smaller budget, trimming excess chunks. | + +> [!TIP] +> Use **immutable snapshots** (`mutable=False`) for sharing reproducible, read-only reference archives or distributing datasets that should never modify local storage. Use **mutable snapshots** (`mutable=True`) when users should be able to expand the local cache with newly fetched regions over time. -## Filling an array from several writers +## Retrieve scattered points -A Caterva2 array can be *written*, one chunk at a time, by as many processes as it has chunks. Lay the array out empty first — {func}`blosc2.uninit` writes a couple of hundred bytes whatever the shape — upload it to the server, then have each writer post the chunks it owns: +A remote array maps coordinate arrays and boolean masks to the blocks that contain their selected points: + +```python +a[rows, :100] +a[mask] +``` + +For Caterva2, a bare {ref}`C2Array` can be substantially more efficient for one-off point queries: it sends coordinates to the server, which evaluates the selection and returns only the selected values. +Prefer direct `C2Array` indexing for sparse, one-off point retrieval; prefer a {ref}`RemoteArray` when reuse through a local cache matters. + +## Handle remote changes + +### Standalone arrays and Caterva2 sources + +A persistent cache records the source identity when one is available. +On a later `blosc2.open()` with the same `cache_dir` or `cache_path`, a mismatched cache is discarded and rebuilt automatically. + +When constructing a proxy directly in append mode, a mismatch is reported instead: + +```python +p = blosc2.Proxy(source, urlpath="cache.b2nd", mode="a") +# ValueError if cache.b2nd belongs to different remote bytes +``` + +Use `mode="w"` to start that cache again. +If a source cannot provide an identity, compatibility is checked only from shape, dtype, chunks, and blocks. +Use a fresh cache when such a source may have changed without changing its geometry. + +For a replaceable `.b2nd` or Caterva2 source, pass `assume_immutable=False` to check for updates and invalidate stale cached chunks before each operation. + +### Refreshing a RemoteStore + +Remote containers (B2Z, Zarr, and HDF5) are assumed immutable by default. +If a remote container is updated on the server—such as adding new datasets or appending data—call `store.refresh()` to update discovery: + +```python +with blosc2.RemoteStore("https://datasets.example.org/data.h5") as store: + # Refresh remote metadata atomically + store.refresh() + + # Re-access datasets from the refreshed store + array = store["experiment/temperature"] +``` + +- **Atomic update**: `store.refresh()` fetches fresh discovery from the remote source before updating the active generation. If discovery fails, the existing store state remains unchanged. +- **Stale handle safety**: Any child handles (`RemoteArray` leaves or group views) opened *before* `refresh()` become stale. Accessing them raises a `RuntimeError`, prompting you to look them up again from the refreshed store. +- **Immutability rule**: Calling `refresh()` on an immutable reference snapshot (`mutable=False`) is disallowed and raises an error. + +## Fill a Caterva2 array concurrently + +Several writers can fill one Caterva2 array when each chunk is written at most once. +First create and upload an uninitialized array with its final geometry: ```python import blosc2 import numpy as np -# Once, before the writers start: an empty array of the final geometry blosc2.uninit( (1_000_000,), dtype=np.float64, @@ -139,50 +563,39 @@ blosc2.uninit( ) ``` -Upload it with the client that comes with Caterva2: - ```sh cat2-client upload run.b2nd @personal/run.b2nd ``` -Then each writer opens it and posts its own chunks: +Each writer compresses and posts the chunks it owns: ```python import math -import blosc2 - a = blosc2.C2Array("@personal/run.b2nd", urlbase="https://cat2.cloud/demo") -itemsize = a.dtype.itemsize chunk = blosc2.compress2( - data, typesize=itemsize, blocksize=math.prod(a.blocks) * itemsize + data, + typesize=a.dtype.itemsize, + blocksize=math.prod(a.blocks) * a.dtype.itemsize, ) -a.update_chunk(nchunk, chunk) -``` -Each slot is written once. A second write to the same slot raises {class}`blosc2.ChunkAlreadyWritten`, and that refusal is the whole of the coordination — two writers that both think they own a chunk are sorted out by the array, with no lease, lock or registry between them. The loser drops its chunk and moves on: - -```python try: a.update_chunk(nchunk, chunk) except blosc2.ChunkAlreadyWritten: - pass # someone else got there first + pass # another writer completed this slot ``` -Writing into an empty slot appends to the file and moves no other chunk, which is what makes a fill cheap and lets a reader follow one without its cached positions going wrong. {meth}`C2Array.written_chunks() ` says how far it has got, straight out of the file's own index — no endpoint of its own, about 2.5 ms over HTTP: +The server serializes updates, and {meth}`C2Array.written_chunks() ` reports progress from the array's index: ```python -written = a.written_chunks() # one bool per chunk -print(f"{written.sum()}/{written.size} chunks in") +written = a.written_chunks() for nchunk in np.flatnonzero(~written): - ... # the work still to do, after a crash + ... # chunks still missing after a restart ``` -What this buys: the server serializes the writes themselves, so what overlaps is the round trip — which over a network is nearly all of the cost. Against a real server, a fill went from **244 ms per chunk serially to 32 ms with 8 writers, 7.6x**. Over loopback, where there is no round trip to hide, it is 1.0x. +## Use your own transport -## Your own transport - -If your frames live somewhere fsspec does not reach — per-request credentials, a signing proxy, a database column, an in-house gateway — supply one method and you get everything above: +Subclass {ref}`ByteRangeNDSource` when the frame lives behind a transport that fsspec cannot use: ```python import boto3 @@ -191,18 +604,18 @@ import blosc2 class S3Source(blosc2.ByteRangeNDSource): def __init__(self, bucket, key): - self._s3 = boto3.client("s3") - self._bucket, self._key = bucket, key - self.stamp = self._s3.head_object(Bucket=bucket, Key=key)["ETag"] + self.s3 = boto3.client("s3") + self.bucket, self.key = bucket, key + self.stamp = self.s3.head_object(Bucket=bucket, Key=key)["ETag"] super().__init__(f"s3://{bucket}/{key}") def read_range(self, offset, size): - answer = self._s3.get_object( - Bucket=self._bucket, - Key=self._key, + response = self.s3.get_object( + Bucket=self.bucket, + Key=self.key, Range=f"bytes={offset}-{offset + size - 1}", ) - data = answer["Body"].read() + data = response["Body"].read() self.traffic.charge(len(data)) return data @@ -210,18 +623,22 @@ class S3Source(blosc2.ByteRangeNDSource): a = blosc2.Proxy(S3Source("bucket", "big.b2nd"), urlpath="cache.b2nd", mode="a") ``` -(For plain S3 you would just use `blosc2.open("s3://bucket/big.b2nd", lazy=True)`; this is the shape of the thing.) - -Four things to get right: +Initialize the transport before `super().__init__()`, because the base constructor immediately reads the frame header. +Make `read_range()` thread-safe, set `stamp` so persistent caches can detect changes, and charge the bytes read so traffic measurements remain accurate. -- **Set up the transport before `super().__init__()`.** The base constructor calls `read_range()` straight away to read the file's header. -- **`read_range()` must be thread-safe.** It is called from a thread pool so fetches can overlap. A boto3 *client* is fine; a `Session` or resource is not. -- **Set `stamp` if you can.** It is what lets a cache tell that the remote has changed. Without it the cache is kept on geometry alone. -- **Charge what you read.** End `read_range()` with `self.traffic.charge(len(data))` and your source is counted like the built-in ones — see [Seeing byte savings](#seeing-byte-savings). Skip it and `traffic` reads zero forever, which looks like a free transport rather than an uncounted one. +For ordinary remote access, use `blosc2.open("https://...", lazy=True)` or `blosc2.open("s3://bucket/big.b2nd", lazy=True)`; the custom class only illustrates the transport contract. ## See also -- {doc}`Tutorial 6 <../tutorials/06.remote_proxy>` — the same ground at a slower pace, with output. -- `examples/ndarray/rw-fsspec.py` — every way of reading and writing an fsspec URL, runnable. -- `examples/c2array-traffic.py` — what a remote slice costs in bytes, and what blocks and the cache save, runnable. -- {ref}`C2Array`, {ref}`FsspecNDSource`, {ref}`ByteRangeNDSource`, {ref}`Proxy`, {ref}`Traffic` — the reference pages. +- {doc}`Tutorial 6 <../tutorials/06.remote_proxy>` — a step-by-step introduction with output. +- `examples/remote/s3-access.py` — remote access across Blosc2 (.b2nd, .b2z), Zarr, and HDF5 with timing and network traffic metering. +- `examples/remote/store-browse.py` — inspecting hierarchies, leaf previews, and shared caching across leaves. +- `examples/remote/c2array-get-slice.py` — opening and reading remote Caterva2 arrays via URLPath. +- `examples/remote/c2array-traffic.py` — block, chunk, and cached transfer sizes against Caterva2. +- `examples/remote/c2array_expr.py` — lazy expression evaluation on remote Caterva2 arrays. +- `examples/remote/concurrent-fsspec.py` — concurrent chunk fetching (`max_concurrency`) on high-latency stores. +- `examples/remote/fsspec-cat2-access.py` — one dataset and cache through fsspec and Caterva2. +- `examples/remote/proxy-carray.py` — creating a persistent local disk proxy of a remote Caterva2 array. +- `examples/remote/rw-fsspec.py` — fsspec reading and writing examples. +- {doc}`b2view ` — interactive terminal browser for local and remote containers. +- {ref}`RemoteArray`, {ref}`RemoteStore`, {ref}`RemoteNode`, {ref}`C2Array`, {ref}`B2ZNDSource`, {ref}`ZarrNDSource`, {ref}`HDF5NDSource`, {ref}`FsspecNDSource`, {ref}`ByteRangeNDSource`, {ref}`Proxy`, and {ref}`Traffic` — API reference pages. diff --git a/doc/reference/b2zndsource.rst b/doc/reference/b2zndsource.rst new file mode 100644 index 000000000..a1faf4d98 --- /dev/null +++ b/doc/reference/b2zndsource.rst @@ -0,0 +1,20 @@ +.. _B2ZNDSource: + +B2ZNDSource +=========== + +``B2ZNDSource`` exposes an external NDArray member inside an immutable ``.b2z`` +archive through :ref:`ByteRangeNDSource`. Chunks and blocks are read directly +from the ZIP member using native Blosc2 range reads without decompressing or +downloading the archive. + +The source is assumed immutable (``assume_immutable=True``). It requires an +explicit dataset path pointing to an uncompressed (``ZIP_STORED``) external +NDArray member. Embedded leaves and CTable columns are not supported. + +Install support with ``pip install "blosc2[fsspec]"`` plus the protocol +driver, such as ``s3fs`` for S3. + +.. autoclass:: blosc2.B2ZNDSource + + .. automethod:: __init__ diff --git a/doc/reference/c2array.rst b/doc/reference/c2array.rst index 9f8caaef2..2085ce11e 100644 --- a/doc/reference/c2array.rst +++ b/doc/reference/c2array.rst @@ -3,16 +3,26 @@ C2Array ======= -This is a class for remote arrays. This kind of array can also work as operand on a LazyExpr, LazyUDF or reduction. +This is a class for one array-like dataset addressed through a Caterva2 server. +The dataset may be a standalone ``.b2nd`` array, an HDF5 dataset, an NDArray +leaf inside a ``.b2z`` store, or a lazy/computed array. A ``C2Array`` does not +represent or navigate a whole remote ``TreeStore`` or ``DictStore``; use +Caterva2 to select a leaf and open that leaf's path. This kind of array can also +work as an operand on a LazyExpr, LazyUDF or reduction. :ref:`URLPath` is +Caterva2-only, including when its ``urlbase`` is omitted and inherited from +:func:`blosc2.c2context`. + +For a comparison with byte-oriented fsspec access, see +:doc:`Working with Remote Arrays <../guides/remote_arrays>`. Wrapped in a :ref:`Proxy`, a stored remote array is read at block granularity: the proxy asks for the blocks a slice touches rather than the chunks they live in, which for a multi-megabyte chunk is a small fraction of the bytes. That rests on the server serving the dataset from a file, ``Range`` header and auth cookie both honoured; a dataset it computes instead (a lazy expression, an -HDF5 leaf) is fetched a whole chunk at a time, as everything was before. Which -one this is takes at most one request to find out, and is decided once -- -:meth:`C2Array.block_source` is what answers it. +HDF5 leaf, or a ``.b2z`` member) is fetched a whole chunk at a time, as +everything was before. Which one this is takes at most one request to find out, +and is decided once -- :meth:`C2Array.block_source` is what answers it. A stored remote array can also be *filled*, by as many writers at once as it has chunks. The array is laid out first -- ``blosc2.uninit`` writes a couple of diff --git a/doc/reference/classes.rst b/doc/reference/classes.rst index d06646d04..77f9c5c66 100644 --- a/doc/reference/classes.rst +++ b/doc/reference/classes.rst @@ -32,6 +32,9 @@ without chunk caching. Proxy ProxySource ProxyNDSource + ZarrNDSource + HDF5NDSource + B2ZNDSource SimpleProxy Traffic @@ -108,6 +111,7 @@ codecs, filters, and remote paths. SpecialValue Tuner FPAccuracy + CachePolicy URLPath @@ -138,10 +142,15 @@ container APIs above. list_array objectarray proxy + remotearray + remotestore proxysource proxyndsource byterangendsource fsspecndsource + zarrndsource + hdf5ndsource + b2zndsource simpleproxy traffic embed_store diff --git a/doc/reference/fsspecndsource.rst b/doc/reference/fsspecndsource.rst index deb2a6fb6..d8bc84f3a 100644 --- a/doc/reference/fsspecndsource.rst +++ b/doc/reference/fsspecndsource.rst @@ -7,11 +7,16 @@ A :ref:`ByteRangeNDSource` that serves the chunks of a Blosc2 frame living behind an fsspec URL, reading each one with a range request instead of transferring the whole container. Everything about the frame format, block granularity included, lives in the base class; this adds the fsspec transport. +The URL must name a standalone, contiguous ``.b2nd`` NDArray frame. It cannot +name an HDF5 dataset, a member inside a ``.b2z`` store, a sparse directory +container, or a computed array: fsspec provides bytes, not dataset semantics. +For the Caterva2 alternative and a capability comparison, see +:doc:`Working with Remote Arrays <../guides/remote_arrays>`. For other sources, see :ref:`ProxyNDSource` and :ref:`ProxySource`. -``examples/ndarray/rw-fsspec.py`` is a runnable walkthrough of this and the +``examples/remote/rw-fsspec.py`` is a runnable walkthrough of this and the other two ways to read an fsspec URL, and of writing one back. -``examples/ndarray/concurrent-fsspec.py`` measures ``max_concurrency`` against a +``examples/remote/concurrent-fsspec.py`` measures ``max_concurrency`` against a filesystem with a simulated round trip, since no protocol that runs offline has latency for the thread pool to hide. diff --git a/doc/reference/hdf5ndsource.rst b/doc/reference/hdf5ndsource.rst new file mode 100644 index 000000000..2fc9d9df0 --- /dev/null +++ b/doc/reference/hdf5ndsource.rst @@ -0,0 +1,24 @@ +.. _HDF5NDSource: + +HDF5NDSource +============ + +``HDF5NDSource`` exposes an HDF5 dataset through :ref:`ProxyNDSource` using +``kerchunk`` metadata pre-indexing. Individual chunks are fetched on demand +and converted to Blosc2-compressed chunks stored in the surrounding +:ref:`Proxy` or :ref:`RemoteArray` cache. + +The source is assumed immutable (``assume_immutable=True``). It supports fixed-size +boolean, integer, floating-point, complex, and fixed-length string arrays. +HDF5 filters such as Blosc2 (via ``hdf5plugin``), gzip, and uncompressed datasets +are supported. + +Install local support with ``pip install "blosc2[hdf5]"``. Remote datasets also +need ``blosc2[fsspec]`` and the protocol driver, such as ``s3fs`` for S3. + +.. autofunction:: blosc2.available_datasets + +.. autoclass:: blosc2.HDF5NDSource + + .. automethod:: __init__ + .. automethod:: get_chunk diff --git a/doc/reference/lazyarray.rst b/doc/reference/lazyarray.rst index 9bcce3839..d77844c59 100644 --- a/doc/reference/lazyarray.rst +++ b/doc/reference/lazyarray.rst @@ -14,7 +14,7 @@ You can get an object following the LazyArray API in any of the following ways: The LazyArray object is a thin wrapper around the expression or user-defined function that allows for lazy computation. This means that the expression is not computed until the ``compute`` or ``__getitem__`` methods are called. The ``compute`` method will return a new NDArray object with the result of the expression evaluation. The ``__getitem__`` method will return a NumPy object instead. -LazyArray objects also support user metadata via :attr:`LazyArray.vlmeta`. For +LazyArray objects also support user metadata via :attr:`LazyArray.attrs`. For in-memory objects, this metadata lives on the Python object itself. For persisted LazyArrays reopened from disk, metadata is synchronized with the underlying carrier and survives reopening. @@ -40,7 +40,7 @@ See the `LazyExpr`_ and `LazyUDF`_ sections for more information. Attributes ---------- - .. autoattribute:: vlmeta + .. autoattribute:: attrs .. _LazyExpr: diff --git a/doc/reference/msgpack_serialization.rst b/doc/reference/msgpack_serialization.rst index 9807a5fbe..594c20eca 100644 --- a/doc/reference/msgpack_serialization.rst +++ b/doc/reference/msgpack_serialization.rst @@ -22,6 +22,7 @@ The following objects are serialized by value using - ``ObjectArray`` - ``BatchArray`` - ``EmbedStore`` +- ``RemoteArray`` Structured objects ------------------ @@ -40,12 +41,14 @@ Currently implemented structured kinds are: - ``"ref"`` - ``"c2array"`` +- ``"remote_array"`` +- ``"fsspec"`` - ``"urlpath"`` - ``"dictstore_key"`` - ``"lazyexpr"`` - ``"lazyudf"`` -The ``"urlpath"``, ``"dictstore_key"``, and ``"c2array"`` reference forms map +The ``"urlpath"``, ``"dictstore_key"``, ``"c2array"``, and ``"fsspec"`` reference forms map directly onto the public :class:`blosc2.Ref` type. ``C2Array`` @@ -58,6 +61,16 @@ Remote arrays are serialized as lightweight references with: Authentication data is intentionally not serialized. +``RemoteArray`` +--------------- + +Remote proxies use a B2ND carrier containing a versioned Caterva2 or fsspec +source descriptor, a ``"none"`` or ``"disk"`` cache policy, and the finite +disk-cache limit. A disk-caching carrier may also contain fetched compressed +chunks and its cache bookkeeping. Saving includes valid warm chunks by default; +``include_cache=False`` produces a cold carrier. Local paths, live filesystem +objects, and credentials are intentionally not serialized. + Persistent local operands ------------------------- @@ -93,6 +106,7 @@ Only durable reference-style operands are supported: - persistent local Blosc2 operands reopenable from ``urlpath`` - remote ``C2Array`` operands +- ``RemoteArray`` operands for fsspec, Zarr, or Caterva2 references - ``DictStore`` members reopenable from ``(.b2d|.b2z, key)`` Purely in-memory operands are intentionally rejected. This keeps msgpack @@ -119,6 +133,7 @@ Supported operands are the same durable reference-style operands used for - persistent local Blosc2 operands reopenable from ``urlpath`` - remote ``C2Array`` operands +- ``RemoteArray`` operands for fsspec, Zarr, or Caterva2 references - ``DictStore`` members reopenable from ``(.b2d|.b2z, key)`` Plain Python ``LazyUDF`` callables are intentionally not serialized by diff --git a/doc/reference/ndarray.rst b/doc/reference/ndarray.rst index de44ad4d9..555678217 100644 --- a/doc/reference/ndarray.rst +++ b/doc/reference/ndarray.rst @@ -6,6 +6,9 @@ NDArray The multidimensional data array class. Instances may be constructed using the constructor functions in the list below `NDArrayConstructors`_. In addition, all the functions from the :ref:`LazyArray` section can be used with NDArray instances. +Use ``array.attrs`` for user-defined metadata (see :ref:`MsgpackSerialization`). +It is a direct alias for ``array.vlmeta`` and uses the same persistent storage. + .. currentmodule:: blosc2 .. autoclass:: NDArray diff --git a/doc/reference/ref.rst b/doc/reference/ref.rst index 120f89cf0..8962d313a 100644 --- a/doc/reference/ref.rst +++ b/doc/reference/ref.rst @@ -14,6 +14,9 @@ Currently supported reference kinds are: - ``"urlpath"`` for persistent local objects - ``"dictstore_key"`` for members inside ``.b2d`` / ``.b2z`` ``DictStore`` containers - ``"c2array"`` for remote ``C2Array`` objects +- ``"fsspec"`` for fsspec sources used by ``RemoteArray`` objects +- ``"zarr"`` for remote Zarr arrays +- ``"b2z"`` and ``"hdf5"`` for remote container datasets, with the dataset path in ``key`` Use :meth:`Ref.open` to resolve a reference back into a live object. @@ -38,11 +41,11 @@ Example # A Ref can itself be persisted, for example as variable-length metadata # in another persistent Blosc2 object. catalog = blosc2.zeros(1, urlpath=catalog_path, mode="w") - catalog.schunk.vlmeta["array_ref"] = ref + catalog.schunk.attrs["array_ref"] = ref # Reopen the metadata holder and resolve the persisted reference. catalog = blosc2.open(catalog_path, mode="r") - restored_ref = catalog.schunk.vlmeta["array_ref"] + restored_ref = catalog.schunk.attrs["array_ref"] reopened = restored_ref.open() print(reopened[:]) # [0 1 2 3 4] diff --git a/doc/reference/remotearray.rst b/doc/reference/remotearray.rst new file mode 100644 index 000000000..1071ac5f9 --- /dev/null +++ b/doc/reference/remotearray.rst @@ -0,0 +1,233 @@ +.. _RemoteArray: + +RemoteArray +=========== + +``RemoteArray`` is a persistable proxy for one remote B2ND, B2Z, Zarr, or HDF5 array. It +accepts an fsspec URL or a Caterva2 :ref:`URLPath`. With disk caching enabled, +its B2ND carrier is both the portable descriptor and the bounded compressed-data +cache. + +The default policy is :attr:`blosc2.CachePolicy.NONE`: each operation reads the +remote data it needs and no fetched data is retained afterwards. Saving such an +object writes only its source descriptor and array geometry. + +.. code-block:: python + + remote = blosc2.RemoteArray( + "s3://public-bucket/dataset.b2nd", + cache_policy=blosc2.CachePolicy.NONE, + ) + remote.save("dataset-reference.b2nd") + +A Caterva2 dataset is named with :class:`blosc2.URLPath` rather than an fsspec +URL: + +.. code-block:: python + + remote = blosc2.RemoteArray( + blosc2.URLPath( + "@public/dataset.b2nd", + urlbase="https://example.org/caterva2", + ) + ) + +By default, ``RemoteArray`` assumes its source is immutable and skips remote +identity checks before reads. For a replaceable single-file or Caterva2 source, +pass ``assume_immutable=False`` to refresh its identity and invalidate stale +cached data before each operation. + +Zarr URLs use a different contract: a ``.zarr`` path component selects +:ref:`ZarrNDSource`, or pass ``source_format="zarr"`` for a suffix-free path. +The URL names one array, including its path inside a hierarchy. Zarr sources are +assumed immutable for the lifetime of every cache; replacing data beneath the +same URL may mix stale and new chunks. Use a new URL or replace the cache when +publishing a new dataset. Mutable Zarr stores are not supported. + +.. code-block:: python + + remote = blosc2.open( + "s3://public-bucket/hierarchy.zarr/d0/a1", + lazy=True, + storage_options={"anon": True}, + ) + +HDF5 URLs (``.h5``, ``.hdf5``, or ``source_format="hdf5"``) select :ref:`HDF5NDSource`. +Datasets within an HDF5 container can be specified via standard slash syntax (``.../file.h5/dataset``), +the double-colon separator (``.../file.h5::dataset``), or the ``dataset="dataset"`` argument. +Zarr containers similarly accept all three forms (``.../file.zarr/dataset``, ``.../file.zarr::dataset``, +or ``dataset="dataset"``). +HDF5 datasets are read through ``kerchunk`` metadata pre-indexing. Like Zarr, HDF5 sources +are assumed immutable (``assume_immutable=True``); mutable HDF5 sources are not supported. +Pre-computed kerchunk references can be supplied via ``refs`` to avoid remote scanning. + +.. code-block:: python + + remote = blosc2.open( + "s3://public-bucket/hierarchy.h5/d0/d1/a2", + lazy=True, + storage_options={"profile": "blosc2"}, + ) + # Equivalent to "s3://public-bucket/hierarchy.h5::d0/d1/a2" + # or blosc2.open("s3://public-bucket/hierarchy.h5", lazy=True, dataset="d0/d1/a2", ...) + +B2Z archives +------------ + +An external NDArray inside an immutable ``.b2z`` archive can be selected using +the same three addressing forms: + +.. code-block:: python + + remote = blosc2.open( + "s3://public-bucket/hierarchy.b2z::/d0/a3", + lazy=True, + storage_options={"anon": True}, + ) + values = remote[:10, 0, :5] + # Also accepts hierarchy.b2z/d0/a3 or dataset="d0/a3". + +Use ``source_format="b2z"`` for suffix-free archive URLs. The dataset is a logical +tree key without the member's ``.b2nd`` suffix. The native Blosc2 reader preserves +source chunks, blocks, dtype, and compression parameters; no kerchunk, Zarr, or +HDF5 dependencies are needed. Install the fsspec extra and the protocol backend. + +Opening reads the ZIP directory and selected member's headers. Directory cost +scales with archive member count. An 8 KiB archive tail and 16 KiB member prefix +are prefetched to combine small metadata requests; larger directories or headers +fall back to exact reads. These temporary buffers are released after opening. +Subsequent reads fetch native chunks or blocks +by byte range; repeated cache hits perform no remote reads. Reopening a saved +carrier rereads archive/frame metadata and resolves the member offset afresh. +The optimized reader derives its source stamp from the same metadata response +used to obtain archive size. Caches from the initial v10 reader may therefore +refetch their contents once after upgrading. + +Only unencrypted, ``ZIP_STORED`` external NDArray members are supported. Groups, +embedded leaves inside ``embed.b2e``, other leaf types, and compressed ZIP members +are unsupported. Archives must remain immutable; replacing an archive requires +replacing its cache. Authorized sparse attachment (:meth:`RemoteArray.with_sparse_cache`) +is supported for eligible external NDArray leaves, while Caterva2 federation is not +supported in this version. + +See :ref:`B2ZNDSource` for class details. + +Caching and persistence +----------------------- + +Ephemeral in-memory caching is available through :attr:`blosc2.CachePolicy.MEMORY`. +Fetched chunks are kept in RAM, bounded by a finite 256 MiB compressed-payload limit by default +(customizable via ``max_cache_bytes``) with automatic LRU eviction. + +Persistent caching is available through :attr:`blosc2.CachePolicy.DISK`. +Disk caches have a finite 256 MiB compressed-payload bound by default and can +take an explicit ``max_cache_bytes`` bound, or ``max_cache_bytes=None`` for an +unbounded cache that never evicts chunks. When bounded, the limit is enforced after an +operation completes and therefore does not limit its temporary working set or +returned NumPy array. + +.. code-block:: python + + remote = blosc2.RemoteArray( + "s3://public-bucket/dataset.b2nd", + cache_policy=blosc2.CachePolicy.DISK, + cache_path="dataset-cache.b2nd", + max_cache_bytes=2 * 2**30, + ) + +When opening a remote array via :func:`blosc2.open` with ``lazy=True``, a :class:`RemoteArray` +is always returned: specifying ``cache_dir`` or ``cache_path`` configures it with +:attr:`blosc2.CachePolicy.DISK`, while omitting them configures it with +:attr:`blosc2.CachePolicy.MEMORY`. + +By default, :meth:`RemoteArray.save ` and +:meth:`RemoteArray.to_cframe ` include valid warm +chunks for DISK proxies; MEMORY proxies always export cold carriers. +Pass ``include_cache=False`` for a cold carrier without changing the +warm original. The cache policy and limit remain in both forms; local paths and +authentication data are not serialized. + +Pass ``cache_policy=blosc2.CachePolicy.NONE`` (or another policy) to either +export method to produce a cold carrier with an explicit policy, leaving the +live proxy unchanged. Caterva2 servers accept persisted MEMORY carriers under +opt-in policy but execute them without retained caching (identical to NONE); +older Caterva2 servers reject MEMORY resolution. Use DISK for retained carrier +caching on Caterva2. Cold exports must not overwrite the live disk carrier. + +``fetch()`` and ``afetch()`` prefetch and return the proxy. Eviction may discard +requested chunks; ``materialize(item)`` returns an independent complete NDArray. +The raw ``cache`` is incomplete storage for inspection, not a materialized array. + +Reads and exports on one handle are serialized. Async methods use worker threads; +cancelling an await does not stop a running operation. Separate handles and +processes sharing a carrier need external locking. Unreadable cache files are +preserved and their opening errors are propagated. + +Authentication supplied to a live Caterva2 source is deliberately omitted from +the carrier. Caterva2's first server implementation resolves public HTTPS +sources only; client credentials never travel with the proxy. + +Open a disk-caching carrier in append mode to let misses populate that same +file. Read-only mode can use warm chunks but does not retain misses: + +.. code-block:: python + + cached = blosc2.open("dataset-cache.b2nd", mode="a") + cached[100:200] + +.. warning:: + + Resolving an uploaded remote reference makes the receiving server perform + an outbound request. Caterva2 installations must reject these references by + default unless administrators configure allowed protocols, destinations, + credentials, redirects, and resource limits. Client-side URL checks are not + a server security boundary. + +.. autoclass:: blosc2.RemoteArray + + .. automethod:: __init__ + .. automethod:: __getitem__ + .. automethod:: fetch + .. automethod:: afetch + .. automethod:: get_chunk + .. automethod:: aget_chunk + .. automethod:: save + .. automethod:: materialize + .. automethod:: to_cframe + .. autoattribute:: shape + .. autoattribute:: dtype + .. autoattribute:: ndim + .. autoattribute:: chunks + .. autoattribute:: blocks + .. autoattribute:: cparams + .. autoattribute:: nbytes + .. autoattribute:: meta + .. autoattribute:: attrs + .. autoattribute:: info + .. autoattribute:: cache + .. autoattribute:: cache_bytes + .. autoattribute:: cache_policy + .. autoattribute:: max_cache_bytes + .. autoattribute:: cache_path + .. autoattribute:: cache_status + .. autoattribute:: schunk + .. autoattribute:: source + .. autoattribute:: traffic + .. autoattribute:: urlpath + .. autoattribute:: dataset + +RemoteMetadataMapping +--------------------- + +``RemoteArray.attrs`` returns a read-only mapping that fetches array attributes +only when they are accessed. Use it like a dictionary, or use ``attrs[:]`` to +fetch all attributes at once. + +.. autoclass:: blosc2.RemoteMetadataMapping + :members: + +CachePolicy +----------- + +.. autoclass:: blosc2.CachePolicy + :members: diff --git a/doc/reference/remotestore.rst b/doc/reference/remotestore.rst new file mode 100644 index 000000000..3fa77447d --- /dev/null +++ b/doc/reference/remotestore.rst @@ -0,0 +1,158 @@ +.. _RemoteStore: + +RemoteStore +=========== + +``RemoteStore`` discovers a read-only B2Z, Zarr or HDF5 hierarchy and returns +:ref:`RemoteArray` leaves. Groups and arrays share one source session: a B2Z +archive, an HDF5 reference map, or a Zarr store. Zarr listing remains lazy. + +The default ``CachePolicy.MEMORY`` shares a 256 MiB allowance across all leaves. +Set ``max_cache_bytes`` to a positive integer to change it. ``CachePolicy.NONE`` +retains no payload and rejects a limit. Passing ``cache_dir`` selects DISK when +the policy is omitted; an explicit policy must agree with the cache location. +DISK accepts ``max_cache_bytes=None`` for unbounded retention. +Sources must be immutable. Generic ``blosc2.open(..., lazy=True, dataset=...)`` +continues to open a single array. + +.. code-block:: python + + with blosc2.RemoteStore( + "https://host/data.h5", cache_policy=blosc2.CachePolicy.NONE + ) as store: + print(store.keys()) # immediate children + info = store.get_info("experiment") # metadata only + group = store["experiment"] + array = group["temperature"] + values = array[:100] + result = (array + 273.15).compute() + + # Returned handles own their source lifetime independently. + values = array[:100] + group.close() + array.close() + +Paths are relative to the selected group. ``store["a/b"]`` and +``store["a"]["b"]`` use the same source reader. Each lookup returns an independent +handle. With NONE, repeated reads fetch again; no payload cache is retained. +``dataset="a"`` or a subgroup suffix in the URL selects a group at construction. +An array root must be opened with ``RemoteArray`` instead. + +``keys()`` and ``get_info()`` do not construct leaf readers or payload caches. +Discovery can read archive prefixes, attributes and small HDF5 inline values. +``get_info()`` returns a ``RemoteNode`` with a relative path, a kind (``group``, +``ndarray`` or ``unsupported``), known attributes and a diagnostic. Unknown array +attributes are ``None``; open the array to retrieve them. Unsupported nodes stay +discoverable and raise ``NotImplementedError`` when selected. Missing paths raise +``KeyError``. + +Group ``attrs`` mappings are read-only. ``source`` returns the credential-free +container descriptor and full group path. ``traffic`` is one shared source +counter across all views, including discovery; do not add counts from aliases. +It counts source reads, not connections or every HTTP request: HEAD requests and +failed Zarr probes are excluded. ``cache_bytes`` counts retained compressed chunks +and partial-block duplicates across the store, without double-counting aliases. +The least recently used native chunk is evicted across all leaves when necessary. +Oversized reads return their result before eviction; returned arrays and temporary +buffers are outside the allowance. Closing a leaf preserves its warm cache while +other session handles remain open. With NONE, ``cache_bytes`` is zero and +``max_cache_bytes`` is ``None``. + +Closing a handle, or exiting its context, releases its ownership. Existing child +handles remain usable until closed or garbage-collected. The last handle closes +the owned archive/store wrappers and private HTTP/S3 transport sessions. Operations on an explicitly closed handle raise ``RuntimeError``. +Standalone ``RemoteArray`` exports remain self-contained references, including +the HDF5 reference map when applicable. + +``b2view`` uses ``RemoteStore`` for remote hierarchies with one 64 MiB MEMORY +allowance, and ``RemoteArray`` for selected or directly opened leaves. Switching +selection releases the UI handle while retaining the store's warm chunks. + +For persistent shared caching: + +.. code-block:: python + + with blosc2.RemoteStore("https://host/data.h5", cache_dir="remote-cache") as store: + with store["experiment/temperature"] as array: + values = array[:100] + print(store.cache_bytes, store.metadata_bytes) + store.refresh() # rebuild discovery; old child handles become stale + +Each source and selected root has its own directory under ``cache_dir``. One +owner holds an exclusive operating-system lock until its last dependent handle +closes. Conflicting opens raise ``RuntimeError``, including in other processes; +the operating system releases the lock after a process exits or crashes. + +Reopening restores all previously created leaf caches and trims them against the +new aggregate allowance before returning. The manifest preserves B2Z directory +and bounded metadata reads, one HDF5 reference map, and lazily discovered Zarr +metadata. Metadata reads can contain small inline values or incidental bytes in +bounded prefixes; they are separate from evictable payload. ``metadata_bytes`` +is the encoded manifest size, and is zero without a disk manifest. Credentials +and storage options must be supplied again at runtime. + +The allowance measures compressed retained payload, not filesystem allocation. +Container overhead and old generations are outside it; obsolete generation +files are removed on the next exclusive reopen. Manifests survive payload +eviction. Store backing files are private implementation details; use array +``save()`` or ``to_cframe()`` for standalone exports, optionally with +``include_cache=True``. + +Sources must remain immutable until explicit root ``refresh()``. Refresh builds +replacement discovery before publishing a new generation; failed discovery or +publication leaves existing handles valid. Successful refresh makes existing +child groups and arrays stale, requiring fresh lookups. Corrupt manifests and +B2Z validator mismatches raise an error; use a fresh cache directory if the store +cannot be opened. Fully offline reopening is not promised. The lifetime lock +uses POSIX flock or Windows byte locking; Windows execution remains a CI check. + +Shared sparse runtime caches +---------------------------- + +Services and multiple local processes can use ``RemoteStore.with_sparse_cache`` +to keep simultaneous handles to the same private runtime cache: + +.. code-block:: python + + with blosc2.RemoteStore.with_sparse_cache( + "https://host/data.b2z", "shared-runtime", max_cache_bytes=64 << 20 + ) as store: + with store["experiment/temperature"] as array: + values = array[:100] + hit, values = store.read_cached("experiment/temperature", slice(0, 100)) + +This mode stores leaf payload in sparse RemoteArray caches. Each operation +acquires a store-wide OS lock, reloads discovery and leaf accounting, and applies +one aggregate payload allowance. Handles may coexist across processes, while +operations within a store serialize. All users of that directory must use the +shared constructor. A process-local memory cache or the ordinary exclusive +``cache_dir`` constructor must not write to it. + +Manifests and generation pointers are published atomically. A process that dies +during an operation causes the next owner to discard the disposable payload +generation; remote sources are not contacted by offline trimming or manifest recovery. +Ordinary exceptions, such as a missing key, leave existing handles usable when +operation cleanup succeeds. +Refresh publishes a new generation and makes child handles in other processes +stale. Reopen a store handle after another process refreshes it. + +``read_cached`` returns ``(False, None)`` on a miss without fetching missing +payload. ``trim_sparse_cache`` trims leaves offline with a bounded chunk count. +Its first implementation evicts in leaf order; the live aggregate coordinator +handles eviction during ordinary reads. Allocated storage and old generations +are separate from the compressed-payload allowance and belong to the service's +storage accounting and lifecycle management. + +The shared constructor accepts an authorized ``_filesystem`` and validation +callbacks for server use; these runtime objects are never persisted. A portable +``carrier`` archive can seed a new cache once, with source stamp and geometry +checks. ``save`` exports ordinary portable warm/cold archives. Private sparse +directories are not portable store artifacts. This protocol targets processes +sharing a local filesystem, not distributed or network-filesystem ownership. + +.. autoclass:: blosc2.RemoteStore + :members: + :special-members: __getitem__, __iter__ + +.. autoclass:: blosc2.RemoteNode + :members: diff --git a/doc/reference/schunk.rst b/doc/reference/schunk.rst index 856463691..7b3dd9b93 100644 --- a/doc/reference/schunk.rst +++ b/doc/reference/schunk.rst @@ -10,7 +10,31 @@ The basic compressed data container (aka super-chunk). This class consists of a Metadata support ---------------- -``SChunk.vlmeta`` uses the general Blosc2 msgpack extensions. This means +Use ``obj.attrs`` as the recommended interface for user-defined metadata:: + + array = blosc2.zeros(10) + array.attrs["units"] = "kelvin" + print(array.attrs["units"]) + print(array.attrs[:]) + +``attrs`` delegates to ``vlmeta`` on ``NDArray``, ``SChunk``, ``ObjectArray``, +``BatchArray``, ``ListArray``, ``CTable``, ``TreeStore``, ``LazyArray``, ``Proxy`` +and ``RemoteArray``, and on ``ProxyNDSource`` implementations. It preserves +existing persistence, serialization and access rules; it does not introduce +another metadata store or filter operational entries from ordinary containers. +Some objects return a fresh mapping wrapper on each access. + +``vlmeta`` remains supported with its existing behavior and is not deprecated. +Prefer ``attrs`` in new user-facing code. Constructor arguments named ``vlmeta`` +retain their existing names. + +``C2Array`` is the exception: ``attrs`` selects user attributes from the server, +whereas ``vlmeta`` retains raw protocol metadata. Older servers without an +``attrs`` field fall back to raw variable metadata. Neither property writes +changes to the server. ``RemoteArray.attrs`` is read-only. See the +:doc:`remote array guide <../guides/remote_arrays>` for details. + +``SChunk.attrs`` uses the general Blosc2 msgpack extensions. This means variable-length metadata can store not only ordinary msgpack-safe Python values, but also the currently supported Blosc2 objects and references, including: @@ -21,8 +45,8 @@ including: - ``LazyExpr`` - ``LazyUDF`` backed by ``@blosc2.dsl_kernel`` -Both single-key access (``schunk.vlmeta["name"]``) and bulk access -(``schunk.vlmeta[:]``) use this serializer. +Both single-key access (``schunk.attrs["name"]``) and bulk access +(``schunk.attrs[:]``) use this serializer. Lazy expressions and supported lazy UDFs still require durable operand references only; purely in-memory operands are intentionally rejected. diff --git a/doc/reference/traffic.rst b/doc/reference/traffic.rst index f25e185f9..99d99cfea 100644 --- a/doc/reference/traffic.rst +++ b/doc/reference/traffic.rst @@ -27,6 +27,6 @@ does with one. It is not private, though -- a transport of your own calls it from ``read_range()`` so that its reads are counted; see :ref:`ByteRangeNDSource` and the *Your own transport* section of the remote-arrays guide. -``examples/c2array-traffic.py`` is a runnable walkthrough -- what a corner slice +``examples/remote/c2array-traffic.py`` is a runnable walkthrough -- what a corner slice of a remote array costs against the chunk holding it, and what the cache saves on the second read. diff --git a/doc/reference/tree_store.rst b/doc/reference/tree_store.rst index d093809e2..7af844d96 100644 --- a/doc/reference/tree_store.rst +++ b/doc/reference/tree_store.rst @@ -105,7 +105,7 @@ Quick example Properties ---------- - .. autoattribute:: vlmeta + .. autoattribute:: attrs Public Members -------------- diff --git a/doc/reference/zarrndsource.rst b/doc/reference/zarrndsource.rst new file mode 100644 index 000000000..0c4794485 --- /dev/null +++ b/doc/reference/zarrndsource.rst @@ -0,0 +1,22 @@ +.. _ZarrNDSource: + +ZarrNDSource +============ + +``ZarrNDSource`` exposes a Zarr v2 or v3 array through :ref:`ProxyNDSource`. +Zarr decodes each logical chunk and Blosc2 stores the converted compressed +chunk in the surrounding :ref:`Proxy` or :ref:`RemoteArray` cache. + +The source is assumed immutable for the cache lifetime. It supports scalar and +zero-length arrays, and fixed-size boolean, integer, floating-point, complex, +structured, string, datetime, and timedelta dtypes. Variable-length strings, +object dtypes, and ZIP stores are not supported. Concurrent reads temporarily +hold decoded chunks and conversion buffers in addition to the compressed cache. + +Install local support with ``pip install "blosc2[zarr]"``. Remote stores also +need ``blosc2[fsspec]`` and the protocol driver, such as ``s3fs``. + +.. autoclass:: blosc2.ZarrNDSource + + .. automethod:: __init__ + .. automethod:: get_chunk diff --git a/doc/tutorials/06.remote_proxy.ipynb b/doc/tutorials/06.remote_proxy.ipynb index 9c1cf8a98..41d8f6bc8 100644 --- a/doc/tutorials/06.remote_proxy.ipynb +++ b/doc/tutorials/06.remote_proxy.ipynb @@ -38,7 +38,7 @@ "metadata": {}, "source": [ "## ``C2Array`` class\n", - "Before we look at proxies, it is first necessary to understand how to use Blosc2 to work with remote data, via the ``C2Array`` class. The class implements a (limited) version of the NDArray interface of which we have already seen a lot in previous tutorials. However, it is really a local pointer to a remote array (stored e.g. on a remote server). This means that we can refer to the data, access certain attribute information about it, download portions of the data and even define it in computational expressions, without having to download the entire array into local memory or disk. This is particularly useful when working with large datasets that cannot fit into memory or would take far too long to transfer over the network.\n", + "Before we look at proxies, it is first necessary to understand how to use Blosc2 to work with remote data, via the ``C2Array`` class. The class implements a (limited) version of the NDArray interface of which we have already seen a lot in previous tutorials. However, it is really a local pointer to a remote array (stored e.g. on a remote server). This means that we can refer to the data, access certain attribute information about it, download portions of the data and even define it in computational expressions, without having to download the entire array into local memory or disk. This is particularly useful when working with large datasets that cannot fit into memory or would take far too long to transfer over the network. The [Working with Remote Arrays](https://www.blosc.org/python-blosc2/guides/remote_arrays.html#choosing-between-fsspec-and-caterva2) guide explains when to use Caterva2's semantic dataset route instead of a byte-oriented fsspec URL.\n", "\n", "However, one limitation of this approach is that every time one wants to download a slice of the dataset, the data is fetched over the network - even if the same slice has been downloaded before. This can lead to inefficiencies, especially when working with large datasets or when the same data is accessed multiple times. Proxies offer a solution to this, whilst still preserving the low storage requirements of the ``C2Array`` class.\n", "\n", diff --git a/examples/c2array-get-slice.py b/examples/remote/c2array-get-slice.py similarity index 100% rename from examples/c2array-get-slice.py rename to examples/remote/c2array-get-slice.py diff --git a/examples/c2array-traffic.py b/examples/remote/c2array-traffic.py similarity index 91% rename from examples/c2array-traffic.py rename to examples/remote/c2array-traffic.py index 60b323b76..ee8b42835 100644 --- a/examples/c2array-traffic.py +++ b/examples/remote/c2array-traffic.py @@ -11,7 +11,7 @@ # the whole chunk: on a fast link the two take about as long, and differ by the # compression ratio in bytes. Bytes are also what a metered link and a shared # server uplink actually run out of, so they are what `Traffic` counts -- at the -# transport, so the frame index and block offsets are in the tally too. +# transport, so metadata, the frame index, and block offsets are in the tally too. import blosc2 @@ -27,9 +27,8 @@ def cost(traffic): array = blosc2.C2Array(path, urlbase=urlbase) print(f"{path}: shape={array.shape} chunks={array.chunks} blocks={array.blocks}") -# Opening a handle costs one `api/info` call, which is metadata rather than data -# and is deliberately not counted -- no slice can avoid it, and no choice of -# granularity changes it. +# Opening a handle costs one `api/info` call, included so this is a complete +# account of everything that crossed the wire. print(f"after opening: {array.traffic}") # -- A proxy reads through the block path, so it pays for what a slice touches. diff --git a/examples/ndarray/c2array_expr.py b/examples/remote/c2array_expr.py similarity index 100% rename from examples/ndarray/c2array_expr.py rename to examples/remote/c2array_expr.py diff --git a/examples/ndarray/concurrent-fsspec.py b/examples/remote/concurrent-fsspec.py similarity index 100% rename from examples/ndarray/concurrent-fsspec.py rename to examples/remote/concurrent-fsspec.py diff --git a/examples/remote/fsspec-cat2-access.py b/examples/remote/fsspec-cat2-access.py new file mode 100644 index 000000000..547c963d6 --- /dev/null +++ b/examples/remote/fsspec-cat2-access.py @@ -0,0 +1,114 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Compare lazy access to the same array through fsspec and Caterva2. + +The HTTPS path needs the fsspec extra. Install it with: + + pip install "blosc2[fsspec]" + +By default, caches are kept under ``./fsspec-cat2-cache``. Run the example again to +see the first data access served by the cache left by the previous process. +""" + +import argparse +from pathlib import Path +from time import perf_counter + +import numpy as np + +import blosc2 + +# Using the Caterva2 API +CATERVA2_URL = blosc2.URLPath( + "@public/examples/cube-1k-1k-1k.b2nd", + urlbase="https://cat2.cloud/demo", +) +# ...and also using the fsspec path via fetch URL in Caterva2 +FSSPEC_URL = "https://cat2.cloud/demo/api/fetch/@public/examples/cube-1k-1k-1k.b2nd" +# The same contents are published in this Backblaze B2 bucket with a ``-2`` suffix. +# FSSPEC_URL = "https://f001.backblazeb2.com/file/blosc2/cube-1k-1k-1k-2.b2nd" + +SLICE = np.s_[100:110, 200:300, 400:500] + + +def traffic_text(traffic: blosc2.Traffic | None) -> str: + if traffic is None: + return "traffic unavailable" + request_word = "request" if traffic.requests == 1 else "requests" + return f"{traffic.requests} {request_word}, {traffic.nbytes / 2**20:.3f} MiB" + + +def size_text(size: int) -> str: + return f"{size / 2**20:.3f} MiB" + + +def benchmark(label: str, urlpath, cache_dir: Path) -> np.ndarray: + start = perf_counter() + array = blosc2.open(urlpath, lazy=True, cache_dir=cache_dir) + open_time = perf_counter() - start + open_traffic = traffic_text(array.traffic) + + metadata = (array.shape, array.dtype, array.chunks, array.blocks) + cache_path = Path(array.urlpath).resolve() + cache_status = array.cache_status + + array.traffic.reset() + start = perf_counter() + data = array[SLICE] + first_read_time = perf_counter() - start + first_traffic = traffic_text(array.traffic) + cache_size = cache_path.stat().st_size + + # Open a fresh remote handle over the same on-disk cache. This demonstrates + # that cached data survives the Proxy object, not merely one array access. + del array + start = perf_counter() + reopened = blosc2.open(urlpath, lazy=True, cache_dir=cache_dir) + reopen_time = perf_counter() - start + reopen_traffic = traffic_text(reopened.traffic) + + reopened.traffic.reset() + start = perf_counter() + cached = reopened[SLICE] + cached_read_time = perf_counter() - start + cached_traffic = traffic_text(reopened.traffic) + np.testing.assert_array_equal(cached, data) + + print(f"\n{label}") + print(f" metadata: shape={metadata[0]}, dtype={metadata[1]}") + print(f" chunks={metadata[2]}, blocks={metadata[3]}") + print(f" persistent cache: {cache_path} ({cache_status})") + print(f" {'open + remote metadata:':<27}{open_time * 1000:.0f} ms ({open_traffic})") + print(f" {'first data slice:':<27}{first_read_time * 1000:.0f} ms ({first_traffic})") + print(f" {'cache after slice:':<27}{size_text(cache_size)}") + print(f" {'reopen + remote metadata:':<27}{reopen_time * 1000:.0f} ms ({reopen_traffic})") + print(f" {'same slice after reopen:':<27}{cached_read_time * 1000:.0f} ms ({cached_traffic})") + return data + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--cache-dir", + type=Path, + default=Path("fsspec-cat2-cache"), + help="persistent cache root (default: ./fsspec-cat2-cache)", + ) + args = parser.parse_args() + root = args.cache_dir + + print(f"Persistent cache root: {root.resolve()}") + print("Run this command again to reuse these cache files.") + cat2_data = benchmark("Caterva2", CATERVA2_URL, root / "caterva2") + fsspec_data = benchmark("fsspec over HTTPS", FSSPEC_URL, root / "fsspec") + np.testing.assert_array_equal(cat2_data, fsspec_data) + print("\nBoth services returned identical data.") + + +if __name__ == "__main__": + main() diff --git a/examples/ndarray/proxy-carray.py b/examples/remote/proxy-carray.py similarity index 100% rename from examples/ndarray/proxy-carray.py rename to examples/remote/proxy-carray.py diff --git a/examples/ndarray/rw-fsspec.py b/examples/remote/rw-fsspec.py similarity index 95% rename from examples/ndarray/rw-fsspec.py rename to examples/remote/rw-fsspec.py index c7aa286ca..1d7c09da0 100644 --- a/examples/ndarray/rw-fsspec.py +++ b/examples/remote/rw-fsspec.py @@ -43,7 +43,7 @@ # starts from the copy that is already there. Cached copies are checked # against the remote on every open, so a replaced array is never served # from a stale cache. - c = blosc2.open(urlpath, cache_storage=cachedir, mmap_mode="r") + c = blosc2.open(urlpath, cache_dir=cachedir, mmap_mode="r") print(f"read cached: {c.shape} (mmapped from {cachedir})") np.testing.assert_array_equal(c[:], a[:]) @@ -51,7 +51,7 @@ # is and each slice fetches only what it touches -- the chunks it lands in, # or just the blocks inside them when the chunks are large enough for that # to pay. This is what you want for an array too big to download. - d = blosc2.open(urlpath, lazy=True, cache_storage=cachedir) + d = blosc2.open(urlpath, lazy=True, cache_dir=cachedir) print(f"read lazy: {type(d).__name__} {d.shape} {d.dtype}") # Only the two chunks covering rows 15..25 are fetched here diff --git a/examples/remote/s3-access.py b/examples/remote/s3-access.py new file mode 100755 index 000000000..4894f7dba --- /dev/null +++ b/examples/remote/s3-access.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Open a remote S3 array (Blosc2 .b2nd/.b2z, Zarr .zarr, or HDF5 .h5) and print sample data. + +Usage: + python s3-access.py [--profile PROFILE] [--endpoint-url ENDPOINT_URL] + +Examples: + python s3-access.py s3://blosc2/cube-1k-1k-1k.b2nd + python s3-access.py s3://blosc2/cube-1k-1k-1k.zarr + python s3-access.py s3://blosc2/cube-1k-1k-1k-1shard.zarr + python s3-access.py s3://blosc2/hierarchy.zarr/d0/d1/a2 + python s3-access.py s3://blosc2/hierarchy.zarr::d0/d1/a2 + python s3-access.py s3://blosc2/hierarchy.h5/d0/d1/a2 + python s3-access.py s3://blosc2/hierarchy.h5::d0/d1/a2 + python s3-access.py s3://blosc2/hierarchy.b2z::/d0/a3 +""" + +from __future__ import annotations + +import argparse +import sys +import time +from typing import Any + +import blosc2 + +DEFAULT_PROFILE = "blosc2" +DEFAULT_ENDPOINT_URL = "https://s3.us-west-001.backblazeb2.com" + + +def get_sample_slice(arr: Any) -> Any: + """Extract a representative small sample slice regardless of array dimensionality.""" + ndim = getattr(arr, "ndim", None) + if ndim is None: + ndim = len(arr.shape) if hasattr(arr, "shape") else 0 + if ndim == 0: + return arr[()] + if ndim == 1: + return arr[: min(10, arr.shape[0])] + if ndim == 2: + return arr[: min(10, arr.shape[0]), : min(5, arr.shape[1])] + + idx: list[Any] = [slice(0, min(10, arr.shape[0]))] + for _ in range(ndim - 2): + idx.append(0) + idx.append(slice(0, min(5, arr.shape[-1]))) + return arr[tuple(idx)] + + +class Traffic: + """Track data transferred over the network.""" + + def __init__(self, nbytes: int = 0) -> None: + self.nbytes = nbytes + + def charge(self, n: int) -> None: + self.nbytes += n + + +class TrackingFile: + """Wrapper around a file-like object to track bytes read.""" + + def __init__(self, f: Any, traffic: Traffic) -> None: + self._f = f + self._traffic = traffic + + def read(self, *args: Any, **kwargs: Any) -> Any: + data = self._f.read(*args, **kwargs) + if data: + self._traffic.charge(len(data)) + return data + + def readinto(self, b: Any) -> Any: + n = self._f.readinto(b) + if n: + self._traffic.charge(n) + return n + + def __getattr__(self, name: str) -> Any: + return getattr(self._f, name) + + +def open_remote_array( + url: str, + profile: str = DEFAULT_PROFILE, + endpoint_url: str = DEFAULT_ENDPOINT_URL, +) -> tuple[str, Any]: + """Open remote array depending on extension (.b2nd vs .zarr/.zip vs .h5). + + Returns (format_name, array_object). + """ + storage_options = { + "profile": profile, + "endpoint_url": endpoint_url, + } + + clean_url = url.split("::", 1)[0].rstrip("/") + if clean_url.endswith((".zarr.zip", ".zip")): + import fsspec + import zarr + from zarr.storage import ZipStore + + traffic = Traffic() + raw_f = fsspec.open(url, "rb", **storage_options).open() + tf = TrackingFile(raw_f, traffic) + store = ZipStore(tf, mode="r") + arr = zarr.open(store=store) + arr.traffic = traffic + return "Zarr (Zip)", arr + + # If the URL targets an HDF5 container without a dataset path, list available datasets + base_url, detected_dataset, hint = blosc2.remote_array.parse_container_url(url) + if hint == "hdf5" and detected_dataset is None: + available = blosc2.available_datasets(base_url, storage_options=storage_options) + raise ValueError( + f"HDF5 files require specifying the dataset path using '/dataset_name' or '::dataset_name' " + f"(e.g. {base_url}/d0/d1/a2 or {base_url}::d0/d1/a2). Available datasets: {available}" + ) + + arr = blosc2.open(url, lazy=True, storage_options=storage_options) + kind = arr.source.get("kind", "") + if kind == "hdf5": + label = "HDF5" + elif kind == "zarr": + label = "Zarr" + elif kind == "b2z": + label = "Blosc2 B2Z" + else: + label = "Blosc2" + return f"{label} (Lazy RemoteArray)", arr + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Open a remote array in S3 (.b2nd or .zarr) and print metadata and data.", + ) + parser.add_argument("url", help="Remote S3 URL (e.g. s3://blosc2/cube.b2nd or s3://blosc2/cube.zarr)") + parser.add_argument( + "--profile", + default=DEFAULT_PROFILE, + help=f"AWS CLI credential profile (default: '{DEFAULT_PROFILE}')", + ) + parser.add_argument( + "--endpoint-url", + default=DEFAULT_ENDPOINT_URL, + help=f"S3 endpoint URL (default: '{DEFAULT_ENDPOINT_URL}')", + ) + + args = parser.parse_args() + + print(f"Accessing: {args.url}") + t0 = time.perf_counter() + try: + fmt, arr = open_remote_array( + url=args.url, + profile=args.profile, + endpoint_url=args.endpoint_url, + ) + except Exception as exc: + print(f"Error opening remote array: {exc}", file=sys.stderr) + return 1 + t_open = time.perf_counter() - t0 + + def get_traffic_bytes() -> int | None: + traffic = getattr(arr, "traffic", None) + if traffic is None: + src = getattr(arr, "src", None) + traffic = getattr(src, "traffic", None) + if traffic is not None and hasattr(traffic, "nbytes"): + return int(traffic.nbytes) + return None + + b_open = get_traffic_bytes() + + print(f"\n[Format: {fmt}]") + if hasattr(arr, "info"): + print(arr.info, end="") + else: + print(f"{'shape':<12} : {arr.shape}") + print(f"{'dtype':<12} : {arr.dtype}") + chunks = getattr(arr, "chunks", None) + if chunks is not None: + print(f"{'chunks':<12} : {chunks}") + blocks = getattr(arr, "blocks", None) + if blocks is not None: + print(f"{'blocks':<12} : {blocks}") + meta = getattr(arr, "meta", None) + if meta: + print(f"{'meta':<12} : {dict(meta)}") + vlmeta = getattr(arr, "vlmeta", None) + if vlmeta is not None: + print(f"{'vlmeta':<12} : {dict(vlmeta) if vlmeta else {}}") + + print("\nSample slice data (1st fetch):") + t0 = time.perf_counter() + sample = get_sample_slice(arr) + t_fetch1 = time.perf_counter() - t0 + b_after_fetch1 = get_traffic_bytes() + b_fetch1 = (b_after_fetch1 - b_open) if (b_after_fetch1 is not None and b_open is not None) else None + print(sample) + + # Re-fetch the same slice to test caching behavior + t0 = time.perf_counter() + _ = get_sample_slice(arr) + t_fetch2 = time.perf_counter() - t0 + b_after_fetch2 = get_traffic_bytes() + b_fetch2 = ( + (b_after_fetch2 - b_after_fetch1) + if (b_after_fetch2 is not None and b_after_fetch1 is not None) + else None + ) + + print("\nTiming & Network Traffic:") + if b_open is not None: + print(f" - Metadata open : {t_open * 1000:7.1f} ms ({b_open / 1024:8.2f} KB transferred)") + else: + print(f" - Metadata open : {t_open * 1000:7.1f} ms") + + if b_fetch1 is not None: + print(f" - 1st slice fetch: {t_fetch1 * 1000:7.1f} ms ({b_fetch1 / 1024:8.2f} KB transferred)") + else: + print(f" - 1st slice fetch: {t_fetch1 * 1000:7.1f} ms") + + if b_fetch2 is not None: + tag = " (cache hit!)" if b_fetch2 == 0 else "" + print( + f" - 2nd slice fetch: {t_fetch2 * 1000:7.1f} ms ({b_fetch2 / 1024:8.2f} KB transferred){tag}" + ) + else: + print(f" - 2nd slice fetch: {t_fetch2 * 1000:7.1f} ms") + + if b_after_fetch2 is not None: + print(f" - Total network : {b_after_fetch2 / 1024:8.2f} KB transferred from S3") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/remote/store-browse.py b/examples/remote/store-browse.py new file mode 100644 index 000000000..2e6c9ed4d --- /dev/null +++ b/examples/remote/store-browse.py @@ -0,0 +1,33 @@ +"""List a remote hierarchy and optionally preview a leaf, with shared caching. + +Usage: python examples/remote/store-browse.py https://host/data.h5 --dataset group/a +Add --cache-dir ./remote-cache to retain discovery and payload across runs. +""" + +import argparse + +import blosc2 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("url", help="Remote B2Z, Zarr or HDF5 root/group URL") + parser.add_argument("--dataset", help="Array path relative to the selected group") + parser.add_argument("--cache-dir", help="Enable persistent DISK caching") + args = parser.parse_args() + with blosc2.RemoteStore(args.url, cache_dir=args.cache_dir, max_cache_bytes=64 << 20) as store: + for name in store: + print(name, store.get_info(name).kind) + if args.dataset: + with store[args.dataset] as array: + selection = tuple(slice(0, min(3, size)) for size in array.shape) + print(array[selection]) + # The store retains the leaf cache after the handle closes. + with store[args.dataset] as array: + print(array[selection]) + print("Retained payload bytes:", store.cache_bytes) + print("Manifest bytes:", store.metadata_bytes) + + +if __name__ == "__main__": + main() diff --git a/examples/vlmeta.py b/examples/vlmeta.py index af90f763a..91eb2d5c4 100644 --- a/examples/vlmeta.py +++ b/examples/vlmeta.py @@ -17,18 +17,18 @@ assert nchunks_ == (i + 1) # Initially the vlmeta is empty -print(len(schunk.vlmeta)) +print(len(schunk.attrs)) # Add a vlmeta -schunk.vlmeta["meta1"] = "first vlmetalayer" -print(schunk.vlmeta.getall()) +schunk.attrs["meta1"] = "first vlmetalayer" +print(schunk.attrs.getall()) # Update the vlmeta -schunk.vlmeta["meta1"] = "new vlmetalayer" -print(schunk.vlmeta.getall()) +schunk.attrs["meta1"] = "new vlmetalayer" +print(schunk.attrs.getall()) # Add another vlmeta -schunk.vlmeta["vlmeta2"] = "second vlmeta" +schunk.attrs["vlmeta2"] = "second vlmeta" # Check that it has been added -assert "vlmeta2" in schunk.vlmeta +assert "vlmeta2" in schunk.attrs # Delete a vlmeta -del schunk.vlmeta["vlmeta2"] -assert "vlmeta2" not in schunk.vlmeta +del schunk.attrs["vlmeta2"] +assert "vlmeta2" not in schunk.attrs diff --git a/plans/remote-proxy-v10.md b/plans/remote-proxy-v10.md new file mode 100644 index 000000000..dc3ce96c7 --- /dev/null +++ b/plans/remote-proxy-v10.md @@ -0,0 +1,353 @@ +# Remote proxy v10: native array reads inside remote B2Z hierarchies + +Status: implemented and validated (2026-09-08). Authorized B2Z sparse attachment remains explicitly +unsupported, as permitted by this plan; ordinary RemoteProxy caches and direct +Proxy persistence are supported. + +The implementation below is the delivered v10 scope. The remaining items in +Deferred work are not prerequisites for this version. + +## Implementation results + +- Added `B2ZNDSource` using a seekable view with bounded opening buffers for ZIP + discovery and bounded native Blosc2 range reads for the selected member. +- Integrated all three addressing forms, source descriptors, cache identity, + carrier reopening, and durable B2Z operand references for saved expressions. +- Added validation for member bounds, ZIP64, duplicate/encrypted/compressed + members, malformed headers, and unsupported object carriers. +- Updated the remote proxy API documentation and S3 example. +- Default suite: 9750 passed, 29 skipped. Subsequently added dependency-isolation + coverage passed with the focused B2Z suite (36 passed, 1 network test deselected). + Focused existing remote/Proxy regressions: 299 passed, 6 deselected. + Final combined B2Z/remote regressions after URL-parser edge-case checks: + 300 passed, 7 deselected. Ruff and whitespace checks passed. +- The exact S3 example succeeded against the remote archive: 8.85 KiB for opening + (2592.3 ms), 76.42 KiB for the first slice (921.0 ms), and zero bytes for the + cached repeat (0.9 ms). Values and native geometry match the supplied example. + These are individual observed timings, not performance guarantees. +- Tests needing localhost servers and the S3 check were run outside the network + sandbox. All Python commands used the `blosc2` conda environment. + +## Opening optimization measurements + +Three fresh-process S3 runs per version against `hierarchy.b2z::/d0/a3`: + +| Metric | Initial reader | Buffered reader | +|--------|---------------:|----------------:| +| Median opening time | 2268.9 ms | 1197.3 ms | +| Opening requests | 2 HEAD + 7 GET | 1 HEAD + 2 GET | +| Opening payload | 9067 bytes | 24576 bytes | +| First-slice payload | 78257 bytes | 78257 bytes | +| Repeated-slice payload | 0 bytes | 0 bytes | + +Opening is approximately 47% faster in these measurements. An 8 KiB tail and +16 KiB member prefix combine ZIP and native-frame header reads, while larger +directories, comments, and extra fields retain exact-read fallbacks. Buffers +are released after source construction; payload cache ownership is unchanged. +Source stamps now use the existing archive-info response instead of a second +identity lookup. This changes stamps from the initial reader, so older warm +caches may refetch once. Values, geometry, and cached-repeat behavior were +verified on every measurement. These timings do not promise a first-slice speedup. +Validation: 39 focused B2Z tests passed; the default suite passed with 9754 tests +and 29 skips. Ruff and whitespace checks passed. + +## Objective + +Open an external NDArray leaf inside an immutable remote `.b2z` archive as a +`RemoteProxy`, fetching only the archive metadata and Blosc2 byte ranges needed +for reads. Reuse native Blosc2 chunk/block reading and the existing cache policies. + +```python +arr = blosc2.open( + "s3://blosc2/hierarchy.b2z::/d0/a3", + lazy=True, + storage_options={ + "profile": "blosc2", + "endpoint_url": "https://s3.us-west-001.backblazeb2.com", + }, +) +values = arr[:10, 0, :5] +``` + +This extends the dataset addressing introduced in v9. It does not require +kerchunk, Zarr, a Caterva2 server, archive extraction, or conversion of source +chunks into a different storage format. + +## Scope and fixed decisions + +- Support read-only access to external NDArray leaves stored as `ZIP_STORED` + members in `.b2z` archives. This covers the default TreeStore layout. +- Require `assume_immutable=True`; reject mutable B2Z sources explicitly. +- Require an explicit dataset path. Do not silently choose the first array. +- Preserve native shape, chunks, blocks, dtype, and compression parameters. +- Reuse `ByteRangeNDSource` and the existing Proxy/RemoteProxy cache machinery, + including its current whole-chunk versus block-read decisions. +- Use Python's `zipfile` for archive directory parsing and ZIP64 support. + Do not write another ZIP directory parser. +- Use the existing optional fsspec dependency and protocol backends. Add no + mandatory dependencies and no C/Cython changes unless a demonstrated blocker + requires them. +- Preserve existing local store opening and standalone remote frame behavior. +- Full hierarchy browsing and embedded leaves are outside this version. + +## Evidence and reusable code + +`src/blosc2/dict_store.py` already writes external leaves as uncompressed ZIP +members containing self-contained Blosc2 frames. Its `member_window()` method +returns `(offset, length)` for a local external leaf. `_get_zip_offsets()` +calculates member data offsets using the local ZIP header, and +`_logical_key_from_relpath()` defines the external-member-to-logical-key mapping. +Reuse these rules without constructing a local DictStore for a remote archive. + +`src/blosc2/proxy_source.py` provides `ByteRangeNDSource`, which interprets a +frame using a `read_range(offset, size)` transport. `FsspecNDSource` supplies +the current fsspec transport, identity handling, and traffic accounting. + +`src/blosc2/core.py` provides `parse_container_url()`. Public remote opening +and option validation live in `src/blosc2/schunk.py`. Source descriptors, +reconstruction, identity, and sparse attachment live in +`src/blosc2/remote_proxy.py`. + +A read-only diagnostic against the local `hierarchy.b2z` found 12 external +`.b2nd` members and `embed.b2e`, all `ZIP_STORED`. The member `d0/a3.b2nd` +starts at byte 939145 and occupies 312995 bytes. A temporary +`ByteRangeNDSource` subclass translating reads into that member, wrapped in +the existing `Proxy`, reproduced the sample values with: + +| Operation | Bytes read | +|-----------|-----------:| +| Frame metadata | 8192 | +| First `[:10, 0, :5]` slice | 78257 | +| Repeated slice | 0 | + +These are local range-read measurements, excluding ZIP discovery. They are +evidence that the native reader can handle the member, not an S3 performance +result or an assertion that the remote archive is identical. + +## Public API and addressing + +Support the same three addressing forms as v9: + +```python +blosc2.open("s3://bucket/hierarchy.b2z::/d0/a3", lazy=True) +blosc2.open("s3://bucket/hierarchy.b2z/d0/a3", lazy=True) +blosc2.open("s3://bucket/hierarchy.b2z", dataset="d0/a3", lazy=True) + +blosc2.RemoteProxy( + "s3://bucket/archive", + source_format="b2z", + dataset="d0/a3", + cache_policy=blosc2.CachePolicy.MEMORY, +) +``` + +Add `"b2z"` as an explicit source format. Infer it from a `.b2z` URL path +component, including nested dataset syntax; inspect parsed URL paths rather +than query-string text. Explicit source-format selection retains precedence. +Preserve fsspec protocol-chain handling when interpreting `::`. + +Normalize optional leading/trailing dataset slashes to the same canonical +identity. Reject conflicting URL and keyword dataset specifications. Use the +existing TreeStore key rules; reject invalid traversal or malformed paths +rather than normalizing them into another leaf. A logical key is not a raw ZIP +filename: resolve `d0/a3` to the canonical external member `d0/a3.b2nd`. + +Missing dataset, missing leaf, group selection, and unsupported leaf kinds +must produce actionable errors. A known embedded-only leaf should explain the +scope limitation if cheaply identifiable; do not add remote EmbedStore reading +just to improve an error message. Otherwise report that no supported external +NDArray exists at the selected path. + +Dataset selection through this remote path requires `lazy=True`. Existing +non-lazy whole-archive localization, where supported, remains unchanged. + +## Archive discovery and bounded frame reads + +Implement a small B2Z source adapter around the existing native frame reader. +Prefer a focused adapter in `src/blosc2/b2z_source.py`, reusing fsspec transport +code where practical. Avoid a general archive framework or unrelated reader +refactoring. The adapter must also accept a supplied filesystem internally for +deterministic tests and authorized transport attachment. + +### Opening + +1. Resolve the filesystem and archive path with the existing fsspec conventions. +2. Open a seekable read-only view for `zipfile.ZipFile`, with buffering explicitly + controlled so a seek does not cause a large default read-ahead or full download. +3. Read the ZIP directory and locate the selected canonical array member. + Central-directory work scales with archive member count; do not open every + member or construct the complete TreeStore. +4. Reject duplicate matches, encrypted members, and compression methods other + than `ZIP_STORED`. Check the selected local header and its consistency with + the directory before using its data offset. Reuse the local-header offset + calculation, with explicit short-read and signature validation. +5. Validate the member window against the archive size. Account for ZIP64 using + `zipfile`'s decoded metadata; stored compressed and uncompressed sizes must + agree. Reject corrupt or impossible windows before frame construction. +6. Initialize the native frame reader over this member window. Validate that + it holds a supported NDArray and that its declared frame length fits inside + the member. Do not fetch the selected array in full to inspect it. + +Close discovery handles after resolving the member; payload reads should use +stateless range requests suitable for the existing concurrent fetch scheduler. + +### Reading + +Translate frame-relative reads as: + +```text +archive_start = member_offset + frame_offset +``` + +Bound each read by the member length. Preserve normal end-of-file short-read +semantics for speculative header/tail reads, while letting existing frame +validation reject truncated required data. Reject negative or invalid ranges. +No frame operation may read an adjacent member by running past its window. + +Reuse `get_chunk()`, block layout parsing, index handling, and fetch scheduling +from the native source. Do not decompress/recompress every chunk through the +HDF5/Zarr conversion path. Existing block assembly may still apply when the +cache fetches partial chunks. + +Traffic accounting must include directory discovery, local-header reads, frame +metadata, indexes, and payloads exactly once. Measure bytes at the transport +boundary when buffering is present; counting bytes returned by a buffered file +can conceal read-ahead. ZIP metadata reads may overlap payload bytes in a small +archive, but opening must not intentionally materialize array members. + +## Identity, cache policies, and persistence + +Use a distinct source descriptor: + +```json +{ + "kind": "b2z", + "version": 1, + "urlpath": "s3://bucket/hierarchy.b2z", + "dataset": "d0/a3", + "assume_immutable": true +} +``` + +The canonical identity includes both archive URL and dataset. Two leaves with +identical geometry must never collide in automatic cache paths or source-spec +comparisons. Keep credentials and live storage options out of persisted metadata. +Apply the existing persistable-URL validation and fail-closed descriptor checks. + +Resolve the member window afresh on reopening in the first implementation. +Do not persist an unchecked byte offset as the authority for future reads. +Combine archive identity, selected member/window, and native frame interpretation +as needed for a stable source stamp using existing identity conventions. Retain +geometry validation on reopening. Do not claim to detect payload-only replacement +under the immutable contract; replacing the archive at the same identity can +serve stale data and requires replacing its cache. + +Update all relevant paths together: + +- Source opening, format validation, dataset normalization, and `urlpath`. +- Descriptor validation, `_source_identity()`, stamps, and payload reconstruction. +- NONE, MEMORY, bounded/unbounded DISK, and automatic cache locations. +- `save()`, `to_cframe()`, carrier reopening, and sparse-cache reconstruction. +- Direct persistent `Proxy` metadata/reconstruction if the adapter is exposed + for direct use; never serialize it as a standalone frame at the archive URL. + +Reuse existing retention and eviction behavior. Preserve storage options during +in-process reconstruction; fresh processes resolve their own credentials. +Repeated cache hits under the immutable contract must perform no remote payload +or metadata reads. Reopening may reread ZIP and frame metadata; eliminating +those reads is not a requirement for v10. + +## Authorized transports and Caterva2 boundary + +Actual Caterva2 federation remains a separate integration task. Within +Python-Blosc2, either support B2Z explicitly through the existing authorized +sparse-attachment interface or reject it explicitly until that path is complete. +Do not let a subclass pass a broad FsspecNDSource check while losing its dataset +identity or member bounds. + +If supported, validate the supplied concrete source against the full descriptor +before cache access, including hits. Directory discovery, header reads, payload +reads, and any reconstruction must retain the supplied filesystem and must not +fall back to unrestricted URL opening. Archive members are byte windows, not +external references to follow. Preserve existing server transport restrictions. + +## Implementation sequence and validation + +### 1. Source adapter + +Create small temporary TreeStore archives using existing test conventions. +Exercise a selected external NDArray through `Proxy`, using counted local or +memory-backed fsspec transport. Confirm native values and geometry for nested +keys, multiple chunks, edge chunks, and representative fixed-size dtypes. +Check scalar and empty arrays where the native reader supports them, preserving +clear errors for any existing native limitations. + +Verify selected-member bounds, malformed/truncated archives, missing members, +duplicate selected names, compressed/encrypted members, and non-NDArray leaves. +Cover ZIP64 local-header behavior without allocating a multi-gigabyte fixture. +Ensure concurrent reads do not share an unsafe seek position. + +### 2. Public dispatch and cache integration + +Update `parse_container_url()`, `blosc2.open()`, and `RemoteProxy` together. +Test all addressing forms, leading slashes, explicit suffix-free format, +conflicting dataset specifications, missing datasets, and mutable-source +rejection. Preserve existing HDF5/Zarr/standalone Blosc2 dispatch tests. + +Assert metadata-only opening with transport read logs on an archive large enough +to distinguish directory/header reads from downloading members. Verify correct +slices, zero reads on repeated hits, absent-only fetches on overlapping slices, +and refetch after eviction. Exercise all cache policies using existing tests +and helpers rather than duplicating their complete suites. + +### 3. Persistence and attachment + +Test cold/warm carrier reopening, memory exports, bounded disk caches, and +source identity isolation between same-shaped leaves. Verify credential +exclusion, geometry mismatch rejection, materialization, and a simple lazy +expression round trip. Test direct persistent Proxy use if supported. + +For authorized attachment, use a supplied fake filesystem and make unrestricted +opening raise. Exercise directory discovery, misses, hits, and source-descriptor +mismatch. If attachment is deferred, test its explicit rejection instead. + +### 4. Documentation and example + +Extend the existing remote proxy documentation and `examples/remote/s3-access.py` +with `.b2z` dataset selection. Keep the metadata, cold/warm slice, and traffic +reporting comparable across formats. Document immutable archives, external +NDArray-only scope, directory-discovery cost, and unsupported embedded leaves. + +Run the requested S3 example when network access and credentials are available. +Check sample values against the known data and compare with the HDF5 example; +report measured metadata bytes, first-read bytes/latency, and warm-cache behavior. +Do not assume the local archive's byte offsets match the remote copy. Keep public +S3 tests marked `network` and outside the default suite. + +### 5. Final checks + +Use the `blosc2` conda environment for all Python, tests, and build commands. +Run focused B2Z, fsspec, Proxy, RemoteProxy, and URL parsing tests, followed by the +default suite and repository lint checks. Verify unrelated local use and native +remote B2ND reads still work without Zarr, kerchunk, or h5py. Record network or +optional-dependency checks that could not run. + +## Completion criteria + +- [x] The requested `.b2z::/d0/a3` example opens as a RemoteProxy and returns correct + values without downloading or extracting the archive in full. +- [x] Native frame reads remain within the selected member, preserving native + geometry and the existing chunk/block fetch behavior. +- [x] Opening reads archive/frame metadata; warm slice hits perform zero remote reads. +- [x] Traffic counters include discovery and payload transport without double counting. +- [x] Cache policies, exports, and reopening retain the selected dataset identity. +- [x] Unsupported representations fail clearly; authorized attachment either retains + its transport and descriptor constraints or rejects B2Z explicitly. +- [x] Existing local stores and remote Blosc2/HDF5/Zarr behavior remain intact. + +## Deferred work + +Hierarchy browsing/discovery APIs, group metadata, remote `.b2d` hierarchies, +embedded leaves inside `embed.b2e`, SChunk/ObjectArray/BatchArray/CTable leaves, +Caterva2 reference leaves, compressed ZIP members, mutable archives, archive +writing, persisted ZIP indexes, and Caterva2 server-side federation. diff --git a/plans/remote-proxy-v11.md b/plans/remote-proxy-v11.md new file mode 100644 index 000000000..16ab0fe88 --- /dev/null +++ b/plans/remote-proxy-v11.md @@ -0,0 +1,117 @@ +# RemoteProxy v11: Caterva2 user attributes + +## Goal + +Expose consistent user attributes for arrays served through Caterva2, including +ordinary Blosc2 arrays, B2Z leaves, HDF5 leaves, and saved RemoteProxy arrays. +Reuse `/api/info` so reading attributes requires no additional endpoint or request. + +The preceding Python-Blosc2 change adds `RemoteProxy.attrs` as a read-only alias +for `RemoteProxy.vlmeta` and filters Kerchunk's `_ARRAY_DIMENSIONS` from +`HDF5NDSource.vlmeta`. + +## Current behavior and gap + +Caterva2 already exposes variable metadata through `schunk.vlmeta` in array +metadata responses and through `File.vlmeta` in its Python client. HDF5 adapters +already copy dataset attributes into their backing Blosc2 array's variable +metadata. The web metadata panel displays this mapping directly. + +Saved RemoteProxy arrays are different: `services/server.py:get_info` replaces +their variable metadata with only the `b2o` descriptor to avoid exposing binary +cache bookkeeping. This also discards the saved user attributes, which reside +under `_b2o_user_vlmeta` in the carrier. Python-Blosc2 already provides +`blosc2.b2objects.read_b2object_user_vlmeta()` to retrieve them. + +## Proposed changes + +### 1. Add public attributes to the existing response + +Add an `attrs` mapping to the relevant `/api/info` metadata models. Populate it +in Caterva2's shared metadata builder (`services/srv_utils.py:read_metadata`) +so standalone arrays and container leaves use the same behavior: + +- Ordinary Blosc2 arrays: expose user variable metadata. +- HDF5 leaves and legacy HDF5 proxies: expose dataset attributes, excluding + adapter bookkeeping such as `_ftype` and `_dsetname`. +- Saved RemoteProxy arrays: use `read_b2object_user_vlmeta()` on the raw carrier, + without resolving or fetching the remote array. + +Preserve the existing `schunk.vlmeta` response contract, including control +information used by existing clients. Keep public attributes separate from +proxy descriptors, cache bookkeeping, and fill protocol fields. Identify +internal fields explicitly; do not remove every underscore-prefixed user key. + +Keep the new field optional during compatibility handling: distinguish an +absent field from an explicitly empty mapping. Ensure peer responses and their +model conversion preserve the field. + +### 2. Expose attributes in the clients + +Add `.attrs` to Caterva2's `File` class (inherited by its dataset objects) and +Python-Blosc2's `C2Array`. Prefer the new response field; fall back to the +existing variable metadata when talking to an older server. Reuse each client's +existing metadata cache and refresh behavior. + +Update `RemoteProxy`'s Caterva2 metadata path to consume these public attributes. +Its `.attrs` and `.vlmeta` must continue to return the same read-only mapping. +Preserve existing `.vlmeta` compatibility in the lower-level clients, since +their callers may depend on protocol fields there. + +### 3. Update the web metadata panel and documentation + +Have `services/templates/includes/info_metadata.html` display `attrs`, with a +fallback for older metadata responses. Label the section "Attributes" and avoid +showing proxy descriptors or cache bookkeeping as user attributes. + +Document the new response field, client access, read-only remote behavior, and +older-server fallback. Explain that saved proxy attributes reflect the metadata +stored in the carrier; this change does not introduce remote metadata refreshes +on the server. + +### 4. Verify the complete route + +Extend existing tests rather than introducing a new test harness: + +- `/api/info` preserves scalar and nested user attributes for ordinary Blosc2 + arrays, B2Z leaves, HDF5 leaves, and saved RemoteProxy arrays. +- Saved proxies expose their stored user attributes without resolving their + source or exposing cache internals. +- Existing descriptor and fill protocol metadata remain available to callers + that use the existing response fields. +- Caterva2 `.attrs`, C2Array `.attrs`, and RemoteProxy `.attrs` return the + expected attributes; RemoteProxy `.attrs` and `.vlmeta` share their cache. +- Missing `attrs` falls back for older servers; explicit empty `attrs` stays + empty. Peer forwarding and the web metadata panel preserve the result. + +Run the affected API, container, HDF5, remote-proxy, and client tests in the +repositories' prescribed environments, together with their lint checks. + +## Scope + +Implementation spans `/Users/faltet/ironArray/caterva2` and +`/Users/faltet/blosc/python-blosc2`. The implementation is complete in both +working trees. + +Attribute writes, a separate metadata endpoint, new serialization formats, and +server-side refresh of saved remote attributes are outside this change. + +## Implementation and validation + +Implemented the public `attrs` field, both client properties, RemoteProxy's +Caterva2 integration, and the web metadata panel. The file-less HDF5 leaf +adapter also needed to copy attributes using the existing HDF5 conversion; +its previous implementation copied only array geometry. + +Validation in the `blosc2` conda environment: + +- Python-Blosc2 RemoteProxy, open-C2Array, and HDF5 source modules: 121 passed. +- Focused Caterva2 metadata, client, HDF5, B2Z, table, saved-proxy, panel, and + peer checks: 21 passed. +- Broader Caterva2 API, HDF5, TreeStore, CTable, and RemoteProxy modules, plus + attribute checks: 296 passed, 110 skipped, two failures. +- Both failures reproduced against an unchanged Caterva2 HEAD snapshot: + `test_remote_proxy_download_can_omit_cache[CachePolicy.DISK]` encounters the + disabled-resolution policy on warm download, and + `test_dir_named_like_container` encounters `dataset requires lazy=True`. +- Ruff lint, formatting, and diff whitespace checks pass for the changed files. diff --git a/plans/remote-proxy-v12.md b/plans/remote-proxy-v12.md new file mode 100644 index 000000000..2bebb9066 --- /dev/null +++ b/plans/remote-proxy-v12.md @@ -0,0 +1,353 @@ +# RemoteProxy v12: remote hierarchy browsing in b2view + +Status: implemented and validated on 2026-09-09. + +## Goal + +Browse a remote container from its root, expand groups, inspect attributes, and +preview selected arrays without downloading the complete container first: + +```sh +b2view --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com s3://blosc2/hierarchy.b2z +b2view --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com s3://blosc2/hierarchy.zarr +b2view --profile blosc2 --endpoint-url https://s3.us-west-001.backblazeb2.com s3://blosc2/hierarchy.h5 +``` + +Support a URL pointing at a subgroup as well as the container root. Preserve the +existing standalone presentation when the URL selects an array: no tree panel, +no redundant internal root path, and the original source in the header. + +All three formats are feasible using existing dependencies and leaf readers. +B2Z needs archive discovery; Zarr needs group discovery; HDF5 needs its existing +Kerchunk references retained and reused across dataset selections. The main +shared work is connecting discovery to the browser without opening every leaf. + +## Current behavior and reusable pieces + +- `b2view/model.py:StoreBrowser` calls `blosc2.open()` and treats only a + `TreeStore` as a hierarchy. Its previews already handle remote array objects. +- `StoreBrowser.list_children()` queries descendants and opens terminal nodes + to classify them. That is unsuitable for remote discovery and also cannot + use the absence of descendants to distinguish an empty group from an array. +- Group metadata currently reads the store's attributes, rather than necessarily + the selected group's attributes. A remote adapter must return attributes for + the actual selected node. +- `b2view/app.py` already displays trees, expands nodes on demand, navigates to + an initial path, and hides the tree for standalone objects. Opening, listing, + and panel updates currently include synchronous calls on the UI thread. +- `core.py:parse_container_url()` handles container and dataset addressing. + Reuse it, including its distinction between dataset syntax and chained fsspec + URLs; do not add another suffix parser in the application. +- A whole remote B2Z currently bypasses lazy opening in `StoreBrowser`. Without + a local cache, the generic fsspec path in `schunk.py` reads the entire object + and passes ZIP bytes to `from_cframe()`. This is the wrong opening path. +- `b2z_source.py:B2ZNDSource` already reads the ZIP directory through a bounded, + seekable range view and opens external NDArray members lazily. It validates + member windows and uses native Blosc2 chunk/block reads. It requires a leaf + path and rejects embedded leaves and object carriers. +- `dict_store.py` defines canonical external member names and logical key + mapping. TreeStore also uses `embed.b2e` for information that cannot be + reconstructed from external member names alone. +- `zarr_source.py:ZarrNDSource` already opens an fsspec-backed Zarr array and + exposes its attributes. It explicitly rejects groups. +- `hdf5_source.py:available_datasets()` and `HDF5NDSource` already translate HDF5 + into Kerchunk references. The source accepts an existing reference dictionary, + recognizes `.zgroup` and `.zarray`, and opens arrays through a reference store. + Repeating translation for each selected dataset would waste substantial work. +- The recent b2view Zarr fix initializes the Numcodecs Blosc mutex before + Textual captures stderr. Group-root startup must preserve this behavior, and + HDF5 leaves using that decoder need the same fresh-process check. + +## Scope and decisions + +1. Deliver read-only b2view browsing first. Keep `RemoteProxy` an array operand; + do not make it represent groups or implement mutable remote TreeStore. +2. Use a small internal hierarchy adapter, shared by the three remote formats. + Do not subclass `TreeStore`: its local storage and mutation contracts do not + describe these sources. Do not introduce a plugin registry or public store + API for this release. +3. Keep existing `blosc2.open(..., lazy=True, dataset=...)` array behavior. + b2view explicitly selects the hierarchy adapter for container/group targets. + The erroneous generic B2Z root path also needs a shared dispatch guard so + other callers receive a useful error instead of a full download followed by + a frame-decoding failure. Existing explicit `cache_dir` localization remains + available; this plan does not make it b2view's default. +4. Opening and expanding a hierarchy may read metadata and list object keys. + It must not materialize every dataset or fetch array payloads merely to + classify children. Metadata cost can scale with container size; it is not + necessarily a constant number of bytes or requests. +5. Preview support matches the existing leaf readers. Unsupported datasets + remain visible with an explanatory message; one unsupported leaf must not + prevent browsing its siblings. +6. Keep remote sources immutable for a browsing session. Explicit refresh + rebuilds discovery state and invalidates cached node metadata and leaf + objects. Live mutation detection and persistent hierarchy caches are deferred. +7. Use the existing optional fsspec, Zarr, and HDF5/Kerchunk dependency groups. + B2Z browsing must work without Zarr, h5py, or Kerchunk installed. + +## Browser integration + +Introduce one internal module, tentatively `b2view/hierarchy.py`, for discovery +and format adapters. Keep format-specific range/translation helpers in their +existing source modules when also used by array readers. + +The browser needs only these operations: + +- Resolve whether the requested node is a group, supported array, or unsupported + object, independently of whether it has children. +- List direct children with their logical path, display name, kind, and expansion + capability. Reuse `NodeInfo` where practical; do not load data for classification. +- Read one node's metadata and user attributes. +- Open a selected supported leaf using the existing array source/proxy machinery. +- Close resources and rebuild the session on refresh. + +Keep these operations internal and concrete; a base class is unnecessary unless +implementation reveals meaningful shared behavior. `StoreBrowser` remains the +UI-facing adapter and routes hierarchy operations through this small surface. +Audit all uses of `self.store` and `is_tree`, especially `_get_object()`, `kind()`, +`get_info()`, `list_children()`, and `close()`. Existing local TreeStore and CTable +query behavior must remain intact. + +Separate node kind from expansion state. An empty group is still a group and +shows its own attributes. For a remote group whose children have not been +listed, allow expansion without recursively probing its descendants. Omit the +descendant count when obtaining it would require walking the whole hierarchy; +show direct-child counts after listing, rather than inventing a zero count. + +Use browser-relative paths for subtree views: `/` is the requested group, and +leaf resolution joins that view to the actual container path. The header keeps +the supplied source URL; the metadata path identifies the selected node within +the view. Handle root, trailing slash, subgroup, and direct-array URLs explicitly. +Preserve query strings, credentials, and storage options when resolving leaves; +never build child URLs by appending strings after a URL query or fragment. + +Retain discovery metadata for the session. Reuse the existing current leaf proxy +on repeated panel reads; avoid retaining an unbounded number of payload caches +while traversing a large tree. Start with the selected leaf and release it when +selection changes, unless existing cache ownership already provides a bound. + +## B2Z discovery + +### First milestone: external arrays and inferred groups + +Extract only the archive-opening/index functionality needed by both discovery +and `B2ZNDSource`. Preserve bounded range reads, ZIP64 support, malformed-header +checks, member validation, traffic accounting, and existing leaf cache identity. +Use `zipfile`, not a new ZIP parser. Share archive identity and directory results +within the browser session rather than rediscovering them on every click. + +Map canonical member names to logical keys using DictStore's existing rules. +Construct parent groups from the paths, hide storage implementation members, +and list direct children without opening the corresponding frames. Resolve a +selected external NDArray through the existing B2Z leaf reader. Validate the +selected frame before preview; a `.b2nd` suffix is not proof of a plain NDArray. + +Reject ambiguous duplicate logical keys, unsafe paths, and leaf/group collisions +with an actionable archive error. Preserve the existing reader's rejection of +encrypted or ZIP-compressed array members. Surface unsupported ordinary object +types as unavailable leaves where they can be identified reliably. + +This milestone is sufficient for the supplied hierarchy if it contains ordinary +external arrays. Verify its actual directory before claiming that coverage. + +### Complete the hierarchy's metadata + +Inspect the actual `embed.b2e` layout and TreeStore subtree metadata conventions +before implementation. External filenames alone do not establish empty groups, +embedded leaves, group attributes, or logical CTable object boundaries. + +Reuse EmbedStore decoding to read the index and group metadata required for +discovery. Determine whether these can be read through a bounded member window +without loading the entire embedded payload. Do not assume `embed.b2e` is small +or silently download it wholesale. Record the measured access pattern in tests. + +Show explicit empty groups and the correct root/subgroup attributes when the +stored format records them. If the format does not persist empty groups, document +that limitation rather than synthesizing them. Identify embedded objects and +CTable roots from their metadata and show an unsupported-preview message; do not +expose CTable column carriers as if they were an ordinary user group. + +If bounded embedded-index access needs a larger storage-layer change, deliver the +external-array milestone with a clear partial-discovery indication and document +the missing metadata. Do not call that full B2Z hierarchy support. Remote embedded +payload and CTable preview support are separate follow-up work. + +## Zarr discovery + +Open the requested node through Zarr's existing read-only fsspec store path and +distinguish arrays from groups using node metadata. Reuse that store and its +filesystem across navigation; keep leaf reads in `ZarrNDSource` and the existing +proxy implementation. + +Use Zarr's supported group/member APIs for immediate children and group attributes. +Verify the exact installed API and supported dependency versions during +implementation. Support both Zarr v2 and v3 storage layouts already covered by +the array reader. Use consolidated metadata when available and supported; fall +back to normal metadata discovery without requiring users to consolidate stores. + +Avoid recursive enumeration of chunk objects. An unconsolidated remote store may +require LIST requests and metadata requests for child nodes, and some backends +cannot list at all. Report missing listing capability/permission clearly. Direct +array URLs should keep working when the user can read an array but cannot list +its parent. Preserve empty groups and per-group attributes. + +List datasets with unsupported dtypes/codecs, but defer decoding to selection and +show the source reader's specific limitation. Do not create another Zarr decoder +or promise finer data-fetch granularity than the current reader provides. + +## HDF5 discovery + +Translate the file to Kerchunk references once per browsing session, reusing the +existing translation and filter-registration path. Factor the shared operation +out of `available_datasets()` and `_load_or_scan_refs()` only as needed; do not +add an independent HDF5 traversal implementation inside b2view. + +Build the hierarchy from `.zgroup`, `.zarray`, and `.zattrs` entries, including +explicit empty groups. Reuse those references when opening the selected dataset +through `HDF5NDSource` or the existing proxy path that accepts `refs`. Passing a +fresh URL alone must not trigger another translation for each leaf. Preserve +dataset identity and original source information when using the shared refs. + +Display root and group attributes from their reference metadata, filtering only +known adapter bookkeeping, consistently with existing leaf attributes. Measure +the initial scan: HDF5 translation can visit metadata across the file, enumerate +many chunk references, and inline some small values. Promise no full-file +localization, not zero payload bytes or metadata cost independent of file size. + +Check how the installed translator handles unsupported datasets, hard-link +aliases, soft/external links, and cycles. Do not recursively follow arbitrary +links or contact additional external sources as part of listing. Prefer visible +unsupported nodes where the translator supplies them. If translation omits an +object or aborts the whole scan, expose/document that limitation; isolating +unsupported siblings is a completion requirement for the supported test matrix. +Do not describe the resulting view as covering every HDF5 object type. + +## UI responsiveness, failures, and lifecycle + +Run remote opening, expansion, metadata loading, and array previews through +Textual workers, following existing worker patterns in the app. Keep local-only +operations simple. Capture selection/slice state before dispatch and apply results +on the UI thread only if they still match the active request and browser session. + +Provide loading state for startup and expansion, keep navigation and quit usable, +and make failed group listings retryable. Do not add paths to `loaded_paths` until +listing succeeds. A stale worker must not repaint an old selection or repopulate +a tree after refresh. Cancellation may not interrupt an underlying blocking read; +discard stale results and close their resources after that read completes. + +Refresh must replace the discovery session, clear loaded paths, discard old leaf +caches, and attempt to restore the selected logical path, falling back to the +view root if it vanished. Ensure normal shutdown, failed startup, and refresh +close owned handles without closing a filesystem still in use by another worker. +Keep credentials in runtime storage options and out of errors or serialized state. + +## Implementation order and checkpoints + +1. **Reproduce and characterize.** Add a small valid in-memory B2Z root case that + demonstrates the bad dispatch, and inspect representative fixtures for all + three formats. Confirm embedded metadata and HDF5 translation behavior. +2. **B2Z external-array vertical slice.** Implement shared archive discovery, + minimal browser routing, root/subgroup navigation, and selected-array preview. + Guard the generic non-lazy B2Z path and preserve explicit local caching. +3. **Finish browser behavior.** Add remote workers, stale-result handling, refresh, + failures, correct node attributes, and empty-group behavior. Complete the B2Z + metadata milestone or explicitly mark its bounded-index blocker. +4. **Zarr groups.** Add discovery through native group APIs and reuse leaf readers. + Exercise v2/v3, consolidated/unconsolidated metadata, and restricted listing. +5. **HDF5 groups.** Reuse one translation and reference dictionary across selections; + verify attributes, empty groups, supported filters, and unsupported objects. +6. **Validate and document.** Run the targeted suite and real remote commands, + record traffic and startup observations, and update user-facing examples. + +The earlier one-to-two-day estimate applies only to the narrow B2Z external-array +browser. It does not cover all three formats, complete embedded metadata support, +or the UI lifecycle work above. Re-estimate after the first checkpoint resolves +the embedded-index and translator limitations. + +## Validation + +Extend existing pytest modules and use local temporary containers plus fsspec's +memory filesystem for deterministic checks. Use a counting filesystem/store to +test actual reads, not merely calls to adapter methods. + +- Root and subgroup discovery returns stable direct children with correct kinds; + empty groups remain groups. Selected group attributes differ from root attrs. +- Equivalent small B2Z, Zarr, and HDF5 arrays produce identical bounded previews + through root navigation and direct leaf URLs, including slicing and paging. +- B2Z startup does not request the complete archive or read all array frames. + Selecting one array does not fetch sibling payloads. Cover large ZIP directories, + malformed/duplicate entries, object carriers, and embedded metadata boundaries. +- Zarr discovery does not decode chunks; cover v2/v3, consolidated metadata, + unconsolidated metadata, listing failure, empty groups, and unsupported leaves. +- HDF5 translation runs once per session, not per leaf; test multiple selections, + group attributes, empty groups, unsupported/link cases, and refresh translation. +- Root-open dispatch errors occur before a bulk download. Explicit cached B2Z + opening and existing direct remote leaf opening retain their behavior. +- Fresh-process TUI tests cover compressed Zarr and HDF5 leaves reached from a + group root, preserving the decoder-mutex regression check. +- Headless TUI tests verify tree visibility, keyboard focus, expansion, initial + path navigation, selected path display, refresh, quit, and retry after failure. + Use controlled slow reads to verify that stale results cannot repaint a newer + selection or session and that shutdown does not race resource cleanup. +- Existing local TreeStore, standalone NDArray/CTable, and remote leaf tests pass. + Verify optional dependency isolation so B2Z does not acquire Zarr/HDF5 imports. + +Run Python and all tests in the `blosc2` conda environment. A starting focused run: + +```sh +conda run --no-capture-output -n blosc2 pytest tests/b2view tests/test_b2z_source.py tests/test_zarr_source.py tests/test_hdf5_source.py -m 'not network and not heavy' -q +``` + +Include new discovery tests and affected generic-open/remote-proxy tests in that +run. Explicitly include TUI markers, which the default configuration excludes. +Run Ruff and whitespace checks on the changed files; broaden testing when shared +open/source code changes warrant it. + +Finally run the three commands in Goal against available remote fixtures with +the supplied profile/endpoint. For each, verify root, nested group, array values, +attributes, refresh, and direct leaf behavior. Record first-open metadata bytes, +request counts, first-preview bytes, and repeated-preview behavior separately. +Keep credentials out of fixtures and logs. Do not claim support based only on a +mocked tree or one successful array read. + +## Documentation and completion criteria + +Update `doc/guides/b2view.rst` with remote root/subgroup examples, dependency +requirements, listing permissions, read-only behavior, and format limitations. +Clarify that b2view's hierarchy adapter does not change the array-only contract +of lazy `blosc2.open()` or introduce a persisted RemoteProxy hierarchy descriptor. + +The full v12 plan is complete when all three supported hierarchy views navigate +correctly, selected arrays reuse existing lazy readers, metadata and unsupported +nodes are represented honestly, UI operations remain responsive, and the tests +above pass. A delivered B2Z external-array milestone should be labeled as such +until the remaining metadata work is complete. + +Deferred: remote writes, full remote TreeStore/CTable semantics, embedded B2Z +payload previews, cross-container links, recursive search, persistent hierarchy +indexes, live watching, and support for additional codecs/dtypes beyond the leaf +readers. These are not prerequisites for the three-format read-only browser. + +## Implementation record + +The remote fixtures in the goal were exercised through the headless b2view CLI +from the root and a subgroup, and direct array URLs were exercised through +``StoreBrowser``. Nested navigation, attributes, array values, refresh, and quit +passed for B2Z, Zarr, and HDF5. The measured root opens were: + +| Format | Initial metadata bytes | Initial S3 operations | First preview bytes | Repeated preview bytes | +| --- | ---: | --- | ---: | ---: | +| B2Z | 8,192 | 1 HEAD, 1 GET | 32,225 | 0 | +| Zarr | 132 | 7 HEAD, 5 LIST, 2 GET | 12,420 | 0 | +| HDF5 | 52,976 | 1 HEAD, 54 GET | 13,662 | 0 | + +Zarr discovery grew to 6,515 bytes after expanding through ``/d0/d1``. HDF5 +translation remained at 52,976 bytes through hierarchy expansion because its +references were built once at session start. The request counts reflect the +installed s3fs/Zarr/Kerchunk versions and the fixture's current metadata layout; +they are observations rather than API guarantees. + +B2Z embedded arrays and remote CTable previews remain unavailable as planned. +Embedded group attributes are read with bounded native chunk access; layouts +whose metadata lives in native chunks larger than 1 MiB show an explicit partial +metadata notice. Soft and external HDF5 links and group cycles are not followed. diff --git a/plans/remote-proxy-v13.md b/plans/remote-proxy-v13.md new file mode 100644 index 000000000..3b30bfb66 --- /dev/null +++ b/plans/remote-proxy-v13.md @@ -0,0 +1,444 @@ +# RemoteArray and RemoteStore v13: shared caching and public hierarchies + +Status: implemented and locally validated, 2026-09-10. Steps 1–6 complete. +RemoteStore supports NONE, shared MEMORY and DISK caching. Windows execution +remains subject to the existing CI matrix; see validation limits below. + +## Implementation progress + +- Step 1: renamed the class/module to `RemoteArray`/`remote_array`, the public + export, maintained references, tests and reference page. Persisted object tags + now use `remote_array`, including carrier dispatch and warm-cache validation. + No compatibility alias was added. Existing Proxy tutorial filenames remain + unchanged because they describe the original Proxy API. +- Validation in the `blosc2` conda environment: 497 focused tests passed across + RemoteArray, Proxy, expressions, object serialization, B2Z/Zarr/HDF5/fsspec, + browser model and Caterva2 blocks. Local HTTP/mock S3 tests initially hit + sandbox socket restrictions; all affected tests passed outside that restriction. + Ruff check and format check passed for all 21 changed Python files; + `git diff --check` passed. +- Additional core/ndarray/schunk doctests: 72 passed, 2 skipped, 48 failed. + Failures include unchanged examples with ambiguous array truth values and + an incomplete `if` block. These are outside the rename milestone; the full + default suite, marked network tests and Textual UI tests have not been run. +- Step 2: added public `RemoteStore` and `RemoteNode`, with immediate-child + iteration, relative-path lookup, read-only attributes, unsupported-node + diagnostics, source descriptors and shared discovery traffic. B2Z archives, + HDF5 reference maps and Zarr stores/readers are reused across leaves and aliases. + Enumeration constructs no leaf readers or payload caches. B2Z attachment checks + archive/source identity; the existing sparse-attachment restriction remains. +- Discovery now lives in `remote_store.py`. Step 5 removes the temporary + `b2view/hierarchy.py` presentation adapter. Independent store/array handles own resource lifetime: + closing a parent leaves children usable, and the last close or garbage collection + closes owned archive/store wrappers. Step 4 adds private filesystem ownership + and explicit HTTP/S3 session cleanup. +- Step-2 validation in `blosc2`: 437 focused array/source/serialization/browser + tests passed, with 2 Zarr-specific cases skipped for other formats. Another + 23 Textual UI tests passed. Coverage includes Zarr v2/v3, one HDF5 translation, + no-cache repeat reads, expressions, standalone exports and independent close/GC. + Ruff check/format passed for all 8 affected Python files; whitespace checks passed. + No new live-network benchmark or full default-suite run was performed for step 2. +- Step 3: RemoteStore defaults to MEMORY with one 256 MiB allowance and accepts + a positive `max_cache_bytes`. NONE remains uncached and rejects an explicit limit. + Existing Proxy compressed-size and partial-block maps supply aggregate accounting; + one coordinator orders native chunks across leaves. Aliases share the same Proxy, + and closing a leaf preserves warm payload while session handles remain alive. + Bounded standalone Proxies use the same coordinator privately. Operations enforce + the budget after returning data internally and on failures; failed backing-store + eviction keeps its byte charge until a later successful eviction. +- Step-3 validation in `blosc2`: 272 focused store/array/Proxy/fsspec/expression tests + passed, 2 skipped and 4 deselected, including local HTTP fixtures. New cases cover + B2Z, HDF5, Zarr v2/v3, cross-leaf LRU, warm reopen, aggregate partial-block charges, + oversized concurrent reads, publication/eviction failures and limit validation. + Ruff check/format and whitespace checks passed. No live benchmark or full-suite + rerun was performed for this milestone. +- Step 4: `cache_dir` selects DISK when policy is omitted, with source-derived + directories, generation-tagged per-leaf Proxy files, and one exclusive lifetime + OS lock. Reopening restores every retained leaf before aggregate trimming; + DISK permits an unbounded `None` limit. Standalone exports remain self-contained, + including warm payload and one embedded HDF5 reference map when requested. +- Atomic versioned MessagePack manifests preserve discovery and safe source identity. + B2Z restores directory/locator data by replaying captured bounded metadata reads + through the existing ZIP reader (including its offset/member validation), rather + than inventing a parallel ZIP index representation. These prefixes may include + incidental payload bytes but are only replayed for metadata, never for ordinary + chunk reads. HDF5 stores one shared map; Zarr preserves metadata keys and lazy + listing state without recursively discovering unvisited groups. +- Root `refresh()` builds replacement discovery before publishing a generation; + old child handles become stale only after success. Failed publication leaves the + current generation usable. Obsolete generation files are removed on the next + exclusive reopen. Payload accounting excludes filesystem allocation and metadata; + `metadata_bytes` exposes encoded manifest size. Private filesystem clients are + reused without Zarr cloning them, and owned HTTP/S3 sessions close on last release. +- Step-4 validation in `blosc2`: 306 focused tests passed, 2 skipped and 8 deselected. + Tests cover B2Z/HDF5/Zarr v2/v3 disk reopening without remote byte reads, aggregate + smaller-limit trimming, lock exclusion and process-kill release, dependent lifetime, + corrupt manifests, failed refresh and warm standalone export. A local HTTP HDF5 + regression verifies request-free warm reopen and actual session closure. Ruff and + whitespace checks passed. POSIX locking ran on macOS; Windows byte locking still + needs Windows CI. No live S3 benchmark or full default-suite run was performed. +- Step 5: b2view uses public RemoteStore/RemoteArray handles, with one 64 MiB + MEMORY budget per remote browsing session. Selecting another leaf or group + closes the old UI handle while preserving warm payload in the store. Browser + presentation converts public node metadata directly; the internal RemoteHierarchy + adapter is deleted. An internal array-root opening flag preserves single-pass + discovery for direct leaves without changing public RemoteStore group validation + or generic `blosc2.open` dispatch. Direct Zarr leaves do not require parent LIST. +- Step-5 validation: 246 focused browser/store/array tests passed, 2 skipped and + 34 deselected; all 34 marked Textual tests passed separately. Coverage includes + B2Z/HDF5/Zarr warm revisits, subtree-relative paths, empty/unsupported nodes, + navigation during blocked reads, refresh, listing retries, shutdown and fresh + process codec initialization. Ruff and whitespace checks passed. The new + `examples/remote/store-browse.py` ran against a memory-backed B2Z with a DISK + cache, and browser/reference/remote-array guides describe shared retention. +- Step 6: the full default `conda run --no-capture-output -n blosc2 pytest -q` + run passed on macOS: **9,853 passed, 31 skipped in 28.93 seconds**. The run used + the repository's default parallel workers and permitted local HTTP/subprocess + fixtures. Ruff lint and format checks passed for all 12 maintained Python files + changed in steps 3–5; whitespace checks passed. No implementation changes were + needed after this validation run. +- Validation limits: the 34 Textual tests passed separately in step 5; default + pytest excludes marked Textual, heavy and network tests. The explicit standalone + core/ndarray/schunk doctest invocation reported earlier was not rerun; its + previously recorded failures are not claimed fixed by the default-suite result. + Windows byte locking is implemented and covered by tests in the existing Windows + CI matrix, but was not executed locally. No new live S3/HTTPS benchmark, fully + offline validation, or heavy-test run was performed. The recorded live-network + measurements remain the standalone RemoteArray baseline, not a RemoteStore + performance claim. +- Measurement follow-up: added `bench/remote_array_traffic.py` with its report + and raw JSONL results. It measures cold/warm requests, connections and response + body bytes for B2Z, Zarr and HDF5 over S3 and HTTPS. Fixed HTTPS HDF5 discovery + seekability in `scan_hdf5_refs` and added a regression test. This is a standalone + RemoteArray baseline; it does not measure RemoteStore behavior. + +## Measured remote baseline (2026-09-09) + +Live Backblaze B2 measurements cover all three formats over S3 and direct HTTPS, +with three fresh-process trials per case (18 trials total). Each opens `d0/a3` +from `hierarchy.{b2z,zarr,h5}`, reads `[0, :100, :100]`, then repeats that slice +on the same handle. All sources have shape `(10, 1000, 1000)`, dtype `int32` and +chunks `(2, 500, 500)`; the returned slice is 40,000 bytes. The MEMORY allowance +is 64 MiB, and each handle retains 15,598 compressed cache bytes. + +Cold totals include opening/discovery and the first slice. Times are medians; +variable request counts are ranges. Connections are newly established client +connections, separate from HTTP request attempts. Downloaded bytes count response +bodies, including metadata and error responses, but exclude HTTP headers and +TLS/TCP overhead. + +| Format | Transport | Cold requests | New connections | Downloaded bytes | Cold time | Warm time | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| `.b2z` | S3 | 5 | 1 | 40,817 | 1.683 s | 0.989 ms | +| `.b2z` | HTTPS | 5 | 1 | 40,817 | 1.419 s | 1.004 ms | +| `.zarr` | S3 | 8 | 5 | 13,433 | 1.918 s | 0.364 ms | +| `.zarr` | HTTPS | 4–5 | 3 | 12,875 | 0.904 s | 0.392 ms | +| `.h5` | S3 | 58 | 1 | 60,731 | 11.839 s | 0.595 ms | +| `.h5` | HTTPS | 58 | 1 | 60,731 | 11.829 s | 0.541 ms | + +Every warm read required **zero requests, zero new connections and zero downloaded +bytes**. All values and warm-cache assertions passed. These are same-handle MEMORY +hits, not disk reopen or RemoteStore manifest measurements. Service/CDN and OS DNS +caches were not cleared. Latency outliers and the extra Zarr HTTPS GET remain in +the raw results; this small, regular fixture does not establish general throughput +or compression rankings. + +Implications for the remaining implementation: + +- HDF5 discovery alone costs 57 requests and 47,332 bytes, followed by one data + request of 13,399 bytes. Reusing its reference map across leaves and disk reopens + is the main opportunity demonstrated by this baseline. Validate that manifest + restoration removes repeated translation, rather than merely retaining payloads. +- S3 and HTTPS have similar HDF5 costs here, but are not interchangeable performance + baselines. Zarr's S3 route adds HEAD requests; keep both transports in subsequent + comparisons and separate discovery from slice traffic. +- Built-in `array.traffic` is not a complete transport tally: standalone HDF5 + opening reports 392 bytes versus 47,332 received body bytes because its Kerchunk + scan is outside that counter. HEAD requests and failed Zarr metadata probes are + also absent. During shared-source extraction, include discovery in root accounting + once and document counter semantics. Use transport instrumentation to verify + manifest reuse rather than treating the existing counter as total HTTP traffic. + +The benchmark exposed and fixed HTTPS HDF5 scanning: `block_size=0` selected a +non-seekable HTTP stream. `scan_hdf5_refs` now uses +`block_size=1, cache_type="none"`, preserving seekability without read-ahead. +The 155 existing focused fsspec/HDF5 tests and the new HTTP scan/warm-slice regression +test passed; Ruff and whitespace checks passed. Live network behavior was exercised +by the 18 benchmark trials; the marked network pytest suite, full default suite and +Textual UI suite remain unrun for this follow-up. + +See the [full measurement report](../bench/remote_array_traffic.md), +[reproducible benchmark](../bench/remote_array_traffic.py), and +[raw trial results](../bench/remote_array_traffic.jsonl). + +## Goal + +Rename the unreleased `RemoteProxy` to `RemoteArray` and introduce a read-only +`RemoteStore` for remote B2Z, Zarr, and HDF5 hierarchies. Arrays selected from a +store share one cache budget. Persist discovery metadata so reopening a disk +cache avoids repeating archive discovery or HDF5 translation. Migrate b2view to +these public APIs and remove its internal hierarchy implementation. + +Keep this a library and browser change. Caterva2 integration, customer quotas, +and concurrent processes sharing a writable disk cache are out of scope. + +## Target public behavior + +```python +store = blosc2.RemoteStore( + "https://host/data.h5", + cache_policy=blosc2.CachePolicy.MEMORY, + max_cache_bytes=256 << 20, +) +store.keys() # immediate children; metadata only +group = store["experiment"] # RemoteStore view +arr = group["temperature"] # RemoteArray +values = arr[:100] # fetch and cache on demand +result = (arr + 273.15).compute() # existing array expressions + +disk_store = blosc2.RemoteStore( + "https://host/data.h5", + cache_dir="./cache", # selects DISK when policy is omitted + max_cache_bytes=256 << 20, # total retained payload for this store +) + +direct = blosc2.RemoteArray( + "https://host/data.h5", + dataset="experiment/temperature", + cache_policy=blosc2.CachePolicy.MEMORY, +) +``` + +- `RemoteArray` retains current array operations, format support, serialization, + and standalone cache defaults. Rename the class, module, exports, references, + tests, and documentation without a compatibility alias. Historical plans need + not be renamed. No migration support for unreleased RemoteProxy artifacts is + required; update current persisted-type dispatch consistently. +- `RemoteStore` defaults to MEMORY, with a 256 MiB store-wide allowance. An omitted + policy plus `cache_dir` selects DISK; contradictory explicit options raise. + Reuse array validation: MEMORY requires a finite positive limit, DISK permits + `None` for unbounded retention, and NONE takes no payload limit or disk cache. +- Group views and arrays share the root store's policy, budget, and resources. + No independent per-leaf limit overrides in v13. A separately constructed + RemoteArray remains independent. +- Expose immediate-child iteration/`keys()`, string-path lookup, group attributes, + node kind and unsupported-node diagnostics, source identity, and cache/traffic + information. Keep UI rendering types out of the library API. Do not implement + mutation methods or subclass local TreeStore merely to reuse its interface. +- Paths are relative to the selected group. Both `store["a/b"]` and + `store["a"]["b"]` resolve the same source and cache entry. Missing paths raise + KeyError; unsupported objects remain discoverable and raise a useful error + when opened as arrays. +- Keep `blosc2.open(..., lazy=True, dataset=...)` array behavior, now returning + RemoteArray. The explicit RemoteStore constructor is the new hierarchy entry + point; do not broaden generic root-opening dispatch or alter existing eager + localization behavior as an incidental part of this change. + +## Responsibilities and reuse + +RemoteArray selects and fetches array data through existing source readers and +Proxy machinery. RemoteStore discovers hierarchy metadata and supplies shared +source resources and cache ownership. A small internal cache coordinator owns +aggregate accounting and eviction; it does not decode formats or fetch arrays. + +Start by tracing all cache mutation and access paths in `proxy.py` and +`remote_proxy.py`, including partial blocks, hits, fill operations, expressions, +serialization, and trimming. Reuse their fetched maps, compressed caches, +mutation recovery, and eviction primitives. Do not add a second payload cache, +generic backend plugin framework, or replacement chunk reader. + +Extract `b2view/hierarchy.py` discovery into `remote_store.py`. Retain helpers in +the existing B2Z, Zarr, and HDF5 source modules when used by both discovery and +array readers. Replace imports of browser NodeInfo/ObjectInfo and rendering +helpers with library metadata that StoreBrowser translates for the UI. + +Reuse one B2Z archive/index, one HDF5 reference map, and the appropriate Zarr +store per root session. Add the smallest internal source-attachment path needed +for store arrays. In particular, do not reopen every leaf from its URL or bypass +the current B2Z attachment restriction without validating source identity. + +## Shared payload cache + +Use existing per-array backing storage with common ownership and eviction. +Different array geometries do not need to fit into a single B2ND carrier. + +- Identify entries by canonical dataset identity and native cache chunk number. + Preserve partial-block fetched maps and charge the actual retained compressed + representation. Replacing a partial chunk updates its charge rather than + counting both versions. Whole-chunk eviction is sufficient for v13. +- Maintain one LRU across array caches, touching entries on hits and publication. + Repeated handles and group aliases reuse a dataset's cache state. Do not create + a fresh Proxy/cache every time the same array is looked up. +- Enforce the aggregate limit after operations, including failures that retain + data, following the current post-operation limit contract. An oversized chunk + may be fetched to serve a read and then evicted. The budget does not cap output + arrays, decompression buffers, metadata, or transient peak memory. +- Retain warm payloads when the browser changes selection. Cache ownership must + not depend on retaining the selected UI handle. Conversely, enumerating a + hierarchy must not allocate carriers/fetched maps for every array. +- Keep standalone behavior on the same code path with a private cache owner. + Avoid duplicating accounting and applying both a per-array and store limit. +- Store `cache_bytes` reports the aggregate; array `cache_bytes` reports that + leaf's contribution. Clearly document that a store-derived array's configured + limit belongs to the store. Shared transport traffic is reported once at the + root; do not sum multiple views of the same counter. + +Initially serialize store cache operations within one process using a shared +reentrant lock if needed. Preserve concurrency inside existing remote fetch +operations. Establish one lock order before adding hooks to avoid cross-array +eviction deadlocks. Mark any deliberate coarse locking with a ponytail comment +describing its throughput ceiling and possible later refinement. + +## Disk layout, ownership, and reopening + +Treat `cache_dir` as a parent containing source-derived store directories, not +as a single shared cache for all URLs. A store directory contains one manifest +and lazily created per-array payload caches. Reuse current carrier/storage +primitives; a packed store cache format is unnecessary. + +The payload bound is per store, not per parent directory, customer, or physical +filesystem. File overhead and stale physical allocation are not compressed +payload accounting. Do not describe this bound as a strict disk quota. + +Only one independently opened owner may use a given store cache directory at +a time. Group views and arrays share that owner. Use a lifetime exclusive lock +to reject conflicting opens clearly, including another process; this is an +exclusion guard, not concurrent shared-cache support. Reuse an existing suitable +locking facility where possible. Verify crash release and supported platforms; +do not implement a stale PID-file lock or silently run without exclusion. +Different source-derived directories under the same parent remain independent. + +On reopening, include retained payload from all leaf caches in accounting, even +if those leaves have not been selected in this session. Restore sizes from +existing bookkeeping and trim locally if a smaller limit was requested. Exact +LRU order need not survive a restart; a deterministic cold ordering is sufficient. +No remote payload fetch is needed merely to account for or evict local data. + +## Persistent discovery manifest + +Keep discovery metadata separate from evictable payloads: + +| Format | Persisted discovery information | +| --- | --- | +| B2Z | Logical hierarchy, attributes, unsupported boundaries, ZIP member offsets/lengths and metadata needed to reopen bounded members | +| HDF5 | One Kerchunk reference map, hierarchy, attributes, unsupported diagnostics | +| Zarr | Discovered groups/arrays, attributes and decoding/layout metadata, including consolidated metadata where available | + +Store a versioned, validated source descriptor and serializable discovery data; +never pickle live filesystem, archive, Zarr, or browser objects. Reuse current +serialization dependencies and URL portability validation. Credentials and +storage options remain runtime inputs; audit reference-map URLs as well as the +top-level URL so secrets are not copied into manifests. + +Restore source readers from persisted locators through explicit internal hooks. +Saving offsets without teaching readers to reuse them does not satisfy this +milestone. HDF5 arrays share the manifest's map rather than embedding a duplicate +map in every store-owned carrier. Keep standalone array export self-contained +under its existing contract; store-owned backing files are implementation +details, not automatically portable standalone exports. + +Zarr discovery remains lazy. Persist what is known and which groups have been +listed; absence from a partial manifest is not evidence that a child is missing. +Do not force a recursive scan on open or close to produce a complete manifest. + +Publish manifests atomically with a temporary file and replacement. Tie locator +metadata and payloads to the same source identity/generation. Validate manifest +schema, paths, offsets and cache identities before using them. An invalid or +interrupted manifest must never cause old locators to be used with new payloads; +rebuild disposable state safely or report an actionable error. + +Keep manifests when payloads are evicted. Expose `metadata_bytes` as the encoded +manifest size, distinct from payload bytes and Python memory consumption. Large +HDF5 reference maps remain possible and are outside `max_cache_bytes`. + +## Source assumptions and lifetime + +Support immutable sources, retaining current format restrictions. Do not claim +automatic change detection for Zarr/HDF5 or derive a content version solely from +matching shape/dtype. Reuse reliable validators where already available. + +Provide an explicit root refresh that rebuilds discovery and invalidates dependent +payloads as one generation change. Existing child handles become stale and raise +a clear error requiring lookup again; do not silently mix source generations or +reinterpret an old array handle with a different shape. Build replacement +metadata before publishing it so a failed refresh does not publish a half-state. + +Provide context-manager/close behavior. Closing a handle releases its ownership; +arrays and group views already returned remain usable until individually closed +or released. The last dependent handle releases resources and the disk ownership +lock. Ensure owner/cache references do not create a cycle that keeps the lock +alive indefinitely. Operations on an explicitly closed handle raise clearly. + +Persistent caching does not initially promise fully offline reopening. Metadata +validation or source initialization may still contact the remote service; state +that limitation while proving that a valid manifest avoids repeated discovery. + +## b2view migration + +- Use RemoteStore for remote roots/groups and RemoteArray for direct leaves. +- Set one 64 MiB MEMORY budget for a browsing store, preserving the current + preview allowance while allowing warm revisits across arrays. +- Translate public metadata into browser presentation types in StoreBrowser. + Preserve empty groups, subtree-relative navigation, attributes, unsupported + object messages, source headers, traffic reporting and standalone presentation. +- Preserve direct-array opening without listing its parent; preserve codec + initialization required before Textual captures stderr. +- Remove the old internal RemoteHierarchy implementation after migration. Keep + local TreeStore, CTable, plotting and query behavior intact. + +## Implementation sequence + +1. Rename RemoteProxy throughout maintained source/tests/docs and update current + serialization dispatch. Run focused array/proxy/expression checks. +2. Extract public RemoteStore discovery and resource ownership. Return RemoteArray + leaves through shared source readers; verify discovery does not fetch arrays. +3. Introduce minimal shared accounting/eviction hooks in existing cache machinery. + Deliver and validate NONE and aggregate MEMORY behavior first. +4. Add store DISK layout, exclusive ownership, persistent manifest restoration, + aggregate reopening/eviction and explicit refresh. Preserve standalone exports. +5. Migrate b2view, remove its internal implementation, and update reference docs, + remote-array guide and examples with the two-type model and cache semantics. +6. Run focused and default repository checks; record actual results and limitations + before marking this plan implemented. + +## Validation and acceptance + +Use the `blosc2` conda environment for every Python/test/build command. Extend +existing source, Proxy, serialization, expression and b2view tests rather than +creating another test framework. Prefer memory-backed fsspec and local fixtures; +network validation remains separately marked. + +Required behavioral coverage: + +- All three formats: root/subgroup lookup, aliases, attributes, empty groups, + supported arrays, unsupported siblings, slicing and array expressions. +- Existing B2Z ZIP64/range checks, partial-block reads and restrictions; existing + Zarr layouts/codecs and HDF5 filter/Kerchunk behavior remain intact. +- Read A, read B, revisit A under generous and restrictive shared budgets. Verify + both returned values and transport hits/misses, aggregate bounds and cross-array + eviction. Cover duplicate handles, partial chunk replacement and oversized reads. +- NONE retains no payload; enumeration creates no per-leaf payload caches. +- DISK reopen reuses discovery and warm chunks, counts unopened leaf caches, and + applies a smaller budget without remote payload reads. HDF5 translation happens + once initially and is not repeated on a valid manifest reopen or leaf selection. +- Manifest-only retention after payload eviction, partial Zarr discovery, malformed + manifests, interrupted publication and generation refresh with stale handles. +- Conflicting owner opens fail in one process and in a subprocess. Lock release + after normal close and process termination permits a later open. +- Returned arrays survive closing their originating store handle; final resource + release closes transport handles and releases the disk lock. +- Standalone RemoteArray persistence/export/expression behavior and b2view's local + and remote navigation/preview behavior pass their existing regression checks. + +Run the focused suites after each relevant phase, then the default pytest suite +and Ruff checks for changed files. Do not mark network or offline behavior tested +unless explicitly exercised. No implementation or tests are run by writing this +plan alone. + +## Deferred + +Caterva2 integration; customer quotas; multiple processes concurrently using the +same writable cache; distributed locking; cache services; strict physical-disk +limits; automatic mutable-source invalidation; fully offline reopening; remote +writes; embedded B2Z array/CTable payload support; per-array reservations; public +cache-backend plugins. None is a prerequisite for this version. diff --git a/plans/remote-proxy-v14.md b/plans/remote-proxy-v14.md new file mode 100644 index 000000000..6f72b943c --- /dev/null +++ b/plans/remote-proxy-v14.md @@ -0,0 +1,394 @@ +# RemoteStore v14: TreeStore-backed caches and portable exports + +Status: implemented and locally validated, 2026-09-10. All implementation steps +and acceptance criteria complete. TreeStore-backed DISK caches, portable .b2z +reference exports (`RemoteStore.save()`), root marker dispatch via `blosc2.open()`, +immutable (`mutable=False`) and mutable (`mutable=True`) cache modes, allowance +validation, and single-HDF5-map reuse are fully implemented and verified. + +## Implementation progress + +- Step 1 (Root schema, mutability model, and TreeStore leaf alignment): + - Updated `DictStore._external_ext` and `DictStore._is_external_value` in `src/blosc2/dict_store.py` + so store-owned `RemoteArray` leaves externalize with `.b2nd` instead of `.b2f`. + - Defined the versioned `"b2remote_store"` root marker in `embed.b2e.meta` alongside `"b2tree"`, + and the `b2remote_manifest` dictionary stored in `embed.b2e.vlmeta`. + - Added boolean `.mutable` property (defaulting to `False` for future exports) and `.is_cache_mutable` + property to `RemoteStore` and `RemoteArray`. + - Group views inherit and propagate their owner's `.mutable` setting; independent arrays own theirs. + - Guarded `_enforce_cache_limit` in `src/blosc2/proxy.py` when `_schunk_cache.mode == 'r'`, avoiding + any cache-touch or eviction writes to read-only carriers. + - In `src/blosc2/remote_array.py`, updated `__getitem__` and `_serialized_operation` to evaluate + cache misses transiently on immutable caches without writing to disk. + +- Step 2 (TreeStore-backed DISK layout): + - In `src/blosc2/remote_store_cache.py`, migrated `StoreDiskCache` to store active generations as + genuine TreeStore directories: `.b2d/` containing `embed.b2e` and logical leaf files + `.b2d/.b2nd`. + - Replaced legacy `manifest.msgpack` with atomic `active_generation.json`. + - Updated cache discard logic to remove obsolete `.b2d` generations and reject incompatible legacy + development caches with actionable errors. + +- Step 3 (Portable `.b2z` reference export): + - Implemented `RemoteStore.save(destination, *, include_cache=True, mutable=None, overwrite=False)` + and `RemoteStore.to_cframe()` in `src/blosc2/remote_store.py`. + - Implemented export precedence: explicit `save(mutable=...)`, then assigned `.mutable` value, + otherwise `False`. + - Added subtree export support (`store["group"].save(...)`) with remapped root dataset and relative child keys. + - Preserves empty groups, attributes, and unsupported-node diagnostics. + - Validates destination extension (`.b2z`), rejects directory destinations, enforces `overwrite` policy, + and rejects destinations located inside the live cache storage directory. + - Validates retained cache payload against declared `max_cache_bytes` at save-time. + +- Step 4 (Root marker dispatch and shared-owner restoration): + - Updated `_open_special_store` in `src/blosc2/schunk.py` to detect `"b2remote_store"` in `embed.b2e` + metadata and route `blosc2.open("snapshot.b2z")` directly to `RemoteStore._open_artifact`. + - **Immutable snapshots (`mutable=False`)**: served directly in-place from `.b2z` member offsets + or `.b2d` paths without disk writes. Verified on `chmod 0o444` read-only files with byte-for-byte + SHA-256 preservation across hits and misses. Misses are fetched transiently. + - **Mutable snapshots (`mutable=True`)**: safely staged into an independent writable runtime storage + directory under standard policy and LRU eviction, keeping the original `.b2z` file untouched. + - Budget validation: opening an immutable snapshot with an allowance smaller than its retained payload + is rejected with an actionable error (`"smaller than retained immutable payload"`). Opening a warm + snapshot with `CachePolicy.NONE` is rejected. Mutable snapshots opened with a smaller requested budget + are trimmed via LRU eviction before returning. + - Preserves single HDF5 Kerchunk reference map across leaves without repeated translation on reopen. + - Refreshing an immutable artifact is rejected with an actionable error. + +- Step 5 (Comprehensive acceptance tests and validation): + - Full default test suite after review fixes passed on macOS: **9,918 passed, 34 skipped in 30.33 seconds**. + - Focused remote store, array, fsspec, proxy, and `b2view` hierarchy tests after review fixes: + **371 passed, 5 skipped, 8 deselected in 7.29 seconds**. + - `b2view` hierarchy tests: **16 passed**. + - Ruff formatting and linting: zero errors across `src/blosc2` and `tests`. + - Added acceptance tests in `tests/test_remote_store.py` covering: + - B2Z, HDF5, and Zarr v2/v3 export and reopen (warm, cold, immutable, mutable). + - `chmod 0o444` read-only byte preservation across hits and misses. + - Budget validation, eviction trimming on mutable reopen, and rejection of smaller budgets on immutable snapshots. + - Subtree exports with remapped root datasets and relative child keys. + - Single HDF5 Kerchunk reference map reuse without repeated translation on reopen. + - `.mutable` property hierarchy inheritance and validation. + - Save destination validation (extension, directory, collision, and live cache nesting). + +- Review follow-up (completed 2026-09-10): + - Validate artifact generations, manifest paths and archive members before staging; reject directory + symlinks. Reuse generation validation wherever generation values form disk-cache paths. + - Release the cache owner and lock on extraction failure, and publish the active generation only + after successful restoration. Regression tests verify reopening the same cache after failure. + - Make immutable `RemoteArray.get_chunk()` and `aget_chunk()` misses transient, preserving retained + payload and fetched-state metadata. Reject `fetch()`, `afetch()`, and `trim_cache()` on immutable caches. + - Attach immutable carriers read-only even when opened with mode `"a"`, and avoid eviction on cached reads. + - Reject standalone immutable RemoteArray payloads exceeding their persisted cache allowance on both + file and CFrame reopen. Regression tests verify unchanged carrier bytes and cache accounting. + - Ruff lint/format checks on the five files changed during review and `git diff --check` passed. + +## Objective + +Represent a RemoteStore DISK cache generation as a genuine TreeStore directory +(`.b2d`) and add `RemoteStore.save()` to produce a portable `.b2z` reference +archive. Transport the source description, discovered hierarchy and optional warm +payload together, preserving shared RemoteStore ownership and cache accounting +when reopened. + +An export remains a remote reference. Missing regions require access to the +original source and runtime credentials. Saving must not download every array +or claim fully offline operation. + +## Prototype evidence + +A local exploratory test used existing APIs to insert two RemoteArray leaves +into a TreeStore `.b2d`, pack it with `to_b2z()`, delete the original runtime cache +and staging directory, move the archive, and reopen it as a TreeStore. + +| Original backend | Exported archive bytes | Warm slice source bytes | +| --- | ---: | ---: | +| B2Z | 5,999 | 0 | +| Zarr v3 | 3,686 | 0 | +| HDF5 | 4,895 | 0 | + +Uncached slices returned correct values by fetching from the original sources. +An empty group and its attributes also survived. These were small synthetic +arrays on fsspec's memory filesystem, in one process; they are not network +benchmarks or proof of cross-machine portability. + +The experiment exposed the remaining work: + +- Reopened leaves are independent RemoteArrays, with independent cache owners. +- Each exported HDF5 leaf duplicates the reference map; the example stored two + compressed copies of 546 bytes each. +- Direct TreeStore insertion currently gives RemoteArray leaves a `.b2f` suffix. + Ensure store-owned NDArray carriers use the appropriate `.b2nd` representation. +- Existing carrier-copy branches need auditing for warm-payload preservation; + packing a tree alone does not define RemoteStore serialization semantics. + +## Public behavior + +Public API and dispatch: + +```python +with blosc2.RemoteStore(url, cache_dir="remote-cache") as store: + with store["experiment/temperature"] as array: + values = array[:100] + store.mutable = False # optional: immutable is already the export default + store.save("snapshot.b2z", include_cache=True) + +# Root marker dispatch: recognize the remote-store root marker. +with blosc2.open("snapshot.b2z") as restored: + array = restored["experiment/temperature"] + values = array[:100] + array.close() +``` + +- `save(destination, *, include_cache=True, mutable=None, overwrite=False)` writes `.b2z`. + Do not add directory export or extra packaging options unless needed. +- The default includes retained payload only. `include_cache=False` preserves + reference/discovery metadata and omits fetched data, without clearing the live + cache. Inline HDF5 values and incidental metadata-prefix bytes remain metadata. +- Support saves from NONE, MEMORY and DISK stores through one export path. + Saving a group view should export that subtree with relative keys and the + correct original-source root. Reject invalid destinations before copying data. +- Preserve empty groups, attributes, unsupported-node diagnostics and partial + discovery state. Exporting a partially discovered Zarr hierarchy must not force + a recursive listing; unknown children remain discoverable from the source. +- Standalone RemoteArray exports remain self-contained. Store-owned carriers may + depend on the root manifest, but exporting one leaf must still produce a valid + independent RemoteArray reference, including HDF5 references when required. +- Archives contain neither credentials nor machine-specific cache paths. + Credentials and storage options are supplied by the receiving process. + +## Persisted cache mutability + +Agreed API: RemoteStore and RemoteArray expose a boolean `.mutable` property +that sets the default for future exports. Do not add `mutable` to constructors. +New live objects retain their existing cache-policy behavior; their export default +is `False`. The property is not a runtime freeze/unfreeze switch. + +- Export precedence is: explicit `save(mutable=...)`, then an explicitly assigned + `.mutable` value, otherwise `False`. Use an omitted sentinel internally (shown + as `None` in the proposed signature). Apply the same rule to RemoteArray's + `to_cframe()`. Re-saving an opened artifact without either explicit choice also + defaults to `False`; its loaded runtime mode is separate from the export default. +- Neither the setter nor an export override changes the current object's cache + behavior, contents or remote source. Validate boolean assignments. Store views + share the owner's export default; independent arrays own theirs. +- On opening an artifact saved with `mutable=False`, serve its included regions + without changing chunks, fetched maps, manifest or LRU metadata. Fetch misses + transiently without retaining them across operations. Repeated misses may fetch + again. Source arrays remain read-only under the immutable-source contract. +- On opening an artifact saved with `mutable=True`, permit cache fills, eviction + and bookkeeping in writable runtime storage under the existing policy, budget + and ownership rules. ZIP exports use existing extraction/staging machinery + initially; the transported archive changes only through an explicit save. +- `include_cache` and `mutable` are independent. A cold immutable export is a + streaming reference. To obtain a writable version of an immutable artifact, + export a separate artifact with `mutable=True` and reopen it; setting the export + default alone does not make the currently opened cache writable. +- Refresh must not silently replace an immutable artifact's runtime generation. + Use a new live session or a separately exported writable artifact. Mutability + is a behavior declaration, not a security boundary or filesystem permission. + +The remote-proxy branch is unreleased. Update its descriptors and fixtures +directly; no migration or compatibility shim for prior RemoteArray/RemoteStore +artifacts is required. Store-derived leaves and group views expose the root's +setting; setters update the shared owner rather than creating per-leaf overrides. +Independent RemoteArray objects own their setting. Version the resulting schema +and reject unsupported artifacts clearly. + +## TreeStore-backed DISK layout + +Keep source-derived ownership directories and generation boundaries from v13. +Make the active generation a TreeStore, rather than making lock files and old +generations part of its logical hierarchy: + +```text +cache_dir/ +└── / + ├── owner.lock + ├── + └── .b2d/ + ├── embed.b2e + └── experiment/ + └── temperature.b2nd +``` + +The exact manifest placement is an implementation decision. Prefer existing +TreeStore metadata/storage primitives: one reserved root descriptor and one +shared discovery record. A compressed metadata object may suit large HDF5 maps +better than a large attribute. Do not duplicate the map in per-leaf carriers or +maintain two competing hierarchy/manifest authorities. + +Retain logical dataset paths instead of hashed leaf filenames where TreeStore +can safely represent them. Check collisions with TreeStore's reserved names, +attribute files and object boundaries before committing to a mapping. Either +provide an unambiguous reversible mapping for such paths or reject them clearly; +never silently hide a valid source dataset. + +Reuse existing per-array Proxy caches, fetched maps, partial-block bookkeeping, +dirty-state recovery and aggregate LRU coordinator. The storage layout changes; +the payload-fetch and eviction algorithms should not need replacement. + +## Root descriptor and shared manifest + +Introduce a versioned remote-store root marker distinguishable from an ordinary +TreeStore and existing object roots. It should identify: + +- The portable source descriptor, selected root and immutable-source contract. +- The discovery generation and references to shared backend metadata. +- Known node kinds, attributes, diagnostics and which groups have been listed. +- Dataset-to-carrier associations and each carrier's generation/source identity. +- Requested cache policy and aggregate allowance, subject to explicit receiving + process overrides decided below. +- Cache mutability, independent of source immutability and payload inclusion. + +Preserve v13 backend metadata reuse: B2Z directory/header locators, one HDF5 +Kerchunk map, and lazily acquired Zarr decoding and listing metadata. Keep +metadata separate from evictable payload. Define `metadata_bytes` as encoded +descriptor/discovery bytes, excluding ZIP and filesystem overhead. + +Validate schemas, paths, identities, reference URLs and byte-range bounds before +attaching readers or trusting fetched maps. Use existing serialization helpers; +never pickle filesystem, archive, browser or source-reader objects. + +## Export transaction + +1. Validate destination, overwrite policy and subtree scope. Reject destinations + inside live cache storage when they could overwrite or recursively package it. +2. Hold the existing store operation lock while capturing a consistent generation + and cache state. A first implementation may copy under that lock; annotate the + throughput limitation and defer background snapshot machinery. +3. Build a portable TreeStore snapshot using existing carriers and metadata. + Include only the active generation and requested subtree. Preserve fetched + maps with their matching compressed chunks, including partial chunks. +4. Reuse TreeStore/DictStore `to_b2z()` and its atomic temporary-file replacement. + Do not blindly archive the v13 ownership directory: its lock, stale generations, + temporary files and absolute-path state do not belong in an export. +5. Clean temporary state after success or failure. An interrupted save must leave + an existing destination and the live RemoteStore usable. + +Prefer the same export machinery for MEMORY and DISK snapshots. Do not promote +export to an implicit full-data materialization operation. Establish whether any +source metadata reads are necessary and test/document them separately from +payload fetches. + +## Reopening and lifetime + +Teach the local archive-opening path to recognize the root marker and construct +one RemoteStore owner. Ordinary `.b2z` TreeStores must retain their current +dispatch and behavior. Agreed reopening decisions: + +- Use `blosc2.open("snapshot.b2z")` as the primary local entry point, recognizing + the RemoteStore root descriptor automatically. +- Mutable archives use existing safe extraction/staging into independently owned + writable runtime storage. The transported archive remains unchanged until an + explicit save. Do not add an immutable-base/writable-overlay cache framework. +- Included immutable payload is part of the cache budget. There is no exempt + snapshot tier: `cache_bytes` includes all retained cached payload, whether loaded + from the artifact or subsequently fetched, across every leaf. +- The receiving process supplies any writable cache location and runtime storage + options; never persist an exporting machine's absolute cache path. Retain the + artifact's requested allowance unless the receiver explicitly overrides it. +- Audit read-only attachment so open and hits do not write Proxy bookkeeping. + A read-only file mode alone does not establish immutable-cache behavior. + +Before returning the restored store, account for all included leaf payloads, +including leaves not selected by the receiver. Writable runtime caches apply a +smaller requested budget by local eviction before returning. An immutable snapshot +cannot be trimmed in place: reject an allowance smaller than its retained payload +with an actionable error. The caller can select a sufficient allowance or produce +a separate smaller/cold export. Do not silently exceed the allowance or exclude +immutable bytes from accounting. Immutable snapshots retain no new misses; mutable +caches can replace imported chunks through the ordinary aggregate LRU. + +The bound remains compressed retained payload, including charged partial-block +duplicates, rather than total `.b2z` file size or physical disk allocation. +Manifest bytes, container overhead, output arrays and transient buffers remain +outside it, as in v13. Do not introduce a second allowance for the imported data. +Validate the snapshot's retained payload against its declared finite allowance +when saving as well as opening; preserve DISK's explicit unbounded option. + +NONE keeps its no-retained-payload meaning. A conflicting request to reopen a warm +artifact under NONE must not create an exempt payload tier; reject it clearly and +direct callers to a cold export. Merely reading an artifact's metadata or inline +source values does not turn it into a retained payload cache. + +Aliases must share one leaf cache, one aggregate coordinator, one +discovery session and one HDF5 map. Never fall back silently to independent +RemoteArray owners while reporting a store-wide limit. + +The archive remains unchanged by reads, eviction or refresh. Writable runtime +storage must have independent ownership. Preserve last-handle cleanup, process +exclusion, source-generation validation and stale-child errors after refresh. +Close all extracted-file/archive resources only after dependent handles finish. + +Remote range-opening of an exported `.b2z` is a separate decision. Current B2Z +readers reject object carriers as plain NDArray members. Local transport and +reopening must work first; do not promise hosted-reference traversal without +explicit reader support and credential/range validation. + +## Existing formats and unreleased artifacts + +- Keep ordinary TreeStore/DictStore archives and standalone RemoteArray carriers + working, including existing eager localization and direct-array opening. +- Treat current v13 hashed directories and remote-reference artifacts as disposable + development state. No migration is required; reject incompatible state clearly + and rebuild fixtures for the new schema. +- Version the root schema and provide useful errors for unknown versions, missing + manifests, mismatched generations and malformed carriers. +- Audit DictStore insertion, embedding and externalization paths before changing + accepted object types or suffix selection. Avoid broad serialization refactors. + +## Implementation sequence + +1. [x] Settle root schema, reserved-path handling, local reopen API and policy rules. + Trace TreeStore packing/opening and RemoteArray carrier-copy paths end to end. +2. [x] Make DISK generations real TreeStores while preserving v13 locking, refresh, + aggregate accounting and metadata reuse. Reject incompatible development caches. +3. [x] Add atomic RemoteStore `.b2z` export with cold/warm and subtree variants. +4. [x] Add root-marker dispatch and shared-owner restoration. Preserve standalone leaf + exports and ordinary TreeStore behavior. +5. [x] Add round-trip tests, documentation and an executable transport example; run + focused suites, default pytest and Ruff in the `blosc2` conda environment. + +## Acceptance tests + +- B2Z, HDF5 and Zarr v2/v3: export, move to another directory, remove the original + runtime cache, and reopen in a fresh process. Use local HTTP fixtures so the + source remains reachable across processes without relying on memory filesystem + state. Verify values for warm slices and misses separately. +- Count actual source requests/bytes where applicable. Warm reads must avoid + payload downloads; metadata/source validation traffic must be reported honestly. +- Verify one shared owner/budget after reopening, A/B/A warm revisits, restrictive + aggregate limits, aliases, partial chunks, oversized reads and unselected caches. +- Count imported immutable payload in `cache_bytes` and the allowance. Reject + oversized immutable snapshots or conflicting NONE requests; trim writable + runtime copies under a smaller allowance without modifying the original archive. + Check combined imported/new payload bounds and save-time allowance validation. +- Preserve one HDF5 reference map per store and prove no repeated translation on + valid reopen. Independently exported HDF5 leaves must still carry their own map. +- Cold export excludes fetched payload without mutating the live cache. Warm + export never marks absent or partially copied data as fetched. +- Immutable exports perform no hidden chunk, bitmap, manifest or recency writes + during open, hits or misses. Test genuinely read-only files, repeated misses, + and byte-for-byte artifact preservation. Mutable exports retain new data in + writable runtime storage. Test persisted defaults, explicit writable copies, + export-default changes, shared-owner propagation and budget/policy conflicts for + both public types. Verify that omitted export arguments inherit the current + setting and explicit overrides do not mutate the live object. +- Cover empty groups, attributes, unsupported nodes, subtree exports, reserved + names and unlisted Zarr groups. Saving must not enumerate an entire lazy source. +- Inject export/publication failures; verify destination atomicity, cleanup and + live-cache usability. Reject unsafe archive paths, leaked credentials, invalid + locators and mismatched generations before use. +- Verify receiver lifetime/exclusion, smaller-budget restoration and refresh; + ensure the transported archive remains unchanged. +- Regress ordinary TreeStore packing, RemoteArray persistence and b2view behavior. + Record platform and network validation limits rather than claiming unrun checks. + +## Non-goals + +Full-source downloads, guaranteed offline archives, remote writes, automatic +mutable-source detection, concurrent writable owners, strict filesystem quotas, +new compression/container formats, and a generic cache-backend framework. + +This proposal is a storage/serialization revision, not a reason to replace the +working v13 reader and cache machinery. Review the open decisions before starting. diff --git a/plans/remote-proxy-v2.md b/plans/remote-proxy-v2.md new file mode 100644 index 000000000..b84497b53 --- /dev/null +++ b/plans/remote-proxy-v2.md @@ -0,0 +1,427 @@ +# Plan: Self-Caching RemoteProxy + +## Status + +Superseded by remote-proxy-v3.md for the Python client API and cache policies. + +Implemented and validated in Python-Blosc2 and Caterva2. The product and +persistence contracts are settled; the phase checklist and test matrix below +describe the delivered version-1 behavior. + +## Purpose + +Allow Caterva2 to host a small `.b2nd` proxy for a public remote B2ND array. +When Caterva2 reads missing regions, the proxy may retain the fetched compressed +chunks inside that same `.b2nd` file, up to a configured storage limit. + +The proxy file is therefore a mutable persistent cache, not an immutable remote +reference plus a separate server cache. + +## Scope + +The first version supports: + +- direct RemoteProxy `.b2nd` files +- one public, contiguous B2ND source available over HTTPS +- public S3 objects through their HTTPS object URLs +- operation with embedded caching disabled or enabled +- a bound on compressed chunk payload retained in each proxy file +- reuse of cached chunks after process and server restarts +- warm-carrier and cache-free export/download forms +- invalidation when the remote object identity changes +- Caterva2's existing `api/info`, `api/fetch`, `api/chunk`, and physical + download endpoints + +The first version does not support: + +- credentials, signed URLs, cookies, custom headers, or private sources +- native `s3://` resolution in Caterva2 +- redirects or arbitrary fsspec protocol chains +- remote references embedded in expressions or other object graphs +- cumulative network-byte or request-count budgets +- a new Caterva2 HTTP endpoint for proxy creation or cache management +- a separate RemoteProxy-specific server quota + +Ordinary connection timeouts, remote-read concurrency, destination policy, and +structural validation remain in scope. + +## Core Decisions + +### The carrier is the cache + +The uploaded `.b2nd` contains both: + +- stable identity: versioned source descriptor and array geometry +- mutable state: fetched compressed chunks, source stamp, fetched bitmap, + compressed-size accounting, and LRU bookkeeping + +Reads may change the file's contents, size, modification time, and cache +metadata. The source descriptor and geometry must not change as a consequence +of reading or eviction. + +### Cache data is disposable + +Cached chunks are an optimization, never the authority for array identity. +Deleting every cached chunk leaves a valid proxy. If the source identity changes, +the cache is emptied logically before any old chunk can be served. + +### Reuse the existing Proxy cache engine + +`blosc2.Proxy` already implements the required persistent-cache mechanisms: + +- UNINIT chunks for cache misses +- chunk/block fetched bitmaps +- remote source stamps +- compressed-byte accounting +- whole-chunk LRU eviction +- reopening and adopting a valid cache + +RemoteProxy should reuse these mechanics with its own carrier passed as the +Proxy cache. It should not implement a second cache engine. + +### RemoteProxy has no memory-cache policy + +RemoteProxy supports only `CachePolicy.NONE` and `CachePolicy.DISK`. A retained +RemoteProxy cache belongs in its carrier, so a `MEMORY` policy would violate the +self-caching artifact model and could multiply memory use across Caterva2 +workers and hosted proxies. + +Remove `CachePolicy.MEMORY` while the API is still unreleased. Existing +`blosc2.open(remote, lazy=True)` process-local memory caching continues to +return the established generic `Proxy`; users who explicitly need an ephemeral +memory cache can also construct that type directly. Caterva2 therefore never +needs to translate, reject, or impose a quota on a persisted memory-cache +policy because no valid RemoteProxy carrier can contain one. + +### The proxy specifies caching; the customer quota bounds storage + +The persisted proxy specifies whether it retains misses and its maximum cached +compressed payload. This behavior travels with the proxy instead of being +selected independently by each Caterva2 server. + +Each customer owns a virtual Caterva2 server whose users share one state +directory and one server-wide quota. Embedded proxy-cache growth is charged to +that existing customer quota just like other stored data. No user-level +attribution or separate RemoteProxy quota is needed. + +When the customer quota cannot accommodate another cached chunk, Caterva2 still +serves the remotely fetched result but skips retaining that chunk. A proxy with +cache policy `NONE` never retains misses. Already embedded valid chunks remain +readable without an outbound fetch. + +### Public HTTPS remains the network boundary + +Caterva2 keeps the security work already implemented: + +- remote resolution disabled by default +- exact administrator host allowlists +- HTTPS only +- public-address validation and DNS pinning +- redirects disabled +- URL credentials, queries, and fragments rejected +- descriptor inspection before generic Blosc2 object decoding +- carrier/source geometry validation + +These controls remain necessary because cache misses still cause outbound +requests selected by uploaded data. + +## Carrier Format + +The RemoteProxy format has not been released, so define version 1 directly as +the self-caching format. No upgrade path or compatibility contract is needed +for the earlier development-only reference format: + +```python +{ + "kind": "remote_proxy", + "version": 1, + "source": { + "kind": "fsspec", + "version": 1, + "urlpath": "https://datasets.example.org/array.b2nd", + }, + "cache_policy": "disk", + "max_cache_bytes": 268435456, +} +``` + +The ordinary B2ND chunk slots form the cache. Proxy-owned variable metadata +records the source stamp, fetched bitmap, cached sizes, and any persisted index +needed by the remote reader. + +Do not serialize: + +- server cache limits or server policy +- local paths +- credentials or request configuration +- locks, sessions, or live filesystem objects + +Do not add `proxy-source` metadata merely to trigger the legacy open path. The +`remote_proxy` B2 object kind remains the authoritative discriminator; its +decoder can construct a `Proxy` over the carrier after the source has been +resolved safely. + +## Python-Blosc2 Behavior + +### Creation + +Creating and saving a `NONE` RemoteProxy produces a metadata-sized carrier whose +data chunks are UNINIT: + +```python +proxy = blosc2.RemoteProxy("https://datasets.example.org/array.b2nd") +proxy.save("array-proxy.b2nd") +``` + +To create a self-caching carrier directly, select `DISK`, its path, and an +optional finite limit: + +```python +proxy = blosc2.RemoteProxy( + "https://datasets.example.org/array.b2nd", + cache_policy=blosc2.CachePolicy.DISK, + cache_path="array-proxy.b2nd", + max_cache_bytes=256 * 2**20, +) +``` + +Opening that carrier in append mode uses the carrier itself as the persistent +cache; it does not require a second `cache_path`. Read-only mode can consume +warm chunks but does not retain misses. + +### Reads + +For each requested chunk or block: + +1. Validate or refresh the remote source identity. +2. Serve a valid fetched entry from the carrier when available. +3. Fetch missing compressed bytes from the authorized source. +4. Return the requested logical result. +5. If writes are enabled, retain fetched chunks and enforce the configured cap. + +Eviction replaces complete least-recently-used chunks with UNINIT and updates +the fetched bitmap atomically enough that an interrupted write cannot cause an +unfetched chunk to be trusted. + +### Saving, CFrames, and downloads + +By default, `save()` and `to_cframe()` serialize the current physical carrier, +including valid warm cache data. Downloading the proxy `.b2nd` from Caterva2 +likewise returns the physical self-caching carrier with its warm chunks. A proxy +with no retained data remains metadata-sized. Credentials and runtime server +policy are never included. + +This is distinct from `api/fetch`: fetching returns the requested logical array +or slice, not the physical proxy/cache file. A concurrent physical download must +take the carrier lock or copy under that lock so the downloaded B2ND is +internally consistent. + +An explicit `include_cache=False` option produces a cold carrier without +mutating the warm source proxy: + +```python +proxy.save("cold-proxy.b2nd", include_cache=False) +frame = proxy.to_cframe(include_cache=False) +``` + +Caterva2 exposes the same choice as an optional `include_cache=false` parameter +on its existing physical download operation. Cold export preserves descriptor, +geometry, compression parameters, cache policy, and cache limit, while replacing +cached chunks with UNINIT and clearing fetched bitmaps, cached-size/LRU state, +stored remote indexes, and source stamps. It does not contact the remote source. + +### Source changes + +ETag or another stable source token is stored with the cache. Before serving a +cached entry after reopening, compare the current source token with the stored +token. A mismatch clears the fetched bitmap and accounting before reading data. +Geometry mismatch remains a hard error rather than an automatic rewrite of the +carrier's identity. + +If an HTTPS source supplies no stable validator, persistent reuse across +independent opens is unsafe. Such a source may be read without retention, or its +cache must be treated as empty on every new open. + +## Caterva2 Behavior + +### Existing API surface + +No new HTTP endpoint is needed: + +- upload stores the proxy through the existing upload path +- `api/info` reads local geometry and descriptor metadata without an outbound + request +- `api/fetch` resolves the source under policy and serves slices or indices +- `api/chunk` can serve a compressed chunk while applying the same cache and + quota rules +- physical download accepts `include_cache=false` to export a cold copy + +The physical carrier must not be mistaken for a complete materialized array +frame when serving a whole logical-array fetch. + +### Configuration and customer quota + +The existing remote-source security configuration remains: + +```toml +[server.remote_proxy] +enabled = true +allowed_hosts = ["datasets.example.org"] +timeout = 30 +max_concurrency = 8 +``` + +No Caterva2-specific cache limit is added. `max_cache_bytes` belongs to the +proxy payload and measures retained compressed chunk payload for that carrier; +small fixed metadata/index overhead is excluded. A `DISK` proxy must specify a +positive finite limit. + +Caterva2's existing server-wide `quota` is the aggregate storage bound for one +customer's virtual server. Automatic proxy fills must join the same disk-usage +checking and accounting path currently used by uploads and explicit chunk +writes. The check must cover concurrent fills rather than merely noticing the +larger file during a later state-directory scan. + +### Opening and resolution + +Caterva2 continues to inspect the carrier before `blosc2.open()` can resolve an +untrusted source. After HTTPS authorization and geometry validation, it opens +the carrier in the appropriate mode and constructs the existing Proxy cache +engine over the secure `FsspecNDSource`. + +- `NONE`: misses are not retained +- `DISK`: the carrier is opened append/write; misses populate it and its own LRU + eviction enforces `max_cache_bytes` +- customer quota exhausted: the result is served, but a miss is not retained + +### Concurrency + +Because reads may now write, Caterva2 must synchronize access per carrier. The +safe initial rule is one active cache-mutating operation per carrier, including +across server workers. Read-only metadata inspection need not take the write +lock. Locking must cover source-stamp validation, fetched-bitmap changes, chunk +writes, eviction, and bookkeeping persistence. + +The lock must follow Caterva2's existing dataset mutation/locking conventions +where possible. Process-local Python locks alone are insufficient when multiple +workers can open the same file. + +### HTTP metadata and ETags + +A cache fill changes the physical file mtime but not the logical remote-array +identity. API validators must not accidentally present cache churn as a user +dataset edit. The implementation must distinguish: + +- physical carrier identity, relevant when downloading or backing up the proxy +- logical array identity, derived from descriptor plus current source stamp + +This can initially be conservative by changing the API ETag after cache writes, +but clients must never receive stale logical data. A stable logical ETag is a +later optimization. `api/info` exposes the portable `b2o` descriptor but not +binary fetched bitmaps, source stamps, or other cache-engine bookkeeping. + +## Changes To The Current Caterva2 Slice + +Keep: + +- policy configuration and default deny +- raw pre-resolution carrier inspection +- HTTPS validation, public DNS pinning, and disabled redirects +- geometry/rank/chunk validation +- existing `api/info` and `api/fetch` dispatch +- rejection of embedded remote references + +Replace or revise: + +- replace operation-scoped `ServerRemoteProxy` assembly with a Proxy backed by + the uploaded carrier +- remove the requirement that persisted policy is `none` +- open data requests with controlled write access when caching is enabled +- replace byte-for-byte immutability assertions with bounded-cache assertions +- document physical mutation and source-stamp invalidation + +## Implementation Phases + +### Phase 0: Format and cache prototype + +- [x] Finalize the version-1 self-caching payload and metadata invariants. +- [x] Prototype `Proxy(src, _cache=carrier, _max_cache_bytes=...)` over a RemoteProxy + carrier. +- [x] Confirm eviction reclaims physical storage with bounded overhead. + +### Phase 1: Python self-caching carrier + +- [x] Make the decoded RemoteProxy retain its carrier. +- [x] Reuse the carrier as the Proxy cache for persistent mode. +- [x] Preserve and serialize fetched bitmap, stamp, sizes, and LRU state. +- [x] Make `save()` and `to_cframe()` include valid warm chunks. +- [x] Add non-destructive `include_cache=False` cold exports. + +### Phase 2: Caterva2 cache integration + +- [x] Replace no-retention resolution with the carrier-backed Proxy. +- [x] Integrate automatic cache fills with the existing customer-server quota + checks and accounting; skip retention when no quota remains. +- [x] Add per-carrier cross-worker locking. +- [x] Keep metadata inspection local and source resolution default-deny. +- [x] Validate persisted cache policy and require a finite positive per-proxy limit. + +### Phase 3: Tests and documentation + +- [x] Update Python and Caterva2 API documentation. +- [x] Add cold-read, warm-read, restart, eviction, and source-change tests. +- [x] Add warm and cold physical download/export tests. +- [x] Add concurrent-fill and interrupted-write tests. +- [x] Verify default-deny and HTTPS security tests still pass. +- [x] Verify existing legacy Proxy caches still reopen in the full suites. + +## Test Matrix + +- Empty carrier is metadata-sized and reports complete local geometry. +- First read fetches and embeds the required compressed chunks. +- Repeated covered read causes no remote data traffic. +- Reopening the same file reuses its embedded cache. +- A `NONE` proxy leaves carrier bytes and mtime unchanged. +- Bounded caching evicts whole LRU chunks and stays within payload limit plus + documented fixed overhead. +- `save()`/`to_cframe()` preserve valid warm chunks. +- `include_cache=False` exports a metadata-sized cold carrier without mutating + the warm proxy or contacting its source. +- Physical download preserves valid warm chunks by default and supports a cold + copy, while `api/fetch` returns logical array data. +- Changed source stamp invalidates all old cached entries before use. +- Changed source geometry raises without rewriting carrier identity. +- Missing source validator cannot silently reuse cache across opens. +- Concurrent reads cannot corrupt chunks, bitmaps, or accounting. +- Cache fills count against the customer virtual server's existing quota; when + it is exhausted, reads succeed without retaining additional chunks. +- Caterva2 default deny, destination allowlist, DNS pinning, redirect rejection, + and credential rejection remain effective. +- Existing info/fetch clients need no endpoint changes. + +## Acceptance Criteria + +1. A RemoteProxy `.b2nd` is both the portable descriptor and its persistent + bounded cache. +2. Caterva2 can host it through existing info/fetch APIs. +3. Public HTTPS cache misses are resolved only through server policy. +4. Warm chunks survive restart and avoid remote data traffic. +5. The proxy's compressed-payload cap is enforced by whole-chunk LRU + eviction. +6. Cache mutation never changes source descriptor or geometry. +7. Replaced sources cannot cause stale cached data to be served. +8. Concurrent requests cannot corrupt the carrier. +9. No credentials or server-specific runtime configuration are serialized. +10. Existing persistent Proxy caches retain documented compatibility. +11. RemoteProxy exposes no memory-cache policy; existing generic Proxy memory + behavior remains unchanged. +12. Users can download or export a cold proxy without altering its warm carrier. + +## Settled Product Decisions + +No RemoteProxy format has shipped, so payload version 1 is the self-caching +format and has no upgrade path. RemoteProxy accepts only `NONE` and `DISK`; +`MEMORY` is removed before release. Ordinary `save()`, `to_cframe()`, and +physical proxy downloads preserve valid warm cache data by default. Explicit +`include_cache=False` creates or downloads a cold copy without modifying the +hosted proxy. Logical `api/fetch` operations continue to return array data. diff --git a/plans/remote-proxy-v3.md b/plans/remote-proxy-v3.md new file mode 100644 index 000000000..671819267 --- /dev/null +++ b/plans/remote-proxy-v3.md @@ -0,0 +1,100 @@ +# Plan: Unified RemoteProxy with Memory and Disk Caching (v3) + +## Status + +Implemented in Python-Blosc2. This document supersedes v2 for the client API. + +Review follow-up: fetch/afetch return the proxy after prefetching; materialize +returns an independent NDArray. Reads and exports on one handle are serialized, +and async methods use worker threads (cancellation does not interrupt a running +fetch). Independent handles/processes require external carrier locking. +Floating Caterva2 references force identity refresh, including completed arrays. +Failed cache opens preserve existing files. Explicit export cache_policy selects +a cold NONE, MEMORY, or DISK carrier without changing the live policy. +Memory-only runtime URLs need not be portable, but export and source descriptor +access validate portability. Cache limits exclude peak working memory and results. + +## Purpose + +Unify all lazy remote dataset access under `RemoteProxy`. Previously, lazy opens returned a legacy `Proxy` unless `cache_policy` or `max_cache_bytes` explicitly selected `RemoteProxy`, including when disk storage options were supplied. + +In v3, `blosc2.open(url, lazy=True)` **always** returns a `RemoteProxy`. To achieve this cleanly while maintaining the fast, ephemeral in-memory caching behavior users expect, `CachePolicy.MEMORY` is reinstated as a first-class policy alongside `CachePolicy.DISK` and `CachePolicy.NONE`. + +## Core Decisions + +### 1. Unified Return Type: Always `RemoteProxy` + +`blosc2.open(url, lazy=True)` unconditionally returns an instance of `blosc2.RemoteProxy`: +- When neither `cache_dir` nor `cache_path` is specified: defaults to `CachePolicy.MEMORY`. +- When `cache_dir` or `cache_path` is specified: defaults to `CachePolicy.DISK`. +- When `cache_policy` is passed explicitly: respects the requested policy (`NONE`, `MEMORY`, or `DISK`). + +Users interact with a single, consistent API (`.source`, `.info`, `.traffic`, `.cache_policy`, `.cache_bytes`, `.max_cache_bytes`, `.fetch()`, `.save()`). + +### 2. First-Class Cache Policies + +`blosc2.CachePolicy` provides three explicit retention policies: + +1. **`NONE`**: + - Stateless floating reference. + - Fetches only data required for the current operation and retains no cached chunks. + - `max_cache_bytes` must be `None`. + +2. **`MEMORY`**: + - Ephemeral in-memory cache held in client RAM during the process lifetime. + - Bounded by `max_cache_bytes` (defaults to 256 MiB). + - Automatically applies LRU chunk eviction when retained payload exceeds `max_cache_bytes`. + - Requires no local files or directories (`cache_dir` and `cache_path` must be `None`). + +3. **`DISK`**: + - Persistent carrier cache on disk. + - Bounded by `max_cache_bytes` (defaults to 256 MiB) with LRU chunk eviction. + - Requires `cache_dir` or `cache_path` when creating from a remote URL. + +### 3. Server-Side Protection in Caterva2 +*(Updated in v4: Caterva2 accepts persisted `MEMORY` carriers under opt-in policy but executes them using the same no-retention path as `NONE`, avoiding unmanaged server RAM caching while preserving the requested limit for downloads; older Caterva2 servers still reject `MEMORY` resolution).* + +Caterva2 maintains its strict server-side gate in `caterva2/services/remote_proxy.py`: +- Carriers uploaded to Caterva2 with `cache_policy` other than `"none"`, `"memory"`, or `"disk"` raise `RemoteProxyDenied` (HTTP 403). +- MEMORY carriers execute without retained array-data caching across operations. +- Server resource controls, default-deny policy, and secure filesystem restrictions remain in effect. + +### 4. Direct Carrier Export & Deserialization + +- For `CachePolicy.MEMORY`: + - `save()` or `to_cframe()` exports the carrier structure and metadata with `cache_policy: "memory"`. + - Since in-memory chunks are process-local, only the cold descriptor is persisted. + - When reopened via `blosc2.open("saved.b2nd")`, it initializes as an in-memory `RemoteProxy` ready to cache misses in RAM. +- For `CachePolicy.DISK`: + - `save()` and `to_cframe()` include warm cached chunks by default unless `include_cache=False`. +- For `CachePolicy.NONE`: + - Persisted as a cold reference without data. + +### 5. `fetch()` and `afetch()` Support on `RemoteProxy` + +`RemoteProxy` exposes `fetch(item=None)` and `afetch(item=None)`: +- When caching is enabled (`MEMORY` or `DISK`), prefetches through the cache engine and returns the proxy. Requested chunks may already have been evicted; materialize() returns an independent complete array when needed. +- When `cache_policy` is `NONE`, raises `NotImplementedError`. +- Exposes `cache` property returning the cache container or `None`. + +## Implementation Tasks + +1. **Enum & Policies (`src/blosc2/__init__.py`)**: + - Re-introduce `MEMORY = "memory"` in `CachePolicy`. + +2. **RemoteProxy Implementation (`src/blosc2/remote_proxy.py`)**: + - Update `_normalize_limit` to accept `_POLICY_DEFAULT` (256 MiB) for `CachePolicy.MEMORY`. + - Update `__init__` validation: ensure `cache_dir` and `cache_path` are only used with `DISK`. + - In `_attach_carrier_cache`: instantiate an in-memory `Proxy` with `_max_cache_bytes` when `cache_policy is CachePolicy.MEMORY`. + - In `_export_carrier`: return `_to_b2object_carrier()` for `MEMORY` (no disk carrier to export). + - In `_from_payload`: decode `CachePolicy.MEMORY` with positive `max_cache_bytes`, instantiating with in-memory cache. + - Add `fetch(item=None)`, `async afetch(item=None)`, and `@property def cache`. + +3. **Open Integration (`src/blosc2/schunk.py`)**: + - In `_remote_proxy_options`: when `lazy=True`, always return options (defaulting to `DISK` if disk options are present, else `MEMORY`). + - In `_open_fsspec_url` and `_open_c2_urlpath`: always return `RemoteProxy` for `lazy=True`. + +4. **Documentation & Tests**: + - Update `doc/guides/remote_arrays.md` and docstrings to describe `CachePolicy.MEMORY`. + - Update tests in `tests/test_remote_proxy.py`, `tests/test_fsspec.py`, and `tests/ndarray/test_c2array_blocks.py`. + - Run full pytest suite and lint checks. diff --git a/plans/remote-proxy-v4.md b/plans/remote-proxy-v4.md new file mode 100644 index 000000000..dd9637101 --- /dev/null +++ b/plans/remote-proxy-v4.md @@ -0,0 +1,372 @@ +# Plan: Caterva2 executes MEMORY RemoteProxy carriers without retained caching and supports unbounded DISK cache (v4) + +## Status and decision + +Implemented in Caterva2 and Python-Blosc2. This extends v3's client API, +replaces its server rejection of MEMORY carriers, and adds support for unbounded +persistent DISK caching (`max_cache_bytes=None`). It does not supersede the +existing HTTPS security boundary. + +1. Keep Python-Blosc2's lazy remote-open default at MEMORY with a 256 MiB retained + compressed-payload limit. Accept persisted MEMORY carriers in Caterva2, but + execute them using the same no-retention path as NONE. Preserve the original + MEMORY policy and limit in the uploaded file and its downloads. +2. Support `max_cache_bytes=None` for `CachePolicy.DISK` in both Python-Blosc2 and + Caterva2. Passing `None` disables LRU cache eviction (unbounded cache size). + On Caterva2, an unbounded DISK cache operates without eviction when no server + customer quota is set. With a customer storage quota, carriers are read-only: + valid warm chunks are reused and misses are served without retention. + +No server memory-cache registry, aggregate memory-cache quota, new configuration +knob, carrier format version, or Python-Blosc2 runtime-construction hook is needed. +This removes retained server data caching for MEMORY, not temporary memory use. + +## Inspected baseline + +Inspected on 2026-09-05: + +- Python-Blosc2 checkout: `/Users/faltet/blosc/python-blosc2`, HEAD + `dfb193e706f4f5801d8116cc4941aada8204cae9`, including the local v3 review changes. +- Caterva2 checkout: `/Users/faltet/ironArray/caterva2`, HEAD + `7478babd6e66dd32413ebe20691f931a8355215c`. No tracked diff was reported there + during inspection. + +Caterva2 file paths below are relative to that repository. Function names are +the primary implementation anchors; line numbers may move. + +### Current resolver and runtime construction + +`caterva2/services/remote_proxy.py` already separates secure resolution from +runtime execution: + +1. `inspect()` opens the physical carrier with `raw_carrier()`, inspects its B2 + marker and payload, and avoids generic B2-object decoding. Inspection uses + carrier locks; a lock sidecar may be created even for observational access. +2. `_validated_source()` checks default-deny configuration, exact payload/source + fields, format versions, cache policy, and URL restrictions. It currently + accepts only NONE and DISK. DISK requires a positive integer limit, excluding + booleans; NONE requires a null limit. +3. `resolve()` validates public DNS answers, creates a pinned HTTPS filesystem, + and passes that exact filesystem to `blosc2.FsspecNDSource(..., _filesystem=fs)`. + It validates source/carrier geometry and rank, logical-byte, and chunk limits. +4. It returns `ServerRemoteProxy(source, expected, carrier, payload)` directly. + It does **not** construct a Python `blosc2.RemoteProxy` from a URL or decode + its persisted policy through `RemoteProxy._from_payload()`. + +`ServerRemoteProxy` currently copies the payload's `cache_policy` and +`max_cache_bytes`. Its `_backend()` uses a temporary +`blosc2.Proxy(self.src, _refresh_source=False)` whenever policy is not DISK or +the operation's disk-cache allowance is zero. That temporary cache is local to +the operation. `read()` assembles the result through it, while `get_chunk()` +reads directly from the authorized source on this path. + +Only the DISK path opens the carrier in append mode and attaches it as `_cache`. +`current_cache_bytes()` already returns zero for non-DISK policies. + +**Consequence:** accepting MEMORY in validation would already reach the desired +non-disk branch. Nevertheless, explicitly normalizing the runtime policy makes +the contract visible and avoids relying on an accidental "anything but disk" +fallback. + +### API, quota, metadata, and download integration + +- `services/server.py::open_b2()` calls `inspect()` and `resolve()` before generic + `blosc2.open()`, translating resolution denial to HTTP 403. +- `remote_proxy_cache_limit()` returns zero for non-DISK runtimes. DISK remains + subject to the carrier cap and existing customer storage quota. +- `read_remote_proxy()` applies dataset locking and disk-growth accounting to + slice/index reads. The `api/chunk` branch similarly obtains a cache allowance + and invokes `ServerRemoteProxy.get_chunk()` in a worker thread. +- `services/srv_utils.py::read_metadata()` inspects the raw carrier without + resolving its source. `api/info` marks RemoteProxy data as + `accept_ranges="none"` and exposes only its portable `b2o` variable metadata, + removing cache-engine bookkeeping from the response. +- The physical download path inspects the carrier and calls `export_cframe()` + under locks. Warm export serializes the raw carrier; cold export constructs a + cold carrier with the original payload. Neither resolves the source. + +These paths already support the separation between logical reads and physical +downloads required by this proposal. + +## Behavior contract + +| Stored policy | Stored limit | Caterva2 effective policy | Retained server data | Python reopen | +| --- | --- | --- | --- | --- | +| NONE | null | NONE | None | NONE | +| MEMORY | positive integer; default 268435456 | NONE | None between operations | MEMORY with original limit | +| DISK | positive integer | DISK | Bounded fills without quota; read-only cache with quota | DISK with positive limit | +| DISK | null (unbounded) | DISK | Unbounded fills without quota; read-only cache with quota | DISK with unbounded cache (no eviction) | + +For a MEMORY carrier: + +- Each independent read fetches the required upstream data again; there is no + reusable server array-data cache across reads or requests. +- Temporary compressed assembly, metadata, transport state, decompression + buffers, and output may exist. Do not assert that literally no memory is used + or that every HTTP request is duplicated identically across operations. +- The stored positive limit is validated and preserved, but does not set a + runtime memory budget or allocate that amount on the server. +- Neither reads nor quota handling populate the physical carrier or change its + payload. Carrier bytes, size, and mtime remain unchanged by those reads; + lock-sidecar activity is outside that invariant. +- Physical downloads preserve the requested policy and limit. A cold download + also preserves them; `include_cache=false` is not a policy override. +- Embedded chunk data in a crafted MEMORY carrier is ignored for logical reads. + Physical warm download can preserve those original bytes; cold download drops + cache state. Do not trust them merely because the carrier is structurally valid. +- Actual source replacement between API requests is observed through fresh + resolution. V4 does not promise an atomic snapshot across multiple upstream + reads or add source refresh inside a long-lived ServerRemoteProxy instance. + +For an unbounded DISK carrier (`max_cache_bytes=None`): + +- In Python-Blosc2, chunks fetched into the carrier are retained without LRU + eviction (`proxy.cache_bytes` reflects `carrier.schunk.cbytes`). +- In Caterva2, `_validated_source()` accepts `max_cache_bytes: null` for `disk`. +- When customer quota is enabled, `remote_proxy_cache_limit()` returns zero: + disk caches are consumed read-only and misses use temporary assembly. Payload + limits cannot reserve physical metadata growth across all writers and workers. + With quota disabled, the proxy operates without a limit (`None`). +- `ServerRemoteProxy.current_cache_bytes()` uses `carrier.schunk.cbytes`, not + potentially stale or user-supplied size tables. Unbounded Proxy writes remove + old `proxy-cache-sizes` metadata so later bounded readers rebuild accounting. +- Exporting a DISK proxy to DISK preserves `max_cache_bytes=None`; exporting to + MEMORY falls back to `DEFAULT_DISK_CACHE_BYTES` (since MEMORY requires a finite + positive integer limit); exporting to NONE sets `max_cache_bytes=None`. + +## Implementation steps + +### 1. Accept and validate MEMORY descriptors + +In `_validated_source()`: + +- Keep the exact field, version, source-kind, and default-deny checks. +- MEMORY requires a positive integer; DISK accepts a positive integer or null. + Reject missing fields, booleans, floats, strings, zero, and negatives. +- Keep `"none"` restricted to a null limit and reject unknown policy strings. +- Update denial messages so MEMORY is recognized rather than described as an + unsupported policy. +- Preserve the input payload; do not rewrite it to NONE or remove its limit. + +All URL checks still apply to MEMORY: public credential-free HTTPS only, exact +host allowlist, no query or fragment, no user information, and no custom source +fields. Client support for runtime signed URLs does not extend server support. + +### 2. Make runtime policy explicit in ServerRemoteProxy + +Use the existing constructor with the already authorized source. Proposed +initialization, after validation by `resolve()`: + +```python +self.requested_cache_policy = payload["cache_policy"] +self.requested_max_cache_bytes = payload["max_cache_bytes"] +self.cache_policy = ( + "none" if self.requested_cache_policy == "memory" else self.requested_cache_policy +) +self.max_cache_bytes = ( + self.requested_max_cache_bytes if self.cache_policy == "disk" else None +) +``` + +Here `cache_policy` remains the execution-facing attribute used by existing +backend and quota branches. Optionally provide a read-only +`effective_cache_policy` alias for diagnostics; do not introduce two mutable +execution-policy fields. Constructor documentation must distinguish requested +and effective values. If adding a mapping helper, reject unknown policies rather +than silently converting them to NONE, and reuse it for optional diagnostics. + +Do not mutate `payload`, `carrier.schunk.vlmeta`, or any process-global resolver. +Do not call `blosc2.RemoteProxy(url)` or `blosc2.open(url)` to perform conversion: +that could recreate the source outside the authorized filesystem boundary. + +Keep `_backend()`, `read()`, `get_chunk()`, and `current_cache_bytes()` on their +existing non-DISK paths. A temporary generic Proxy for slice assembly is correct +NONE behavior; no persistent MEMORY backend is ever attached to the runtime. + +### 3. Audit server consumers without adding cache infrastructure + +Confirm all branches consuming `ServerRemoteProxy.cache_policy` use the effective +value. Preserve `remote_proxy_cache_limit()`'s zero result for effective NONE. +Keep existing dataset and carrier locking; removing locks is not part of v4. +Exercise both full/sliced/fancy-index fetches and compressed chunk requests. + +No automatic cache-growth charge should occur for MEMORY reads. Ordinary upload +storage and existing lock files remain subject to current server accounting. +Keep DISK quota enforcement, invalidation, and serialization unchanged. + +### 4. Preserve metadata and exports + +Keep `api/info`'s stored `b2o` descriptor untouched and `accept_ranges="none"`. +Do not insert requested/effective fields inside `b2o`: Python-Blosc2 validates +that payload's exact field set. + +Recommended initial scope: expose requested/effective values on the internal +runtime and document their mapping; keep the public metadata schema unchanged. +Public diagnostics can be a follow-up. If included now, add a declared optional +response-model field outside the portable descriptor, with client compatibility +tests. Describe it as the configured execution mapping, not proof that resolution +is enabled or that a particular URL is authorized. Metadata must remain local +and usable when resolution is disabled. + +Physical downloads must continue using `export_cframe(carrier, original_payload)` +and must never serialize a normalized runtime object or normalized payload. + +### 5. Documentation and compatibility + +Update Caterva2 `doc/utilities/cat2-server.md` and comments in +`caterva2-server.sample.toml`: MEMORY carriers are accepted under the same source +policy but execute without retained caching. No new TOML option is introduced. + +Update Python-Blosc2 `doc/guides/remote_arrays.md`, +`doc/reference/remoteproxy.rst`, and the status of the v3 server-policy section +when the server change ships. Explain client/server performance differences and +the unchanged 256 MiB client default. Describe older Caterva2 servers as still +rejecting MEMORY resolution, rather than claiming universal support. + +No B2 object version bump is needed: MEMORY is already a Python-Blosc2 v3 policy. +Check Caterva2's declared Python-Blosc2 dependency and release floor before +shipping. Reopening downloaded MEMORY carriers requires a client version with +MEMORY support. Do not infer that version solely from `hasattr(RemoteProxy)`. + +### 6. Support unbounded DISK cache (`max_cache_bytes=None`) + +In Python-Blosc2: + +- Update `_normalize_limit()` in `blosc2/remote_proxy.py`: allow `value is None` + when `policy is CachePolicy.DISK`, returning `None`. Continue strictly requiring + a positive integer for `MEMORY`, and forbidding limits for `NONE`. +- Update `_export_carrier()`: when exporting with `cache_policy=CachePolicy.DISK`, + preserve `self.max_cache_bytes` (which can be `None`), rather than coercing it + to `DEFAULT_DISK_CACHE_BYTES`. When exporting to `MEMORY`, fall back to + `DEFAULT_DISK_CACHE_BYTES` (since MEMORY requires a finite positive integer limit). +- Add `_validate_payload_limit()` helper and update `_from_payload()`: allow + `max_cache_bytes: null` for persisted DISK payloads, while continuing to reject + booleans, strings, zero, and negative values. +- Document unbounded DISK caching in docstrings (`RemoteProxy`, `blosc2.open`) and + Sphinx/Myst documentation (`doc/reference/remoteproxy.rst`, `doc/guides/remote_arrays.md`). +- Add tests in `tests/test_remote_proxy.py`: test initialization, rejection of + `None` for MEMORY, rejection of non-integers/negatives for DISK, persistence round-trip + and payload validation, and verify that chunks are retained without eviction. + +In Caterva2: + +- In `caterva2/services/remote_proxy.py::_validated_source()`: accept + `max_cache_bytes: null` for `disk`, while rejecting invalid types (booleans, + strings, non-positive numbers). Keep `memory` strictly requiring positive integers. +- In `caterva2/services/remote_proxy.py::ServerRemoteProxy`: + - In `current_cache_bytes()`: use physical compressed payload (`schunk.cbytes`) + rather than trusting persisted accounting tables. + - In `_backend()`: handle `self.max_cache_bytes is None` when applying `cache_limit`, + avoiding `TypeError` in `min()`. +- In `caterva2/services/server.py::remote_proxy_cache_limit()`: when + `proxy.max_cache_bytes is None`, return `None` if no customer quota is configured; + if customer storage quota is enabled, return zero and reuse disk carriers read-only. +- Update docs and sample config (`doc/utilities/cat2-server.md`, `caterva2-server.sample.toml`). +- Add tests in `caterva2/tests/test_remote_proxy.py` and `caterva2/tests/test_api.py`. + +## Verification plan + +Extend `caterva2/tests/test_remote_proxy.py` using its deterministic fixtures: + +1. Replace the old blanket MEMORY-rejection expectation in + `test_cache_specification_is_strict`. Add valid MEMORY cases with default and + custom limits, invalid-limit cases for MEMORY and DISK, and unknown policies. +2. Parameterize default-deny and unsafe-destination tests over all three policies + with valid corresponding limits. Preserve private-address, pinned resolver, + redirect-disabled, source-field, and embedded-reference rejection coverage. +3. Extend `test_allowed_source_is_resolved_with_the_secure_filesystem` to MEMORY. + Assert identity of the supplied `_filesystem`, requested MEMORY/effective NONE, + and no call to the ordinary RemoteProxy constructor or generic decoder. +4. Extend `_server_proxy` fixtures to create MEMORY carriers. Read the same slice + twice on one runtime and after reconstructing the runtime. Assert correct + values and upstream chunk/block data calls for each read. Repeat for chunks. + Instrument data operations, not just aggregate metadata/request counters. +5. Compare carrier bytes, size, and mtime before and after reads; allow sidecar + creation. Assert no attached reusable backend, current cache bytes zero, and + zero disk-cache allowance regardless of storage quota. +6. Exercise a MEMORY carrier containing synthetic warm cache bookkeeping/data; + logical reads must still use the authorized source and ignore those chunks. +7. Preserve the payload through both warm and cold `export_cframe()` calls and + reopen the exported MEMORY artifact in Python. Verify its original policy, + limit, correct data, and client cache reuse with a controlled source fixture. +8. Keep existing DISK warm-reopen, cold-export, zero-quota, concurrent-fill, and + secure-filesystem tests passing. Add source/geometry replacement cases across + resolutions for MEMORY without claiming stronger within-operation consistency. + +Extend `caterva2/tests/test_api.py`: + +- Parameterize existing discovery/default-deny and physical-download tests for + MEMORY; info/download must work without outbound resolution even when disabled. +- Add enabled-resolution fetch and chunk coverage using a controlled source and + server fixture. The current API tests use a running server; monkeypatching only + the client test process does not patch that server. Use an in-process fixture + with injection at the authorized filesystem boundary, or a controlled HTTPS + fixture with explicitly test-scoped DNS classification. +- Assert requested descriptor preservation, logical fetch results rather than + carrier placeholders, repeated upstream data reads, no carrier mutation, and + unchanged range-advertisement behavior. + +### Verification results + +The results below record the original v4 implementation. Review follow-up replaces +approximate quota clamping with read-only disk caches on quota-enabled servers, +invalidates stale size tables during unbounded writes, and restores test globals +using monkeypatch. Strict automatic growth under quota remains deferred until all +writers share physical-storage reservations; it is not delivered by payload LRU +limits. Follow-up regressions cover transitions, stale tables, warm read-only hits, +misses, and concurrent reads of different carriers. + +Follow-up verification in the blosc2 environment: 133 passed, 1 skipped in the +combined targeted client/resolver/API run; 213 passed in the broader Python +proxy/fsspec/expression regression run. Ruff lint and formatting checks pass in +each repository using its own working directory and configuration. + +The implementation is verified across both repositories in the `blosc2` conda environment: + +1. **Python-Blosc2**: + - `pytest tests/test_remote_proxy.py`: 62 passed in 1.27s. + - Tested MEMORY/DISK/NONE policy configurations, parameter defaults, strict descriptor validation, + unbounded DISK caching without LRU eviction (`test_unlimited_disk_cache_does_not_evict`), + persisted payload decode/reopen (`test_reference_accepts_disk_policy_with_none_limit_in_payload`), + and invalid payload limits (`test_reference_rejects_invalid_disk_limit_in_payload`). + - `ruff check` and `ruff format --check`: all checks passed cleanly. + +2. **Caterva2**: + - `pytest caterva2/tests/test_remote_proxy.py`: 58 passed in 1.01s. + - Covered default-deny across policies, allowed HTTPS destinations, strict cache specification, + secure filesystem pinning, ServerRemoteProxy MEMORY execution without retained caching, + carrier file immutability, synthetic cached chunk rejection, warm/cold cframe export and Python reopen, + geometry replacement detection, customer quota clamping for bounded and unbounded proxies, + unbounded DISK server caching without eviction (`test_unlimited_disk_server_proxy_caches_without_eviction`), + and concurrent fills. + - `pytest caterva2/tests/test_api.py`: 126 passed, 109 skipped in 5.05s. + Included discovery of NONE/MEMORY/DISK (bounded and unbounded), resolution denial before open, + and in-process ASGI resolution and fetch (`test_remote_proxy_memory_enabled_resolution_fetch_and_chunk`). + - `ruff check` and `ruff format --check`: all checks passed cleanly. + +## Limits and deferred work + +Existing rank, logical-byte, chunk-count, timeout, and concurrency controls remain +in effect. They are not an aggregate process RAM cap or an operation-wide network +budget. MEMORY-to-NONE does not solve large result allocation, many concurrent +requests, multi-worker peak memory, or a source changing during range assembly. +Do not describe it as immunity from memory exhaustion. + +Server MEMORY retention, shared cache registries, memory quotas, signed/private +server sources, nested reference resolution, and broader snapshot guarantees +remain separate future designs. The small policy translation proposed here +should not expand those boundaries. + +## Acceptance criteria + +- [x] Valid public HTTPS MEMORY carriers resolve under the existing opt-in policy. +- [x] Their runtime uses the already authorized source and the existing NONE path. +- [x] Repeated logical reads retain no array-data cache between operations and do not + modify the uploaded carrier. +- [x] Stored/downloaded policy remains MEMORY with its original positive limit, and + a compatible Python client restores MEMORY behavior. +- [x] `CachePolicy.DISK` supports `max_cache_bytes=None` (unbounded cache) in Python-Blosc2 + and Caterva2, bypassing LRU eviction while respecting Caterva2 customer storage quota if configured. +- [x] NONE and DISK behavior and all security gates remain intact. +- [x] No Python-Blosc2 construction hook, memory quota knob, or carrier format change + is introduced merely to implement this mapping. diff --git a/plans/remote-proxy-v5.md b/plans/remote-proxy-v5.md new file mode 100644 index 000000000..3096b79df --- /dev/null +++ b/plans/remote-proxy-v5.md @@ -0,0 +1,453 @@ +# Plan: Shared storage reservations and growable remote proxies under customer quota (v5) + +## Status and objective + +First staged-replacement implementation added to Caterva2. The design below +records the original proposal; the implementation decisions and remaining work +in the next section supersede its open choices. This is not a claim that every +benchmark, optimization, or observability item below is complete. + +## Implementation decisions and remaining work + +- `services/storage_quota.py` uses standard-library `sqlite3`, WAL, FULL + synchronous transactions, schema version 1, and separate `account`, `objects`, + and `operations` tables. Connections are short-lived and never shared between + threads/processes. Usage is summed transactionally rather than duplicated in + independently updated counters. +- Owned `public`, `shared`, and `personal` regular files are charged by `st_size`. + Peer storage retains its separate budget. Media, authentication files, + SQLite/WAL, directories, and lock sidecars are excluded operational storage. + This deliberately replaces the old whole-state-directory accounting boundary. +- Candidates are built in memory, then reserve positive final-size growth and + their entire staging-file size before disk writes. `quota_work_bytes` defaults + to `"1G"` and bounds aggregate coordinated disk staging. It does not bound RAM, + HTTP request spooling, operational metadata, or old inodes held by readers; + customer quota is not a guarantee against filling the underlying volume. +- Atomic replacement is the selected correctness reference. Stable per-path OS + locks fence publishers and recovery; generation checks reject stale snapshots. + No heartbeat or TTL frees a live writer's reservation. Recovery syncs the + surviving target, cleans operation-owned staging, then records actual size. + Admission pressure also attempts recovery of other dead workers' operations. +- A shared initialization barrier fences startup inventory against publishers. + Startup reconciles offline changes and reduced quotas. External online writers + are unsupported. All workers must use the same configuration. Local + `publish_root` cannot point into the server's state directory under quota. +- Quota-enabled uploads/imports, chunk writes, append, expression output, + notebooks, HDF5 proxy creation, copying/moving, deletion, publishing metadata, + and archive extraction use shared admission. Directory/archive operations are + file-by-file, not transactional batches; moves copy before deleting and need + capacity for both. Reserved `.b2lock` sidecars may keep removed directories + from disappearing. Arbitrary out-of-band writes from user code are not covered. +- Both logical slices and compressed chunk requests retain DISK misses only + after admission; denials, contention, and publication failure retain the + fetched result. Finite and unlimited payload caps remain distinct from quota. + MEMORY still executes as NONE; portable descriptors and client defaults do + not change. No new Python-Blosc2 runtime API was required. +- One bounded reclamation pass can cold-replace up to four previously validated + DISK carriers, oldest first, after recovery and an admission retry. Descriptor + and user metadata survive; cache bitmaps, indexes, and payload are cleared + together. Generation checks fence stale victims. Ordinary data is not evicted. +- `StorageQuota.usage()` reports committed, reserved, and working bytes. Public + admin endpoints, persistent denial/recovery counters, precise reclaimable-byte + reporting, finer victim fairness/hysteresis, and optimized partial pruning + remain follow-up work. + +### Validation and performance baseline + +New local tests cover exact admission, metadata overhead, separate staging +budgets, independent-process competition, live-owner recovery fencing, worker +death before/after replacement, offline reconciliation, open reader snapshots, +generation conflicts, source reads on denial/SQLite failure, cross-proxy pruning, +threaded uploads versus fills, and authenticated writer/fetch API integration. +The existing remote-proxy and API tests are also used as regression coverage. +Validation in the blosc2 environment: 89 quota/resolver tests passed with +warnings treated as errors. The authenticated API, chunk-write and HDF5 suites +passed with customer quota temporarily enabled in the isolated test server +(292 passed, 4 skipped); they also passed without quota (292 passed, 4 skipped). +The temporary configuration was restored and was not installed in the checkout. +The publication regression test now waits for its own fill nonce, not a stale +published file left by an earlier test. Basic partial-block-to-whole-chunk +transitions and legacy Python UDF persistence have dedicated quota tests. +This does not yet exercise every host-power-loss boundary, every supported OS, +or a production-volume parallel workload. + +`examples/benchmark_storage_quota.py` in Caterva2 compares staged admission to +the existing non-quota in-place path (not a quota-safe in-place implementation). +One local macOS run with an 8 MiB incompressible array and an in-memory upstream: + +| Chunk size | In-place cold ms/fill | Staged cold ms/fill | Staged warm ms/read | Candidate bytes written | +| --- | ---: | ---: | ---: | ---: | +| 256 KiB | 0.952 | 8.955 | 8.323 | 138,463,971 | +| 1 MiB | 1.712 | 7.918 | 6.958 | 37,756,992 | + +Whole-carrier copying also penalizes warm reads. These are illustrative +microbenchmark measurements, not production performance claims. Efficient warm +reads, a proven-bound in-place path, many-tiny-chunk/partial-block stress tests, +and the broader contention/peak-storage benchmark matrix remain follow-up work. +Large candidates exceeding the staging budget fall back to no retention. + +Restore automatic DISK RemoteProxy cache fills when a Caterva2 customer has a +configured storage quota. Coordinate growth across that customer's proxies, +uploads, and other storage writers, and eventually reclaim disposable cached +chunks across proxies when capacity is needed. + +Keep the v4 read-only behavior as the safe fallback until admission and recovery +are complete: reuse valid warm chunks, serve misses temporarily, and retain +nothing when permission to grow cannot be established. + +## Deployment and execution model + +Confirmed assumptions: + +- One customer's virtual Caterva2 server runs on one host with local storage. +- Its worker processes share one state directory and one customer storage quota. +- Different customers have independent state directories and quota accounts. +- Peer access reaches independent Caterva2 servers on other machines. The local + server accounts for locally retained peer data; no quota database is shared + with the upstream server. +- A proxy does not own a dedicated process or thread. Async requests dispatch + blocking operations to a shared thread pool, and multiple server processes may + handle requests for the same carrier. + +These assumptions permit SQLite coordination across local processes. Sharing the +database across hosts or a network filesystem is outside this design. + +## Existing implementation anchors + +Paths in this section are relative to the Caterva2 repository at +`/Users/faltet/ironArray/caterva2`, inspected during the v4/v5 discussion. + +- `caterva2/services/db.py`: authentication uses SQLAlchemy and aiosqlite with + `/db.sqlite`. The schema currently contains the user table. +- `caterva2/services/server.py::lifespan`: initializes/disposes that database + conditionally on authentication. Storage coordination must not inherit that + dependency on login being enabled. +- `caterva2/services/srv_utils.py::Database`: `/db.json` holds server + state as an in-memory model rewritten to JSON. It is not a transactional quota + store and must not be used for cross-worker reservations. +- `server.py::get_disk_usage`, `get_disk_usage_written`, and + `account_chunk_written`: current accounting uses directory scans and a + process-local counter. These are insufficient for shared admission. +- `server.py::remote_proxy_cache_limit`: currently returns zero under customer + quota. `read_remote_proxy` and the `api/chunk` branch use that allowance. +- `services/remote_proxy.py::ServerRemoteProxy`: receives an already authorized + source, uses the carrier for DISK caching, and supports read-only warm reuse + with temporary miss assembly when cache allowance is zero. +- `server.py::dataset_lock`, `dataset_thread_lock`, and + `remote_proxy.py::carrier_thread_lock`: existing process-local guards accompany + Blosc2 carrier file locks. Preserve their ordering and thread/GIL protections. +- `caterva2/c2cache/peercache.py`: existing peer-cache pruning provides useful + chunk-eviction and recency mechanisms, but its scans and post-growth eviction + do not implement cross-worker storage reservations. + +Python-Blosc2 anchors are `src/blosc2/proxy.py` for fetched bitmaps, physical +chunks, size accounting, and eviction, and `src/blosc2/remote_proxy.py` for the +portable carrier contract. A new physical-growth planning hook may be needed; +it is not assumed to exist today. + +## Proposed architecture + +Introduce a local storage coordinator, backed by `/storage.sqlite`. +Every quota-consuming server mutation asks it for admission. SQLite coordinates +ownership of capacity; per-carrier locks protect file contents and readers. + +```mermaid +flowchart TD + A["Concurrent requests: proxies, uploads, other writes"] --> C["Storage coordinator"] + C <--> DB[("storage.sqlite: usage, objects, reservations")] + C --> D{"Capacity available?"} + D -->|Yes| R["Reserve physical growth"] + R --> W["Mutate under target carrier lock"] + W --> F["Record actual size; release reservation"] + F --> DB + D -->|No| P["Claim disposable cache victim"] + P --> E["Prune under victim carrier lock"] + E --> U["Measure reclaimed space; update usage"] + U --> D + D -->|No eligible victim| N["Proxy read succeeds without retaining misses"] +``` + +Network fetching occurs outside SQLite transactions. Successful reservations +remain recorded while filesystem work runs; they do not require an open database +transaction. Pruning must be bounded so repeated admission retries cannot loop +indefinitely under pressure. + +Use a dedicated database rather than extending authentication tables. This +separates lifecycle, schema migration, and frequent accounting traffic from user +management while reusing the installed SQLite/aiosqlite infrastructure. + +## Accounting contract + +For one quota account, the admission invariant is: + +```text +committed chargeable storage + outstanding reserved growth <= customer quota +``` + +If existing data already exceeds a newly configured or reduced quota, preserve +user data, prohibit positive-growth admissions, and allow reads and safe pruning. +Do not pretend the invariant already holds during that reconciliation state. + +Distinguish two different limits: + +- The carrier's `max_cache_bytes` bounds retained compressed cache payload. A + null DISK limit disables its own LRU bound. +- Customer quota bounds chargeable physical file growth, including carrier + headers, indexes, bitmaps, and metadata. It cannot be implemented by passing + remaining customer capacity as a compressed-payload limit. + +An unlimited carrier has no private cache cap, but it never bypasses customer +admission. MEMORY continues to execute as NONE on Caterva2; no server memory +cache registry or memory quota is introduced. + +### Scope decisions required before implementation + +Define chargeable files once and reuse that definition for migration, scanning, +admission, and reconciliation. Inventory public, shared, personal, peer-cache, +temporary, and internal state files. Existing peer-cache budgets are separate +policies; do not accidentally exclude their physical files from customer quota +if they are currently counted. Either make their writers participate or explicitly +document and implement a different accounting boundary. + +Decide whether quota means apparent file length (matching current `st_size` +accounting) or allocated filesystem blocks. Proposed first version: retain +`st_size` semantics and describe this as chargeable stored bytes, not a guarantee +against exhausting the underlying volume. + +Database/WAL/SHM files, lock sidecars, and working storage need an explicit policy. +Recommended: distinguish managed data quota from bounded operational headroom, +and do not charge recursive growth of the quota ledger through its own ledger. +The operational budget must still be provisioned and bounded. This is a deliberate +accounting-policy decision, not permission to create unlimited temporary files. + +## Database model and lifecycle + +Suggested minimal tables (final SQL and migration strategy are implementation work): + +| Table | Core fields | Role | +| --- | --- | --- | +| `storage_usage` | account ID, quota, committed bytes, reserved bytes, reconciliation state | One account per customer state directory | +| `objects` | stable object ID, relative path, generation, measured bytes, type, cache eligibility, coarse last-use time | Identify storage and pruning candidates | +| `operations` | operation ID, object ID, owner token, expected generation, reserved bytes, state, heartbeat, recovery metadata | Durable reservations and mutation intent | +| `schema_version` | version | Explicit migrations independent of authentication | + +Avoid counting the same reservation both in the usage row and operation rows +without transactional updates and a reconciliation check. Add nonnegative-value +constraints and uniqueness rules for live object mutation claims. + +Initialize storage coordination for quota-enabled servers regardless of login. +Use one database engine/pool per process; do not share connections across process +forks. All workers must see the same configured quota. WAL and a bounded busy +timeout are candidates; configure them explicitly and verify their behavior in +multi-process tests. Use durable transaction settings suitable for reservations. + +Use short write transactions, such as `BEGIN IMMEDIATE`, to check capacity and +create/update the reservation atomically. Handle SQLite contention explicitly; +cache admission may fall back to no retention after bounded retries. + +Do not place network transfers, filesystem scans, carrier copying, or waits for +file locks inside a SQLite transaction. Do not update database recency per block +read; batch/coarsen touches so hot reads do not serialize on the SQLite writer. + +## Growth workflow + +1. Resolve the remote source through the existing authorized filesystem path. + Fetch missing data into operation-scoped buffers outside database transactions. + Apply existing request/resource controls and a bounded working-storage policy. +2. Prepare a candidate change or conservative physical-growth bound. Acquire the + target carrier's existing mutation guards and verify source identity, geometry, + and carrier generation. If preparation used an earlier generation, revalidate + or rebuild before admission. +3. In a short database transaction, verify object ownership/generation and reserve + positive physical growth if capacity permits. Persist mutation intent and an + operation token. Commit the database transaction before filesystem mutation. +4. Perform the admitted mutation under the carrier lock. Never exceed the reserved + physical-growth bound. If more space is required, acquire an additional + reservation before that growth or abandon the candidate safely. +5. Measure the result and finalize in a short transaction: update object size and + generation, adjust committed bytes, consume/release the reservation, and mark + the operation complete. +6. Return the logical result whether retention was admitted or not. Failure of + retention must not turn a successfully fetched result into incomplete data. + +Uploads and ordinary user writes differ at step 6: admission denial returns the +existing appropriate quota error rather than silently losing a requested write. +Replacement credits old storage only when it is actually replaced; two live +copies during staging must not be counted as one if working storage is included. + +## Determining physical growth: prototype before selecting the production path + +### Option A: staged carrier replacement + +Build a candidate carrier, measure its serialized file size, and publish it only +after admission and generation validation. This provides an exact final-size +baseline and keeps rejected candidates from modifying the live carrier. + +Costs and required checks: + +- Copying/rebuilding large carriers per small fill may be prohibitive. Batch + chunk admissions where practical and measure write amplification. +- Temporary space must be bounded before candidate creation; final-size + admission alone does not bound peak physical occupancy. +- Atomic rename does not by itself make SQLite and filesystem state atomic. +- Verify how carrier sidecar locks, already open handles, inode replacement, + downloads, and platform-specific rename behavior interact. All relevant readers + must obey a compatible lifetime/locking protocol. +- Persist enough intent to distinguish an unpublished candidate from a published + replacement whose database finalization was interrupted. Flush/fsync ordering + and parent-directory durability must be specified. + +Use this as the correctness reference, not an assumption that replacing files is +already safe with the existing open-handle behavior. + +### Option B: reserve a proven bound and mutate in place + +Preferable for frequent chunk fills if Python-Blosc2/C-Blosc2 can provide a +reliable upper bound on all physical growth, including metadata and any temporary +rewrite space. A cache update planning/admission hook may be necessary. + +Do not guess a fixed metadata allowance or reserve only compressed chunk bytes. +Specify interrupted-write behavior, accounting updates, and rollback/recovery +before enabling this path. If a reliable bound cannot be established, retain +the staged reference path or skip retention. + +Benchmark both paths on cold and warm carriers, many tiny chunks, large contiguous +frames, partial blocks, and batches. Select the production path from measured +costs and demonstrated correctness. + +## Pruning across proxies + +After shared admission works, permit reclaiming disposable DISK cache payload +from other proxies. Preserve descriptors, geometry, original data, and user +metadata. Cache eviction must use the cache engine's bookkeeping-aware mechanisms, +not raw chunk replacement that leaves fetched bits or indexes inconsistent. + +Start with coarse per-proxy recency and prune chunks in batches. Keep per-proxy +LRU for its own cap. Define priority so a hot proxy cannot repeatedly strip every +other proxy's working set; use hysteresis/bounded work to avoid fill-prune thrash. + +Pruning sequence: + +1. Select and claim an eligible victim in a short transaction. Do not credit the + space expected to be reclaimed. +2. End the transaction, acquire that carrier's guards, and revalidate the claim, + generation, and eligibility. Skip active or unavailable victims after bounded + waiting. +3. Evict a batch safely and measure the resulting physical file length. If pruning + itself needs temporary growth, account for it before starting. +4. Commit the measured reduction and release the claim. Only now can reservations + spend the freed capacity. + +Avoid holding the requesting carrier's lock while waiting for a victim's lock. +Release/revalidate the requester when necessary. Keep one consistent lock order: +carrier guard may enclose short SQLite work, but an open SQLite transaction must +never wait for a carrier. No operation should hold multiple carrier locks for a +routine pruning pass. + +## Crash recovery and reconciliation + +SQLite transactions do not include B2ND file writes. Model operations explicitly, +for example `reserved -> applying -> completed`, with an aborted/recovering path. +Finalize and abort operations idempotently using their unique tokens. + +Recovery must cover: + +- Reservation committed, filesystem work never started. +- Partial in-place write or partially built candidate. +- Replacement published, final size not recorded in SQLite. +- Pruning completed, reclaimed capacity not yet credited. +- Worker cancellation, process termination, host restart, or ledger unavailability. + +A heartbeat timeout is only a signal to investigate. Do not free a reservation +while its owner might still write. Recovery needs carrier-lock acquisition, +owner/generation fencing, and inspection of actual files. An old worker must +verify its token is still authorized before publishing or mutating after a claim +has been recovered. Prefer conservative over-accounting until reconciliation. + +Initial inventory and recovery scans must not race untracked writers. Establish +an initialization/reconciliation barrier, handle multiple workers starting at +once, and define how explicit out-of-band filesystem edits are detected. External +uncoordinated writers cannot be covered by a strict online quota guarantee. +If the ledger is unavailable or inconsistent, serve reads without new retention +and fail quota-controlled user mutations clearly rather than bypassing admission. + +## Implementation stages + +### Stage 1: shared accounting and reservations + +- Add `services/storage_quota.py` (proposed module), schema migration, startup, + shutdown, inventory, and recoverable reservation primitives. +- Audit every physical writer: uploads, URL imports, chunk writes, array/store + creation, replacements, transformations/unfolding, deletion/rename, and any + chargeable peer-cache or background writes. +- Route all quota-consuming writers through the coordinator. Preserve quota + error semantics for explicit user mutations. +- Test contention and recovery independently of remote cache filling. +- Keep quota-enabled remote caches read-only until the writer audit is complete. + +### Stage 2: admit DISK proxy growth + +- Prototype and benchmark physical-growth strategies, including working storage. +- Integrate the selected strategy into both slice/index and compressed-chunk + requests while retaining the authorized source and existing locks. +- Enforce both the private payload cap and shared physical quota. Support null + DISK limits and finite limits equally. +- Retain misses when reservation succeeds; otherwise return fetched results + without retaining them. No automatic cross-proxy pruning is required yet. +- Preserve warm/cold exports, source invalidation, and MEMORY-to-NONE behavior. + +### Stage 3: coordinated cross-proxy pruning + +- Add coarse recency, victim claims, safe batch eviction, fairness, and bounded + admission retry after actual reclamation. +- Permit pruning for explicit uploads as well as cache fills if that product + policy is accepted; never reclaim ordinary user data automatically. +- Add diagnostics for used/reserved/reclaimable bytes, denied cache admissions, + recovered operations, and pruning. Do not expose credentials or source secrets. + +## Test and benchmark matrix + +- Two processes reserve against one quota: e.g. with 100 MiB free, 70 MiB and + 50 MiB requests cannot both be admitted without intervening reclamation. +- Threads, separate event loops, and separate server workers; no dedicated worker + per proxy assumed. Database startup and schema initialization races included. +- Carrier metadata causes more growth than compressed payload; reservation covers + the full admitted change. Include partial-block and many-small-chunk cases. +- Upload competes with proxy fill, two distinct proxies fill concurrently, and + chunk writes compete with both. All use the same account. +- Quota full, quota reduced below existing usage, ledger busy/unavailable, and + no reclaimable chunks: reads still succeed without cache growth. +- A victim is in use, replaced, deleted, or warmed after candidate selection; + no stale decision corrupts data or frees fictitious capacity. +- Kill a worker at each filesystem/database boundary and recover exactly once; + a paused old worker cannot publish after its claim is fenced out. +- Warm cache reads, bounded/unbounded transitions, source replacement, logical + fetches, compressed chunks, and physical downloads retain v4 correctness. +- Working-storage exhaustion, replacement with live readers, pruning failure, + cancellation, and process restart leave valid carriers or recoverable state. +- Measure database contention, transactions per fill, latency, staging peak space, + bytes rewritten per admitted chunk, and pruning churn under parallel workloads. + +Use deterministic local upstream fixtures and multi-process tests on temporary +local state directories. Run Python and test/build commands in the blosc2 conda +environment. No runtime tests are claimed by this planning document. + +## Acceptance and non-goals + +V5 is complete when quota-enabled servers can safely grow DISK proxy caches, +different workers cannot spend the same capacity, physical growth is admitted +before it happens, and interrupted operations reconcile without under-accounting +or corrupted reads. Cross-proxy pruning only credits verified reclaimed space. + +No new carrier format or changed client MEMORY default is required. Distributed +quota coordination, shared network-filesystem SQLite, private/signed upstream +server access, server MEMORY retention, and hard process-RAM limits are out of +scope. SQLite is the local coordination mechanism, not a substitute for carrier +integrity locks or for a defined filesystem durability protocol. + +## SQLite references + +- [Transactions](https://www.sqlite.org/lang_transaction.html): write transaction + admission, BEGIN IMMEDIATE, and contention behavior. +- [Write-ahead logging](https://www.sqlite.org/wal.html): concurrent readers, + single-writer coordination, checkpoints, and the same-host constraint. diff --git a/plans/remote-proxy-v7.md b/plans/remote-proxy-v7.md new file mode 100644 index 000000000..238770e22 --- /dev/null +++ b/plans/remote-proxy-v7.md @@ -0,0 +1,810 @@ +# Remote proxy v7: Caterva2 sparse-cache lifecycle and soft quota + +## Status and objective + +Implementation checkpoint: Caterva2 now uses the sparse backend as its runtime +default. The implementation includes authorized source attachment, shared quota admission, private +generations, coldify, retirement/cleanup, offline pruning, conservative recovery, +and warm export artifacts. Endpoint benchmarks +and their raw samples live in Caterva2's `examples/benchmarks/remote_proxy_v7.md` +and `remote_proxy_v7_results/`. + +This is the first implementation/measurement iteration, not completion of all +acceptance criteria below. It uses full-generation measurement/fsync after writes, +discards interrupted generations rather than resuming their transitions, and +reserves the full staging budget for one warm export. Incremental mutation +reports, bounded inventories, configurable maintenance tuning, large logical +chunk-count/RSS benchmarks, and power-loss validation remain follow-up work. +The rebuilt C-Blosc2 now removes payload-free sparse chunk files on successful +eviction; zero-length leftovers mentioned below apply to legacy/interrupted state. + +The public carrier remains contiguous and portable, while mutable DISK state lives +in private sparse generations. The implementation is applied to Caterva2's server, +quota coordinator, recovery paths, API tests, and benchmark harness. + +The server-boundary benchmark uses real ASGI routes and the rebuilt C-Blosc2. +For an 8 MiB source with a 1 MiB cache, v7 was 2.42x faster on warm hits and +1.98x faster on partial growth, but 0.57x on eviction-heavy churn. For a 64 MiB +source with a 32 MiB cache, v7 was 2.65x faster on cold fill, 8.82x on warm hits, +1.90x on churn, and 6.73x on partial growth. Raw samples and methodology live +in Caterva2's `examples/benchmarks/remote_proxy_v7.md`. + +## Decisions fixed by v7 + +- Runtime layout is `/.remote-cache///`. + Both identifiers are random UUID hex strings assigned by the server. User + paths and source URLs never appear in private filenames. +- The public path remains a contiguous RemoteProxy carrier. It is the portable + descriptor and never becomes a directory. +- An uploaded warm carrier is migrated once into a new sparse generation. Once + that generation is active, it is the only authoritative cache. Caterva2 then + cold-replaces the public carrier. Both copies remain charged until replacement + completes. +- `NONE` and `MEMORY` continue to execute without server retention. Only requested + `DISK` policy creates a runtime generation. +- Cache lifecycle exists even when customer quota is disabled. Quota controls + admission and pruning; it must not be the owner registry. +- The customer quota is soft for runtime-cache fills. Ordinary dataset writes + retain their existing staged admission and `quota_work_bytes` behavior. +- Charge regular dataset files by their existing `st_size` rule. Charge every + entry in an active, retired, or trash-resident sparse generation by allocated + size (`st_blocks * 512`), falling back to `st_size` where allocated size is + unavailable. Include `chunks.b2frame`, `.b2lock`, numbered chunk files, + directories, and zero-length entries' directory allocation. Stable lifecycle + locks under `.storage` remain operational and excluded. +- Cache pruning uses whole-chunk LRU batches. It never chooses ordinary datasets. +- Copy creates a new cache identity and exports a cold carrier, including when + the source is a legacy warm carrier. Move is implemented as + copy-plus-delete and also starts cold at the destination. This deliberately + gives up cached data to keep current non-atomic directory move semantics and + crash recovery simple. +- Removing or replacing a RemoteProxy retires its active generation. The API may + finish after the generation is atomically moved to private trash, but its bytes + remain charged until background or startup deletion completes. +- Rollback ignores private caches and serves public cold carriers safely. It does + not translate sparse generations back into a mutable public carrier. + +## Scope and non-goals + +The first deployment remains one customer state directory shared by local worker +processes on one host. SQLite and file locks on a network filesystem, shared state +across hosts, hard physical-volume guarantees, and conversion of ordinary writes +to in-place mutation are out of scope. + +Do not weaken the existing HTTPS allowlist, DNS pinning, redirect refusal, +credential-free descriptor validation, geometry limits, or embedded-reference +guard. Every request resolves and authorizes the source before consulting a +private cache, including a cache hit. + +Peer cache storage retains its independent quota and pruning implementation. +Media, SQLite/WAL files, staged candidates, authentication state, and lifecycle +locks keep their existing operational accounting classifications. + +## Python-Blosc2 contract required by Caterva2 + +The Caterva2 integration now uses the supported Python-Blosc2 sparse-cache +contract below rather than reaching through `_proxy` internals: + +1. Let `with_sparse_cache()` accept Caterva2's already-authorized + `FsspecNDSource` together with its credential-free source descriptor. It must + retain that source object and its pinned filesystem; it must not reopen the URL + through default fsspec transport. Validate the descriptor used for persistence + against the supplied source. +2. `cache_contains(item=(), *, nchunk=None) -> bool`, evaluated under the sparse + frame lock, so a pure hit avoids a SQLite reservation and filesystem scan. + Also expose a supported cache-operation context that keeps the frame lock held + across checking and reading, or an atomic `read_cached()` returning a hit/result + pair. A separate check followed by an ordinary mutating read is insufficient. +3. `trim_cache(target_bytes, *, max_chunks) -> CacheMutation`, which evicts no + more than `max_chunks` least-recently-used chunks, updates fetched/block/index + state through Proxy, and returns the affected chunk numbers plus payload sizes + before and after. Persist enough LRU ordering in the sparse metadata for a new + process to make the same coarse ordering decision; do not create one SQLite row + per chunk. +4. A mutation result for reads/fetches containing whether storage changed, + affected chunk numbers, payload bytes before/after, and the index/metadata + files that changed. Caterva2 uses this to stat only affected files. +5. A stable warm export to a destination path while holding one consistent cache + snapshot. Confirm that sparse-to-contiguous `save()` has bounded peak memory; + otherwise add a bounded serializer before enabling warm exports. +6. Offline sparse inspection and dirty recovery without constructing a remote + source. Startup must not call the URL-based constructor or perform outbound + requests. If a damaged frame cannot be recovered offline, retire it and let a + later authorized request create a fresh generation. + +Specify mutation results as a supported value type: affected chunk IDs (including +evictions), old/new payload bytes, changed metadata paths, and created/deleted +entries. Paths must be relative to the generation and validated by the server. +The result must include writes performed during attachment/recovery, not just +explicit fetches. Define `target_bytes` and `max_cache_bytes` as compressed cached +payload limits; customer charge additionally includes allocated metadata and +directory space. An exception may follow a partial mutation: retain durable +accounting intent and reconcile even when no mutation result was returned. + +The existing persistent `proxy-dirty` marker remains the payload-integrity fence. +All processes attach with `with_sparse_cache()`, which enables compatible frame +locking, refreshes shared fetched state, and recovers a marker only after obtaining +the frame lock. Caterva2 adds lifecycle and accounting intent around it. + +The current server scans one generation after a mutation when a precise mutation +report is unavailable, but never scans on a hit. This is the safe initial fallback; +replace it with mutation-result accounting after benchmarking millions of logical +chunks and long-running quota convergence. + +## Private paths and API isolation + +Add these settings-derived paths without adding them to `storage_quota.ROOTS` or +any provider/root registry: + +```text +/.remote-cache/ active and retired generation directories +/.remote-cache/.trash/ atomically detached generations awaiting deletion +/.storage/exports/ controlled warm-export artifacts +``` + +Create them mode `0700` where supported. Reject `.remote-cache` and `.storage` +explicitly in path resolution and writable-path helpers even though public APIs +currently accept only `@public`, `@shared`, and `@personal`. Static mounts, root +listing, dataset walking, providers, the web viewer, archive expansion, and HDF5 +unfolding must never traverse these paths. + +Only the remote-cache registry constructs private paths. Validate stored relative +paths as exactly `.remote-cache/<32 hex>/<32 hex>` or +`.remote-cache/.trash/<32 hex>` before opening, renaming, or deleting them. Reject +symlinks at every component and never follow links during measurement or cleanup. + +## SQLite schema version 2 + +Initialize `storage.sqlite` at schema version 2 inside the existing startup +initialization guard. Preserve `objects`, `operations`, and `account` for ordinary +file publication. The existing `objects.cache` column remains for compatibility; +sparse runtime bytes never use it. + +Add: + +```sql +CREATE TABLE remote_objects ( + object_id TEXT PRIMARY KEY, + path TEXT UNIQUE, + carrier_generation TEXT NOT NULL, + spec_hash TEXT NOT NULL, + source_stamp TEXT, + active_generation TEXT, + parent_charge_bytes INTEGER NOT NULL DEFAULT 0 CHECK(parent_charge_bytes >= 0), + updated REAL NOT NULL +); + +CREATE TABLE remote_generations ( + generation_id TEXT PRIMARY KEY, + object_id TEXT NOT NULL, + relpath TEXT UNIQUE NOT NULL, + state TEXT NOT NULL CHECK(state IN + ('building', 'active', 'retired', 'trash')), + spec_hash TEXT NOT NULL, + source_stamp TEXT NOT NULL, + max_cache_bytes INTEGER, + payload_bytes INTEGER NOT NULL CHECK(payload_bytes >= 0), + charge_bytes INTEGER NOT NULL CHECK(charge_bytes >= 0), + inode_count INTEGER NOT NULL CHECK(inode_count >= 0), + touched REAL NOT NULL, + created REAL NOT NULL +); + +CREATE UNIQUE INDEX one_active_remote_generation +ON remote_generations(object_id) WHERE state='active'; + +CREATE TABLE remote_operations ( + id TEXT PRIMARY KEY, + generation_id TEXT NOT NULL UNIQUE, + kind TEXT NOT NULL CHECK(kind IN + ('build', 'fill', 'prune', 'retire', 'delete', 'coldify')), + estimate INTEGER NOT NULL CHECK(estimate >= 0), + previous_charge INTEGER NOT NULL CHECK(previous_charge >= 0), + details BLOB NOT NULL, + started REAL NOT NULL +); + +CREATE TABLE remote_work ( + id TEXT PRIMARY KEY, + kind TEXT NOT NULL CHECK(kind IN ('export', 'rebuild')), + reserved INTEGER NOT NULL CHECK(reserved >= 0), + relpath TEXT NOT NULL, + started REAL NOT NULL +); + +CREATE TABLE remote_orphans ( + id TEXT PRIMARY KEY, + relpath TEXT UNIQUE NOT NULL, + charge_bytes INTEGER NOT NULL CHECK(charge_bytes >= 0), + inode_count INTEGER NOT NULL CHECK(inode_count >= 0), + updated REAL NOT NULL +); +``` + +Add `cache_fill_suspended INTEGER NOT NULL DEFAULT 0` to `account`. Use `NULL` +quota internally to mean unlimited if the coordinator is instantiated only for +lifecycle. Alternatively retain the public `settings.quota == 0` convention and +branch admission before reading the account limit; do not encode unlimited as an +arbitrarily large integer. + +Maintain the invariant that `remote_objects.active_generation` is either NULL or +names that object's sole `state='active'` row. Change both fields in the same +transaction whenever a generation activates or retires. + +`remote_objects.path` is the current binding, not permanent ownership history. +On deletion/replacement set the old object's path to NULL and retire its active +generation in the same transaction. Its UUID continues to own retired/trash rows +while a new object can claim the same public path. Keep the former path in the +operation details for recovery. Every lookup must revalidate its binding after +acquiring the object lock; acquiring a lock for an obsolete lookup is not enough. + +`remote_operations.details` is a versioned, credential-free msgpack record. For +build/coldify it records the expected public signature, expected cold artifact +digest, spec/stamp, and associated ordinary-publication operation ID; for +retire/delete it records old and intended trash paths. Insert intent before the +first filesystem mutation. Recovery must distinguish its own completed coldify +from an unrelated replacement with the same descriptor. + +`spec_hash` is SHA-256 over canonical msgpack or sorted JSON containing the +credential-free RemoteProxy payload, shape, dtype string, chunks, blocks, and +runtime cache-format version. `source_stamp` is the stable validator observed from +the already-authorized source. `carrier_generation` is the existing +`storage_quota.signature()` JSON for the public carrier. + +Do not enable SQLite foreign-key cascades for filesystem ownership. Filesystem +removal must be recoverable and charged until it succeeds; explicit state +transitions are clearer than a database cascade that forgets live bytes. + +Extend `StorageQuota.usage()` diagnostics with: + +```text +dataset_used sum(objects.size) +remote_cache_used generation charge + object-parent charge + orphan charge +used dataset_used + remote_cache_used +reserved existing publication growth + remote operation estimates +working existing candidates + remote_work reservations +cache_fill_suspended +``` + +Keep existing keys for API compatibility. + +Update every ordinary `StorageQuota.publish()` admission query to include +`remote_cache_used` and outstanding remote estimates. Likewise, remote fill +admission includes ordinary publication reservations. This is one shared account: +an upload and a cache fill cannot independently spend the same headroom. When no +quota is configured, the coordinator still provides compare-and-swap publication +for internal coldify and lifecycle operations but skips customer-capacity denial. + +## Locks and ordering + +Use stable lifecycle locks under `.storage`, keyed by object or generation UUID, +in addition to Python-Blosc2's sparse-frame `.b2lock`: + +- Dataset path locks protect public carrier compare-and-swap. +- Object locks protect path binding, active-generation replacement, and coldify. +- Generation locks protect registry state, retirement, trash rename, measurement, + and deletion. +- Sparse frame locks protect chunks and their index/bitmaps. + +Rules: + +1. Never acquire an OS lock from inside a SQLite transaction. +2. For multiple dataset paths, acquire path locks in lexical relative-path order. +3. Acquire dataset path, object, generation, then sparse frame lock in that order. +4. Begin any short SQLite transaction only after required OS locks are held; no + code elsewhere may hold SQLite and then wait for those locks. +5. Perform DNS, network, serialization, directory walking, fsync, rename, and + deletion outside SQLite transactions. +6. Never use elapsed time to steal ownership. Recovery attempts the stable OS lock + non-blocking; failure means a live worker may still own the operation. + +The initialization guard precedes path/object/generation locks. Acquire global +migration/prune guards before their subordinate locks and only non-blocking. +Refactor `publish()` into an outer lock-acquiring wrapper and an internal +`publish_locked()` with an explicit caller-held path-lock contract. Coldify and +CRUD hooks call the latter; reacquiring the same file lock through a second file +descriptor can deadlock. Do not trigger recursive global pruning from inside a +generation guard. Queue it after releasing request locks. + +For v7, hold an exclusive generation guard for every sparse operation, including +attachment, cached reads, fills, export, and recovery. Close cache handles before +releasing it; no process may retain an open sparse handle across retirement or +trash deletion. This intentionally serializes operations on one generation. +Different generations remain concurrent. Optimize reader leases only after +benchmarking, with a separate documented deletion protocol. + +The process-local `dataset_lock()` remains useful for avoiding event-loop thread +contention, but correctness must depend only on cross-process locks and SQLite. + +## Resolution and generation attachment + +Refactor `caterva2/services/remote_proxy.py` so `resolve()` receives the public +carrier path and storage coordinator. Preserve the current policy validation and +source creation before registry lookup. + +For every request: + +1. Inspect the public carrier without resolving embedded references. +2. Validate the descriptor and authorize the HTTPS destination. +3. Construct the pinned remote source, obtain its stable stamp, and validate + geometry and configured limits. +4. Compute `spec_hash` and snapshot the public carrier generation. +5. Acquire the path lock, recheck the signature against the authorized snapshot, + then find/create the binding under its object lock. If the snapshot changed, + release locks and restart resolution with a bounded retry count; never attach + storage authorized for the previous carrier to its replacement. +6. Reuse the active generation only when object ID, spec hash, source stamp, + requested DISK policy, and runtime format all match. +7. If the source stamp, descriptor, geometry, policy, or externally observed + carrier generation changed, build a new generation and retire the old one. +8. Attach with `blosc2.RemoteProxy.with_sparse_cache(authorized_source, + runtime_cache_path, source_descriptor=payload["source"], + carrier=warm_carrier, max_cache_bytes=requested_limit)`. + +An old `ServerRemoteProxy` retains its generation ID. Before publishing any cache +mutation it revalidates that the row is still active while holding the generation +lock. If it has been retired, it returns source data without retention. Existing +readers may finish; an old writer cannot mutate or reactivate a newer generation. + +A source with no stable stamp always executes without retention. Do not create a +runtime directory for it. + +## First attachment and warm-carrier migration + +Warm migration is lazy: it happens on the first authorized operation after upload, +not during upload, so uploading a descriptor never initiates an outbound request. +Only one process obtains a non-blocking global migration lock before duplicating a +warm carrier; concurrent requests serve its valid warm chunks read-only and fetch +misses without retention until migration finishes. + +Under path, object, and new-generation locks: + +1. Insert a `building` generation and `build` operation with a coarse estimate in + a short transaction. +2. Close the transaction and call `with_sparse_cache(authorized_source, ..., + source_descriptor=payload["source"], carrier=carrier)`. The helper copies only + fetched state validated against the authorized source stamp. +3. Fsync the sparse generation and its parent, measure its full charge, and mark it + active. Retire any previous active generation in the same transaction and + release the estimate. +4. Build a cold contiguous carrier from the immutable public snapshot, preserving + the RemoteProxy payload, fixed metadata, and every non-reserved user vlmeta. + Remove fetched, cache-size, proxy-index, source-stamp, and dirty bookkeeping. +5. Cold-replace the public carrier through `StorageQuota.publish()` using its + original signature. This is a shrinking ordinary write and retains existing crash + recovery. +6. Update `carrier_generation` to the resulting signature. If compare-and-swap + loses to a user replacement, keep the new sparse generation retired and never + bind it to the replacement. + +The active sparse cache is authoritative from step 3. A crash before coldify may +leave both warm copies, which is safe and fully charged. Startup recovery retries +coldify only when path, descriptor hash, source stamp, and recorded public +generation still match. It never reimports the warm carrier into an existing +generation. + +Warm migration is necessary lifecycle work rather than a discretionary cache fill, +so customer quota may be exceeded temporarily by the duplicate. Record its full +estimate so other workers do not interpret that headroom as free. Before copying, +check configured operational free-space headroom; on denial, ENOSPC, or another +failure, retire and clean the partial generation, leave the public warm carrier +unchanged, and continue serving it read-only. A later request may retry migration. + +Use `publish_locked()` for step 5 because the path guard is already held. Record +a `coldify` intent before publishing, with its ordinary-publication operation ID +durable before candidate rename. Reconciliation recovers this intent before +interpreting a changed public signature as an external replacement. A failed +coldify after successful activation leaves the active cache authoritative and +retries only coldify; it must not restart seed migration. The partial-generation +cleanup fallback above applies to failures before activation. + +Pending coldify is persistent work even after a successful build has released its +estimate. Atomically replace the build operation with coldify intent on activation; +the unique operation-per-generation constraint must never erase unfinished work. +Subsequent requests serve that generation read-only until coldify completes, so a +fill cannot overwrite its recovery record. Back off retries on staging failures. + +## Soft-quota fill protocol + +Route `ServerRemoteProxy.quota_read()` through the sparse generation for every +retained DISK cache. A denied or failed retention attempt falls back to a +no-retention read and still returns the logical result. + +For a logical slice or compressed chunk request: + +1. Authorize and resolve the source as described above. No cache hit skips this. +2. Acquire the generation guard, revalidate its active binding, and keep it until + handles are closed and accounting is finalized. Check and read a hit atomically + under the frame lock; update coarse recency at most once per ten seconds and + avoid SQLite admission. Recency persistence is best-effort and cannot turn a + successful cached read into an error. +3. For a miss, compute a conservative estimate from missing compressed blocks or + chunks and allocated metadata growth. Bound only the payload component by + remaining per-proxy `max_cache_bytes` where finite. Estimates + coordinate workers; they are not physical-growth guarantees. +4. In a short `BEGIN IMMEDIATE` transaction, sum dataset charge, generation + charge, existing reservations, and remote estimates. If fills are suspended or + projected usage exceeds quota, refuse retention. Otherwise insert a `fill` + operation. With no configured quota, insert intent with estimate zero for + recovery/accounting only. +5. Close the transaction, then call the supported combined fetch/read-and-mutate + operation under the generation and frame guards. Durable SQLite intent must + precede all writes; Python-Blosc2 brackets payload writes with its dirty marker. + V7 does not assume a separate fetch/publish API. Holding these guards across + network I/O is acceptable initially; benchmark its same-cache contention. +6. Enforce requested `max_cache_bytes` within that operation and obtain the logical + result plus mutation report, including any evictions. A request larger than the + per-proxy limit must still return its full result without retaining all of it. +7. Stat only affected payload and metadata entries, fsync as required, and update + `payload_bytes`, `charge_bytes`, `inode_count`, and `touched`; delete the + operation in one short transaction. +8. If actual aggregate usage exceeds quota, set `cache_fill_suspended=1` and queue + bounded pruning. Return the already assembled logical result. + +Any SQLite admission failure, busy timeout, stale generation, cache-write error, +ENOSPC, or retention denial falls back to a no-retention read. If data was already +assembled, return it directly; otherwise refetch without retention. Upstream +authorization, validation, or fetch errors still propagate normally. + +Use a shared low watermark of 90% of configured quota. Once suspended, admit no +new discretionary cache fill until pruning or reconciliation observes usage at or +below that watermark. If ordinary dataset bytes alone exceed the watermark, cache +fills remain suspended. Do not claim a fixed maximum overshoot. + +Admission estimates cover positive allocated growth, including index and directory +changes; a payload limit of zero does not imply zero filesystem overhead. Do not +subtract proposed evictions until they have actually freed charge. A finite +payload cap only bounds the payload component of the estimate. + +## Accounting and reconciliation + +Maintain charge from Python-Blosc2 mutation results where available. The current +release still performs a full-generation measurement after mutations as a safe +fallback; a cache hit does not stat every chunk. Coarse recency updates occur at +most every ten seconds. + +Run full reconciliation: + +- at coordinator startup under the initialization guard; +- after recovery of a dead operation; +- after an unexpected mutation/accounting error; +- periodically, default every five minutes, in bounded generation batches; +- on an administrator diagnostic request. + +Reconciliation walks `.remote-cache` without following symlinks, measures files +and directories, repairs charge/inode counts, discovers registered directories +missing on disk, and moves unregistered validly-named directories to trash before +deletion. It never adopts an orphan as active from pathname alone. + +It also checks every `remote_objects.path` against the current public file without +making outbound requests. A missing file, changed carrier signature, non-RemoteProxy +replacement, or changed descriptor/geometry retires the bound generation. A +matching descriptor whose remote source changed is detected later during an +authorized request, when obtaining the source stamp is permitted. + +Skip NULL path bindings. Reconcile under the corresponding generation guard and +recover pending lifecycle/publication intents before comparing signatures. Do +not classify a live `building` directory as an orphan. Record discovered orphan +charge and durable cleanup intent before trash movement; missing files release +charge only after guarded verification. Charge object-parent directories once +per object in a separate accounting total included in `remote_cache_used`, rather +than once per generation. Shared cache/trash roots remain operational overhead. + +Missing active storage retires the generation and makes subsequent reads cold. +Malformed paths, symlinks, descriptor mismatches, or unreadable sparse metadata +are quarantined in trash and remain charged until removed. A reduced quota sets +fill suspension and schedules pruning; startup does not delete ordinary datasets. + +## Batched pruning + +Trigger pruning after an overshooting fill, quota reduction, periodic +reconciliation above quota, or an admission refusal. Only one process obtains the +non-blocking global prune lock; others return their logical results. + +Select active DISK generations ordered by coarse `touched`, excluding the +generation serving the triggering result. For each candidate: + +1. Acquire its generation lock non-blocking and revalidate active state. +2. Compute bytes still needed to reach the aggregate low watermark and call + `trim_cache(max(0, candidate_payload_bytes - needed), max_chunks=64)`. +3. Finalize affected-file charge and recency in a short transaction. +4. Stop after 64 chunks total, four generations, 100 ms of mutation work, or + aggregate usage at/below the low watermark, whichever comes first. + +Schedule another batch if still over quota. Empty generations remain valid cold +caches; zero-length chunk files and directory/index overhead remain charged. +Whole-generation retirement is a fallback only for corrupt, stale, or deleted +objects, not normal quota pressure. + +The triggering generation is excluded only while its request guard is held. A +later background batch must include it; otherwise a customer with one cache can +remain suspended forever. Recompute actual allocated bytes after each batch: +payload bytes evicted are not necessarily physical bytes reclaimed. If no payload +can be reclaimed, stop rescheduling immediate batches, retain suspension, and +report irreducible metadata/dataset charge. Retry on periodic reconciliation or +a storage change. The 100 ms limit is checked between chunks and cannot bound a +single blocking filesystem operation. + +Record a `prune` intent before calling `trim_cache()` and leave it for recovery +if eviction or accounting fails. Exclude generations with pending coldify/build +work; the unique operation row is never replaced by a competing operation. + +## Export and download behavior + +`include_cache=false` reads the public cold carrier snapshot and returns it without +opening or mutating the sparse generation. Before first migration, when the public +carrier may still be warm, build a cold snapshot locally by stripping its cache +state without resolving the source. Preserve requested policy and limit, user +metadata, and existing credential-free descriptor semantics. + +Default warm export uses a controlled artifact: + +1. Authorize the source and bind the active generation. +2. Reserve estimated artifact bytes in `remote_work` against + `quota_work_bytes`. A cold export remains available if this reservation fails. +3. Create `.storage/exports/.b2nd` with `O_EXCL` and mode `0600`. +4. Under the generation/frame snapshot guard, serialize a contiguous carrier that + merges valid runtime state and preserves public user metadata. +5. Fsync and close the artifact, record its exact length and a strong ETag derived + from its bytes or immutable operation identity plus digest, then release the + generation guard. +6. Serve all full and range responses from that one artifact. Never range-read a + changing sparse directory. +7. Remove the artifact and `remote_work` row after response completion or + cancellation. Startup removes artifacts whose OS owner lock is obtainable. + +Acquire a stable per-export owner lock before inserting its reservation and hold +it through streaming and cleanup. Preserve actual artifact charge/reservation if +unlink fails. Cold snapshots that need serialization use the same work budget and +cleanup protocol. Reserve work jointly with ordinary publication candidates; +adjust an underestimated reservation before further growth or abort the export. +A staging failure returns a documented capacity error for a requested warm export; +do not silently substitute a cold result. Each HTTP request creates its own +snapshot: honor `If-Range` against that snapshot's ETag and return a full response +when it differs. Cross-request artifact reuse is outside v7 scope. + +Do not hold a SQLite transaction while serializing or streaming. Export artifacts +are operational working storage and not customer `used` bytes, but they consume +the separately configured work budget. Document that soft customer quota cannot +prevent ENOSPC and that warm export may fail when staging headroom is unavailable. + +## Dataset lifecycle + +Route every coordinated public/shared/personal mutation through remote-cache +lifecycle hooks in `server.py` and `storage_quota.py`: + +- **Upload or replacement:** after successful public publish, compare the prior + registry binding. Retire old generations even when the new carrier has the same + URL or descriptor. The replacement receives a new object ID on first resolve. +- **Delete:** acquire the dataset/object/generation guards, remove the public file + through ordinary publication, mark the generation retired, atomically rename its + directory to `.remote-cache/.trash/`, then delete outside the + locks. Charge remains until deletion is measured complete. +- **Directory delete:** retain current file-by-file semantics; each RemoteProxy + child executes the same retirement path. +- **Copy:** export a cold portable carrier locally, including from a legacy warm + source, preserving descriptor, policy, limit, and user metadata. Do not resolve + its source. Never share an object ID or sparse directory. +- **Move:** use current copy-plus-delete behavior. Destination starts with a new + identity; source cache retires after the source generation is successfully + removed. +- **Append/update/resize:** these already reject B2 object carriers where + applicable. Any generic replacement route must still invoke retirement. +- **Customer removal/state cleanup:** retire every `remote_objects` row, move all + generations to trash under their guards, and keep their charge until deletion. + +Centralize these hooks in the storage coordinator rather than adding endpoint-only +cleanup. CLI, web, API, HDF5 workflows, and future writers must receive identical +behavior. + +Remove `remote_objects` only after its public path is gone or no longer a matching +RemoteProxy and all of its generation rows have been deleted. A row with retired +or trash storage remains the ownership anchor during cleanup. + +## Recovery state machine + +At startup and on demand, inspect each `remote_operations` row. Attempt the +generation lock non-blocking; skip it if busy. + +- `build`: if the directory is absent, delete the generation and release estimate. + If present, use offline inspection/recovery to validate recorded spec/stamp and + measure it. Activate only a durably completed build whose public binding still + matches; retire an incomplete or ambiguous build. Source freshness is checked + again on the next authorized request. +- `fill` or `prune`: recover offline under the frame lock. Python-Blosc2 clears untrusted + fetched state left by a dirty owner. Measure the generation, finalize charge, + clear the reservation, and leave it active only if its registry binding matches. +- `retire`: complete the active-to-retired transition and trash rename. +- `delete`: finish trash deletion, then remove the charged generation row. +- `coldify`: recover the public-path publication first, then compare the resulting + carrier. Update its generation only if signature/digest and publication intent + identify the expected cold artifact; otherwise + retire the cache rather than binding it to unknown bytes. + +Recovery is idempotent. At every state, the public cold/warm carrier remains an +independent source descriptor, so discarding a private generation cannot lose user +data. Never trust a fetched bit merely because SQLite says a fill completed; the +sparse frame and its dirty marker are authoritative for payload integrity. + +## Configuration, diagnostics, and operations + +The runtime has no public cache-backend selector. Sparse v7 is always used for +retained DISK caches; `NONE` and `MEMORY` retain nothing on the server. Keep only +the operational tuning settings that are implemented: + +```toml +[server.remote_proxy] +cache_maintenance_seconds = 60 +cache_low_watermark = 0.90 +cache_reconcile_seconds = 300 +cache_prune_chunks = 64 +cache_prune_generations = 4 +cache_min_free_bytes = "1G" +``` + +Sparse mode requires schema v2 and the pinned Python-Blosc2 API. The public carrier +remains the rollback-safe cold descriptor: discarding a private generation never +requires translating it back into a mutable public carrier. + +The free-space check requires `free - estimated_operation_bytes` to remain above +`cache_min_free_bytes` before migration, rebuild, or export starts. It is a +best-effort ENOSPC guard rather than a reservation against unrelated processes. + +Expose authenticated diagnostics containing aggregate dataset/cache charge, +outstanding estimates, work reservations, suspended state, active/retired/trash +generation counts, inode count, oldest recency, overshoot bytes and age, recovery +count, prune work, and last reconciliation. Do not expose source URLs, opaque +filesystem paths, or cache contents in the web viewer. + +Log generation IDs and hashed object IDs, not credential-bearing URLs. Emit +metrics for fill hit/miss/refusal, actual-versus-estimated growth, dirty recovery, +pruned chunks/bytes, trash backlog, export size/time/RSS, quota overshoot magnitude, +and time to return below the low watermark. + +Start maintenance tasks in the server lifespan and cancel/join them on shutdown. +Each worker may wake a task, but non-blocking global guards elect one executor for +each batch. Run blocking cache/filesystem work in the existing worker-thread path, +never on the event loop. Keep a persistent cursor for bounded reconciliation; +resume failed trash deletion with capped exponential backoff. Cancellation of an +HTTP request must not release generation/owner locks while its worker thread still +mutates storage. Let it finish accounting or leave recoverable intent before +closing handles. Cleanup failures remain visible and charged. + +Define the durability boundary explicitly: after intent commit, write and fsync +payload/index state in the order required by Python-Blosc2, durably clear its dirty +marker, and only then finalize SQLite accounting. Fsync both parents of trash +renames. Validate this ordering against the actual helper implementation before +claiming power-loss safety; SIGKILL tests alone establish process-death behavior. + +## Implementation sequence + +1. Land and pin the supported Python-Blosc2 accounting, pruning, and export APIs. +2. Add schema-v2 lifecycle initialization and combined dataset/cache accounting. +3. Implement private path validation, locks, operation recovery, reconciliation, + generation creation, warm migration, coldify, retirement, and trash cleanup. +4. Route sparse hits/fills, logical slices, and `/api/chunk` through soft + admission with no-retention fallback. +5. Add bounded pruning, hysteresis, maintenance scheduling, diagnostics, and + warm export artifacts with ranges, ETags, cancellation, and work reservations. +6. Run fault-injection, multiprocess, endpoint, and v5 comparison benchmarks. + This sequence is complete for the current release candidate; the remaining + work is listed under follow-up items below. + +## Follow-up items after the current implementation + +These items are useful improvements, but do not block the current sparse-default +release candidate: + +- Replace full-generation post-mutation measurement with incremental accounting + from Python-Blosc2 mutation reports. The current scan is correct and bounded by + the generation size, but it adds latency to fills and pruning. +- Add bounded reconciliation cursors and benchmark millions of logical chunks, + sparse metadata, inode counts, peak RSS, and long-running quota convergence. +- Measure multiprocess throughput and same-generation contention separately from + the existing correctness tests. +- Add power-loss testing for filesystem and SQLite durability. Process-death + recovery is covered; the implementation does not claim crash atomicity after a + power failure. +- Tune staging estimates and warm-export reservations instead of reserving the + full work budget for one export. +- Retain the internal contiguous compatibility path for deterministic comparison + and explicit rollback tests; it is not a public Caterva2 deployment setting. + +## Caterva2 module map + +- `caterva2/services/remote_proxy.py`: authorized-source attachment, generation + binding, sparse read/chunk routing, warm migration, coldify, pruning adapter, + export snapshot, and no-retention fallback. +- `caterva2/services/storage_quota.py`: schema migration, always-available storage + coordinator, combined admission, private measurement, operation recovery, + reconciliation, retirement, trash deletion, and diagnostics. +- `caterva2/services/server.py`: initialize the coordinator, pass public paths into + resolution, replace `quota_read()` routing, and call centralized lifecycle hooks + from write/remove/copy/move paths and download endpoints. +- `caterva2/services/settings.py`, `caterva2-server.sample.toml`, and + `doc/utilities/cat2-server.md`: runtime tuning, hysteresis, reconciliation, work-budget, + accounting, cleanup, and operational-headroom configuration/documentation. +- `caterva2/tests/test_remote_proxy.py`: source policy, sparse behavior, generation + binding, migration, export, and fallback tests. +- `caterva2/tests/test_storage_quota.py` and + `caterva2/tests/test_storage_quota_api.py`: schema, multiprocess admission, + lifecycle, recovery, pruning, endpoint, and quota-accounting tests. +- `examples/benchmarks/remote_proxy_v7.py` and + `examples/benchmarks/remote_proxy_v7.md`: server-boundary v5/v7 comparison using + the same deterministic local range source and machine-readable raw results. + +## Required tests + +Use deterministic local range-capable sources before external HTTPS tests. + +Add targeted regressions for the review's implementation boundaries: + +- Replace/delete/recreate the same public path while old trash cannot be removed; + a new UUID can bind the path and both generations remain charged. +- Pause between hit detection and data extraction while another process attempts + eviction/deletion; no unreserved fill or access to removed handles occurs. +- Crash after cold publication but before binding finalization; recovery recognizes + the exact artifact, while a same-descriptor user replacement gets a new identity. +- Run startup recovery with all outbound transport constructors forbidden. +- Overshoot with one cache, and with only irreducible metadata remaining; pruning + either reaches the watermark or reports suspension without a busy loop. +- Cancel an in-flight read/export with a live worker thread; locks and reservations + remain owned until mutation/stream cleanup ends. +- Verify schema initialization and quota-disabled lifecycle. + +- Two and eight processes filling the same generation, different generations, + and overlapping partial blocks; readers overlap eviction without corrupt or + zero-filled results. +- Competing fills and ordinary uploads cannot all treat the same quota headroom as + free. Actual overshoot is charged and converges through bounded pruning. +- Worker death before/after every dirty marker, chunk truncate/write, index write, + fetched bitmap, accounting finalization, active-generation switch, cold publish, + trash rename, and deletion boundary. +- Recovery never steals a live worker's reservation or generation and is + idempotent across repeated restarts. +- Source replacement with identical geometry creates a new generation. Geometry, + descriptor, policy, and limit changes cannot attach stale storage. +- Authorization executes on cache hits; allowlist removal immediately prevents + use of already-warm data. +- Warm migration preserves valid uploaded chunks and user metadata, coldifies the + public carrier, counts both copies during the transition, and never resurrects + an evicted seed chunk. +- `include_cache=false` is cold and non-mutating. Warm full/range downloads use one + stable artifact and ETag; cancellation, timeout, restart, and ENOSPC remove or + recover reservations and artifacts. +- Delete, directory delete, replacement, move, copy, and customer removal produce + the lifecycle specified above. Private bytes remain charged until trash removal. +- Startup inventory includes registered and orphan private generations while API, + root listings, search, providers, and the web viewer never reveal them. +- Bounded and unlimited per-proxy limits remain distinct from customer quota. + MEMORY and NONE retain nothing. +- SQLite busy/error, malformed registry rows, missing directories, symlinks, + permission failures, and quota disabled/enabled transitions preserve logical + reads and produce reconciled accounting. + +## Benchmarks and acceptance + +Repeat the v6 cold fill, warm hit, LRU churn, and round-robin partial-block tests +through Caterva2 endpoints. Add concurrent same/different proxy fills, millions of +logical chunks with few resident chunks, pruning, startup reconciliation, cold and +warm export, range download, peak RSS, metadata operations, inode count, actual +allocated blocks, estimate error, and quota overshoot duration. + +Accept v7 when: + +- ordinary fills, partial growth, and eviction never rewrite unrelated cached + payload; +- operation memory does not scale with the remaining carrier tail; +- no stale or dirty fetched bit can serve incomplete data after process death; +- every retained private byte is eventually reflected in account usage, and + overshoot converges without failing logical reads; +- delete and replacement cannot leave an unregistered, uncharged active cache; +- public carriers remain portable and private directories remain unreachable from + API and web namespaces; +- warm/cold export semantics, authorization, validators, geometry checks, MEMORY + behavior, and client defaults remain compatible; +- discarding sparse state can serve the public carrier without migration. + +No hard RAM, fixed overshoot, or physical-volume guarantee is implied. Operational +monitoring and free-space headroom remain required even with correct soft-quota +accounting. diff --git a/plans/remote-proxy-v8.md b/plans/remote-proxy-v8.md new file mode 100644 index 000000000..b187cbe2a --- /dev/null +++ b/plans/remote-proxy-v8.md @@ -0,0 +1,371 @@ +# Remote proxy v8: remote arrays with immutable-by-default Blosc2 caching + +## Objective and status + +Status: implemented in this branch. + +Add `ZarrNDSource`, a `ProxyNDSource` adapter that reads remote Zarr arrays and +returns Blosc2 compressed chunks. Reuse the existing `Proxy` and `RemoteProxy` +cache implementations, including bounded memory caches, portable B2ND carriers, +and the sparse runtime caches introduced for Caterva2 in v7. + +RemoteProxy sources are assumed immutable by default. Callers following a mutable +single-file source can opt into identity checks with `assume_immutable=False`. +Zarr remains immutable-only in this version because changes to one object do not +provide an identity for the complete array. Zarr remains an optional dependency, +like fsspec. The adapter belongs in Python-Blosc2 and must be usable without a +Caterva2 server. + +## Fixed decisions + +- Read Zarr through Zarr-Python's public array/store APIs. Do not implement a + second metadata parser, chunk-key encoder, codec pipeline, or shard reader. +- Cache converted Blosc2 chunks in existing B2ND containers. Do not use Zarr's + `CacheStore` or maintain a second cache of Zarr objects. +- Fetch and convert whole logical Zarr chunks. A shard is a storage object, not + the cache chunk shape. Let Zarr resolve chunk reads within shards. +- Support Zarr format 2 and 3 through a supported Zarr-Python 3 release. This + does not require supporting the older Zarr-Python 2 API. +- Support fixed-size NumPy dtypes representable by B2ND, including fixed-width + strings, temporal values, and structured records. Reject object-bearing and + variable-length dtypes explicitly before creating a cache. +- `RemoteProxy(..., assume_immutable=True)` is the default for every source and + skips metadata polling before reads. `False` retains identity refresh and cache + invalidation for mutable single-file and Caterva2 sources. +- A Zarr source identity names immutable contents for the lifetime of its caches. + No per-chunk ETag checks, TTL, or mutable Zarr-store support. +- Apply the same immutable default and persisted flag to Blosc2 and Caterva2 + source descriptors. +- Do not change C/Cython code unless implementation exposes a demonstrated + blocker that cannot be handled by existing Python APIs. + +## Existing code to reuse + +`src/blosc2/proxy_source.py` defines `ProxyNDSource`: shape, chunks, blocks, +dtype, compression parameters, and `get_chunk(nchunk)`. Its optional block-range +interface is unnecessary for this adapter. `Traffic` already provides +thread-safe counters for received bytes and data-bearing requests. + +`src/blosc2/proxy.py` creates a B2ND cache from that geometry, fetches missing +chunks, inserts compressed chunks, and tracks retention and eviction. These +mechanisms should remain the owners of cache state. + +`src/blosc2/remote_proxy.py` handles source descriptors, geometry validation, +source stamps, carriers, and sparse attachment. Its URL-string branch currently +always creates `FsspecNDSource`. Source identity, payload reconstruction, and +authorized attachment currently distinguish only fsspec and Caterva2 sources. + +`src/blosc2/schunk.py` owns the public `blosc2.open()` remote dispatch and cache +options. Changes must reach this shared path, not just the S3 example. + +`pyproject.toml` already defines `zarr = ["zarr"]` and a separate `fsspec` extra. +Reuse these extras. The existing example already opens Zarr stores and measures +their reads, but its Zarr branch returns an uncached Zarr array. + +## Proposed public opening API + +Add `source_format=None` to `RemoteProxy` and accept it through +`blosc2.open(..., lazy=True)`. Initially allow `None`, `"blosc2"`, and `"zarr"`. +Add `assume_immutable=True` to both entry points and persist it in every source +descriptor. Users following replaceable sources must set it to `False`. + +```python +arr = blosc2.open( + "s3://blosc2/cube-1k-1k-1k.zarr", + lazy=True, + source_format="zarr", + cache_policy=blosc2.CachePolicy.MEMORY, + storage_options={"anon": True}, +) +values = arr[:10, 0, :5] + +arr = blosc2.RemoteProxy( + "s3://bucket/hierarchy.zarr/d0/d1/a2", + source_format="zarr", + cache_policy=blosc2.CachePolicy.DISK, + cache_path="array-cache.b2nd", +) +``` + +An explicit format supports URLs without a `.zarr` suffix. With no explicit +format, recognize a `.zarr` path component, including a trailing slash and nested +array paths; retain the existing Blosc2 default for other remote URLs. Inspect +the parsed URL path, not query-string text. Explicit selection wins over the +heuristic. Do not catch arbitrary opening errors and retry another format. + +The URL names the array itself, including its path within any hierarchy. Opening +a group raises an actionable error asking for an array path. Do not recursively +discover or select an array. ZIP-backed Zarr is outside this first version and +must not be misclassified as ordinary directory/object-store Zarr. + +Keep existing cache-policy defaults: direct `RemoteProxy` and `blosc2.open` +already have their own defaults. Adding Zarr must not silently change them. +Reject incompatible format options on Caterva2 `URLPath` inputs and reject Zarr +opening without `lazy=True` in this release. Existing local Blosc2 opening stays +unchanged; a direct adapter can accept a local store for testing. + +## Adapter implementation + +Create `src/blosc2/zarr_source.py` and export `ZarrNDSource` from `blosc2`. +The module may import NumPy and existing Blosc2 helpers at module scope, but must +not import Zarr or fsspec until construction needs them. + +### Construction and metadata + +1. Import optional dependencies with actionable errors. +2. Open a read-only store and use `zarr.open_array(..., mode="r")`. +3. Read metadata only: shape, logical chunks, dtype, and the metadata required + to validate compatibility. Construction must not fetch array payloads. +4. Normalize geometry to Python tuples and dtype to a NumPy dtype. Validate + dimensions, positive chunk extents, dtype support, and Blosc2 size limits + before allocating conversion buffers or cache containers. +5. Set `serves_blocks = False`. Expose the existing concurrency and traffic + conventions so `Proxy` can fetch chunks through its current executor. + +Do not treat Zarr's `arr.blocks` as a block shape: it is an indexing interface. +Choose cache blocks with the existing Blosc2 partition helper, supplying the +chosen chunk shape. Use normal Blosc2 compression defaults initially. These +parameters describe the converted cache, not the original Zarr compressor. + +Cache partitioning must be reproducible across reopening. Persisted carrier +chunks and blocks are authoritative when restoring a cache; pass that geometry +through the adapter reconstruction path and validate it against the source. +Do not accidentally reject old caches because an automatic block-size heuristic +or a default compression setting changed in a later library release. + +### `get_chunk(nchunk)` + +1. Validate the chunk number and map it to C-order logical chunk coordinates. + Reuse an existing coordinate helper if one fits; otherwise use NumPy's + unraveling with the grid obtained by ceiling-dividing shape by chunks. +2. Compute the clipped array slice for that chunk and read it through Zarr. +3. Normalize the decoded result to the advertised dtype and contiguous order. + Preserve numeric values, including endian conversion where needed; never + reinterpret foreign-endian bytes as native values. +4. For edge chunks, place valid values in an initialized full-chunk buffer. + Padding lies outside the logical array and must never contain uninitialized + memory. Missing Zarr chunks inside the logical array are filled by Zarr using + the declared fill value, not by substituting zero in the adapter. +5. Encode a single-chunk B2ND temporary with the cache's exact chunks, blocks, + dtype, and compression parameters, then return `get_chunk(0)` from it. + This delegates multidimensional block layout and padding to Blosc2. + +Keep each conversion buffer and temporary container local to the call. Reuse +`Proxy`'s fetch scheduling; do not add another thread pool or a shared mutable +scratch array. Investigate Zarr's internal concurrency when measuring peak +memory: the outer limit bounds adapter chunk calls, not necessarily every +internal store request. Add an asynchronous adapter method only if an existing +public async path requires it, and use Zarr's supported scheduling mechanism. + +Document the initial memory ceiling: concurrently decoded whole chunks plus +conversion/compression buffers. `max_cache_bytes` limits retained compressed +payload, not transient decoded memory or process RSS. + +### Supported representations + +Cover all fixed-size dtypes supported by B2ND, both Zarr formats, alternative +codec pipelines, nonzero fill values, edge chunks, scalar arrays, zero-length +arrays, and Zarr v3 sharding. Let Zarr decode storage order and transpose codecs. + +Reject object-bearing and variable-length dtype representations with a clear +`TypeError`. Scalar arrays cache their one value; zero-length arrays have no +payload chunks and retain zero cache bytes. Never defer failures until after a +partially written cache. + +## Immutable identity and persistence + +Use a distinct source descriptor, keeping the existing outer RemoteProxy payload +version if its schema remains compatible: + +```json +{ + "kind": "zarr", + "version": 1, + "urlpath": "s3://bucket/hierarchy.zarr/d0/d1/a2", + "assume_immutable": true +} +``` + +Version 1 records `assume_immutable`, which defaults to `true`. Zarr requires it +to remain true in this version. Persisting the option preserves read behavior +after reopening and makes the assumption visible to clients and Caterva2. +Unknown fields, versions, and unsupported values fail closed. + +Generate a stable non-null source stamp from a canonical, credential-free +descriptor plus the adapter encoding version and normalized cache geometry. +Use a deterministic digest, not Python's randomized `hash()`. This stamp denotes +an identity under the immutable contract; it is not a remotely verified content +hash. Include interpretation metadata where practical to detect changed metadata +on reopening, without claiming to detect payload-only mutations. + +Do not implement a remote refresh method that fetches metadata on every read. +Ensure `_prepare_read()` keeps the stable source and cache instead of treating +it as an unstamped source. Existing geometry validation still runs on reopening. +Document that replacing data under the same identity violates the contract and +may serve stale data or mix old cached chunks with newly fetched chunks. A new +dataset should have a new immutable URL; manual cache replacement is the escape +hatch for intentionally reused URLs. + +Update every relevant branch together: `_open_source`, `_source_identity`, +`urlpath`, payload validation/reconstruction, `with_sparse_cache`, carrier +exports, and source-spec comparisons. Preserve live storage options on any +reconstruction within a process, but never persist them. A fresh process must +resolve credentials from its own environment, as with existing sources. + +Check direct `Proxy(ZarrNDSource(...))` metadata too: it must not be labeled as +a local Blosc2 source merely because it exposes `urlpath`. If direct persistent +Proxy caches are supported, add their source reconstruction alongside the +RemoteProxy path; do not produce a cache that saves successfully but reopens +through the wrong backend. + +## Optional dependencies + +- Reuse `blosc2[zarr]`; do not add Zarr to mandatory runtime dependencies. +- Select the minimum Zarr-Python 3 version actually required by the public APIs + used and verify it. Do not require local version 3.3 merely because it is + installed. Adjust the existing extra's version floor only with that evidence. +- Keep fsspec independently optional. Document remote installs as + `pip install "blosc2[zarr,fsspec]"`, adding `s3fs` for S3 as today. +- Import Zarr inside construction or a file-local helper, following the + actionable-error pattern of `core._import_fsspec`. A missing Zarr error should + name `blosc2[zarr]`; missing protocol backends retain their useful errors. +- Importing `blosc2`, using local B2ND arrays, and opening remote B2ND sources + must work without Zarr installed. Local Zarr adapter use should not require + fsspec when the chosen store does not need it. +- Move the S3 example's unconditional Zarr import into the branch that needs + it. Preserve any existing user edits when implementing the example update. +- Optional dependency tests use `pytest.importorskip`; also test the actual + missing-dependency error path separately so skipping cannot hide it. + +## Traffic, example, and performance expectations + +Count encoded bytes received beneath the Zarr decoder. Reuse the example's +instrumentation approach as a reference, but implement the adapter's tracking +through supported store/transport extension points. Cover ordinary reads and +partial/shard reads; avoid double-counting delegated batched operations. +The counter measures received payload bytes, not headers or decoded array size. +Place it below any local cache so a Blosc2 hit charges no remote payload. + +Update `examples/remote/s3-access.py` so directory/object-store Zarr arrays open +through `blosc2.open(..., lazy=True)` and print RemoteProxy cache details. +Display the cache's actual block shape. Keep ZIP behavior explicitly separate. +Metadata, first slice, second slice, and transferred-byte reporting should remain +comparable to the current example. + +The first miss pays Zarr decoding plus Blosc2 encoding. A hit reads the Blosc2 +cache without repeating either remote reads or Zarr decoding. No timing speedup +is promised for a cold read. Warm-cache tests assert zero payload reads, not +a machine-dependent millisecond threshold. Measure cold/warm latency, encoded +network bytes, retained Blosc2 bytes, and transient memory separately. + +## Caterva2 integration boundary + +The Python-Blosc2 implementation comes first. Actual Caterva2 changes belong in +the Caterva2 repository as a subsequent integration phase; v7 remains the cache +lifecycle and quota design. + +Prepare the adapter to accept a server-supplied, already-authorized store or +filesystem internally. Extend authorized sparse attachment to recognize the +concrete Zarr source and validate the exact descriptor against it. Do not accept +arbitrary source objects as implicitly authorized. + +An authorized attachment must retain the supplied transport for metadata, +chunks, shard indexes, and all subsequent reads. It must never reopen the URL +using an unrestricted default filesystem. Keep authorization before cache access, +including hits, as required by the existing Caterva2 contract. + +Caterva2 must authorize an array prefix and validate all derived object paths; +authorizing one metadata URL alone is insufficient for a multi-object source. +Preserve HTTPS allowlists, DNS pinning, redirect refusal, credential isolation, +geometry limits, and embedded-reference restrictions. Confirm the Zarr pipeline +cannot escape the authorized transport/prefix through an unsupported store or +codec; restrict server support explicitly where necessary. + +Once those boundaries are validated, reuse the existing sparse generation, +locking, dirty recovery, eviction, quota, and warm-export paths. Downstream +Caterva2 clients receive ordinary Blosc2 chunks and need no Zarr dependency. +The Caterva2 server reading the external Zarr source needs the optional extra. +An immutable-source stamp does not itself grant authorization. + +## Implementation sequence and checks + +### 1. Adapter and dependency isolation + +Implement the adapter, deferred import, metadata validation, and chunk conversion. +Add local temporary Zarr fixtures in `tests/test_zarr_source.py` and exercise the +adapter through `Proxy`, not only by calling `get_chunk()` directly. + +Verify format 2/3, one and multiple dimensions, exact and edge chunks, missing +chunks/nonzero fill, endian handling, representative supported dtypes, unsupported +dtypes, and sharded format 3. Check scalar/empty geometry explicitly. Include a +codec other than Blosc to establish that conversion does not assume Blosc bytes. + +### 2. RemoteProxy opening and immutable caching + +Implement dispatch, immutable descriptors/stamps, and policy integration. +Use counted local or memory-backed stores for deterministic tests without public +network access. Ensure metadata opening does not fetch chunk data. + +Test explicit/automatic selection, nested arrays, suffix-free explicit URLs, +trailing slashes, group errors, and useful authentication/opening failures. +Test NONE, MEMORY, bounded DISK, and unbounded DISK using existing conventions. +Repeated and overlapping reads must fetch only absent chunks; eviction must +cause a later read to fetch again. A repeated hit must not poll metadata either. +An oversized chunk must obey the existing post-operation retention bound. + +### 3. Persistence and server attachment + +Test cold and warm carrier reopening, MEMORY exports remaining cold, geometry +mismatch rejection, changed encoding identity, source-spec mismatch, and credential +exclusion. Exercise `save()`, `to_cframe()`, materialization, and a simple lazy +expression round trip. Check direct persistent Proxy reconstruction if exposed. + +Exercise sparse attachment and warm seeding with an authorized fake transport. +Patch unrestricted source opening to raise and verify metadata reads, misses, +and hits still follow the authorized path. Preserve existing sparse recovery +and eviction behavior rather than duplicating its tests for every Zarr dtype. + +### 4. Documentation and example + +Document the adapter API, supported arrays, optional installation, cache +representation, immutable contract, and memory/performance limits. Add the new +source to the existing proxy API documentation and example descriptions. +Run the S3 example manually against the supplied unsharded and sharded datasets +when credentials/network access are available; keep public S3 tests marked +`network` and outside the default suite. + +### 5. Validation and handoff + +Use the `blosc2` conda environment for all Python, installation, and tests. +Run focused adapter/RemoteProxy/Proxy/fsspec tests first, then the default suite +and repository lint checks after integration. Use the established pytest fixtures +and parametrization rather than adding a new test framework. + +Validate optional imports in a subprocess with Zarr imports blocked, and with +fsspec blocked for unrelated functionality. Verify the minimum supported Zarr +version in a suitable test environment before setting its requirement floor. +Record any dependency-version or network checks that could not run. + +## Completion criteria + +- A remote Zarr array opens as a RemoteProxy and produces correct slice values. +- Retained payloads are usable Blosc2 chunks with correct B2ND block layout. +- A warm hit performs no remote payload or metadata reads under the immutable + contract; eviction and cache limits retain their existing semantics. +- DISK carriers and sparse caches reopen safely with the correct source and + geometry; credentials are absent from persisted metadata. +- Zarr remains optional and missing dependencies produce actionable errors. +- An authorized server source cannot fall back to unrestricted transport. +- Existing Blosc2/Caterva2 source tests continue to pass. + +## Deferred work + +Mutable-store validation, per-object versions, refresh policies, and TTL are +future changes requiring a new explicit consistency contract. Direct reuse of +compatible compressed Zarr chunks, block-range conversion, shard batching, +custom codec optimizations, variable-length dtypes, ZIP sources, and hierarchy +browsing are deferred until a concrete workload needs them. + +The ponytail choice is one source adapter using existing Zarr decoding and +Blosc2 caching. No cache engine, codec framework, or generalized plugin registry +is needed for this feature. diff --git a/plans/remote-proxy-v9.md b/plans/remote-proxy-v9.md new file mode 100644 index 000000000..0634d4886 --- /dev/null +++ b/plans/remote-proxy-v9.md @@ -0,0 +1,523 @@ +# Remote proxy v9: HDF5 remote arrays via kerchunk pre-indexing + +Status: implemented. + +Added `HDF5NDSource`, a `ProxyNDSource` adapter that reads remote HDF5 datasets +via kerchunk byte-offset indexing and returns Blosc2 compressed chunks. Reused +the existing `Proxy` and `RemoteProxy` cache implementations, including bounded +memory caches, portable B2ND carriers, and the sparse runtime caches introduced +for Caterva2 in v7. + +RemoteProxy sources are assumed immutable by default. HDF5 remains immutable-only +in this version. Kerchunk, h5py, and hdf5plugin are optional dependencies configured +under `blosc2[hdf5]`. Dataset addressing has been unified across both HDF5 and Zarr +to support slash (`/`), double-colon (`::`), and keyword (`dataset=`) specifications. +The adapter lives in Python-Blosc2 and is fully usable without a Caterva2 server. + +## Fixed decisions + +- Read HDF5 through kerchunk pre-indexing: scan metadata once, produce a + reference dict mapping chunk keys to `(url, byte_offset, length)` triples, + then open that reference as a Zarr store via fsspec's `ReferenceFileSystem`. + This bypasses HDF5's chatty B-tree traversal at read time. +- Do not use h5py for chunk-by-chunk data reads over the network. The HDF5 file + format makes 15–50 sequential synchronous seeks just to open, and every + `get_chunk()` call would traverse Python's GIL. Kerchunk eliminates this by + turning HDF5 chunk locations into direct HTTP Range GETs. +- Do not require VirtualiZarr. Kerchunk alone produces the reference dict that + fsspec's `ReferenceFileSystem` and Zarr understand. +- Cache converted Blosc2 chunks in existing B2ND containers. Do not maintain a + second cache of HDF5 objects. +- Fetch and convert whole logical HDF5 chunks. Let kerchunk + Zarr resolve + codec pipelines (including Blosc2 via hdf5plugin, gzip, lzf, etc.). +- Users must specify the dataset path within the HDF5 file. Do not recursively + discover or auto-select a dataset. Opening a group raises an actionable error. +- `RemoteProxy(..., assume_immutable=True)` is the default. Mutable HDF5 sources + are not supported in this version. +- Do not change C/Cython code unless implementation exposes a demonstrated + blocker that cannot be handled by existing Python APIs. + +## Test datasets + +### Local: `hierarchy.h5` + +A 3.3 MB HDF5 file with a nested group structure, 12 datasets total, all +`int32` with shape `(10, 1000, 1000)`, chunks `(2, 500, 500)`, compressed with +Blosc2 (HDF5 filter ID 32026, requires `hdf5plugin` to decode): + +``` +d0/a0, d0/a1, d0/a2, d0/a3 +d0/d1/a0, d0/d1/a1, d0/d1/a2, d0/d1/a3 +d0/d1/d2/a0, d0/d1/d2/a1, d0/d1/d2/a2, d0/d1/d2/a3 +``` + +### Remote: `s3://blosc2/hierarchy.h5` + +The same file, hosted on Backblaze B2 at endpoint +`https://s3.us-west-001.backblazeb2.com`, accessed with the `blosc2` AWS CLI +profile. Available alongside the existing Zarr and Blosc2 test datasets: + +``` +s3://blosc2/ +├── cube-1k-1k-1k.b2nd +├── cube-1k-1k-1k.zarr/ +├── cube-1k-1k-1k-1shard.zarr/ +├── hierarchy.zarr/ +├── hierarchy.b2z +└── hierarchy.h5 ← 3.3 MB, Blosc2-compressed +``` + +## Architecture + +``` +User: blosc2.open("s3://blosc2/hierarchy.h5", lazy=True, + source_format="hdf5", dataset="d0/d1/a2", + storage_options={...}) + │ + ▼ + blosc2.RemoteProxy + source_format == "hdf5" branch + │ + ▼ + HDF5NDSource.__init__ + ┌─────────────────────────┤ + ▼ ▼ + kerchunk.hdf.SingleHdf5ToZarr Stores reference dict + (one-time metadata-only scan) in self._refs (small JSON) + │ + ▼ + fsspec ReferenceFileSystem → zarr.open_array → get_chunk() + (direct HTTP Range GETs (Zarr codec (read slice → + for exact chunk bytes) pipeline) pad → blosc2.asarray) +``` + +After construction, `HDF5NDSource` behaves identically to `ZarrNDSource`: +`serves_blocks = False`, same `get_chunk()` pattern (read slice → pad edge → +`blosc2.asarray()` → return compressed chunk), same `stamp` mechanism. + +## Existing code to reuse + +`src/blosc2/zarr_source.py` defines `ZarrNDSource`, whose `get_chunk()` body is +the exact logic `HDF5NDSource` needs. Factor the chunk-conversion into a shared +helper that both classes call. + +`src/blosc2/proxy_source.py` defines `ProxyNDSource`: shape, chunks, blocks, +dtype, compression parameters, and `get_chunk(nchunk)`. Its optional block-range +interface is unnecessary for this adapter. `Traffic` already provides +thread-safe counters. + +`src/blosc2/proxy.py` creates a B2ND cache, fetches missing chunks, inserts +compressed chunks, and tracks retention and eviction. These remain the owners of +cache state. + +`src/blosc2/remote_proxy.py` handles source descriptors, geometry validation, +source stamps, carriers, and sparse attachment. Its URL-string branch dispatches +to `FsspecNDSource`, `ZarrNDSource`, or `C2Array`. Add an `HDF5NDSource` branch. + +`src/blosc2/schunk.py` owns `blosc2.open()` remote dispatch and cache options. + +`examples/remote/s3-access.py` opens Blosc2 and Zarr arrays from the same +bucket. Extend it to accept `.h5` URLs. + +## New module: `src/blosc2/hdf5_source.py` + +### Construction and metadata + +1. Import `kerchunk.hdf` and `h5py` with actionable errors pointing to + `pip install blosc2[hdf5]`. Import `hdf5plugin` silently if available (needed + for Blosc2-compressed HDF5 chunks; its absence manifests as a codec error + from Zarr/HDF5 rather than an import error). +2. Run `SingleHdf5ToZarr(url, storage_options=storage_options or {}).translate()` + to produce the reference dict. This scans **only metadata** (file headers, + B-tree indices, chunk offset tables). No chunk data crosses the wire. Store + in `self._refs`. +3. Build an fsspec `ReferenceFileSystem` from the reference dict and create a + read-only Zarr store pointing to the specified `dataset` path within the + virtual hierarchy. If `dataset` points to a group, raise `ValueError` with an + actionable message listing available dataset paths. +4. Open with `zarr.open_array(store=..., mode="r")`. Normalize geometry: shape, + chunks, dtype to Python tuples / NumPy dtype. Validate dimensions, positive + chunk extents, dtype support, and Blosc2 size limits — same checks as + `ZarrNDSource._validate_metadata()`. +5. Compute cache block partitioning via `blosc2.compute_chunks_blocks()`. +6. Wrap the store with `_counting_store()` (reuse from `zarr_source.py`) for + traffic accounting beneath the Zarr decoder. +7. Compute deterministic SHA-256 `stamp` from + `{encoding_version, urlpath, dataset, shape, chunks, blocks, dtype}`. + Include `dataset` so two datasets from the same file produce distinct stamps. +8. Set `serves_blocks = False`. Expose concurrency and traffic conventions. + +### `get_chunk(nchunk)` + +Factor the shared chunk-conversion logic out of `ZarrNDSource.get_chunk()` into a +module-level helper in `zarr_source.py`: + +```python +def _zarr_chunk_to_blosc2(array, nchunk, shape, chunks, blocks, dtype, cparams): + """Read a Zarr chunk slice and return it as Blosc2 compressed bytes.""" + ... +``` + +Both `ZarrNDSource` and `HDF5NDSource` call this helper. The existing zarr tests +verify that the extraction does not change behavior. + +### `available_datasets(url, storage_options=None) → list[str]` + +A module-level function that scans the kerchunk reference and returns all dataset +paths within the HDF5 file. Used in error messages when the user passes a group +path or omits `dataset`, and useful for interactive discovery. + +### Properties + +`serves_blocks = False`, `encoding_version = 1`, `shape`, `chunks`, `blocks`, +`dtype`, `cparams`, `urlpath` (the HDF5 URL), `dataset` (the path within the +file), `stamp`, `traffic`, `max_concurrency`. + +## RemoteProxy integration + +### Source format normalization + +`_normalize_source_format()` accepts `"hdf5"` and auto-detects `.h5` / `.hdf5` +URL path suffixes: + +```python +if any(part.endswith((".h5", ".hdf5")) for part in path.split("/")): + return "hdf5" +``` + +### `RemoteProxy.__init__()` — `dataset` parameter + +Add `dataset: str | None = None`. Resolved alongside URL parsing in `_resolve_init_dataset_and_url()`. +Supported for both `hdf5` and `zarr` source formats, unifying internal dataset path handling. +For `zarr`, canonicalizes the underlying store URL to `container.zarr/dataset` while exposing +`RemoteProxy.dataset`. + +### `_open_source()` — `"hdf5"` branch + +```python +if source_format == "hdf5": + if not assume_immutable: + raise NotImplementedError("mutable HDF5 sources are not supported") + if dataset is None: + raise ValueError( + "HDF5 sources require a dataset path (e.g., dataset='d0/d1/a2')" + ) + src = blosc2.HDF5NDSource( + urlpath, dataset, _traffic=traffic, blocks=blocks, cparams=cparams, **kwargs + ) + source = { + "kind": "hdf5", + "version": 1, + "urlpath": urlpath, + "dataset": dataset, + "assume_immutable": assume_immutable, + } +``` + +### Source descriptor: `kind: "hdf5"` + +```json +{ + "kind": "hdf5", + "version": 1, + "urlpath": "s3://blosc2/hierarchy.h5", + "dataset": "d0/d1/a2", + "assume_immutable": true +} +``` + +### `_source_identity()` + +Include `dataset` in the identity so two datasets from the same file have +distinct cache paths: + +```python +if self._source["kind"] == "hdf5": + return f"{self._source['urlpath']}::{self._source['dataset']}" +``` + +Add `"hdf5"` alongside `"fsspec"` and `"zarr"` in all existing branches that +check `self._source["kind"]`. + +### `urlpath` property + +Add `"hdf5"` to the branch returning `self._source["urlpath"]`. + +### `_from_payload()` — deserialize HDF5 carriers + +Add `elif source_kind == "hdf5"` with field validation for +`{kind, version, urlpath, dataset, assume_immutable}`. Pass +`source_format="hdf5"` and `dataset=source["dataset"]` to the constructor. + +## `schunk.py` integration + +### `_validate_fsspec_source_format()` + +Add `"hdf5"` to valid values. Require `lazy=True` for HDF5. + +### `blosc2.open()` — `dataset` parameter and unified container URLs + +Add `dataset=None` to the signature. Forward through `kwargs["dataset"]` to +`RemoteProxy`. Update the docstring. Allow `dataset` for both `hdf5` and `zarr` +formats (requiring `lazy=True`). + +Add URL parsing helpers `split_h5_url` and `parse_container_url` in `src/blosc2/core.py` +to unify dataset specification across formats: +- Slash syntax: `"container.h5/d0/a3"`, `"container.zarr/d0/a3"` +- Double-colon syntax: `"container.h5::d0/a3"`, `"container.zarr::d0/a3"` (including optional leading slash) +- Explicit keyword: `blosc2.open("container.h5", dataset="d0/a3", lazy=True)` + +### All `source_format` validation sites + +Every place that checks `source_format not in {None, "blosc2", "zarr"}` must +add `"hdf5"`. + +## `__init__.py` + +Export `HDF5NDSource` from `blosc2`, add to `__all__`. + +## `pyproject.toml` + +```toml +hdf5 = ["kerchunk", "h5py", "hdf5plugin"] +``` + +Kerchunk pulls in `ujson` automatically. `zarr` and `fsspec` are already covered +by existing extras. `hdf5plugin` is included directly in `hdf5` so that Blosc2-compressed +(filter 32026) and other plugin-compressed HDF5 chunks decode seamlessly. +Remote HDF5 installs: `pip install "blosc2[hdf5,fsspec]" s3fs`. + +## Reference index caching strategy + +The kerchunk reference dict is small (KB–few MB) but the one-time scan costs +seconds over the network. + +### Within a session + +`HDF5NDSource` stores `self._refs` in memory. If `RemoteProxy` reopens the +source within the same process, pass the existing refs through. + +### In the carrier vlmeta + +When persisting a DISK carrier, store the compressed reference JSON in vlmeta: + +```python +import ujson + +carrier.schunk.vlmeta["hdf5-refs"] = blosc2.compress(ujson.dumps(refs).encode()) +``` + +On `_from_payload()` reopening, check for `hdf5-refs` before re-scanning. This +makes warm carrier reopens zero-cost: no network access if the stamp matches. + +### User-supplied reference file + +Accept `refs=` parameter pointing to a pre-computed JSON reference: + +```python +arr = blosc2.open( + "s3://blosc2/hierarchy.h5", + lazy=True, + source_format="hdf5", + dataset="d0/d1/a2", + refs="hierarchy-refs.json", +) +``` + +This skips the kerchunk scan entirely, useful for large files or repeated opens. + +## Test plan + +### Default suite — local HDF5 fixtures (`tests/test_hdf5_source.py`) + +All tests create temporary HDF5 files with `h5py` — no network, no S3. Use +`pytest.importorskip("kerchunk")` and `pytest.importorskip("h5py")`. + +#### Adapter tests (HDF5NDSource directly) + +| Test | What it verifies | +| :--- | :--- | +| `test_hdf5_source_through_proxy` | Create HDF5 → HDF5NDSource → Proxy → slice → compare with h5py | +| `test_hdf5_source_dtypes` | `int32`, `float64`, `bool`, `complex64`, `S6`, `M8[ns]`, structured | +| `test_hdf5_source_edge_chunks` | Shape not divisible by chunk size; edge padding is correct | +| `test_hdf5_source_fill_value` | Sparse datasets with HDF5 fill values | +| `test_hdf5_source_scalar_and_empty` | 0-d scalar and 0-length datasets | +| `test_hdf5_source_multidim` | 1D, 2D, 3D arrays | +| `test_hdf5_source_gzip_compression` | HDF5 datasets compressed with gzip (no hdf5plugin needed) | +| `test_hdf5_source_group_error` | Opening a group → actionable ValueError listing datasets | +| `test_hdf5_source_missing_dataset` | Wrong dataset path → clear error message | +| `test_hdf5_source_available_datasets` | `available_datasets()` returns correct paths | + +#### RemoteProxy integration tests + +| Test | What it verifies | +| :--- | :--- | +| `test_open_hdf5_as_remote_proxy` | `blosc2.open(local_h5, lazy=True, source_format="hdf5", dataset=...)` | +| `test_hdf5_auto_detection` | `.h5` / `.hdf5` suffix triggers auto-detection | +| `test_hdf5_requires_dataset` | Omitting `dataset=` raises ValueError | +| `test_hdf5_requires_lazy` | `lazy=False` with `source_format="hdf5"` raises ValueError | +| `test_hdf5_rejects_mutable` | `assume_immutable=False` raises NotImplementedError | +| `test_hdf5_memory_cache` | `CachePolicy.MEMORY` — second read has zero traffic | +| `test_hdf5_disk_cache` | `CachePolicy.DISK` — carrier reopens warm | +| `test_hdf5_traffic_accounting` | `traffic.nbytes > 0` on cold read, `== 0` on warm hit | + +#### Persistence tests + +| Test | What it verifies | +| :--- | :--- | +| `test_hdf5_carrier_reopens_warm` | Save → reopen → slice without network (refs from vlmeta) | +| `test_hdf5_carrier_save_load` | `save()` / `to_cframe()` round-trip | +| `test_hdf5_source_descriptor` | Correct `kind: "hdf5"` descriptor in payload | +| `test_hdf5_geometry_mismatch` | Changed HDF5 file → geometry validation fails | +| `test_hdf5_refs_in_vlmeta` | Carrier vlmeta contains compressed kerchunk reference | + +#### Dependency isolation tests + +| Test | What it verifies | +| :--- | :--- | +| `test_hdf5_missing_kerchunk_error` | Mock missing kerchunk → ImportError mentioning `blosc2[hdf5]` | +| `test_hdf5_missing_h5py_error` | Mock missing h5py → ImportError | +| `test_blosc2_import_without_hdf5` | `import blosc2` works without kerchunk/h5py installed | + +### Network suite — `s3://blosc2/hierarchy.h5` (`@pytest.mark.network`) + +These tests use the real Backblaze B2 bucket and are excluded from the default +suite. Use the `blosc2` AWS CLI profile and +`endpoint_url=https://s3.us-west-001.backblazeb2.com`. + +| Test | What it verifies | +| :--- | :--- | +| `test_s3_hdf5_open_and_slice` | Open `s3://blosc2/hierarchy.h5`, dataset `d0/d1/a2`, read `[0, :3, :3]`, compare with local `hierarchy.h5` | +| `test_s3_hdf5_cache_hit` | Second read of same slice has zero traffic | +| `test_s3_hdf5_disk_carrier` | DISK cache, save carrier, reopen, read without network | +| `test_s3_hdf5_nested_datasets` | Open `d0/a0`, `d0/d1/a1`, `d0/d1/d2/a3` — all produce correct data | +| `test_s3_hdf5_matches_zarr` | Compare `s3://blosc2/hierarchy.h5::d0/d1/a2` with `s3://blosc2/hierarchy.zarr/d0/d1/a2` — values match | +| `test_s3_hdf5_traffic` | Cold read transfers metadata + chunk bytes; warm read transfers zero | + +Storage options for tests: + +```python +STORAGE_OPTIONS = { + "profile": "blosc2", + "endpoint_url": "https://s3.us-west-001.backblazeb2.com", +} +``` + +### Moto S3 tests (offline, default suite) + +Follow the pattern in `tests/test_fsspec_s3.py`: use moto's `ThreadedMotoServer` +to run a local S3 server, upload a small HDF5 file, and test the full +`blosc2.open()` → `RemoteProxy` → `HDF5NDSource` → slice cycle without network +access. These verify the async/transport path without needing credentials or the +real bucket. + +## Example update: `examples/remote/s3-access.py` + +Add `.h5` / `.hdf5` to the URL dispatch in `open_remote_array()`: + +```python +if clean_url.endswith((".h5", ".hdf5")): + # HDF5: requires dataset path after :: separator + if "::" in url: + h5_url, dataset = url.rsplit("::", 1) + else: + dataset = "d0/d1/a2" # default for hierarchy.h5 + arr = blosc2.open( + h5_url, + lazy=True, + source_format="hdf5", + dataset=dataset, + storage_options=storage_options, + ) + return "HDF5 (Lazy RemoteProxy)", arr +``` + +Usage: + +```bash +python s3-access.py s3://blosc2/hierarchy.h5::d0/d1/a2 +python s3-access.py s3://blosc2/hierarchy.h5::d0/a0 +``` + +## Documentation updates + +| File | Change | +| :--- | :--- | +| `doc/reference/remoteproxy.rst` | Add `HDF5NDSource`, document `dataset` param | +| `doc/reference/classes.rst` | Add `HDF5NDSource` to class list | +| `doc/guides/remote_arrays.md` | Add HDF5 section with usage example and kerchunk explanation | +| `doc/getting_started/installation.rst` | Document `pip install "blosc2[hdf5,fsspec]" s3fs hdf5plugin` | + +## Implementation sequence and checks + +### 1. Extract shared chunk-conversion helper + +Factor `_zarr_chunk_to_blosc2()` out of `ZarrNDSource.get_chunk()`. Verify all +existing zarr tests pass unchanged. + +### 2. Implement `hdf5_source.py` + +Build `HDF5NDSource` and `available_datasets()`. Export from `__init__.py`. +Write adapter-level tests using local temporary HDF5 fixtures (gzip compression +only — no hdf5plugin needed for test fixtures). + +### 3. Integrate into RemoteProxy + +Add `source_format="hdf5"` dispatch, `dataset` parameter, `kind: "hdf5"` +descriptor, `_from_payload()` reconstruction. Write integration tests using +`memory://` or local files. + +### 4. Carrier persistence with reference caching + +Store kerchunk refs in vlmeta. Write persistence tests: save, reopen, verify +warm reads work without network. + +### 5. Wire into `blosc2.open()` and `schunk.py` + +Add `dataset=` parameter, validation, auto-detection. Write `open()` tests. + +### 6. Optional dependency extra + isolation tests + +Add `hdf5` extra to `pyproject.toml`. Write isolation tests. + +### 7. Moto S3 integration tests + +Add tests to `tests/test_hdf5_source.py` (or a separate file) using moto's +local S3 server with an uploaded HDF5 fixture. + +### 8. Network S3 tests + +Add `@pytest.mark.network` tests reading `s3://blosc2/hierarchy.h5`, comparing +with local `hierarchy.h5` and with `s3://blosc2/hierarchy.zarr`. + +### 9. Example and documentation + +Update `examples/remote/s3-access.py`, update docs. + +### 10. Validation and handoff + +Use the `blosc2` conda environment for all Python, installation, and tests. Run +focused adapter/RemoteProxy/Proxy tests first, then the default suite and +repository lint checks. Validate optional imports in a subprocess with kerchunk +imports blocked. + +## Completion criteria (All Verified) + +- [x] A remote HDF5 dataset opens as a RemoteProxy and produces correct slice values. +- [x] The kerchunk reference is generated once (metadata-only scan) and cached in + the carrier vlmeta for warm reopens. +- [x] Retained payloads are usable Blosc2 chunks with correct B2ND block layout. +- [x] A warm hit performs no remote payload or metadata reads. +- [x] DISK carriers and sparse caches reopen safely; credentials are absent from + persisted metadata. +- [x] `s3://blosc2/hierarchy.h5::d0/d1/a2` matches `s3://blosc2/hierarchy.zarr/d0/d1/a2`. +- [x] Kerchunk, h5py, and hdf5plugin are optional in `blosc2[hdf5]`; missing dependencies produce actionable errors. +- [x] Dataset addressing is unified across HDF5 and Zarr (`container.ext/dataset`, `container.ext::dataset`, `dataset="..."`). +- [x] Existing Blosc2/Zarr/Caterva2 source tests continue to pass (full suite passing). + +## Deferred work + +Mutable-store validation, refresh policies, per-dataset ETags, hierarchy +browsing/discovery, VirtualiZarr integration, ZIP-embedded HDF5, variable-length +dtypes, and Caterva2 server-side HDF5 federation are deferred until a concrete +workload needs them. diff --git a/plans/remote-proxy.md b/plans/remote-proxy.md new file mode 100644 index 000000000..bc7a9923d --- /dev/null +++ b/plans/remote-proxy.md @@ -0,0 +1,938 @@ +# Plan: Persistable `RemoteProxy` + +> **Superseded:** The self-caching carrier design in +> [`remote-proxy-v3.md`](remote-proxy-v3.md) is the authoritative client implementation +> plan. This document records the earlier immutable-reference design. + +## Motivation + +Python-Blosc2 can already access remote B2ND arrays lazily: + +```python +array = blosc2.open( + remote_urlpath, + lazy=True, + cache_path="mycache.b2nd", +) +``` + +The resulting `mycache.b2nd` can be uploaded to Caterva2 and acts as a proxy, +but it is fundamentally a **persistent cache**. As remote chunks are read, +the file acquires compressed data plus cache bookkeeping. This is useful for +offline reuse, but it is not the right representation when the desired object +is only a small, immutable reference to a remote array. + +The missing abstraction is a persistable proxy whose stored B2ND file contains +the remote source description and array geometry, but never becomes the data +cache. When uploaded to Caterva2, reads should be forwarded to the referenced +Caterva2 or fsspec source under an explicit server-side security policy. + +## Decision Summary + +Introduce a new public `RemoteProxy` type rather than extending +`SimpleProxy` or broadening `C2Array`: + +- `SimpleProxy` remains the generic, non-persistable adapter for arbitrary + array-like Python objects. +- `C2Array` remains the direct Caterva2 client object and its existing + persistence format remains supported. +- `Proxy` remains the implementation for reusable memory and disk caches. +- `RemoteProxy` is a persistable, backend-neutral remote reference with an + explicit `CachePolicy`. + +The public cache policy should be an enum: + +```python +class CachePolicy(Enum): + NONE = "none" + MEMORY = "memory" + DISK = "disk" +``` + +Python cannot parse `CachePolicy.None` because `None` is a keyword, so the +public spelling must be `blosc2.CachePolicy.NONE`. Uppercase enum members also +match the enum style already used by Python-Blosc2. + +## Implementation Status + +The Python-Blosc2 client implementation on this branch covers the core +RemoteProxy design and the first four implementation phases: + +- `CachePolicy.NONE`, `MEMORY`, and `DISK` are public and validated at runtime. +- `RemoteProxy` supports Caterva2 `URLPath`/`C2Array` sources and contiguous + single-file fsspec B2ND URLs. +- `NONE` uses direct Caterva2 indexing or operation-scoped fsspec assembly; + `MEMORY` and `DISK` reuse the existing `Proxy` machinery. +- Memory caches default to a 256 MiB post-operation compressed-payload bound; + disk caches are unlimited by default and support the same optional LRU bound. +- Remote lazy open accepts explicit cache policies and limits while preserving + the pre-existing `blosc2.open(..., lazy=True)` `Proxy` behavior when neither + is supplied. +- Reference-only `remote_proxy` carriers, strict source/geometry validation, + authentication omission, fsspec `Ref` values, LazyExpr operands, and + ObjectArray/BatchArray msgpack round trips are implemented and documented. +- Client-side URL safety checks reject local filesystem URLs, chained fsspec + URLs, user information, fragments, and credential-like query parameters. +- Floating references refresh their source identity before data operations, + reject geometry changes that happen after a carrier is opened, and discard + memory or disk cache state when the object at a source URL is replaced. +- Caterva2 now discovers direct `remote_proxy` carriers without resolving them + and denies data access by default. Its initial opt-in resolver is limited to + credential-free HTTPS destinations on an exact administrator allowlist; it + pins public DNS results, disables redirects, and bounds time, geometry, chunk + count, and concurrency. + +The following remain future work or deliberate follow-ups: + +- Opening a local reference carrier with an explicit runtime policy (for + example, `blosc2.open("reference.b2nd", cache_policy=...)`) is not yet a + convenience path. Reconstructing `RemoteProxy(reference.urlpath, ...)` is + the current explicit opt-in. +- Cache-oriented `fetch()`/`afetch()` methods are not exposed on `RemoteProxy` + under `NONE`; a separate materialization API can be designed later. +- Caterva2 operation-wide request budgets and reference-chain resolution/cycle + handling remain to be implemented. Remote references embedded inside stored + expressions are rejected until they can use the same secure resolver. +- Pinned reference semantics, broader fsspec/server protocol allowlists, and + any `C2Array.save(as_remote_proxy=True)` convenience are future decisions. + +## Open Work Queue And Decisions + +The remaining server work should be tackled in the following order. The server +scope stays limited to public, credential-free HTTPS while resource controls +and recursive resolution are completed. + +### 1. Operation-wide resource budgets + +**Recommended next implementation.** The current Caterva2 policy bounds source +geometry, chunk count, concurrency, and each HTTP request's timeout. It does not +yet bound the aggregate work caused by one client operation. + +Add one request-scoped budget object shared by metadata discovery and all range +reads. It should account for: + +- total upstream response bytes, including metadata and unsuccessful responses +- range-request count and retry count +- elapsed wall-clock time, with the remaining deadline passed to each request +- live upstream tasks, with cancellation when the downstream request ends + +Exhaustion must abort the operation with a deterministic, non-sensitive error. +The budget should be enforced below `FsspecNDSource`, at the authorized +filesystem/transport boundary, so every request is charged independently of +the slice assembly strategy. Initial limits should be server-wide configuration +defaults; per-user limits can be added independently of remote credentials. + +Open implementation question: fsspec does not expose all transport accounting +through a stable public hook. Prefer a narrow Caterva2 HTTP filesystem wrapper +that charges opened responses and retries rather than adding server policy to +Python-Blosc2's general-purpose `Traffic` diagnostics. + +### 2. Public-only source boundary + +**Decision:** Caterva2 RemoteProxy resolution supports only remote arrays that +are publicly readable without credentials. The server does not select, store, +or forward credentials and does not accept credential profile names, provider +settings, endpoint overrides, headers, cookies, signed query parameters, or +arbitrary fsspec storage options from a carrier. + +The server protocol is HTTPS. Public S3 objects are supported through their +ordinary public HTTPS URLs. Native `s3://` resolution is deliberately out of +scope, even for anonymous buckets, so Caterva2 does not need provider-specific +filesystem configuration, endpoints, or bucket policy logic. + +Exact administrator host allowlists, public-address validation, DNS pinning, +disabled redirects, and rejection of descriptor-provided options remain +mandatory. Private sources, native object-store protocols, authenticated +upstream sessions, and tenant-scoped remote credentials are non-goals. Adding +any of them later requires a separate security design and must not weaken the +public HTTPS path by default. + +### 3. Nested references and stored expressions + +Direct `RemoteProxy` carriers are supported. Remote references inside persisted +`LazyExpr`/`LazyUDF` objects are intentionally rejected before generic Blosc2 +decoding, because decoding currently resolves operands without a Caterva2 +policy context. + +Supporting them requires a resolver/context injection point in Python-Blosc2, +then a Caterva2 graph traversal that carries one operation budget, records +normalized source identities, rejects cycles/self-reference, and enforces a +small maximum depth and hop count. Arbitrary nesting in other container types +must remain denied or unresolved until it goes through the same traversal. + +Open design question: make authorized resolution an explicit argument/context +to the B2 object decoder rather than installing a process-global resolver. A +process-global callback cannot safely represent concurrent requests or policy +contexts. + +### 4. Compatibility and release floor + +Caterva2 currently refuses to enable remote resolution when its installed +Python-Blosc2 lacks `RemoteProxy`. Before releasing the Caterva2 feature, raise +its declared minimum Python-Blosc2 version to the first release containing the +carrier format and secure filesystem-injection hooks. Keep the runtime check for +clear diagnostics in mixed deployments. + +### 5. End-to-end transport validation + +Add a controlled HTTPS integration fixture with real TLS, DNS resolution, and +range responses. It should exercise successful reads, byte/range budget +exhaustion, timeout/cancellation, changed ETag/geometry, and attempts to redirect +or resolve to a denied address. External public services should remain optional +network tests; the security assertions need a deterministic local harness whose +address classification is explicitly test-configurable. + +## Goals + +The first implementation should allow this workflow: + +```python +proxy = blosc2.RemoteProxy( + "https://datasets.example.org/dataset.b2nd", + cache_policy=blosc2.CachePolicy.NONE, +) +proxy.save("dataset-proxy.b2nd") + +# Upload dataset-proxy.b2nd to a suitably configured Caterva2 server. +``` + +It should also work for a Caterva2 source: + +```python +proxy = blosc2.RemoteProxy( + blosc2.URLPath( + "@public/dataset.b2nd", + urlbase="https://example.org/caterva2", + ), + cache_policy=blosc2.CachePolicy.NONE, +) +``` + +The saved file should: + +- be metadata-sized rather than proportional to the remote array +- reopen as `RemoteProxy` +- preserve shape, dtype, chunks, blocks, and the safe source descriptor +- serve slices and chunks from either Caterva2 or supported fsspec URLs +- remain byte-for-byte unchanged after reads under `CachePolicy.NONE` +- never contain credentials + +## Non-Goals For The First Version + +- Persisting arbitrary `SimpleProxy` sources or Python callables. +- Serializing fsspec filesystem instances or arbitrary `storage_options`. +- Embedding authentication tokens, cookies, cloud keys, or signed credentials. +- Proxying every fsspec object type. Initially support a single remote + contiguous B2ND frame; remote directory stores such as `.b2d` can be added + after their object and authorization semantics are designed. +- Making an uploaded descriptor safe merely through client-side URL + validation. Caterva2 must enforce its own resolution policy. +- Changing the existing on-disk cache proxy format. + +## Why `RemoteProxy` Should Be A Separate Type + +### Do not make `SimpleProxy` persistable + +`SimpleProxy` accepts a broad array-like object with `shape`, `dtype`, and +`__getitem__`. Most such objects have no stable or safe reconstruction recipe. +Making the class conditionally persistable only for Caterva2 and fsspec inputs +would give one public type two substantially different contracts and invite +accidental serialization of arbitrary Python state. + +`RemoteProxy` can instead require a constrained, versioned source descriptor. +This makes persistence an invariant of the type rather than a special case. + +### Do not make `C2Array` backend-neutral + +`C2Array` models Caterva2 operations and authentication. Teaching it about +fsspec URLs would mix the Caterva2 protocol with byte-range filesystem access. +It would also leave no natural home for the cache policy shared by both +backends. + +### Reuse implementation, not identity + +`RemoteProxy` should delegate to existing components: + +- `C2Array` for Caterva2 reads +- `FsspecNDSource` for fsspec metadata, chunks, and byte ranges +- `Proxy` for reusable memory or disk caching + +It should not duplicate those implementations, and it should not itself be a +subclass of `SimpleProxy` unless that inheritance remains strictly an internal +convenience with no effect on serialization. + +## Public API + +### Construction + +Proposed primary constructor: + +```python +proxy = blosc2.RemoteProxy( + urlpath, + cache_policy=blosc2.CachePolicy.NONE, + cache_path=None, + cache_dir=None, + max_cache_bytes=None, +) +``` + +The omitted `max_cache_bytes` value uses the policy-dependent default; an +explicit `None` requests an unlimited MEMORY or DISK cache. + +The constructor should discover the source kind and remote array metadata. A +future explicit `source_kind=` escape hatch can be added if URL recognition is +ambiguous, but should not be needed initially. + +`RemoteProxy` should expose at least: + +- `shape`, `dtype`, `chunks`, `blocks`, and `cparams` +- `urlpath` and a read-only normalized source descriptor +- `cache_policy` +- synchronous `__getitem__` and `get_chunk()` +- asynchronous counterparts where the selected backend supports them +- traffic information compatible with the current remote-access diagnostics +- `to_cframe()` and `save(urlpath)` + +It should participate as an expression operand in the same way as `C2Array` +and other remote array-like operands. + +### Integration with `blosc2.open` + +Once the explicit class is stable, extend the existing remote open path: + +```python +blosc2.open( + remote_urlpath, + lazy=True, + cache_policy=blosc2.CachePolicy.NONE, +) +``` + +Recommended compatibility mapping: + +| Arguments | Effective policy | Result | +| --- | --- | --- | +| `lazy=True` only | `MEMORY` | Preserve current process-local lazy cache behavior | +| `lazy=True, cache_policy=NONE` | `NONE` | `RemoteProxy` with no retained data cache | +| `lazy=True, cache_policy=MEMORY` | `MEMORY` | `RemoteProxy` backed by the current memory `Proxy` | +| `lazy=True, cache_path=...` | `DISK` | Preserve current persistent-cache behavior | +| `lazy=True, cache_dir=...` | `DISK` | Preserve current persistent-cache behavior | +| `lazy=True, cache_policy=DISK, cache_path/cache_dir=...` | `DISK` | Explicit persistent cache | + +Validation rules: + +- `NONE` or `MEMORY` combined with `cache_path`/`cache_dir` is an error. +- `DISK` without a cache location is an error unless a documented automatic + cache-location policy is deliberately introduced. +- `max_cache_bytes` is invalid with `NONE`, defaults to 256 MiB with `MEMORY`, + and defaults to unlimited with `DISK`. A positive explicit value bounds + either memory or disk cache payload. The implementation needs an internal + sentinel to distinguish an omitted policy-dependent default from explicit + `None`, which means unlimited. +- Supplying both `cache_path` and `cache_dir` continues to follow the existing + validation rule. +- The Python API should require a `CachePolicy` instance rather than expose + several string aliases. The serialized payload uses stable lowercase string + values so it is independent of Python enum internals. + +The default constructor policy for an explicit `RemoteProxy` should be `NONE`, +because its defining purpose is a reference-only proxy. The default for the +pre-existing `blosc2.open(..., lazy=True)` call should remain `MEMORY` to avoid +a silent performance regression. + +## Precise Cache Semantics + +The word "none" must describe retained cache state, not prohibit every +temporary buffer. fsspec range reads need somewhere to assemble compressed +blocks for a slice. + +### `CachePolicy.NONE` + +- No fetched chunk or block is retained between independent operations. +- No fetched bitmap, chunk payload, or cache index is written to the carrier. +- The serialized carrier is never used as scratch space. +- Temporary buffers may exist in memory for the duration of one operation. +- If an operation requires an NDArray workspace, it is operation-scoped and + discarded before returning. +- Repeating the same data read is expected to contact the remote source again. +- Metadata may be retained in the live Python object's immutable fields; + reopening the carrier does not imply a data fetch. + +### `CachePolicy.MEMORY` + +- Fetched data may be retained for the lifetime of the Python object. +- Repeating a covered read should be served without remote data traffic. +- Retained compressed payload is limited to 256 MiB by default. The caller may + select another positive `max_cache_bytes`, or explicitly request an + unlimited cache with `None` through an API representation that distinguishes + it from an omitted policy-dependent default. +- After each operation, least-recently-used chunks are evicted until the + retained payload is within the bound. +- `save()` still writes a reference-only carrier with persisted policy `NONE`, + not a snapshot of this process-local cache. +- Closing or dropping the object loses the cache. + +### `CachePolicy.DISK` + +- Reuse the existing persistent `Proxy` cache behavior and format. +- The disk cache is unlimited by default for compatibility and because its + purpose is cross-process reuse. An explicit positive `max_cache_bytes` + enables the same post-operation LRU bound as the memory cache. +- Eviction replaces whole cached chunks with `UNINIT`, clears their fetched + bitmap entries, and shrinks the live `.b2nd` payload. Compact contiguous + files may need to move later compressed data, so frequent disk eviction can + be more expensive than memory eviction. +- The disk cache and a reference carrier are distinct concepts. +- `RemoteProxy.save()` should always save the reference-only representation and + normalize the persisted policy to `NONE`. Memory and disk policies describe + the live process, not portable behavior to impose on another machine. +- The configured cache lives at `cache_path` or under `cache_dir`; it is not the + destination passed to `RemoteProxy.save()`. +- Existing `Proxy` cache files continue reopening as `Proxy`, not + `RemoteProxy`. + +This distinction prevents a supposedly portable descriptor from silently +growing or containing a partial snapshot because it happened to be read before +upload. + +### Meaning and accounting of `max_cache_bytes` + +The bound applies after an operation completes and covers retained compressed +cache payload, including partial chunks and duplicated hot partial-block +payloads. It does not bound: + +- the compressed working set needed to complete the current operation +- in-flight concurrent responses +- decompression and assembly buffers +- the NumPy result returned to the caller +- total process RSS, because an allocator may retain freed arenas + +Eviction must happen only after the requested result has been assembled. The +current proxy sequence fetches all required regions before reading the result +from its cache, so evicting during that fetch could discard an early chunk and +produce an incorrect result. + +LRU granularity is one chunk even when only some blocks in that chunk are +cached. Cache hits refresh recency as well as remote fetches. A chunk larger +than the bound may be used for the current operation and then evicted, leaving +the retained payload below the limit. + +For a reopened disk cache, exact recency from a previous process need not be +persisted initially. Existing fetched chunks are seeded in deterministic chunk +order as older than chunks touched by the new process. This preserves the +bound and correctness without rewriting LRU metadata after every read; it only +reduces eviction quality immediately after reopen. + +## Source Model + +Only explicitly supported, reconstructable source kinds should be serialized. +Extend the reference model with a versioned fsspec source kind while retaining +the existing Caterva2 reference kind. + +Conceptually: + +```python +RemoteSource = Caterva2SourceRef | FsspecSourceRef +``` + +Suggested descriptors: + +```python +{ + "kind": "caterva2", + "version": 1, + "path": "@public/dataset.b2nd", + "urlbase": "https://example.org/caterva2", +} +``` + +```python +{ + "kind": "fsspec", + "version": 1, + "urlpath": "https://datasets.example.org/dataset.b2nd", +} +``` + +The Python client may still use a public S3 descriptor directly when its local +fsspec environment is configured for anonymous access: + +```python +{ + "kind": "fsspec", + "version": 1, + "urlpath": "s3://public-bucket/dataset.b2nd", +} +``` + +This source form is not accepted by Caterva2; an uploaded reference must use +the public object's HTTPS URL. + +The source reference must contain only location and format information. It +must not include headers, bearer tokens, passwords, signed query parameters, +filesystem objects, or arbitrary fsspec keyword arguments. + +The current `Ref.from_object(Proxy)` behavior should not be changed as part of +this feature: persisted lazy-expression operands may rely on it unwrapping to +the proxy cache. Add a dedicated remote-source encoder/resolver instead of +silently changing generic `Ref` semantics. + +## Serialized B2 Object Format + +Use the existing B2 object carrier mechanism with a new object kind, +`remote_proxy`. The carrier is an empty, structurally valid NDArray containing +array geometry in its normal metadata and a versioned B2 object payload in +variable-length metadata. + +Example payload: + +```python +{ + "kind": "remote_proxy", + "version": 1, + "source": { + "kind": "fsspec", + "version": 1, + "urlpath": "https://datasets.example.org/dataset.b2nd", + }, + "cache_policy": "none", +} +``` + +The carrier should include: + +- shape and dtype +- chunk and block geometry +- compression parameters needed to interpret fetched chunks +- the `remote_proxy` payload + +Version 1 always writes `"cache_policy": "none"`. This field makes the +reference-only behavior explicit and leaves room for future policy negotiation, +but a carrier must not request server memory or disk use based on the creating +process's runtime policy. A caller reopening locally can select a runtime cache +policy through an explicit future open override; without one, it remains +`NONE`. + +It must not include: + +- `proxy-source` +- `proxy-fetched` or `proxy-index` +- fetched compressed chunks +- access credentials or client configuration + +`process_opened_object()` should dispatch `b2o.kind == "remote_proxy"` to the +new decoder. The existing `proxy-source` check must continue to identify +legacy/current persistent cache proxies before generic B2 object dispatch. + +### Source identity and mutation + +Version 1 may optionally store a non-secret source stamp such as an ETag, +content length, or backend revision when available. It should not rely on one +being available for every backend. + +Recommended initial semantics are a **floating reference with structural +validation**: + +- reads see the source's current contents +- the source must still match the carrier's shape, dtype, chunks, and blocks +- a mismatch raises a clear stale-reference error before returning data + +A future pinned mode can require an exact source stamp for reproducibility. It +should be a separate, explicit option rather than an accidental consequence of +metadata captured at creation time. + +## Runtime Design + +### Common `RemoteProxy` layer + +`RemoteProxy` owns: + +- the normalized source descriptor +- immutable array geometry captured in the carrier +- the selected `CachePolicy` +- a backend adapter +- optional memory/disk cache state according to policy +- the policy-dependent retained-cache limit and chunk-level LRU state + +The public layer validates the source metadata against the carrier and provides +consistent indexing, persistence, traffic reporting, and error behavior. + +### Caterva2 backend + +For `NONE`, delegate slices and chunks directly to `C2Array`; no assembly cache +is necessary. For `MEMORY` and `DISK`, either continue direct delegation when +it satisfies the operation or wrap it with the existing cache proxy machinery. +The policy must still determine whether results are reusable between calls. + +The serialized source should reuse the existing `C2Array` reference fields, +including the rule that authentication tokens are not persisted. + +### fsspec backend + +Use `FsspecNDSource` for metadata, chunk, and range access. Because this source +does not provide general `__getitem__`, slicing needs an assembly layer: + +- `NONE`: construct an in-memory operation-scoped workspace for the requested + slice/chunks, return the result, then discard the workspace +- `MEMORY`: use one in-memory `Proxy` cache retained by `RemoteProxy` +- `DISK`: use the existing persistent `Proxy` at the configured cache location, + with optional post-operation LRU eviction + +The first implementation should favor correctness and a clean policy boundary. +An optimized no-cache slice assembler can replace the operation-scoped proxy +later without changing the public or serialized formats. + +### Fetch APIs + +The first implementation deliberately keeps cache-oriented `fetch()` and +`afetch()` on `Proxy` rather than exposing them on `RemoteProxy`. This avoids +an ambiguous operation under `CachePolicy.NONE`: indexing and `get_chunk()` +return results while retaining no reusable data, whereas a future named +materialization API can explicitly return a new `NDArray` without changing the +reference carrier. Any such API must not silently convert a no-cache proxy +into a retained cache. + +## Caterva2 Server Contract + +Serving an uploaded `RemoteProxy` requires explicit support in Caterva2. The +Python-Blosc2 carrier is only a descriptor; Caterva2 is responsible for deciding +whether and how it may be resolved. + +Recommended request lifecycle: + +1. Upload stores the carrier as an immutable reference object. +2. Catalog and shape/dtype inspection may use the carrier metadata without an + outbound request. +3. On first data access, the server parses and validates the descriptor against + its configured remote-source policy. +4. It resolves current remote metadata and verifies the carrier geometry. +5. It performs only the byte ranges or slices needed for the request. +6. Any temporary assembly state lives outside the uploaded dataset and is + discarded for `NONE`. +7. The carrier's contents, size, and modification time remain unchanged. + +An installation may optionally validate reachability at upload time, but that +cannot replace validation at read time: DNS, redirects, credentials, and the +remote object can all change later. + +Until Caterva2 implements the security and resource controls below, it should +reject `remote_proxy` carriers by default rather than resolve arbitrary URLs. + +## Security Requirements + +An uploaded remote reference asks the server to make outbound requests chosen +by a client. This is an SSRF and resource-exhaustion boundary, not merely a new +file type. + +### Protocol and destination policy + +Caterva2 should: + +- deny all source protocols by default +- enable only administrator-configured protocols, for example `https` or `s3` +- reject local and process-oriented schemes such as `file`, `memory`, and + arbitrary chained fsspec URLs +- optionally allow only configured hosts, ports, buckets, and key prefixes +- resolve hostnames and reject loopback, link-local, private, multicast, and + cloud-instance-metadata destinations unless explicitly authorized +- recheck the destination after every redirect and cap redirect count +- protect against DNS rebinding by validating the actual connection target, + not only the submitted hostname +- normalize URLs before policy checks to prevent parser or encoding bypasses + +Support for fsspec protocol chaining such as archive-over-network URLs should +be out of scope initially because every layer expands the policy surface. + +### Credential-free boundary + +- Never serialize client credentials in the B2 object. +- Strip or reject user-info, sensitive query parameters, custom headers, + cookies, tokens, and arbitrary `storage_options` at creation and upload. +- Do not configure or select upstream credentials in the RemoteProxy resolver. +- Do not add authentication headers, cookies, signed queries, endpoint + overrides, or provider-specific storage options. +- Avoid reflecting secrets or sensitive internal response bodies in errors. + +Only publicly readable HTTPS references are supported by Caterva2. Private +references are rejected; client credentials cannot make an uploaded proxy +portable safely. + +### Resource limits + +Caterva2 should configure and enforce: + +- connection, read, and total request timeouts +- maximum redirects, range requests, retries, and concurrency per operation +- maximum bytes fetched for metadata and for one user request +- maximum rank, shape, logical `nbytes`, chunk count, and metadata size +- decompression and expansion limits before allocating output buffers +- per-user or per-tenant rate and bandwidth limits +- cancellation of upstream requests when the client request is cancelled + +Carrier geometry is untrusted input and must be validated before multiplication +or allocation. + +### Reference graphs + +A remote target may itself be another proxy, possibly pointing back to the +original object. The server must enforce: + +- maximum proxy depth and total remote hops +- cycle detection using normalized source identities +- rejection of direct or indirect self-references +- one cumulative resource budget across the entire reference chain + +### Request isolation and observability + +- Do not share operation-scoped assembly state across client requests. +- Log descriptor identity, resolved destination, bytes, request count, timing, + and policy decision without logging secrets. +- Expose actionable but non-sensitive failures for denied destinations, + unavailable sources, stale geometry, and exhausted limits. + +## Compatibility And Migration + +- Existing persisted `C2Array` carriers keep their current kind and decoder. +- `C2Array.save()` remains unchanged initially; it may delegate internally in a + later cleanup but should not silently start emitting `remote_proxy`. +- Existing persistent cache files using `proxy-source` keep their current + format and reopen behavior. +- `SimpleProxy` remains non-persistable. +- Existing `blosc2.open(remote, lazy=True)` behavior remains memory-cached. +- Existing `cache_path` and `cache_dir` calls remain disk-cached. +- The new B2 object kind must fail clearly on older readers, as other unknown B2 + object kinds do, without being mistaken for an ordinary empty NDArray. +- Stored policy values are stable lowercase strings; decoder code maps these to + enum members and rejects unknown values rather than guessing. + +## Proposed Code Organization + +### New module + +Add `src/blosc2/remote_proxy.py` containing: + +- `RemoteProxy` +- source normalization and safe descriptor validation used by the client +- backend selection +- policy-specific runtime adapters +- B2 object payload encode/decode helpers where this avoids import cycles + +Client-side validation improves error messages and prevents accidentally +writing credentials, but must be documented as distinct from Caterva2's +authoritative server-side policy. + +### Existing modules + +- `src/blosc2/__init__.py` + - define or re-export `CachePolicy` + - export `RemoteProxy` + - add both to `__all__` +- `src/blosc2/ref.py` + - add a versioned fsspec reference representation or a dedicated + remote-source reference helper + - do not change generic `Proxy` unwrapping semantics +- `src/blosc2/b2objects.py` + - encode and decode the `remote_proxy` B2 object kind + - build the metadata-only carrier +- `src/blosc2/schunk.py` + - recognize the new B2 object during `blosc2.open()` dispatch + - preserve precedence of persistent cache-proxy detection +- `src/blosc2/proxy.py` + - expose reusable internal assembly/cache pieces only as needed + - leave `SimpleProxy`'s public contract unchanged +- `src/blosc2/c2array.py` + - expose any small backend-neutral hooks needed by `RemoteProxy` + - retain its existing public persistence behavior + +If defining `CachePolicy` in `__init__.py` creates import cycles, place it in a +small non-private core module and re-export it from `blosc2`. The public name +and enum values are the compatibility surface, not its physical module. + +## Implementation Phases + +### Phase 0: Settle contracts + +- Confirm that `NONE` means no reusable data across independent operations, + while permitting operation-scoped memory. +- Decide `fetch()`/`afetch()` behavior under `NONE`. +- Confirm floating-reference semantics and structural validation. +- Agree on the Caterva2 protocol/host policy and initially supported fsspec + schemes. +- Version and document the `remote_proxy` payload before writing code. + +### Phase 1: Enum and source descriptors + +- Add and export `CachePolicy` with `NONE`, `MEMORY`, and `DISK`. +- Add normalized Caterva2 and fsspec descriptor creation. +- Reject secrets and unsupported URL constructions. +- Add round-trip tests for descriptors and enum payload values. + +### Phase 2: Runtime `RemoteProxy` + +- Implement construction and metadata discovery. +- Implement Caterva2 reads for all policies. +- Implement fsspec reads with operation-scoped assembly for `NONE`. +- Reuse existing memory and persistent `Proxy` caching for the other policies. +- Add traffic accounting and synchronous/asynchronous behavior. +- Verify source geometry before serving data. +- Add shared chunk-level LRU accounting and post-operation eviction for bounded + memory and disk caches. + +### Phase 3: Persistence + +- Add `remote_proxy` carrier encoding, `to_cframe()`, and `save()`. +- Add open-time dispatch and decoding. +- Guarantee reference-only persistence independently of runtime cache state, + normalizing the saved policy to `NONE`. +- Verify that reads never mutate a `NONE` carrier. + +### Phase 4: `blosc2.open` integration + +- Add `cache_policy` to the remote lazy-open path. +- Preserve old defaults and infer `DISK` from existing cache-location + arguments. +- Add conflict validation and focused regression tests. +- Keep the explicit `RemoteProxy` constructor available so descriptor creation + does not depend on overloaded `open()` behavior. + +### Phase 5: Caterva2 support + +- **Completed baseline:** direct `remote_proxy` discovery and read dispatch, + default-deny configuration, exact HTTPS host allowlists, public-address + validation, DNS pinning, disabled redirects, structural limits, and immutable + carrier handling. +- **5a:** add operation-wide byte, request, retry, deadline, and cancellation + budgets with enforcement at the authorized transport boundary. +- **5b:** add request-scoped observability and optional per-client rate limits + without introducing upstream credentials or retained cross-request state. +- **5c:** add policy-aware nested-reference decoding, graph depth/hop limits, + cycle detection, and one cumulative budget across the graph. +- Extend the security suite alongside each subphase; unsupported protocols and + embedded references remain denied until their corresponding subphase lands. + +This phase may live in the Caterva2 repository, but the feature should not be +presented as safe for arbitrary uploads until both sides are complete. + +### Phase 6: Documentation and examples + +- Add a `RemoteProxy` API page and include it in the reference toctree. +- Document the three cache policies and their lifetime guarantees. +- Add examples for a public Caterva2 source and an allowed fsspec HTTPS source; + add anonymous S3 if its server adapter is included. +- Document that private and credential-bearing sources are unsupported. +- Add a Caterva2 administrator guide for the security policy and operational + limits. + +## Test Plan + +### Unit tests + +- `CachePolicy` exports and serialized values. +- `RemoteProxy` metadata and indexing for mocked Caterva2 and fsspec sources. +- Descriptor normalization and rejection of credentials/unsupported schemes. +- B2 object cframe and file round trips for each supported source kind. +- Unknown payload versions, source kinds, and policy values fail clearly. +- Source geometry changes produce a stale-reference error. +- `RemoteProxy` works as a lazy-expression operand. + +### Cache-policy tests + +- Under `NONE`, two identical reads each cause remote data traffic. +- Under `NONE`, carrier size, bytes, mtime, and metadata are unchanged after + reads. +- Under `MEMORY`, an identical covered second read causes no remote data + traffic within the same object lifetime. +- The default memory cache retains at most 256 MiB of compressed payload after + each operation. +- A bounded memory cache evicts least-recently-used chunks and refetches them + when accessed again. +- Reopening a `MEMORY` carrier starts with an empty memory cache. +- Under `DISK`, a covered read survives close/reopen through the configured + cache path. +- A disk cache is unlimited by default; with an explicit bound its live payload + and file size fall after LRU eviction, allowing only fixed metadata overhead. +- Reopening a bounded disk cache preserves the limit and seeds deterministic + recency without requiring persistent timestamps. +- `RemoteProxy.save()` remains reference-only after memory or disk cache use. +- Invalid policy/cache-location combinations raise deterministic errors. + +### Compatibility tests + +- Existing `C2Array` cframes/files still reopen as `C2Array`. +- Existing `proxy-source` cache files still reopen as `Proxy` and preserve + fetched state. +- Existing remote `lazy=True` calls retain their current default caching. +- Existing lazy-expression serialization involving `Proxy` operands is + unchanged. +- The default non-network suite uses local mocks or a local range-capable HTTP + fixture; real services remain under the `network` marker. + +### Caterva2 security tests + +- Default-deny behavior for all remote descriptors. +- Allowed public destination succeeds. +- Local file, loopback, private/link-local addresses, cloud metadata endpoints, + forbidden ports, and disallowed buckets/prefixes are rejected. +- Redirect from an allowed URL to a denied destination is rejected. +- DNS rebinding or changed resolution is rejected at connection time. +- User-info, sensitive query data, custom credentials, and chained fsspec + protocols are rejected. +- Cycles and excessive reference depth are rejected. +- Byte, range, concurrency, timeout, decompression, and allocation limits are + enforced. +- One request cannot observe or reuse another request's temporary assembly + state. + +## Acceptance Criteria + +The feature is ready when all of the following hold: + +1. A user can persist and reopen a `RemoteProxy` for both Caterva2 and one + supported fsspec-backed single-file B2ND source. +2. A `NONE` carrier stays metadata-sized and byte-for-byte unchanged after any + supported read. +3. Repeated reads demonstrate observably different traffic behavior for + `NONE`, `MEMORY`, and `DISK`. +4. Bounded memory and disk caches evict whole least-recently-used chunks after + each operation without affecting the returned result; the default bound is + 256 MiB for memory and unlimited for disk. +5. `RemoteProxy.save()` never serializes fetched data or credentials. +6. Legacy C2Array and persistent cache-proxy files keep their behavior. +7. Caterva2 rejects remote proxies by default and resolves only public, + credential-free HTTPS sources allowed by administrator destination policy. +8. Geometry changes, denied destinations, and resource-limit failures produce + clear errors. +9. API and administrator documentation explain both caching semantics and the + outbound-request security boundary. + +## Future Considerations + +- Native `s3://` support is out of scope for Caterva2; public S3 objects use + their HTTPS URLs. Reconsidering native object-store protocols requires a + separate proposal. +- Reconsidering authenticated sources requires a separate security plan; it is + not an incremental configuration switch. +- When should pinned references be added, and what exact mismatch exception + should they raise? +- Should a future `C2Array.save(as_remote_proxy=True)` convenience exist, or is + the explicit `RemoteProxy(c2array.urlpath)` conversion clearer? +- Should local URLs remain accepted by Python-Blosc2 for testing while + Caterva2 rejects them, or should client-side construction enforce remote-only + schemes everywhere? + +## Initial Slice (Completed) + +The smallest end-to-end path was implemented as follows: + +1. `CachePolicy` and explicit `RemoteProxy` construction. +2. Public, single-file HTTPS/fsspec B2ND source. +3. `CachePolicy.NONE` with operation-scoped in-memory assembly. +4. `remote_proxy` cframe/file round trip. +5. Local mocked tests proving the carrier is immutable and repeated reads do + not reuse data. + +Caterva2-source support and memory/disk policy integration are also implemented +in the Python client. Caterva2 implements the default-deny, public HTTPS server +baseline described in Phase 5. The remaining server work is ordered in +`Open Work Queue And Decisions` and the Phase 5 subphases above. diff --git a/pyproject.toml b/pyproject.toml index f52615b1c..99908de8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,8 @@ documentation = "https://www.blosc.org/python-blosc2/python-blosc2.html" [project.optional-dependencies] parquet = ["pyarrow"] +zarr = ["zarr>=3.0.9"] +hdf5 = ["kerchunk", "h5py", "hdf5plugin"] # The b2view terminal viewer (the `b2view` script) is opt-in: most users want # blosc2 only as a compression library, and the TUI stack has no use under # wasm32 (no TTY). Install with `pip install "blosc2[tui]"`. This also pulls @@ -59,13 +61,16 @@ tui = ["textual", "textual-plotext"] # Adds the high-res 'h' view on top of [tui], rendering a real matplotlib image # (kitty/iTerm2/sixel, or half-cells elsewhere) — matplotlib is the heavy part. hires = ["blosc2[tui]", "textual-image", "matplotlib"] -# Read/write single-file containers through any fsspec URL (s3://, gs://, zip://, -# memory://...). The protocol backends (s3fs, gcsfs, adlfs...) are the caller's -# install: `pip install "blosc2[fsspec]" s3fs`. -fsspec = ["fsspec"] +# Read/write single-file containers through any fsspec URL (https://, s3://, +# gs://, zip://, memory://...). HTTP support is included; the other protocol +# backends (s3fs, gcsfs, adlfs...) are the caller's install: +# `pip install "blosc2[fsspec]" s3fs`. +fsspec = ["fsspec[http]"] [project.scripts] parquet-to-blosc2 = "blosc2.cli.parquet_to_blosc2:main" +b2nd-to-zarr = "blosc2.cli.b2nd_to_zarr:main" +blosc2-to-zarr = "blosc2.cli.b2nd_to_zarr:main" b2view = "blosc2.b2view.cli:main" [dependency-groups] @@ -74,6 +79,7 @@ dev = [ "h5py", "hdf5plugin", "jupyterlab", + "kerchunk", "matplotlib", "pandas", "plotly", @@ -91,6 +97,11 @@ test = [ # feature silently skips in CI. memory:// covers the protocol-generic paths, # and is also the configuration most users installing [fsspec] are in. "fsspec; platform_machine != 'wasm32'", + # Exercise the optional remote array sources instead of silently skipping + # tests/test_zarr_source.py and tests/test_hdf5_source.py. + "zarr>=3.0.9; platform_machine != 'wasm32'", + "kerchunk; platform_machine != 'wasm32'", + "h5py; platform_machine != 'wasm32'", # tests/test_fsspec_s3.py needs a real S3 endpoint (moto, served locally, so # still offline) and a real *async* backend (s3fs). memory:// is neither, and # cannot see the class of bug that lives there: awaiting an async filesystem's diff --git a/src/blosc2/__init__.py b/src/blosc2/__init__.py index 307e3bde0..d80b5fb8e 100644 --- a/src/blosc2/__init__.py +++ b/src/blosc2/__init__.py @@ -230,6 +230,14 @@ class IndexKind(Enum): OPSI = "opsi" +class CachePolicy(Enum): + """Retention policy for data read through a remote proxy.""" + + NONE = "none" + MEMORY = "memory" + DISK = "disk" + + from .blosc2_ext import ( DEFINED_CODECS_STOP, EXTENDED_HEADER_LENGTH, @@ -595,6 +603,9 @@ def _raise(exc): FsspecNDSource, Traffic, ) +from .zarr_source import ZarrNDSource +from .b2z_source import B2ZNDSource +from .hdf5_source import HDF5NDSource, available_datasets from .indexing import Index from .schunk import SChunk, load, open @@ -608,6 +619,8 @@ def _raise(exc): jit, as_simpleproxy, ) +from .remote_array import RemoteMetadataMapping, RemoteArray +from .remote_store import RemoteNode, RemoteStore from . import linalg from .linalg import tensordot, vecdot, permute_dims, matrix_transpose, matmul, transpose, diagonal, outer from .utils import linalg_funcs as linalg_funcs_list @@ -873,6 +886,7 @@ def _raise(exc): "BatchArray", # Enums "Codec", + "CachePolicy", "DParams", "DictStore", "EmbedStore", @@ -890,12 +904,19 @@ def _raise(exc): "Operand", "ByteRangeNDSource", "FsspecNDSource", + "B2ZNDSource", "Traffic", + "ZarrNDSource", + "HDF5NDSource", "Proxy", "ProxyNDField", "ProxyNDSource", "ProxySource", "Ref", + "RemoteMetadataMapping", + "RemoteArray", + "RemoteNode", + "RemoteStore", "SChunk", "SimpleProxy", "SpecialValue", @@ -919,6 +940,7 @@ def _raise(exc): "any", "arange", "array", + "available_datasets", "arccos", "arccosh", "arcsin", diff --git a/src/blosc2/b2objects.py b/src/blosc2/b2objects.py index 375a1c9f3..1f1292c5d 100644 --- a/src/blosc2/b2objects.py +++ b/src/blosc2/b2objects.py @@ -116,7 +116,7 @@ def encode_b2object_payload(obj) -> dict[str, Any] | None: return None -def decode_b2object_payload(payload: dict[str, Any], *, carrier_path=None): +def decode_b2object_payload(payload: dict[str, Any], *, carrier_path=None, carrier=None): kind = payload.get("kind") version = payload.get("version") if version != _B2OBJECT_VERSION: @@ -124,6 +124,10 @@ def decode_b2object_payload(payload: dict[str, Any], *, carrier_path=None): if kind == "c2array": ref = blosc2.Ref.from_dict(payload) return ref.open() + if kind == "remote_array": + if carrier is None: + raise ValueError("A persisted RemoteArray requires its B2ND carrier") + return blosc2.RemoteArray._from_payload(payload, carrier) if kind == "lazyexpr": return decode_structured_lazyexpr(payload, carrier_path=carrier_path) if kind == "lazyudf": @@ -227,7 +231,7 @@ def open_b2object(obj): schunk = getattr(obj, "schunk", obj) if getattr(schunk, "urlpath", None) is not None: carrier_path = pathlib.Path(schunk.urlpath).parent - opened = decode_b2object_payload(payload, carrier_path=carrier_path) + opened = decode_b2object_payload(payload, carrier_path=carrier_path, carrier=obj) if isinstance(opened, blosc2.LazyExpr | blosc2.LazyUDF): opened.array = obj opened.schunk = schunk diff --git a/src/blosc2/b2view/app.py b/src/blosc2/b2view/app.py index f86426f82..0eacd582e 100644 --- a/src/blosc2/b2view/app.py +++ b/src/blosc2/b2view/app.py @@ -2,9 +2,12 @@ from __future__ import annotations +import asyncio import contextlib +import copy import io import os +import threading from typing import TYPE_CHECKING, Any, ClassVar import numpy as np @@ -53,6 +56,7 @@ make_metadata_renderable, make_preview_renderables, ) +from blosc2.core import is_fsspec_url, parse_container_url if TYPE_CHECKING: from textual import events @@ -1968,9 +1972,9 @@ class B2ViewApp(App): CSS = """ #main { height: 1fr; } #tree-pane { width: 35%; border: solid $primary; } - #right-pane { width: 65%; } + #right-pane { width: 1fr; } #top-row { height: 40%; } - #meta-pane, #vlmeta-pane { width: 50%; border: solid $secondary; } + #meta-pane, #attrs-pane { width: 50%; border: solid $secondary; } #data-pane { height: 60%; border: solid $secondary; } #tree { height: 1fr; } #data-header { height: auto; padding: 0 1; } @@ -1978,11 +1982,12 @@ class B2ViewApp(App): #data-table { width: 1fr; height: 1fr; } #row-scrollbar { width: 1; height: 1fr; color: $primary; } #col-scrollbar { height: 1; width: 1fr; color: $primary; } - #meta-scroll, #vlmeta-scroll, #data-scroll { height: 1fr; padding: 0 1; } - #tree-pane:focus-within, #meta-pane:focus-within, #vlmeta-pane:focus-within, #data-pane:focus-within { border: heavy $accent; } + #meta-scroll, #attrs-scroll, #data-scroll { height: 1fr; padding: 0 1; } + #tree-pane:focus-within, #meta-pane:focus-within, #attrs-pane:focus-within, #data-pane:focus-within { border: heavy $accent; } B2ViewPanel.-maximized, #tree-pane.-maximized, #meta-pane.-maximized, + #attrs-pane.-maximized, #data-pane.-maximized { width: 1fr; height: 1fr; } """ @@ -2021,10 +2026,23 @@ def __init__( preview_cols: int = 10, download_url: str | None = None, info_url: str | None = None, + storage_options: dict[str, Any] | None = None, + cache_dir: str | None = None, + max_cache_bytes: int | None = None, ): super().__init__() self.sub_title = f"Python-Blosc2 {blosc2.__version__}" # shown beside the title in the header + if parse_container_url(urlpath)[2] in {"zarr", "hdf5"}: + # Initialize before Textual captures stderr (fileno=-1), which + # prevents multiprocessing's resource tracker from starting. + with contextlib.suppress(ImportError): + from numcodecs.blosc import get_mutex + + get_mutex() self.urlpath = urlpath + self.storage_options = storage_options + self.cache_dir = cache_dir + self.max_cache_bytes = max_cache_bytes self.download_url = download_url # when set, fetch urlpath before browsing self.info_url = info_url # optional: metadata endpoint giving the size # Header label: the path as given on the CLI, or the @public-relative @@ -2036,7 +2054,20 @@ def __init__( self.preview_rows = preview_rows self.preview_cols = preview_cols self.browser: StoreBrowser | None = None + # Set when a remote browser is closed on its own thread (on_unmount); + # lets teardown wait for the cache-dir lock to be released. + self._browser_close_thread: threading.Thread | None = None self.loaded_paths: set[str] = set() + self._remote = is_fsspec_url(urlpath) + self._remote_session = 0 + self._remote_request = 0 + self._remote_page_request = 0 + self._remote_page_pending = False + self._remote_col_end = None + self._remote_children = {} + self._listing_paths = set() + self._selected_info = None + self._closing = False self.selected_path = "/" self.table_page: dict | None = None self.table_buffer: dict | None = None @@ -2069,10 +2100,10 @@ def compose(self) -> ComposeResult: meta_pane.border_title = "meta" with VerticalScroll(id="meta-scroll", can_focus=True): yield Static("Select a node", id="metadata") - with B2ViewPanel(id="vlmeta-pane") as vlmeta_pane: - vlmeta_pane.border_title = "vlmeta" - with VerticalScroll(id="vlmeta-scroll", can_focus=True): - yield Static("", id="vlmetadata") + with B2ViewPanel(id="attrs-pane") as attrs_pane: + attrs_pane.border_title = "attrs" + with VerticalScroll(id="attrs-scroll", can_focus=True): + yield Static("", id="attrs-data") with B2ViewPanel(id="data-pane") as data_pane: data_pane.border_title = "data" data_pane.border_subtitle = ( @@ -2126,7 +2157,21 @@ def _after_download(self, result: bool | str) -> None: def _start_browsing(self) -> None: """Open the bundle and populate the tree (the normal startup path).""" - self.browser = StoreBrowser(self.urlpath) + if self._remote: + self.query_one("#metadata", Static).update("Loading remote container…") + self._open_remote(self._remote_session, self.start_path) + return + browser_kwargs: dict[str, Any] = { + "storage_options": self.storage_options, + "cache_dir": self.cache_dir, + } + if self.max_cache_bytes is not None: + browser_kwargs["max_cache_bytes"] = self.max_cache_bytes + self.browser = StoreBrowser(self.urlpath, **browser_kwargs) + self._populate_browser() + + def _populate_browser(self) -> None: + self.query_one("#tree-pane").display = self.browser.is_tree self.query_one(B2ViewHeader).set_filename(self._header_label) tree = self.query_one("#tree", Tree) tree.root.data = "/" @@ -2155,10 +2200,13 @@ def _apply_start_focus(self) -> None: def _focus_panel_by_name(self, name: str) -> None: """Focus a panel by its user-facing name.""" + if name == "tree" and not self.query_one("#tree-pane").display: + name = "data" panel_map = { "tree": lambda: self.query_one("#tree", Tree), "meta": lambda: self.query_one("#meta-scroll", VerticalScroll), - "vlmeta": lambda: self.query_one("#vlmeta-scroll", VerticalScroll), + "attrs": lambda: self.query_one("#attrs-scroll", VerticalScroll), + "vlmeta": lambda: self.query_one("#attrs-scroll", VerticalScroll), "data": lambda: ( self.query_one("#data-table", DataTable) if self.query_one("#data-table-row", Horizontal).display @@ -2201,19 +2249,154 @@ def _do_select(): self.call_after_refresh(_do_select) + @staticmethod + def _close_browser(browser): + with browser.io_lock: + browser.close() + def on_unmount(self) -> None: + self._closing = True + self._remote_session += 1 if self.browser is not None: - self.browser.close() + if self._remote: + closer = threading.Thread(target=self._close_browser, args=(self.browser,), daemon=True) + self._browser_close_thread = closer + closer.start() + else: + self.browser.close() + + async def _shutdown(self) -> None: + await super()._shutdown() + if self._browser_close_thread is not None: + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, self._browser_close_thread.join, 30.0) + + def run(self, *args, **kwargs): + try: + return super().run(*args, **kwargs) + finally: + if self._browser_close_thread is not None: + self._browser_close_thread.join(timeout=30.0) + + def wait_for_close(self, timeout: float = 30.0) -> None: + """Wait for any asynchronous remote browser shutdown to complete.""" + if self._browser_close_thread is not None: + self._browser_close_thread.join(timeout=timeout) + + def _deliver_remote(self, session, callback, *args): + """A blocking read may finish after cancellation, refresh, or shutdown.""" + if self._closing or session != self._remote_session: + return False + + def deliver(): + if self._closing or session != self._remote_session: + return False + callback(*args) + return True + + try: + return self.call_from_thread(deliver) + except RuntimeError: + return False + + @work(thread=True, exit_on_error=False) + def _open_remote(self, session, start_path): + browser = None + try: + # A refresh may still be closing the previous browser, which owns + # the exclusive cache lock until close() returns. Wait for it so + # the new store cannot fail with "cache is already owned". + closer = self._browser_close_thread + if closer is not None: + closer.join() + browser_kwargs: dict[str, Any] = { + "storage_options": self.storage_options, + "cache_dir": self.cache_dir, + } + if self.max_cache_bytes is not None: + browser_kwargs["max_cache_bytes"] = self.max_cache_bytes + browser = StoreBrowser(self.urlpath, **browser_kwargs) + children = {} + if browser.is_tree: + parent = "/" + children[parent] = browser.list_children(parent) + for part in start_path.strip("/").split("/"): + target = parent.rstrip("/") + "/" + part + found = next((c for c in children[parent] if c.path == target), None) + if found is None or found.kind != "group": + break + parent = target + children[parent] = browser.list_children(parent) + if not self._deliver_remote(session, self._finish_remote_open, browser, children): + browser.close() + except Exception as exc: + if browser is not None: + browser.close() + self._deliver_remote(session, self._remote_error, exc) + + def _finish_remote_open(self, browser, children): + self.browser = browser + self._remote_children = children + self._populate_browser() + + def _remote_error(self, exc): + # Transport exceptions can include signed URLs or credentials. Keep + # source-specific limitations, but remove runtime URLs and option values. + import re + + message = re.sub(r"[a-zA-Z][a-zA-Z0-9+.-]*://[^\s'\"]+", "", str(exc)) + + def redact(options): + nonlocal message + for value in options.values(): + if isinstance(value, dict): + redact(value) + elif isinstance(value, str) and value: + message = message.replace(value, "