From d1052108cf508e966c6c44ebbbabf273dfab9c0e Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Sat, 31 Oct 2026 22:45:25 +0100 Subject: [PATCH 01/21] ci(examples): run example scripts once in their own job (#4586) * ci(examples): run example scripts once in their own job Each example's PEP 723 header defines its environment, so running tests/test_examples.py in every pytest matrix job repeated the same work. Its header-rewriting helper also depended on the test environment's packaging and tomlkit, which broke the min_deps job on Python 3.12.3. A new `just examples` recipe uses `uv add --script` to point each header at the local checkout, and one CI job runs it. `uv add` resolves the header, and that showed two examples declaring requires-python >=3.11 although zarr requires 3.12. Those headers now say >=3.12. tomlkit was only used by the removed test, so it is dropped from the test dependency group. Assisted-by: ClaudeCode:claude-opus-5-5 * test(examples): run the example scripts through pytest The previous commit ran the examples from a bash loop, which stops at the first failing script, so one broken example hid the others. The examples now run from tests/test_examples.py again, one test per script. The tests are marked `examples` and skipped unless pytest gets --run-examples, as slow_hypothesis tests are with --run-slow-hypothesis, so the matrix jobs skip them. `just examples` passes the option, and the CI job runs it in the test.py3.13-minimal hatch environment. Assisted-by: ClaudeCode:claude-opus-5-5 * chore(just): rename the examples recipe to test-examples `just examples` could mean listing or building the examples. `test-examples` says what the recipe does, and follows the `-` naming of recipes like `coverage-html` and `docs-build`. Assisted-by: ClaudeCode:claude-opus-5-5 --- .github/workflows/test.yml | 29 +++++- Justfile | 4 + examples/README.md | 2 +- .../codec_pipeline_performance.py | 2 +- .../sharding_coalescing.py | 2 +- pyproject.toml | 2 +- tests/conftest.py | 26 +++-- tests/test_examples.py | 98 ++++++------------- uv.lock | 15 --- 9 files changed, 84 insertions(+), 96 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a4141d7356..3f995e8b5e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -201,6 +201,32 @@ jobs: run: | HATCH_ENV=test.py3.13-minimal just benchmark + examples: + name: Example scripts + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 # required for hatch version discovery when uv builds zarr + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.13' + cache: 'pip' + - name: Install uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + - name: Install Hatch + run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 + - name: Run Examples + run: | + HATCH_ENV=test.py3.13-minimal just test-examples + test-complete: name: Test complete @@ -209,7 +235,8 @@ jobs: test, test-upstream-and-min-deps, doctests, - benchmarks + benchmarks, + examples ] if: always() runs-on: ubuntu-latest diff --git a/Justfile b/Justfile index b3f187e231..df96003eef 100644 --- a/Justfile +++ b/Justfile @@ -64,6 +64,10 @@ hypothesis *args: doctest *args: hatch run doctest:pytest tests/test_docs.py -v "$@" +# Run each example script against the local checkout, in its PEP 723 environment +test-examples *args: + hatch run {{ quote(hatch_env) }}:pytest --run-examples tests/test_examples.py -v "$@" + # Run the benchmark suite benchmark *args: hatch run {{ quote(hatch_env) }}:pytest --benchmark-enable tests/benchmarks "$@" diff --git a/examples/README.md b/examples/README.md index a6b5fa2179..fa664d2c58 100644 --- a/examples/README.md +++ b/examples/README.md @@ -19,7 +19,7 @@ examples/ To add a new example: 1. Create a new subdirectory: `examples/my_example/` -2. Add your Python code: `examples/my_example/my_example.py` +2. Add your Python code: `examples/my_example/my_example.py`. Start it with a [PEP 723](https://peps.python.org/pep-0723/) header that lists its dependencies, including `zarr`. `just test-examples` runs every example against your local checkout, and CI runs it on every pull request. 3. Create documentation: `examples/my_example/README.md` 4. Create a documentation page at `docs/user-guide/examples/my_example.md`. The documentation page should simply link to the `README.md` and the source code, e.g.: diff --git a/examples/codec_pipeline_performance/codec_pipeline_performance.py b/examples/codec_pipeline_performance/codec_pipeline_performance.py index b821f3e6a7..d146c51f4a 100644 --- a/examples/codec_pipeline_performance/codec_pipeline_performance.py +++ b/examples/codec_pipeline_performance/codec_pipeline_performance.py @@ -1,5 +1,5 @@ # /// script -# requires-python = ">=3.11" +# requires-python = ">=3.12" # dependencies = [ # "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", # "numpy", diff --git a/examples/sharding_coalescing/sharding_coalescing.py b/examples/sharding_coalescing/sharding_coalescing.py index 5da62ed806..66838d2255 100644 --- a/examples/sharding_coalescing/sharding_coalescing.py +++ b/examples/sharding_coalescing/sharding_coalescing.py @@ -1,5 +1,5 @@ # /// script -# requires-python = ">=3.11" +# requires-python = ">=3.12" # dependencies = [ # "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", # "numpy", diff --git a/pyproject.toml b/pyproject.toml index 2554180f38..136f757dac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,7 +121,6 @@ test = [ "pytest-xdist==3.8.0", "pytest-benchmark==5.3.0", "pytest-codspeed==5.0.3", - "tomlkit==0.15.2", "uv==0.13.5", ] remote-tests = [ @@ -463,6 +462,7 @@ markers = [ "gpu: mark a test as requiring CuPy and GPU", "s3: mark a test as requiring a (mock) S3 backend via moto", "slow_hypothesis: slow hypothesis tests", + "examples: runs a script in examples/; skipped unless --run-examples is passed", ] [tool.repo-review] diff --git a/tests/conftest.py b/tests/conftest.py index 3e87a724e0..89eb90a1d8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -262,15 +262,29 @@ def pytest_addoption(parser: Any) -> None: default=False, help="run slow hypothesis tests", ) + parser.addoption( + "--run-examples", + action="store_true", + default=False, + help="run the scripts in examples/", + ) + + +# Tests with one of these markers are skipped unless the matching option is passed. +_OPT_IN_MARKERS = { + "slow_hypothesis": "--run-slow-hypothesis", + "examples": "--run-examples", +} def pytest_collection_modifyitems(config: Any, items: Any) -> None: - if config.getoption("--run-slow-hypothesis"): - return - skip_slow_hyp = pytest.mark.skip(reason="need --run-slow-hypothesis option to run") - for item in items: - if "slow_hypothesis" in item.keywords: - item.add_marker(skip_slow_hyp) + for marker, option in _OPT_IN_MARKERS.items(): + if config.getoption(option): + continue + skip = pytest.mark.skip(reason=f"need {option} option to run") + for item in items: + if marker in item.keywords: + item.add_marker(skip) settings.register_profile( diff --git a/tests/test_examples.py b/tests/test_examples.py index a6634e8cc6..2421a3d0ca 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1,87 +1,45 @@ from __future__ import annotations -import re import subprocess -import sys from pathlib import Path -from typing import Final import pytest -import tomlkit -from packaging.requirements import Requirement -examples_dir = "examples" -script_paths = tuple(Path(examples_dir).rglob("*.py")) +ZARR_PROJECT_PATH = Path(__file__).parent.parent +EXAMPLE_PATHS = tuple(sorted((ZARR_PROJECT_PATH / "examples").glob("*/*.py"))) -PEP_723_REGEX: Final = r"(?m)^# /// (?P[a-zA-Z0-9-]+)$\s(?P(^#(| .*)$\s)+)^# ///$" -# This is the absolute path to the local Zarr installation. Moving this test to a different directory will break it. -ZARR_PROJECT_PATH = Path(".").absolute() +def test_example_paths() -> None: + """The example scripts are found, so the parametrized test below is not empty.""" + assert len(EXAMPLE_PATHS) > 0 -def set_dep(script: str, dependency: str) -> str: +@pytest.mark.examples +@pytest.mark.parametrize("script_path", EXAMPLE_PATHS, ids=lambda path: path.stem) +def test_example_runs(script_path: Path, tmp_path: Path) -> None: """ - Set a dependency in a PEP-723 script header. - If the package is already in the list, it will be replaced. - If the package is not already in the list, it will be added. + Each example runs against this checkout, in the environment its PEP 723 header defines. - Source code modified from - https://packaging.python.org/en/latest/specifications/inline-script-metadata/#reference-implementation + The test copies the script and uses `uv add` to point the copy's zarr requirement at + this checkout. `uv add` also resolves the header, so a header that cannot be + satisfied fails here. """ - match = re.search(PEP_723_REGEX, script) - - if match is None: - raise ValueError(f"PEP-723 header not found in {script}") - - content = "".join( - line[2:] if line.startswith("# ") else line[1:] - for line in match.group("content").splitlines(keepends=True) - ) - - config = tomlkit.parse(content) - for idx, dep in enumerate(tuple(config["dependencies"])): - if Requirement(dep).name == Requirement(dependency).name: - config["dependencies"][idx] = dependency - - new_content = "".join( - f"# {line}" if line.strip() else f"#{line}" - for line in tomlkit.dumps(config).splitlines(keepends=True) - ) - - start, end = match.span("content") - return script[:start] + new_content + script[end:] - - -def resave_script(source_path: Path, dest_path: Path) -> None: - """ - Read a script from source_path and save it to dest_path after inserting the absolute path to the - local Zarr project directory in the PEP-723 header. - """ - source_text = source_path.read_text() - dest_text = set_dep(source_text, f"zarr @ file:///{ZARR_PROJECT_PATH}") - dest_path.write_text(dest_text) - - -def test_script_paths() -> None: - """ - Test that our test fixture is working properly and collecting script paths. - """ - assert len(script_paths) > 0 - - -@pytest.mark.skipif( - sys.platform == "win32", reason="This test fails for unknown reasons on Windows in CI." -) -@pytest.mark.parametrize("script_path", script_paths) -def test_scripts_can_run(script_path: Path, tmp_path: Path) -> None: dest_path = tmp_path / script_path.name - # We resave the script after inserting the absolute path to the local Zarr project directory, - # and then test its behavior. - # This allows the example to be useful to users who don't have Zarr installed, but also testable. - resave_script(script_path, dest_path) - result = subprocess.run( - ["uv", "run", "--refresh", str(dest_path)], capture_output=True, text=True, check=False + dest_path.write_text(script_path.read_text()) + + add = subprocess.run( + ["uv", "add", "--quiet", "--script", str(dest_path), str(ZARR_PROJECT_PATH)], + capture_output=True, + text=True, + check=False, ) - assert result.returncode == 0, ( - f"Script at {script_path} failed to run. Output: {result.stdout} Error: {result.stderr}" + assert add.returncode == 0, f"uv add failed for {script_path}:\n{add.stderr}" + + # `--reinstall-package` rebuilds zarr, so the run sees the current source. + run = subprocess.run( + ["uv", "run", "--reinstall-package", "zarr", str(dest_path)], + capture_output=True, + text=True, + check=False, ) + assert run.returncode == 0, f"{script_path} failed:\n{run.stdout}\n{run.stderr}" diff --git a/uv.lock b/uv.lock index f8094ec90e..e78182c154 100644 --- a/uv.lock +++ b/uv.lock @@ -3435,15 +3435,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e6/34/ebdc18bae6aa14fbee1a08b63c015c72b64868ff7dae68808ab500c492e2/tinycss2-1.4.0-py3-none-any.whl", hash = "sha256:3a49cf47b7675da0b15d0c6e1df8df4ebd96e9394bb905a5775adb0d884c5289", size = 26610, upload-time = "2024-10-24T14:58:28.029Z" }, ] -[[package]] -name = "tomlkit" -version = "0.15.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/08/72/fc01563df95c189a9f65850096bdc205bc2fe37da6c740cd64afaf18d07e/tomlkit-0.15.2.tar.gz", hash = "sha256:83e32755ba1d20d9340bd069ee97e9f6232ba2346b63ffcba425de01f7e327a6", size = 190522, upload-time = "2026-10-16T04:07:08.166Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/46/7d84138a60a8bd796825b2d38ebe92d0d16501ca57d501b7c5d677401bbc/tomlkit-0.15.2-py3-none-any.whl", hash = "sha256:05562412163a3037b94497385fb7f9204c637bc605af7a48199e41cf2272f437", size = 52225, upload-time = "2026-10-16T04:07:09.216Z" }, -] - [[package]] name = "towncrier" version = "26.9.0" @@ -3846,7 +3837,6 @@ dev = [ { name = "requests" }, { name = "ruff" }, { name = "s3fs" }, - { name = "tomlkit" }, { name = "towncrier" }, { name = "universal-pathlib" }, { name = "uv" }, @@ -3888,7 +3878,6 @@ remote-tests = [ { name = "pytest-xdist" }, { name = "requests" }, { name = "s3fs" }, - { name = "tomlkit" }, { name = "uv" }, ] test = [ @@ -3903,7 +3892,6 @@ test = [ { name = "pytest-cov" }, { name = "pytest-reportlog" }, { name = "pytest-xdist" }, - { name = "tomlkit" }, { name = "uv" }, ] @@ -3956,7 +3944,6 @@ dev = [ { name = "requests", specifier = "==2.34.2" }, { name = "ruff", specifier = "==0.17.1" }, { name = "s3fs", specifier = ">=2023.10.0" }, - { name = "tomlkit", specifier = "==0.15.2" }, { name = "towncrier", specifier = "==26.9.0" }, { name = "universal-pathlib" }, { name = "uv", specifier = "==0.13.5" }, @@ -3996,7 +3983,6 @@ remote-tests = [ { name = "pytest-xdist", specifier = "==3.8.0" }, { name = "requests", specifier = "==2.34.2" }, { name = "s3fs", specifier = ">=2023.10.0" }, - { name = "tomlkit", specifier = "==0.15.2" }, { name = "uv", specifier = "==0.13.5" }, ] test = [ @@ -4011,6 +3997,5 @@ test = [ { name = "pytest-cov", specifier = "==7.1.0" }, { name = "pytest-reportlog", specifier = "==1.0.0" }, { name = "pytest-xdist", specifier = "==3.8.0" }, - { name = "tomlkit", specifier = "==0.15.2" }, { name = "uv", specifier = "==0.13.5" }, ] From 4a72dd42a5b1cbaf0d8b3248dd3723df73708268 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Sun, 1 Nov 2026 09:42:38 +0100 Subject: [PATCH 02/21] feat(zarr-metadata): add Hypothesis strategies for metadata fields (#4575) * feat(zarr-metadata): add Hypothesis strategies for metadata fields Add `zarr_metadata.testing.strategies`, behind a new `testing` extra. It ships Hypothesis strategies that draw valid Zarr v3 metadata fields, one for every codec, data type, chunk grid and chunk key encoding the package defines that holds no other field. - Each kind's strategy takes what its configurations depend on: a data type's fill values take its configuration, a chunk grid takes the array shape, and a codec takes the `Chunk` it is handed. - `fields` draws every envelope spelling the spec allows, or only the canonical one. - Strategies are a mapping keyed by definition, so an extension author can add strategies for their own definitions. The tests read every draw back in `CORE_AND_EXTENSIONS` and require no problems. They also check: - that every definition in scope has a strategy, apart from the three that hold nested fields; - reach, with targets derived from each configuration's TypedDict: each optional key present and absent, each `Literal` value, and a non-canonical spelling where one exists; - fill-value spellings and envelope forms. Strategies follow the spec where the package is looser. Core data types are drawn only as bare strings, and regular-grid chunk lengths are never zero ("Chunk sizes must be greater than zero"), although the package's rules accept both today. Assisted-by: ClaudeCode:claude-opus-5-5 * chore(zarr-metadata): number the strategies changelog fragment after #414 Assisted-by: ClaudeCode:claude-opus-5-5 * chore(zarr-metadata): number the strategies changelog fragment after PR 4575 The fragment was numbered after fork PR 414, so the changelog linked to an unrelated upstream PR. Assisted-by: ClaudeCode:claude-opus-5-5 * fix(zarr-metadata): draw a fresh empty configuration on every draw `st.just({})` (crc32c) and `st.just(EmptyConfiguration())` (every data type with nothing to configure) handed out one dict for every draw, and `fields` puts the configuration in the field as it is. A test that changed the configuration it drew changed every later draw, which Hypothesis reports as a flaky strategy. Both are now `st.builds(EmptyConfiguration)`. Export `Configurations` and `NOTHING_KNOWN` from `zarr_metadata.testing.strategies`: they appear in the signatures of `DefinitionStrategy.unconstrained` and `codec_configurations`. Assisted-by: ClaudeCode:claude-opus-5-5 * fix(zarr-metadata): check every drawn field, and draw hex scale_offset values `fields` read a field only when asked for the canonical spelling, so a caller's configurations the definition refuses were drawn silently in the default spelling. Every field is now read, in a scope built once per call rather than once per draw, and a refused one raises in both spellings. A definition no Zarr v3 scope holds, such as a v2 codec, is a `TypeError` when `fields` is called, not a `StopIteration` on a draw. A float `scale_offset` codec's `offset` and `scale` are fill values of its type, so they are now also drawn as the hex string of their bits. The docstring no longer claims a non-zero scale makes decoding defined: a tiny one still overflows. The codec chunk test now draws chunks of an unknown data type and with unknown lengths, which every codec takes. Assisted-by: ClaudeCode:claude-opus-5-5 * fix(zarr-metadata): draw scale_offset for every number type, and widen Strategies keys `scale_offset` drew nothing for complex numbers, numpy times and data types the package does not define, though its rules refuse only data types whose values are not numbers, and a numpy time "is compatible with any codec that supports arrays of signed 64-bit integers". A numpy time is now drawn as that integer, a complex number as a pair of finite floats, and a type the package does not define takes the identity configuration. None now means only what `CodecStrategy` says it means: no configuration takes the chunk. `Strategies` is keyed by `Any`: a mapping's key type is invariant, so a mapping an extension author builds, keyed by a `CodecDefinition`, was not a `Mapping[Definition[Any], ...]` to pyright. A rectilinear dimension of unknown length may now be drawn with no edges, which its rules accept. Assisted-by: ClaudeCode:claude-opus-5-5 * refactor(zarr-metadata): share the strategies' empty configuration and rank cap `EMPTY_CONFIGURATIONS` and `MAX_RANK` are defined once, in `_strategy`, rather than once in each module that draws them. A numpy time's `scale_offset` values are now drawn finite, never the integer numpy reads as `NaT`, as the docstring says. The `fields` docstring now says that a codec's chunk rules are not checked, since no chunk is known. Assisted-by: ClaudeCode:claude-opus-5-5 * test(zarr-metadata): cover scale_offset over a data type the package does not define The identity configuration `scale_offset` draws for a data type the package does not define had no test, since the chunk test draws only the package's own data types. A test now reads an extension `bfloat16` in a scope of its own and checks the codec takes every draw. Also splits the `_scope_of` error message across two literals and drops a doubled blank line in `_data_type`. Assisted-by: ClaudeCode:claude-opus-5-5 --------- Co-authored-by: Claude Opus 5.5 --- .../zarr-metadata/changes/4575.feature.md | 1 + packages/zarr-metadata/docs/api/testing.md | 8 + packages/zarr-metadata/mkdocs.yml | 1 + packages/zarr-metadata/pyproject.toml | 5 + .../src/zarr_metadata/testing/__init__.py | 6 + .../src/zarr_metadata/testing/_chunk_grid.py | 105 +++++ .../testing/_chunk_key_encoding.py | 40 ++ .../src/zarr_metadata/testing/_codec.py | 179 ++++++++ .../src/zarr_metadata/testing/_data_type.py | 200 ++++++++ .../src/zarr_metadata/testing/_strategy.py | 147 ++++++ .../src/zarr_metadata/testing/strategies.py | 299 ++++++++++++ .../zarr-metadata/tests/test_public_api.py | 8 + .../zarr-metadata/tests/testing/__init__.py | 0 .../tests/testing/test_strategies.py | 432 ++++++++++++++++++ packages/zarr-metadata/uv.lock | 7 + 15 files changed, 1438 insertions(+) create mode 100644 packages/zarr-metadata/changes/4575.feature.md create mode 100644 packages/zarr-metadata/docs/api/testing.md create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/__init__.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/_codec.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/strategies.py create mode 100644 packages/zarr-metadata/tests/testing/__init__.py create mode 100644 packages/zarr-metadata/tests/testing/test_strategies.py diff --git a/packages/zarr-metadata/changes/4575.feature.md b/packages/zarr-metadata/changes/4575.feature.md new file mode 100644 index 0000000000..bd1d87ab40 --- /dev/null +++ b/packages/zarr-metadata/changes/4575.feature.md @@ -0,0 +1 @@ +Added `zarr_metadata.testing.strategies`, Hypothesis strategies that draw valid Zarr v3 metadata fields, behind a new `testing` extra (`pip install zarr-metadata[testing]`). There is one strategy for every codec, data type, chunk grid and chunk key encoding the package defines that holds no other field. `fields` draws every envelope spelling the spec allows, or only the canonical one with `spellings="canonical"`. Codecs can be drawn for the `Chunk` they are handed, chunk grids for an array shape, and fill values for a data type. The strategies are a mapping keyed by definition, so an extension author can add strategies for their own definitions. diff --git a/packages/zarr-metadata/docs/api/testing.md b/packages/zarr-metadata/docs/api/testing.md new file mode 100644 index 0000000000..a5d6145258 --- /dev/null +++ b/packages/zarr-metadata/docs/api/testing.md @@ -0,0 +1,8 @@ +--- +title: testing.strategies +--- + +::: zarr_metadata.testing.strategies + options: + inherited_members: false + show_if_no_docstring: false diff --git a/packages/zarr-metadata/mkdocs.yml b/packages/zarr-metadata/mkdocs.yml index 10464e3cf0..b19c0932d2 100644 --- a/packages/zarr-metadata/mkdocs.yml +++ b/packages/zarr-metadata/mkdocs.yml @@ -28,6 +28,7 @@ nav: - ' zarr_metadata.model': api/model.md - ' zarr_metadata.problems': api/problems.md - ' zarr_metadata.pydantic': api/pydantic.md + - ' zarr_metadata.testing.strategies': api/testing.md - ' zarr_metadata.typed_json': api/typed_json.md - ' zarr_metadata.v2': api/v2.md - ' zarr_metadata.v3': diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index abd93bc729..25d1c7aead 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -43,6 +43,11 @@ dependencies = [ "typing_extensions>=4.16", ] +[project.optional-dependencies] +# `zarr_metadata.testing.strategies`. 6.85: first release with +# `characters(exclude_categories=...)`. +testing = ["hypothesis>=6.85"] + [project.urls] Homepage = "https://github.com/zarr-developers/zarr-python" Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata" diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/__init__.py b/packages/zarr-metadata/src/zarr_metadata/testing/__init__.py new file mode 100644 index 0000000000..271a4dfe51 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/__init__.py @@ -0,0 +1,6 @@ +"""Testing utilities for code that reads or writes Zarr metadata. + +`zarr_metadata.testing.strategies` holds Hypothesis strategies that draw +valid metadata. It needs the `testing` extra: +`pip install zarr-metadata[testing]`. +""" diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py new file mode 100644 index 0000000000..b7528a3907 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py @@ -0,0 +1,105 @@ +"""Strategies for each chunk grid this package defines. + +Over an array of a known shape, a grid is drawn to fit it: a chunk +length for each dimension, never zero ("Chunk sizes must be greater +than zero", https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-grids/regular-grid/index.rst#L40), +and rectilinear edges that cover each dimension. Over no shape, what is +drawn is what the grid's rules accept. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from hypothesis import strategies as st + +from zarr_metadata.testing._strategy import MAX_RANK, ChunkGridStrategy +from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID +from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import ChunkGridDefinition + +_MAX_LENGTH: Final = 16 +"""The longest chunk drawn along a dimension whose length is unknown.""" + +_MAX_RUNS: Final = 5 +"""The most runs of equal lengths a rectilinear dimension is drawn with.""" + +_MAX_COUNT: Final = 4 +"""The most chunks a run is drawn with.""" + + +def _length(extent: int | None) -> st.SearchStrategy[int]: + """A chunk length along a dimension of `extent`: at least 1, and up to twice the extent when it is known and not 0.""" + if extent is None or extent == 0: + return st.integers(1, _MAX_LENGTH) + return st.integers(1, 2 * extent) + + +def _regular(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]: + """A chunk length for each dimension.""" + if shape is None: + lengths = st.lists(_length(None), max_size=MAX_RANK).map(tuple) + else: + lengths = st.tuples(*(_length(extent) for extent in shape)) + return lengths.map(lambda chunk_shape: {"chunk_shape": chunk_shape}) + + +def _spelled_run(draw: st.DrawFn, size: int, count: int) -> list[JSONValue]: + """`count` chunks of `size`: one `[size, count]` pair, or the size written once for each.""" + if draw(st.booleans()): + return [(size, count)] + return [size] * count + + +@st.composite +def _edges(draw: st.DrawFn, extent: int | None) -> JSONValue: + """Edge lengths along a dimension, covering `extent`, in runs spelled as pairs or written out. + + Runs are drawn apart, so two of one size can sit side by side and a + pair can count one chunk: spellings the canonical form merges. A + dimension of length zero, or of a length not known, may be covered by + no edges at all. + """ + min_runs = 0 if extent is None or extent == 0 else 1 + runs = draw( + st.lists( + st.tuples(st.integers(1, _MAX_LENGTH), st.integers(1, _MAX_COUNT)), + min_size=min_runs, + max_size=_MAX_RUNS, + ) + ) + covered = sum(size * count for size, count in runs) + if extent is not None and covered < extent: + runs.append((extent - covered, 1)) + spelled: list[JSONValue] = [] + for size, count in runs: + spelled.extend(_spelled_run(draw, size, count)) + return tuple(spelled) + + +def _rectilinear_dimension(extent: int | None) -> st.SearchStrategy[JSONValue]: + """A bare length, which repeats until it covers the dimension, or edges that cover it.""" + return _length(extent) | _edges(extent) + + +def _rectilinear(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]: + """Inline chunk shapes, one entry for each dimension.""" + if shape is None: + dimensions = st.lists(_rectilinear_dimension(None), max_size=MAX_RANK).map(tuple) + else: + dimensions = st.tuples(*(_rectilinear_dimension(extent) for extent in shape)) + return dimensions.map(lambda chunk_shapes: {"kind": "inline", "chunk_shapes": chunk_shapes}) + + +CHUNK_GRID_STRATEGIES: Final[Mapping[ChunkGridDefinition[Any], ChunkGridStrategy]] = { + REGULAR_CHUNK_GRID: ChunkGridStrategy(configurations=_regular), + RECTILINEAR_CHUNK_GRID: ChunkGridStrategy(configurations=_rectilinear), +} +"""A strategy for each chunk grid.""" + +__all__ = ["CHUNK_GRID_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py new file mode 100644 index 0000000000..9c525118c9 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py @@ -0,0 +1,40 @@ +"""Strategies for each chunk key encoding this package defines.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from hypothesis import strategies as st + +from zarr_metadata.testing._strategy import ChunkKeyEncodingStrategy +from zarr_metadata.v3.chunk_key_encoding.default import ( + DEFAULT_CHUNK_KEY_ENCODING, + DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR, +) +from zarr_metadata.v3.chunk_key_encoding.v2 import ( + V2_CHUNK_KEY_ENCODING, + V2_CHUNK_KEY_ENCODING_SEPARATOR, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.v3._definition import ChunkKeyEncodingDefinition + +CHUNK_KEY_ENCODING_STRATEGIES: Final[ + Mapping[ChunkKeyEncodingDefinition[Any], ChunkKeyEncodingStrategy] +] = { + DEFAULT_CHUNK_KEY_ENCODING: ChunkKeyEncodingStrategy( + configurations=st.fixed_dictionaries( + {}, optional={"separator": st.sampled_from(DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR)} + ) + ), + V2_CHUNK_KEY_ENCODING: ChunkKeyEncodingStrategy( + configurations=st.fixed_dictionaries( + {}, optional={"separator": st.sampled_from(V2_CHUNK_KEY_ENCODING_SEPARATOR)} + ) + ), +} +"""A strategy for each chunk key encoding: a `separator`, or none, which means the encoding's default.""" + +__all__ = ["CHUNK_KEY_ENCODING_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py new file mode 100644 index 0000000000..caf7b71a05 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -0,0 +1,179 @@ +"""Strategies for each codec this package defines that holds no other field. + +A codec handed an array draws what fits the chunk: a `bytes` codec an +`endian` for a type whose values take several bytes, a `transpose` an +`order` with an entry for each axis. Handed `Chunk()`, nothing is known +of the chunk, and what is drawn is what the codec's rules accept. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from hypothesis import strategies as st + +from zarr_metadata.definition._definition import AcceptedField +from zarr_metadata.testing._data_type import ( + COMPLEX_WIDTHS, + FLOAT_WIDTHS, + INTEGER_RANGES, + FloatWidth, + finite_floats, + float_spellings, +) +from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, MAX_RANK, CodecStrategy +from zarr_metadata.v3._definition import Chunk, storage_of +from zarr_metadata.v3.codec._arithmetic import NOT_NUMBERS +from zarr_metadata.v3.codec.blosc import BLOSC_CNAME, BLOSC_CODEC, BLOSC_NO_SHUFFLE, BLOSC_SHUFFLE +from zarr_metadata.v3.codec.bytes import BYTES_CODEC, ENDIANNESS +from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC +from zarr_metadata.v3.codec.gzip import GZIP_CODEC +from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC +from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC +from zarr_metadata.v3.codec.zstd import ZSTD_CODEC, ZSTD_MAX_LEVEL, ZSTD_MIN_LEVEL +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import CodecDefinition + +_BLOSC_MAX_TYPESIZE: Final = 255 +"""The widest element c-blosc takes: `BLOSC_MAX_TYPESIZE`.""" + +_BLOSC_MAX_BLOCKSIZE: Final = 2**31 - 1 +"""The largest block c-blosc takes, a signed 32-bit size; 0 lets it choose.""" + +_ENDIAN: Final = st.sampled_from(ENDIANNESS) + +_NUMPY_TIMES: Final = frozenset({NUMPY_DATETIME64_DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE}) +"""The numpy time types, each stored as a signed 64-bit integer.""" + +_NUMPY_TIME_RANGE: Final = (-(2**63) + 1, 2**63 - 1) +"""The signed 64-bit integers but the least, which numpy reads as `NaT`: the finite numpy times.""" + + +def _bytes(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: + """An `endian`, required for a data type whose values take several bytes; None for one whose values vary in size.""" + storage = None if chunk.data_type is None else storage_of(chunk.data_type) + if storage == "variable_length": + return None + if storage == "multi_byte": + return st.fixed_dictionaries({"endian": _ENDIAN}) + return st.fixed_dictionaries({}, optional={"endian": _ENDIAN}) + + +@st.composite +def _blosc(draw: st.DrawFn) -> Mapping[str, JSONValue]: + """A compressor, a level, a shuffle and a block size; and a `typesize`, which only `noshuffle` may leave out. + + Under `noshuffle` the spec says `typesize` "is ignored", so it is + drawn there too, a spelling that is not the simplest. + """ + shuffle = draw(st.sampled_from(BLOSC_SHUFFLE)) + typesize = st.integers(1, _BLOSC_MAX_TYPESIZE) + required: dict[str, st.SearchStrategy[JSONValue]] = { + "cname": st.sampled_from(BLOSC_CNAME), + "clevel": st.integers(0, 9), + "shuffle": st.just(shuffle), + "blocksize": st.integers(0, _BLOSC_MAX_BLOCKSIZE), + } + if shuffle == BLOSC_NO_SHUFFLE: + return draw(st.fixed_dictionaries(required, optional={"typesize": typesize})) + return draw(st.fixed_dictionaries({**required, "typesize": typesize})) + + +def _order(rank: int) -> st.SearchStrategy[Mapping[str, JSONValue]]: + return st.permutations(range(rank)).map(lambda order: {"order": tuple(order)}) + + +def _transpose(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]]: + """A permutation of the chunk's axes; of some number of axes, when the chunk does not say.""" + if chunk.rank is not None: + return _order(chunk.rank) + return st.integers(0, MAX_RANK).flatmap(_order) + + +def _finite_float_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A finite float of `width` bits, as a number or the hex string of its bits.""" + return finite_floats(width).flatmap(lambda value: float_spellings(value, width)) + + +def _non_zero_float_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A finite, non-zero float of `width` bits, as a number or the hex string of its bits.""" + return ( + finite_floats(width) + .filter(lambda value: value != 0) + .flatmap(lambda value: float_spellings(value, width)) + ) + + +def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: + """An `offset` and a `scale`, each a finite value of the data type the codec is handed, the scale not zero; None for a data type whose values are not numbers. + + Each is a fill value of the chunk's data type + (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L39-L43), + a float's as a number or the hex string of its bits, a complex + number's as a pair of them. Each is drawn finite, and the scale + non-zero, so that decoding never divides by zero; a scale small + enough that decoding overflows is still drawn. A numpy time is drawn + as the signed 64-bit integer it is stored as, never `NaT`, since it + "is compatible with any codec that supports arrays of signed 64-bit + integers" + (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120). + A data type this package does not define, whose values it cannot + draw, takes only the configuration with neither, the identity. + Handed a chunk whose data type is unknown, the values are small + non-negative integers, which every integer and float type holds. + """ + data_type = chunk.data_type + definition = data_type.definition if isinstance(data_type, AcceptedField) else None + if definition is None: + offset: st.SearchStrategy[JSONValue] = st.integers(0, 127) + scale: st.SearchStrategy[JSONValue] = st.integers(1, 127) + elif definition.name in NOT_NUMBERS: + return None + elif definition in INTEGER_RANGES or definition in _NUMPY_TIMES: + low, high = INTEGER_RANGES.get(definition, _NUMPY_TIME_RANGE) + offset = st.integers(low, high) + scale = st.integers(low, high).filter(lambda value: value != 0) + elif definition in FLOAT_WIDTHS: + offset = _finite_float_values(FLOAT_WIDTHS[definition]) + scale = _non_zero_float_values(FLOAT_WIDTHS[definition]) + elif definition in COMPLEX_WIDTHS: + width = COMPLEX_WIDTHS[definition] + offset = st.tuples(_finite_float_values(width), _finite_float_values(width)) + # Non-zero in either part is a non-zero complex number. + scale = st.one_of( + st.tuples(_non_zero_float_values(width), _finite_float_values(width)), + st.tuples(_finite_float_values(width), _non_zero_float_values(width)), + ) + else: + return EMPTY_CONFIGURATIONS + return st.fixed_dictionaries({}, optional={"offset": offset, "scale": scale}) + + +def _unchunked(configurations: st.SearchStrategy[Mapping[str, JSONValue]]) -> CodecStrategy: + """A codec whose configurations do not depend on the chunk it is handed.""" + return CodecStrategy(configurations=lambda _chunk: configurations) + + +CODEC_STRATEGIES: Final[Mapping[CodecDefinition[Any], CodecStrategy]] = { + BLOSC_CODEC: _unchunked(_blosc()), + BYTES_CODEC: CodecStrategy(configurations=_bytes), + CRC32C_CODEC: _unchunked(EMPTY_CONFIGURATIONS), + GZIP_CODEC: _unchunked(st.fixed_dictionaries({"level": st.integers(0, 9)})), + SCALE_OFFSET_CODEC: CodecStrategy(configurations=_scale_offset), + TRANSPOSE_CODEC: CodecStrategy(configurations=_transpose), + ZSTD_CODEC: _unchunked( + st.fixed_dictionaries( + {"level": st.integers(ZSTD_MIN_LEVEL, ZSTD_MAX_LEVEL)}, + optional={"checksum": st.booleans()}, + ) + ), +} +"""A strategy for each codec that holds no other field.""" + +__all__ = ["CODEC_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py new file mode 100644 index 0000000000..dd06df28a0 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py @@ -0,0 +1,200 @@ +"""Strategies for each data type this package defines that holds no other field. + +A fill value is drawn in every spelling the data type's spec gives it: +a float's as a number, a named non-finite value or a hex string of its +bits, a `bytes` value's as an array of byte values or base64. +""" + +from __future__ import annotations + +import base64 +import struct +from typing import TYPE_CHECKING, Any, Final, Literal, cast, get_args + +from hypothesis import strategies as st + +from zarr_metadata._numpy import NUMPY_TIME_MAX_SCALE_FACTOR +from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, DataTypeStrategy +from zarr_metadata.v3.data_type._float import FloatSpecialFillValue +from zarr_metadata.v3.data_type._numpy_time import NUMPY_TIME_UNIT +from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE +from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE +from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE +from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE +from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE +from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE +from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE +from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE +from zarr_metadata.v3.data_type.int16 import INT16_DATA_TYPE +from zarr_metadata.v3.data_type.int32 import INT32_DATA_TYPE +from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE +from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE, RawBytesConfiguration +from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE +from zarr_metadata.v3.data_type.uint8 import UINT8_DATA_TYPE +from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE +from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE +from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import DataTypeDefinition + +FloatWidth = Literal[16, 32, 64] + +INTEGER_RANGES: Final[Mapping[DataTypeDefinition[Any], tuple[int, int]]] = { + INT8_DATA_TYPE: (-(2**7), 2**7 - 1), + INT16_DATA_TYPE: (-(2**15), 2**15 - 1), + INT32_DATA_TYPE: (-(2**31), 2**31 - 1), + INT64_DATA_TYPE: (-(2**63), 2**63 - 1), + UINT8_DATA_TYPE: (0, 2**8 - 1), + UINT16_DATA_TYPE: (0, 2**16 - 1), + UINT32_DATA_TYPE: (0, 2**32 - 1), + UINT64_DATA_TYPE: (0, 2**64 - 1), +} +"""The values each integer data type holds.""" + +FLOAT_WIDTHS: Final[Mapping[DataTypeDefinition[Any], FloatWidth]] = { + FLOAT16_DATA_TYPE: 16, + FLOAT32_DATA_TYPE: 32, + FLOAT64_DATA_TYPE: 64, +} +"""The width in bits of each floating-point data type.""" + +COMPLEX_WIDTHS: Final[Mapping[DataTypeDefinition[Any], FloatWidth]] = { + COMPLEX64_DATA_TYPE: 32, + COMPLEX128_DATA_TYPE: 64, +} +"""The width in bits of each part of each complex data type.""" + +_MAX_RAW_BYTES: Final = 32 +"""The widest raw bits drawn, in bytes: `r256`.""" + +_MAX_BYTES: Final = 16 +"""The longest `bytes` fill value drawn.""" + + +def finite_floats(width: FloatWidth) -> st.SearchStrategy[float]: + """Finite numbers a float of `width` bits holds exactly.""" + return st.floats(width=width, allow_nan=False, allow_infinity=False) + + +_PACKED: Final[Mapping[FloatWidth, str]] = {16: ">e", 32: ">f", 64: ">d"} +"""How `struct` packs a float of each width, most significant byte first.""" + + +def _hex_spellings(digits: str) -> st.SearchStrategy[str]: + """`0x` and the digits, the letters in either case.""" + return st.sampled_from((f"0x{digits}", f"0x{digits.upper()}")) + + +def float_spellings(value: float, width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """`value`, a float of `width` bits, as a number or as the hex string of its bits.""" + return st.just(value) | _hex_spellings(struct.pack(_PACKED[width], value).hex()) + + +def _float_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A number, an integer the type holds exactly, a named non-finite value, or the bits in hex. + + A float's fill value is any of these + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79). + Numbers are drawn from those the type holds, so none is rounded: a + `float16` takes `1e300` by the letter of the spec, and a reader is + free to make of it what it will. + """ + # Every integer of magnitude up to 2**11 is a float16, the narrowest type. + numbers = finite_floats(width) | st.integers(-(2**11), 2**11) + named = st.sampled_from(get_args(FloatSpecialFillValue)) + bits = st.integers(0, 2**width - 1).flatmap( + lambda value: _hex_spellings(f"{value:0{width // 4}x}") + ) + return st.one_of(numbers, named, bits) + + +def _fill_values_of(values: st.SearchStrategy[JSONValue]) -> DataTypeStrategy: + """A data type with nothing to configure, whose fill values are `values`.""" + return DataTypeStrategy( + configurations=EMPTY_CONFIGURATIONS, fill_values=lambda _configuration: values + ) + + +def _byte_values(size: int) -> st.SearchStrategy[JSONValue]: + """`size` byte values, each an integer in `[0, 255]`, as a JSON array.""" + return st.lists(st.integers(0, 255), min_size=size, max_size=size).map( + lambda values: cast("JSONValue", tuple(values)) + ) + + +def _raw_bytes_fill_values(configuration: Mapping[str, JSONValue]) -> st.SearchStrategy[JSONValue]: + """One byte value for each 8 bits of the size.""" + return _byte_values(cast("RawBytesConfiguration", configuration)["bits"] // 8) + + +def _base64(value: bytes) -> JSONValue: + return base64.b64encode(value).decode("ascii") + + +def _complex_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A pair of float fill values of `width` bits, real then imaginary.""" + component = _float_fill_values(width) + return st.tuples(component, component).map(lambda pair: cast("JSONValue", pair)) + + +_BYTES_FILL_VALUES: Final = st.integers(0, _MAX_BYTES).flatmap(_byte_values) | st.binary( + max_size=_MAX_BYTES +).map(_base64) +"""A `bytes` fill value: byte values, or the bytes in standard-alphabet base64.""" + +_TEXT: Final = st.text(st.characters(exclude_categories=("Cs",)), max_size=16) +"""Text a JSON document can hold: no lone surrogate, which is no Unicode scalar value.""" + +_NUMPY_TIME: Final = DataTypeStrategy( + configurations=st.fixed_dictionaries( + { + "unit": st.sampled_from(NUMPY_TIME_UNIT), + "scale_factor": st.integers(1, NUMPY_TIME_MAX_SCALE_FACTOR), + } + ), + fill_values=lambda _configuration: st.integers(-(2**63), 2**63 - 1) | st.just("NaT"), +) +"""Both numpy time types: a unit, and how many of it one tick is; a signed 64-bit count of ticks, or `"NaT"`.""" + +DATA_TYPE_STRATEGIES: Final[Mapping[DataTypeDefinition[Any], DataTypeStrategy]] = { + BOOL_DATA_TYPE: _fill_values_of(st.booleans()), + **{ + definition: _fill_values_of(st.integers(low, high)) + for definition, (low, high) in INTEGER_RANGES.items() + }, + **{ + definition: _fill_values_of(_float_fill_values(width)) + for definition, width in FLOAT_WIDTHS.items() + }, + **{ + definition: _fill_values_of(_complex_fill_values(width)) + for definition, width in COMPLEX_WIDTHS.items() + }, + RAW_BYTES_DATA_TYPE: DataTypeStrategy( + configurations=st.integers(1, _MAX_RAW_BYTES).map( + lambda size: RawBytesConfiguration(bits=8 * size) + ), + fill_values=_raw_bytes_fill_values, + ), + BYTES_DATA_TYPE: _fill_values_of(_BYTES_FILL_VALUES), + STRING_DATA_TYPE: _fill_values_of(_TEXT), + NUMPY_DATETIME64_DATA_TYPE: _NUMPY_TIME, + NUMPY_TIMEDELTA64_DATA_TYPE: _NUMPY_TIME, +} +"""A strategy for each data type that holds no other field.""" + +__all__ = [ + "COMPLEX_WIDTHS", + "DATA_TYPE_STRATEGIES", + "FLOAT_WIDTHS", + "INTEGER_RANGES", + "FloatWidth", + "finite_floats", + "float_spellings", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py new file mode 100644 index 0000000000..73de0f8f5f --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py @@ -0,0 +1,147 @@ +"""What draws one definition's configurations: a record for each kind of definition. + +Each record holds what its kind's configurations depend on, the way +each kind of definition does: a data type's fill values depend on its +configuration, a chunk grid's on the shape of the array, a codec's on +the chunk it is handed. A chunk key encoding's depend on nothing. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, ClassVar, Final, TypeAlias, cast + +from hypothesis import strategies as st + +from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata.v3._definition import ( + Chunk, + ChunkGridDefinition, + ChunkKeyEncodingDefinition, + CodecDefinition, + DataTypeDefinition, +) + +if TYPE_CHECKING: + from hypothesis.strategies import SearchStrategy + + from zarr_metadata._common import JSONValue + from zarr_metadata.definition._definition import Definition + +Configurations: TypeAlias = "SearchStrategy[Mapping[str, JSONValue]]" +"""Configurations of one definition, each the mapping every configuration is.""" + +NOTHING_KNOWN: Final = Chunk() +"""A chunk nothing is known of, which every codec takes.""" + +MAX_RANK: Final = 5 +"""The most axes drawn where the number of axes is not known.""" + +EMPTY_CONFIGURATIONS: Final = st.builds(EmptyConfiguration) +"""A fresh empty configuration for each draw, so a test that changes one changes no other.""" + + +class DefinitionStrategy(ABC): + """How to draw the configurations of one kind of definition.""" + + __slots__ = () + + kind: ClassVar[type[Definition[Any]]] + """The kind of definition whose configurations it draws.""" + + @abstractmethod + def unconstrained(self) -> Configurations: + """Configurations when nothing is known of what they depend on: what the definition's rules accept.""" + + +@dataclass(frozen=True, kw_only=True, slots=True) +class DataTypeStrategy(DefinitionStrategy): + """How to draw a data type's configurations, and the fill values an array of one takes.""" + + kind: ClassVar[type[Definition[Any]]] = DataTypeDefinition + + configurations: Configurations + """Configurations its definition accepts.""" + fill_values: Callable[[Mapping[str, JSONValue]], SearchStrategy[JSONValue]] + """Fill values of the data type a configuration describes.""" + + def unconstrained(self) -> Configurations: + return self.configurations + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ChunkGridStrategy(DefinitionStrategy): + """How to draw a chunk grid's configurations over an array of a shape.""" + + kind: ClassVar[type[Definition[Any]]] = ChunkGridDefinition + + configurations: Callable[[tuple[int, ...] | None], Configurations] + """Configurations of grids that fit an array of the shape. + + Handed None, no shape is known, and what is drawn is what the grid's + rules accept, whatever shape it is put over. + """ + + def unconstrained(self) -> Configurations: + return self.configurations(None) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ChunkKeyEncodingStrategy(DefinitionStrategy): + """How to draw a chunk key encoding's configurations.""" + + kind: ClassVar[type[Definition[Any]]] = ChunkKeyEncodingDefinition + + configurations: Configurations + """Configurations its definition accepts.""" + + def unconstrained(self) -> Configurations: + return self.configurations + + +@dataclass(frozen=True, kw_only=True, slots=True) +class CodecStrategy(DefinitionStrategy): + """How to draw a codec's configurations, handed a chunk. + + Every codec takes `Chunk()`, a chunk nothing is known of, so a + strategy that draws nothing for it is refused when it is built. + """ + + kind: ClassVar[type[Definition[Any]]] = CodecDefinition + + configurations: Callable[[Chunk], Configurations | None] + """Configurations of the codec handed the chunk; None when no configuration takes it. + + A `bytes` codec handed a `string` chunk is None, since the spec gives + that data type a codec of its own. + """ + + def __post_init__(self) -> None: + if self.configurations(NOTHING_KNOWN) is None: + msg = "a codec strategy draws configurations for Chunk(), a chunk nothing is known of; got None" + raise ValueError(msg) + + def unconstrained(self) -> Configurations: + return cast("Configurations", self.configurations(NOTHING_KNOWN)) + + +Strategies: TypeAlias = Mapping[Any, DefinitionStrategy] +"""How to draw each definition's configurations, by the definition. + +The key is `Any`, not `Definition[Any]`: a mapping's key type is +invariant, so `{MY_CODEC: CodecStrategy(...)}`, a mapping keyed by a +`CodecDefinition`, would not be a `Mapping[Definition[Any], ...]`. +""" + +__all__ = [ + "NOTHING_KNOWN", + "ChunkGridStrategy", + "ChunkKeyEncodingStrategy", + "CodecStrategy", + "Configurations", + "DataTypeStrategy", + "DefinitionStrategy", + "Strategies", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py new file mode 100644 index 0000000000..fd13c43839 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py @@ -0,0 +1,299 @@ +"""Hypothesis strategies for valid Zarr v3 metadata fields. + +Every strategy here draws metadata its definition accepts, and that the +spec allows: a reader, writer or converter can test itself against the +whole space of a codec's, data type's, chunk grid's or chunk key +encoding's configurations instead of the handful its authors thought to +write down. + +```python +from hypothesis import given +from zarr_metadata.testing.strategies import fields +from zarr_metadata.v3.codec.blosc import BLOSC_CODEC + +@given(fields(BLOSC_CODEC)) +def test_my_reader_reads_every_blosc_codec(field): + my_reader.read_codec(field) +``` + +What a configuration depends on, each kind's function takes: +`fill_values` a data type's configuration, `chunk_grid_configurations` +the shape of the array, `codec_configurations` the `Chunk` the codec is +handed. Left out, nothing is known of it, and what is drawn is what the +definition's rules accept. `fields` wraps configurations in the +envelopes a document writes them in. + +How each definition's configurations are drawn is a value, +`Strategies`, keyed by the definition: `DEFAULT_STRATEGIES` holds one +for each definition this package defines that holds no other field. An +extension author adds their own with +`DEFAULT_STRATEGIES | {MY_CODEC: CodecStrategy(...)}`. + +Configurations are drawn at the data level the TypedDicts declare, JSON +arrays as tuples; `json.dumps` takes them as they are. + +Requires the `testing` extra (`pip install zarr-metadata[testing]`). +""" + +from __future__ import annotations + +from functools import partial +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast + +from hypothesis import strategies as st + +from zarr_metadata._sentinel import UNSET +from zarr_metadata.definition._definition import Definition, as_kind, canonicalize, kind_of +from zarr_metadata.testing._chunk_grid import CHUNK_GRID_STRATEGIES +from zarr_metadata.testing._chunk_key_encoding import CHUNK_KEY_ENCODING_STRATEGIES +from zarr_metadata.testing._codec import CODEC_STRATEGIES +from zarr_metadata.testing._data_type import DATA_TYPE_STRATEGIES +from zarr_metadata.testing._strategy import ( + NOTHING_KNOWN, + ChunkGridStrategy, + ChunkKeyEncodingStrategy, + CodecStrategy, + Configurations, + DataTypeStrategy, + DefinitionStrategy, + Strategies, +) +from zarr_metadata.v3._context import ZarrV3Context +from zarr_metadata.v3._core import CORE +from zarr_metadata.v3._definition import Chunk, DataTypeDefinition + +if TYPE_CHECKING: + from collections.abc import Mapping + + from hypothesis.strategies import SearchStrategy + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3 import ZarrV3MetadataFieldJSON + from zarr_metadata.v3._context import ZarrV3ContextSlots + from zarr_metadata.v3._definition import ( + ChunkGridDefinition, + ChunkKeyEncodingDefinition, + CodecDefinition, + ) + +C = TypeVar("C") + +Spellings = Literal["all", "canonical"] +"""How fields are written: every spelling the spec allows, or only the simplest. + +`"all"` draws each envelope the spec gives a field -- a bare name, an +object without a configuration, an empty configuration, a +`must_understand` of `true` -- and configurations that say what a +simpler one says, as a `blosc` `typesize` under `noshuffle` does. +`"canonical"` draws each in the spelling `canonicalize` gives it, which +is what a writer that writes the fewest words writes. +""" + +DEFAULT_STRATEGIES: Final[MappingProxyType[Definition[Any], DefinitionStrategy]] = MappingProxyType( + { + **DATA_TYPE_STRATEGIES, + **CHUNK_GRID_STRATEGIES, + **CHUNK_KEY_ENCODING_STRATEGIES, + **CODEC_STRATEGIES, + } +) +"""A strategy for each definition this package defines that holds no other field.""" + + +def _strategy_for(definition: Definition[Any], strategies: Strategies) -> DefinitionStrategy: + """The strategy `strategies` holds for `definition`, which draws configurations of its kind.""" + found = strategies.get(definition) + if found is None: + msg = f"no strategy for {definition.name!r}; pass one in `strategies`" + raise LookupError(msg) + if not isinstance(definition, found.kind): + msg = ( + f"the strategy for {definition.name!r}, a {type(definition).__name__}, draws " + f"configurations of a {found.kind.__name__}: {found!r}" + ) + raise TypeError(msg) + return found + + +def data_type_configurations( + definition: DataTypeDefinition[C], *, strategies: Strategies = DEFAULT_STRATEGIES +) -> SearchStrategy[C]: + """Configurations of the data type `definition` defines.""" + found = cast("DataTypeStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations) + + +def fill_values( + definition: DataTypeDefinition[C], + configuration: C, + *, + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[JSONValue]: + """Fill values of the data type `definition` and `configuration` describe, in every spelling the spec gives them.""" + found = cast("DataTypeStrategy", _strategy_for(definition, strategies)) + return found.fill_values(cast("Mapping[str, JSONValue]", configuration)) + + +def chunk_grid_configurations( + definition: ChunkGridDefinition[C], + shape: tuple[int, ...] | None = None, + *, + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[C]: + """Configurations of the chunk grid `definition` defines, that fit an array of `shape`. + + With no shape, what is drawn is what the grid's rules accept. + """ + found = cast("ChunkGridStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations(shape)) + + +def chunk_key_encoding_configurations( + definition: ChunkKeyEncodingDefinition[C], *, strategies: Strategies = DEFAULT_STRATEGIES +) -> SearchStrategy[C]: + """Configurations of the chunk key encoding `definition` defines.""" + found = cast("ChunkKeyEncodingStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations) + + +def codec_configurations( + definition: CodecDefinition[C], + chunk: Chunk = NOTHING_KNOWN, + *, + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[C] | None: + """Configurations of the codec `definition` defines, handed `chunk`; None when none takes it. + + With no chunk, nothing is known of it, and what is drawn is what the + codec's rules accept. + """ + found = cast("CodecStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C] | None", found.configurations(chunk)) + + +def _written_bare(definition: Definition[Any]) -> bool: + """Whether the spec has the field written as its bare name: a data type it defines itself. + + "If the data type is defined in this specification, then the value + must be the data type identifier provided as a string" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L509-L512). + """ + if not isinstance(definition, DataTypeDefinition): + return False + return CORE.claimant(DataTypeDefinition, definition.name) is definition + + +@st.composite +def _any_spelling( + draw: st.DrawFn, definition: Definition[Any], configurations: SearchStrategy[Any] +) -> ZarrV3MetadataFieldJSON: + """A configuration of `definition` in one of the envelopes the spec allows it. + + A configuration with nothing in it is written as the bare name, an + object without one, or an object with an empty one; any object may + say `must_understand` is `true`, which is what absence means. A name + that carries the configuration, as `r16` does, and a data type the + spec defines itself, are written as the bare name. + """ + configuration = cast("Mapping[str, JSONValue]", draw(configurations)) + carried = definition._carrying_name(configuration) # pyright: ignore[reportPrivateUsage] + if carried is not None: + return carried + if _written_bare(definition): + return definition.name + empty = len(configuration) == 0 + if empty and draw(st.booleans()): + return definition.name + field: dict[str, JSONValue] = {"name": definition.name} + if not empty or draw(st.booleans()): + field["configuration"] = configuration + if draw(st.booleans()): + field["must_understand"] = True + return cast("ZarrV3MetadataFieldJSON", field) + + +def _scope_of(definition: Definition[Any]) -> ZarrV3Context: + """The scope that claims `definition` and nothing else; a `TypeError` for a definition no slot of a Zarr v3 scope holds.""" + for slot, kind in ZarrV3Context.kinds.items(): + if isinstance(definition, kind): + return ZarrV3Context(**cast("ZarrV3ContextSlots", {slot: (definition,)})) + msg = ( + f"{definition.name!r} is a {type(definition).__name__}, " + "which no slot of a Zarr v3 scope holds" + ) + raise TypeError(msg) + + +def _checked( + field: ZarrV3MetadataFieldJSON, + *, + definition: Definition[Any], + kind: type[Definition[Any]], + scope: ZarrV3Context, + spellings: Spellings, +) -> ZarrV3MetadataFieldJSON: + """`field`, or its canonical spelling, once it reads without a problem in `scope`; a `ValueError` when it does not.""" + canonical, problems = canonicalize(field, kind, scope) + if len(problems) != 0: + msg = f"{field!r}, drawn for {definition.name!r}, which is refused: {problems!r}" + raise ValueError(msg) + if spellings == "canonical": + return cast("ZarrV3MetadataFieldJSON", canonical) + return field + + +def fields( + definition: Definition[C], + configurations: SearchStrategy[C] | UNSET = UNSET, + *, + spellings: Spellings = "all", + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[ZarrV3MetadataFieldJSON]: + """Metadata fields of `definition`: its name and a configuration, in an envelope a document writes. + + `configurations` is what to draw the configuration from, by default + the definition's own with nothing known of what they depend on: pass + `codec_configurations(definition, chunk)` for a codec handed a chunk, + once it is known not to be None. `spellings` says whether every + spelling the spec allows is drawn, or only the simplest. + + Each field is read in a scope of its definition alone, with no chunk + known, and one its definition's rules refuse, as they refuse one + drawn from `configurations` they do not accept, raises a + `ValueError`. A codec's chunk rules are not checked: what fits the + chunk is what `codec_configurations` draws. + """ + drawn: Configurations | SearchStrategy[C] = ( + _strategy_for(definition, strategies).unconstrained() + if configurations is UNSET + else configurations + ) + checked = partial( + _checked, + definition=definition, + kind=as_kind(kind_of(definition)), + scope=_scope_of(definition), + spellings=spellings, + ) + return _any_spelling(definition, drawn).map(checked) + + +__all__ = [ + "DEFAULT_STRATEGIES", + "NOTHING_KNOWN", + "ChunkGridStrategy", + "ChunkKeyEncodingStrategy", + "CodecStrategy", + "Configurations", + "DataTypeStrategy", + "DefinitionStrategy", + "Spellings", + "Strategies", + "chunk_grid_configurations", + "chunk_key_encoding_configurations", + "codec_configurations", + "data_type_configurations", + "fields", + "fill_values", +] diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index e0095aec31..0baad413c2 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -277,6 +277,9 @@ def test_all_is_grouped_and_unique() -> None: # example, `ZarrV3CodecField`, read in the same scope as the field that # contains it. "Field", + # How to draw one kind of definition's configurations, in + # `zarr_metadata.testing.strategies`: `CodecStrategy`. + "Strategy", ) _EXTENSION_NAME = re.compile(r"^(?:[A-Z][a-z0-9]*)+?(?:" + "|".join(_EXTENSION_ROLES) + r")$") @@ -294,6 +297,9 @@ def test_all_is_grouped_and_unique() -> None: # two definitions of one name. "ClaimKey", "Conflict", + # What one definition's configurations are drawn from, in + # `zarr_metadata.testing.strategies`. + "Configurations", "ScopeConflictError", # A reader's problems, as the signatures of every reader name them. "Problems", @@ -339,6 +345,8 @@ def test_all_is_grouped_and_unique() -> None: "ScalarMap", "ScalarMapEntry", "ShardingIndexLocation", + "Spellings", + "Strategies", "Struct", "StructField", "ValidationProblem", diff --git a/packages/zarr-metadata/tests/testing/__init__.py b/packages/zarr-metadata/tests/testing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py new file mode 100644 index 0000000000..c0d8aa0e3d --- /dev/null +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -0,0 +1,432 @@ +"""The strategies against the definitions they draw for, and against what they ought to reach. + +Every draw is read in `CORE_AND_EXTENSIONS` and must read without a +problem, by the definition it was drawn for: the field alone, a fill +value beside its data type, a chunk grid over the shape it was drawn +for, a codec handed the chunk it was drawn for. A strategy that only +drew valid metadata could draw almost none, so reach is checked too, +with targets worked out from each configuration's TypedDict rather than +from the strategies: every optional key present and absent, every +`Literal` value, and for a definition with a simplest spelling, a +configuration it simplifies. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, Literal, cast, get_args, get_origin + +import pytest +from hypothesis import event, find, given, settings +from hypothesis import strategies as st + +from zarr_metadata._typed_json import typeddict_keys +from zarr_metadata.definition import ( + AcceptedField, + Definition, + EmptyConfiguration, + fill_value_problems, + resolve, +) +from zarr_metadata.definition._definition import as_kind, canonicalize, kind_of, unchanged +from zarr_metadata.testing.strategies import ( + DEFAULT_STRATEGIES, + ChunkKeyEncodingStrategy, + CodecStrategy, + Spellings, + chunk_grid_configurations, + codec_configurations, + data_type_configurations, + fields, + fill_values, +) +from zarr_metadata.v2.definition import CORE_V2 +from zarr_metadata.v3._definition import chunk_grid_lengths +from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID +from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC +from zarr_metadata.v3.codec.gzip import GZIP_CODEC +from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC +from zarr_metadata.v3.codec.sharding_indexed import SHARDING_INDEXED_CODEC +from zarr_metadata.v3.data_type._float import FloatSpecialFillValue +from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE +from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE +from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE +from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE +from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE +from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE +from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE +from zarr_metadata.v3.definition import ( + CORE, + CORE_AND_EXTENSIONS, + Chunk, + ChunkGridDefinition, + CodecDefinition, + DataTypeDefinition, + ZarrV3Context, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3 import ZarrV3MetadataFieldJSON + +_HOLDING_FIELDS = frozenset({SHARDING_INDEXED_CODEC, CAST_VALUE_CODEC, STRUCT_DATA_TYPE}) +"""The definitions whose configurations hold other fields, which have no strategy yet.""" + +_IN_SCOPE = frozenset( + ( + *CORE_AND_EXTENSIONS.data_types.values(), + *CORE_AND_EXTENSIONS.chunk_grids.values(), + *CORE_AND_EXTENSIONS.chunk_key_encodings.values(), + *CORE_AND_EXTENSIONS.codecs.values(), + *CORE_AND_EXTENSIONS.storage_transformers.values(), + ) +) +"""The definitions at the spec's extension points that `CORE_AND_EXTENSIONS` holds; members of a document it defines are not fields.""" +_CORE_DATA_TYPES = frozenset(CORE.data_types.values()) + +_DEFINITIONS = sorted(DEFAULT_STRATEGIES, key=lambda definition: definition.name) +_DATA_TYPES = [d for d in _DEFINITIONS if isinstance(d, DataTypeDefinition)] +_CHUNK_GRIDS = [d for d in _DEFINITIONS if isinstance(d, ChunkGridDefinition)] +_CODECS = [d for d in _DEFINITIONS if isinstance(d, CodecDefinition)] + +_SHAPES = st.lists(st.integers(0, 12), max_size=4).map(tuple) + + +def _id(value: object) -> str: + if isinstance(value, Definition): + return f"{type(value).__name__}:{value.name}" + return str(value) + + +def _read(definition: Definition[Any], field: object) -> Any: # noqa: ANN401 + """`field` read in `CORE_AND_EXTENSIONS`, asserted to read without a problem by `definition`.""" + resolved, problems = resolve(field, as_kind(kind_of(definition)), CORE_AND_EXTENSIONS) + assert problems == () + assert isinstance(resolved, AcceptedField) + assert resolved.definition is definition + return resolved + + +def test_every_definition_has_a_strategy() -> None: + """Each definition in scope has a strategy, but those that hold other fields; and none is out of scope.""" + assert _IN_SCOPE - frozenset(DEFAULT_STRATEGIES) == _HOLDING_FIELDS + assert frozenset(DEFAULT_STRATEGIES) <= _IN_SCOPE + + +@pytest.mark.parametrize("spellings", get_args(Spellings)) +@pytest.mark.parametrize("definition", _DEFINITIONS, ids=_id) +@given(data=st.data()) +def test_fields_read( + definition: Definition[Any], spellings: Spellings, data: st.DataObject +) -> None: + """A drawn field reads by its definition, through JSON too; a canonical one is its own simplest spelling.""" + field = data.draw(fields(definition, spellings=spellings)) + _read(definition, field) + _read(definition, json.loads(json.dumps(field))) + if definition in _CORE_DATA_TYPES: + assert isinstance(field, str) + if spellings == "canonical": + canonical, _ = canonicalize(field, as_kind(kind_of(definition)), CORE_AND_EXTENSIONS) + assert canonical == field + + +@pytest.mark.parametrize("definition", _DEFINITIONS, ids=_id) +@given(data=st.data()) +def test_a_changed_draw_changes_no_other(definition: Definition[Any], data: st.DataObject) -> None: + """Each draw is a fresh value: a test that changes the configuration it drew does not change the next draw, which Hypothesis would report as flaky.""" + field = data.draw(fields(definition)) + if isinstance(field, dict) and "configuration" in field: + configuration = cast("dict[str, JSONValue]", field["configuration"]) + assert "changed" not in configuration + configuration["changed"] = True + + +@pytest.mark.parametrize("definition", _DATA_TYPES, ids=_id) +@given(data=st.data()) +def test_fill_values_fit_their_data_type( + definition: DataTypeDefinition[Any], data: st.DataObject +) -> None: + configuration = data.draw(data_type_configurations(definition)) + data_type = _read(definition, data.draw(fields(definition, st.just(configuration)))) + value = data.draw(fill_values(definition, configuration)) + assert fill_value_problems(data_type, value) == () + assert fill_value_problems(data_type, json.loads(json.dumps(value))) == () + + +@pytest.mark.parametrize("definition", _CHUNK_GRIDS, ids=_id) +@given(data=st.data(), shape=_SHAPES) +def test_chunk_grids_fit_their_shape( + definition: ChunkGridDefinition[Any], data: st.DataObject, shape: tuple[int, ...] +) -> None: + configurations = chunk_grid_configurations(definition, shape) + chunk_grid = _read(definition, data.draw(fields(definition, configurations))) + lengths, problems = chunk_grid_lengths(chunk_grid, shape) + assert problems == () + # "Chunk sizes must be greater than zero", which the regular grid's rules do not yet refuse. + assert all(0 not in axis for axis in lengths if axis is not None) + + +@st.composite +def _chunks(draw: st.DrawFn) -> Chunk: + """A chunk of a data type drawn from the strategies, or of none known, with lengths known along some axes, all, or none.""" + definition = draw(st.none() | st.sampled_from(_DATA_TYPES)) + data_type = None if definition is None else _read(definition, draw(fields(definition))) + axis = st.none() | st.integers(1, 8).map(lambda length: frozenset({length})) + lengths = draw(st.none() | st.lists(axis, max_size=4).map(tuple)) + return Chunk(lengths, data_type) + + +@pytest.mark.parametrize("definition", _CODECS, ids=_id) +@given(data=st.data(), chunk=_chunks()) +def test_codecs_take_their_chunk( + definition: CodecDefinition[Any], data: st.DataObject, chunk: Chunk +) -> None: + """A codec drawn for a chunk takes it; one drawn for none is one the spec does not give it, which needs the data type known.""" + configurations = codec_configurations(definition, chunk) + if configurations is None: + assert chunk.data_type is not None + assert chunk.data_type.definition is not None + data_type_name = chunk.data_type.definition.name + event(f"takes no {data_type_name}") + codec = _read(definition, data.draw(fields(definition))) + assert list(definition.chunk_rules(codec.configuration, codec.nested, chunk)) != [] + return + codec = _read(definition, data.draw(fields(definition, configurations))) + assert list(definition.chunk_rules(codec.configuration, codec.nested, chunk)) == [] + + +Target = tuple[Literal["present", "absent"], str] | tuple[Literal["value"], str, object] +FieldForm = Literal["bare", "object", "configured", "must_understand"] + + +def _configuration_targets( + definition: Definition[Any], +) -> list[Target | tuple[Literal["simplified"]]]: + """What a strategy for `definition` ought to reach, by its configuration's TypedDict.""" + targets: list[Target | tuple[Literal["simplified"]]] = [] + for key, (annotation, required) in typeddict_keys(definition.configuration).members.items(): + if not required: + targets.extend([("present", key), ("absent", key)]) + if get_origin(annotation) is Literal: + targets.extend(("value", key, value) for value in get_args(annotation)) + if definition.canonical is not unchanged: + targets.append(("simplified",)) + return targets + + +def _field_forms(definition: Definition[Any]) -> list[FieldForm]: + """The envelopes the spec allows a field of `definition` to be written in.""" + if definition in _CORE_DATA_TYPES: + return [] # only ever bare, which `test_fields_read` asserts + if definition.requires_configuration: + return ["configured", "must_understand"] + return ["bare", "object", "configured", "must_understand"] + + +def _form_of(field: ZarrV3MetadataFieldJSON) -> set[FieldForm]: + if isinstance(field, str): + return {"bare"} + forms: set[FieldForm] = {"configured" if "configuration" in field else "object"} + if "must_understand" in field: + forms.add("must_understand") + return forms + + +def _hits( + definition: Definition[Any], + target: Target | tuple[Literal["simplified"]], + configuration: Mapping[str, JSONValue], +) -> bool: + match target: + case ("present", key): + return key in configuration + case ("absent", key): + return key not in configuration + case ("value", key, value): + return configuration.get(key) == value + case ("simplified",): + return definition.canonical(configuration) != configuration + + +_REACH = settings(max_examples=2000, database=None) + + +@pytest.mark.parametrize( + ("definition", "target"), + [(d, target) for d in _DEFINITIONS for target in _configuration_targets(d)], + ids=_id, +) +def test_configurations_reach( + definition: Definition[Any], target: Target | tuple[Literal["simplified"]] +) -> None: + configurations = fields(definition).map(lambda field: _read(definition, field).configuration) + find( + configurations, + lambda configuration: _hits(definition, target, configuration), + settings=_REACH, + ) + + +@pytest.mark.parametrize( + ("definition", "form"), + [(d, form) for d in _DEFINITIONS for form in _field_forms(d)], + ids=_id, +) +def test_fields_reach(definition: Definition[Any], form: FieldForm) -> None: + find(fields(definition), lambda field: form in _form_of(field), settings=_REACH) + + +def test_an_added_strategy_is_drawn_from() -> None: + """A strategy joined to the defaults, as an extension author adds one, is the one drawn from.""" + level_three = CodecStrategy(configurations=lambda _chunk: st.just({"level": 3})) + strategies = DEFAULT_STRATEGIES | {GZIP_CODEC: level_three} + field = find(fields(GZIP_CODEC, strategies=strategies), lambda _field: True, settings=_REACH) + assert _read(GZIP_CODEC, field).configuration == {"level": 3} + + +def test_a_definition_without_a_strategy_is_a_lookup_error() -> None: + with pytest.raises(LookupError, match="no strategy for 'sharding_indexed'"): + fields(SHARDING_INDEXED_CODEC) + + +def test_a_strategy_of_another_kind_is_a_type_error() -> None: + # Unannotated, as an extension author writes it: a mapping keyed by a + # `CodecDefinition` is a `Strategies`. + strategies = {GZIP_CODEC: ChunkKeyEncodingStrategy(configurations=st.just({}))} + with pytest.raises(TypeError, match="draws configurations of a ChunkKeyEncodingDefinition"): + codec_configurations(GZIP_CODEC, strategies=strategies) + + +def test_a_codec_strategy_refusing_an_unknown_chunk_is_a_value_error() -> None: + with pytest.raises(ValueError, match=r"Chunk\(\), a chunk nothing is known of"): + CodecStrategy(configurations=lambda _chunk: None) + + +@pytest.mark.parametrize("spellings", get_args(Spellings)) +def test_a_refused_draw_is_a_value_error(spellings: Spellings) -> None: + refused = fields(GZIP_CODEC, st.just({"level": 99}), spellings=spellings) + with pytest.raises(ValueError, match="which is refused"): + find(refused, lambda _field: True, settings=settings(database=None)) + + +_BFLOAT16 = DataTypeDefinition(name="bfloat16", configuration=EmptyConfiguration) +"""A number type this package does not define, as an extension author would.""" + + +@given(data=st.data()) +def test_scale_offset_takes_a_data_type_the_package_does_not_define(data: st.DataObject) -> None: + """Values of a type the package does not define cannot be drawn, so the configuration is the identity, which its rules take.""" + data_type, problems = resolve( + "bfloat16", DataTypeDefinition, ZarrV3Context(data_types=[_BFLOAT16]) + ) + assert problems == () + chunk = Chunk(None, data_type) + configurations = codec_configurations(SCALE_OFFSET_CODEC, chunk) + assert configurations is not None + configuration = data.draw(configurations) + assert configuration == {} + assert list(SCALE_OFFSET_CODEC.chunk_rules(configuration, {}, chunk)) == [] + + +def test_a_definition_no_v3_scope_holds_is_a_type_error() -> None: + zlib = CORE_V2.codecs["zlib"] + with pytest.raises(TypeError, match="which no slot of a Zarr v3 scope holds"): + fields(zlib, st.just({})) + + +@pytest.mark.parametrize( + "definition", [FLOAT16_DATA_TYPE, FLOAT32_DATA_TYPE, FLOAT64_DATA_TYPE], ids=_id +) +@pytest.mark.parametrize("member", ["offset", "scale"]) +def test_scale_offset_reaches_a_value_written_in_hex( + definition: DataTypeDefinition[Any], member: str +) -> None: + """A float's `offset` and `scale` are fill values of its type, which the bits in hex spell too.""" + chunk = Chunk(None, _read(definition, definition.name)) + configurations = codec_configurations(SCALE_OFFSET_CODEC, chunk) + assert configurations is not None + find( + configurations, + lambda configuration: isinstance(configuration.get(member), str), + settings=_REACH, + ) + + +def _has_pair(configuration: Mapping[str, Any]) -> bool: + return any( + isinstance(entry, tuple) + for dimension in configuration["chunk_shapes"] + if isinstance(dimension, tuple) + for entry in cast("tuple[object, ...]", dimension) + ) + + +def test_rectilinear_edges_reach_a_run_written_as_a_pair() -> None: + """The `[size, count]` spelling, which no TypedDict target asks for: it is one arm of a union.""" + find(chunk_grid_configurations(RECTILINEAR_CHUNK_GRID), _has_pair, settings=_REACH) + + +FillValueSpelling = Literal[ + "number", "named", "hex lowercase", "hex uppercase", "array", "base64", "NaT" +] + +_FILL_VALUE_SPELLINGS: list[tuple[DataTypeDefinition[Any], FillValueSpelling]] = [ + *( + (definition, spelling) + for definition in ( + FLOAT16_DATA_TYPE, + FLOAT32_DATA_TYPE, + FLOAT64_DATA_TYPE, + COMPLEX64_DATA_TYPE, + COMPLEX128_DATA_TYPE, + ) + for spelling in get_args(FillValueSpelling)[:4] + ), + (BYTES_DATA_TYPE, "array"), + (BYTES_DATA_TYPE, "base64"), + (NUMPY_DATETIME64_DATA_TYPE, "number"), + (NUMPY_DATETIME64_DATA_TYPE, "NaT"), + (NUMPY_TIMEDELTA64_DATA_TYPE, "number"), + (NUMPY_TIMEDELTA64_DATA_TYPE, "NaT"), +] +"""The spellings the specs give a fill value, by the data types whose fill values take them.""" + + +def _spellings_of(value: object) -> set[FillValueSpelling]: + """How a fill value is spelled; a complex one, how either component is.""" + if isinstance(value, bool): + return set() + if isinstance(value, int | float): + return {"number"} + if isinstance(value, tuple): + parts = cast("tuple[object, ...]", value) + if all(isinstance(part, int) for part in parts): + return {"array"} + return {spelling for part in parts for spelling in _spellings_of(part)} + if not isinstance(value, str): + return set() + if value == "NaT": + return {"NaT"} + if value in get_args(FloatSpecialFillValue): + return {"named"} + if value.startswith("0x"): + digits = value[2:] + return { + *(("hex lowercase",) if any(c in "abcdef" for c in digits) else ()), + *(("hex uppercase",) if any(c in "ABCDEF" for c in digits) else ()), + } + return {"base64"} + + +@pytest.mark.parametrize(("definition", "spelling"), _FILL_VALUE_SPELLINGS, ids=_id) +def test_fill_values_reach( + definition: DataTypeDefinition[Any], spelling: FillValueSpelling +) -> None: + values = data_type_configurations(definition).flatmap( + lambda configuration: fill_values(definition, configuration) + ) + find(values, lambda value: spelling in _spellings_of(value), settings=_REACH) diff --git a/packages/zarr-metadata/uv.lock b/packages/zarr-metadata/uv.lock index 9c21f5a4c9..1679e33309 100644 --- a/packages/zarr-metadata/uv.lock +++ b/packages/zarr-metadata/uv.lock @@ -1233,6 +1233,11 @@ dependencies = [ { name = "typing-extensions" }, ] +[package.optional-dependencies] +testing = [ + { name = "hypothesis" }, +] + [package.dev-dependencies] docs = [ { name = "griffe-inherited-docstrings" }, @@ -1254,8 +1259,10 @@ test = [ [package.metadata] requires-dist = [ { name = "annotated-types", specifier = ">=0.6" }, + { name = "hypothesis", marker = "extra == 'testing'", specifier = ">=6.85" }, { name = "typing-extensions", specifier = ">=4.16" }, ] +provides-extras = ["testing"] [package.metadata.requires-dev] docs = [ From be4d9d4316fb81b259efede10dcf2967433fc317 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Sun, 1 Nov 2026 09:42:49 +0100 Subject: [PATCH 03/21] chore(deps): bump `cast-value.rs` (#4584) * chore(deps): bump cast-value.rs to >=0.5.2 * chore(deps): raise the runtime cast-value-rs floor to 0.5.1 The previous commit raised the packaging floor to 0.5.1 but left the runtime check at 0.4.2. cast-value-rs 0.5.0 makes casts exact at integer and float range limits; 0.5.0 was never published to PyPI, so the floor is 0.5.1. Also update the error message, the version-check test cases, and uv.lock. Assisted-by: ClaudeCode:claude-opus-5-5 * test(cast_value): derive version-check cases from the minimum The cases hard-coded versions near the floor, so every floor bump meant rewriting them. Build them from CAST_VALUE_RS_MIN_VERSION instead, and drop cases that repeated the same comparison. A floor bump now touches only pyproject.toml and the constant. Assisted-by: ClaudeCode:claude-opus-5-5 --- pyproject.toml | 2 +- src/zarr/codecs/cast_value.py | 14 +-- tests/test_codecs/test_cast_value.py | 17 ++-- uv.lock | 142 ++++++++++++++++----------- 4 files changed, 102 insertions(+), 73 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 136f757dac..c95ecb6ee8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,7 +84,7 @@ remote = [ gpu = [ "cupy-cuda12x; sys_platform != 'darwin'", ] -cast-value-rs = ["cast-value-rs>=0.4.2"] +cast-value-rs = ["cast-value-rs>=0.5.1"] cli = ["typer"] optional = ["universal-pathlib"] diff --git a/src/zarr/codecs/cast_value.py b/src/zarr/codecs/cast_value.py index 629b4e7a27..3d465070b8 100644 --- a/src/zarr/codecs/cast_value.py +++ b/src/zarr/codecs/cast_value.py @@ -5,7 +5,7 @@ rounding, out-of-range handling, and explicit scalar mappings. Requires the optional ``cast-value-rs`` package for the actual casting -logic. Install it with: ``pip install 'cast-value-rs>=0.4.2'``. +logic. Install it with: ``pip install 'cast-value-rs>=0.5.1'``. """ from __future__ import annotations @@ -125,10 +125,12 @@ def parse_scalar_map(obj: ScalarMapJSON | ScalarMap) -> ScalarMap: # Backend: cast-value-rs # --------------------------------------------------------------------------- -# Versions below this silently transpose input arrays that are not row-major - +# Versions below 0.4.2 silently transpose input arrays that are not row-major - # the layout `transpose` hands to the next codec - writing corrupted data with -# no error. Keep in sync with the `cast-value-rs` extra in pyproject.toml. -CAST_VALUE_RS_MIN_VERSION: Final = "0.4.2" +# no error. Versions below 0.5.0 give wrong results for some casts at integer +# and float range limits. 0.5.0 was never published to PyPI, so the floor is +# 0.5.1. Keep in sync with the `cast-value-rs` extra in pyproject.toml. +CAST_VALUE_RS_MIN_VERSION: Final = "0.5.1" _INSTALL_HINT: Final = f"Install it with: pip install 'cast-value-rs>={CAST_VALUE_RS_MIN_VERSION}'" @@ -144,8 +146,8 @@ def _check_backend_version() -> str | None: if Version(installed) < Version(CAST_VALUE_RS_MIN_VERSION): return ( f"The cast_value codec requires cast-value-rs >= {CAST_VALUE_RS_MIN_VERSION}, " - f"but version {installed} is installed. Earlier versions silently corrupt data " - f"when the input array is not row-major. {_INSTALL_HINT}" + f"but version {installed} is installed. Earlier versions can write wrong " + f"values without raising an error. {_INSTALL_HINT}" ) return None diff --git a/tests/test_codecs/test_cast_value.py b/tests/test_codecs/test_cast_value.py index c2e78770d9..d6c703767f 100644 --- a/tests/test_codecs/test_cast_value.py +++ b/tests/test_codecs/test_cast_value.py @@ -9,7 +9,7 @@ import zarr from tests.conftest import Expect, ExpectFail from zarr.codecs import BytesCodec, TransposeCodec -from zarr.codecs.cast_value import CastValue +from zarr.codecs.cast_value import CAST_VALUE_RS_MIN_VERSION, CastValue from zarr.storage import MemoryStore try: @@ -490,15 +490,14 @@ def test_parse_scalar_map(case: Expect[Any, Any]) -> None: @pytest.mark.parametrize( "case", [ - Expect(input="0.4.2", output=None, id="exactly-minimum"), - Expect(input="0.4.3", output=None, id="newer-patch"), - Expect(input="0.5.0", output=None, id="newer-minor"), - Expect(input="1.0.0", output=None, id="newer-major"), - Expect(input="0.4.2.post1", output=None, id="post-release"), + Expect(input=CAST_VALUE_RS_MIN_VERSION, output=None, id="exactly-minimum"), + Expect(input=f"{CAST_VALUE_RS_MIN_VERSION}.post1", output=None, id="post-release"), + Expect( + input=f"{CAST_VALUE_RS_MIN_VERSION}.dev1", + output=f"{CAST_VALUE_RS_MIN_VERSION}.dev1", + id="pre-release-of-minimum", + ), Expect(input="0.4.0", output="0.4.0", id="known-corrupting"), - Expect(input="0.4.1", output="0.4.1", id="one-patch-below"), - Expect(input="0.3.0", output="0.3.0", id="older-minor"), - Expect(input="0.4.2.dev1", output="0.4.2.dev1", id="pre-release-of-minimum"), ], ids=lambda c: c.id, ) diff --git a/uv.lock b/uv.lock index e78182c154..05d88533ec 100644 --- a/uv.lock +++ b/uv.lock @@ -441,63 +441,91 @@ wheels = [ [[package]] name = "cast-value-rs" -version = "0.4.2" +version = "0.5.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/56/a4/634d1917ac49ea69291471e87eee9c1aa45d71d6da9b2cd5e1bfe82729de/cast_value_rs-0.4.2.tar.gz", hash = "sha256:fd621b8dd4f7e93bbffdb119882d85e2731c2f0fb1d30de85f29e8e04c044822", size = 86862, upload-time = "2026-08-13T09:16:59.199Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/67/c7/9db59d415df2d1c9f905c4da951fe023c6259a162584ef68bd81c171d448/cast_value_rs-0.4.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6d8e1c6d7b0fec6b1060f83867d20ebd0cb1ac6fa2954cd82f418a31a96ac504", size = 494354, upload-time = "2026-08-13T09:14:41.627Z" }, - { url = "https://files.pythonhosted.org/packages/8f/53/10ece0b4ba4a0c156674c2db7e5d05ce9b04efe72e191417546e111d645c/cast_value_rs-0.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8c953321ff7954e0fe1b25c1baf96f0a9cae0961b8a56ce63dc7b94e2ea7d85c", size = 458463, upload-time = "2026-08-13T09:14:43.371Z" }, - { url = "https://files.pythonhosted.org/packages/91/0b/e50ac2eb271eb6c34402b4abe317c795bc7d650d332d0bf422ae868aa2cd/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68492eef745691c80754cd027e216dbd5f7a4ce2e07fddfded3ee59bd947afb5", size = 487628, upload-time = "2026-08-13T09:14:44.86Z" }, - { url = "https://files.pythonhosted.org/packages/a9/0f/69befc7905af2e73a2739b26444787cd528331c041f37235f3853bf15f75/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fbf145ed6f724fa7f5ddc9715f78e0d450ac141fc05299c83b2e7d627793ddaf", size = 555538, upload-time = "2026-08-13T09:14:46.305Z" }, - { url = "https://files.pythonhosted.org/packages/4a/1b/4105fabec3268139bda3555422808211ec29428e17fa56566c1af7957c04/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73fb23256dda16e4e9f49bfedebe0e068f00ece85091b526c102618f66144216", size = 650896, upload-time = "2026-08-13T09:14:48.049Z" }, - { url = "https://files.pythonhosted.org/packages/83/2a/452cf37675af02c3ac8bfa01fc962cb744a1392c51bccfb6d1faca423067/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8184135c78ebcabb904574a1b3bc5704c131140feae1b58d65c888fe08b46029", size = 561036, upload-time = "2026-08-13T09:14:49.431Z" }, - { url = "https://files.pythonhosted.org/packages/d4/87/edb215d17da304a6b36489fc51b8eb785c5b041c2fcc5f200f16adde3133/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:686ea86aad1ffebd0b6793549043108600593a85032457eb88a3a0513a4100f5", size = 549187, upload-time = "2026-08-13T09:14:50.801Z" }, - { url = "https://files.pythonhosted.org/packages/63/82/0f9963ef0bcb434cd1244cb2620123c2d71763d429d017ceb98fbb6faf48/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d6c425d2803e13408755a0022faa9efd0ab0becdf3573a1a8a7af64b1adee4ba", size = 591027, upload-time = "2026-08-13T09:14:52.281Z" }, - { url = "https://files.pythonhosted.org/packages/90/97/b489f8048491f2ece21bf99d5a1fea3bbfaf3cd0e9f555ac4a0352c11d81/cast_value_rs-0.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:92556f1bb9d6b222736d967228565ec745c640c29bc291d1cb730a1011d3dcac", size = 665201, upload-time = "2026-08-13T09:14:53.769Z" }, - { url = "https://files.pythonhosted.org/packages/ce/9f/fabffd9788f0b2d3f49ca4582b8859ab58b929b1b8837ec0be1263399279/cast_value_rs-0.4.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1b0881c23a8f5ff1dc6c70542179f2fc9191e0b7fa7a02f65316fb48904cd8d1", size = 831085, upload-time = "2026-08-13T09:14:55.244Z" }, - { url = "https://files.pythonhosted.org/packages/b5/66/ca653bf9b51eb624615623611db28ec9428a351259fcf5625c5520faf759/cast_value_rs-0.4.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8a67648bf47d9b740388efaf6f05185f22dea783e97eba895efdd1dbafe8e3cc", size = 788462, upload-time = "2026-08-13T09:14:56.666Z" }, - { url = "https://files.pythonhosted.org/packages/f1/54/3a9280f791c8765a54f7814a010d2f460c7d5f5483be3e5634f4d9b38d75/cast_value_rs-0.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:decf297bc576675fff2139885fb88f84844e27b99e5d28eaa01909db7a327c4b", size = 753055, upload-time = "2026-08-13T09:14:58.302Z" }, - { url = "https://files.pythonhosted.org/packages/06/26/09f4618dbab2de8ea427e817c349b8391b4d559316de899e9b54dde6143e/cast_value_rs-0.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:79a8c13af39bf65014dd6567dc5fa7c42dfda246e4335ab52814f3dd6fb42cac", size = 442068, upload-time = "2026-08-13T09:15:00.414Z" }, - { url = "https://files.pythonhosted.org/packages/fc/49/7def5933625ff3c98e7d20cbd7de0aa1e83098ffa425a9df087764657767/cast_value_rs-0.4.2-cp312-cp312-win_arm64.whl", hash = "sha256:b8af7de3639271ad9e62aba4d15ddd22819c706bd7c7859663fdbb022397bdad", size = 397846, upload-time = "2026-08-13T09:15:02.022Z" }, - { url = "https://files.pythonhosted.org/packages/6f/fc/1f48ae9ea1f10caf3bbc31b288d08137835d8965188f6ddcccdb3c7d398e/cast_value_rs-0.4.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:297445abc64891a2e62af93e4f35bb280f874ac548646d0a6b04a86d45e636a5", size = 494394, upload-time = "2026-08-13T09:15:03.604Z" }, - { url = "https://files.pythonhosted.org/packages/7f/f3/d633218979cd8839fcfbade64a277c0ff9d8ff59d21bbd8a59f1467085b4/cast_value_rs-0.4.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b8b6e0781b4ce61a559cc050d836a52157bd30b1f59d63fdeee79fbc19ab7e57", size = 458677, upload-time = "2026-08-13T09:15:04.947Z" }, - { url = "https://files.pythonhosted.org/packages/97/e4/96d72ccf5e00dd7af7bf55c96313df2ffba1f34a224f10430716232775c0/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8399dc89570c01537c07adc03fc2fc0e43a517be8a198e2f9e3f8a70dd003be0", size = 487289, upload-time = "2026-08-13T09:15:06.493Z" }, - { url = "https://files.pythonhosted.org/packages/93/30/18313de8c1d3826e9a6abd883b7a7981b19391f674dcd52ff2635d19f8ee/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:939417d72c9945ce24eb959e6b2cd32f7aa7d0a50a06a0fac484056b10b650dd", size = 555221, upload-time = "2026-08-13T09:15:08.28Z" }, - { url = "https://files.pythonhosted.org/packages/e9/1f/ed22b2e536ccbd4c5b234d869fc8f478347a45f7b940273023a0d6013134/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:101318e594ee998a5ccbc5ea7ed045cbd9fd22d45cac467c745fbd30309f1343", size = 650972, upload-time = "2026-08-13T09:15:10.103Z" }, - { url = "https://files.pythonhosted.org/packages/d7/d2/39f9ff9b678dfb01d206aec6be0f486137d5e9669ebadf0d1f356d2a7324/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d23bac26b2c2fe6a2368d10fbb072a71671f6e0d4d8cf9e893f62ed3530bbd1b", size = 560869, upload-time = "2026-08-13T09:15:11.602Z" }, - { url = "https://files.pythonhosted.org/packages/60/18/f63e8378db66b5a99e96c1422c68ba02a4e35de34355b90cf6d169d99ae6/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9d799a20d7517974d61d7c4dd54e5d7ef439e726012c087bada31bc644b1535", size = 548885, upload-time = "2026-08-13T09:15:13.18Z" }, - { url = "https://files.pythonhosted.org/packages/98/af/12349007b670b11e16d3be51c29ab2398550b2653dae692a5d788d3dda87/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b9f0f04815efcc47592474e91f61f24810139396696f0690f3d0bf0e6ff3bb4a", size = 590846, upload-time = "2026-08-13T09:15:14.67Z" }, - { url = "https://files.pythonhosted.org/packages/a3/cf/6a39df104923bc397ae58f90e21d1e87edd62d17f24fe3d6777e6c4d3e5f/cast_value_rs-0.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e69f81dab0b47c74c1f30c79198c412938b6c41510b8124dbd08fb36d13a8807", size = 665064, upload-time = "2026-08-13T09:15:16.527Z" }, - { url = "https://files.pythonhosted.org/packages/36/df/0a9ac5bae4e1d3b7b4a79795054d822c26bdbc3bedad71fe13e8ce666c4b/cast_value_rs-0.4.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:59e15ec495e8707447180e0111e65e083464da60236edd5c3387efedce3da997", size = 830909, upload-time = "2026-08-13T09:15:18.088Z" }, - { url = "https://files.pythonhosted.org/packages/2f/7d/9e99c969dd76087dbf731667982d8bb6b390d29c8f37c09dd330657b0487/cast_value_rs-0.4.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f9e787397588fabb28a146a28159150f44e8ef85a25abe364f989044f1b7159d", size = 788217, upload-time = "2026-08-13T09:15:19.706Z" }, - { url = "https://files.pythonhosted.org/packages/21/a9/5d7410d37be26bed1dc740c885a0c7fdb213b93e9b13ac76fb3282aa50ea/cast_value_rs-0.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c66ca6ad21758993830a9f4d24263065727bf767e4524d79ded0c0b61e3da9d", size = 752817, upload-time = "2026-08-13T09:15:21.197Z" }, - { url = "https://files.pythonhosted.org/packages/65/10/9c18cb01104bb211c9b0d9a956fae3f1a40f743c35bbc6763d0114ec36a7/cast_value_rs-0.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:26fe305d5bd35d21c5ee72c5651892895447e175ba3019a24713c565d965c0ef", size = 442025, upload-time = "2026-08-13T09:15:22.575Z" }, - { url = "https://files.pythonhosted.org/packages/64/49/42a0b48a71aca3ebb0abd0948fa9d484723c3562af30a8d6904b874abb2f/cast_value_rs-0.4.2-cp313-cp313-win_arm64.whl", hash = "sha256:9ce6678fc4b1bdc459178225347acd8019f1d22bca1e13ab86cc2a3e4ecb2f46", size = 397871, upload-time = "2026-08-13T09:15:23.906Z" }, - { url = "https://files.pythonhosted.org/packages/be/e9/baf39cd3991962705ffd055fce071e23e7707f62df3fb44af2ffe0e84d11/cast_value_rs-0.4.2-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:cefdcfd4d2b95e3b35fd98b3d3d865be9b3ef1bc1264db3bc1b8c50c5e8ac999", size = 495672, upload-time = "2026-08-13T09:15:25.168Z" }, - { url = "https://files.pythonhosted.org/packages/ed/9f/7ac6bf95d160f3fd89020aa952ffb4a49216b85a3fda02f84bc2c2612a2f/cast_value_rs-0.4.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cbcec2cfd199c1e404f5bda71b273b4a7470762f4b55c1c21f09d60d2762e339", size = 458480, upload-time = "2026-08-13T09:15:26.647Z" }, - { url = "https://files.pythonhosted.org/packages/65/e5/46c2985ccbae41c8140f04f2acc9080761f10e8d2f8101c46083e632f812/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c224a9b30c2521cd06eac4bb1ac3860f41c902695e6f806344bc039d7a8a1d93", size = 486807, upload-time = "2026-08-13T09:15:28.129Z" }, - { url = "https://files.pythonhosted.org/packages/18/13/97851c45b275fbc9d42b5d5a340fd5ea430525b60c2eacfe6a36399d69ef/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1eaa7c3d5fe418a74928f78514f9baeaeb8e31b93db96b1761edd6144f0d2eff", size = 555335, upload-time = "2026-08-13T09:15:29.648Z" }, - { url = "https://files.pythonhosted.org/packages/7f/b3/c84c1e820786ddb062b162112b80e5dfc1b5b54b11125618f38c1e18d075/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bddec46b36f29964c8d4bbe82b8270575fbe1c0fb41c404bb3da1778f390a24", size = 650347, upload-time = "2026-08-13T09:15:31.044Z" }, - { url = "https://files.pythonhosted.org/packages/60/38/52b4cc31570384e0286e28a03164a357d53a8aba7421ef83e500843d5449/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab3fd77144d26e2da2a113f862c05ea1cf465df347daf5deb6cf1689253fddfa", size = 561498, upload-time = "2026-08-13T09:15:32.485Z" }, - { url = "https://files.pythonhosted.org/packages/41/c8/313e1d016b0adfccd1ead2bb28d2c94313ab62534bd6e11db19a480d3b55/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4affe8668a2f17764ed978c2dddb2b49554f36a71389e6928a8a5523c1a4173", size = 548676, upload-time = "2026-08-13T09:15:33.912Z" }, - { url = "https://files.pythonhosted.org/packages/81/8b/a11f8e514598ebfc084507b0db390ecbdc6bfcffbf91ae666e0de93e6c25/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a67316b00ca01a78865d222badef2b1cb6c60ad5ebd360bf4fa3b224e87b8ffa", size = 590803, upload-time = "2026-08-13T09:15:35.384Z" }, - { url = "https://files.pythonhosted.org/packages/94/1e/ed86da6fe69eff312cc83e348cf4715d4168ab2699783febd24bfd477cd8/cast_value_rs-0.4.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9946054c77d7c5df579e9e99ebebd4d8a220cb04a6af1b6467112dfe07eee72d", size = 664118, upload-time = "2026-08-13T09:15:36.797Z" }, - { url = "https://files.pythonhosted.org/packages/0b/01/58a3eadff1233bd98d5809e2f6b5dc6d1b8c4c770664145f57fd0bdce756/cast_value_rs-0.4.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:95b064458233a7ef0525481eccfd2d37e86620d877ab1535ae809d37355cf131", size = 830686, upload-time = "2026-08-13T09:15:38.645Z" }, - { url = "https://files.pythonhosted.org/packages/e6/40/a234d9d06013a0bee08f662ab083b3a55decfa2dbb315094fdf06693d425/cast_value_rs-0.4.2-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:d7de068a1dcc65d1aaf4e93228198b588776bd25b62ef42478ab7efe5245a6f3", size = 787835, upload-time = "2026-08-13T09:15:40.078Z" }, - { url = "https://files.pythonhosted.org/packages/d6/72/93f906345d03972de52e3e74dd4ed12b56aab53bec85cc96020690ba6dbe/cast_value_rs-0.4.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae0cbdd44290a74e34519eb990b33bf0f8e96f0fde95ffc799913ff1aab27719", size = 753039, upload-time = "2026-08-13T09:15:41.702Z" }, - { url = "https://files.pythonhosted.org/packages/e3/a5/47b6db4f68da086db3f3768775f545b9b7f660cab8f61039c91de1f813c4/cast_value_rs-0.4.2-cp314-cp314-win32.whl", hash = "sha256:d438517530d3a8d6bc810ab6dee831c48b45287435513fea1b0476b5d4fda07d", size = 387092, upload-time = "2026-08-13T09:15:43.218Z" }, - { url = "https://files.pythonhosted.org/packages/d3/9c/5f71a5e94875bb999646227a583792dbbe9a2b257df0d961e8a2da879f9e/cast_value_rs-0.4.2-cp314-cp314-win_amd64.whl", hash = "sha256:7e7bb9faa4820b7eb0d99074fd61efff3a7392e72b6abe4b41c9d6b6d14fe457", size = 442126, upload-time = "2026-08-13T09:15:44.623Z" }, - { url = "https://files.pythonhosted.org/packages/95/1c/8e7546a044da8e92eafbb12594169301cc94e2a071fd3e34204595d0a05f/cast_value_rs-0.4.2-cp314-cp314-win_arm64.whl", hash = "sha256:1f57e762a06102d61e940548a6ab841e05ad1f51f1c4e2aa26bc27a8c59a5abc", size = 398200, upload-time = "2026-08-13T09:15:46.117Z" }, - { url = "https://files.pythonhosted.org/packages/a0/09/527646deeb622e5a597e4271e9e06d9d7cd24d64de77b8cb4ce250128c18/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:230f6440db74d2d11e513060d0e72aa3f897546465bc778d30aaeac90f8e733e", size = 485197, upload-time = "2026-08-13T09:15:47.622Z" }, - { url = "https://files.pythonhosted.org/packages/b9/79/aaa713cb151a902439b670a4774ea76d0a5dadf1f03954f6a34af2a6cb1e/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:37538170dc1fedbfd62dd56670e443b881474e7eda7f583a3c989c83a9ff15fc", size = 538669, upload-time = "2026-08-13T09:15:49.148Z" }, - { url = "https://files.pythonhosted.org/packages/d3/b1/2cf4f4f13f74caaa4dab4ddf4924fdeeffb39920d5ec30a80112aa041efa/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d00b9cef0d38d0c5631ba63270c38c5d7c85aa87c609644f204919f939f0c9", size = 649341, upload-time = "2026-08-13T09:15:50.614Z" }, - { url = "https://files.pythonhosted.org/packages/39/8b/1faa6b27cda3dd793635c7945118e49188b6714d6bf23bb072efd718e467/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:89434eebb7a12f66e99382e08c32648f055e24d9fce66274adef6620af2aebbd", size = 555253, upload-time = "2026-08-13T09:15:52.252Z" }, - { url = "https://files.pythonhosted.org/packages/8b/c2/d31f177025b51a3fda70c5888b92f5fd30ec53bc2df2c046fc9a2717cd36/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:704171aec75f417f5a70a3eb5f107fbfc79183620093c9d9074290920a5981f1", size = 547964, upload-time = "2026-08-13T09:15:53.669Z" }, - { url = "https://files.pythonhosted.org/packages/64/99/521c2d6347ca39deef237909dfee61c6d2cb806dcc20d74636e209bc646e/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2add2599d299436ec161661a8e119a69742c763a0936e542db92b1ce9d6e0c04", size = 574778, upload-time = "2026-08-13T09:15:55.479Z" }, - { url = "https://files.pythonhosted.org/packages/f0/70/35c7cb37839454aeb93b9576b42daebd8249dd59e3834cd1be020b4b0660/cast_value_rs-0.4.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:da5df79a926bdcc1f5494612f621027970a9702eb49d556068025166d4c0ace2", size = 662461, upload-time = "2026-08-13T09:15:56.97Z" }, - { url = "https://files.pythonhosted.org/packages/24/4f/a71bfaea1286bd12cc912733d650bd89a07c4430ee715202551c6e3eb0b0/cast_value_rs-0.4.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:5552e3c80d43de7a8a821793cb57da0dd3720a377051a5bf86bcabe094e87777", size = 814743, upload-time = "2026-08-13T09:15:58.652Z" }, - { url = "https://files.pythonhosted.org/packages/94/a3/b04e0f42dafeae30604abca079cce303ebb69ba24f3c4debbf211ecc781c/cast_value_rs-0.4.2-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:73978c6dd2ce45a05a59ec7090df8b3f2a579ee86db9b1c7df21ede17f32189e", size = 782340, upload-time = "2026-08-13T09:16:00.135Z" }, - { url = "https://files.pythonhosted.org/packages/3a/6c/913de75054092de796509dc04b92fc6ac1bf83dc86901f1f482667dead16/cast_value_rs-0.4.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:45d630592afb18880d4df0d7afb28b8cfc0ba6aa742d8db115f616df284b453a", size = 754068, upload-time = "2026-08-13T09:16:01.572Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/a2/25/151a7ac3e68a7bfa76d91dcbe318828dfa227841cd3ca1809890c052b0d6/cast_value_rs-0.5.1.tar.gz", hash = "sha256:67220f38cf791489b93ecee1e33370428326a523a211760faa36dd6f9b60a07b", size = 91796, upload-time = "2026-10-31T08:45:39.516Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/01/3b5bd177228cfcaf72a08a42f0b2184d029845edabb337b4b32c825be5ac/cast_value_rs-0.5.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f6410d5f5ca3b1c5a5ddb9e54f0064d2a1486ec8cdb187d6df344cb8294bba51", size = 514726, upload-time = "2026-10-31T08:43:15.438Z" }, + { url = "https://files.pythonhosted.org/packages/8e/fe/b0863f6769c8396a247ba30a728c041c094634890c9fd698349711fb5498/cast_value_rs-0.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d10b5ff6fd0d07133f5143384837f692d4034a113947e8b0bc4309aa39ab66f1", size = 474619, upload-time = "2026-10-31T08:43:17.336Z" }, + { url = "https://files.pythonhosted.org/packages/9b/0b/3820f21a9f5c9d71cad9dba22274cee693962bc9a503a5abbe0d94c80704/cast_value_rs-0.5.1-cp312-cp312-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:a43ca46ebdff1aaa9260b61b4527db72040123010aab1ec281fe8dcfced2ad1f", size = 563294, upload-time = "2026-10-31T08:43:18.812Z" }, + { url = "https://files.pythonhosted.org/packages/65/f9/3bcb27ea7ee0766eaffee2e75461a416beafccafde35c6816d152eebd906/cast_value_rs-0.5.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b167288f74bc2aa1201ed9b3595c767cdb67abe289cd36c1c78b5f394d94eda3", size = 477658, upload-time = "2026-10-31T08:43:20.248Z" }, + { url = "https://files.pythonhosted.org/packages/f0/04/187ff973140d1cd77d0a8067f2ac96e4693d50f3cbd206ca8127019edef8/cast_value_rs-0.5.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:7aecf24a3444ea02e082f94a4c469ff527c2f580eb71cb702f091ebc3c19c5d2", size = 536492, upload-time = "2026-10-31T08:43:21.733Z" }, + { url = "https://files.pythonhosted.org/packages/48/02/b2dbf5e4ed5caecc4f1686d5511bdff3769db5f28f262acf8f22d62feba8/cast_value_rs-0.5.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4351299e8fdeb83cbd34c0827a29fd8c8b160a3226cae774d3e682a0d9acc3e1", size = 632370, upload-time = "2026-10-31T08:43:23.687Z" }, + { url = "https://files.pythonhosted.org/packages/db/44/f168d64613bb9a42595c4e55cedcf94b81fa4c47bb142c71970a7d6359e8/cast_value_rs-0.5.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a8bfb2dd64d8cf51cddff9abb396f0d5bac118a8065c69f9c0a5b223bcd3a2cd", size = 558721, upload-time = "2026-10-31T08:43:25.189Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b3/383147f4e9e3dc04ab8e392c282653bedc871fb96344f6d773a23d6345f9/cast_value_rs-0.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:891fc3844dc25980a688dee92f7042d332ca6966ca952347cd335339b0ad1504", size = 525962, upload-time = "2026-10-31T08:43:26.784Z" }, + { url = "https://files.pythonhosted.org/packages/e1/d3/7e84f237a128047398cb786fc2ea1e4b0bc52f07cbdb6e8bf47d9d0df635/cast_value_rs-0.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f973626e9d6d31da4784dbc53d30c8c200724bf4f244d6dc890c79a479c01871", size = 654291, upload-time = "2026-10-31T08:43:28.259Z" }, + { url = "https://files.pythonhosted.org/packages/8d/5c/3e9442c4aa6d6b1e45e901bbd4f591df1729acc7842b02eb4c4a7c5429f6/cast_value_rs-0.5.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cd7c41110ffe110f5364a04e2b2a82a7dbaa57166e5b7c2893b9970d4e871c33", size = 812705, upload-time = "2026-10-31T08:43:29.696Z" }, + { url = "https://files.pythonhosted.org/packages/43/7c/3e1b5742839ba9a97ba4e2cffb02758c8209b462f00474b08dc69537edc6/cast_value_rs-0.5.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2d56b5b7cfeeea8737772bf1ccb19835111cbf669d773591826ec53fb551dbeb", size = 766586, upload-time = "2026-10-31T08:43:31.22Z" }, + { url = "https://files.pythonhosted.org/packages/3f/e0/ffa2b2fbe125f9f0fd3f2c3cd4c818317d7523a2c02a9b6ab0d48d76644c/cast_value_rs-0.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b14d256361bb449562194c4f12c51f9b7745900412f37c84c1f759e6eab362fc", size = 730423, upload-time = "2026-10-31T08:43:32.782Z" }, + { url = "https://files.pythonhosted.org/packages/40/7a/42c1ac8202610e48204a07f98f7055515c2b7d860c6fd8de5101e45c7974/cast_value_rs-0.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:ee3e305f27dd0f2c3334eb80c874bdd5262f0a707ad60b4ff2857a3c9a07b5a3", size = 401775, upload-time = "2026-10-31T08:43:34.402Z" }, + { url = "https://files.pythonhosted.org/packages/ac/e0/9b85c3da33c11f0ef38b03878bfe4b55ab06d4282bc3b2db0be94f30f2c6/cast_value_rs-0.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:6bc12f62e3a890243322999596d7001cf730cc59599538ad284f5e4508101bbc", size = 370508, upload-time = "2026-10-31T08:43:36.084Z" }, + { url = "https://files.pythonhosted.org/packages/42/ce/e881f4ca411676ee6918ad74ba57639a5a0b0fa0ef43f3c6dc037c1dd5e2/cast_value_rs-0.5.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:32ff4e7e5c2bcbc3fe85314bd15553321ad43f4869e7015a45e05f1d131964d0", size = 514776, upload-time = "2026-10-31T08:43:37.558Z" }, + { url = "https://files.pythonhosted.org/packages/80/cb/37300e5c5e2bf2396883aa9fdfd90e2c9cea6b33dfc1a69b04b4e723a973/cast_value_rs-0.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2689fe1558d5ecbd115bbd2bb97f3a1f2a458f3ebc656f268ceb94624b4dda26", size = 474544, upload-time = "2026-10-31T08:43:39.12Z" }, + { url = "https://files.pythonhosted.org/packages/b8/e5/c45635cb3da6b1ab9d06e57bd63e294362948d9ad6380a75570c4686ffb8/cast_value_rs-0.5.1-cp313-cp313-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:71e419e6f4799c187fffbe8c6a96bc1c07d17ccc63a8d77f1fcf74f01ce6f98e", size = 563316, upload-time = "2026-10-31T08:43:40.906Z" }, + { url = "https://files.pythonhosted.org/packages/cf/5d/da4920da8731987c016730cb527688d5eccfd0660dec1ad387ac5e97a996/cast_value_rs-0.5.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a0fa4372466e59197be1c0ab17a909d16e9108fc5327ca38d8519e014fa6bf70", size = 476817, upload-time = "2026-10-31T08:43:42.322Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b0/afabc7883058a18f6e3849a260e6c9faacbfa864d4ec78cadf2d7c8db49d/cast_value_rs-0.5.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2a3acfbcc7e4cf29ea45cbf2f481199dd19be6f101e5a519614708cfb8c9d9ce", size = 536511, upload-time = "2026-10-31T08:43:43.788Z" }, + { url = "https://files.pythonhosted.org/packages/98/d5/0806712a64a7b117bbd24e7b2aface573f31a6b8977b0ae50d5a449d0621/cast_value_rs-0.5.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f7701b7d680689b901c2480289af095c3065c28731066a98d0bf4058ff1d7ec9", size = 631482, upload-time = "2026-10-31T08:43:45.193Z" }, + { url = "https://files.pythonhosted.org/packages/0d/81/b24b7c75eab58e0f653c62a037955868907f8d782963d88b8c3cd1ab8fba/cast_value_rs-0.5.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:f895ffcf8c24870a92e8697619ef8b091d87159cb354ca3168ed72b092e81907", size = 558991, upload-time = "2026-10-31T08:43:46.766Z" }, + { url = "https://files.pythonhosted.org/packages/14/b3/f7c034c10bc88ae99ae00c282690aeed534a978fa591fb5ba7d7947adca2/cast_value_rs-0.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c5acd38e2e8d0aa2873881efb624f076c6286b04af030e61aa124769fa75d06", size = 525925, upload-time = "2026-10-31T08:43:48.247Z" }, + { url = "https://files.pythonhosted.org/packages/ed/32/6bbbbbfe1352395243120ee77d364c54684613fa0e8c9e59b82571553816/cast_value_rs-0.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2373117423aee9df307b8ff87af9487faa6324124fc30eb358aa9f5630462f0f", size = 653682, upload-time = "2026-10-31T08:43:49.685Z" }, + { url = "https://files.pythonhosted.org/packages/14/2c/81f025c5ff0acb2ffa5ea3dccc69e7a6ba62fc432376fc1df72b33ddde5e/cast_value_rs-0.5.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:fa1cda7c516ecb5219b9b109603ba09193bdb1ca9b3a64599e9f597dda576327", size = 812907, upload-time = "2026-10-31T08:43:51.151Z" }, + { url = "https://files.pythonhosted.org/packages/3d/cf/9baf257796769718ee29e2c0e372d44dc9bf1682edc3e9b2053fe2a0d094/cast_value_rs-0.5.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b619762eace293242b43286c3fedb8b83660effaac016afca400f7852f71c0da", size = 766871, upload-time = "2026-10-31T08:43:52.97Z" }, + { url = "https://files.pythonhosted.org/packages/59/27/acbfb113c69afb86bac85b38ba872d95f7c61ecbea3ff350ab5e8e70c050/cast_value_rs-0.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18b3ddcf763d5208e2f8ac9db659837c67d77e22c44b7315e924b7351ed6e08c", size = 730422, upload-time = "2026-10-31T08:43:54.922Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e9/7a097d2844b3541b36946cdeb51430b2cf1f287026e054ac0240b46649ab/cast_value_rs-0.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:0fdc504d6f46b1f49b7c7fd1bf108b605d9d9ecb6054ba578d3f6a38843d6fce", size = 402047, upload-time = "2026-10-31T08:43:56.312Z" }, + { url = "https://files.pythonhosted.org/packages/c1/71/ad4dbde8cee35f86c660473f6bcb98641236c7b506a4b481bd4954a208b1/cast_value_rs-0.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:b0ef63e39dc55c5210d31e3fc2377668c12db86659af6212474338ab1168c0a8", size = 370621, upload-time = "2026-10-31T08:43:57.832Z" }, + { url = "https://files.pythonhosted.org/packages/61/7b/4f3aac163a600c3a70be75d9b5652b0482c71117daac7c0276c040e60cf6/cast_value_rs-0.5.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:7f52ddc9536a2319caa0358088cdb552bd5c3ea0d576dd4bef5378dd87605f94", size = 519536, upload-time = "2026-10-31T08:43:59.456Z" }, + { url = "https://files.pythonhosted.org/packages/8c/fd/b2b7456241e86e485de36bf7f659be592fe2eaf93aa0df4d2f4b60b37de9/cast_value_rs-0.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4048177e1200023661ef37c0dacedb40d39465cae41f6c8e0da70a64cd98ddb2", size = 476470, upload-time = "2026-10-31T08:44:01.182Z" }, + { url = "https://files.pythonhosted.org/packages/66/f3/74ce87a4064388862485d6d5394553a2764de3af5e1f7568cb19e19a0111/cast_value_rs-0.5.1-cp314-cp314-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:ef3fa404ed66e5f33c24cef003071f509b7e1f36a7f748182e62eda9e20941da", size = 563457, upload-time = "2026-10-31T08:44:02.813Z" }, + { url = "https://files.pythonhosted.org/packages/4b/1f/fc7a5e1b0e6af62ee5b48e57c50bcef62713900467b09bcb87a56fbf3fd2/cast_value_rs-0.5.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:946f69a6e586e0d1f8c395fc97e7a5556b64dbba87a8a34bef112dbbb9690373", size = 478965, upload-time = "2026-10-31T08:44:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/6c/b1/39ce0bfd2ba25cbda8113eed86afa77f6eedd348d711a067352a73cba2e2/cast_value_rs-0.5.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:18460ef2f1883238083c6bbd50c2b4be41afc5384de0ad815a7b1483e8b2e491", size = 537191, upload-time = "2026-10-31T08:44:06.348Z" }, + { url = "https://files.pythonhosted.org/packages/4f/13/8c3f1d0a9593a15add6e0973226ab9e8ddeafad4aeb464bdac3811eae5c3/cast_value_rs-0.5.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:44d6d057327b96e1fef29ce10510a6ecc9612874d4df30e3e43e0540844b1bf2", size = 633461, upload-time = "2026-10-31T08:44:07.787Z" }, + { url = "https://files.pythonhosted.org/packages/49/65/fb101465f33896a1454e5a959c9c50099e6d6a37db4d4d899cf924d766e1/cast_value_rs-0.5.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d3708705c38210d9ba5c6d75433532ca98744f36a5844b07e926a7511b7c0c3b", size = 559760, upload-time = "2026-10-31T08:44:09.529Z" }, + { url = "https://files.pythonhosted.org/packages/3c/24/9171671d9b14febdbc94810a61f33276bdc1dbc591c0c8d954d0f05a4086/cast_value_rs-0.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:20bb849f145eb1d7cff568e84dd2970481c25fcdbae92d3bfb6f041750e92a48", size = 527072, upload-time = "2026-10-31T08:44:11.26Z" }, + { url = "https://files.pythonhosted.org/packages/9b/bc/f1e902a8b24b04564a45cea3471be06b4a1ba185f65553be4e26f7dd5a7e/cast_value_rs-0.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b6705f2c17704925a516392fbdc194fa1a3010829bd0800f205fd673d15b9751", size = 655678, upload-time = "2026-10-31T08:44:12.986Z" }, + { url = "https://files.pythonhosted.org/packages/31/50/6ddedc98d93acd6d0fde9bfe6bf17ec5fe83485a00e0dc946b026fde2b31/cast_value_rs-0.5.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:7b98cf7330ab7378f5497c215a7597e974e6224ebaaa46744eb6ec10c7cc6b55", size = 813526, upload-time = "2026-10-31T08:44:14.439Z" }, + { url = "https://files.pythonhosted.org/packages/29/c1/b3ec914d461603d2d36dc1eb445710e58c90436b6efa1e9d99493adbbbe4/cast_value_rs-0.5.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:d4229f74c608b895ff7ebae51ea894384c4f8a639f1129df394ed86ac4f7fcf2", size = 767696, upload-time = "2026-10-31T08:44:16.427Z" }, + { url = "https://files.pythonhosted.org/packages/85/ce/fb4f4ec09a80ebcd5c9e15ef6bb9943edf394653c0f97762d67eab5fda3d/cast_value_rs-0.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:60f4d3fdf33b44a294942938f49618d1908af8051ae80a2bb3ac7a81bf8cb563", size = 731352, upload-time = "2026-10-31T08:44:18.044Z" }, + { url = "https://files.pythonhosted.org/packages/4b/88/41ce6ec2663e40a28e2e9c8e35e08962a904909fed7ad45d1f6ae1f54c47/cast_value_rs-0.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:d9b5f2a2a8e0c7ca9dec0741e0d7a011fff2e1f7c36273ce4edd806ad7f5b5be", size = 404653, upload-time = "2026-10-31T08:44:19.421Z" }, + { url = "https://files.pythonhosted.org/packages/b3/27/80a353c489247f3fcb1555e2bb8fa87cfa273fdb12f6bafecf6fcf39313b/cast_value_rs-0.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:94ca8cd384413459399f3de50e383007090e46d55a89e554dc5f8cf2df4633e1", size = 371183, upload-time = "2026-10-31T08:44:20.766Z" }, + { url = "https://files.pythonhosted.org/packages/63/d7/a1a6f2ad4112592a342e0291accd7d7503d00f7f77afba7bfa746568440b/cast_value_rs-0.5.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:85511ddd1e54f5d065b2cec047cb90765006b104fab44ce2394364b91257d3ff", size = 491276, upload-time = "2026-10-31T08:44:22.216Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d2/7a6f34c0a425f5b5264aa72982ae975394ff56af7667c4175da402538967/cast_value_rs-0.5.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e7588000b116674a94b62b09668afd4047ae1cfa6ca23fa217e2a0f9cc7c5d9", size = 659458, upload-time = "2026-10-31T08:44:23.938Z" }, + { url = "https://files.pythonhosted.org/packages/e7/95/14ebfec2e8bc9b5f4c4119f3bf341ac8d4af1e4f80cc469dc899627b3095/cast_value_rs-0.5.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5695711e9c5abe0bd1663e1cca65135ef6521c4aa9b0f66c3ea5dfa6a520d0d4", size = 578621, upload-time = "2026-10-31T08:44:25.344Z" }, + { url = "https://files.pythonhosted.org/packages/31/bc/79e87e32286543f83cdcc3922dd7346fc5f59391a7f296d20efa77c1b091/cast_value_rs-0.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3394e3114361b4f7898dfc6e06da75373ae1a37ce042d40bd87a2e9e0aed5201", size = 552219, upload-time = "2026-10-31T08:44:27.024Z" }, + { url = "https://files.pythonhosted.org/packages/e7/61/2fdad5258778bff2dd557030f1712e7f9fc82be3b4504235111d57111123/cast_value_rs-0.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:57f08679dbca2f45c15ffd21de75da7891be7bfb7fa04ffe4cadcc10740dde49", size = 668370, upload-time = "2026-10-31T08:44:28.511Z" }, + { url = "https://files.pythonhosted.org/packages/72/c7/e3fd8da9d62a6fafb7ac35b54753bbade989463588769d24d65147d10573/cast_value_rs-0.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea5de15f91e9b59c7077ddee0903de11982440e3ed261995e0173f0d4dfb35db", size = 756835, upload-time = "2026-10-31T08:44:29.986Z" }, + { url = "https://files.pythonhosted.org/packages/4a/02/61bdbf5c06fce0dcdda04fbd8a2e6b224c5329c2b05089df2ec8b87206f7/cast_value_rs-0.5.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:3bf1543670fd8a3e70b40bfe099848ad938e79c2583335be44328466e9452f60", size = 476694, upload-time = "2026-10-31T08:44:31.419Z" }, + { url = "https://files.pythonhosted.org/packages/1a/e3/1d842641d5f2ddd7d8debcd26cc372cb048bb5a0c032e011608c3f4cf8e9/cast_value_rs-0.5.1-cp315-cp315-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:d64f290dc2f440a67717cb5051552b509403f3d8ef5b9690876dc514acd13486", size = 563843, upload-time = "2026-10-31T08:44:33.063Z" }, + { url = "https://files.pythonhosted.org/packages/8a/0c/c3c1b448e97fe9684865bcd6534273cef3ae242d41706f8a05bfa8fab372/cast_value_rs-0.5.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7fafc0825b4f675fe55d9d22f480043da2b920ce474b4dc53162306c64bca3e4", size = 479009, upload-time = "2026-10-31T08:44:34.718Z" }, + { url = "https://files.pythonhosted.org/packages/20/5f/91e9a654ac1fbd299ec6c6b6e0dd43f244438debfeb810e519cb7322c6da/cast_value_rs-0.5.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2d997464871e9c5ad2fa5a950de15390b134abd5a2071fd5e9fb2b27fe97abda", size = 537179, upload-time = "2026-10-31T08:44:36.188Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8c/844d1a4d7bd0f8a8908b1f311d14a414c11fb2ce8445eb9a88621d528b0b/cast_value_rs-0.5.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:ea69d3d3e1a0aeb428268bff34e2bd88b3ccf5d48200ef77e91e700b637c8380", size = 634858, upload-time = "2026-10-31T08:44:37.65Z" }, + { url = "https://files.pythonhosted.org/packages/23/c1/93384916c505767f270240cb7f84bb838d486bf2a95e7a12a70267a0c03f/cast_value_rs-0.5.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9c6d156cc51ba7cc6083bc6c13751df9865b40fb22cb01de0e0de510c3b1c60b", size = 559196, upload-time = "2026-10-31T08:44:39.205Z" }, + { url = "https://files.pythonhosted.org/packages/50/e8/17541402fe90d8241ce4c8c228d81dba4dc2cf15f9a447d0a3d42f15d6db/cast_value_rs-0.5.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:63ff7854dd69c00e3d113c1cf80e137ab62b703e1d5a3933a6f317e48629aac2", size = 527247, upload-time = "2026-10-31T08:44:40.665Z" }, + { url = "https://files.pythonhosted.org/packages/ed/f2/7791c4cc6348daf07ae32be323a387c3abf3693e2d34540810ab0bfa2ee8/cast_value_rs-0.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:306f35ad3cd2cf6dfad817d47ff4f1d72c30857654da264170d5ab0f4cd16ba3", size = 656202, upload-time = "2026-10-31T08:44:42.272Z" }, + { url = "https://files.pythonhosted.org/packages/20/0a/35f4cadb0012a1f03aa34b4c4dedbc040fa6f2a8ded51791aa54a911078b/cast_value_rs-0.5.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:01bb73fe09e64f6d7b98efed7e527870b7c5b506248728532c0bb6fb96eb870a", size = 813923, upload-time = "2026-10-31T08:44:43.751Z" }, + { url = "https://files.pythonhosted.org/packages/23/b9/cfc4993e242375217aaec5c4732adc537280538f99d229c92c1cc4a0bccf/cast_value_rs-0.5.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:79115de49db59bb715dfb97dd6cb565ed937bb0f4ab65bf7a85c115aeb279865", size = 767624, upload-time = "2026-10-31T08:44:45.189Z" }, + { url = "https://files.pythonhosted.org/packages/86/73/b4ad932c157b66d95e94e325aa79669667fe9fcee82cd2d4f148c04a9a63/cast_value_rs-0.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:fdb09db5d2585407125ada3b5bb9025cadb2fe7fc467e50839a6ded43add14de", size = 731376, upload-time = "2026-10-31T08:44:46.594Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e2/9e4153d8efa644059126a03ed1957174baf9b8453de6f1ddabc97436dc09/cast_value_rs-0.5.1-cp315-cp315-win32.whl", hash = "sha256:0176678830df81c30cc931624b0a352544bf40a89883b0651471f76389900114", size = 347664, upload-time = "2026-10-31T08:44:48.142Z" }, + { url = "https://files.pythonhosted.org/packages/81/c8/54e5e9a13ab37bb26b9e769892ac79dd7bae30dca847353d5a9948bf5a21/cast_value_rs-0.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:50b335dd32f4ab9da85c0b07627bf2dfd9c123e6c281eb67c3ad812ecb96cc66", size = 404766, upload-time = "2026-10-31T08:44:49.632Z" }, + { url = "https://files.pythonhosted.org/packages/31/9c/2b8d62dd4720aa96e88b5a5b499eba91dea6fb6d5f2c88ccca77563af4a1/cast_value_rs-0.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:39a2017df8ac0ac80fc6049920a569c7ef9ffb452958e51d7690003d241f3b41", size = 371573, upload-time = "2026-10-31T08:44:51.008Z" }, + { url = "https://files.pythonhosted.org/packages/d2/61/a1bc81db4af793e7268854065f1e238d548ebf107a12c276cf1296525647/cast_value_rs-0.5.1-cp315-cp315t-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:95f1d08db5067bef51ac65b9addcec901132d4dc51bee089f5a50513c07c4e16", size = 594666, upload-time = "2026-10-31T08:44:52.621Z" }, + { url = "https://files.pythonhosted.org/packages/ac/bf/cd460b4f80d0c7ca531f3bda9596a9902b8847b986fed351210b9e1a3d63/cast_value_rs-0.5.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bc4f9995a6c9629c1b625c81309a22584a38738d8a4bf19d53d703e877d0a625", size = 491256, upload-time = "2026-10-31T08:44:53.958Z" }, + { url = "https://files.pythonhosted.org/packages/55/f9/82558521a5be44dca82f14d32ed39e268aaca7a8c55d855e07cbe1839d01/cast_value_rs-0.5.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:d1fa3b07d705a625ff1effb6cf7068290a04b6fb342ffdb2734666563c4a5842", size = 555869, upload-time = "2026-10-31T08:44:55.579Z" }, + { url = "https://files.pythonhosted.org/packages/38/c7/0345a1d127f86672383abcf201f465915b1c7551726da61d0c7a7a39a075/cast_value_rs-0.5.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:82fb5958e1bddd1b6b59e5f320b7060a02c28ea4dfba959cf0199e92c8451fc5", size = 658025, upload-time = "2026-10-31T08:44:57.02Z" }, + { url = "https://files.pythonhosted.org/packages/16/32/0884b5a9debc5d45fee87732aec96e720ec9e312b14f352c85d811d5ecb6/cast_value_rs-0.5.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:0aee9b55d721ab9aca8e9ca882e904d25d43562139d228ca8190605b19b77d4d", size = 578598, upload-time = "2026-10-31T08:44:58.622Z" }, + { url = "https://files.pythonhosted.org/packages/fb/01/da585bb0fb303495779123d1f681fcbc219b00e33c23b0055ad803e586da/cast_value_rs-0.5.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:76fcd3c6322bdbd24be9772c66c7aec314ad48e801c14857f8553ad42884733a", size = 552414, upload-time = "2026-10-31T08:45:00.145Z" }, + { url = "https://files.pythonhosted.org/packages/5c/71/28da1e78625ae61110f03688d92a4c77f1e1e5a72dfd266901f4ba79c3a0/cast_value_rs-0.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:eef6fa6aa0dbbf0a31a50bd3e4482f95a662a9a1bbcc2cb4d47adb68189ed94c", size = 668386, upload-time = "2026-10-31T08:45:01.72Z" }, + { url = "https://files.pythonhosted.org/packages/d2/03/5016bec95d999a884fb2f49746e8f2b705f522677701051111d1442cc7c2/cast_value_rs-0.5.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:c27411eeb713daf0afe0a3d8025fdc1cc32191d15df0c0ac09f229cf3cd97d1b", size = 831458, upload-time = "2026-10-31T08:45:03.37Z" }, + { url = "https://files.pythonhosted.org/packages/44/db/cb3369d143c5eb5f998fd78e2772f985a3f73b30452a0be043ee83e62916/cast_value_rs-0.5.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:fdb1bdca4c472199a410ade27c3123b9c8aca4c5dd6a91597a5ca740a00d8eca", size = 796713, upload-time = "2026-10-31T08:45:04.845Z" }, + { url = "https://files.pythonhosted.org/packages/f6/2b/73ea6609be5583e648a6fcc723dc66296b0753bd98cbd10a3afca49c8da3/cast_value_rs-0.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:825d422a0e4f79a1103e8fae70f45a3244ca437ce317bb0d49bec6b5a4fccb6a", size = 757395, upload-time = "2026-10-31T08:45:06.343Z" }, + { url = "https://files.pythonhosted.org/packages/c3/4a/317620763de8d2061638cb30a10871100f2585e45a65a97bb54e1f0d1a29/cast_value_rs-0.5.1-pp312-pypy312_pp80-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:5a0eed236a5f65a5a4c84598b5cbfb77d6b295785684ef6912b47c2267429768", size = 587381, upload-time = "2026-10-31T08:45:25.09Z" }, + { url = "https://files.pythonhosted.org/packages/9c/bb/449c2fbcc5238cf24fd4c74d05d8d457287eee8ed7e6436dd8f730745f17/cast_value_rs-0.5.1-pp312-pypy312_pp80-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5d072bc28e8f9fa25c2bd7fc73ed5566a2f752af38e38203ef142a9258353ff4", size = 485865, upload-time = "2026-10-31T08:45:26.455Z" }, + { url = "https://files.pythonhosted.org/packages/e5/cf/8113580800507c4c5a35a7876c4f2dccd2d05bbd84fd0820ad74489223c4/cast_value_rs-0.5.1-pp312-pypy312_pp80-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:55a474d9495588216625f960eb9ac5d281d5463222c5dc717745020f914444e8", size = 547762, upload-time = "2026-10-31T08:45:28.065Z" }, + { url = "https://files.pythonhosted.org/packages/34/28/3119cdb31d40ea7fa5165a5bcb9e141db0d2a4737c393a09a473ab608a73/cast_value_rs-0.5.1-pp312-pypy312_pp80-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:57bbaa75c2b19e4e276cb26c98a94a08e80c657398be4dd28e149aa3e99af671", size = 650458, upload-time = "2026-10-31T08:45:30.123Z" }, + { url = "https://files.pythonhosted.org/packages/34/04/40206336236ccbc59638a0e87383f1bf4c76a8feea89e48f0dac2b44d463/cast_value_rs-0.5.1-pp312-pypy312_pp80-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1f2fcb8e0487f5a0f9cbca9bb2d5ab15f227f77acaa7ea23b5dbda7e58179084", size = 568208, upload-time = "2026-10-31T08:45:31.545Z" }, + { url = "https://files.pythonhosted.org/packages/e4/31/8a26430d6a1e0f6b6f47b2c5faf30dc30a098d9db2e0734ad8fd44f4ed99/cast_value_rs-0.5.1-pp312-pypy312_pp80-musllinux_1_2_aarch64.whl", hash = "sha256:740e7fadb7d4884451d5b75ad4410ae2508c7a09f682797462c910abb285cc95", size = 663319, upload-time = "2026-10-31T08:45:33.133Z" }, + { url = "https://files.pythonhosted.org/packages/73/d2/705418aa24017bd6bf574f96d42132fd61cb0a69dbf495129e7dfb1ca1d7/cast_value_rs-0.5.1-pp312-pypy312_pp80-musllinux_1_2_armv7l.whl", hash = "sha256:7ec196b6debda0ae9cbded8528951e0f8d2be57eb464028568733afd32aa5b2c", size = 823564, upload-time = "2026-10-31T08:45:34.543Z" }, + { url = "https://files.pythonhosted.org/packages/e3/e4/3c29de8fa19ddf9f97acfd9f44d93423da9596b5c2bcc27a9f22e2840026/cast_value_rs-0.5.1-pp312-pypy312_pp80-musllinux_1_2_i686.whl", hash = "sha256:8d69bddf06b53ca6f543d49ec64e99aeda569d208808f4b2b0829044a10e6299", size = 787628, upload-time = "2026-10-31T08:45:36.069Z" }, + { url = "https://files.pythonhosted.org/packages/cd/9f/f5bd9736239a12106e726de7b1eac0341d4b0a3e611e120452fd4c827999/cast_value_rs-0.5.1-pp312-pypy312_pp80-musllinux_1_2_x86_64.whl", hash = "sha256:a3b162f91f060845743ef8519e86643b3cfa92fd4404ab7e988c2a09c2b45432", size = 745735, upload-time = "2026-10-31T08:45:37.706Z" }, ] [[package]] @@ -3897,7 +3925,7 @@ test = [ [package.metadata] requires-dist = [ - { name = "cast-value-rs", marker = "extra == 'cast-value-rs'", specifier = ">=0.4.2" }, + { name = "cast-value-rs", marker = "extra == 'cast-value-rs'", specifier = ">=0.5.1" }, { name = "cupy-cuda12x", marker = "sys_platform != 'darwin' and extra == 'gpu'" }, { name = "donfig", specifier = ">=0.8" }, { name = "fsspec", marker = "extra == 'remote'", specifier = ">=2023.10.0" }, From 268a4707643641f72391955a810cc4ec299941ea Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 1 Nov 2026 10:46:23 +0100 Subject: [PATCH 04/21] refactor(zarr-metadata)!: call a JSON key a field and an extension-point value a declaration Match zarrs and the v3 spec. A name/value pair in a JSON metadata object is now a field everywhere (it was a member), which frees "field" from its old term-of-art meaning: a value at an extension point that a scope resolves, which is now a declaration. The public names for that concept follow: AcceptedField and its siblings become AcceptedDeclaration etc., the ZarrV3*Field aliases become ZarrV3*Declaration, ZarrV3MetadataFieldJSON becomes ZarrV3DeclarationJSON, a reading's fields() becomes declarations(), and unclaimed_fields becomes unclaimed_declarations. Messages that said member now say field. The pickle-rebuild functions keep their names so existing pickles load. Assisted-by: ClaudeCode:claude-opus-5-5 --- packages/zarr-metadata/README.md | 96 +- .../changes/+field-terminology.removal.md | 10 + .../zarr-metadata/changes/4574.bugfix.1.md | 2 +- .../zarr-metadata/changes/4574.bugfix.10.md | 2 +- .../zarr-metadata/changes/4574.bugfix.5.md | 4 +- .../zarr-metadata/changes/4574.bugfix.7.md | 2 +- .../zarr-metadata/changes/4574.bugfix.9.md | 8 +- packages/zarr-metadata/docs/api/index.md | 6 +- packages/zarr-metadata/docs/api/v3/index.md | 2 +- packages/zarr-metadata/docs/context.md | 28 +- packages/zarr-metadata/docs/index.md | 96 +- .../zarr-metadata/docs/parsing-validation.md | 28 +- packages/zarr-metadata/pyproject.toml | 2 +- .../src/zarr_metadata/__init__.py | 6 +- .../src/zarr_metadata/_common.py | 16 +- .../zarr-metadata/src/zarr_metadata/_json.py | 38 +- .../src/zarr_metadata/_pydantic_schema.py | 18 +- .../src/zarr_metadata/_typed_json.py | 96 +- .../src/zarr_metadata/definition/__init__.py | 114 +-- .../src/zarr_metadata/definition/_context.py | 6 +- .../zarr_metadata/definition/_definition.py | 832 +++++++++--------- .../src/zarr_metadata/definition/_envelope.py | 32 +- .../src/zarr_metadata/definition/_scope.py | 88 +- .../src/zarr_metadata/json_schema.py | 28 +- .../src/zarr_metadata/model/__init__.py | 4 +- .../src/zarr_metadata/model/_array_v2.py | 250 +++--- .../src/zarr_metadata/model/_array_v3.py | 349 ++++---- .../src/zarr_metadata/model/_group_v2.py | 70 +- .../src/zarr_metadata/model/_group_v3.py | 382 ++++---- .../src/zarr_metadata/model/_repair.py | 46 +- .../src/zarr_metadata/model/_shared.py | 128 +-- .../src/zarr_metadata/pydantic.py | 4 +- .../src/zarr_metadata/typed_json.py | 2 +- .../src/zarr_metadata/v2/_context.py | 10 +- .../{_field_json.py => _declaration_json.py} | 0 .../src/zarr_metadata/v2/_definition.py | 26 +- .../src/zarr_metadata/v2/array.py | 4 +- .../src/zarr_metadata/v2/codec/__init__.py | 2 +- .../zarr_metadata/v2/data_type/__init__.py | 2 +- .../src/zarr_metadata/v2/data_type/struct.py | 30 +- .../src/zarr_metadata/v2/definition.py | 22 +- .../src/zarr_metadata/v2/group.py | 2 +- .../src/zarr_metadata/v3/__init__.py | 6 +- .../src/zarr_metadata/v3/_common.py | 64 +- .../src/zarr_metadata/v3/_context.py | 24 +- .../src/zarr_metadata/v3/_core.py | 2 +- .../src/zarr_metadata/v3/_definition.py | 171 ++-- .../src/zarr_metadata/v3/_json_schema.py | 90 +- .../src/zarr_metadata/v3/_pipeline.py | 58 +- .../src/zarr_metadata/v3/array.py | 28 +- .../src/zarr_metadata/v3/codec/__init__.py | 6 +- .../src/zarr_metadata/v3/codec/blosc.py | 2 +- .../src/zarr_metadata/v3/codec/cast_value.py | 16 +- .../zarr_metadata/v3/codec/scale_offset.py | 10 +- .../v3/codec/sharding_indexed.py | 16 +- .../src/zarr_metadata/v3/consolidated.py | 6 +- .../src/zarr_metadata/v3/data_type/raw.py | 2 +- .../src/zarr_metadata/v3/data_type/struct.py | 16 +- .../src/zarr_metadata/v3/definition.py | 64 +- .../zarr-metadata/tests/model/test_array.py | 178 ++-- .../tests/model/test_consolidating.py | 26 +- .../tests/model/test_construction.py | 92 +- .../tests/model/test_extension_points.py | 20 +- .../tests/model/test_extra_fields.py | 70 +- .../zarr-metadata/tests/model/test_group.py | 193 ++-- .../zarr-metadata/tests/model/test_pair.py | 126 +-- .../zarr-metadata/tests/model/test_pair_v2.py | 38 +- .../tests/model/test_problem_invariants.py | 20 +- .../tests/model/test_pydantic.py | 10 +- .../tests/model/test_pydantic_module.py | 40 +- .../tests/model/test_read_array_metadata.py | 136 +-- .../model/test_read_array_metadata_v2.py | 38 +- .../tests/model/test_read_node_metadata_v2.py | 32 +- .../zarr-metadata/tests/model/test_repair.py | 6 +- .../tests/model/test_scope_threading.py | 2 +- .../tests/model/test_scope_threading_v2.py | 4 +- .../tests/model/test_store_json.py | 12 +- ...elds.py => test_unclaimed_declarations.py} | 54 +- packages/zarr-metadata/tests/test_contexts.py | 8 +- .../zarr-metadata/tests/test_json_schema.py | 110 +-- .../zarr-metadata/tests/test_problem_data.py | 8 +- .../zarr-metadata/tests/test_public_api.py | 20 +- .../zarr-metadata/tests/test_typed_json.py | 88 +- .../tests/test_typed_json_properties.py | 101 ++- .../zarr-metadata/tests/v2/test_codecs.py | 50 +- .../zarr-metadata/tests/v2/test_data_types.py | 22 +- .../zarr-metadata/tests/v2/test_definition.py | 16 +- packages/zarr-metadata/tests/v2/test_kinds.py | 12 +- .../tests/v3/test_definitions.py | 501 ++++++----- .../tests/v3/test_every_definition.py | 119 +-- .../tests/v3/test_fill_values.py | 14 +- .../tests/v3/test_grid_shapes.py | 4 +- packages/zarr-metadata/tests/v3/test_kinds.py | 30 +- .../zarr-metadata/tests/v3/test_pipelines.py | 56 +- .../tests/v3/test_rule_locations.py | 8 +- packages/zarr-metadata/tests/v3/test_scope.py | 78 +- .../zarr-metadata/tests/v3/test_sharding.py | 12 +- .../zarr-metadata/tests/v3/test_storage.py | 2 +- 98 files changed, 2979 insertions(+), 2851 deletions(-) create mode 100644 packages/zarr-metadata/changes/+field-terminology.removal.md rename packages/zarr-metadata/src/zarr_metadata/v2/{_field_json.py => _declaration_json.py} (100%) rename packages/zarr-metadata/tests/model/{test_unclaimed_fields.py => test_unclaimed_declarations.py} (71%) diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index 3c5513d1dd..2cdb843a7a 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -48,7 +48,7 @@ closely model the content of the Zarr specifications, such as: - **Problems** ([`zarr_metadata.problems`](https://zarr-metadata.readthedocs.io/en/latest/api/problems/)): the problem values every reader returns, and the error that carries them. - **JSON Schema** ([`zarr_metadata.json_schema`](https://zarr-metadata.readthedocs.io/en/latest/api/json_schema/)): the - JSON Schema of a `TypedDict`, a v3 field, or a whole `zarr.json`. + JSON Schema of a `TypedDict`, a v3 declaration, or a whole `zarr.json`. - **Optional Pydantic integration** ([`zarr_metadata.pydantic`](https://zarr-metadata.readthedocs.io/en/latest/api/pydantic/), requires Pydantic 2.13 or newer): each model as a Pydantic field type that validates raw documents through the same strict parser. @@ -89,7 +89,7 @@ closely model the content of the Zarr specifications, such as: Note that a bare `TypeAdapter` over a public `TypedDict` is a **coercive** shape adapter, not a Zarr conformance validator; it may coerce values or discard - members that the strict model parser rejects. This is better suited for handling input provided + fields that the strict model parser rejects. This is better suited for handling input provided from Python objects (e.g., user-authored dicts) than JSON. ## Validation boundary @@ -97,7 +97,7 @@ closely model the content of the Zarr specifications, such as: The readers and model constructors enforce the declared document structure and a small set of context-free consistency rules, e.g.: - fixed format literals, -- finite JSON numbers outside `attributes` and the members nothing in scope reads +- finite JSON numbers outside `attributes` and the fields nothing in scope reads - non-negative dimensions - one `dimension_names` entry per array dimension. @@ -129,7 +129,7 @@ Some choices that the spec text leaves open, or answers in two ways: does not interpret attributes, and zarr-python and xarray write these numbers there (for example, a CF `_FillValue`). The models read them, and `to_key_value` writes them back as bare tokens, which a strict JSON - parser rejects. So may a member the spec does not define when no + parser rejects. So may a field the spec does not define when no definition in scope claims it: nothing reads it, so it is kept as written. `check`, from `zarr_metadata.typed_json`, rejects a non-finite number anywhere, including in attributes. @@ -151,8 +151,8 @@ Some choices that the spec text leaves open, or answers in two ways: including codecs and storage transformers, although the core spec names only the data type, chunk grid and chunk key encoding. A reader that skips a codec reads wrong bytes, just as a reader that skips a data type - reads wrong values. On an unknown top-level member, `must_understand` - keeps its meaning: a reader can skip that member. + reads wrong values. On an unknown top-level field, `must_understand` + keeps its meaning: a reader can skip that field. - **A v2 datetime or timedelta dtype must have a unit**, as in `, configuration=mappingproxy({'level': 99})) +# UnclaimedDeclaration(json=mappingproxy({'name': 'gzip', 'configuration': mappingproxy({'level': 99})}), name='gzip', read_as=, configuration=mappingproxy({'level': 99})) ``` In this example, our context object only contained one codec definition, for the `bytes` codec. But the array metadata document contained three codecs, named `"bytes"`, `"gzip"` and `"unknown"`. @@ -82,12 +82,12 @@ Now read the same document in the new context. `read_array_metadata_v3` returns ```python test="true" session="context" reading = read_array_metadata_v3(document, context=new_context) -for loc, field in reading.fields(): +for loc, declaration in reading.declarations(): if loc[0] == "codecs": - print(loc, type(field).__name__) -# ('codecs', 0) AcceptedField -# ('codecs', 1) RefusedField -# ('codecs', 2) UnclaimedField + print(loc, type(declaration).__name__) +# ('codecs', 0) AcceptedDeclaration +# ('codecs', 1) RefusedDeclaration +# ('codecs', 2) UnclaimedDeclaration for problem in reading.problems: print(problem.loc, problem.kind, problem.message) @@ -99,9 +99,9 @@ print(reading.metadata) These are the three scenarios: -- `bytes` is parsed to an `AcceptedField`. A definition in the context claims the name `bytes`, and the configuration passes its checks. -- The malformed `gzip` is parsed to a `RefusedField`. A definition claims the name `gzip`, but the configuration fails one of its checks: a `level` of 99 is outside the range gzip allows. The read reports the problem, with its location in the document, and builds no model, so `reading.metadata` is `None`. Building the model directly raises the same problem as a `MetadataValidationError`. -- The `unknown` codec is parsed to an `UnclaimedField`. Nothing in the context claims the name, so it is kept as written and not checked. This is not a problem: the spec lets a document name codecs a reader does not know. +- `bytes` is parsed to an `AcceptedDeclaration`. A definition in the context claims the name `bytes`, and the configuration passes its checks. +- The malformed `gzip` is parsed to a `RefusedDeclaration`. A definition claims the name `gzip`, but the configuration fails one of its checks: a `level` of 99 is outside the range gzip allows. The read reports the problem, with its location in the document, and builds no model, so `reading.metadata` is `None`. Building the model directly raises the same problem as a `MetadataValidationError`. +- The `unknown` codec is parsed to an `UnclaimedDeclaration`. Nothing in the context claims the name, so it is kept as written and not checked. This is not a problem: the spec lets a document name codecs a reader does not know. So one JSON document can be valid in one context and invalid in another. This is why a model is a document _and_ the context it was read in: the context defines the interpretation of the document. A model keeps its context, and moving it to another context reads the document again. Our `small` model was valid in its original context, but it is not valid in the new one: @@ -113,16 +113,16 @@ except MetadataValidationError as error: # [('codecs', 1, 'configuration', 'level')] ``` -An unclaimed field is never an error, but a reader that does not understand a codec cannot decode the array's chunks. `unclaimed_fields` lists every field that nothing in the context claims, so a reader can check that it understands everything before it opens the array. With a valid `gzip` level, the document reads in the new context, and only `unknown` is left: +An unclaimed declaration is never an error, but a reader that does not understand a codec cannot decode the array's chunks. `unclaimed_declarations` lists every declaration that nothing in the context claims, so a reader can check that it understands everything before it opens the array. With a valid `gzip` level, the document reads in the new context, and only `unknown` is left: ```python test="true" session="context" fixed_codecs = (codecs[0], {"name": "gzip", "configuration": {"level": 1}}, codecs[2]) fixed = ZarrV3ArrayMetadata({**document, "codecs": fixed_codecs}, context=new_context) -print(list(fixed.unclaimed_fields)) +print(list(fixed.unclaimed_declarations)) # [('codecs', 2)] # In the small context, gzip is unclaimed too. -print(list(small.unclaimed_fields)) +print(list(small.unclaimed_declarations)) # [('codecs', 1), ('codecs', 2)] ``` diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md index b3936156d0..d6c9403518 100644 --- a/packages/zarr-metadata/docs/index.md +++ b/packages/zarr-metadata/docs/index.md @@ -46,7 +46,7 @@ closely model the content of the Zarr specifications, such as: - **Problems** ([`zarr_metadata.problems`](api/problems.md)): the problem values every reader returns, and the error that carries them. - **JSON Schema** ([`zarr_metadata.json_schema`](api/json_schema.md)): the - JSON Schema of a `TypedDict`, a v3 field, or a whole `zarr.json`. + JSON Schema of a `TypedDict`, a v3 declaration, or a whole `zarr.json`. - **Optional Pydantic integration** ([`zarr_metadata.pydantic`](api/pydantic.md), requires Pydantic 2.13 or newer): each model as a Pydantic field type that validates raw documents through the same strict parser. @@ -87,7 +87,7 @@ closely model the content of the Zarr specifications, such as: Note that a bare `TypeAdapter` over a public `TypedDict` is a **coercive** shape adapter, not a Zarr conformance validator; it may coerce values or discard - members that the strict model parser rejects. This is better suited for handling input provided + fields that the strict model parser rejects. This is better suited for handling input provided from Python objects (e.g., user-authored dicts) than JSON. ## Validation boundary @@ -95,7 +95,7 @@ closely model the content of the Zarr specifications, such as: The readers and model constructors enforce the declared document structure and a small set of context-free consistency rules, e.g.: - fixed format literals, -- finite JSON numbers outside `attributes` and the members nothing in scope reads +- finite JSON numbers outside `attributes` and the fields nothing in scope reads - non-negative dimensions - one `dimension_names` entry per array dimension. @@ -131,7 +131,7 @@ Some choices that the spec text leaves open, or answers in two ways: does not interpret attributes, and zarr-python and xarray write these numbers there (for example, a CF `_FillValue`). The models read them, and `to_key_value` writes them back as bare tokens, which a strict JSON - parser rejects. So may a member the spec does not define when no + parser rejects. So may a field the spec does not define when no definition in scope claims it: nothing reads it, so it is kept as written. `check`, from `zarr_metadata.typed_json`, rejects a non-finite number anywhere, including in attributes. @@ -153,8 +153,8 @@ Some choices that the spec text leaves open, or answers in two ways: including codecs and storage transformers, although the core spec names only the data type, chunk grid and chunk key encoding. A reader that skips a codec reads wrong bytes, just as a reader that skips a data type - reads wrong values. On an unknown top-level member, `must_understand` - keeps its meaning: a reader can skip that member. + reads wrong values. On an unknown top-level field, `must_understand` + keeps its meaning: a reader can skip that field. - **A v2 datetime or timedelta dtype must have a unit**, as in `=0.6", diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index bc57d90fd0..5b45fb198a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -7,7 +7,7 @@ - `zarr_metadata.v3` and `zarr_metadata.v2`: the JSON types of each document, extension and data type; - `zarr_metadata.model`: the document models and their readers; -- `zarr_metadata.definition`: definitions, scopes and the field reader +- `zarr_metadata.definition`: definitions, scopes and the declaration reader that both formats share; - `zarr_metadata.problems`: the problems every reader reports; - `zarr_metadata.typed_json`: `check`, the type checker for JSON; @@ -62,7 +62,7 @@ ZarrV2GroupMetadataStoreKey, ZarrV2ZGroupJSON, ) -from zarr_metadata.v3 import ZarrV3ExtensionField, ZarrV3MetadataFieldJSON, ZarrV3NamedConfigJSON +from zarr_metadata.v3 import ZarrV3DeclarationJSON, ZarrV3ExtensionField, ZarrV3NamedConfigJSON from zarr_metadata.v3.array import ( ZARR_V3_ARRAY_METADATA_STORE_KEY, ZarrV3ArrayMetadataJSON, @@ -405,12 +405,12 @@ "ZarrV3ConsolidatedMetadataEnvelopeJSON", "ZarrV3ConsolidatedMetadataInput", "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3DeclarationJSON", "ZarrV3ExtensionField", "ZarrV3GroupMetadata", "ZarrV3GroupMetadataJSON", "ZarrV3GroupMetadataStoreKey", "ZarrV3GroupMetadataUpdate", - "ZarrV3MetadataFieldJSON", "ZarrV3NamedConfigJSON", "ZarrV3NodeMetadataInput", "ZstdCodecMetadata", diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index e866f907b4..9c9e14e4f6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -2,8 +2,8 @@ JSON types that the shared reading code needs. `JSONValue` is the type of any JSON value. `ZarrV3NamedConfigJSON` is the -`{name, configuration, must_understand}` envelope of a v3 field. It is v3's, -but it lives here because the shared field reader in +`{name, configuration, must_understand}` envelope of a v3 declaration. It is v3's, +but it lives here because the shared declaration reader in `zarr_metadata.definition` uses the envelope as its default. Public homes: `zarr_metadata.typed_json` for `JSONValue`, `zarr_metadata.v3` for the envelope. @@ -38,15 +38,15 @@ The spec only requires attributes to be JSON values, and zarr-python writes these numbers as bare tokens, so a reader accepts them where a -document holds user data. A configuration member annotated with this -alias is checked that way; a member annotated `JSONValue` must hold +document holds user data. A configuration field annotated with this +alias is checked that way; a field annotated `JSONValue` must hold finite numbers, as RFC 8259 requires. """ class ZarrV3NamedConfigJSON(TypedDict, closed=True): """ - Externally-tagged union member for a metadata field. + Externally-tagged union member for a declaration. The optional `configuration` mapping holds arbitrary JSON-encodable values. `must_understand` is implicitly true when absent. @@ -54,7 +54,7 @@ class ZarrV3NamedConfigJSON(TypedDict, closed=True): `name` and `configuration` are `ReadOnly` (PEP 705) so that concrete entity types — `BloscCodecObject`, `RegularChunkGridObject`, and the rest — are assignable to this type, and therefore to - `ZarrV3MetadataFieldJSON`. Without `ReadOnly` both items are invariant, + `ZarrV3DeclarationJSON`. Without `ReadOnly` both items are invariant, so a concrete `name: Literal["blosc"]` does not satisfy `name: str`, and a required `configuration` does not satisfy a `NotRequired` one. That made the package's own codec types unusable in the very fields they @@ -63,10 +63,10 @@ class ZarrV3NamedConfigJSON(TypedDict, closed=True): requires the narrowed type to be assignable to the input type. `must_understand` stays writable: nothing needs to narrow it, and - keeping it mutable lets writers set it on an already-constructed field. + keeping it mutable lets writers set it on an already-constructed declaration. The type is `closed` (PEP 728): the spec's named-configuration envelope - has exactly these three members, and closing it is also what makes this + has exactly these three fields, and closing it is also what makes this type — and every concrete entity type embedding it, e.g. the `sharding_indexed` configuration's inner `codecs` list — assignable to `Mapping[str, JSONValue]` (i.e. usable as a `JSONValue`). diff --git a/packages/zarr-metadata/src/zarr_metadata/_json.py b/packages/zarr-metadata/src/zarr_metadata/_json.py index bc18f39d6f..56f74ff977 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_json.py @@ -250,13 +250,13 @@ def prefixed( def value_at(value: object, loc: tuple[str | int, ...]) -> object: - """The value inside `value` at `loc`, where each key names an object member and each index an array element; `UNSET` if nothing is there.""" + """The value inside `value` at `loc`, where each key names an object field and each index an array element; `UNSET` if nothing is there.""" for part in loc: if isinstance(part, str) and is_object(value): - members = value - if part not in members: + fields = value + if part not in fields: return UNSET - value = members[part] + value = fields[part] elif isinstance(part, int) and is_array(value) and 0 <= part < len(value): value = value[part] else: @@ -294,7 +294,7 @@ def with_input( def _holdable(value: object) -> bool: """Whether a problem can hold `value` as its input: JSON, nested no deeper than `JSON_DEPTH`. - A value the validator did not walk (a member it only reports) may be + A value the validator did not walk (a field it only reports) may be nested deeper than that. Such a value is not stored, because `is_canonical_json` rejects it. """ @@ -406,7 +406,7 @@ def nested_past_the_levels(value: object, loc: tuple[str | int, ...]) -> Validat `_refine` applies this rule to each container it reaches, through `_past_the_levels`, which handles a value known to be a container. A reader also calls this for each container it walks without refining - (a document's members, or the consolidated metadata it descends into). + (a document's fields, or the consolidated metadata it descends into). This bounds a chain of documents in the same way as any other nesting, and checks every container at its location, with the same result as `refine_json` of the whole document. @@ -490,23 +490,23 @@ def _refine(value: object, loc: tuple[str | int, ...], *, finite: bool) -> _Refi if len(loc) >= JSON_DEPTH: return None, (_past_the_levels(loc),) if mapping: - # Walk the members here rather than in a separate function, so that + # Walk the fields here rather than in a separate function, so that # each level of nesting costs one frame, `JSON_DEPTH` at most. - members: dict[str, JSONValue] = {} - found_in_members: list[ValidationProblem] = [] + fields: dict[str, JSONValue] = {} + found_in_fields: list[ValidationProblem] = [] for key, item in cast("Mapping[object, object]", value).items(): if not isinstance(key, str): - found_in_members.append( + found_in_fields.append( ValidationProblem( loc, f"non-string key {shown_key(key)} in JSON object", "invalid_type" ) ) continue - member, found = _refine(item, (*loc, key), finite=finite) - found_in_members.extend(found) + field, found = _refine(item, (*loc, key), finite=finite) + found_in_fields.extend(found) if len(found) == 0: - members[key] = member - return (members if len(found_in_members) == 0 else None), tuple(found_in_members) + fields[key] = field + return (fields if len(found_in_fields) == 0 else None), tuple(found_in_fields) entries: list[JSONValue] = [] found_in_entries: list[ValidationProblem] = [] for index, item in enumerate(cast("Sequence[object]", value)): @@ -736,13 +736,13 @@ def frozen(value: JSONValue) -> JSONValue: if cls in _SCALARS: return value if cls is dict or isinstance(value, Mapping): - members = cast("Mapping[str, JSONValue]", value) + fields = cast("Mapping[str, JSONValue]", value) return cast( "JSONValue", MappingProxyType( { key: item if type(item) in _SCALARS else frozen(item) - for key, item in members.items() + for key, item in fields.items() } ), ) @@ -765,10 +765,10 @@ def copied(value: JSONValue) -> JSONValue: does, so any refined value can be copied, however deep it is. """ if isinstance(value, Mapping): - members: dict[str, JSONValue] = {} + fields: dict[str, JSONValue] = {} for key, item in value.items(): - members[key] = copied(item) - return members + fields[key] = copied(item) + return fields if isinstance(value, (tuple, list)): # Use `map`, not a comprehension. Before Python 3.12 a comprehension # uses its own frame; this keeps it to one frame per level. diff --git a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py index 1b1d7bbda5..c7b2074ea2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py @@ -55,11 +55,9 @@ class ZarrV3MandatoryNamedConfigJSON(TypedDict, closed=True): must_understand: NotRequired[Literal[True]] -ZarrV3MetadataFieldJSON = ExtensionName | ZarrV3NamedConfigJSON -ZarrV3MandatoryMetadataFieldJSON = ExtensionName | ZarrV3MandatoryNamedConfigJSON -ZarrV3CodecPipelineJSON = Annotated[ - tuple[ZarrV3MandatoryMetadataFieldJSON, ...], Field(min_length=1) -] +ZarrV3DeclarationJSON = ExtensionName | ZarrV3NamedConfigJSON +ZarrV3MandatoryDeclarationJSON = ExtensionName | ZarrV3MandatoryNamedConfigJSON +ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MandatoryDeclarationJSON, ...], Field(min_length=1)] ZarrV2FilterPipelineJSON = tuple[ZarrV2CodecMetadata, ...] @@ -68,14 +66,14 @@ class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue): zarr_format: Literal[3] node_type: Literal["array"] - data_type: ZarrV3MandatoryMetadataFieldJSON + data_type: ZarrV3MandatoryDeclarationJSON shape: tuple[NonNegativeInt, ...] - chunk_grid: ZarrV3MandatoryMetadataFieldJSON - chunk_key_encoding: ZarrV3MandatoryMetadataFieldJSON + chunk_grid: ZarrV3MandatoryDeclarationJSON + chunk_key_encoding: ZarrV3MandatoryDeclarationJSON fill_value: JSONValue codecs: ZarrV3CodecPipelineJSON attributes: NotRequired[Mapping[str, JSONValue]] - storage_transformers: NotRequired[tuple[ZarrV3MandatoryMetadataFieldJSON, ...]] + storage_transformers: NotRequired[tuple[ZarrV3MandatoryDeclarationJSON, ...]] dimension_names: NotRequired[tuple[str | None, ...]] @@ -146,6 +144,6 @@ class ZarrV2ConsolidatedMetadataJSON(TypedDict, closed=True): "ZarrV3ArrayMetadataJSON", "ZarrV3ConsolidatedMetadataEnvelopeJSON", "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3DeclarationJSON", "ZarrV3GroupMetadataJSON", - "ZarrV3MetadataFieldJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py index 280bae918a..f858a066a3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py @@ -23,7 +23,7 @@ `check` is the full entry point for a caller that has a JSON value and a TypedDict. The public module `zarr_metadata.typed_json` exports it. -This module does not know what a metadata field is. A caller with its own +This module does not know what a declaration is. A caller with its own shapes passes a `leaf`. The parser asks the leaf first for every annotation at every depth, and uses the parser the leaf returns without changes. Parsers are compiled once per annotation and are pure functions @@ -247,7 +247,7 @@ def alias_value(alias: typing_extensions.TypeAliasType) -> object: class TypedDictKeys: """The keys a TypedDict defines for an object, read as the typing spec defines them. - `members` holds every key the TypedDict declares, including keys from + `fields` holds every key the TypedDict declares, including keys from its bases. For each key it holds the type of the value and whether the key is required. The type has its qualifiers removed and keeps any `Annotated` metadata, since a constraint there is part of the type. @@ -258,14 +258,14 @@ class TypedDictKeys: open. """ - members: Mapping[str, tuple[object, bool]] + fields: Mapping[str, tuple[object, bool]] extra_items: object declared: bool @property def required(self) -> frozenset[str]: """The keys an object of this TypedDict must have.""" - return frozenset(key for key, (_, required) in self.members.items() if required) + return frozenset(key for key, (_, required) in self.fields.items() if required) @property def closed(self) -> bool: @@ -309,7 +309,7 @@ def typeddict_keys(typeddict: type) -> TypedDictKeys: f"{sorted(required_at_runtime | optional_at_runtime)!r}" ) raise TypeError(msg) - members: dict[str, tuple[object, bool]] = {} + fields: dict[str, tuple[object, bool]] = {} for key, hint in hints.items(): said = qualifiers(hint) if "Required" in said and "NotRequired" in said: @@ -321,9 +321,9 @@ def typeddict_keys(typeddict: type) -> TypedDictKeys: required = False else: required = key in required_at_runtime - members[key] = (unqualified(hint), required) + fields[key] = (unqualified(hint), required) extra_items, declared = _openness(typeddict) - return TypedDictKeys(types.MappingProxyType(members), extra_items, declared) + return TypedDictKeys(types.MappingProxyType(fields), extra_items, declared) @functools.cache @@ -610,7 +610,7 @@ def parse(value: object, loc: Loc) -> Parsed: def one_of(allowed: tuple[object, ...]) -> Parser: - """A member whose type is a closed set of values. + """A field whose type is a closed set of values. A value matches an entry when it is equal and of the same type. JSON `true` is not the integer 1, though in Python `True == 1`. If the @@ -628,7 +628,7 @@ def parse(value: object, loc: Loc) -> Parsed: def sequence_of(element: Parser) -> Parser: - """A member whose type is an array of one element type, parsed element by element.""" + """A field whose type is an array of one element type, parsed element by element.""" def parse(value: object, loc: Loc) -> Parsed: if not is_list_or_tuple(value): @@ -646,7 +646,7 @@ def parse(value: object, loc: Loc) -> Parsed: def fixed_tuple(elements: Sequence[Parser], description: str) -> Parser: - """A member whose type is an array of a fixed length, parsed position by position.""" + """A field whose type is an array of a fixed length, parsed position by position.""" def parse(value: object, loc: Loc) -> Parsed: if not is_list_or_tuple(value): @@ -679,7 +679,7 @@ def any_of( literals: frozenset[int] = frozenset(), allowed: tuple[JSONValue, ...] = (), ) -> Parser: - """A member whose type is a union of shapes, parsed by the matching branch. + """A field whose type is a union of shapes, parsed by the matching branch. The parser tries only the branches whose top-level shape matches the value, and an array of a fixed length only on an array of that length. @@ -757,10 +757,10 @@ def _hashable(value: object) -> bool: return value is None or isinstance(value, (str, int, float)) -def object_of(members: Mapping[str, tuple[Parser, bool]], extra: Parser | None) -> Parser: +def object_of(fields: Mapping[str, tuple[Parser, bool]], extra: Parser | None) -> Parser: """An object with declared keys, each parsed at its own key, and `extra` for every other key. - `members` maps each declared key to its parser and whether it is + `fields` maps each declared key to its parser and whether it is required. A missing required key is a `missing_key` problem. `extra` parses a key the object does not declare. With no `extra` (a closed TypedDict), such a key is an `unknown_key` problem and is left out of @@ -777,16 +777,16 @@ def parse(value: object, loc: Loc) -> Parsed: if not isinstance(key, str): found.extend(problem(loc, f"non-string key {shown(key)}")) continue - if key in members: + if key in fields: continue if extra is None: found.extend(problem((*loc, key), f"unexpected key {key!r}", "unknown_key")) continue parsed[key], problems = extra(entry, (*loc, key)) found.extend(problems) - for key, (member, required) in members.items(): + for key, (field, required) in fields.items(): if key in entries: - parsed[key], problems = member(entries[key], (*loc, key)) + parsed[key], problems = field(entries[key], (*loc, key)) found.extend(problems) elif required: found.extend(problem((*loc, key), f"missing required key {key!r}", "missing_key")) @@ -798,7 +798,7 @@ def parse(value: object, loc: Loc) -> Parsed: def mapping_of(value: Parser) -> Parser: - """A member whose type is an object with any keys, parsed value by value. + """A field whose type is an object with any keys, parsed value by value. The open counterpart of `object_of`: a `Mapping[str, V]` does not restrict the keys, only the type of each value. @@ -976,7 +976,7 @@ def _union(inner: object, leaf: Leaf, building: _Building) -> Parser | None: return None typeddict = strip_annotation(branch)[0] keys = ( - frozenset(typeddict_keys(typeddict).members) + frozenset(typeddict_keys(typeddict).fields) if isinstance(typeddict, type) and is_typeddict(typeddict) else None ) @@ -1024,7 +1024,7 @@ def _tag(arguments: Sequence[object]) -> Tag | None: for key in sorted(shared): picks: dict[tuple[type, object], int] = {} for index, each in enumerate(keys): - annotation = strip_annotation(each.members[key][0])[0] + annotation = strip_annotation(each.fields[key][0])[0] if get_origin(annotation) is not Literal: break values: tuple[object, ...] = get_args(annotation) @@ -1077,21 +1077,21 @@ def _recursive( def _object(typeddict: type, leaf: Leaf, building: _Building) -> Parser | None: def compile_() -> Parser | None: keys = typeddict_keys(typeddict) - members: dict[str, tuple[Parser, bool]] = {} - for key, (annotation, required) in keys.members.items(): - member = _compile(annotation, leaf, building) - if member is None: + fields: dict[str, tuple[Parser, bool]] = {} + for key, (annotation, required) in keys.fields.items(): + field = _compile(annotation, leaf, building) + if field is None: return None - members[key] = (member, required) + fields[key] = (field, required) if keys.closed: - return object_of(members, None) + return object_of(fields, None) if keys.open: # Any JSON value is allowed. A note does not change that, and a # bound cannot apply, but the metadata is still checked. _constrainable(object, constraints_of(strip_annotation(keys.extra_items)[1])) - return object_of(members, _JSON) + return object_of(fields, _JSON) extra = _compile(keys.extra_items, leaf, building) - return None if extra is None else object_of(members, extra) + return None if extra is None else object_of(fields, extra) return _recursive(typeddict, building, compile_) @@ -1221,13 +1221,13 @@ def _unread_in( ) -> tuple[Unread | None, bool]: """The unread part of `typeddict`, looked inside from each TypedDict in `visiting`, itself included, and whether it is surely the part that is unread. - Each TypedDict a member holds, directly, in a union or a container, or + Each TypedDict a field holds, directly, in a union or a container, or through an alias or a `NewType`, is looked inside first, as the - checker meets it first. Then a member that holds a TypedDict is named + checker meets it first. Then a field that holds a TypedDict is named when the checker does not read its own metadata, such as a constraint. - A member that leads back to one of `visiting` may be unread only + A field that leads back to one of `visiting` may be unread only because of what is unread there, which is found at that TypedDict's - own members. Looking inside it again would not end, so such a member + own fields. Looking inside it again would not end, so such a field is named only when nothing else is unread. """ keys = typeddict_keys(typeddict) @@ -1235,13 +1235,13 @@ def _unread_in( if not (keys.closed or keys.open): others.append(("extra_items", keys.extra_items)) fallback: Unread | None = None - for key, annotation in [*((key, member[0]) for key, member in keys.members.items()), *others]: + for key, annotation in [*((key, field[0]) for key, field in keys.fields.items()), *others]: failed: TypeError | None = None try: - member = parser_for(annotation, leaf) + field = parser_for(annotation, leaf) except TypeError as error: - failed, member = error, None - if member is not None: + failed, field = error, None + if field is not None: continue inner, metadata = strip_annotation(annotation) held = _typeddicts_in(inner, frozenset()) @@ -1249,14 +1249,14 @@ def _unread_in( for nested in held: if nested in visiting: continue - # Name the part inside a TypedDict the member holds, whether - # the member's parser failed or there is none. + # Name the part inside a TypedDict the field holds, whether + # the field's parser failed or there is none. try: deeper, sure = _unread_in(nested, leaf, visiting | {nested}) except TypeError as error: # The nested TypedDict cannot be read at all, which this # error says. - return _unread_member(typeddict, key, error), True + return _unread_field(typeddict, key, error), True if deeper is not None: named = Unread(f"{typeddict.__name__}.{key}: {deeper.message}", deeper.constraint) if sure: @@ -1267,12 +1267,12 @@ def _unread_in( if len(held) != 0: own = _metadata_problem(inner, metadata) if own is not None: - return _unread_member(typeddict, key, own), True + return _unread_field(typeddict, key, own), True if leads_back: if fallback is None: - fallback = _unread_member(typeddict, key, failed) + fallback = _unread_field(typeddict, key, failed) continue - return _unread_member(typeddict, key, failed), True + return _unread_field(typeddict, key, failed), True return fallback, False @@ -1301,7 +1301,7 @@ def _typeddicts_in(annotation: object, seen: frozenset[object]) -> tuple[type, . def _metadata_problem(inner: object, metadata: tuple[object, ...]) -> TypeError | None: - """The error the checker raises for `metadata` on a member of type `inner`: a constraint it does not read, or a bound on a type that is not a number; None when it reads the metadata, as it does a note.""" + """The error the checker raises for `metadata` on a field of type `inner`: a constraint it does not read, or a bound on a type that is not a number; None when it reads the metadata, as it does a note.""" try: _constrainable(inner, constraints_of(metadata)) except TypeError as error: @@ -1309,8 +1309,8 @@ def _metadata_problem(inner: object, metadata: tuple[object, ...]) -> TypeError return None -def _unread_member(typeddict: type, key: str, failed: TypeError | None) -> Unread: - """The member `key` of `typeddict` as unread: the error its parser raised, or, when it has no parser, that it is not a shape JSON takes.""" +def _unread_field(typeddict: type, key: str, failed: TypeError | None) -> Unread: + """The field `key` of `typeddict` as unread: the error its parser raised, or, when it has no parser, that it is not a shape JSON takes.""" if failed is not None: constraint = isinstance(failed, UnreadConstraintError) return Unread(f"{typeddict.__name__}.{key}: {failed}", constraint) @@ -1332,7 +1332,7 @@ def check( """Type-check `value` as `shape`, a TypedDict. Returns the typed value or None, and every problem. This function first refines `value` to JSON (arrays as tuples, string - keys, finite floats), then checks it member by member. Each problem is + keys, finite floats), then checks it field by field. Each problem is located under `loc`. The returned value holds only what `shape` allows. A key that a closed TypedDict does not declare is reported as `unknown_key` and left out, and the value is still returned. Any other @@ -1442,11 +1442,11 @@ def object_of(self, typeddict: type) -> JSONSchema: """The inline schema of `typeddict`: its keys, its required keys, and what any other key may hold.""" keys = typeddict_keys(typeddict) schema: JSONSchema = {"type": "object"} - if len(keys.members) != 0: + if len(keys.fields) != 0: schema["properties"] = { - key: self.of(annotation) for key, (annotation, _) in keys.members.items() + key: self.of(annotation) for key, (annotation, _) in keys.fields.items() } - required: list[JSONValue] = [key for key in keys.members if key in keys.required] + required: list[JSONValue] = [key for key in keys.fields if key in keys.required] if len(required) != 0: schema["required"] = required if keys.closed: diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index 6dd9c86173..76d30ee1e7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -1,30 +1,30 @@ -"""Reading metadata fields against their definitions, in either Zarr format. +"""Reading declarations against their definitions, in either Zarr format. Every extension point of a Zarr document (a v3 codec, data type, chunk grid, chunk key encoding or storage transformer; a v2 dtype or numcodecs -codec) is a metadata field: a name, and a configuration whose JSON the +codec) is a declaration: a name, and a configuration whose JSON the extension defines. A **definition** declares what that JSON is and what it may contain. A definition is a value, not a class to subclass. It has: - `name`, the name the metadata uses; - `configuration`, the TypedDict for the configuration's JSON. This is - the single declaration of the JSON: the type checker is compiled from + the single source of the JSON: the type checker is compiled from it, and a checked configuration has it as its static type. It is interpreted as the typing spec defines a TypedDict: `total`, `Required`, `NotRequired`, `closed` and `extra_items` mean what they mean to a type checker, whether or not its module postpones - annotations. A member's type carries its bounds, in the same way + annotations. A field's type carries its bounds, in the same way pydantic reads them: a gzip `level` is `Annotated[int, Interval(ge=0, le=9)]`; - `rules`, a function over that TypedDict that yields the spec violations a type cannot express (such as constraints between - members), located in the configuration. + fields), located in the configuration. The kind of metadata a definition defines is its type. The v3 kinds are in `zarr_metadata.v3.definition`, and the v2 kinds are in `zarr_metadata.v2.definition`. This module holds what both formats share: `Definition` and `WithFillValue`, which the kinds derive from; `resolve`, -which reads a field in a scope; the three results of a reading; and +which reads a declaration in a scope; the three results of a reading; and `Context`, the protocol of a scope. **Reading JSON.** There are three steps. Each feeds the next, and each @@ -34,39 +34,39 @@ type-checks JSON against a TypedDict and needs nothing else. It returns a value of the TypedDict or None, and every problem, each with a location. The value contains only what the TypedDict admits. A - member typed with a field alias is checked as metadata field JSON. + field typed with a declaration alias is checked as declaration JSON. 2. `definition.read_configuration(configuration)` returns the configuration as that definition reads it. It type-checks the - configuration, checks the envelope of each nested field (an extra - member, a `must_understand` of `false`), and then runs the rules, for + configuration, checks the envelope of each nested declaration (an extra + field, a `must_understand` of `false`), and then runs the rules, for one configuration: `GZIP_CODEC.read_configuration({"level": 12})`. -3. `resolve(field, CodecDefinition, CORE_AND_EXTENSIONS)` reads a whole - field in a scope. It checks the envelope, maps the name to a - definition, checks the configuration, and reads each nested field the +3. `resolve(declaration, CodecDefinition, CORE_AND_EXTENSIONS)` reads a whole + declaration in a scope. It checks the envelope, maps the name to a + definition, checks the configuration, and reads each nested declaration the same way. It returns the result of the reading and every problem. The - result is one of three types. `AcceptedField` is a field read by the + result is one of three types. `AcceptedDeclaration` is a declaration read by the definition that claims its name, with the configuration it checked - and allowed, and its nested fields, each read the same way. - `UnclaimedField` is a name that nothing in scope claims. It is not - checked, which is what keeps the format open. `RefusedField` is a - field with problems that explain why it was rejected. Each type has the - field's JSON, the `name` it is written with, the kind it was read as + and allowed, and its nested declarations, each read the same way. + `UnclaimedDeclaration` is a name that nothing in scope claims. It is not + checked, which is what keeps the format open. `RefusedDeclaration` is a + declaration with problems that explain why it was rejected. Each type has the + declaration's JSON, the `name` it is written with, the kind it was read as (`read_as`), the `definition` that claims its name (None for - `UnclaimedField`), and its nested fields as the scope read them - (`nested`). The two types a model can hold, `AcceptedField` and - `UnclaimedField`, have a `configuration` and `to_json()`, which - returns the field as a document writes it. Two of them are equal when - they read the same, however each was spelled. `ResolvedField` is the + `UnclaimedDeclaration`), and its nested declarations as the scope read them + (`nested`). The two types a model can hold, `AcceptedDeclaration` and + `UnclaimedDeclaration`, have a `configuration` and `to_json()`, which + returns the declaration as a document writes it. Two of them are equal when + they read the same, however each was spelled. `ResolvedDeclaration` is the union of the three, for use with `match`. - `configuration_of(field, GZIP_CODEC)` returns the configuration of a - field that `GZIP_CODEC` accepted, typed as its TypedDict - (`GzipCodecConfiguration`), and None for any other field, including - one that `GZIP_CODEC` claimed and refused. A field is read as one of + `configuration_of(declaration, GZIP_CODEC)` returns the configuration of a + declaration that `GZIP_CODEC` accepted, typed as its TypedDict + (`GzipCodecConfiguration`), and None for any other declaration, including + one that `GZIP_CODEC` claimed and refused. A declaration is read as one of the kinds of its format, with or without type arguments. - `resolve(field, Definition, context)` raises `TypeError`, because + `resolve(declaration, Definition, context)` raises `TypeError`, because nothing is filed under `Definition`. A whole v3 array document is read by `read_array_metadata_v3`, in `zarr_metadata.model`. Its - reading returns each field with its location in the document. + reading returns each declaration with its location in the document. from zarr_metadata.definition import resolve from zarr_metadata.v3.codec.gzip import GZIP_CODEC @@ -74,7 +74,7 @@ resolved, problems = resolve({"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE_AND_EXTENSIONS) - resolved # RefusedField(..., definition=CodecDefinition(name='gzip'), ...) + resolved # RefusedDeclaration(..., definition=CodecDefinition(name='gzip'), ...) problems[0].loc # ('configuration', 'level') problems[0].input # 12 dict(problems[0].ctx) # {'ge': 0, 'le': 9} @@ -86,10 +86,10 @@ Problems are values, not exceptions: `ValidationProblem(loc, message, kind)`, with `kind` one of `invalid_type`, `invalid_value`, -`missing_key`, `unknown_key` and `invalid_json`. A field with an +`missing_key`, `unknown_key` and `invalid_json`. A declaration with an unknown key is still read: the key is reported, and the configuration is checked without it. So a consumer that tolerates unknown keys can -filter problems by kind and use the result. The field is valid only +filter problems by kind and use the result. The declaration is valid only when there are no problems at all. Each problem stores the content of its message as data, as pydantic's errors and zod's issues do: `input` is the value found at `loc` (the `12` above), and `ctx` is what was @@ -103,15 +103,15 @@ because `closed` and `extra_items` come from PEP 728, and `typing.TypedDict` does not accept them on the Python versions this package supports. The rules receive the configuration and its nested -fields as the scope read them. A field that is read stores its nested -fields as `AcceptedField.nested`, a `Nested` mapping keyed by location, +declarations as the scope read them. A declaration that is read stores its nested +declarations as `AcceptedDeclaration.nested`, a `Nested` mapping keyed by location, so a struct's rules can access its field types. `read_configuration` -reads in no scope, so it passes no nested fields. A rule's message +reads in no scope, so it passes no nested declarations. A rule's message should show a value as JSON, as the package's own messages do, with `zarr_metadata.problems.shown`: `null`, `[1, 2]`, `"C"`. A rule reports the location of a problem. The value at that location becomes the problem's `input` automatically. Define each function at module top level. A model holds -the definitions that read its fields, so the model can be pickled, and +the definitions that read its declarations, so the model can be pickled, and compares equal after loading, only if its definitions can. A lambda or a closure cannot be pickled, and a `functools.partial` can be pickled but does not compare equal to its loaded copy. @@ -156,7 +156,7 @@ def acme_lz4_rules( such a key must hold that type. With `closed=False`, anything is allowed. A TypedDict that declares none of these is open by default and would accept a misspelled key silently, so a definition rejects it. Its -members must be JSON shapes: `int`, `float`, `bool`, `str`, `None`, +fields must be JSON shapes: `int`, `float`, `bool`, `str`, `None`, `JSONValue`, a `Literal`, `tuple[T, ...]` and `tuple[T1, T2]`, a union, a TypedDict, `Mapping[str, V]`, a `NewType` and a type alias. A number's type may carry bounds, written with annotated-types and read the way @@ -180,11 +180,11 @@ def acme_lz4_rules( function that yields the spec violations in a fill value of that shape that the type cannot express, such as a hex string of the wrong width, or a number of byte values that does not match the size. The rules -receive the configuration, the nested fields as the scope read them, +receive the configuration, the nested declarations as the scope read them, and the typed fill value, so a struct checks each field's fill value against that field's own type. `fill_value_problems(data_type, value)` checks a fill value against a -data type field the scope read. If nothing in scope claims the data +data type declaration the scope read. If nothing in scope claims the data type, the fill value is not checked. A data type that does not declare its fill value accepts any JSON. A fill value can have more than one spelling (`"NaN"` and `"0x7fc00000"` @@ -199,16 +199,16 @@ def acme_lz4_rules( or `UNSET` for a fill value with a problem. A data type that does not declare `fill_value_canonical` keeps each value as written. -**The simplest spelling.** A field without problems has a simplest -equivalent spelling, and fields are compared by it. Each nested field is +**The simplest spelling.** A declaration without problems has a simplest +equivalent spelling, and declarations are compared by it. Each nested declaration is written in its own simplest spelling, then the definition's `canonical` is applied (blosc drops a `typesize` that `noshuffle` ignores, and a rectilinear grid run-length encodes its chunk shapes). The envelope uses the shortest form every reader accepts: a data type with nothing to -configure is its bare name, and any other field is an object, +configure is its bare name, and any other declaration is an object, `{"name": ...}`, because a Zarr v3.0 reader does not accept a short-hand name in `codecs`. Raw bits write their size back into the name, in -decimal, so `r008` becomes `r8`. A field with any problem, including an +decimal, so `r008` becomes `r8`. A declaration with any problem, including an unknown key, has no simplest spelling, because a simpler spelling would erase what its author wrote. The result of `canonical` is checked again, and an invalid result raises `ValueError`, which indicates a bug in the @@ -222,8 +222,8 @@ def acme_lz4_rules( Zarr format a scope reads, and `claimant`, which says which definition of a kind reads a name. That is all `resolve` needs, so any object that provides these two can be a scope. A scope has one slot for each kind of -field its format has, named like the document member that holds such -fields: `ZarrV3Context(codecs=[...], data_types=[...])` builds one, and +declaration its format has, named like the document field that holds such +declarations: `ZarrV3Context(codecs=[...], data_types=[...])` builds one, and `context.codecs["gzip"]` reads one slot. A type checker rejects a definition in the wrong slot, and a scope of one format given to a reader of the other. The constructor and the reader also reject them at @@ -244,32 +244,32 @@ def acme_lz4_rules( A definition checks itself when it is built. Each of these raises a `TypeError` that says what is wrong: a `configuration` that is not a TypedDict, that does not declare how undeclared keys are handled, or -that has a member no checker reads (the error names the TypedDict that -contains it); a `name` that is not a string; a member declared as a +that has a field no checker reads (the error names the TypedDict that +contains it); a `name` that is not a string; an attribute declared as a function (`rules`, `canonical`, `fill_value_rules`, `storage`, `shape_rules`, `chunk_lengths`, `chunk_rules`, `transition`, `pipelines`) that is not a function; a data type's `fill_value` that no -checker reads, or that contains a metadata field, which a value of the +checker reads, or that contains a declaration, which a value of the data type never does; a codec `kind` that is not one of the three, or a `size` that is not `"static"` or `"dynamic"`; a function that is never called for a codec of its kind (chunk rules or pipelines of a bytes -> bytes codec, which receives bytes, or a `transition` of a codec that outputs bytes); a data type named the way raw bits of one size are -written; an extra field named after a member the spec defines, whose +written; an extra field named after a field the spec defines, whose `finite` is not a boolean, or whose configuration does not declare `must_understand`. Building a definition files it nowhere: only a scope built from it does. """ from zarr_metadata.definition._definition import ( - AcceptedField, + AcceptedDeclaration, Context, Definition, EmptyConfiguration, Nested, - RefusedField, - ResolvedField, - UnclaimedField, + RefusedDeclaration, + ResolvedDeclaration, + UnclaimedDeclaration, WithFillValue, canonical_fill_value, canonical_of, @@ -285,17 +285,17 @@ def acme_lz4_rules( ) __all__ = [ - "AcceptedField", + "AcceptedDeclaration", "ClaimKey", "Conflict", "Context", "Definition", "EmptyConfiguration", "Nested", - "RefusedField", - "ResolvedField", + "RefusedDeclaration", + "ResolvedDeclaration", "ScopeConflictError", - "UnclaimedField", + "UnclaimedDeclaration", "WithFillValue", "canonical_fill_value", "canonical_of", diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_context.py b/packages/zarr-metadata/src/zarr_metadata/definition/_context.py index 44a889dd23..f238759a03 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_context.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_context.py @@ -1,8 +1,8 @@ """How a format's scope files its definitions. -No global registry files definitions. Reading a field means +No global registry files definitions. Reading a declaration means deciding which definition its name refers to. That decision depends on -the *scope* the field is read in, not on the definitions themselves. A +the *scope* the declaration is read in, not on the definitions themselves. A scope is a value built from definitions. `Context` is the protocol a reader needs from one. `ZarrV3Context` and `ZarrV2Context`, in each format's `definition` module, are the scopes of each format. They are @@ -56,7 +56,7 @@ def slot_kinds( ) -> Mapping[str, type[K]]: """The kind each slot of a scope holds, read from `slots`: the TypedDict a scope's constructor takes, whose keys are the slots, each annotated `Iterable[K] | Mapping[str, K]`, in that order and with `Mapping` from `collections.abc`, for a kind `K` that `accepts`, the check for a kind of the scope's format. A scope class derives its slots from it, so they are declared once. - Raises `TypeError`, when the scope class is defined, for a member + Raises `TypeError`, when the scope class is defined, for a field annotated any other way, or with a kind that is not of the format. """ kinds: dict[str, type[K]] = {} diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index 5d50f8e605..bba9c1f157 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1,28 +1,28 @@ -"""Reading one metadata field against its definition. +"""Reading one declaration against its definition. There are three steps. Each step feeds the next and needs more input than the step before: 1. The type check needs the value and a TypedDict. `check` runs the checker compiled from the TypedDict's annotations over the JSON. It - checks the shape member by member, locates every problem, and returns + checks the shape field by field, locates every problem, and returns a value of the TypedDict. A key the TypedDict does not declare is - reported and left out. A member that holds another metadata field (a - shard's codecs, a cast's data type) is annotated with a field alias - such as `ZarrV3CodecField`, and is checked as an envelope: a name, or + reported and left out. A field that holds another declaration (a + shard's codecs, a cast's data type) is annotated with a declaration alias + such as `ZarrV3CodecDeclaration`, and is checked as an envelope: a name, or a named configuration. Which definition the name refers to is a question for a scope, so the check does not need one. 2. The rules need the definition. They are plain functions over the checked TypedDict, for constraints that a type cannot express (a - bound, or members that depend on each other). + bound, or fields that depend on each other). `Definition.read_configuration` runs the check and then the rules, for a caller that has one configuration. -3. The reading needs a scope. `resolve` maps the field's name to a +3. The reading needs a scope. `resolve` maps the declaration's name to a definition through a `Context`, checks the configuration, and reads - each nested field found by the check, in the same scope. + each nested declaration found by the check, in the same scope. A definition is a value, not a class to subclass. It holds the name the -metadata uses, the TypedDict that is the single declaration of the +metadata uses, the TypedDict that is the single source of the configuration's JSON, and its rules. It checks itself when it is built. The kind of metadata it defines is its type (`CodecDefinition`, `ChunkGridDefinition`), and a scope files it by that type. Building a @@ -31,7 +31,7 @@ # The hooks of `Definition` (`_named_configuration`, `_envelope_problems` and # the rest) are private: a kind overrides them, and no user calls them. The -# functions in this module that read a field call them, from outside the +# functions in this module that read a declaration call them, from outside the # class body, which pyright reports as private use. This module is where # reading is implemented, so it is allowed here, and nowhere else. # pyright: reportPrivateUsage=false @@ -96,7 +96,7 @@ Any configuration is a `Mapping[str, JSONValue]`. That is the default when a kind is written without a type argument: -`resolve(field, CodecDefinition, context)` is typed as +`resolve(declaration, CodecDefinition, context)` is typed as `CodecDefinition[Mapping[str, JSONValue]]`, with no unknown types. """ @@ -126,18 +126,18 @@ def fill_value_as_written(configuration: object, nested: object, value: T) -> T: class EmptyConfiguration(TypedDict, closed=True): - """The configuration of a definition with nothing to configure. Its field is written with its name only.""" + """The configuration of a definition with nothing to configure. Its declaration is written with its name only.""" -_FIELD_KINDS: Final[dict[TypeAliasType, type[Definition[Any]]]] = {} -"""Maps each field alias to the kind that a member annotated with it is read as. Each kind adds its aliases when its class is built.""" +_DECLARATION_KINDS: Final[dict[TypeAliasType, type[Definition[Any]]]] = {} +"""Maps each declaration alias to the kind that a field annotated with it is read as. Each kind adds its aliases when its class is built.""" @dataclass(frozen=True, kw_only=True, slots=True) class Definition(Generic[C]): """One extension's metadata as JSON: its name, its configuration TypedDict, and its rules. - `configuration` is the TypedDict, and so it is the single declaration + `configuration` is the TypedDict, and so it is the single source of the JSON. The checker is compiled from it, it is the static type of a checked configuration, and a document's author writes to it. It is interpreted as the typing spec defines a TypedDict: `total`, @@ -148,24 +148,24 @@ class Definition(Generic[C]): `closed=False`, anything is allowed. `rules` yields each violation of the spec it finds in a configuration of that type. It receives only a configuration that has passed the check: it contains only what the - TypedDict admits, each member is within the bounds its type declares, - and the nested fields are passed as the scope read them (for example, a - struct's field types). The nested fields are empty when no scope read + TypedDict admits, each field is within the bounds its type declares, + and the nested declarations are passed as the scope read them (for example, a + struct's field types). The nested declarations are empty when no scope read the configuration. `read_configuration` runs both steps, for a caller that has JSON. Each function receives the configuration as a read-only view, not a `dict`. `copy.deepcopy` and `json.dumps` reject this view. A function that simplifies a spelling must build a new mapping, such as - `{**configuration}` without the member, instead of editing its input. + `{**configuration}` without the field, instead of editing its input. `canonical` is where two spellings of the configuration that mean the same thing are reduced to one. A definition built by hand rejects inputs it could not read with: a `configuration` that is not a TypedDict, that does not declare how - undeclared keys are handled, or that has a member no checker reads - (the error names the member); and a `name` or rules of the wrong type. + undeclared keys are handled, or that has a field no checker reads + (the error names the field); and a `name` or rules of the wrong type. """ _is_kind: ClassVar[bool] = False @@ -175,34 +175,34 @@ class Definition(Generic[C]): a kind is a definition of that kind, and does not declare `kind`. """ _format: ClassVar[Literal[2, 3] | None] = None - """The Zarr format of the documents that contain fields of this kind. It is declared with the kind and inherited by its definitions, and it sets the format a scope reads.""" + """The Zarr format of the documents that contain declarations of this kind. It is declared with the kind and inherited by its definitions, and it sets the format a scope reads.""" label: ClassVar[str] = "definition" """The name of the kind in error messages, for example "codec".""" - field_aliases: ClassVar[tuple[TypeAliasType, ...]] = () - """The field aliases used to annotate a configuration member that holds a field of this kind. For a codec: `ZarrV3CodecField` and `ZarrV3StaticCodecField`.""" + declaration_aliases: ClassVar[tuple[TypeAliasType, ...]] = () + """The declaration aliases used to annotate a configuration field that holds a declaration of this kind. For a codec: `ZarrV3CodecDeclaration` and `ZarrV3StaticCodecDeclaration`.""" name: str """The name the metadata uses. A scope files the definition under this name.""" configuration: type[C] """The configuration TypedDict.""" rules: Callable[[C, Nested], Iterable[ValidationProblem]] = no_rules - """The spec violations in a well-typed configuration and its nested fields, located in the configuration.""" + """The spec violations in a well-typed configuration and its nested declarations, located in the configuration.""" canonical: Callable[[C], C] = unchanged """A well-typed, allowed configuration in its simplest equivalent spelling. - This handles only the definition's own members. `canonicalize` puts - each nested field in its own canonical form, because it knows the + This handles only the definition's own fields. `canonicalize` puts + each nested declaration in its own canonical form, because it knows the location of each one. """ @classmethod def _name_problem(cls, name: str, at: Loc) -> ValidationProblem | None: - """The problem to report at `at` when no document of the format may use `name` for a field of this kind; None when it may. For Zarr v3, a name may be used when it is a valid extension name under the spec.""" + """The problem to report at `at` when no document of the format may use `name` for a declaration of this kind; None when it may. For Zarr v3, a name may be used when it is a valid extension name under the spec.""" return name_problem(name, at) @classmethod def _well_named(cls, name: str) -> bool: - """Whether a document of the format may use `name` for a field of this kind, as decided by `_name_problem`.""" + """Whether a document of the format may use `name` for a declaration of this kind, as decided by `_name_problem`.""" return cls._name_problem(name, ()) is None @classmethod @@ -225,17 +225,17 @@ def _carrying_name(self, configuration: Mapping[str, JSONValue]) -> str | None: def _named_configuration( cls, value: object ) -> tuple[str | None, Mapping[str, object] | None, Problems]: - """Split `value` (a field as a document of the format writes it) into `(name, configuration, problems)`, as the module-level `named_configuration` does for a v3 field.""" + """Split `value` (a declaration as a document of the format writes it) into `(name, configuration, problems)`, as the module-level `named_configuration` does for a v3 declaration.""" return named_configuration(value) @classmethod def _envelope_problems(cls, value: object) -> Problems: - """Every reason `value` is not a valid envelope for a field of this kind in this format. The contents of the configuration are not checked.""" + """Every reason `value` is not a valid envelope for a declaration of this kind in this format. The contents of the configuration are not checked.""" return envelope_problems(value, allow_must_understand_false=False) @classmethod def _envelope_json(cls, name: str, configuration: Mapping[str, JSONValue]) -> JSONValue: - """A field with `name` and `configuration`, as a document of the format writes it, in the shortest form every reader accepts. For v3 this is an object.""" + """A declaration with `name` and `configuration`, as a document of the format writes it, in the shortest form every reader accepts. For v3 this is an object.""" if len(configuration) != 0: return {"name": name, "configuration": configuration} return {"name": name} @@ -244,19 +244,19 @@ def _envelope_json(cls, name: str, configuration: Mapping[str, JSONValue]) -> JS def _unclaimed_json( cls, json: JSONValue, name: str, configuration: Mapping[str, JSONValue] ) -> JSONValue: - """A field of this kind that no definition claims, as a document writes it: `json` is the field as written. By default the name and configuration go in the envelope every reader accepts, as for a field a definition read.""" + """A declaration of this kind that no definition claims, as a document writes it: `json` is the declaration as written. By default the name and configuration go in the envelope every reader accepts, as for a declaration a definition read.""" return cls._envelope_json(name, configuration) @classmethod def _refined( cls, data: object, loc: Loc, definition: Definition[Any] | None ) -> tuple[JSONValue | None, Problems]: - """`data`, a field of this kind at `loc`, refined to JSON before it is read, with every reason it is not JSON. `definition` is the definition that claims the field, if any. By default every number must be finite, as RFC 8259 requires.""" + """`data`, a declaration of this kind at `loc`, refined to JSON before it is read, with every reason it is not JSON. `definition` is the definition that claims the declaration, if any. By default every number must be finite, as RFC 8259 requires.""" return refine_json(data, loc) @classmethod def _undeclared_parts(cls, family: str, name: str, keys: Sequence[str]) -> str: - """The message for `name`, a name filed under `family` that encodes the configuration members `keys`, which the definition does not declare. For a kind whose names encode configuration; by default the message names the keys.""" + """The message for `name`, a name filed under `family` that encodes the configuration fields `keys`, which the definition does not declare. For a kind whose names encode configuration; by default the message names the keys.""" return ( f"expected a name that {family!r} reads, got {shown(name)}, which encodes " f"{said(keys)} that it does not declare" @@ -264,22 +264,22 @@ def _undeclared_parts(cls, family: str, name: str, keys: Sequence[str]) -> str: @classmethod def _in_configuration(cls, loc: Loc, place: Loc) -> Loc: - """The location in the document of `place`, a location in the configuration of a field at `loc`: under `configuration` for v3, or under the field itself for a format that writes the parameters next to the name. `place` is `()` for the configuration itself.""" + """The location in the document of `place`, a location in the configuration of a declaration at `loc`: under `configuration` for v3, or under the declaration itself for a format that writes the parameters next to the name. `place` is `()` for the configuration itself.""" return (*loc, "configuration", *place) @classmethod def _name_loc(cls, loc: Loc) -> Loc: - """The location of the name of a field at `loc` that is written as an object: under `name` for v3.""" + """The location of the name of a declaration at `loc` that is written as an object: under `name` for v3.""" return (*loc, "name") @classmethod def _alias_problems( cls, alias: TypeAliasType, definition: Definition[Any], loc: Loc ) -> Problems: - """The problems with using `definition` in a member annotated with `alias`, one of this kind's `field_aliases`. Empty by default. + """The problems with using `definition` in a field annotated with `alias`, one of this kind's `declaration_aliases`. Empty by default. A kind with an alias that admits only some of its definitions - overrides this. For example, `ZarrV3StaticCodecField` admits only + overrides this. For example, `ZarrV3StaticCodecDeclaration` admits only codecs of static size. """ return () @@ -287,7 +287,7 @@ def _alias_problems( def __init_subclass__( cls, *, kind: bool = False, format: Literal[2, 3] | None = None, **kwargs: object ) -> None: - """Register a subclass. `kind=True` declares a kind, with the Zarr `format` of the documents that contain its fields. This marks the class but not its subclasses, as `typing.Protocol` and SQLAlchemy's `__abstract__` do.""" + """Register a subclass. `kind=True` declares a kind, with the Zarr `format` of the documents that contain its declarations. This marks the class but not its subclasses, as `typing.Protocol` and SQLAlchemy's `__abstract__` do.""" # Use the explicit form, not bare `super()`. A dataclass with slots # is rebuilt, and the cell that bare `super()` reads refers to the # discarded class. @@ -299,7 +299,7 @@ def __init_subclass__( if kind: if format not in (2, 3): msg = ( - f"{cls.__name__}: a kind declares the Zarr format its fields belong to, " + f"{cls.__name__}: a kind declares the Zarr format its declarations belong to, " f"format=2 or format=3, got {format!r}" ) raise TypeError(msg) @@ -308,8 +308,8 @@ def __init_subclass__( elif format is not None: msg = f"{cls.__name__}: only a kind declares a format; a definition of a kind has its kind's" raise TypeError(msg) - for alias in cls.__dict__.get("field_aliases", ()): - _FIELD_KINDS[alias] = cls + for alias in cls.__dict__.get("declaration_aliases", ()): + _DECLARATION_KINDS[alias] = cls def __post_init__(self) -> None: refusal = _malformed(self) or self._refusal() @@ -329,7 +329,7 @@ def __repr__(self) -> str: return f"{type(self).__name__}(name={self.name!r})" def _refusal(self) -> str | None: - """A description of what is wrong with the members a kind adds; None if nothing is wrong, or if the kind adds no members.""" + """A description of what is wrong with the attributes a kind adds; None if nothing is wrong, or if the kind adds no attributes.""" return None def _finite_numbers(self) -> bool: @@ -342,11 +342,11 @@ def requires_configuration(self) -> bool: return len(typeddict_keys(self.configuration).required) != 0 def _check_configuration(self, value: object, loc: Loc = ()) -> tuple[C | None, Problems]: - """Type-check `value` as this definition's configuration, and check the envelope of each nested field. + """Type-check `value` as this definition's configuration, and check the envelope of each nested declaration. `zarr_metadata.typed_json.check` does only the type check. This - method also checks the envelope of each metadata field that a - member holds. + method also checks the envelope of each declaration that a + field holds. """ configuration, problems = _configuration_checked( value, self.configuration, loc, finite=self._finite_numbers() @@ -357,13 +357,13 @@ def read_configuration(self, value: object, loc: Loc = ()) -> tuple[C | None, Pr """Type-check `value`, then check it with the rules. Return the configuration if it is valid, and every problem. The rules run only on a configuration that passed the type check, - including its bounds, and whose nested fields are well formed. Such + including its bounds, and whose nested declarations are well formed. Such a configuration contains only what its TypedDict admits. So a rule - never receives a member of the wrong type, a member out of bounds, - or a member the type does not allow. No scope reads the nested - fields here, so the rules see none of them as read. A rule about a - nested field (for example, a struct field whose type has values of - varying size) finds nothing to check. `resolve` reads the field in a + never receives a field of the wrong type, a field out of bounds, + or a field the type does not allow. No scope reads the nested + declarations here, so the rules see none of them as read. A rule about a + nested declaration (for example, a struct field whose type has values of + varying size) finds nothing to check. `resolve` reads the declaration in a scope and runs every rule. """ configuration, problems = self._check_configuration(value, loc) @@ -381,10 +381,10 @@ def _malformed(definition: Definition[Any]) -> str | None: name = cast("object", definition.name) if not isinstance(name, str): return f"a definition's name is a string, got {name!r}" - for member in _function_members(type(definition)): - value = getattr(definition, member) + for attribute in _function_attributes(type(definition)): + value = getattr(definition, attribute) if not callable(value): - return f"{name!r}: {member} is a function, got {value!r}" + return f"{name!r}: {attribute} is a function, got {value!r}" kind = type(definition) filed, _ = kind._filed_under(name) bad = None if filed is None else kind._name_problem(name, ()) @@ -397,11 +397,11 @@ def _malformed(definition: Definition[Any]) -> str | None: @functools.cache -def _function_members(kind: type[Definition[Any]]) -> tuple[str, ...]: - """The members `kind` declares as functions: each member whose annotation is a `Callable`.""" +def _function_attributes(kind: type[Definition[Any]]) -> tuple[str, ...]: + """The attributes `kind` declares as functions: each attribute whose annotation is a `Callable`.""" return tuple( - member - for member, annotation in get_type_hints(kind).items() + attribute + for attribute, annotation in get_type_hints(kind).items() if get_origin(annotation) is Callable ) @@ -427,10 +427,10 @@ class WithFillValue(Definition[C]): so `fill_value_problems` can check a fill value for either format. `fill_value` is the JSON shape of a fill value, as an annotation. The - checker reads it the same way it reads a configuration's members, + checker reads it the same way it reads a configuration's fields, including any range. `fill_value_rules` yields the spec violations in a fill value of that shape that the type cannot express. It receives - the configuration, the nested fields as the scope read them, and the + the configuration, the nested declarations as the scope read them, and the typed fill value. `fill_value_canonical` writes a fill value without problems in the single canonical spelling of its value. So two fill values are the same value of the type exactly when their canonical @@ -454,8 +454,8 @@ def _refusal(self) -> str | None: @functools.cache def _fill_value_parser(annotation: object) -> Parser: - """The checker for a fill value's JSON shape, compiled once. Raises `TypeError` naming any part that no checker reads, or any metadata field in it.""" - return parser(annotation, _no_field) + """The checker for a fill value's JSON shape, compiled once. Raises `TypeError` naming any part that no checker reads, or any declaration in it.""" + return parser(annotation, _no_declaration) def kind_of(definition: object) -> type[Definition[Any]] | None: @@ -470,7 +470,7 @@ def _kind_of_class(cls: type[object]) -> type[Definition[Any]] | None: def format_of(kind: type[Definition[Any]]) -> Literal[2, 3]: - """The Zarr format of the documents that contain fields of `kind`, as declared by the kind. Raises `TypeError` for a class that is not a kind.""" + """The Zarr format of the documents that contain declarations of `kind`, as declared by the kind. Raises `TypeError` for a class that is not a kind.""" found = as_kind(kind)._format if found is None: # a kind cannot be declared without one msg = f"{kind!r} declares no format" @@ -495,10 +495,10 @@ def _kind_named(kind: object) -> type[Definition[Any]] | None: def as_kind(kind: object) -> type[Definition[Any]]: """The kind of metadata that `kind` refers to, with type arguments removed. Raises `TypeError` if it refers to no kind. - A scope files definitions by kind, so a field is read as a kind + A scope files definitions by kind, so a declaration is read as a kind (`CodecDefinition`, or `CodecDefinition[Any]`). It is never read as the base `Definition` or as a class that declares no kind, because - nothing is filed under those, and a field read as one would never be + nothing is filed under those, and a declaration read as one would never be checked. """ try: @@ -509,27 +509,27 @@ def as_kind(kind: object) -> type[Definition[Any]]: if found is not None: return found msg = ( - f"{kind!r} is not a kind of metadata; read a field as one of the kinds of a format, " + f"{kind!r} is not a kind of metadata; read a declaration as one of the kinds of a format, " "such as CodecDefinition" ) raise TypeError(msg) -def field_kind(annotation: object) -> type[Definition[Any]] | None: - """The kind of metadata field held by a member annotated with `annotation` (`CodecDefinition` for `ZarrV3CodecField`), or None if the member holds no field.""" +def declaration_kind(annotation: object) -> type[Definition[Any]] | None: + """The kind of declaration held by a field annotated with `annotation` (`CodecDefinition` for `ZarrV3CodecDeclaration`), or None if the field holds no declaration.""" if not is_alias(annotation): return None - return _FIELD_KINDS.get(annotation) + return _DECLARATION_KINDS.get(annotation) class Context(Protocol): """A scope: the definitions in scope while metadata of one Zarr format is read. - This is what reading a field needs from a scope: the Zarr format of the + This is what reading a declaration needs from a scope: the Zarr format of the documents it reads, and which definition of a kind claims a name. `ZarrV3Context` and `ZarrV2Context` implement it, each with one slot - per kind of field its format has, and so does any other object that - provides these two. `resolve` reads a field in any `Context`. + per kind of declaration its format has, and so does any other object that + provides these two. `resolve` reads a declaration in any `Context`. """ @property @@ -539,10 +539,10 @@ def claimant(self, kind: type[D], name: str) -> D | None: ... @dataclass(frozen=True, slots=True) -class _NestedField: - """A metadata field that the check found inside a configuration: its location, its kind, and its JSON. +class _NestedDeclaration: + """A declaration that the check found inside a configuration: its location, its kind, and its JSON. - The checker returns this in place of the JSON wherever a field alias + The checker returns this in place of the JSON wherever a declaration alias appears. `_checked` collects each one and puts its JSON back. It is carried in the value instead of being recorded separately, so a union branch that did not match leaves nothing behind. @@ -552,44 +552,44 @@ class _NestedField: kind: type[Definition[Any]] json: JSONValue alias: TypeAliasType - """Whether the member that holds it accepts only codecs of static size.""" + """Whether the field that holds it accepts only codecs of static size.""" -def _field(annotation: object) -> Parser | None: - """The one checker shape that this module defines: a member that holds a metadata field. +def _declaration(annotation: object) -> Parser | None: + """The one checker shape that this module defines: a field that holds a declaration. The value is checked only as a bare name or an object, and passed on. - Whoever reads the field (`check` without a scope, `resolve` with one) + Whoever reads the declaration (`check` without a scope, `resolve` with one) checks the envelope and maps the name to a definition. """ - kind = field_kind(annotation) + kind = declaration_kind(annotation) if kind is None or not is_alias(annotation): return None alias = annotation def parse(value: object, loc: Loc) -> Parsed: # The value is refined JSON, which the checker types only as - # `object`. If it is not a field at all, the kind's envelope check + # `object`. If it is not a declaration at all, the kind's envelope check # reports that. - return _NestedField(loc, kind, cast("JSONValue", value), alias), () + return _NestedDeclaration(loc, kind, cast("JSONValue", value), alias), () return parse def _vetting(annotation: object) -> Parser | None: - """`_field`, but rejecting annotations that a definition's configuration cannot contain and still be read. + """`_declaration`, but rejecting annotations that a definition's configuration cannot contain and still be read. - A member typed with the envelope's own TypedDict - (`ZarrV3MetadataFieldJSON`) is checked as plain JSON, so its name would + A field typed with the envelope's own TypedDict + (`ZarrV3DeclarationJSON`) is checked as plain JSON, so its name would never be mapped to a definition and its configuration would never be checked. A TypedDict that does not declare how undeclared keys are handled is open, so an undeclared key would never be reported. """ if annotation is ZarrV3NamedConfigJSON: msg = ( - "a member typed ZarrV3MetadataFieldJSON checks as plain JSON, and is never read as " - "a field; annotate it with the field alias of its kind: " - + ", ".join(alias.__name__ for alias in _FIELD_KINDS) + "a field typed ZarrV3DeclarationJSON checks as plain JSON, and is never read as " + "a declaration; annotate it with the declaration alias of its kind: " + + ", ".join(alias.__name__ for alias in _DECLARATION_KINDS) ) raise TypeError(msg) if ( @@ -604,14 +604,14 @@ def _vetting(annotation: object) -> Parser | None: "on a typing_extensions.TypedDict" ) raise TypeError(msg) - return _field(annotation) + return _declaration(annotation) -def _no_field(annotation: object) -> Parser | None: - """A leaf checker that rejects a field alias. A fill value is a value of its data type and never contains a metadata field.""" - if is_alias(annotation) and field_kind(annotation) is not None: +def _no_declaration(annotation: object) -> Parser | None: + """A leaf checker that rejects a declaration alias. A fill value is a value of its data type and never contains a declaration.""" + if is_alias(annotation) and declaration_kind(annotation) is not None: name = annotation.__name__ - msg = f"{name} holds a metadata field, and a fill value is a value of its data type" + msg = f"{name} holds a declaration, and a fill value is a value of its data type" raise TypeError(msg) return None @@ -626,14 +626,14 @@ def _vet(configuration: type, zarr_format: int | None) -> None: raise TypeError(msg) def vetting(annotation: object) -> Parser | None: - # A document of one format never holds a field of the other, so a + # A document of one format never holds a declaration of the other, so a # definition never reads one. - kind = field_kind(annotation) + kind = declaration_kind(annotation) if kind is not None and zarr_format is not None and format_of(kind) != zarr_format: msg = ( - f"a member typed {getattr(annotation, '__name__', annotation)} holds a field of " + f"a field typed {getattr(annotation, '__name__', annotation)} holds a declaration of " f"a Zarr v{format_of(kind)} document, which a definition of a v{zarr_format} " - "field never reads" + "declaration never reads" ) raise TypeError(msg) return _vetting(annotation) @@ -649,7 +649,7 @@ def vetting(annotation: object) -> Parser | None: @dataclass(frozen=True, slots=True) class _Checker: - """A TypedDict's checker, compiled once, and whether a value of the TypedDict can contain a nested field.""" + """A TypedDict's checker, compiled once, and whether a value of the TypedDict can contain a nested declaration.""" parse: Parser nests: bool @@ -661,7 +661,7 @@ def _checker(shape: type) -> _Checker: met: list[object] = [] def leaf(annotation: object) -> Parser | None: - found = _field(annotation) + found = _declaration(annotation) if found is not None: met.append(annotation) return found @@ -671,28 +671,28 @@ def leaf(annotation: object) -> Parser | None: def _checked( shape: type, value: object, loc: Loc -) -> tuple[object, Problems, tuple[_NestedField, ...]]: - """Check `value` as `shape`. Return the typed value (with each nested field restored as written), every problem, and the nested fields in order.""" +) -> tuple[object, Problems, tuple[_NestedDeclaration, ...]]: + """Check `value` as `shape`. Return the typed value (with each nested declaration restored as written), every problem, and the nested declarations in order.""" typed, found, nested = _typed(shape, value, loc) return _put_back(typed, _as_written, nested), found, nested def _typed( shape: type, value: object, loc: Loc -) -> tuple[object, Problems, tuple[_NestedField, ...]]: - """Check `value` as `shape`, leaving each nested field where the checker found it. Return the typed value, every problem, and the nested fields in order.""" +) -> tuple[object, Problems, tuple[_NestedDeclaration, ...]]: + """Check `value` as `shape`, leaving each nested declaration where the checker found it. Return the typed value, every problem, and the nested declarations in order.""" checker = _checker(shape) typed, found = checker.parse(value, loc) if not checker.nests: return typed, found, () - nested: list[_NestedField] = [] + nested: list[_NestedDeclaration] = [] _collect(typed, nested) return typed, found, tuple(nested) -def _collect(value: object, nested: list[_NestedField]) -> None: - """Append to `nested` each nested field that the checker returned inside `value`, in order.""" - if isinstance(value, _NestedField): +def _collect(value: object, nested: list[_NestedDeclaration]) -> None: + """Append to `nested` each nested declaration that the checker returned inside `value`, in order.""" + if isinstance(value, _NestedDeclaration): nested.append(value) elif is_tuple(value): for entry in value: @@ -702,31 +702,33 @@ def _collect(value: object, nested: list[_NestedField]) -> None: _collect(entry, nested) -def _as_written(field: _NestedField) -> JSONValue: - """A nested field restored as it was written.""" - return field.json +def _as_written(declaration: _NestedDeclaration) -> JSONValue: + """A nested declaration restored as it was written.""" + return declaration.json -def _declared(field: _NestedField) -> JSONValue: - """A nested field restored as it was written, except for envelope members that are not declared. Those are reported and left out, in the same way the checker leaves out a key that a closed TypedDict does not declare.""" - if not isinstance(field.json, Mapping): - return field.json - envelope = cast("Mapping[str, JSONValue]", field.json) +def _declared(declaration: _NestedDeclaration) -> JSONValue: + """A nested declaration restored as it was written, except for envelope fields that are not declared. Those are reported and left out, in the same way the checker leaves out a key that a closed TypedDict does not declare.""" + if not isinstance(declaration.json, Mapping): + return declaration.json + envelope = cast("Mapping[str, JSONValue]", declaration.json) return {key: value for key, value in envelope.items() if key in ENVELOPE_KEYS} def _put_back( - value: object, put: Callable[[_NestedField], JSONValue], nested: Sequence[_NestedField] + value: object, + put: Callable[[_NestedDeclaration], JSONValue], + nested: Sequence[_NestedDeclaration], ) -> object: - """Replace each of `nested` (the fields the checker returned inside `value`) with the result of `put`. Return `value` unchanged when there are none.""" + """Replace each of `nested` (the declarations the checker returned inside `value`) with the result of `put`. Return `value` unchanged when there are none.""" if len(nested) == 0: return value return _putting_back(value, put) -def _putting_back(value: object, put: Callable[[_NestedField], JSONValue]) -> object: - """`_put_back` for a value that contains nested fields: a recursive walk.""" - if isinstance(value, _NestedField): +def _putting_back(value: object, put: Callable[[_NestedDeclaration], JSONValue]) -> object: + """`_put_back` for a value that contains nested declarations: a recursive walk.""" + if isinstance(value, _NestedDeclaration): return put(value) if is_tuple(value): return tuple(_putting_back(entry, put) for entry in value) @@ -741,13 +743,13 @@ def _usable(problems: Sequence[ValidationProblem]) -> bool: def read_only(configuration: C) -> C: - """`configuration` in the form a definition's functions receive it: a read-only view of the field's own configuration, so a function that assigns a member fails. + """`configuration` in the form a definition's functions receive it: a read-only view of the declaration's own configuration, so a function that assigns a field fails. - This function makes only the top-level members read-only. The value a - member holds (for example, an object in a struct's `fields`) is passed + This function makes only the top-level fields read-only. The value a + field holds (for example, an object in a struct's `fields`) is passed as is. While a configuration is being read, that value can be mutable, and a function that writes into it changes the configuration. An - accepted field's configuration is already read-only at every level. + accepted declaration's configuration is already read-only at every level. """ return cast("C", MappingProxyType(cast("Mapping[str, JSONValue]", configuration))) @@ -803,21 +805,21 @@ def _located(prefix: Loc, problems: Iterable[ValidationProblem]) -> Problems: return tuple(dataclasses.replace(each, loc=(*prefix, *each.loc)) for each in found) -def _envelope(field: _NestedField) -> Problems: - """The problems with a nested field's envelope, located at the field.""" - return _located(field.loc, field.kind._envelope_problems(field.json)) +def _envelope(declaration: _NestedDeclaration) -> Problems: + """The problems with a nested declaration's envelope, located at the declaration.""" + return _located(declaration.loc, declaration.kind._envelope_problems(declaration.json)) def _configuration_checked( value: object, shape: type[T], loc: Loc = (), *, finite: bool = True ) -> tuple[T | None, Problems]: - """Check `value` as `shape`, as `typed_json.check` does, and check the envelope of each nested field. + """Check `value` as `shape`, as `typed_json.check` does, and check the envelope of each nested declaration. - A definition's `read_configuration` starts with this step. A member - typed with a field alias holds a metadata field, and its envelope is + A definition's `read_configuration` starts with this step. A field + typed with a declaration alias holds a declaration, and its envelope is checked the same way as in a document. A `must_understand` of `false` is a problem of the configuration, and no value is returned. An extra - member is an unknown key: it is reported and left out, and the value is + field is an unknown key: it is reported and left out, and the value is still returned. This matches how the checker reports and leaves out a key that a closed TypedDict does not declare. A number that is not finite is a problem unless `finite` is false, for a configuration that @@ -828,7 +830,7 @@ def _configuration_checked( if len(problems) != 0: return None, problems typed, found, nested = _typed(shape, refined, loc) - problems = (*found, *(problem for field in nested for problem in _envelope(field))) + problems = (*found, *(problem for declaration in nested for problem in _envelope(declaration))) if not _usable(problems): return None, problems return cast("T", _put_back(typed, _declared, nested)), problems @@ -837,10 +839,10 @@ def _configuration_checked( def named_configuration( value: object, ) -> tuple[str | None, Mapping[str, object] | None, Problems]: - """Split a metadata field into `(name, configuration, problems)`. + """Split a declaration into `(name, configuration, problems)`. - A field is a bare name, or an object that contains a name. A `None` - name means the value is not a metadata field at all. A `None` + A declaration is a bare name, or an object that contains a name. A `None` + name means the value is not a declaration at all. A `None` configuration means the bare spelling was used, or the key was left out. The only problem reported is a configuration that is present but not an object, at `("configuration",)`. @@ -866,80 +868,80 @@ def named_configuration( def _nothing_nested() -> Nested: - """The nested fields of a field that contains no read fields: an empty mapping.""" + """The nested declarations of a declaration that contains no read declarations: an empty mapping.""" return {} @dataclass(frozen=True, slots=True, kw_only=True) -class AcceptedField(Generic[D]): - """A field that a definition in scope read: the name it is written with, the definition, and the configuration the definition allowed. - - A problem with the envelope (an extra member, a `must_understand` of - `false`) is reported with the field, and the field is still read. The - same applies to a problem in a nested field, which belongs to that - nested field, as `nested` shows. Two fields are equal when they read - the same, however each was spelled, as compared by `field_key`. For - example, `"bytes"` and `{"name": "bytes"}` are one field. So are a +class AcceptedDeclaration(Generic[D]): + """A declaration that a definition in scope read: the name it is written with, the definition, and the configuration the definition allowed. + + A problem with the envelope (an extra field, a `must_understand` of + `false`) is reported with the declaration, and the declaration is still read. The + same applies to a problem in a nested declaration, which belongs to that + nested declaration, as `nested` shows. Two declarations are equal when they read + the same, however each was spelled, as compared by `declaration_key`. For + example, `"bytes"` and `{"name": "bytes"}` are one declaration. So are a blosc with and without the `typesize` that `noshuffle` ignores, because - the definition's `canonical` removes it. Equal fields have equal + the definition's `canonical` removes it. Equal declarations have equal hashes. """ json: JSONValue - """The field as written, refined (arrays as tuples). It is not used for equality.""" + """The declaration as written, refined (arrays as tuples). It is not used for equality.""" name: str - """The name the field is written with, for example `"r16"`, even though its definition is filed under `r*`.""" + """The name the declaration is written with, for example `"r16"`, even though its definition is filed under `r*`.""" definition: D - """The definition that read the field.""" + """The definition that read the declaration.""" configuration: Mapping[str, JSONValue] """The configuration, type-checked and allowed by the rules. For raw bits, this is the configuration encoded in the name. - Each nested field is written as that field's `to_json` writes it, so + Each nested declaration is written as that declaration's `to_json` writes it, so the configuration shows what was read, however it was spelled. For example, a shard's `"crc32c"` and `{"name": "crc32c"}` are the same index codec. """ nested: Nested = dataclasses.field(default_factory=_nothing_nested) - """The fields in the configuration, each as the scope read it, keyed by location in the configuration. + """The declarations in the configuration, each as the scope read it, keyed by location in the configuration. For example, a struct's field types are at `("fields", 0, "data_type")` and a shard's codecs are at `("codecs", 0)`. A definition's functions - use this to inspect the fields inside its own configuration. + use this to inspect the declarations inside its own configuration. """ read_as: type[Definition[Any]] = dataclasses.field(init=False, repr=False) - """The kind of metadata the field was read as: the kind of its definition.""" + """The kind of metadata the declaration was read as: the kind of its definition.""" def __eq__(self, other: object) -> bool: - if not is_field(other): + if not is_declaration(other): return NotImplemented - return field_key(self) == field_key(other) + return declaration_key(self) == declaration_key(other) def __hash__(self) -> int: - return hash(field_key(self)) + return hash(declaration_key(self)) def __post_init__(self) -> None: - # Check the annotations at runtime. A field built by hand (an + # Check the annotations at runtime. A declaration built by hand (an # extension may do this) fails here, not later in a function that # trusts it. definition = cast("object", self.definition) kind = kind_of(definition) refusal = ( - f"a field read is read by a definition of a kind, got {definition!r}" + f"a declaration read is read by a definition of a kind, got {definition!r}" if kind is None else _misread(definition, kind, self.name) ) if refusal is not None: raise TypeError(refusal) object.__setattr__(self, "read_as", kind) - # Everything the field returns is read-only at every level, so a - # field cannot reach a state where its key, `==` and `refines` + # Everything the declaration returns is read-only at every level, so a + # declaration cannot reach a state where its key, `==` and `refines` # disagree. object.__setattr__(self, "json", frozen(self.json)) object.__setattr__(self, "configuration", frozen(self.configuration)) object.__setattr__(self, "nested", MappingProxyType(dict(self.nested))) - def __reduce__(self) -> tuple[Callable[..., AcceptedField[Any]], tuple[object, ...]]: - # Read-only views cannot be pickled, so pickle the field from its constructor arguments. + def __reduce__(self) -> tuple[Callable[..., AcceptedDeclaration[Any]], tuple[object, ...]]: + # Read-only views cannot be pickled, so pickle the declaration from its constructor arguments. return _accepted_field, ( copied(self.json), self.name, @@ -949,11 +951,11 @@ def __reduce__(self) -> tuple[Callable[..., AcceptedField[Any]], tuple[object, . ) def to_json(self) -> JSONValue: - """The field as a document writes it, in a form every reader accepts. The configuration is as read, and the result shares no objects with the field. + """The declaration as a document writes it, in a form every reader accepts. The configuration is as read, and the result shares no objects with the declaration. The envelope uses the shortest form every reader accepts. A data type with nothing to configure is written as its bare name, as core - data types have been written since Zarr v3.0. Any other field is an + data types have been written since Zarr v3.0. Any other declaration is an object, `{"name": ...}`, because a Zarr v3.0 reader does not accept a bare name in `codecs` (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L585-L592). @@ -964,31 +966,31 @@ def to_json(self) -> JSONValue: @dataclass(frozen=True, slots=True, kw_only=True) -class UnclaimedField: - """A field that nothing in scope claims: an extension that the scope does not check. Accepting such fields is what keeps the format open. +class UnclaimedDeclaration: + """A declaration that nothing in scope claims: an extension that the scope does not check. Accepting such declarations is what keeps the format open. - It is equal to another unclaimed field when both are of the same kind, + It is equal to another unclaimed declaration when both are of the same kind, have the same name, and a document writes the same JSON for both. - Nothing in scope interprets the field, so it is compared as JSON text, - as `field_key` defines. + Nothing in scope interprets the declaration, so it is compared as JSON text, + as `declaration_key` defines. """ json: JSONValue - """The field as written, refined (arrays as tuples). It is not used for equality.""" + """The declaration as written, refined (arrays as tuples). It is not used for equality.""" name: str """The name that nothing in scope claims.""" read_as: type[Definition[Any]] - """The kind of metadata the field was read as: the kind a claiming definition would have.""" + """The kind of metadata the declaration was read as: the kind a claiming definition would have.""" configuration: Mapping[str, JSONValue] = dataclasses.field(init=False) """The configuration as written, not checked; empty when no configuration is written.""" def __eq__(self, other: object) -> bool: - if not is_field(other): + if not is_declaration(other): return NotImplemented - return field_key(self) == field_key(other) + return declaration_key(self) == declaration_key(other) def __hash__(self) -> int: - return hash(field_key(self)) + return hash(declaration_key(self)) def __post_init__(self) -> None: # Check the annotations at runtime. Remove type arguments from @@ -997,7 +999,7 @@ def __post_init__(self) -> None: name = cast("object", self.name) if not isinstance(name, str) or not self.read_as._well_named(name): msg = ( - "a field nothing in scope claims is named as a document names a " + "a declaration nothing in scope claims is named as a document names a " f"{self.read_as.label}, got {name!r}" ) raise TypeError(msg) @@ -1008,47 +1010,47 @@ def __post_init__(self) -> None: self, "configuration", frozen(cast("Mapping[str, JSONValue]", configuration)) ) - def __reduce__(self) -> tuple[Callable[..., UnclaimedField], tuple[object, ...]]: - # Read-only views cannot be pickled, so pickle the field from its constructor arguments. + def __reduce__(self) -> tuple[Callable[..., UnclaimedDeclaration], tuple[object, ...]]: + # Read-only views cannot be pickled, so pickle the declaration from its constructor arguments. return _unclaimed_field, (copied(self.json), self.name, self.read_as) @property def definition(self) -> None: - """The definition that read the field. Always None, because no definition read it.""" + """The definition that read the declaration. Always None, because no definition read it.""" return None @property def nested(self) -> Nested: - """The nested fields as the scope read them. Always empty, because nothing read the configuration.""" + """The nested declarations as the scope read them. Always empty, because nothing read the configuration.""" return _nothing_nested() def to_json(self) -> JSONValue: - """The field as a document writes it, sharing no objects with the field. The configuration is as written, in the envelope form every reader accepts, as `AcceptedField.to_json` writes it.""" + """The declaration as a document writes it, sharing no objects with the declaration. The configuration is as written, in the envelope form every reader accepts, as `AcceptedDeclaration.to_json` writes it.""" return copied(written_json(self)) @dataclass(frozen=True, slots=True, kw_only=True) -class RefusedField(Generic[D]): - """A field that could not be read: it is not a field at all, not JSON, or rejected by the definition that claims its name. Its problems explain why.""" +class RefusedDeclaration(Generic[D]): + """A declaration that could not be read: it is not a declaration at all, not JSON, or rejected by the definition that claims its name. Its problems explain why.""" json: JSONValue | UNSET - """The field as written, refined (arrays as tuples). `UNSET` when it is not JSON, which no document can contain.""" + """The declaration as written, refined (arrays as tuples). `UNSET` when it is not JSON, which no document can contain.""" name: str | None - """The name the field is written with; None when it has no name.""" + """The name the declaration is written with; None when it has no name.""" read_as: type[Definition[Any]] - """The kind of metadata the field was read as.""" + """The kind of metadata the declaration was read as.""" definition: D | None = None - """The definition that claims the name and rejected the field; None when nothing in scope claims it, or when the field has no name.""" + """The definition that claims the name and rejected the declaration; None when nothing in scope claims it, or when the declaration has no name.""" nested: Nested = dataclasses.field(default_factory=_nothing_nested) - """The nested fields, each as the scope read it; empty when the configuration was not checked against its TypedDict.""" + """The nested declarations, each as the scope read it; empty when the configuration was not checked against its TypedDict.""" def __eq__(self, other: object) -> bool: - if not is_field(other): + if not is_declaration(other): return NotImplemented - return field_key(self) == field_key(other) + return declaration_key(self) == declaration_key(other) def __hash__(self) -> int: - return hash(field_key(self)) + return hash(declaration_key(self)) def __post_init__(self) -> None: # Check the annotations at runtime. Remove type arguments from @@ -1063,8 +1065,8 @@ def __post_init__(self) -> None: object.__setattr__(self, "json", frozen(self.json)) object.__setattr__(self, "nested", MappingProxyType(dict(self.nested))) - def __reduce__(self) -> tuple[Callable[..., RefusedField[Any]], tuple[object, ...]]: - # Read-only views cannot be pickled, so pickle the field from its constructor arguments. + def __reduce__(self) -> tuple[Callable[..., RefusedDeclaration[Any]], tuple[object, ...]]: + # Read-only views cannot be pickled, so pickle the declaration from its constructor arguments. return _refused_field, ( UNSET if self.json is UNSET else copied(self.json), self.name, @@ -1074,28 +1076,34 @@ def __reduce__(self) -> tuple[Callable[..., RefusedField[Any]], tuple[object, .. ) -ResolvedField = TypeAliasType( - "ResolvedField", "AcceptedField[D] | UnclaimedField | RefusedField[D]", type_params=(D,) +ResolvedDeclaration = TypeAliasType( + "ResolvedDeclaration", + "AcceptedDeclaration[D] | UnclaimedDeclaration | RefusedDeclaration[D]", + type_params=(D,), ) -"""One metadata field as a scope read it: accepted by the definition that claims its name, unclaimed, or refused.""" +"""One declaration as a scope read it: accepted by the definition that claims its name, unclaimed, or refused.""" +# The three rebuild functions keep their `_field` names: a pickle looks them +# up by name, so renaming them would break existing pickles. def _accepted_field( json: JSONValue, name: str, definition: Definition[Any], configuration: Mapping[str, JSONValue], nested: Nested, -) -> AcceptedField[Any]: - """Rebuild an accepted field from its pickled arguments.""" - return AcceptedField( +) -> AcceptedDeclaration[Any]: + """Rebuild an accepted declaration from its pickled arguments.""" + return AcceptedDeclaration( json=json, name=name, definition=definition, configuration=configuration, nested=nested ) -def _unclaimed_field(json: JSONValue, name: str, read_as: type[Definition[Any]]) -> UnclaimedField: - """Rebuild an unclaimed field from its pickled arguments.""" - return UnclaimedField(json=json, name=name, read_as=read_as) +def _unclaimed_field( + json: JSONValue, name: str, read_as: type[Definition[Any]] +) -> UnclaimedDeclaration: + """Rebuild an unclaimed declaration from its pickled arguments.""" + return UnclaimedDeclaration(json=json, name=name, read_as=read_as) def _refused_field( @@ -1104,47 +1112,51 @@ def _refused_field( read_as: type[Definition[Any]], definition: Definition[Any] | None, nested: Nested, -) -> RefusedField[Any]: - """Rebuild a refused field from its pickled arguments.""" - return RefusedField(json=json, name=name, read_as=read_as, definition=definition, nested=nested) +) -> RefusedDeclaration[Any]: + """Rebuild a refused declaration from its pickled arguments.""" + return RefusedDeclaration( + json=json, name=name, read_as=read_as, definition=definition, nested=nested + ) -_FIELDS: Final = (AcceptedField, UnclaimedField, RefusedField) -"""The three possible results of reading a field in a scope.""" +_DECLARATIONS: Final = (AcceptedDeclaration, UnclaimedDeclaration, RefusedDeclaration) +"""The three possible results of reading a declaration in a scope.""" def _misread(definition: object, kind: type[Definition[Any]], name: object) -> str | None: - """A description of why `definition` cannot be the definition that read a field named `name` as a `kind`; None when it can.""" + """A description of why `definition` cannot be the definition that read a declaration named `name` as a `kind`; None when it can.""" if not isinstance(definition, kind): - return f"a field read as a {kind.__name__} is read by one, got {definition!r}" + return f"a declaration read as a {kind.__name__} is read by one, got {definition!r}" filed = definition.name if not isinstance(name, str) or filed_under(kind, name)[0] != filed: - return f"a field named {name!r} is read by the definition filed under it, got {filed!r}" + return ( + f"a declaration named {name!r} is read by the definition filed under it, got {filed!r}" + ) return None -def field_key(field: ResolvedField[Any]) -> tuple[object, ...]: - """The key that `==` and `hash` compare for a field. It represents what the field means, not how it was spelled. +def declaration_key(declaration: ResolvedDeclaration[Any]) -> tuple[object, ...]: + """The key that `==` and `hash` compare for a declaration. It represents what the declaration means, not how it was spelled. - An accepted field is compared by the definition that read it, by its + An accepted declaration is compared by the definition that read it, by its configuration in the definition's canonical spelling (as `canonical_of` writes it) as JSON text, and by the key of each nested - field. An unclaimed field is compared by its kind, its name and the + declaration. An unclaimed declaration is compared by its kind, its name and the JSON a document writes for it, as JSON text, because nothing interprets it. - A refused field is compared by what was written and by the definition - that rejected it. So two fields are equal when the parts a reader + A refused declaration is compared by what was written and by the definition + that rejected it. So two declarations are equal when the parts a reader understands read the same, and the parts no reader interprets are written the same way. """ - if isinstance(field, AcceptedField): - definition = cast("Definition[Any]", field.definition) - configuration: JSONValue = dict(field.configuration) - # Each nested field is compared by its own key, so its location is + if isinstance(declaration, AcceptedDeclaration): + definition = cast("Definition[Any]", declaration.definition) + configuration: JSONValue = dict(declaration.configuration) + # Each nested declaration is compared by its own key, so its location is # cleared before the definition's `canonical` sees the rest, in the - # same order as `_canonical_field`. `canonical` simplifies only the - # definition's own members. - for loc in field.nested: + # same order as `_canonical_declaration`. `canonical` simplifies only the + # definition's own fields. + for loc in declaration.nested: configuration = _replaced(configuration, loc, None) view = read_only(cast("Mapping[str, JSONValue]", configuration)) spelled = asked( @@ -1152,144 +1164,154 @@ def field_key(field: ResolvedField[Any]) -> tuple[object, ...]: "canonical", lambda: json_text(cast("JSONValue", dict(definition.canonical(view)))), ) - return ("read", definition, spelled, _nested_key(field.nested)) - if isinstance(field, UnclaimedField): - return ("unclaimed", field.read_as, field.name, json_text(written_json(field))) + return ("read", definition, spelled, _nested_key(declaration.nested)) + if isinstance(declaration, UnclaimedDeclaration): + return ( + "unclaimed", + declaration.read_as, + declaration.name, + json_text(written_json(declaration)), + ) return ( "refused", - field.read_as, - field.name, - field.definition, - UNSET if field.json is UNSET else json_text(field.json), - _nested_key(field.nested), + declaration.read_as, + declaration.name, + declaration.definition, + UNSET if declaration.json is UNSET else json_text(declaration.json), + _nested_key(declaration.nested), ) -def own_key(field: AcceptedField[Any]) -> tuple[object, ...]: - """The part of `field_key` for an accepted field that excludes nested fields: the definition and the canonical spelling of its own members.""" - return field_key(field)[:3] +def own_key(declaration: AcceptedDeclaration[Any]) -> tuple[object, ...]: + """The part of `declaration_key` for an accepted declaration that excludes nested declarations: the definition and the canonical spelling of its own fields.""" + return declaration_key(declaration)[:3] def _nested_key(nested: Nested) -> tuple[tuple[Loc, tuple[object, ...]], ...]: - """The key of each nested field, paired with its location.""" - return tuple((loc, field_key(inner)) for loc, inner in nested.items()) + """The key of each nested declaration, paired with its location.""" + return tuple((loc, declaration_key(inner)) for loc, inner in nested.items()) -def document_json(field: ResolvedField[Any]) -> JSONValue | UNSET: - """A field as a document writes it, using the field's own values: as `to_json` writes it, or as originally written for a refused field. `UNSET` for a refused field that was not JSON. +def document_json(declaration: ResolvedDeclaration[Any]) -> JSONValue | UNSET: + """A declaration as a document writes it, using the declaration's own values: as `to_json` writes it, or as originally written for a refused declaration. `UNSET` for a refused declaration that was not JSON. A writer serializes this without changing it, so it is not copied. `to_json` is a copy of this. """ - if isinstance(field, RefusedField): - return field.json - return written_json(field) + if isinstance(declaration, RefusedDeclaration): + return declaration.json + return written_json(declaration) -def written_json(field: AcceptedField[Any] | UnclaimedField) -> JSONValue: - """A field that a scope accepted or left unclaimed, as a document writes it. This is `document_json` for a field that is JSON.""" - kind = field.read_as - if isinstance(field, UnclaimedField): - return kind._unclaimed_json(field.json, field.name, field.configuration) - if kind._filed_under(field.name)[1] is not None: - return kind._envelope_json(field.name, {}) - return kind._envelope_json(field.name, field.configuration) +def written_json(declaration: AcceptedDeclaration[Any] | UnclaimedDeclaration) -> JSONValue: + """A declaration that a scope accepted or left unclaimed, as a document writes it. This is `document_json` for a declaration that is JSON.""" + kind = declaration.read_as + if isinstance(declaration, UnclaimedDeclaration): + return kind._unclaimed_json(declaration.json, declaration.name, declaration.configuration) + if kind._filed_under(declaration.name)[1] is not None: + return kind._envelope_json(declaration.name, {}) + return kind._envelope_json(declaration.name, declaration.configuration) -Nested: TypeAlias = Mapping[Loc, ResolvedField[Any]] -"""The fields in a configuration, each as the scope read it, keyed by location in the configuration.""" +Nested: TypeAlias = Mapping[Loc, ResolvedDeclaration[Any]] +"""The declarations in a configuration, each as the scope read it, keyed by location in the configuration.""" -def is_field(value: object) -> TypeGuard[ResolvedField[Any]]: - """Whether `value` is a field that a scope read: `AcceptedField`, `UnclaimedField` or `RefusedField`.""" - return isinstance(value, _FIELDS) +def is_declaration(value: object) -> TypeGuard[ResolvedDeclaration[Any]]: + """Whether `value` is a declaration that a scope read: `AcceptedDeclaration`, `UnclaimedDeclaration` or `RefusedDeclaration`.""" + return isinstance(value, _DECLARATIONS) -def held(field: ResolvedField[D] | UNSET) -> AcceptedField[D] | UnclaimedField: - """Return `field`, which must come from a read that found no problems, as `AcceptedField` or `UnclaimedField`. Raises `TypeError` for a refused field or one never read, which such a read cannot produce.""" - if isinstance(field, (AcceptedField, UnclaimedField)): - return field - msg = f"expected a field a read found nothing wrong with, got {field!r}" +def held( + declaration: ResolvedDeclaration[D] | UNSET, +) -> AcceptedDeclaration[D] | UnclaimedDeclaration: + """Return `declaration`, which must come from a read that found no problems, as `AcceptedDeclaration` or `UnclaimedDeclaration`. Raises `TypeError` for a refused declaration or one never read, which such a read cannot produce.""" + if isinstance(declaration, (AcceptedDeclaration, UnclaimedDeclaration)): + return declaration + msg = f"expected a declaration a read found nothing wrong with, got {declaration!r}" raise TypeError(msg) -def fields_of( - resolved: ResolvedField[Any], loc: Loc = () -) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Yield `resolved` (a field a scope read) with its location, then each nested field recursively, each with its location. +def declarations_of( + resolved: ResolvedDeclaration[Any], loc: Loc = () +) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Yield `resolved` (a declaration a scope read) with its location, then each nested declaration recursively, each with its location. - `loc` is the location of `resolved`. A nested field is located where + `loc` is the location of `resolved`. A nested declaration is located where the document writes its place in the configuration, as the kind's `_in_configuration` says, the same way `resolve` locates problems. - Each field's nested fields are its `nested`. A field whose + Each declaration's nested declarations are its `nested`. A declaration whose configuration was not checked has none. """ yield loc, resolved for place, inner in resolved.nested.items(): - yield from fields_of(inner, resolved.read_as._in_configuration(loc, place)) + yield from declarations_of(inner, resolved.read_as._in_configuration(loc, place)) def with_problems( - fields: Iterable[tuple[Loc, ResolvedField[Any]]], problems: Sequence[ValidationProblem] -) -> Iterator[tuple[Loc, ResolvedField[Any], Problems]]: - """Yield each of `fields` with its location and the problems in `problems` located inside it, including inside its nested fields. - - `fields` and `problems` must come from the same read: a reading's - `fields()` and `problems`, or `fields_of` a field and the problems that - `resolve` returned with it. A field's problems include those found when + declarations: Iterable[tuple[Loc, ResolvedDeclaration[Any]]], + problems: Sequence[ValidationProblem], +) -> Iterator[tuple[Loc, ResolvedDeclaration[Any], Problems]]: + """Yield each of `declarations` with its location and the problems in `problems` located inside it, including inside its nested declarations. + + `declarations` and `problems` must come from the same read: a reading's + `declarations()` and `problems`, or `declarations_of` a declaration and the problems that + `resolve` returned with it. A declaration's problems include those found when it was read, and those the document found with it in its position (its place in the pipeline, the chunk it receives, the array's shape). So a - field with no problems is valid in that position, and `canonical_of` + declaration with no problems is valid in that position, and `canonical_of` can write it. This groups the problems at every depth, as zod's `treeifyError` groups issues: a problem with a shard's inner codec - belongs to the inner codec and to the shard. Each field comes before - its nested fields, in the order of `fields_of`, so the last field that - lists a problem is the innermost field containing it. A problem - outside every field (with the fill value or the shape) is not listed - under any field. + belongs to the inner codec and to the shard. Each declaration comes before + its nested declarations, in the order of `declarations_of`, so the last declaration that + lists a problem is the innermost declaration containing it. A problem + outside every declaration (with the fill value or the shape) is not listed + under any declaration. """ - located = list(fields) + located = list(declarations) held: dict[Loc, list[ValidationProblem]] = {loc: [] for loc, _ in located} for found in problems: for depth in range(len(found.loc) + 1): holder = held.get(found.loc[:depth]) if holder is not None: holder.append(found) - for loc, field in located: - yield loc, field, tuple(held[loc]) + for loc, declaration in located: + yield loc, declaration, tuple(held[loc]) -def configuration_of(resolved: ResolvedField[Any], definition: Definition[C]) -> C | None: +def configuration_of(resolved: ResolvedDeclaration[Any], definition: Definition[C]) -> C | None: """The configuration of `resolved`, typed as `definition` declares it, if `definition` accepted it. - `AcceptedField` stores its configuration as a read-only mapping. When - called with the definition that read the field, or one equal to it - (as with a pickled field), this returns a copy of it, a plain `dict` - typed as the TypedDict, which shares nothing with the field. Returns - None for any other field: one that another definition read, one that + `AcceptedDeclaration` stores its configuration as a read-only mapping. When + called with the definition that read the declaration, or one equal to it + (as with a pickled declaration), this returns a copy of it, a plain `dict` + typed as the TypedDict, which shares nothing with the declaration. Returns + None for any other declaration: one that another definition read, one that no definition claims, and one that `definition` claimed and refused. """ - if not isinstance(resolved, AcceptedField) or resolved.definition != definition: + if not isinstance(resolved, AcceptedDeclaration) or resolved.definition != definition: return None return cast("C", copied(resolved.configuration)) -def fill_value_problems(data_type: ResolvedField[F], value: object, loc: Loc = ()) -> Problems: - """The problems with `value` as a fill value of `data_type` (a data type field a scope read). +def fill_value_problems( + data_type: ResolvedDeclaration[F], value: object, loc: Loc = () +) -> Problems: + """The problems with `value` as a fill value of `data_type` (a data type declaration a scope read). `value` is refined to JSON first. If it is not JSON, that is reported first, whatever the data type. It is then checked against the JSON shape that the data type's definition declares, and by the fill value rules, in the same way `read_configuration` reads a configuration: a key the shape does not declare is reported and left out, and the rules - still check the rest. The rules see the nested fields of the + still check the rest. The rules see the nested declarations of the configuration as the scope read them, so a struct checks each field's fill value against that field's own type. If the scope did not read the data type (it is out of scope or invalid), a JSON fill value is not checked. `loc` prefixes every problem. """ refined, problems = refine_json(value, loc) - if len(problems) != 0 or not isinstance(data_type, AcceptedField): + if len(problems) != 0 or not isinstance(data_type, AcceptedDeclaration): return with_input(problems, value, loc) definition, configuration = data_type.definition, data_type.configuration typed, problems = _fill_value_parser(definition.fill_value)(refined, loc) @@ -1303,8 +1325,8 @@ def fill_value_problems(data_type: ResolvedField[F], value: object, loc: Loc = ( return with_input((*problems, *refused), value, loc) -def canonical_fill_value(data_type: ResolvedField[F], value: object) -> JSONValue | UNSET: - """`value` (a fill value of `data_type`, a data type field a scope read) in the single canonical spelling of its value; `UNSET` when it has a problem. +def canonical_fill_value(data_type: ResolvedDeclaration[F], value: object) -> JSONValue | UNSET: + """`value` (a fill value of `data_type`, a data type declaration a scope read) in the single canonical spelling of its value; `UNSET` when it has a problem. The spelling comes from the data type's `fill_value_canonical`. Two fill values of a data type are the same value exactly when their @@ -1312,7 +1334,7 @@ def canonical_fill_value(data_type: ResolvedField[F], value: object) -> JSONValu `json.dumps` writes it. `==` is not the same test, because it treats `-0.0` as equal to `0.0`. A fill value with a problem found by `fill_value_problems` has no canonical spelling, just as `canonical_of` - returns none for a field with a problem. The result is `UNSET` and not + returns none for a declaration with a problem. The result is `UNSET` and not `None`, because `None` is the JSON `null`, which is a valid fill value for a data type the scope did not read (such a data type keeps a fill value as written). @@ -1323,14 +1345,14 @@ def canonical_fill_value(data_type: ResolvedField[F], value: object) -> JSONValu return spelled_canonically(data_type, refined) -def spelled_canonically(data_type: ResolvedField[F], value: JSONValue) -> JSONValue: +def spelled_canonically(data_type: ResolvedDeclaration[F], value: JSONValue) -> JSONValue: """`value` (a fill value of `data_type` with no problems) in its canonical spelling, as `canonical_fill_value` returns it, without checking it again. The data type's `fill_value_canonical` is the extension author's code. Its result is checked to be JSON. If it raises an error, a note is added that says which data type's canonical spelling raised it. """ - if not isinstance(data_type, AcceptedField): + if not isinstance(data_type, AcceptedDeclaration): return value definition, configuration = data_type.definition, data_type.configuration spelled = asked( @@ -1353,28 +1375,28 @@ def spelled_canonically(data_type: ResolvedField[F], value: JSONValue) -> JSONVa def resolve( data: object, kind: type[D], context: Context, loc: Loc = () -) -> tuple[ResolvedField[D], Problems]: - """Read `data` (one metadata field) as a `kind` in `context`. Return the result of the reading, and every problem. +) -> tuple[ResolvedDeclaration[D], Problems]: + """Read `data` (one declaration) as a `kind` in `context`. Return the result of the reading, and every problem. - This runs all three steps for one field. `data` is refined to JSON and - its envelope is checked. An extra member, a `configuration` that is not + This runs all three steps for one declaration. `data` is refined to JSON and + its envelope is checked. An extra field, a `configuration` that is not an object, and a `must_understand` that is not a boolean or is `false` are each a problem. The name is mapped to a definition in `context`. The configuration is checked against its TypedDict and by its rules. - Each nested field found by the check is read the same way, in the same - scope. A problem in a nested field belongs to that field and is - reported at its location, as with a document's fields. The result is - an `AcceptedField` read by the definition that claims the name; an - `UnclaimedField` when nothing in scope claims the name (an extension + Each nested declaration found by the check is read the same way, in the same + scope. A problem in a nested declaration belongs to that declaration and is + reported at its location, as with a document's declarations. The result is + an `AcceptedDeclaration` read by the definition that claims the name; an + `UnclaimedDeclaration` when nothing in scope claims the name (an extension with no model, not checked, which is what keeps the format open); or a - `RefusedField`, with the problems that explain why. `loc` prefixes + `RefusedDeclaration`, with the problems that explain why. `loc` prefixes every problem. `kind` must be one of the kinds of a format (for v3, those `ZarrV3Kind` names; for v2, those `ZarrV2Kind` names), with or without type arguments, and `context` a scope of that format. Anything else raises `TypeError`. """ if not _is_scope(context): - msg = f"a field is read in a scope, such as CORE_AND_EXTENSIONS, got {described(context)}" + msg = f"a declaration is read in a scope, such as CORE_AND_EXTENSIONS, got {described(context)}" raise TypeError(msg) asked = as_kind(kind) _check_format(asked, context) @@ -1391,7 +1413,7 @@ def _check_format(kind: type[Definition[Any]], context: Context) -> None: scope_format = context.zarr_format if scope_format != format_of(kind): msg = ( - f"a {kind.label} is a field of a Zarr v{format_of(kind)} document, read in a scope " + f"a {kind.label} is a declaration of a Zarr v{format_of(kind)} document, read in a scope " f"of that format, got a v{scope_format} scope" ) raise TypeError(msg) @@ -1432,12 +1454,12 @@ def resolve_refined( kind: type[D], context: Context, loc: Loc = (), -) -> tuple[ResolvedField[D], Problems]: +) -> tuple[ResolvedDeclaration[D], Problems]: """`resolve` for `data` that the caller has already refined. `refined` and `problems` are the result of `refine_json(data, loc)`. - This is for a reader that refines a document's members once and keeps + This is for a reader that refines a document's fields once and keeps the result (the document a model holds is built from it), instead of - refining each field again here. `problems` shows whether `data` is + refining each declaration again here. `problems` shows whether `data` is JSON. If it contains any problems, `refined` is not used. """ asked = as_kind(kind) @@ -1450,22 +1472,22 @@ def resolve_refined( name = asked._named_configuration(data)[0] bad = None if name is None else asked._name_problem(name, asked._name_loc(loc)) claimant = None if name is None or bad is not None else context.claimant(asked, name) - refused = RefusedField(json=UNSET, name=name, read_as=asked, definition=claimant) + refused = RefusedDeclaration(json=UNSET, name=name, read_as=asked, definition=claimant) found = problems if bad is None else (bad, *problems) - return cast("ResolvedField[D]", refused), with_input(found, data, loc) - resolved, found = _resolve_field(refined, asked, context, loc) - return cast("ResolvedField[D]", resolved), with_input(found, data, loc) + return cast("ResolvedDeclaration[D]", refused), with_input(found, data, loc) + resolved, found = _resolve_declaration(refined, asked, context, loc) + return cast("ResolvedDeclaration[D]", resolved), with_input(found, data, loc) -def _resolve_field( +def _resolve_declaration( data: JSONValue, kind: type[Definition[Any]], context: Context, loc: Loc -) -> tuple[ResolvedField[Definition[Any]], Problems]: - """Check the envelope of a refined field, then read the field. +) -> tuple[ResolvedDeclaration[Definition[Any]], Problems]: + """Check the envelope of a refined declaration, then read the declaration. The result of the reading depends only on the configuration. An extra - member or a `must_understand` of `false` does not affect the - configuration, so it is reported alongside the field that was read, - and later layers can still check that field. + field or a `must_understand` of `false` does not affect the + configuration, so it is reported alongside the declaration that was read, + and later layers can still check that declaration. """ envelope = _located(loc, kind._envelope_problems(data)) resolved, found = _read(data, kind, context, loc) @@ -1474,58 +1496,60 @@ def _resolve_field( def _read( data: JSONValue, kind: type[Definition[Any]], context: Context, loc: Loc -) -> tuple[ResolvedField[Definition[Any]], Problems]: +) -> tuple[ResolvedDeclaration[Definition[Any]], Problems]: name, given, malformed = kind._named_configuration(data) if name is None: - return RefusedField(json=data, name=None, read_as=kind), () + return RefusedDeclaration(json=data, name=None, read_as=kind), () if not kind._well_named(name): # The envelope check that every reader runs first reports this. No # definition is looked up for the name. - return RefusedField(json=data, name=name, read_as=kind), () + return RefusedDeclaration(json=data, name=name, read_as=kind), () definition = context.claimant(kind, name) if len(malformed) != 0: # The configuration is not an object, which the envelope problems # report. The name still determines which definition claims the - # field. - return RefusedField(json=data, name=name, read_as=kind, definition=definition), () + # declaration. + return RefusedDeclaration(json=data, name=name, read_as=kind, definition=definition), () if definition is None: - return UnclaimedField(json=data, name=name, read_as=kind), () + return UnclaimedDeclaration(json=data, name=name, read_as=kind), () _, carried = kind._filed_under(name) if carried is not None: return _read_carried(data, name, kind, definition, given, carried, loc) at = kind._in_configuration(loc, ()) if given is None and definition.requires_configuration: missing = problem(at, f"{name!r} requires a configuration", "missing_key") - return RefusedField(json=data, name=name, read_as=kind, definition=definition), missing + return RefusedDeclaration( + json=data, name=name, read_as=kind, definition=definition + ), missing typed, found, nested = _typed(definition.configuration, {} if given is None else given, at) - # Read the nested fields first, each one stack frame deeper, so the + # Read the nested declarations first, each one stack frame deeper, so the # rules see them as the scope read them. Each is put back as a document # writes it, so the configuration shows what was read, however each was # spelled. Their problems are reported after the problems from the # rules. - within: dict[Loc, ResolvedField[Any]] = {} + within: dict[Loc, ResolvedDeclaration[Any]] = {} written: dict[Loc, JSONValue] = {} inside: list[ValidationProblem] = [] - for field in nested: - place = field.loc[len(at) :] - placed = dataclasses.replace(field, loc=kind._in_configuration(loc, place)) + for declaration in nested: + place = declaration.loc[len(at) :] + placed = dataclasses.replace(declaration, loc=kind._in_configuration(loc, place)) inside.extend(_envelope(placed)) - inner, found_inside = _read(field.json, field.kind, context, placed.loc) + inner, found_inside = _read(declaration.json, declaration.kind, context, placed.loc) within[place] = inner - # The input was refined JSON, so the result is JSON, or a field + # The input was refined JSON, so the result is JSON, or a declaration # refused for its contents, never for not being JSON. - written[field.loc] = cast("JSONValue", document_json(inner)) + written[declaration.loc] = cast("JSONValue", document_json(inner)) inside.extend(found_inside) inside.extend(_sized(placed, inner)) - # The rules may look up a nested field by its name, so they run only - # when every nested field has a name. Any other problem with a nested - # field belongs to that field and is reported at its location, as with - # a document's fields. - sound = _usable(found) and all(_named(field) for field in nested) + # The rules may look up a nested declaration by its name, so they run only + # when every nested declaration has a name. Any other problem with a nested + # declaration belongs to that declaration and is reported at its location, as with + # a document's declarations. + sound = _usable(found) and all(_named(declaration) for declaration in nested) configuration = ( cast( "Mapping[str, JSONValue]", - _put_back(typed, lambda field: written[field.loc], nested), + _put_back(typed, lambda declaration: written[declaration.loc], nested), ) if sound else None @@ -1537,36 +1561,36 @@ def _read( ) placed = _in_document(kind, loc, given is not None, own) if configuration is None or not _usable(placed): - refused = RefusedField( + refused = RefusedDeclaration( json=data, name=name, read_as=kind, definition=definition, nested=within ) return refused, (*placed, *inside) - read = AcceptedField( + read = AcceptedDeclaration( json=data, name=name, definition=definition, configuration=configuration, nested=within ) return read, (*placed, *inside) def in_document( - field: ResolvedField[Any], loc: Loc, problems: Sequence[ValidationProblem] + declaration: ResolvedDeclaration[Any], loc: Loc, problems: Sequence[ValidationProblem] ) -> Problems: - """`problems`, which a definition's functions found in the configuration of `field` (a field at `loc`), located under the configuration as the kind locates it, each moved to where the document writes it, as `resolve` places a definition's rules.""" - written = field.read_as._named_configuration(field.json)[1] is not None - return _in_document(field.read_as, loc, written, problems) + """`problems`, which a definition's functions found in the configuration of `declaration` (a declaration at `loc`), located under the configuration as the kind locates it, each moved to where the document writes it, as `resolve` places a definition's rules.""" + written = declaration.read_as._named_configuration(declaration.json)[1] is not None + return _in_document(declaration.read_as, loc, written, problems) def _in_document( kind: type[Definition[Any]], loc: Loc, written: bool, problems: Sequence[ValidationProblem] ) -> Problems: - """`problems`, found in the configuration of a field of `kind` at `loc` and located under the configuration's location, each moved to where the document writes its place, as `kind._in_configuration` says. + """`problems`, found in the configuration of a declaration of `kind` at `loc` and located under the configuration's location, each moved to where the document writes its place, as `kind._in_configuration` says. - `written` says whether the document writes a configuration. A field + `written` says whether the document writes a configuration. A declaration whose configuration has a location of its own, and which writes none (a v3 codec written as its name alone, or with no `configuration`), - has an empty configuration that the document does not hold. A member + has an empty configuration that the document does not hold. A field found missing from it is then the configuration itself, missing, as - for a field that requires a configuration, and a problem with the - configuration as a whole is the field's. + for a declaration that requires a configuration, and a problem with the + configuration as a whole is the declaration's. """ at = kind._in_configuration(loc, ()) absent = not written and at != loc @@ -1581,9 +1605,9 @@ def _in_document( return tuple(placed) -def _named(field: _NestedField) -> bool: - """Whether a nested field has a name, and its configuration (if any) is an object.""" - name, _, malformed = field.kind._named_configuration(field.json) +def _named(declaration: _NestedDeclaration) -> bool: + """Whether a nested declaration has a name, and its configuration (if any) is an object.""" + name, _, malformed = declaration.kind._named_configuration(declaration.json) return name is not None and len(malformed) == 0 @@ -1595,13 +1619,13 @@ def _read_carried( given: Mapping[str, object] | None, carried: Mapping[str, JSONValue], loc: Loc, -) -> tuple[ResolvedField[Definition[Any]], Problems]: - """Read a field whose name encodes its configuration (raw bits, such as `r16`) with the definition its name is filed under. +) -> tuple[ResolvedDeclaration[Definition[Any]], Problems]: + """Read a declaration whose name encodes its configuration (raw bits, such as `r16`) with the definition its name is filed under. The document wrote a name, so a problem with the configuration encoded in the name (such as a size that is not a positive multiple of 8) is a - problem of the field. A configuration written next to the name must be - empty, so each member in it is an undeclared key. + problem of the declaration. A configuration written next to the name must be + empty, so each field in it is an undeclared key. """ _, beside, _ = _checked( EmptyConfiguration, @@ -1612,13 +1636,13 @@ def _read_carried( # a wrong value of the name, not an unknown key a reader may leave out. # The parts it does not declare are reported in one problem, by how the # name spells them, and the rest is read. - members = typeddict_keys(definition.configuration).members - declared = {key: value for key, value in carried.items() if key in members} - undeclared = [key for key in carried if key not in members] + fields = typeddict_keys(definition.configuration).fields + declared = {key: value for key, value in carried.items() if key in fields} + undeclared = [key for key in carried if key not in fields] configuration, judged = definition.read_configuration(declared) # A problem with the configuration encoded in the name belongs to the - # field. It is located at the field, where the name is, and the - # `input` and `ctx` that described a configuration member do not apply + # declaration. It is located at the declaration, where the name is, and the + # `input` and `ctx` that described a configuration field do not apply # to the name, so they are cleared. Each is a wrong value of the name: # a part it lacks is not a missing key. problems = ( @@ -1631,17 +1655,17 @@ def _read_carried( if len(undeclared) != 0: message = kind._undeclared_parts(definition.name, name, undeclared) problems = (*problems, ValidationProblem(loc, message, "invalid_value")) - # Any problem with the encoded configuration refuses the field. + # Any problem with the encoded configuration refuses the declaration. if configuration is None or not _usable(beside) or len(judged) != 0 or len(undeclared) != 0: - refused = RefusedField(json=data, name=name, read_as=kind, definition=definition) + refused = RefusedDeclaration(json=data, name=name, read_as=kind, definition=definition) return refused, problems - return AcceptedField( + return AcceptedDeclaration( json=data, name=name, definition=definition, configuration=configuration ), problems -def _sized(field: _NestedField, inner: ResolvedField[Any]) -> Problems: - """The problems with the definition that read `inner` in the member that holds `field`, as the kind of `field` reports them for its alias. +def _sized(declaration: _NestedDeclaration, inner: ResolvedDeclaration[Any]) -> Problems: + """The problems with the definition that read `inner` in the field that holds `declaration`, as the kind of `declaration` reports them for its alias. A name that nothing in scope claims is not checked. Nothing is known about it. @@ -1649,33 +1673,33 @@ def _sized(field: _NestedField, inner: ResolvedField[Any]) -> Problems: definition = inner.definition if definition is None: return () - return field.kind._alias_problems(field.alias, definition, field.loc) + return declaration.kind._alias_problems(declaration.alias, definition, declaration.loc) def canonicalize( data: object, kind: type[D], context: Context, loc: Loc = () ) -> tuple[JSONValue | None, Problems]: - """`data` (one metadata field) in its simplest equivalent spelling, and every problem. + """`data` (one declaration) in its simplest equivalent spelling, and every problem. - Only a field without problems has a simplest spelling. A simplest + Only a declaration without problems has a simplest spelling. A simplest spelling expresses what the author wrote in fewer words. A key the - TypedDict does not declare, an extra envelope member, or a + TypedDict does not declare, an extra envelope field, or a `must_understand` of `false` is something the author wrote that the - simplest spelling would erase. So a field with any problem returns - None, with its problems. Otherwise each nested field is written in its + simplest spelling would erase. So a declaration with any problem returns + None, with its problems. Otherwise each nested declaration is written in its own simplest spelling, and then the definition's `canonical` handles the rest. Its result is checked again, so a `canonical` that returns an invalid configuration raises `ValueError`, which indicates a bug in - the definition, not in the field. The envelope uses the shortest form + the definition, not in the declaration. The envelope uses the shortest form every reader accepts. A data type with nothing to configure is written as its bare name, as core data types have been written since Zarr - v3.0. Any other field is an object, `{"name": ...}`, because a Zarr + v3.0. Any other declaration is an object, `{"name": ...}`, because a Zarr v3.0 reader does not accept a short-hand name in `codecs` (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L585-L592). There is no `must_understand`, because leaving it out means `true`. A name that nothing in scope claims keeps the configuration it was written with, because only its own definition could decide how to - simplify it. For a field that a scope has already read, use + simplify it. For a declaration that a scope has already read, use `canonical_of` with its problems. """ resolved, problems = resolve(data, kind, context, loc) @@ -1683,17 +1707,17 @@ def canonicalize( def canonical_of( - resolved: ResolvedField[Any], problems: Sequence[ValidationProblem] + resolved: ResolvedDeclaration[Any], problems: Sequence[ValidationProblem] ) -> JSONValue | None: - """`resolved` (a field a scope read) in its simplest equivalent spelling, as `canonicalize` writes it; None when it has a problem. + """`resolved` (a declaration a scope read) in its simplest equivalent spelling, as `canonicalize` writes it; None when it has a problem. - `problems` are the field's problems, as returned by `resolve`, or as - `with_problems` returns them for each field of a reading. Only a field + `problems` are the declaration's problems, as returned by `resolve`, or as + `with_problems` returns them for each declaration of a reading. Only a declaration with no problems has a simplest spelling. A simpler spelling of a - field with a problem would erase what its author wrote (such as a key + declaration with a problem would erase what its author wrote (such as a key its TypedDict does not declare), or would write something invalid. The result is built from what the scope read, without reading the - field again, and is plain JSON that shares nothing with the field. + declaration again, and is plain JSON that shares nothing with the declaration. """ if len(problems) != 0: return None @@ -1701,17 +1725,17 @@ def canonical_of( return None if simplest is None else copied(simplest) -def _simplest(field: ResolvedField[Any]) -> JSONValue | None: - """A field in its simplest spelling; None when the field or one of its nested fields was refused, because a refused field has no simplest spelling.""" - if isinstance(field, AcceptedField): - return _canonical_field(field) - if isinstance(field, UnclaimedField): - return field.to_json() +def _simplest(declaration: ResolvedDeclaration[Any]) -> JSONValue | None: + """A declaration in its simplest spelling; None when the declaration or one of its nested declarations was refused, because a refused declaration has no simplest spelling.""" + if isinstance(declaration, AcceptedDeclaration): + return _canonical_declaration(declaration) + if isinstance(declaration, UnclaimedDeclaration): + return declaration.to_json() return None -def _canonical_field(resolved: AcceptedField[Any]) -> JSONValue | None: - """An accepted field in its simplest equivalent spelling: nested fields first, then its own members. None when a nested field was refused.""" +def _canonical_declaration(resolved: AcceptedDeclaration[Any]) -> JSONValue | None: + """An accepted declaration in its simplest equivalent spelling: nested declarations first, then its own fields. None when a nested declaration was refused.""" definition, name = resolved.definition, resolved.name configuration: JSONValue = dict(resolved.configuration) for loc, inner in resolved.nested.items(): @@ -1755,16 +1779,16 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: __all__ = [ - "AcceptedField", + "AcceptedDeclaration", "C", "Context", "Definition", "EmptyConfiguration", "Nested", "Problems", - "RefusedField", - "ResolvedField", - "UnclaimedField", + "RefusedDeclaration", + "ResolvedDeclaration", + "UnclaimedDeclaration", "WithFillValue", "as_kind", "asked", @@ -1772,17 +1796,17 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "canonical_of", "canonicalize", "configuration_of", + "declaration_key", + "declaration_kind", + "declarations_of", "document_json", - "field_key", - "field_kind", - "fields_of", "filed_under", "fill_value_as_written", "fill_value_problems", "format_of", "held", "in_document", - "is_field", + "is_declaration", "kind_of", "named_configuration", "no_rules", diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_envelope.py b/packages/zarr-metadata/src/zarr_metadata/definition/_envelope.py index 01c9cdfc46..00216dda2c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_envelope.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_envelope.py @@ -1,4 +1,4 @@ -"""The default field envelope: how a field is written as a name or as an object with `name`, `configuration` and `must_understand`, and which names are valid extension names. +"""The default declaration envelope: how a declaration is written as a name or as an object with `name`, `configuration` and `must_understand`, and which names are valid extension names. This is the envelope Zarr v3 uses. `Definition` uses it by default, and a kind of another format (the v2 kinds) overrides each method that uses it. @@ -11,7 +11,7 @@ from zarr_metadata._json import ValidationProblem, is_object, shown, shown_key ENVELOPE_KEYS: frozenset[str] = frozenset({"name", "configuration", "must_understand"}) -"""The keys a metadata field's envelope declares. Any other key is an unknown key.""" +"""The keys a declaration's envelope declares. Any other key is an unknown key.""" _EXTENSION_NAME: Final = re.compile(r"[a-z][a-z0-9_.-]+") @@ -61,12 +61,12 @@ def name_problem(name: str, at: tuple[str | int, ...]) -> ValidationProblem | No def envelope_problems( value: object, *, allow_must_understand_false: bool ) -> tuple[ValidationProblem, ...]: - """Every reason `value` is not a valid v3 metadata field envelope. The contents of the configuration are not checked. + """Every reason `value` is not a valid v3 declaration envelope. The contents of the configuration are not checked. The envelope is the part around the configuration: a `name` that is a valid extension name under the spec, a `configuration` that is an object with string keys, a boolean `must_understand`, and no other - keys. `resolve` calls this on a field it has already narrowed to JSON, + keys. `resolve` calls this on a declaration it has already narrowed to JSON, so the configuration is walked only once. """ if isinstance(value, str): @@ -76,29 +76,27 @@ def envelope_problems( return ( ValidationProblem( (), - f"expected a metadata field (string or extension object), got {shown(value)}", + f"expected a declaration (string or extension object), got {shown(value)}", "invalid_type", ), ) - field = value + declaration = value problems: list[ValidationProblem] = [] - for key in field: + for key in declaration: if not isinstance(key, str): problems.append( - ValidationProblem( - (), f"non-string metadata field key {shown_key(key)}", "invalid_type" - ) + ValidationProblem((), f"non-string field key {shown_key(key)}", "invalid_type") ) elif key not in ENVELOPE_KEYS: problems.append(ValidationProblem((key,), f"unexpected key {key!r}", "unknown_key")) - if "name" not in field: + if "name" not in declaration: problems.append(ValidationProblem(("name",), "missing required key", "missing_key")) - elif not isinstance(field["name"], str): + elif not isinstance(declaration["name"], str): problems.append(ValidationProblem(("name",), "expected a string name", "invalid_type")) - elif (bad := name_problem(field["name"], ("name",))) is not None: + elif (bad := name_problem(declaration["name"], ("name",))) is not None: problems.append(bad) - if "configuration" in field: - configuration = field["configuration"] + if "configuration" in declaration: + configuration = declaration["configuration"] if not is_object(configuration): problems.append( ValidationProblem(("configuration",), "expected an object", "invalid_type") @@ -107,8 +105,8 @@ def envelope_problems( problems.append( ValidationProblem(("configuration",), "expected string keys", "invalid_type") ) - if "must_understand" in field: - must_understand = field["must_understand"] + if "must_understand" in declaration: + must_understand = declaration["must_understand"] if not isinstance(must_understand, bool): problems.append( ValidationProblem(("must_understand",), "expected a boolean", "invalid_type") diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py index 11ad7e1563..4afb9bad45 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py @@ -20,13 +20,13 @@ from typing import TYPE_CHECKING, Any, Literal, TypeAlias, cast from zarr_metadata.definition._definition import ( - AcceptedField, + AcceptedDeclaration, Definition, - RefusedField, - UnclaimedField, + RefusedDeclaration, + UnclaimedDeclaration, as_kind, - field_key, - fields_of, + declaration_key, + declarations_of, filed_under, format_of, own_key, @@ -37,7 +37,7 @@ from collections.abc import Callable, Iterable, Sequence from zarr_metadata._typed_json import Loc - from zarr_metadata.definition._definition import D, ResolvedField + from zarr_metadata.definition._definition import D, ResolvedDeclaration ClaimKey: TypeAlias = tuple[type[Definition[Any]], str] """A kind and the name a definition is filed under. A scope's `claimant` looks up definitions by this key.""" @@ -101,31 +101,31 @@ def __reduce__( return type(self), (self.conflicts,), dict(self.__dict__) -def claim_key(field: ResolvedField[Any]) -> ClaimKey | None: - """The key `field` is claimed under: its kind and the name its definition is filed under (for example `r*` for `r16`). Returns None for a field with no name.""" - if field.name is None: +def claim_key(declaration: ResolvedDeclaration[Any]) -> ClaimKey | None: + """The key `declaration` is claimed under: its kind and the name its definition is filed under (for example `r*` for `r16`). Returns None for a declaration with no name.""" + if declaration.name is None: return None - filed, _ = filed_under(field.read_as, field.name) - return None if filed is None else (field.read_as, filed) + filed, _ = filed_under(declaration.read_as, declaration.name) + return None if filed is None else (declaration.read_as, filed) def claims_of( - fields: Iterable[tuple[Loc, ResolvedField[Any]]], + declarations: Iterable[tuple[Loc, ResolvedDeclaration[Any]]], ) -> dict[ClaimKey, Definition[Any] | None]: - """The claims that `fields` (each paired with its location) record for each name: the definition that read the name, or None if the name was unclaimed. + """The claims that `declarations` (each paired with its location) record for each name: the definition that read the name, or None if the name was unclaimed. - `fields` must come from one reading, as returned by `fields_of` or a - reading's `fields()`. If two fields claim the same name with different + `declarations` must come from one reading, as returned by `declarations_of` or a + reading's `declarations()`. If two declarations claim the same name with different definitions, this raises `ScopeConflictError`, because no single scope could have read them both. """ claims: dict[ClaimKey, Definition[Any] | None] = {} conflicts: list[Conflict] = [] - for loc, field in fields: - key = claim_key(field) + for loc, declaration in declarations: + key = claim_key(declaration) if key is None: continue - definition = field.definition + definition = declaration.definition if key in claims and claims[key] != definition: conflicts.append(Conflict(key, claims[key], definition, loc)) continue @@ -135,54 +135,54 @@ def claims_of( return claims -def refines(field: ResolvedField[Any], other: ResolvedField[Any]) -> bool: - """Whether `field` contains everything `other` contains: it reads the same where both read, and it may also read what `other` left unclaimed. +def refines(declaration: ResolvedDeclaration[Any], other: ResolvedDeclaration[Any]) -> bool: + """Whether `declaration` contains everything `other` contains: it reads the same where both read, and it may also read what `other` left unclaimed. This is the order in which one reading of a document refines another. - Suppose `other` is unclaimed and `field` is accepted. This counts as a - gain when re-reading the unclaimed field's JSON, with `field`'s - definition and the definitions of the fields nested in `field`, - produces `field`. Two spellings of one configuration count as one gain, - because `==` treats them as one field, so the order is transitive + Suppose `other` is unclaimed and `declaration` is accepted. This counts as a + gain when re-reading the unclaimed declaration's JSON, with `declaration`'s + definition and the definitions of the declarations nested in `declaration`, + produces `declaration`. Two spellings of one configuration count as one gain, + because `==` treats them as one declaration, so the order is transitive through equality. The reverse direction (accepted to unclaimed) is a loss. One name read by two different definitions is a conflict. A - refused field refines only itself. Two fields that refine each other + refused declaration refines only itself. Two declarations that refine each other are equal. """ - if isinstance(field, RefusedField) or isinstance(other, RefusedField): - return field == other - if isinstance(other, UnclaimedField): - if isinstance(field, UnclaimedField): - return field_key(field) == field_key(other) - return claim_key(field) == claim_key(other) and _gained(field, other) - if isinstance(field, UnclaimedField): + if isinstance(declaration, RefusedDeclaration) or isinstance(other, RefusedDeclaration): + return declaration == other + if isinstance(other, UnclaimedDeclaration): + if isinstance(declaration, UnclaimedDeclaration): + return declaration_key(declaration) == declaration_key(other) + return claim_key(declaration) == claim_key(other) and _gained(declaration, other) + if isinstance(declaration, UnclaimedDeclaration): return False - if field.definition != other.definition or own_key(field) != own_key(other): + if declaration.definition != other.definition or own_key(declaration) != own_key(other): return False - if set(field.nested) != set(other.nested): + if set(declaration.nested) != set(other.nested): return False - return all(refines(field.nested[loc], other.nested[loc]) for loc in field.nested) + return all(refines(declaration.nested[loc], other.nested[loc]) for loc in declaration.nested) -def _gained(field: AcceptedField[Any], other: UnclaimedField) -> bool: - """Whether reading `other` in the scope formed by `field`'s own claims produces `field`. This is the test for a gain.""" - again, _ = resolve(other.json, field.read_as, _Claimed.of(field)) - return again == field +def _gained(declaration: AcceptedDeclaration[Any], other: UnclaimedDeclaration) -> bool: + """Whether reading `other` in the scope formed by `declaration`'s own claims produces `declaration`. This is the test for a gain.""" + again, _ = resolve(other.json, declaration.read_as, _Claimed.of(declaration)) + return again == declaration @dataclass(frozen=True, slots=True) class _Claimed: - """The scope formed by a field's own claims: its definition and the definitions of its nested fields, keyed by kind and filed name. `_gained` re-reads a field in this scope.""" + """The scope formed by a declaration's own claims: its definition and the definitions of its nested declarations, keyed by kind and filed name. `_gained` re-reads a declaration in this scope.""" filed: Mapping[ClaimKey, Definition[Any]] zarr_format: Literal[2, 3] @classmethod - def of(cls, field: AcceptedField[Any]) -> _Claimed: - claimed = claims_of(fields_of(field)) + def of(cls, declaration: AcceptedDeclaration[Any]) -> _Claimed: + claimed = claims_of(declarations_of(declaration)) return cls( {key: definition for key, definition in claimed.items() if definition is not None}, - format_of(field.read_as), + format_of(declaration.read_as), ) def claimant(self, kind: type[D], name: str) -> D | None: diff --git a/packages/zarr-metadata/src/zarr_metadata/json_schema.py b/packages/zarr-metadata/src/zarr_metadata/json_schema.py index 9b7443a8f4..168ca34cc0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/json_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/json_schema.py @@ -1,9 +1,9 @@ -"""JSON Schema output: a TypedDict, one Zarr v3 field, or a whole v3 `zarr.json`. +"""JSON Schema output: a TypedDict, one Zarr v3 declaration, or a whole v3 `zarr.json`. Each function writes JSON Schema draft 2020-12, for a validator in another language or for an editor that checks a document while it is -written. A schema describes each member on its own, so it does not -contain the rules that relate members to each other, and a document it +written. A schema describes each field on its own, so it does not +contain the rules that relate fields to each other, and a document it accepts may still have a problem. **A TypedDict.** `json_schema(typed_dict)` writes the rules that @@ -23,34 +23,34 @@ # 'properties': {'level': {'type': 'integer', 'minimum': 0, 'maximum': 9}}, # 'required': ['level'], 'additionalProperties': False} -**One v3 field.** `field_json_schema_v3(kind, context=None)` writes one field -read as `kind` in the scope `context`: a field that a definition in +**One v3 declaration.** `declaration_json_schema_v3(kind, context=None)` writes one declaration +read as `kind` in the scope `context`: a declaration that a definition in scope reads, or a name that no definition claims, with any configuration. **A whole `zarr.json`.** `node_metadata_json_schema_v3(context=None)` writes a whole v3 `zarr.json`, array or group, as the scope reads it -(`CORE_AND_EXTENSIONS` when none is given). Its fields are written +(`CORE_AND_EXTENSIONS` when none is given). Its declarations are written as the scope reads them. For each definition, the schema accepts its -field (its name, its configuration as its TypedDict declares it, +declaration (its name, its configuration as its TypedDict declares it, including bounds, a `must_understand` of `true`, and its bare name when it needs no configuration). It also accepts a name that nothing in scope -claims, with any configuration. A nested field is written in the same -scope, and a member that accepts only codecs of static size accepts -only those. The rules are not in the schema, so a field it accepts may -still have a problem. Any field that `resolve` reads without a problem +claims, with any configuration. A nested declaration is written in the same +scope, and a field that accepts only codecs of static size accepts +only those. The rules are not in the schema, so a declaration it accepts may +still have a problem. Any declaration that `resolve` reads without a problem is accepted by the schema, as JSON (arrays as lists, as a parser -returns them). Each configuration TypedDict and each field alias is +returns them). Each configuration TypedDict and each declaration alias is written once, in `$defs`, under its name. The fill value must match its data type's fill value schema. """ from zarr_metadata._typed_json import JSONSchema, json_schema -from zarr_metadata.v3._json_schema import field_json_schema_v3, node_metadata_json_schema_v3 +from zarr_metadata.v3._json_schema import declaration_json_schema_v3, node_metadata_json_schema_v3 __all__ = [ "JSONSchema", - "field_json_schema_v3", + "declaration_json_schema_v3", "json_schema", "node_metadata_json_schema_v3", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 98776c7c5d..f824894203 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -26,7 +26,7 @@ They undo each bug with `repair_node_metadata_v3` or `repair_node_metadata_v2` before the strict read, and report what they changed. `zarr_metadata.json_schema` writes a JSON Schema of what the v3 -readers read, except for the rules. One field on its own is read with +readers read, except for the rules. One declaration on its own is read with `zarr_metadata.definition.resolve`. The model constructors `from_json` and `from_key_value` raise @@ -36,7 +36,7 @@ `ZarrV3ArrayMetadata(document, context=None)` reads the document in the scope and raises `MetadataValidationError` with every problem, so an invalid model is never built. `to_json` returns the document as written. -`update` reads new members in the model's own scope. `with_context` and +`update` reads new fields in the model's own scope. `with_context` and `refined_in` read the document in another scope. `to_key_value` writes the model as it is. A v3 group reads its `consolidated_metadata` as an envelope and does not read the documents it lists, because consolidated diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index 7768e68210..fa260fae0e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -29,11 +29,11 @@ from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._context import disagreements, scoped from zarr_metadata.definition._definition import ( - AcceptedField, - ResolvedField, - UnclaimedField, - field_key, - fields_of, + AcceptedDeclaration, + ResolvedDeclaration, + UnclaimedDeclaration, + declaration_key, + declarations_of, fill_value_problems, held, resolve, @@ -41,7 +41,7 @@ spelled_canonically, ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of -from zarr_metadata.definition._scope import refines as refines_field +from zarr_metadata.definition._scope import refines as refines_declaration from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, @@ -49,18 +49,18 @@ check_literal, dimension_lengths, dump_store_json, + field_changes, load_store_json, located_conflicts, - member_changes, missing_keys, - other_members, + other_fields, reading_of, - unclaimed_fields_of, + unclaimed_declarations_of, ) # A runtime import: `get_type_hints` resolves the public signatures. from zarr_metadata.v2._context import ZarrV2Context # noqa: TC001 -from zarr_metadata.v2._field_json import ZarrV2DataTypeMetadata +from zarr_metadata.v2._declaration_json import ZarrV2DataTypeMetadata from zarr_metadata.v2.array import ( ZARR_V2_ARRAY_METADATA_STORE_KEY, ZarrV2ArrayDimensionSeparator, @@ -97,7 +97,7 @@ _ARRAY_METADATA_STANDARD_KEYS_V2: Final[frozenset[str]] = ( _ARRAY_METADATA_REQUIRED_KEYS_V2 | _ARRAY_METADATA_OPTIONAL_KEYS_V2 ) -"""The members the spec defines in a v2 array document, as the model merges it, with `attributes` from `.zattrs`: these are never given in `extra_fields`.""" +"""The fields the spec defines in a v2 array document, as the model merges it, with `attributes` from `.zattrs`: these are never given in `extra_fields`.""" def _is_canonical_dtype_v2(value: object) -> bool: @@ -118,8 +118,8 @@ def _is_canonical_dtype_v2(value: object) -> bool: @dataclass(frozen=True, slots=True) -class ArrayMembersV2: - """The members other than fields of a v2 array document that a read found no problems with, refined as the model holds them.""" +class ArrayFieldsV2: + """The fields other than declarations of a v2 array document that a read found no problems with, refined as the model holds them.""" shape: tuple[int, ...] chunks: tuple[int, ...] @@ -129,32 +129,32 @@ class ArrayMembersV2: attributes: dict[str, JSONValue] | UNSET extra_fields: dict[str, JSONValue] document: dict[str, JSONValue] - """The refined document that the model holds: each member as the read refined it, in the order the document writes them.""" + """The refined document that the model holds: each field as the read refined it, in the order the document writes them.""" @dataclass(frozen=True, slots=True) class ZarrV2ArrayMetadataReading: """A v2 array document as a scope read it, whatever it holds: its dtype, compressor and filters, every problem, and the model when there are no problems. - A field the document does not hold is `UNSET`, and so is a `filters` + A declaration the document does not hold is `UNSET`, and so is a `filters` that is not an array, which holds no filters to read. A `compressor` - or `filters` written as `null` is None. Each problem and each field is + or `filters` written as `null` is None. Each problem and each declaration is located in what was read: the document, or the files of a node that `read_node_metadata_v2` reads, where `at` is `(".zarray",)`. """ - dtype: ResolvedField[ZarrV2DataTypeDefinition[Any]] | UNSET = UNSET + dtype: ResolvedDeclaration[ZarrV2DataTypeDefinition[Any]] | UNSET = UNSET """The dtype, as the scope read it.""" - compressor: ResolvedField[ZarrV2CodecDefinition[Any]] | UNSET | None = UNSET + compressor: ResolvedDeclaration[ZarrV2CodecDefinition[Any]] | UNSET | None = UNSET """The compressor, as the scope read it; None when written as `null`.""" - filters: tuple[ResolvedField[ZarrV2CodecDefinition[Any]], ...] | UNSET | None = UNSET + filters: tuple[ResolvedDeclaration[ZarrV2CodecDefinition[Any]], ...] | UNSET | None = UNSET """The filters, each as the scope read it; None when written as `null`.""" problems: tuple[ValidationProblem, ...] = () """Every reason the document is not valid.""" metadata: ZarrV2ArrayMetadata | None = None - """The document's model, which holds these fields, when there is no problem; None otherwise.""" + """The document's model, which holds these declarations, when there is no problem; None otherwise.""" at: Loc = () - """Where the document's fields are in what was read: `()` for a document read alone, `(".zarray",)` for the files of a node. A model's own reading is of its merged document, at `()`.""" + """Where the document's declarations are in what was read: `()` for a document read alone, `(".zarray",)` for the files of a node. A model's own reading is of its merged document, at `()`.""" def __reduce__(self) -> str | tuple[object, ...]: # Same as the v3 reading: pickle through the model, when there is one. @@ -162,26 +162,26 @@ def __reduce__(self) -> str | tuple[object, ...]: return (_reading_at, (self.metadata, self.at)) return object.__reduce__(self) - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Each field in the document, as the scope read it, with its location under `at`: the dtype, followed by a struct's record types, then the compressor, then each filter at its index.""" + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Each declaration in the document, as the scope read it, with its location under `at`: the dtype, followed by a struct's record types, then the compressor, then each filter at its index.""" if self.dtype is not UNSET: - yield from fields_of(self.dtype, (*self.at, "dtype")) + yield from declarations_of(self.dtype, (*self.at, "dtype")) if self.compressor is not UNSET and self.compressor is not None: - yield from fields_of(self.compressor, (*self.at, "compressor")) + yield from declarations_of(self.compressor, (*self.at, "compressor")) if self.filters is not UNSET and self.filters is not None: for index, entry in enumerate(self.filters): - yield from fields_of(entry, (*self.at, "filters", index)) + yield from declarations_of(entry, (*self.at, "filters", index)) def _reading_at(model: ZarrV2ArrayMetadata, at: Loc) -> ZarrV2ArrayMetadataReading: - """The reading that `model` holds, with its fields at `at`. Unpickling a reading that held a model rebuilds it with this function.""" + """The reading that `model` holds, with its declarations at `at`. Unpickling a reading that held a model rebuilds it with this function.""" return dataclasses.replace(cast("ZarrV2ArrayMetadataReading", reading_of(model)), at=at) def read_array_v2( value: object, context: ZarrV2Context, *, at: Loc = () -) -> tuple[ZarrV2ArrayMetadataReading, ArrayMembersV2 | None]: - """Read `value`, a v2 array document, once in `context`. Return the reading, and the other members when there are no problems. +) -> tuple[ZarrV2ArrayMetadataReading, ArrayFieldsV2 | None]: + """Read `value`, a v2 array document, once in `context`. Return the reading, and the other fields when there are no problems. `dtype`, `compressor` and `filters` are read in `context`. A dtype or codec that the scope refuses is a problem. One that the scope does @@ -196,13 +196,13 @@ def read_array_v2( # Unlike the group document ("Other keys MUST NOT be present", # https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313), the v2 array document is open: other keys "SHOULD NOT be # present within the metadata object and SHOULD be ignored by - # implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), so a member outside + # implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), so a field outside # _ARRAY_METADATA_STANDARD_KEYS_V2 is not a problem just for being there. - # Ignored does not mean unchecked: as in v3, the member must be JSON, - # and its key must be a string. Like a v3 member that nothing claims, + # Ignored does not mean unchecked: as in v3, the field must be JSON, + # and its key must be a string. Like a v3 field that nothing claims, # it is not read, so it may hold any number, as attributes may. problems: list[ValidationProblem] = list(missing_keys(_ARRAY_METADATA_REQUIRED_KEYS_V2, doc)) - extra_fields, found = other_members(doc, _ARRAY_METADATA_STANDARD_KEYS_V2) + extra_fields, found = other_fields(doc, _ARRAY_METADATA_STANDARD_KEYS_V2) problems.extend(found) problems.extend(check_literal(doc, "zarr_format", 2)) shape, shape_problems = dimension_lengths(doc, "shape") @@ -217,7 +217,7 @@ def read_array_v2( "invalid_value", ) ) - # Each member is refined once, here, and the refined value is kept. + # Each field is refined once, here, and the refined value is kept. # The document the model holds is built from these values, so nothing # is walked twice. refined: dict[str, JSONValue] = { @@ -226,7 +226,7 @@ def read_array_v2( "shape": shape, "chunks": chunks, } - dtype: ResolvedField[ZarrV2DataTypeDefinition[Any]] | UNSET = UNSET + dtype: ResolvedDeclaration[ZarrV2DataTypeDefinition[Any]] | UNSET = UNSET if "dtype" in doc: # A typestr is read by its family's definition, and a list of field # records as a struct. @@ -242,7 +242,7 @@ def read_array_v2( refined["order"] = order if len(found) != 0 or order not in ("C", "F"): problems.append(outside_of(("order",), doc["order"], ("C", "F"))) - compressor: ResolvedField[ZarrV2CodecDefinition[Any]] | UNSET | None = UNSET + compressor: ResolvedDeclaration[ZarrV2CodecDefinition[Any]] | UNSET | None = UNSET if "compressor" in doc: compressor = None refined["compressor"] = None @@ -257,7 +257,7 @@ def read_array_v2( ("compressor",), ) problems.extend(found) - filters: tuple[ResolvedField[ZarrV2CodecDefinition[Any]], ...] | UNSET | None = UNSET + filters: tuple[ResolvedDeclaration[ZarrV2CodecDefinition[Any]], ...] | UNSET | None = UNSET if "filters" in doc: entries = doc["filters"] if entries is None: @@ -276,7 +276,7 @@ def read_array_v2( else: # "A list of JSON objects providing codec configurations, or # null" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79): an empty list is a list. - read: list[ResolvedField[ZarrV2CodecDefinition[Any]]] = [] + read: list[ResolvedDeclaration[ZarrV2CodecDefinition[Any]]] = [] items: list[JSONValue] = [] for index, item in enumerate(entries): refined_item, found = refine_json(item, ("filters", index)) @@ -323,7 +323,7 @@ def read_array_v2( ) if len(problems) != 0 or shape is None or chunks is None or attributes is None: return reading, None - members = ArrayMembersV2( + fields = ArrayFieldsV2( shape=shape, chunks=chunks, fill_value=fill_value, @@ -335,7 +335,7 @@ def read_array_v2( extra_fields=extra_fields, document={key: refined[key] for key in cast("Mapping[str, object]", doc)}, ) - return reading, members + return reading, fields NODE_FILES_V2: Final = ( @@ -370,8 +370,8 @@ def node_document_v2( `files` must hold `key`. Returns the document and the problems found in merging, located in `files`. A file that holds `attributes` itself - gives `beside_attributes`, at the member in that file, since - attributes belong in `.zattrs`, and the document without that member, + gives `beside_attributes`, at the field in that file, since + attributes belong in `.zattrs`, and the document without that field, so that its reader still reports every other problem. A file that is not an object is returned as it is, for the document's reader to report. @@ -391,7 +391,7 @@ def node_document_v2( def read_array_node_v2( files: Mapping[str, object], context: ZarrV2Context ) -> ZarrV2ArrayMetadataReading: - """Read the v2 array node whose files are `files`, which hold its `.zarray`, as `read_node_metadata_v2` reads it: each problem and each field located in the file that holds it.""" + """Read the v2 array node whose files are `files`, which hold its `.zarray`, as `read_node_metadata_v2` reads it: each problem and each declaration located in the file that holds it.""" document, problems = node_document_v2( files, ZARR_V2_ARRAY_METADATA_STORE_KEY, ARRAY_FILE_ATTRIBUTES ) @@ -431,7 +431,7 @@ def in_files( f"belong in {ZARR_V2_ATTRIBUTES_STORE_KEY}", "invalid_value", ) -"""The problem of a `.zarray` that holds `attributes`, which belong in `.zattrs`. A `.zarray` may hold members the spec does not define, so the key is not unknown, but this one is taken by the attributes.""" +"""The problem of a `.zarray` that holds `attributes`, which belong in `.zattrs`. A `.zarray` may hold fields the spec does not define, so the key is not unknown, but this one is taken by the attributes.""" def read_array_metadata_v2( @@ -440,29 +440,29 @@ def read_array_metadata_v2( """Read `value`, a v2 array document, in `context` (`CORE_V2` when none is given), whatever it holds. The result holds everything the read finds: the dtype, the compressor - and each filter as the scope read them (an `AcceptedField` read by - the definition that claims the typestr or id, an `UnclaimedField`, or - a `RefusedField`), every problem, and, when there are no problems, + and each filter as the scope read them (an `AcceptedDeclaration` read by + the definition that claims the typestr or id, an `UnclaimedDeclaration`, or + a `RefusedDeclaration`), every problem, and, when there are no problems, the document's model. A dtype or codec that the scope refuses is a problem. One that the scope does not claim is not a problem. `fill_value` is checked against the dtype the scope read. """ context = scoped(context, CORE_V2) - reading, members = read_array_v2(value, context) - if members is None: + reading, fields = read_array_v2(value, context) + if fields is None: return reading - model = ZarrV2ArrayMetadata._of(members.document, context, reading, members) # pyright: ignore[reportPrivateUsage] + model = ZarrV2ArrayMetadata._of(fields.document, context, reading, fields) # pyright: ignore[reportPrivateUsage] return model.reading class ZarrV2ArrayMetadataUpdate(TypedDict, total=False, closed=True): - """The members `ZarrV2ArrayMetadata.update` replaces: each in the form a document writes it, or `UNSET` to remove a member that a document may omit. + """The fields `ZarrV2ArrayMetadata.update` replaces: each in the form a document writes it, or `UNSET` to remove a field that a document may omit. - The members a document may omit are `attributes` (meaning no - `.zattrs`), `dimension_separator` (read as `"."`), and any member the + The fields a document may omit are `attributes` (meaning no + `.zattrs`), `dimension_separator` (read as `"."`), and any field the spec does not define. Those go in `extra_fields`, by key: each one given is set, or removed when given as `UNSET`, and the others are - kept. A member is given by its own name only when the spec defines it, + kept. A field is given by its own name only when the spec defines it, so a misspelled name is refused. """ @@ -482,39 +482,39 @@ class ZarrV2ArrayMetadata(Keyed): """A v2 array document, and the scope it was read in. Like the v3 models, this model is a pair. `to_json` returns the merged - document (the `.zarray` members, plus `attributes` when a `.zattrs` + document (the `.zarray` fields, plus `attributes` when a `.zattrs` holds them) as written, refined. A `.zarray` that omits `dimension_separator` means `"."` by the v2 convention (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L81-L86): the `dimension_separator` property returns `"."`, and the model compares equal to one whose document writes `"."`, but `to_json` writes the document without the key. `dtype`, `compressor` and - each filter are views of the reading: an `AcceptedField` read by the + each filter are views of the reading: an `AcceptedDeclaration` read by the definition in scope that claims the typestr or id, or an - `UnclaimedField`. `attributes` is empty when no `.zattrs` exists, as + `UnclaimedDeclaration`. `attributes` is empty when no `.zattrs` exists, as in a v3 model. The document still tells the two apart: `to_json` and `to_key_value` write `attributes` and `.zattrs` only when a `.zattrs` - exists, even an empty one. A member the spec does not define is kept + exists, even an empty one. A field the spec does not define is kept in `extra_fields`. A model is only built by reading: the constructor reads `document` in `context` and raises `MetadataValidationError` with every problem, so no model is invalid. Two models are equal when their documents mean the same thing in - their scopes, as the model key defines. `update` reads new members + their scopes, as the model key defines. `update` reads new fields in the model's own scope. `with_context` and `refined_in` read the document in another scope. A model pickles as its pair. """ - __slots__ = ("_context", "_document", "_members", "_reading", "_shown") + __slots__ = ("_context", "_document", "_fields", "_reading", "_shown") zarr_format: Final[Literal[2]] = 2 # noqa: PYI064 - a bare Final does not narrow a union of models node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: context = scoped(context, CORE_V2) - reading, members = read_array_v2(document, context) - if members is None: + reading, fields = read_array_v2(document, context) + if fields is None: raise MetadataValidationError(reading.problems) - self._adopt(members.document, context, reading, members) + self._adopt(fields.document, context, reading, fields) @classmethod def _of( @@ -522,11 +522,11 @@ def _of( document: dict[str, JSONValue], context: ZarrV2Context, reading: ZarrV2ArrayMetadataReading, - members: ArrayMembersV2, + fields: ArrayFieldsV2, ) -> ZarrV2ArrayMetadata: """Build a model of a document that a read found no problems with, holding that reading, without reading it again. The readers in this package build models through this method; pyright reports these calls as private use.""" model = object.__new__(cls) - model._adopt(document, context, reading, members) + model._adopt(document, context, reading, fields) return model def _adopt( @@ -534,24 +534,24 @@ def _adopt( document: dict[str, JSONValue], context: ZarrV2Context, reading: ZarrV2ArrayMetadataReading, - members: ArrayMembersV2, + fields: ArrayFieldsV2, ) -> None: self._document = document self._context = context self._reading = dataclasses.replace(reading, metadata=self) - self._members = members - # The members the model exposes, read-only at every level. + self._fields = fields + # The fields the model exposes, read-only at every level. self._shown = ( - frozen(members.fill_value), - frozen({} if members.attributes is UNSET else members.attributes), - frozen(members.extra_fields), + frozen(fields.fill_value), + frozen({} if fields.attributes is UNSET else fields.attributes), + frozen(fields.extra_fields), ) # --- the pair --------------------------------------------------------- @property def context(self) -> ZarrV2Context: - """The scope the document was read in. `update` reads new members in this scope.""" + """The scope the document was read in. `update` reads new fields in this scope.""" return self._context @property @@ -589,45 +589,47 @@ def __repr__(self) -> str: def _key_of(self) -> tuple[object, ...]: """The key `==` and `hash` compare for a v2 array model: what its document means. - The key holds each field's `field_key`, and the fill value as JSON + The key holds each declaration's `declaration_key`, and the fill value as JSON text, in its canonical spelling when a definition in scope read the - dtype. Every other member is included as it is, with JSON members + dtype. Every other field is included as it is, with JSON fields as text. No `.zattrs` counts as empty attributes, since both mean that the array has none, as in v3. """ - members = self._members + fields = self._fields return ( - members.shape, - members.chunks, - members.order, - members.dimension_separator, + fields.shape, + fields.chunks, + fields.order, + fields.dimension_separator, self._fill_value_key(), - field_key(self.dtype), - None if self.compressor is None else field_key(self.compressor), - None if self.filters is None else tuple(field_key(entry) for entry in self.filters), - json_text({} if members.attributes is UNSET else members.attributes), - json_text(members.extra_fields), + declaration_key(self.dtype), + None if self.compressor is None else declaration_key(self.compressor), + None + if self.filters is None + else tuple(declaration_key(entry) for entry in self.filters), + json_text({} if fields.attributes is UNSET else fields.attributes), + json_text(fields.extra_fields), ) def _plain_key( - self, dtype: AcceptedField[ZarrV2DataTypeDefinition[Any]] | UnclaimedField + self, dtype: AcceptedDeclaration[ZarrV2DataTypeDefinition[Any]] | UnclaimedDeclaration ) -> tuple[object, ...]: - """The members other than fields that `refines` compares. The fill value is in the spelling of `dtype`, the dtype of the more informed side.""" - members = self._members + """The fields other than declarations that `refines` compares. The fill value is in the spelling of `dtype`, the dtype of the more informed side.""" + fields = self._fields return ( - members.shape, - members.chunks, - members.order, - members.dimension_separator, - json_text(spelled_canonically(dtype, members.fill_value)), - json_text({} if members.attributes is UNSET else members.attributes), - json_text(members.extra_fields), + fields.shape, + fields.chunks, + fields.order, + fields.dimension_separator, + json_text(spelled_canonically(dtype, fields.fill_value)), + json_text({} if fields.attributes is UNSET else fields.attributes), + json_text(fields.extra_fields), ) def _fill_value_key(self) -> str: """The fill value as `==` compares it, as JSON text: in its canonical spelling when a definition in scope read the dtype, and as written otherwise.""" - fill_value = self._members.fill_value - if isinstance(self.dtype, AcceptedField): + fill_value = self._fields.fill_value + if isinstance(self.dtype, AcceptedDeclaration): return json_text(spelled_canonically(self.dtype, fill_value)) return json_text(fill_value) @@ -639,12 +641,12 @@ def __reduce__(self) -> tuple[type[ZarrV2ArrayMetadata], tuple[object, ZarrV2Con @property def shape(self) -> tuple[int, ...]: """The array's shape.""" - return self._members.shape + return self._fields.shape @property def chunks(self) -> tuple[int, ...]: """The shape of each chunk.""" - return self._members.chunks + return self._fields.chunks @property def fill_value(self) -> JSONValue: @@ -654,23 +656,23 @@ def fill_value(self) -> JSONValue: @property def order(self) -> ZarrV2ArrayOrder: """The in-chunk layout, `"C"` or `"F"`.""" - return self._members.order + return self._fields.order @property def dimension_separator(self) -> ZarrV2ArrayDimensionSeparator: """The separator between chunk indices in a chunk key: `"."` when the document writes none.""" - return self._members.dimension_separator + return self._fields.dimension_separator @property - def unclaimed_fields(self) -> dict[Loc, UnclaimedField]: - """Every field nothing in scope claims, its dtype or a codec, located as `reading.fields()` locates it: at its place in the document, where a struct dtype's field types are at their record and position, as their problems are. + def unclaimed_declarations(self) -> dict[Loc, UnclaimedDeclaration]: + """Every declaration nothing in scope claims, its dtype or a codec, located as `reading.declarations()` locates it: at its place in the document, where a struct dtype's field types are at their record and position, as their problems are. This answers whether a reader can open the array: a reader cannot decode data with a dtype or a codec it does not understand, and the scope is what the reader understands. The model only reports these - fields; refusing to open is up to the reader. + declarations; refusing to open is up to the reader. """ - return unclaimed_fields_of(self._reading.fields(), ()) + return unclaimed_declarations_of(self._reading.declarations(), ()) @property def attributes(self) -> Mapping[str, JSONValue]: @@ -679,16 +681,18 @@ def attributes(self) -> Mapping[str, JSONValue]: @property def extra_fields(self) -> Mapping[str, JSONValue]: - """Every member the spec does not define, as written, read-only at every level.""" + """Every field the spec does not define, as written, read-only at every level.""" return self._shown[2] @property - def dtype(self) -> AcceptedField[ZarrV2DataTypeDefinition[Any]] | UnclaimedField: + def dtype(self) -> AcceptedDeclaration[ZarrV2DataTypeDefinition[Any]] | UnclaimedDeclaration: """The dtype as the scope read it: read by its family's definition, or unclaimed.""" return held(self._reading.dtype) @property - def compressor(self) -> AcceptedField[ZarrV2CodecDefinition[Any]] | UnclaimedField | None: + def compressor( + self, + ) -> AcceptedDeclaration[ZarrV2CodecDefinition[Any]] | UnclaimedDeclaration | None: """The compressor as the scope read it; None when written as `null`.""" compressor = self._reading.compressor return None if compressor is None else held(compressor) @@ -696,7 +700,7 @@ def compressor(self) -> AcceptedField[ZarrV2CodecDefinition[Any]] | UnclaimedFie @property def filters( self, - ) -> tuple[AcceptedField[ZarrV2CodecDefinition[Any]] | UnclaimedField, ...] | None: + ) -> tuple[AcceptedDeclaration[ZarrV2CodecDefinition[Any]] | UnclaimedDeclaration, ...] | None: """The filters, each as the scope read it; None when written as `null`.""" filters = self._reading.filters if filters is None: @@ -708,16 +712,14 @@ def filters( # --- changing --------------------------------------------------------- - def update(self, **members: Unpack[ZarrV2ArrayMetadataUpdate]) -> ZarrV2ArrayMetadata: - """Return this model with `members` (as JSON) replacing the document's members, read in this model's own scope. A member set to `UNSET` is removed. + def update(self, **fields: Unpack[ZarrV2ArrayMetadataUpdate]) -> ZarrV2ArrayMetadata: + """Return this model with `fields` (as JSON) replacing the document's fields, read in this model's own scope. A field set to `UNSET` is removed. Raises `MetadataValidationError` when the resulting document has a - problem. So members that depend on each other must be passed + problem. So fields that depend on each other must be passed together, for example a `dtype` with a fill value of that dtype. """ - changes = member_changes( - members, ZarrV2ArrayMetadataUpdate, _ARRAY_METADATA_STANDARD_KEYS_V2 - ) + changes = field_changes(fields, ZarrV2ArrayMetadataUpdate, _ARRAY_METADATA_STANDARD_KEYS_V2) document: dict[str, object] = {**self._document, **changes} for key, value in changes.items(): if value is UNSET: @@ -732,8 +734,8 @@ def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetad every claim identically. """ context = scoped(context, CORE_V2) - if disagreements(context, claims_of(self._reading.fields())).agrees: - return self._of(self._document, context, self._reading, self._members) + if disagreements(context, claims_of(self._reading.declarations())).agrees: + return self._of(self._document, context, self._reading, self._fields) return type(self)(self._document, context=context) def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadata: @@ -746,13 +748,15 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadat that `context` claims rejects what was written. """ context = scoped(context, CORE_V2) - found = disagreements(context, claims_of(self._reading.fields())) + found = disagreements(context, claims_of(self._reading.declarations())) if len(found.conflicts) != 0: - raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) + raise ScopeConflictError( + located_conflicts(self._reading.declarations(), found.conflicts) + ) return self.with_context(context) def refines(self, other: ZarrV2ArrayMetadata) -> bool: - """Whether this model holds everything `other` holds. Each field must refine its counterpart. A `null` compressor or `null` filters refine only a `null`. Every other member must be the same, with the fill value compared in the spelling of the more informed dtype. If that dtype rejects the other model's fill value, this is not a refinement.""" + """Whether this model holds everything `other` holds. Each declaration must refine its counterpart. A `null` compressor or `null` filters refine only a `null`. Every other field must be the same, with the fill value compared in the spelling of the more informed dtype. If that dtype rejects the other model's fill value, this is not a refinement.""" if type(other) is not type(self): return False if (self.compressor is None) != (other.compressor is None): @@ -766,9 +770,9 @@ def refines(self, other: ZarrV2ArrayMetadata) -> bool: pairs = [(self.dtype, other.dtype), *zip(mine, theirs, strict=True)] if self.compressor is not None and other.compressor is not None: pairs.append((self.compressor, other.compressor)) - if not all(refines_field(one, another) for one, another in pairs): + if not all(refines_declaration(one, another) for one, another in pairs): return False - if len(fill_value_problems(self.dtype, other._members.fill_value)) != 0: + if len(fill_value_problems(self.dtype, other._fields.fill_value)) != 0: return False return self._plain_key(self.dtype) == other._plain_key(self.dtype) @@ -778,7 +782,7 @@ def refines(self, other: ZarrV2ArrayMetadata) -> bool: def create_default( cls, *, context: ZarrV2Context | None = None, **overrides: Unpack[ZarrV2ArrayMetadataUpdate] ) -> ZarrV2ArrayMetadata: - """Return a scalar `|u1` array, with any document members in `overrides` applied, read in `context`. + """Return a scalar `|u1` array, with any document fields in `overrides` applied, read in `context`. Raises `MetadataValidationError` when the resulting document has a problem. Overriding `shape` without `chunks` sets `chunks` equal to @@ -799,7 +803,7 @@ def create_default( "filters": None, "dimension_separator": ".", } - given = member_changes( + given = field_changes( overrides, ZarrV2ArrayMetadataUpdate, _ARRAY_METADATA_STANDARD_KEYS_V2 ) if "shape" in given and "chunks" not in given: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 53d12254bc..01a705c126 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -30,20 +30,20 @@ from zarr_metadata._typed_json import typeddict_keys from zarr_metadata.definition._context import disagreements, scoped from zarr_metadata.definition._definition import ( - AcceptedField, + AcceptedDeclaration, Definition, - ResolvedField, - UnclaimedField, - field_key, - field_kind, - fields_of, + ResolvedDeclaration, + UnclaimedDeclaration, + declaration_key, + declaration_kind, + declarations_of, fill_value_problems, held, resolve_refined, spelled_canonically, ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of -from zarr_metadata.definition._scope import refines as refines_field +from zarr_metadata.definition._scope import refines as refines_declaration from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, @@ -51,14 +51,14 @@ check_literal, dimension_lengths, dump_store_json, + field_changes, load_store_json, located_conflicts, - member_changes, missing_keys, reading_of, - unclaimed_fields_of, + unclaimed_declarations_of, ) -from zarr_metadata.v3._common import ZarrV3ExtensionField, ZarrV3MetadataFieldJSON +from zarr_metadata.v3._common import ZarrV3DeclarationJSON, ZarrV3ExtensionField # A runtime import: `get_type_hints` resolves the public signatures. from zarr_metadata.v3._context import ZarrV3Context # noqa: TC001 @@ -87,19 +87,19 @@ from zarr_metadata.v3.array import ZarrV3ArrayMetadataStoreKey -# The members a v3 array document must have. The members the spec defines -# are `ArrayExtraFieldDefinition.spec_members`; any other is an extra field. +# The fields a v3 array document must have. The fields the spec defines +# are `ArrayExtraFieldDefinition.spec_fields`; any other is an extra field. _ARRAY_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( ZarrV3ArrayMetadataJSON.__required_keys__ ) -def members_past_the_levels( +def fields_past_the_levels( doc: Mapping[object, object], at: Loc ) -> dict[object, ValidationProblem]: - """Each member of `doc` (a document located at `at`) that is a container nested deeper than a reader walks, mapped to its problem, located in the document. + """Each field of `doc` (a document located at `at`) that is a container nested deeper than a reader walks, mapped to its problem, located in the document. - A reader reports these members and does not walk into them. This + A reader reports these fields and does not walk into them. This matches what `refine_json` would report for the whole document: a document one level before the depth cap can hold scalars, and nothing else. @@ -113,26 +113,27 @@ def members_past_the_levels( return past -_MEMBERS_V3: Final = typeddict_keys(ZarrV3ArrayMetadataJSON).members +_FIELDS_V3: Final = typeddict_keys(ZarrV3ArrayMetadataJSON).fields _EXTENSION_POINTS_V3: Final[tuple[tuple[str, type[Definition[Any]]], ...]] = tuple( (key, kind) - for key, (annotation, _) in _MEMBERS_V3.items() - if (kind := field_kind(annotation)) is not None + for key, (annotation, _) in _FIELDS_V3.items() + if (kind := declaration_kind(annotation)) is not None ) -"""The single extension points of a v3 array document, and the kind each is read as: each member that the TypedDict annotates with a field alias.""" +"""The single extension points of a v3 array document, and the kind each is read as: each field that the TypedDict annotates with a declaration alias.""" _EXTENSION_LISTS_V3: Final[tuple[tuple[str, type[Definition[Any]]], ...]] = tuple( (key, kind) - for key, (annotation, _) in _MEMBERS_V3.items() - if get_origin(annotation) is tuple and (kind := field_kind(get_args(annotation)[0])) is not None + for key, (annotation, _) in _FIELDS_V3.items() + if get_origin(annotation) is tuple + and (kind := declaration_kind(get_args(annotation)[0])) is not None ) -"""The lists of extension points of a v3 array document, and the kind each entry is read as: each member that the TypedDict annotates as a tuple of a field alias.""" +"""The lists of extension points of a v3 array document, and the kind each entry is read as: each field that the TypedDict annotates as a tuple of a declaration alias.""" -def no_extra_fields() -> Mapping[str, ResolvedField[Any]]: +def no_extra_fields() -> Mapping[str, ResolvedDeclaration[Any]]: """The extra fields of a reading of a document that has none.""" return MappingProxyType({}) @@ -141,30 +142,30 @@ def no_extra_fields() -> Mapping[str, ResolvedField[Any]]: class ZarrV3ArrayMetadataReading: """A v3 array document as a scope read it, whatever it holds: each extension point, its codecs as a pipeline, every problem, and the model when there are no problems. - A field the document does not have is `UNSET`. A list of fields that + A declaration the document does not have is `UNSET`. A list of declarations that the document does not have as a list is empty. """ - data_type: ResolvedField[DataTypeDefinition[Any]] | UNSET = UNSET + data_type: ResolvedDeclaration[DataTypeDefinition[Any]] | UNSET = UNSET """The data type, as the scope read it.""" - chunk_grid: ResolvedField[ChunkGridDefinition[Any]] | UNSET = UNSET + chunk_grid: ResolvedDeclaration[ChunkGridDefinition[Any]] | UNSET = UNSET """The chunk grid, as the scope read it.""" - chunk_key_encoding: ResolvedField[ChunkKeyEncodingDefinition[Any]] | UNSET = UNSET + chunk_key_encoding: ResolvedDeclaration[ChunkKeyEncodingDefinition[Any]] | UNSET = UNSET """The chunk key encoding, as the scope read it.""" chunk: Chunk = dataclasses.field(default_factory=Chunk) """The chunks the first codec receives: the length of the grid's chunks along each axis of the shape, and the data type.""" pipeline: tuple[Stage, ...] = () """The codecs, read as a pipeline: each as the scope read it, with the chunk it receives.""" - storage_transformers: tuple[ResolvedField[StorageTransformerDefinition[Any]], ...] = () + storage_transformers: tuple[ResolvedDeclaration[StorageTransformerDefinition[Any]], ...] = () """The storage transformers, each as the scope read it.""" - extra_fields: Mapping[str, ResolvedField[ArrayExtraFieldDefinition[Any]]] = dataclasses.field( - default_factory=no_extra_fields + extra_fields: Mapping[str, ResolvedDeclaration[ArrayExtraFieldDefinition[Any]]] = ( + dataclasses.field(default_factory=no_extra_fields) ) - """Each member the spec does not define, as the scope read it, by key: an `UnclaimedField` when no definition in scope claims the key.""" + """Each field the spec does not define, as the scope read it, by key: an `UnclaimedDeclaration` when no definition in scope claims the key.""" problems: tuple[ValidationProblem, ...] = () """Every reason the document is not valid.""" metadata: ZarrV3ArrayMetadata | None = None - """The document's model, which holds these fields, when there is no problem; None otherwise.""" + """The document's model, which holds these declarations, when there is no problem; None otherwise.""" def __reduce__(self) -> str | tuple[object, ...]: # A reading that holds a model pickles and copies the same way the @@ -177,29 +178,29 @@ def __reduce__(self) -> str | tuple[object, ...]: values = {item.name: getattr(self, item.name) for item in dataclasses.fields(self)} return (_array_reading, ({**values, "extra_fields": dict(self.extra_fields)},)) - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Each field in the document, as the scope read it, with its location in the document. + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Each declaration in the document, as the scope read it, with its location in the document. The order is: the extension points, then each codec and storage transformer at its index, then each extra field at its key. Each is - followed by the fields nested in it, as `fields_of` returns them: a + followed by the declarations nested in it, as `declarations_of` returns them: a shard's codecs, or a struct's field types. `with_problems` returns - each field with its problems. + each declaration with its problems. """ - own: tuple[tuple[str, ResolvedField[Any] | UNSET], ...] = ( + own: tuple[tuple[str, ResolvedDeclaration[Any] | UNSET], ...] = ( ("data_type", self.data_type), ("chunk_grid", self.chunk_grid), ("chunk_key_encoding", self.chunk_key_encoding), ) - for key, field in own: - if field is not UNSET: - yield from fields_of(field, (key,)) + for key, declaration in own: + if declaration is not UNSET: + yield from declarations_of(declaration, (key,)) for index, stage in enumerate(self.pipeline): - yield from fields_of(stage.codec, ("codecs", index)) + yield from declarations_of(stage.codec, ("codecs", index)) for index, transformer in enumerate(self.storage_transformers): - yield from fields_of(transformer, ("storage_transformers", index)) + yield from declarations_of(transformer, ("storage_transformers", index)) for key, extra in self.extra_fields.items(): - yield from fields_of(extra, (key,)) + yield from declarations_of(extra, (key,)) def _array_reading(values: dict[str, Any]) -> ZarrV3ArrayMetadataReading: @@ -209,8 +210,8 @@ def _array_reading(values: dict[str, Any]) -> ZarrV3ArrayMetadataReading: @dataclass(frozen=True, slots=True) -class ArrayMembersV3: - """The members other than fields of a v3 array document that a read found no problems with, refined as the model holds them.""" +class ArrayFieldsV3: + """The fields other than declarations of a v3 array document that a read found no problems with, refined as the model holds them.""" shape: tuple[int, ...] fill_value: JSONValue @@ -218,16 +219,16 @@ class ArrayMembersV3: attributes: dict[str, JSONValue] extra_fields: dict[str, JSONValue] document: dict[str, JSONValue] - """The refined document that the model holds: each member as the read refined it, in the order the document writes them.""" + """The refined document that the model holds: each field as the read refined it, in the order the document writes them.""" def read_array_v3( value: object, context: ZarrV3Context, *, at: Loc = () -) -> tuple[ZarrV3ArrayMetadataReading, ArrayMembersV3 | None]: - """Read `value`, a v3 array document, in `context`, without building its model. Return the reading and the other members, refined; the members are None when the document has a problem. +) -> tuple[ZarrV3ArrayMetadataReading, ArrayFieldsV3 | None]: + """Read `value`, a v3 array document, in `context`, without building its model. Return the reading and the other fields, refined; the fields are None when the document has a problem. `read_array_metadata_v3` builds the model from these two results. A - field object in the document (an `AcceptedField` built by hand) is + declaration object in the document (an `AcceptedDeclaration` built by hand) is not JSON, and is refused as such. `at` is the location of this document within the input document; a document in consolidated metadata is three levels below its group's document. Nesting depth @@ -238,7 +239,7 @@ def read_array_v3( return ZarrV3ArrayMetadataReading(problems=not_an_object(value)), None doc = value problems: list[ValidationProblem] = list(missing_keys(_ARRAY_METADATA_REQUIRED_KEYS_V3, doc)) - past = members_past_the_levels(doc, at) + past = fields_past_the_levels(doc, at) problems.extend(past.values()) whole, doc = doc, {key: item for key, item in doc.items() if key not in past} extra_fields, extra, found = read_extra_fields_v3(doc, ArrayExtraFieldDefinition, context, at) @@ -258,7 +259,7 @@ def read_array_v3( # `context` that claims the name. `must_understand: false` keeps its # meaning where it has one: on an unknown top-level extension *field*, # which a reader really can skip. - # Each member is refined once, here, and the refined value is kept. + # Each field is refined once, here, and the refined value is kept. # The document the model holds is built from these values, so nothing # is walked twice. refined: dict[str, JSONValue] = { @@ -267,7 +268,7 @@ def read_array_v3( "node_type": "array", "shape": shape, } - read: dict[str, ResolvedField[Any]] = {} + read: dict[str, ResolvedDeclaration[Any]] = {} for key, kind in _EXTENSION_POINTS_V3: if key in doc: refined[key], found = refine_json(doc[key], (*at, key)) @@ -292,7 +293,7 @@ def read_array_v3( if "chunk_grid" in read and shape is not None: lengths, found = chunk_grid_lengths(read["chunk_grid"], shape, ("chunk_grid",)) problems.extend(found) - listed: dict[str, list[ResolvedField[Any]]] = {} + listed: dict[str, list[ResolvedDeclaration[Any]]] = {} for key, kind in _EXTENSION_LISTS_V3: if key in doc: entries = doc[key] @@ -327,7 +328,7 @@ def read_array_v3( if "dimension_names" in doc: # Simple typed sequences (dimension_names, shape, chunks) report a single # field-level loc, not per-bad-item locs; per-index locs are reserved for - # the metadata-field lists (codecs, storage_transformers). + # the declaration lists (codecs, storage_transformers). names = doc["dimension_names"] if not is_array(names): problems.append( @@ -365,7 +366,7 @@ def read_array_v3( # each string is as JSON writes it. dimension_names = cast("tuple[str | None, ...]", refine_json(doc["dimension_names"])[0]) refined["dimension_names"] = dimension_names - members = ArrayMembersV3( + fields = ArrayFieldsV3( shape=shape, fill_value=fill_value, dimension_names=dimension_names, @@ -373,7 +374,7 @@ def read_array_v3( extra_fields=extra_fields, document={key: refined[key] for key in cast("Mapping[str, object]", doc)}, ) - return reading, members + return reading, fields def read_extra_fields_v3( @@ -381,19 +382,21 @@ def read_extra_fields_v3( kind: type[ExtraFieldDefinition[Any]], context: ZarrV3Context, at: Loc, -) -> tuple[dict[str, JSONValue], dict[str, ResolvedField[Any]], tuple[ValidationProblem, ...]]: - """Read each member of `doc` (a document at `at`) that is not one of the `spec_members` of `kind` as an extra field of `kind`. +) -> tuple[ + dict[str, JSONValue], dict[str, ResolvedDeclaration[Any]], tuple[ValidationProblem, ...] +]: + """Read each field of `doc` (a document at `at`) that is not one of the `spec_fields` of `kind` as an extra field of `kind`. - Returns three values: each member that is JSON, refined; each member + Returns three values: each field that is JSON, refined; each field as the scope read it, by key; and every problem, located in `doc`. A - member that a definition in `context` claims is read by that + field that a definition in `context` claims is read by that definition, and must be an object whose numbers are finite, unless the - definition reads user data. Any other member is not read: it is an - `UnclaimedField`, and, like attributes, need only be JSON as Python's + definition reads user data. Any other field is not read: it is an + `UnclaimedDeclaration`, and, like attributes, need only be JSON as Python's `json` module reads it. """ values: dict[str, JSONValue] = {} - fields: dict[str, ResolvedField[Any]] = {} + declarations: dict[str, ResolvedDeclaration[Any]] = {} problems: list[ValidationProblem] = [] for key, value in doc.items(): if not isinstance(key, str): @@ -401,23 +404,23 @@ def read_extra_fields_v3( ValidationProblem((), f"non-string top-level key {shown_key(key)}", "invalid_type") ) continue - if key in kind.spec_members: + if key in kind.spec_fields: continue - member = {key: value} + field = {key: value} definition = context.claimant(kind, key) - refined, found = kind._refined(member, (*at, key), definition) # pyright: ignore[reportPrivateUsage] + refined, found = kind._refined(field, (*at, key), definition) # pyright: ignore[reportPrivateUsage] if isinstance(refined, dict): values[key] = refined[key] if definition is None: - fields[key] = UnclaimedField(json=refined, name=key, read_as=kind) + declarations[key] = UnclaimedDeclaration(json=refined, name=key, read_as=kind) continue - fields[key], found = resolve_refined(member, refined, found, kind, context, (*at, key)) + declarations[key], found = resolve_refined(field, refined, found, kind, context, (*at, key)) problems.extend(within(found, at)) - return values, fields, tuple(problems) + return values, declarations, tuple(problems) def must_understand_subset( - fields: Mapping[str, AcceptedField[Any] | UnclaimedField], + declarations: Mapping[str, AcceptedDeclaration[Any] | UnclaimedDeclaration], values: Mapping[str, ZarrV3ExtensionField], ) -> dict[str, ZarrV3ExtensionField]: """The extra fields, by key and as written, that no definition in scope reads and that a reader is obligated to understand. @@ -433,12 +436,12 @@ def must_understand_subset( """ return { name: values[name] - for name, field in fields.items() - if isinstance(field, UnclaimedField) and not _waived(values[name]) + for name, declaration in declarations.items() + if isinstance(declaration, UnclaimedDeclaration) and not _waived(values[name]) } -def waived_members(values: Mapping[str, ZarrV3ExtensionField]) -> list[str]: +def waived_fields(values: Mapping[str, ZarrV3ExtensionField]) -> list[str]: """The keys of the extra fields in `values` (by key, as written) that say `"must_understand": false`, which a reader may ignore.""" return [key for key, value in values.items() if _waived(value)] @@ -449,23 +452,23 @@ def _waived(value: ZarrV3ExtensionField) -> bool: class ZarrV3ArrayMetadataUpdate(TypedDict, total=False, closed=True): - """The members `ZarrV3ArrayMetadata.update` replaces: each in the form a document writes it, or `UNSET` to remove a member that a document may omit. + """The fields `ZarrV3ArrayMetadata.update` replaces: each in the form a document writes it, or `UNSET` to remove a field that a document may omit. - The members a document may omit are `dimension_names`, `attributes`, - `storage_transformers`, and any member the spec does not define. Those + The fields a document may omit are `dimension_names`, `attributes`, + `storage_transformers`, and any field the spec does not define. Those go in `extra_fields`, by key: each one given is set, or removed when - given as `UNSET`, and the others are kept. A member is given by its own + given as `UNSET`, and the others are kept. A field is given by its own name only when the spec defines it, so a misspelled name is refused. """ shape: tuple[int, ...] - data_type: ZarrV3MetadataFieldJSON - chunk_grid: ZarrV3MetadataFieldJSON - chunk_key_encoding: ZarrV3MetadataFieldJSON + data_type: ZarrV3DeclarationJSON + chunk_grid: ZarrV3DeclarationJSON + chunk_key_encoding: ZarrV3DeclarationJSON fill_value: JSONValue - codecs: tuple[ZarrV3MetadataFieldJSON, ...] + codecs: tuple[ZarrV3DeclarationJSON, ...] attributes: Mapping[str, JSONValue] | UNSET - storage_transformers: tuple[ZarrV3MetadataFieldJSON, ...] | UNSET + storage_transformers: tuple[ZarrV3DeclarationJSON, ...] | UNSET dimension_names: tuple[str | None, ...] | UNSET extra_fields: Mapping[str, ZarrV3ExtensionField | UNSET] @@ -475,35 +478,35 @@ class ZarrV3ArrayMetadata(Keyed): The model is this pair. `to_json` returns the document as written, refined (arrays as tuples, string keys), and `context` is the scope. - Every typed member is a view of the reading of the document in the + Every typed attribute is a view of the reading of the document in the scope. `data_type`, `chunk_grid`, `chunk_key_encoding`, and each codec - and storage transformer are fields as the scope read them: an - `AcceptedField` (read by the definition that claims its name) or an - `UnclaimedField`, and so is each member in `extra_fields`, which the + and storage transformer are declarations as the scope read them: an + `AcceptedDeclaration` (read by the definition that claims its name) or an + `UnclaimedDeclaration`, and so is each field in `extra_fields`, which the spec does not define. `shape`, `fill_value`, `dimension_names` and `attributes` are the values the read refined. A model is only built by reading: the constructor reads `document` in `context` and raises `MetadataValidationError` with every problem, so no model is invalid. Two models are equal when their documents mean the same thing in their scopes, as the model key defines. The scope - itself is not compared. `update` reads new members in the model's own + itself is not compared. `update` reads new fields in the model's own scope. `with_context` and `refined_in` read the document in another scope. A model pickles as its pair, provided the definitions in its scope can be pickled, which requires their functions to be defined at the top level of a module. """ - __slots__ = ("_context", "_document", "_members", "_reading", "_shown") + __slots__ = ("_context", "_document", "_fields", "_reading", "_shown") zarr_format: Final[Literal[3]] = 3 # noqa: PYI064 - a bare Final does not narrow a union of models node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: context = scoped(context, CORE_AND_EXTENSIONS) - reading, members = read_array_v3(document, context) - if members is None: + reading, fields = read_array_v3(document, context) + if fields is None: raise MetadataValidationError(reading.problems) - self._adopt(members.document, context, reading, members) + self._adopt(fields.document, context, reading, fields) @classmethod def _of( @@ -511,11 +514,11 @@ def _of( document: dict[str, JSONValue], context: ZarrV3Context, reading: ZarrV3ArrayMetadataReading, - members: ArrayMembersV3, + fields: ArrayFieldsV3, ) -> ZarrV3ArrayMetadata: """Build a model of a document that a read found no problems with, holding that reading, without reading it again. The readers in this package build models through this method; pyright reports these calls as private use.""" model = object.__new__(cls) - model._adopt(document, context, reading, members) + model._adopt(document, context, reading, fields) return model def _adopt( @@ -523,19 +526,19 @@ def _adopt( document: dict[str, JSONValue], context: ZarrV3Context, reading: ZarrV3ArrayMetadataReading, - members: ArrayMembersV3, + fields: ArrayFieldsV3, ) -> None: self._document = document self._context = context # The reading holds this model, in the form `read_array_metadata_v3` # returns it, no matter how the model was built. self._reading = dataclasses.replace(reading, metadata=self) - self._members = members - # The members the model exposes, read-only at every level. + self._fields = fields + # The fields the model exposes, read-only at every level. self._shown = ( - frozen(members.fill_value), - frozen(members.attributes), - frozen(members.extra_fields), + frozen(fields.fill_value), + frozen(fields.attributes), + frozen(fields.extra_fields), MappingProxyType({key: held(extra) for key, extra in reading.extra_fields.items()}), ) @@ -543,12 +546,12 @@ def _adopt( @property def context(self) -> ZarrV3Context: - """The scope the document was read in. `update` reads new members in this scope.""" + """The scope the document was read in. `update` reads new fields in this scope.""" return self._context @property def reading(self) -> ZarrV3ArrayMetadataReading: - """The document as the scope read it: each field, the pipeline, and the chunk each codec receives.""" + """The document as the scope read it: each declaration, the pipeline, and the chunk each codec receives.""" return self._reading def to_json(self) -> ZarrV3ArrayMetadataJSON: @@ -571,18 +574,18 @@ def __repr__(self) -> str: def _key_of(self) -> tuple[object, ...]: """The key `==` and `hash` compare for a v3 array model: what its document means, as computed by `array_key_of`.""" - return array_key_of(self._reading, self._members) + return array_key_of(self._reading, self._fields) def _plain_key( - self, data_type: AcceptedField[DataTypeDefinition[Any]] | UnclaimedField + self, data_type: AcceptedDeclaration[DataTypeDefinition[Any]] | UnclaimedDeclaration ) -> tuple[object, ...]: - """The members other than fields that `refines` compares. The fill value is in the spelling of `data_type`, the data type of the more informed side.""" - members = self._members + """The fields other than declarations that `refines` compares. The fill value is in the spelling of `data_type`, the data type of the more informed side.""" + fields = self._fields return ( - members.shape, - json_text(spelled_canonically(data_type, members.fill_value)), - members.dimension_names, - json_text(members.attributes), + fields.shape, + json_text(spelled_canonically(data_type, fields.fill_value)), + fields.dimension_names, + json_text(fields.attributes), ) def __reduce__(self) -> tuple[type[ZarrV3ArrayMetadata], tuple[object, ZarrV3Context]]: @@ -595,7 +598,7 @@ def __reduce__(self) -> tuple[type[ZarrV3ArrayMetadata], tuple[object, ZarrV3Con @property def shape(self) -> tuple[int, ...]: """The array's shape.""" - return self._members.shape + return self._fields.shape @property def fill_value(self) -> JSONValue: @@ -605,7 +608,7 @@ def fill_value(self) -> JSONValue: @property def dimension_names(self) -> tuple[str | None, ...] | UNSET: """The dimension names; `UNSET` when the document writes none.""" - return self._members.dimension_names + return self._fields.dimension_names @property def attributes(self) -> Mapping[str, JSONValue]: @@ -615,27 +618,29 @@ def attributes(self) -> Mapping[str, JSONValue]: @property def extra_fields( self, - ) -> Mapping[str, AcceptedField[ArrayExtraFieldDefinition[Any]] | UnclaimedField]: - """Each member the spec does not define, by key, as the scope read it: an `AcceptedField` when a definition in scope claims the key, and an `UnclaimedField` otherwise.""" + ) -> Mapping[str, AcceptedDeclaration[ArrayExtraFieldDefinition[Any]] | UnclaimedDeclaration]: + """Each field the spec does not define, by key, as the scope read it: an `AcceptedDeclaration` when a definition in scope claims the key, and an `UnclaimedDeclaration` otherwise.""" return self._shown[3] @property - def unclaimed_fields(self) -> dict[Loc, UnclaimedField]: - """Every field nothing in scope claims, at any depth, by its location in the document, except an extra member that says `"must_understand": false` and every field inside it, claimed or not, since a reader may ignore such a member whole. + def unclaimed_declarations(self) -> dict[Loc, UnclaimedDeclaration]: + """Every declaration nothing in scope claims, at any depth, by its location in the document, except an extra field that says `"must_understand": false` and every declaration inside it, claimed or not, since a reader may ignore such a field whole. This answers whether a reader can open the array: the spec requires - a reader to refuse a node with a field it does not understand, and + a reader to refuse a node with a declaration it does not understand, and the scope is what the reader understands. A codec inside a shard is located inside the shard's configuration. The model only reports - these fields; refusing to open is up to the reader. + these declarations; refusing to open is up to the reader. """ - return unclaimed_fields_of(self._reading.fields(), waived_members(self._shown[2])) + return unclaimed_declarations_of( + self._reading.declarations(), waived_fields(self._shown[2]) + ) @property def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: """The extra fields, by key and as written, that no definition in scope reads and that a reader is obligated to understand. - Each unclaimed member of `extra_fields` not explicitly waived with + Each unclaimed field of `extra_fields` not explicitly waived with `must_understand: false` (the spec's implicit-true rule, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578). The scope is what the reader recognizes, so a compliant reader MUST fail to open the array when this is not empty. The model only reports these fields; @@ -644,44 +649,48 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: return must_understand_subset(self.extra_fields, self._shown[2]) @property - def data_type(self) -> AcceptedField[DataTypeDefinition[Any]] | UnclaimedField: + def data_type(self) -> AcceptedDeclaration[DataTypeDefinition[Any]] | UnclaimedDeclaration: """The data type, as the scope read it.""" return held(self._reading.data_type) @property - def chunk_grid(self) -> AcceptedField[ChunkGridDefinition[Any]] | UnclaimedField: + def chunk_grid(self) -> AcceptedDeclaration[ChunkGridDefinition[Any]] | UnclaimedDeclaration: """The chunk grid, as the scope read it.""" return held(self._reading.chunk_grid) @property - def chunk_key_encoding(self) -> AcceptedField[ChunkKeyEncodingDefinition[Any]] | UnclaimedField: + def chunk_key_encoding( + self, + ) -> AcceptedDeclaration[ChunkKeyEncodingDefinition[Any]] | UnclaimedDeclaration: """The chunk key encoding, as the scope read it.""" return held(self._reading.chunk_key_encoding) @property - def codecs(self) -> tuple[AcceptedField[CodecDefinition[Any]] | UnclaimedField, ...]: + def codecs( + self, + ) -> tuple[AcceptedDeclaration[CodecDefinition[Any]] | UnclaimedDeclaration, ...]: """The codecs, each as the scope read it, in pipeline order.""" return tuple(held(stage.codec) for stage in self._reading.pipeline) @property def storage_transformers( self, - ) -> tuple[AcceptedField[StorageTransformerDefinition[Any]] | UnclaimedField, ...]: + ) -> tuple[AcceptedDeclaration[StorageTransformerDefinition[Any]] | UnclaimedDeclaration, ...]: """The storage transformers, each as the scope read it.""" return tuple(held(entry) for entry in self._reading.storage_transformers) # --- changing --------------------------------------------------------- - def update(self, **members: Unpack[ZarrV3ArrayMetadataUpdate]) -> ZarrV3ArrayMetadata: - """Return this model with `members` (as JSON) replacing the document's members, read in this model's own scope. A member set to `UNSET` is removed. A member the spec does not define is given in `extra_fields`. + def update(self, **fields: Unpack[ZarrV3ArrayMetadataUpdate]) -> ZarrV3ArrayMetadata: + """Return this model with `fields` (as JSON) replacing the document's fields, read in this model's own scope. A field set to `UNSET` is removed. A field the spec does not define is given in `extra_fields`. - Raises `TypeError` for a member the update does not take. Raises + Raises `TypeError` for a field the update does not take. Raises `MetadataValidationError` when the resulting document has a - problem. So members that depend on each other must be passed + problem. So fields that depend on each other must be passed together, for example a `shape` with a grid that fits it. """ - changes = member_changes( - members, ZarrV3ArrayMetadataUpdate, ArrayExtraFieldDefinition.spec_members + changes = field_changes( + fields, ZarrV3ArrayMetadataUpdate, ArrayExtraFieldDefinition.spec_fields ) document: dict[str, object] = {**self._document, **changes} for key, value in changes.items(): @@ -697,8 +706,8 @@ def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetad every claim identically. """ context = scoped(context, CORE_AND_EXTENSIONS) - if disagreements(context, claims_of(self._reading.fields())).agrees: - return self._of(self._document, context, self._reading, self._members) + if disagreements(context, claims_of(self._reading.declarations())).agrees: + return self._of(self._document, context, self._reading, self._fields) return type(self)(self._document, context=context) def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadata: @@ -714,13 +723,15 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadat any scope. """ context = scoped(context, CORE_AND_EXTENSIONS) - found = disagreements(context, claims_of(self._reading.fields())) + found = disagreements(context, claims_of(self._reading.declarations())) if len(found.conflicts) != 0: - raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) + raise ScopeConflictError( + located_conflicts(self._reading.declarations(), found.conflicts) + ) return self.with_context(context) def refines(self, other: ZarrV3ArrayMetadata) -> bool: - """Whether this model holds everything `other` holds. Each field must refine its counterpart, in the order `refines` defines for fields (including the fields nested in each field). Every other member must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement.""" + """Whether this model holds everything `other` holds. Each declaration must refine its counterpart, in the order `refines` defines for declarations (including the declarations nested in each declaration). Every other field must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement.""" if type(other) is not type(self): return False if len(self.codecs) != len(other.codecs) or len(self.storage_transformers) != len( @@ -737,9 +748,9 @@ def refines(self, other: ZarrV3ArrayMetadata) -> bool: *zip(self.storage_transformers, other.storage_transformers, strict=True), *((self.extra_fields[key], other.extra_fields[key]) for key in self.extra_fields), ) - if not all(refines_field(mine, theirs) for mine, theirs in pairs): + if not all(refines_declaration(mine, theirs) for mine, theirs in pairs): return False - if len(fill_value_problems(self.data_type, other._members.fill_value)) != 0: + if len(fill_value_problems(self.data_type, other._fields.fill_value)) != 0: return False return self._plain_key(self.data_type) == other._plain_key(self.data_type) @@ -752,10 +763,10 @@ def create_default( context: ZarrV3Context | None = None, **overrides: Unpack[ZarrV3ArrayMetadataUpdate], ) -> ZarrV3ArrayMetadata: - """Return a scalar `uint8` array, with any document members in `overrides` applied, read in `context`. `overrides` takes what `update` takes, including `UNSET` to drop a member. + """Return a scalar `uint8` array, with any document fields in `overrides` applied, read in `context`. `overrides` takes what `update` takes, including `UNSET` to drop a field. Raises `MetadataValidationError` when the resulting document has a - problem. So members that depend on each other must be passed + problem. So fields that depend on each other must be passed together: a data type with a fill value of that type, or a grid with the shape it fits. The default codec is `bytes` with little `endian`, which accepts a data type of any fixed size. Overriding @@ -767,8 +778,8 @@ def create_default( """ # The grid is derived only from a valid shape. An invalid shape # derives no grid, and the read reports it. - changes = member_changes( - overrides, ZarrV3ArrayMetadataUpdate, ArrayExtraFieldDefinition.spec_members + changes = field_changes( + overrides, ZarrV3ArrayMetadataUpdate, ArrayExtraFieldDefinition.spec_fields ) lengths, _ = dimension_lengths(changes, "shape") document: dict[str, object] = { @@ -815,40 +826,38 @@ def from_key_value( return cls(load_store_json(mapping, ZARR_V3_ARRAY_METADATA_STORE_KEY), context=context) -def array_key_of( - reading: ZarrV3ArrayMetadataReading, members: ArrayMembersV3 -) -> tuple[object, ...]: - """A key for what a v3 array document means, from its `reading` and refined `members`. `==` and `hash` compare this key for the model, and two listings of one node are compared by it. +def array_key_of(reading: ZarrV3ArrayMetadataReading, fields: ArrayFieldsV3) -> tuple[object, ...]: + """A key for what a v3 array document means, from its `reading` and refined `fields`. `==` and `hash` compare this key for the model, and two listings of one node are compared by it. - The key holds each field's `field_key`, and the fill value as JSON + The key holds each declaration's `declaration_key`, and the fill value as JSON text: in its canonical spelling when a definition in scope read the - data type, and as written otherwise. Every other member is included - as it is, with JSON members as text. + data type, and as written otherwise. Every other field is included + as it is, with JSON fields as text. """ data_type = held(reading.data_type) fill_value = ( - spelled_canonically(data_type, members.fill_value) - if isinstance(data_type, AcceptedField) - else members.fill_value + spelled_canonically(data_type, fields.fill_value) + if isinstance(data_type, AcceptedDeclaration) + else fields.fill_value ) return ( - members.shape, + fields.shape, json_text(fill_value), - field_key(data_type), - field_key(held(reading.chunk_grid)), - tuple(field_key(held(stage.codec)) for stage in reading.pipeline), - field_key(held(reading.chunk_key_encoding)), - members.dimension_names, - json_text(members.attributes), - tuple(field_key(held(entry)) for entry in reading.storage_transformers), + declaration_key(data_type), + declaration_key(held(reading.chunk_grid)), + tuple(declaration_key(held(stage.codec)) for stage in reading.pipeline), + declaration_key(held(reading.chunk_key_encoding)), + fields.dimension_names, + json_text(fields.attributes), + tuple(declaration_key(held(entry)) for entry in reading.storage_transformers), extra_fields_key(reading.extra_fields), ) -def extra_fields_key(extra_fields: Mapping[str, ResolvedField[Any]]) -> tuple[object, ...]: - """A key for what a document's extra fields mean: each key with its field's `field_key`, sorted by key.""" +def extra_fields_key(extra_fields: Mapping[str, ResolvedDeclaration[Any]]) -> tuple[object, ...]: + """A key for what a document's extra fields mean: each key with its declaration's `declaration_key`, sorted by key.""" return tuple( - (key, field_key(extra)) + (key, declaration_key(extra)) for key, extra in sorted(extra_fields.items(), key=lambda item: item[0]) ) @@ -859,8 +868,8 @@ def read_array_metadata_v3( """Read `value`, a v3 array document, in `context`, whatever it holds. The result holds everything the read finds: each extension point as - `context` read it (an `AcceptedField` read by the definition that - claims its name, an `UnclaimedField`, or a `RefusedField`), the chunks + `context` read it (an `AcceptedDeclaration` read by the definition that + claims its name, an `UnclaimedDeclaration`, or a `RefusedDeclaration`), the chunks the codecs receive, each codec with the chunk it receives, every problem, and, when there are no problems, the document's model, which holds the same reading. @@ -884,13 +893,13 @@ def read_array_metadata_v3( declare `must_understand: false`; the model reports these as `must_understand_fields`. - To apply a policy over the fields (for example, allow only the core - spec's fields), iterate over the reading's `fields()`. A value that - is not an object has no fields. + To apply a policy over the declarations (for example, allow only the core + spec's declarations), iterate over the reading's `declarations()`. A value that + is not an object has no declarations. """ context = scoped(context, CORE_AND_EXTENSIONS) - reading, members = read_array_v3(value, context) - if members is None: + reading, fields = read_array_v3(value, context) + if fields is None: return reading - model = ZarrV3ArrayMetadata._of(members.document, context, reading, members) # pyright: ignore[reportPrivateUsage] + model = ZarrV3ArrayMetadata._of(fields.document, context, reading, fields) # pyright: ignore[reportPrivateUsage] return model.reading diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py index eb290a6cb7..b5b361903f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py @@ -49,9 +49,9 @@ StoreKey, check_literal, dump_store_json, + field_changes, load_store_json, located_conflicts, - member_changes, missing_keys, reading_of, unexpected_keys, @@ -70,7 +70,7 @@ from collections.abc import Iterator from zarr_metadata._typed_json import Loc - from zarr_metadata.definition._definition import ResolvedField, UnclaimedField + from zarr_metadata.definition._definition import ResolvedDeclaration, UnclaimedDeclaration from zarr_metadata.v2.attributes import ZarrV2AttributesStoreKey from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataStoreKey from zarr_metadata.v2.group import ZarrV2GroupMetadataStoreKey @@ -92,7 +92,7 @@ def _group_problems_v2(value: object) -> tuple[ValidationProblem, ...]: - """Every reason `value` is not a v2 group document in the merged form: the `.zgroup` members, plus an optional `attributes` object from `.zattrs`. A group has no fields for a scope to read, so no scope is needed.""" + """Every reason `value` is not a v2 group document in the merged form: the `.zgroup` fields, plus an optional `attributes` object from `.zattrs`. A group has no declarations for a scope to read, so no scope is needed.""" if not is_object(value): return not_an_object(value) doc = value @@ -110,20 +110,20 @@ def _group_problems_v2(value: object) -> tuple[ValidationProblem, ...]: f"belong in {ZARR_V2_ATTRIBUTES_STORE_KEY}", "invalid_value", ) -"""The problem of a `.zgroup` that holds `attributes`, which belong in `.zattrs`. The kind is the one `.zarray` gives the same member: the key is not unknown, but taken by the attributes, and a caller who tolerates an unknown key would drop the attributes the member holds.""" +"""The problem of a `.zgroup` that holds `attributes`, which belong in `.zattrs`. The kind is the one `.zarray` gives the same field: the key is not unknown, but taken by the attributes, and a caller who tolerates an unknown key would drop the attributes the field holds.""" @dataclass(frozen=True, slots=True) class ZarrV2GroupMetadataReading: - """A v2 group document as a scope read it: every problem, and the model when there are no problems. A group has no fields for a scope to read.""" + """A v2 group document as a scope read it: every problem, and the model when there are no problems. A group has no declarations for a scope to read.""" problems: tuple[ValidationProblem, ...] = () """Every reason the document is not valid.""" metadata: ZarrV2GroupMetadata | None = None """The document's model when there is no problem; None otherwise.""" - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Its fields as read: none, since a group has no fields.""" + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Its declarations as read: none, since a group has no declarations.""" return iter(()) def __reduce__(self) -> str | tuple[object, ...]: @@ -136,7 +136,7 @@ def __reduce__(self) -> str | tuple[object, ...]: def read_group_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2GroupMetadataReading: - """Read `value`, a v2 group document in the merged form (`.zgroup` plus `attributes` from `.zattrs`), in `context` (`CORE_V2` when none is given), whatever it holds: every problem, and the model when there are none. A group has no fields for a scope to read; `context` is taken for consistency with every other v2 reader.""" + """Read `value`, a v2 group document in the merged form (`.zgroup` plus `attributes` from `.zattrs`), in `context` (`CORE_V2` when none is given), whatever it holds: every problem, and the model when there are none. A group has no declarations for a scope to read; `context` is taken for consistency with every other v2 reader.""" context = scoped(context, CORE_V2) problems = _group_problems_v2(value) if len(problems) != 0: @@ -158,8 +158,8 @@ def metadata(self) -> None: """Its model: always None, since there is no node to model.""" return None - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Its fields as read: none, since no document is read.""" + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Its declarations as read: none, since no document is read.""" return iter(()) @@ -188,9 +188,9 @@ def read_node_metadata_v2( reading of its node type with no model, and with that problem followed by the problems of the rest of the file. Each problem is located in the file that holds it, such as `(".zarray", "order")` or - `(".zattrs", "x")`, and so is each field of an array's reading, whose + `(".zattrs", "x")`, and so is each declaration of an array's reading, whose `at` is `(".zarray",)`. The model reads its merged document, so its - own `reading` locates fields there, at `("dtype",)`. This is the + own `reading` locates declarations there, at `("dtype",)`. This is the v2 counterpart of `read_node_metadata_v3`, which reads one `zarr.json` and its `node_type`. """ @@ -245,7 +245,7 @@ def node_metadata_from_key_value_v2( class ZarrV2GroupMetadataUpdate(TypedDict, total=False): - """The members `ZarrV2GroupMetadata.update` replaces: `attributes` in the form a `.zattrs` writes them, or `UNSET` for no `.zattrs`.""" + """The fields `ZarrV2GroupMetadata.update` replaces: `attributes` in the form a `.zattrs` writes them, or `UNSET` for no `.zattrs`.""" attributes: Mapping[str, JSONValue] | UNSET @@ -258,7 +258,7 @@ class ZarrV2GroupMetadata(Keyed): them) as written, refined. `attributes` is empty when no `.zattrs` exists, as in a v3 model. The document still tells the two apart: `to_json` and `to_key_value` write `attributes` and `.zattrs` only when - a `.zattrs` exists, even an empty one. A group has no fields + a `.zattrs` exists, even an empty one. A group has no declarations for a scope to read, so the scope is held only for consistency with the other models: `update` reads new attributes in it, and no other scope conflicts with the reading. A model is only built by reading: @@ -313,8 +313,8 @@ def context(self) -> ZarrV2Context: return self._context @property - def unclaimed_fields(self) -> dict[Loc, UnclaimedField]: - """Every field nothing in scope claims. A v2 group holds no fields, so this is always empty; it is here so a reader can ask any node the same question.""" + def unclaimed_declarations(self) -> dict[Loc, UnclaimedDeclaration]: + """Every declaration nothing in scope claims. A v2 group holds no declarations, so this is always empty; it is here so a reader can ask any node the same question.""" return {} @property @@ -357,9 +357,9 @@ def _key_of(self) -> tuple[object, ...]: def __reduce__(self) -> tuple[type[ZarrV2GroupMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) - def update(self, **members: Unpack[ZarrV2GroupMetadataUpdate]) -> ZarrV2GroupMetadata: + def update(self, **fields: Unpack[ZarrV2GroupMetadataUpdate]) -> ZarrV2GroupMetadata: """Return this model with `attributes` replacing the document's attributes, read in this model's own scope. `UNSET` removes them. Raises `MetadataValidationError` when the resulting document has a problem.""" - changes = member_changes(members, ZarrV2GroupMetadataUpdate, frozenset()) + changes = field_changes(fields, ZarrV2GroupMetadataUpdate, frozenset()) document: dict[str, object] = {**self._document, **changes} for key, value in changes.items(): if value is UNSET: @@ -372,7 +372,7 @@ def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetad return self._of(self._document, context) def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: - """Return this document read in `context`. A group has no fields, so no scope conflicts with its reading, and this is the same as `with_context`.""" + """Return this document read in `context`. A group has no declarations, so no scope conflicts with its reading, and this is the same as `with_context`.""" return self.with_context(context) def refines(self, other: ZarrV2GroupMetadata) -> bool: @@ -384,7 +384,7 @@ def create_default( cls, *, context: ZarrV2Context | None = None, **overrides: Unpack[ZarrV2GroupMetadataUpdate] ) -> ZarrV2GroupMetadata: """Return a group with no `.zattrs`, with `overrides` applied, read in `context`. Raises `MetadataValidationError` when the resulting document has a problem.""" - changes = member_changes(overrides, ZarrV2GroupMetadataUpdate, frozenset()) + changes = field_changes(overrides, ZarrV2GroupMetadataUpdate, frozenset()) given = {key: value for key, value in changes.items() if value is not UNSET} return cls({"zarr_format": 2, **given}, context=context) @@ -433,11 +433,11 @@ class ZarrV2ConsolidatedMetadataReading: metadata: ZarrV2ConsolidatedMetadata | None = None """The document's model when there is no problem; None otherwise.""" - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Each field of each array node, as read, located in the `.zmetadata`: under the node's `.zarray` entry, at the key the document writes for it. Nothing when there is no model, and nothing for a group, which has no fields.""" + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Each declaration of each array node, as read, located in the `.zmetadata`: under the node's `.zarray` entry, at the key the document writes for it. Nothing when there is no model, and nothing for a group, which has no declarations.""" if self.metadata is None: return - yield from self.metadata._located_fields() # pyright: ignore[reportPrivateUsage] + yield from self.metadata._located_declarations() # pyright: ignore[reportPrivateUsage] def __reduce__(self) -> str | tuple[object, ...]: # Same as the other readings: pickle through the model, when there is one. @@ -563,13 +563,13 @@ def _other_entries_text(self) -> str: consumed = {key for names in self._files.values() for key in names.values()} return json_text({key: value for key, value in entries.items() if key not in consumed}) - def _located_fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Each field of each array node, as read, located in the document: under the node's `.zarray` entry, at the key the document writes for it.""" + def _located_declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Each declaration of each array node, as read, located in the document: under the node's `.zarray` entry, at the key the document writes for it.""" for path, node in self._metadata.items(): if isinstance(node, ZarrV2ArrayMetadata): key = self._files[path][ZARR_V2_ARRAY_METADATA_STORE_KEY] - for loc, field in node.reading.fields(): - yield ("metadata", key, *loc), field + for loc, declaration in node.reading.declarations(): + yield ("metadata", key, *loc), declaration def __reduce__(self) -> tuple[type[ZarrV2ConsolidatedMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) @@ -591,14 +591,14 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2Consolidated for path, node in self._metadata.items(): if not isinstance(node, ZarrV2ArrayMetadata): continue - found = disagreements(context, claims_of(node.reading.fields())) + found = disagreements(context, claims_of(node.reading.declarations())) key = self._files[path][ZARR_V2_ARRAY_METADATA_STORE_KEY] conflicts.extend( dataclasses.replace( conflict, loc=("metadata", key, *(() if conflict.loc is None else conflict.loc)), ) - for conflict in located_conflicts(node.reading.fields(), found.conflicts) + for conflict in located_conflicts(node.reading.declarations(), found.conflicts) ) if len(conflicts) != 0: raise ScopeConflictError(conflicts) @@ -710,7 +710,7 @@ def _read_consolidated_v2( Each entry is read as the kind of document its key names. An entry for a node's file is refined as user data, and its reader judges each value the spec interprets: a `.zattrs` entry is user data, and a - `.zarray` or `.zgroup` reader refuses a non-finite number in a member + `.zarray` or `.zgroup` reader refuses a non-finite number in a field the spec defines, as it does in a store. Any other entry must be JSON by RFC 8259. A `.zarray` or `.zgroup` entry is read as that document, merged with its sibling `.zattrs`. Its problems are located under its own entry, @@ -761,9 +761,9 @@ def _read_consolidated_v2( if len(problems) != 0: nodes = {} files = {path: by_path[path] for path in nodes} - members: dict[str, JSONValue] = {"zarr_consolidated_format": 1, "metadata": refined} + fields: dict[str, JSONValue] = {"zarr_consolidated_format": 1, "metadata": refined} # The keys in the order the document writes them, as the v3 models keep them. - document = {key: members[key] for key in doc if isinstance(key, str) and key in members} + document = {key: fields[key] for key in doc if isinstance(key, str) and key in fields} return document, nodes, files, with_input(problems, doc) @@ -833,10 +833,10 @@ def _read_node_v2( merged["attributes"] = entries[zattrs] if zarray is not None: - reading, members = read_array_v2(merged, context) - if members is None or len(beside) != 0: + reading, fields = read_array_v2(merged, context) + if fields is None or len(beside) != 0: return None, [*beside, *prefixed("metadata", in_files(reading.problems, key, entries))] - return ZarrV2ArrayMetadata._of(members.document, context, reading, members), [] # pyright: ignore[reportPrivateUsage] + return ZarrV2ArrayMetadata._of(fields.document, context, reading, fields), [] # pyright: ignore[reportPrivateUsage] found = _group_problems_v2(merged) if len(found) != 0 or len(beside) != 0: return None, [*beside, *prefixed("metadata", in_files(found, key, entries))] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index 155d8e8019..d95dd99079 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -39,7 +39,7 @@ from zarr_metadata._json import prefixed as _prefix from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._context import disagreements, scoped -from zarr_metadata.definition._definition import UnclaimedField, fields_of, held +from zarr_metadata.definition._definition import UnclaimedDeclaration, declarations_of, held from zarr_metadata.definition._scope import ( Conflict, ScopeConflictError, @@ -47,20 +47,20 @@ definition_said, kind_name, ) -from zarr_metadata.definition._scope import refines as refines_field +from zarr_metadata.definition._scope import refines as refines_declaration from zarr_metadata.model._array_v3 import ( - ArrayMembersV3, + ArrayFieldsV3, ZarrV3ArrayMetadata, ZarrV3ArrayMetadataReading, array_key_of, extra_fields_key, - members_past_the_levels, + fields_past_the_levels, must_understand_subset, no_extra_fields, read_array_metadata_v3, read_array_v3, read_extra_fields_v3, - waived_members, + waived_fields, ) from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( @@ -68,12 +68,12 @@ attributes_of, check_literal, dump_store_json, + field_changes, load_store_json, located_conflicts, - member_changes, missing_keys, reading_of, - unclaimed_fields_of, + unclaimed_declarations_of, unexpected_keys, ) from zarr_metadata.v3._common import ZarrV3ExtensionField @@ -94,13 +94,13 @@ from zarr_metadata._hierarchy import NodeType from zarr_metadata._typed_json import Loc - from zarr_metadata.definition._definition import AcceptedField, ResolvedField + from zarr_metadata.definition._definition import AcceptedDeclaration, ResolvedDeclaration from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataStoreKey -# The members a v3 group document must have. The members the spec defines -# are `GroupExtraFieldDefinition.spec_members`; any other is an extra field. +# The fields a v3 group document must have. The fields the spec defines +# are `GroupExtraFieldDefinition.spec_fields`; any other is an extra field. _GROUP_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( ZarrV3GroupMetadataJSON.__required_keys__ ) @@ -122,7 +122,7 @@ class ZarrV3ConsolidatedMetadataInput(TypedDict, closed=True): - """The `consolidated_metadata` member as `ZarrV3ConsolidatedMetadata` accepts it: in the form a document writes it, with each entry a document or a node model. + """The `consolidated_metadata` field as `ZarrV3ConsolidatedMetadata` accepts it: in the form a document writes it, with each entry a document or a node model. A node model is accepted when the scope of the consolidated metadata reads every one of its claims identically, or claims names the @@ -138,7 +138,7 @@ class ZarrV3ConsolidatedMetadataInput(TypedDict, closed=True): class ZarrV3GroupMetadataUpdate(TypedDict, total=False, closed=True): - """The members `ZarrV3GroupMetadata.update` replaces: each in the form a document writes it, or `UNSET` to remove it. + """The fields `ZarrV3GroupMetadata.update` replaces: each in the form a document writes it, or `UNSET` to remove it. `consolidated_metadata` is typed as the reference implementation writes it, or as any JSON object, such as another group's @@ -147,7 +147,7 @@ class ZarrV3GroupMetadataUpdate(TypedDict, total=False, closed=True): group's scope, as a listing of node models is read, and its document is written. A group does not read the documents its consolidated metadata lists, so build that model to check them before they are - written. Any other member the spec does not define goes in + written. Any other field the spec does not define goes in `extra_fields`, by key: each one given is set, or removed when given as `UNSET`, and the others are kept. """ @@ -168,29 +168,29 @@ class ZarrV3GroupMetadata(Keyed): Like `ZarrV3ArrayMetadata`, the model is this pair. `to_json` returns the document as written, refined, and `context` is the scope. `attributes` and `consolidated_metadata` are views of the values the - read refined. Each member the spec does not define is an extra field, + read refined. Each field the spec does not define is an extra field, in `extra_fields`, as the scope read it. `CORE` reads a - `consolidated_metadata` member as the envelope the core spec fixes, - and no definition reads the documents the member lists: they follow a + `consolidated_metadata` field as the envelope the core spec fixes, + and no definition reads the documents the field lists: they follow a convention of the reference implementation, not the core spec. To - read them, pass the member to `ZarrV3ConsolidatedMetadata` or + read them, pass the field to `ZarrV3ConsolidatedMetadata` or `read_consolidated_metadata_v3`, or read one listed document with `read_node_metadata_v3`. A model is only built by reading: the constructor reads `document` in `context` and raises `MetadataValidationError` with every problem. """ - __slots__ = ("_context", "_document", "_members", "_reading", "_shown") + __slots__ = ("_context", "_document", "_fields", "_reading", "_shown") zarr_format: Final[Literal[3]] = 3 # noqa: PYI064 - a bare Final does not narrow a union of models node_type: Final[Literal["group"]] = "group" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: context = scoped(context, CORE_AND_EXTENSIONS) - reading, members = read_group_v3(document, context) - if members is None or len(reading.problems) != 0: + reading, fields = read_group_v3(document, context) + if fields is None or len(reading.problems) != 0: raise MetadataValidationError(reading.problems) - self._adopt(_document_of(members), context, reading, members) + self._adopt(_document_of(fields), context, reading, fields) @classmethod def _of( @@ -198,11 +198,11 @@ def _of( document: dict[str, JSONValue], context: ZarrV3Context, reading: ZarrV3GroupMetadataReading, - members: GroupMembersV3, + fields: GroupFieldsV3, ) -> ZarrV3GroupMetadata: """Build a model of a document that a read found no problems with, holding that reading, without reading it again. The readers in this package build models through this method; pyright reports these calls as private use.""" model = object.__new__(cls) - model._adopt(document, context, reading, members) + model._adopt(document, context, reading, fields) return model def _adopt( @@ -210,19 +210,19 @@ def _adopt( document: dict[str, JSONValue], context: ZarrV3Context, reading: ZarrV3GroupMetadataReading, - members: GroupMembersV3, + fields: GroupFieldsV3, ) -> None: self._document = document self._context = context - self._members = members - if members.attributes is None: - msg = "a group model holds members a read found nothing wrong with" + self._fields = fields + if fields.attributes is None: + msg = "a group model holds fields a read found nothing wrong with" raise TypeError(msg) - # The members the model exposes, read-only at every level. + # The fields the model exposes, read-only at every level. self._shown = ( - frozen(members.attributes), - frozen(members.extra_fields), - UNSET if members.consolidated is UNSET else frozen(members.consolidated), + frozen(fields.attributes), + frozen(fields.extra_fields), + UNSET if fields.consolidated is UNSET else frozen(fields.consolidated), MappingProxyType({key: held(extra) for key, extra in reading.extra_fields.items()}), ) # The reading holds this model, no matter how the model was built. @@ -232,7 +232,7 @@ def _adopt( @property def context(self) -> ZarrV3Context: - """The scope the document was read in. `update` reads new members in this scope.""" + """The scope the document was read in. `update` reads new fields in this scope.""" return self._context @property @@ -262,7 +262,7 @@ def __repr__(self) -> str: def _key_of(self) -> tuple[object, ...]: """The key `==` and `hash` compare for a v3 group model: its attributes as JSON text, and what each extra field means, as `extra_fields_key` computes it.""" return ( - json_text(self._members.attributes), + json_text(self._fields.attributes), extra_fields_key(self._reading.extra_fields), ) @@ -280,15 +280,15 @@ def attributes(self) -> Mapping[str, JSONValue]: @property def extra_fields( self, - ) -> Mapping[str, AcceptedField[GroupExtraFieldDefinition[Any]] | UnclaimedField]: - """Each member the spec does not define, by key, as the scope read it: an `AcceptedField` when a definition in scope claims the key (as `CORE` claims `consolidated_metadata`), and an `UnclaimedField` otherwise.""" + ) -> Mapping[str, AcceptedDeclaration[GroupExtraFieldDefinition[Any]] | UnclaimedDeclaration]: + """Each field the spec does not define, by key, as the scope read it: an `AcceptedDeclaration` when a definition in scope claims the key (as `CORE` claims `consolidated_metadata`), and an `UnclaimedDeclaration` otherwise.""" return self._shown[3] @property def consolidated_metadata(self) -> Mapping[str, JSONValue] | UNSET: - """The `consolidated_metadata` member as written, refined, and read-only at every level. `UNSET` when the document writes none, or writes one that is not an object, which only a scope that leaves the member unclaimed allows. + """The `consolidated_metadata` field as written, refined, and read-only at every level. `UNSET` when the document writes none, or writes one that is not an object, which only a scope that leaves the field unclaimed allows. - The documents it lists are not read. Pass the member to + The documents it lists are not read. Pass the field to `ZarrV3ConsolidatedMetadata` to read them all, or read one with `read_node_metadata_v3`. """ @@ -298,7 +298,7 @@ def consolidated_metadata(self) -> Mapping[str, JSONValue] | UNSET: def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: """The extra fields, by key and as written, that no definition in scope reads and that a reader is obligated to understand. - Each unclaimed member of `extra_fields` not explicitly waived with + Each unclaimed field of `extra_fields` not explicitly waived with `must_understand: false` (the spec's implicit-true rule, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578). The scope is what the reader recognizes, so a compliant reader MUST fail to open the group when this is not empty. The model only reports these fields; @@ -307,21 +307,23 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: return must_understand_subset(self.extra_fields, self._shown[1]) @property - def unclaimed_fields(self) -> dict[Loc, UnclaimedField]: - """Every field nothing in scope claims, at any depth, by its location in the document, except an extra member that says `"must_understand": false` and every field inside it, claimed or not, since a reader may ignore such a member whole. + def unclaimed_declarations(self) -> dict[Loc, UnclaimedDeclaration]: + """Every declaration nothing in scope claims, at any depth, by its location in the document, except an extra field that says `"must_understand": false` and every declaration inside it, claimed or not, since a reader may ignore such a field whole. This answers whether a reader can open the group: the spec requires - a reader to refuse a node with a field it does not understand, and - the scope is what the reader understands. The fields of the + a reader to refuse a node with a declaration it does not understand, and + the scope is what the reader understands. The declarations of the documents that the group's `consolidated_metadata` lists are not included, because the group does not read those documents; read - them with `consolidated()`. The model only reports these fields; + them with `consolidated()`. The model only reports these declarations; refusing to open is up to the reader. """ - return unclaimed_fields_of(self._reading.fields(), waived_members(self._shown[1])) + return unclaimed_declarations_of( + self._reading.declarations(), waived_fields(self._shown[1]) + ) def consolidated(self) -> ZarrV3ConsolidatedMetadata | None: - """The documents this group's `consolidated_metadata` lists, read in this group's own scope; None when the group writes no such member. + """The documents this group's `consolidated_metadata` lists, read in this group's own scope; None when the group writes no such field. A group opens without reading them, so a document with a problem fails only here. Raises `MetadataValidationError` with every problem @@ -345,17 +347,17 @@ def consolidated(self) -> ZarrV3ConsolidatedMetadata | None: # --- changing --------------------------------------------------------- - def update(self, **members: Unpack[ZarrV3GroupMetadataUpdate]) -> ZarrV3GroupMetadata: - """Return this model with `members` (as JSON) replacing the document's members, read in this model's own scope. A member set to `UNSET` is removed. A member the spec does not define is given in `extra_fields`. + def update(self, **fields: Unpack[ZarrV3GroupMetadataUpdate]) -> ZarrV3GroupMetadata: + """Return this model with `fields` (as JSON) replacing the document's fields, read in this model's own scope. A field set to `UNSET` is removed. A field the spec does not define is given in `extra_fields`. A `ZarrV3ConsolidatedMetadata` given as `consolidated_metadata` is read in this model's scope and written as its document. Raises - `TypeError` for a member the update does not take, and + `TypeError` for a field the update does not take, and `MetadataValidationError` when the resulting document has a problem, such as a listed model that conflicts with this scope. """ - changes = member_changes( - members, ZarrV3GroupMetadataUpdate, GroupExtraFieldDefinition.spec_members + changes = field_changes( + fields, ZarrV3GroupMetadataUpdate, GroupExtraFieldDefinition.spec_fields ) document: dict[str, object] = {**self._document, **_written(changes, self._context)} for key, value in changes.items(): @@ -366,8 +368,8 @@ def update(self, **members: Unpack[ZarrV3GroupMetadataUpdate]) -> ZarrV3GroupMet def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadata: """Return this document read in `context`, whatever that changes. Raises `MetadataValidationError` when the document has a problem in `context`. The existing reading is reused when `context` reads every claim identically.""" context = scoped(context, CORE_AND_EXTENSIONS) - if disagreements(context, claims_of(self._reading.fields())).agrees: - return self._of(self._document, context, self._reading, self._members) + if disagreements(context, claims_of(self._reading.declarations())).agrees: + return self._of(self._document, context, self._reading, self._fields) return type(self)(self._document, context=context) def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadata: @@ -383,21 +385,23 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadat that they read the same there. """ context = scoped(context, CORE_AND_EXTENSIONS) - found = disagreements(context, claims_of(self._reading.fields())) + found = disagreements(context, claims_of(self._reading.declarations())) if len(found.conflicts) != 0: - raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) + raise ScopeConflictError( + located_conflicts(self._reading.declarations(), found.conflicts) + ) return self.with_context(context) def refines(self, other: ZarrV3GroupMetadata) -> bool: - """Whether this group holds everything `other` holds: the same attributes, and the same extra fields, each refining its counterpart in the order `refines` defines for fields. False when `other` is not a v3 group.""" + """Whether this group holds everything `other` holds: the same attributes, and the same extra fields, each refining its counterpart in the order `refines` defines for declarations. False when `other` is not a v3 group.""" if type(other) is not type(self): return False - if json_text(self._members.attributes) != json_text(other._members.attributes): + if json_text(self._fields.attributes) != json_text(other._fields.attributes): return False mine, theirs = self.extra_fields, other.extra_fields if mine.keys() != theirs.keys(): return False - return all(refines_field(mine[key], theirs[key]) for key in mine) + return all(refines_declaration(mine[key], theirs[key]) for key in mine) # --- constructors ----------------------------------------------------- @@ -408,10 +412,10 @@ def create_default( context: ZarrV3Context | None = None, **overrides: Unpack[ZarrV3GroupMetadataUpdate], ) -> ZarrV3GroupMetadata: - """Return a group with no attributes, with any document members in `overrides` applied, read in `context`. `overrides` takes what `update` takes, including `UNSET` to drop a member and a `ZarrV3ConsolidatedMetadata` as `consolidated_metadata`. Raises `MetadataValidationError` when the resulting document has a problem.""" + """Return a group with no attributes, with any document fields in `overrides` applied, read in `context`. `overrides` takes what `update` takes, including `UNSET` to drop a field and a `ZarrV3ConsolidatedMetadata` as `consolidated_metadata`. Raises `MetadataValidationError` when the resulting document has a problem.""" context = scoped(context, CORE_AND_EXTENSIONS) - changes = member_changes( - overrides, ZarrV3GroupMetadataUpdate, GroupExtraFieldDefinition.spec_members + changes = field_changes( + overrides, ZarrV3GroupMetadataUpdate, GroupExtraFieldDefinition.spec_fields ) given = {key: value for key, value in changes.items() if value is not UNSET} document = {"zarr_format": 3, "node_type": "group", **_written(given, context)} @@ -426,9 +430,9 @@ def from_json( Raises `MetadataValidationError` with every problem the read finds. A `consolidated_metadata` of `null`, which zarr-python 3.0 and 3.1 wrote, is a value the document wrote that is not an object. In a - scope that claims the member, as `CORE` does, this is a problem, + scope that claims the field, as `CORE` does, this is a problem, because the spec requires an object. Use - `read_repaired_node_metadata_v3` to read such a store. A member the + `read_repaired_node_metadata_v3` to read such a store. A field the spec does not define is kept in `extra_fields`. """ return cls(data, context=context) @@ -447,8 +451,8 @@ def from_key_value( return cls(load_store_json(mapping, ZARR_V3_GROUP_METADATA_STORE_KEY), context=context) -def _written(members: Mapping[str, object], context: ZarrV3Context) -> dict[str, object]: - """`members`, the members given to `update` or `create_default` of a group read in `context`, with a `ZarrV3ConsolidatedMetadata` given as `consolidated_metadata` replaced by its document. +def _written(fields: Mapping[str, object], context: ZarrV3Context) -> dict[str, object]: + """`fields`, the fields given to `update` or `create_default` of a group read in `context`, with a `ZarrV3ConsolidatedMetadata` given as `consolidated_metadata` replaced by its document. The models the consolidated metadata lists are read in `context` first, as a listing of node models is read: a gain is read again, and a @@ -456,9 +460,9 @@ def _written(members: Mapping[str, object], context: ZarrV3Context) -> dict[str, writes a listing that its own scope reads differently. Raises `MetadataValidationError` with those problems, located in the group. """ - given = members.get(ZARR_V3_CONSOLIDATED_METADATA_KEY) + given = fields.get(ZARR_V3_CONSOLIDATED_METADATA_KEY) if not isinstance(given, ZarrV3ConsolidatedMetadata): - return dict(members) + return dict(fields) if given.context != context: try: given = ZarrV3ConsolidatedMetadata( @@ -468,16 +472,16 @@ def _written(members: Mapping[str, object], context: ZarrV3Context) -> dict[str, raise MetadataValidationError( prefixed(ZARR_V3_CONSOLIDATED_METADATA_KEY, error.problems) ) from error - return {**members, ZARR_V3_CONSOLIDATED_METADATA_KEY: given.to_json()} + return {**fields, ZARR_V3_CONSOLIDATED_METADATA_KEY: given.to_json()} class ZarrV3ConsolidatedMetadata(Keyed): - """A group's inline `consolidated_metadata` member with every document it lists read, and the scope they were read in. + """A group's inline `consolidated_metadata` field with every document it lists read, and the scope they were read in. Models the reference-implementation convention where consolidated metadata is embedded in a group's `zarr.json`. A group's own read - checks only the member's envelope. This model reads each document the - member lists, once, as the array or group its `node_type` declares, + checks only the field's envelope. This model reads each document the + field lists, once, as the array or group its `node_type` declares, and checks that the documents and the group form a hierarchy with the group as its root. Build it to check consolidated metadata before `ZarrV3GroupMetadata.update` writes it into a group. `metadata` maps @@ -486,7 +490,7 @@ class ZarrV3ConsolidatedMetadata(Keyed): the node at `/a/b` is at `a/b`. `kind` is `inline` and `must_understand` is `False`, as the convention declares. A model is only built by reading: the constructor raises - `MetadataValidationError` with every problem, located in the member. + `MetadataValidationError` with every problem, located in the field. """ __slots__ = ("_context", "_document", "_metadata") @@ -494,16 +498,16 @@ class ZarrV3ConsolidatedMetadata(Keyed): kind: Final = "inline" must_understand: Final = False - def __init__(self, member: object, context: ZarrV3Context | None = None) -> None: + def __init__(self, value: object, context: ZarrV3Context | None = None) -> None: context = scoped(context, CORE_AND_EXTENSIONS) - # The member is always under a group's key, even when read on its + # The field is always under a group's key, even when read on its # own, so nesting depth is counted from there, as in the group. - readings, members, document, problems = _read_consolidated_v3( - member, context, (ZARR_V3_CONSOLIDATED_METADATA_KEY,) + readings, fields, document, problems = _read_consolidated_v3( + value, context, (ZARR_V3_CONSOLIDATED_METADATA_KEY,) ) if len(problems) != 0 or document is None: raise MetadataValidationError(problems) - self._adopt(document, context, _nested_models(context, readings, members)) + self._adopt(document, context, _nested_models(context, readings, fields)) @classmethod def _of( @@ -512,7 +516,7 @@ def _of( context: ZarrV3Context, metadata: dict[str, ZarrV3NodeMetadata], ) -> ZarrV3ConsolidatedMetadata: - """Build the model of a member that a read found no problems with, holding the models that read built. The readers in this package build models through this method; pyright reports these calls as private use.""" + """Build the model of a field that a read found no problems with, holding the models that read built. The readers in this package build models through this method; pyright reports these calls as private use.""" model = object.__new__(cls) model._adopt(document, context, metadata) return model @@ -529,7 +533,7 @@ def _adopt( @property def reading(self) -> ZarrV3ConsolidatedMetadataReading: - """The member as the scope read it: no problems, and this model.""" + """The field as the scope read it: no problems, and this model.""" return ZarrV3ConsolidatedMetadataReading(metadata=self) @property @@ -543,7 +547,7 @@ def metadata(self) -> Mapping[str, ZarrV3NodeMetadata]: return MappingProxyType(self._metadata) def to_json(self) -> ZarrV3ConsolidatedMetadataJSON: - """The member as written, refined. The result shares no objects with the model.""" + """The field as written, refined. The result shares no objects with the model.""" return cast("ZarrV3ConsolidatedMetadataJSON", copied(self._document)) def __repr__(self) -> str: @@ -560,10 +564,10 @@ def __reduce__(self) -> tuple[type[ZarrV3ConsolidatedMetadata], tuple[object, Za return type(self), (self._document, self._context) def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ConsolidatedMetadata: - """Return this member with every document read in `context`, whatever that changes. Raises `MetadataValidationError` when a document has a problem in `context`. The existing readings are reused when `context` reads every claim of every document identically.""" + """Return this field with every document read in `context`, whatever that changes. Raises `MetadataValidationError` when a document has a problem in `context`. The existing readings are reused when `context` reads every claim of every document identically.""" context = scoped(context, CORE_AND_EXTENSIONS) if all( - disagreements(context, claims_of(node.reading.fields())).agrees + disagreements(context, claims_of(node.reading.declarations())).agrees for node in self._metadata.values() ): moved = {path: node.with_context(context) for path, node in self._metadata.items()} @@ -571,7 +575,7 @@ def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3Consolidat return type(self)(self._document, context=context) def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ConsolidatedMetadata: - """Return this member with every document read in `context`. `context` may claim names this scope left unclaimed, but must not contradict any claim. + """Return this field with every document read in `context`. `context` may claim names this scope left unclaimed, but must not contradict any claim. Raises `ScopeConflictError` naming each conflict, located at the document's path. Raises `MetadataValidationError` when a gain @@ -584,8 +588,8 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3Consolidated ) for path, node in self._metadata.items() for conflict in located_conflicts( - node.reading.fields(), - disagreements(context, claims_of(node.reading.fields())).conflicts, + node.reading.declarations(), + disagreements(context, claims_of(node.reading.declarations())).conflicts, ) ] if len(conflicts) != 0: @@ -606,7 +610,7 @@ def refines(self, other: ZarrV3ConsolidatedMetadata) -> bool: def from_json( cls, data: object, *, context: ZarrV3Context | None = None ) -> ZarrV3ConsolidatedMetadata: - """Return the model of `data`, a group's `consolidated_metadata` member. Each document is read once in `context`, as the array or group its `node_type` declares. Raises `MetadataValidationError` with every problem found.""" + """Return the model of `data`, a group's `consolidated_metadata` field. Each document is read once in `context`, as the array or group its `node_type` declares. Raises `MetadataValidationError` with every problem found.""" return cls(data, context=context) @@ -625,15 +629,15 @@ class ZarrV3GroupMetadataReading: """Every reason the document is not valid.""" metadata: ZarrV3GroupMetadata | None = None """The document's model when there is no problem; None otherwise.""" - extra_fields: Mapping[str, ResolvedField[GroupExtraFieldDefinition[Any]]] = dataclasses.field( - default_factory=no_extra_fields + extra_fields: Mapping[str, ResolvedDeclaration[GroupExtraFieldDefinition[Any]]] = ( + dataclasses.field(default_factory=no_extra_fields) ) - """Each member the spec does not define, as the scope read it, by key: an `UnclaimedField` when no definition in scope claims the key.""" + """Each field the spec does not define, as the scope read it, by key: an `UnclaimedDeclaration` when no definition in scope claims the key.""" - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Each field in the document, as the scope read it, with its location: the extra fields, each at its key.""" + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Each declaration in the document, as the scope read it, with its location: the extra fields, each at its key.""" for key, extra in self.extra_fields.items(): - yield from fields_of(extra, (key,)) + yield from declarations_of(extra, (key,)) def __reduce__(self) -> str | tuple[object, ...]: # Same as the other readings: pickle through the model, when there @@ -646,7 +650,7 @@ def __reduce__(self) -> str | tuple[object, ...]: def _group_reading( problems: tuple[ValidationProblem, ...], - extra_fields: dict[str, ResolvedField[GroupExtraFieldDefinition[Any]]], + extra_fields: dict[str, ResolvedDeclaration[GroupExtraFieldDefinition[Any]]], ) -> ZarrV3GroupMetadataReading: """Rebuild a group reading without a model from the values `ZarrV3GroupMetadataReading.__reduce__` returns, with its extra fields held read-only.""" return ZarrV3GroupMetadataReading( @@ -656,20 +660,20 @@ def _group_reading( @dataclass(frozen=True, slots=True) class ZarrV3ConsolidatedMetadataReading: - """A group's `consolidated_metadata` member as a scope read it: every problem, and the model when there are no problems.""" + """A group's `consolidated_metadata` field as a scope read it: every problem, and the model when there are no problems.""" problems: tuple[ValidationProblem, ...] = () - """Every reason the member is not valid.""" + """Every reason the field is not valid.""" metadata: ZarrV3ConsolidatedMetadata | None = None - """The member's model when there is no problem; None otherwise.""" + """The field's model when there is no problem; None otherwise.""" - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Each field of each document, as read, located in the member: under `metadata` and the document's path. Nothing when there is no model.""" + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Each declaration of each document, as read, located in the field: under `metadata` and the document's path. Nothing when there is no model.""" if self.metadata is None: return for path, node in self.metadata.metadata.items(): - for loc, field in node.reading.fields(): - yield ("metadata", path, *loc), field + for loc, declaration in node.reading.declarations(): + yield ("metadata", path, *loc), declaration def __reduce__(self) -> str | tuple[object, ...]: # Same as the other readings: pickle through the model, when there is one. @@ -681,15 +685,15 @@ def __reduce__(self) -> str | tuple[object, ...]: def read_consolidated_metadata_v3( value: object, *, context: ZarrV3Context | None = None ) -> ZarrV3ConsolidatedMetadataReading: - """Read `value`, a group's `consolidated_metadata` member, in `context` (`CORE_AND_EXTENSIONS` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV3ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" + """Read `value`, a group's `consolidated_metadata` field, in `context` (`CORE_AND_EXTENSIONS` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV3ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" context = scoped(context, CORE_AND_EXTENSIONS) - readings, members, document, problems = _read_consolidated_v3( + readings, fields, document, problems = _read_consolidated_v3( value, context, (ZARR_V3_CONSOLIDATED_METADATA_KEY,) ) if len(problems) != 0 or document is None: return ZarrV3ConsolidatedMetadataReading(problems=problems) model = ZarrV3ConsolidatedMetadata._of( # pyright: ignore[reportPrivateUsage] - document, context, _nested_models(context, readings, members) + document, context, _nested_models(context, readings, fields) ) return model.reading @@ -711,8 +715,8 @@ def metadata(self) -> None: """Its model: always None, since without a node type there is no model to choose.""" return None - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Its fields as read: none, since no field is read.""" + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Its declarations as read: none, since no declaration is read.""" return iter(()) @@ -784,8 +788,8 @@ def node_metadata_from_key_value_v3( def _read_node_v3( value: object, context: ZarrV3Context, at: tuple[str | int, ...] = () -) -> tuple[ZarrV3NodeMetadataReading, ArrayMembersV3 | GroupMembersV3 | None]: - """Read `value` as `read_node_metadata_v3` does, without building models, and return its refined members. +) -> tuple[ZarrV3NodeMetadataReading, ArrayFieldsV3 | GroupFieldsV3 | None]: + """Read `value` as `read_node_metadata_v3` does, without building models, and return its refined fields. `at` is the location of `value` within the input document, so nesting depth is counted from the root of the input document. A document @@ -831,17 +835,17 @@ def _node_type(value: object) -> tuple[str | None, tuple[ValidationProblem, ...] @dataclass(frozen=True, slots=True) -class GroupMembersV3: - """The refined members of a v3 group document, as the model holds them.""" +class GroupFieldsV3: + """The refined fields of a v3 group document, as the model holds them.""" attributes: dict[str, JSONValue] | None """The attributes; None when they have a problem.""" extra_fields: dict[str, JSONValue] - """Each member the spec does not define that is JSON.""" + """Each field the spec does not define that is JSON.""" consolidated: dict[str, JSONValue] | UNSET - """The `consolidated_metadata` member, refined, with the documents it lists not read; UNSET when the document writes none, or the member has a problem.""" + """The `consolidated_metadata` field, refined, with the documents it lists not read; UNSET when the document writes none, or the field has a problem.""" document: dict[str, JSONValue] | None - """The refined document that the model holds: each member as the read refined it, in the order the document writes them; None when the read found a problem.""" + """The refined document that the model holds: each field as the read refined it, in the order the document writes them; None when the read found a problem.""" def read_group_metadata_v3( @@ -852,7 +856,7 @@ def read_group_metadata_v3( Unknown top-level keys are allowed (they map to `extra_fields`). A reader must understand each one that does not declare `must_understand: false`; the model reports these as - `must_understand_fields`. A `consolidated_metadata` member, if + `must_understand_fields`. A `consolidated_metadata` field, if present and claimed by the scope, as `CORE` claims it, is checked against the envelope the core spec fixes. The documents it lists are not read; `read_consolidated_metadata_v3` reads them. This is the @@ -860,27 +864,27 @@ def read_group_metadata_v3( instead of raising them. """ context = scoped(context, CORE_AND_EXTENSIONS) - reading, members = read_group_v3(value, context) - if members is None or members.document is None: + reading, fields = read_group_v3(value, context) + if fields is None or fields.document is None: return reading - return ZarrV3GroupMetadata._of(members.document, context, reading, members).reading # pyright: ignore[reportPrivateUsage] + return ZarrV3GroupMetadata._of(fields.document, context, reading, fields).reading # pyright: ignore[reportPrivateUsage] def read_group_v3( value: object, context: ZarrV3Context, *, at: tuple[str | int, ...] = () -) -> tuple[ZarrV3GroupMetadataReading, GroupMembersV3 | None]: - """Read `value`, a v3 group document, in `context`, without building a model. Return the reading and the refined members; the members are None when `value` is not an object. +) -> tuple[ZarrV3GroupMetadataReading, GroupFieldsV3 | None]: + """Read `value`, a v3 group document, in `context`, without building a model. Return the reading and the refined fields; the fields are None when `value` is not an object. `at` is the location of this document within the input document, as in `read_array_v3`, so nesting depth is counted from the outermost - root. Each member the spec does not define is read as an extra field + root. Each field the spec does not define is read as an extra field of the group, by the definition in `context` that claims its key. """ if not is_object(value): return ZarrV3GroupMetadataReading(problems=not_an_object(value)), None doc = value found: list[ValidationProblem] = list(missing_keys(_GROUP_METADATA_REQUIRED_KEYS_V3, doc)) - past = members_past_the_levels(doc, at) + past = fields_past_the_levels(doc, at) found.extend(past.values()) whole, doc = doc, {key: item for key, item in doc.items() if key not in past} extra_fields, extra, others = read_extra_fields_v3(doc, GroupExtraFieldDefinition, context, at) @@ -891,14 +895,14 @@ def read_group_v3( if "attributes" in doc: attributes, problems = attributes_of(doc["attributes"], at) found.extend(problems) - member = extra_fields.get(ZARR_V3_CONSOLIDATED_METADATA_KEY) - consolidated: dict[str, JSONValue] | UNSET = member if isinstance(member, dict) else UNSET + field = extra_fields.get(ZARR_V3_CONSOLIDATED_METADATA_KEY) + consolidated: dict[str, JSONValue] | UNSET = field if isinstance(field, dict) else UNSET reading = ZarrV3GroupMetadataReading( problems=with_input(found, whole), extra_fields=MappingProxyType(extra) ) document: dict[str, JSONValue] | None = None if len(found) == 0 and attributes is not None: - # The document the model holds: each member as refined above, in + # The document the model holds: each field as refined above, in # the document's order, built from what was read instead of being # refined again. refined: dict[str, JSONValue] = { @@ -908,36 +912,36 @@ def read_group_v3( "attributes": attributes, } document = {key: refined[key] for key in cast("Mapping[str, object]", doc)} - return reading, GroupMembersV3(attributes, extra_fields, consolidated, document) + return reading, GroupFieldsV3(attributes, extra_fields, consolidated, document) -_CONSOLIDATED_MEMBERS: Final = ("kind", "must_understand", "metadata") -"""The members of an inline `consolidated_metadata`, in the order the convention declares them.""" +_CONSOLIDATED_FIELDS: Final = ("kind", "must_understand", "metadata") +"""The fields of an inline `consolidated_metadata`, in the order the convention declares them.""" _CONSOLIDATED_ENVELOPE: Final[dict[str, JSONValue]] = {"kind": "inline", "must_understand": False} -"""The fixed values the convention declares for the member.""" +"""The fixed values the convention declares for the field.""" -def _envelope_problems(member: Mapping[object, object]) -> list[ValidationProblem]: - """Why `member`, a `consolidated_metadata` object, is not the envelope the core spec fixes: `kind` `"inline"`, `must_understand` false and an object at `metadata`, and nothing else. The entries of `metadata` are not checked. Each problem is located in the member.""" - # Missing members are reported in the order the envelope declares them. +def _envelope_problems(field: Mapping[object, object]) -> list[ValidationProblem]: + """Why `field`, a `consolidated_metadata` object, is not the envelope the core spec fixes: `kind` `"inline"`, `must_understand` false and an object at `metadata`, and nothing else. The entries of `metadata` are not checked. Each problem is located in the field.""" + # Missing fields are reported in the order the envelope declares them. problems: list[ValidationProblem] = [ ValidationProblem((key,), "missing required key", "missing_key") - for key in _CONSOLIDATED_MEMBERS - if key not in member + for key in _CONSOLIDATED_FIELDS + if key not in field ] - problems.extend(unexpected_keys(frozenset(_CONSOLIDATED_MEMBERS), member)) - problems.extend(check_literal(member, "kind", "inline")) - problems.extend(check_literal(member, "must_understand", False)) - if "metadata" in member and not is_object(member["metadata"]): + problems.extend(unexpected_keys(frozenset(_CONSOLIDATED_FIELDS), field)) + problems.extend(check_literal(field, "kind", "inline")) + problems.extend(check_literal(field, "must_understand", False)) + if "metadata" in field and not is_object(field["metadata"]): problems.append(ValidationProblem(("metadata",), "expected an object", "invalid_type")) return problems def _read_node_model( entry: ZarrV3ArrayMetadata | ZarrV3GroupMetadata, context: ZarrV3Context, at: Loc -) -> tuple[ZarrV3NodeMetadataReading, ArrayMembersV3 | GroupMembersV3 | None]: +) -> tuple[ZarrV3NodeMetadataReading, ArrayFieldsV3 | GroupFieldsV3 | None]: """Read `entry`, a node model given in place of a listed document, in `context`. Returns the model's own reading when `context` reads every one of its @@ -950,13 +954,13 @@ def _read_node_model( location. """ document = entry.to_json() - found = disagreements(context, claims_of(entry.reading.fields())) + found = disagreements(context, claims_of(entry.reading.declarations())) if len(found.conflicts) != 0: # Read the document again in this scope, so the reading belongs to # it, like every other reading the consolidated metadata holds. - # The conflicts become its problems, each at the field's location. + # The conflicts become its problems, each at the declaration's location. reading, _ = _read_node_v3(document, context, at) - written = {loc: field.name for loc, field in entry.reading.fields()} + written = {loc: declaration.name for loc, declaration in entry.reading.declarations()} problems = tuple( ValidationProblem( conflict.loc if conflict.loc is not None else (), @@ -965,7 +969,7 @@ def _read_node_model( ), "invalid_value", ) - for conflict in located_conflicts(entry.reading.fields(), found.conflicts) + for conflict in located_conflicts(entry.reading.declarations(), found.conflicts) ) # List the conflicts first: they are the cause of what the re-read finds. return _with_problems(reading, (*problems, *reading.problems)), None @@ -977,7 +981,7 @@ def _read_node_model( if len(past) != 0: return _read_node_v3(document, context, at) moved = entry.with_context(context) - return moved.reading, moved._members # pyright: ignore[reportPrivateUsage] + return moved.reading, moved._fields # pyright: ignore[reportPrivateUsage] return _read_node_v3(document, context, at) @@ -1009,31 +1013,31 @@ def _with_problems( @dataclass(frozen=True, slots=True) class _Listing: - """The documents a group listed in consolidated metadata lists in its own `consolidated_metadata`, as read: the reading of each, and the members of each that a model can be built from, keyed by path.""" + """The documents a group listed in consolidated metadata lists in its own `consolidated_metadata`, as read: the reading of each, and the fields of each that a model can be built from, keyed by path.""" readings: Mapping[str, ZarrV3NodeMetadataReading] - members: Mapping[str, ArrayMembersV3 | GroupMembersV3] + fields: Mapping[str, ArrayFieldsV3 | GroupFieldsV3] def _read_consolidated_v3( value: object, context: ZarrV3Context, at: tuple[str | int, ...] = () ) -> tuple[ dict[str, ZarrV3NodeMetadataReading], - dict[str, ArrayMembersV3 | GroupMembersV3], + dict[str, ArrayFieldsV3 | GroupFieldsV3], dict[str, JSONValue] | None, tuple[ValidationProblem, ...], ]: - """Read an inline `consolidated_metadata` member in `context`, with every document it lists. + """Read an inline `consolidated_metadata` field in `context`, with every document it lists. Returns four values: the reading of each document it holds, read - once as `read_node_metadata_v3` reads it, keyed by path; the members - of each document that a model can be built from; the refined member + once as `read_node_metadata_v3` reads it, keyed by path; the fields + of each document that a model can be built from; the refined field (each document as its read refined it) when there are no problems, - else None; and every problem, located in the member. A listed group + else None; and every problem, located in the field. A listed group that has its own listing has that listing read the same way, so that it can be checked against this one. `at` is the location of the - member within the input document. Each container the reader descends - into (the member, its `metadata`, and each document) is checked at + field within the input document. Each container the reader descends + into (the field, its `metadata`, and each document) is checked at its location, as `_refine` checks a container, so a chain of documents is bounded by the maximum nesting depth. """ @@ -1050,7 +1054,7 @@ def _read_consolidated_v3( env = value problems = _envelope_problems(env) readings: dict[str, ZarrV3NodeMetadataReading] = {} - members: dict[str, ArrayMembersV3 | GroupMembersV3] = {} + fields: dict[str, ArrayFieldsV3 | GroupFieldsV3] = {} listings: dict[str, _Listing] = {} node_types: dict[str, NodeType | None] = {} entries = env.get("metadata") @@ -1077,10 +1081,10 @@ def _read_consolidated_v3( else: readings[key], child = _read_node_v3(entry, context, here) if child is not None: - members[key] = child + fields[key] = child problems.extend(_prefix("metadata", _prefix(key, readings[key].problems))) if ( - isinstance(child, GroupMembersV3) + isinstance(child, GroupFieldsV3) and child.document is not None and child.consolidated is not UNSET and claims_listings @@ -1088,12 +1092,12 @@ def _read_consolidated_v3( # A listed group with no problem of its own, which lists # documents too, in a scope that reads a listing: read its # listing, located in its document. In a scope that leaves - # the member unclaimed, the listed group does not read it, + # the field unclaimed, the listed group does not read it, # so neither does this. - nested, nested_members, _, inside = _read_consolidated_v3( + nested, nested_fields, _, inside = _read_consolidated_v3( child.consolidated, context, (*here, ZARR_V3_CONSOLIDATED_METADATA_KEY) ) - listings[key] = _Listing(nested, nested_members) + listings[key] = _Listing(nested, nested_fields) problems.extend( _prefix( "metadata", _prefix(key, _prefix(ZARR_V3_CONSOLIDATED_METADATA_KEY, inside)) @@ -1104,28 +1108,28 @@ def _read_consolidated_v3( problems.extend(_hierarchy_problems(node_types)) for key, listing in listings.items(): if key in node_types: - problems.extend(_nested_listing_problems(key, listing, readings, members, node_types)) + problems.extend(_nested_listing_problems(key, listing, readings, fields, node_types)) refined: dict[str, JSONValue] | None = None if len(problems) == 0: - # There are no problems, so every entry has members and a - # document. The envelope has the declared values, in the member's + # There are no problems, so every entry has fields and a + # document. The envelope has the declared values, in the field's # key order. documents = { - path: _document_of(members[path]) for path in cast("Mapping[str, object]", entries) + path: _document_of(fields[path]) for path in cast("Mapping[str, object]", entries) } held: dict[str, JSONValue] = {**_CONSOLIDATED_ENVELOPE, "metadata": documents} refined = {key: held[key] for key in cast("Mapping[str, object]", env)} # Each problem carries the value found at its location, as every other # reader's problems do. - return readings, members, refined, with_input(problems, value) + return readings, fields, refined, with_input(problems, value) -def _document_of(members: ArrayMembersV3 | GroupMembersV3) -> dict[str, JSONValue]: - """The refined document held by `members`, the members of a node that a read found no problems with. Raises `TypeError` for a group's members with a problem, which such a read rules out.""" - if members.document is None: - msg = "expected the members of a document a read found nothing wrong with" +def _document_of(fields: ArrayFieldsV3 | GroupFieldsV3) -> dict[str, JSONValue]: + """The refined document held by `fields`, the fields of a node that a read found no problems with. Raises `TypeError` for a group's fields with a problem, which such a read rules out.""" + if fields.document is None: + msg = "expected the fields of a document a read found nothing wrong with" raise TypeError(msg) - return members.document + return fields.document def _key_problems(key: str) -> list[ValidationProblem]: @@ -1147,7 +1151,7 @@ def _nested_listing_problems( key: str, listing: _Listing, readings: Mapping[str, ZarrV3NodeMetadataReading], - members: Mapping[str, ArrayMembersV3 | GroupMembersV3], + fields: Mapping[str, ArrayFieldsV3 | GroupFieldsV3], node_types: Mapping[str, NodeType | None], ) -> list[ValidationProblem]: """Check `listing`, the own consolidated listing of the group at `key`, against the outer group's flat listing (`readings` and their `node_types`). Each problem is located at the nested entry. @@ -1185,19 +1189,19 @@ def _nested_listing_problems( ) problems.append(ValidationProblem(here, message, "invalid_value")) continue - own = listing.members.get(path) - flat_members = members.get(joined) - # A document with a problem of its own may still have members (a + own = listing.fields.get(path) + flat_fields = fields.get(joined) + # A document with a problem of its own may still have fields (a # group whose attributes are not an object has them, without the # attributes), so the problems decide whether the two are compared. # An unknown key leaves the rest of the document read, so a copy # with only that is still compared. if ( own is not None - and flat_members is not None + and flat_fields is not None and _read_whole(entry) and _read_whole(readings[joined]) - and not _read_alike(readings[joined], flat_members, entry, own) + and not _read_alike(readings[joined], flat_fields, entry, own) ): message = ( f"expected the document the group lists at {shown(f'/{joined}')}, got one " @@ -1208,36 +1212,36 @@ def _nested_listing_problems( def _read_whole(reading: ZarrV3NodeMetadataReading) -> bool: - """Whether every member of the document `reading` read was read: it has no problem but unknown keys, which are reported and left out.""" + """Whether every field of the document `reading` read was read: it has no problem but unknown keys, which are reported and left out.""" return all(found.kind == "unknown_key" for found in reading.problems) def _read_alike( reading: ZarrV3NodeMetadataReading, - members: ArrayMembersV3 | GroupMembersV3, + fields: ArrayFieldsV3 | GroupFieldsV3, other: ZarrV3NodeMetadataReading, - other_members: ArrayMembersV3 | GroupMembersV3, + other_fields: ArrayFieldsV3 | GroupFieldsV3, ) -> bool: """Whether two documents of one node read the same. Arrays are compared by their model keys. Groups are compared by their attributes and extra fields only; each group's listing is checked separately, at its own location.""" if isinstance(reading, ZarrV3ArrayMetadataReading) and isinstance( other, ZarrV3ArrayMetadataReading ): - if not isinstance(members, ArrayMembersV3) or not isinstance(other_members, ArrayMembersV3): + if not isinstance(fields, ArrayFieldsV3) or not isinstance(other_fields, ArrayFieldsV3): return True - return array_key_of(reading, members) == array_key_of(other, other_members) - if isinstance(members, GroupMembersV3) and isinstance(other_members, GroupMembersV3): - return (json_text(members.attributes), json_text(_unlisted(members))) == ( - json_text(other_members.attributes), - json_text(_unlisted(other_members)), + return array_key_of(reading, fields) == array_key_of(other, other_fields) + if isinstance(fields, GroupFieldsV3) and isinstance(other_fields, GroupFieldsV3): + return (json_text(fields.attributes), json_text(_unlisted(fields))) == ( + json_text(other_fields.attributes), + json_text(_unlisted(other_fields)), ) return True -def _unlisted(members: GroupMembersV3) -> dict[str, JSONValue]: - """The extra fields of a group's members, without its own consolidated listing.""" +def _unlisted(fields: GroupFieldsV3) -> dict[str, JSONValue]: + """The extra fields of a group's fields, without its own consolidated listing.""" return { key: value - for key, value in members.extra_fields.items() + for key, value in fields.extra_fields.items() if key != ZARR_V3_CONSOLIDATED_METADATA_KEY } @@ -1282,23 +1286,23 @@ def _below_faults(path: str) -> list[str]: def _nested_models( context: ZarrV3Context, readings: Mapping[str, ZarrV3NodeMetadataReading], - members: Mapping[str, ArrayMembersV3 | GroupMembersV3], + fields: Mapping[str, ArrayFieldsV3 | GroupFieldsV3], ) -> dict[str, ZarrV3NodeMetadata]: - """The model, in `context`, of each document in `members` (the members of a `metadata` member) that a model can be built from. Reuses the model its reading already holds when that model is in `context`. Otherwise builds the model from its reading and members, which hold the refined document, without reading it again.""" + """The model, in `context`, of each document in `fields` (the fields of a `metadata` field) that a model can be built from. Reuses the model its reading already holds when that model is in `context`. Otherwise builds the model from its reading and fields, which hold the refined document, without reading it again.""" models: dict[str, ZarrV3NodeMetadata] = {} - for path, child in members.items(): + for path, child in fields.items(): reading = readings[path] held = reading.metadata if held is not None and held.context is context: models[path] = held continue - if isinstance(reading, ZarrV3ArrayMetadataReading) and isinstance(child, ArrayMembersV3): + if isinstance(reading, ZarrV3ArrayMetadataReading) and isinstance(child, ArrayFieldsV3): models[path] = ZarrV3ArrayMetadata._of( # pyright: ignore[reportPrivateUsage] child.document, context, reading, child ) elif ( isinstance(reading, ZarrV3GroupMetadataReading) - and isinstance(child, GroupMembersV3) + and isinstance(child, GroupFieldsV3) and child.document is not None ): models[path] = ZarrV3GroupMetadata._of( # pyright: ignore[reportPrivateUsage] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_repair.py b/packages/zarr-metadata/src/zarr_metadata/model/_repair.py index d16663b1fe..35ced80197 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_repair.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_repair.py @@ -92,7 +92,7 @@ class ZarrV3ZeroChunkRegularGridJSON(TypedDict): class ZarrV3ZeroChunkArrayMetadataJSON(TypedDict): - """The members of an array document the `zero_chunk_length` repair reads. + """The fields of an array document the `zero_chunk_length` repair reads. zarr-python 3.0 and 3.1 wrote a chunk length of 0 along a dimension of length 0, which the regular grid refuses: "Chunk sizes must be greater @@ -107,11 +107,11 @@ class ZarrV3ZeroChunkArrayMetadataJSON(TypedDict): class ZarrV3NullConsolidatedGroupMetadataJSON(TypedDict): - """The members of a group document the `null_consolidated_metadata` repair reads. + """The fields of a group document the `null_consolidated_metadata` repair reads. zarr-python 3.0 and 3.1 wrote `"consolidated_metadata": null` on a group - they had not consolidated. The convention does not allow this: the member - is an object, or absent. The repair removes the member, which is what the + they had not consolidated. The convention does not allow this: the field + is an object, or absent. The repair removes the field, which is what the writer meant. """ @@ -120,7 +120,7 @@ class ZarrV3NullConsolidatedGroupMetadataJSON(TypedDict): class ZarrV3NonFiniteFillValueArrayMetadataJSON(TypedDict): - """The members of an array document the v3 `non_finite_fill_value` repair reads. + """The fields of an array document the v3 `non_finite_fill_value` repair reads. zarr-python 3.0.0 to 3.0.6 wrote the fill value of a float64 array, in the documents a group's consolidated metadata lists, as a bare `NaN`, @@ -139,7 +139,7 @@ def repair_node_metadata_v3(value: object) -> tuple[object, tuple[Repair, ...]]: Each document in the consolidated metadata is repaired too. A group does not read those documents, but `ZarrV3ConsolidatedMetadata` reads - them from the repaired group's member. Content that no repair applies + them from the repaired group's field. Content that no repair applies to is left as it is. `value` is not modified: a repaired document is a new object, which shares its unchanged parts with `value`. A document with none of the bugs is returned as it is, with no repairs. @@ -169,15 +169,15 @@ def _non_finite_fill_value_v3( ) -> Mapping[str, object]: """`document` with a non-finite fill value of a float data type, or complex fill value component, written by its name.""" array, problems = check( - _members(document, ("node_type", "data_type")), ZarrV3NonFiniteFillValueArrayMetadataJSON + _fields(document, ("node_type", "data_type")), ZarrV3NonFiniteFillValueArrayMetadataJSON ) if array is None or len(problems) != 0 or "fill_value" not in document: return document return _non_finite_named(document, array["data_type"].startswith("float"), at, repairs) -def _members(document: Mapping[str, object], keys: tuple[str, ...]) -> dict[str, object]: - """Only the members of `document` that a repair reads, so the rest is not walked.""" +def _fields(document: Mapping[str, object], keys: tuple[str, ...]) -> dict[str, object]: + """Only the fields of `document` that a repair reads, so the rest is not walked.""" return {key: document[key] for key in keys if key in document} @@ -185,7 +185,7 @@ def _zero_chunk_length( document: Mapping[str, object], at: Loc, repairs: list[Repair] ) -> Mapping[str, object]: array, problems = check( - _members(document, ("node_type", "shape", "chunk_grid")), ZarrV3ZeroChunkArrayMetadataJSON + _fields(document, ("node_type", "shape", "chunk_grid")), ZarrV3ZeroChunkArrayMetadataJSON ) if array is None or len(problems) != 0: return document @@ -218,7 +218,7 @@ def _null_consolidated_metadata( document: Mapping[str, object], at: Loc, repairs: list[Repair] ) -> Mapping[str, object]: group, problems = check( - _members(document, ("node_type", ZARR_V3_CONSOLIDATED_METADATA_KEY)), + _fields(document, ("node_type", ZARR_V3_CONSOLIDATED_METADATA_KEY)), ZarrV3NullConsolidatedGroupMetadataJSON, ) if group is None or len(problems) != 0: @@ -237,10 +237,10 @@ def _consolidated( document: Mapping[str, object], at: Loc, repairs: list[Repair] ) -> Mapping[str, object]: """`document` with each document in its consolidated metadata repaired, if it has any.""" - member = document.get(ZARR_V3_CONSOLIDATED_METADATA_KEY) - if not is_object(member): + field = document.get(ZARR_V3_CONSOLIDATED_METADATA_KEY) + if not is_object(field): return document - envelope = member + envelope = field entries = envelope.get("metadata") if not is_object(entries): return document @@ -286,7 +286,7 @@ def read_repaired_node_metadata_v3( class ZarrV2ZGroupWithConsolidatedMetadataJSON(TypedDict): - """A `.zgroup` entry of a `.zmetadata`, below the root, as zarr-python 3.x writes it: with a `consolidated_metadata` member, which a v2 group document does not allow.""" + """A `.zgroup` entry of a `.zmetadata`, below the root, as zarr-python 3.x writes it: with a `consolidated_metadata` field, which a v2 group document does not allow.""" zarr_format: Literal[2] consolidated_metadata: Mapping[str, JSONValue] @@ -295,9 +295,9 @@ class ZarrV2ZGroupWithConsolidatedMetadataJSON(TypedDict): def repair_consolidated_metadata_v2(value: object) -> tuple[object, tuple[Repair, ...]]: """Undo each known writer bug in `value`, a v2 `.zmetadata`, and return the result with the list of changes. - zarr-python 3.x writes a `consolidated_metadata` member into each + zarr-python 3.x writes a `consolidated_metadata` field into each `.zgroup` entry below the root. This function removes it. The root's - entry is left alone, since no writer puts the member there. Each + entry is left alone, since no writer puts the field there. Each `.zarray` entry is repaired as `repair_node_metadata_v2` repairs a `.zarray`. Content that no repair applies to is left as it is, and `value` is not modified. A document with none of the bugs is returned @@ -314,7 +314,7 @@ def repair_consolidated_metadata_v2(value: object) -> tuple[object, tuple[Repair for key, entry in entries.items(): if isinstance(key, str): path, _, name = key.rpartition("/") - # Below the root only: no writer puts the member in the root's .zgroup. + # Below the root only: no writer puts the field in the root's .zgroup. if name == ZARR_V2_GROUP_METADATA_STORE_KEY and path.strip("/") != "": entry = _without_consolidated_metadata(entry, ("metadata", key), repairs) if name == ZARR_V2_ARRAY_METADATA_STORE_KEY: @@ -326,12 +326,12 @@ def repair_consolidated_metadata_v2(value: object) -> tuple[object, tuple[Repair def _without_consolidated_metadata(entry: object, at: Loc, repairs: list[Repair]) -> object: - """`entry`, a `.zgroup` entry, without the member zarr-python 3.x writes into it, if the entry has that member.""" + """`entry`, a `.zgroup` entry, without the field zarr-python 3.x writes into it, if the entry has that field.""" if not is_json_object(entry): return entry group = entry shaped, problems = check( - _members(group, ("zarr_format", ZARR_V3_CONSOLIDATED_METADATA_KEY)), + _fields(group, ("zarr_format", ZARR_V3_CONSOLIDATED_METADATA_KEY)), ZarrV2ZGroupWithConsolidatedMetadataJSON, ) if shaped is None or len(problems) != 0: @@ -356,7 +356,7 @@ def _without_consolidated_metadata(entry: object, at: Loc, repairs: list[Repair] class ZarrV2NonFiniteFillValueArrayMetadataJSON(TypedDict): - """The members of a `.zarray` that the `non_finite_fill_value` repair reads, other than the fill value: a float or complex `dtype`. + """The fields of a `.zarray` that the `non_finite_fill_value` repair reads, other than the fill value: a float or complex `dtype`. zarr-python 3.0.0 to 3.0.6 wrote a non-finite fill value of a float array as a bare token, `"fill_value": NaN`, and each non-finite @@ -366,7 +366,7 @@ class ZarrV2NonFiniteFillValueArrayMetadataJSON(TypedDict): (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L178-L190), and so does zarr-python since 3.0.7. Python's `json` module reads the tokens as floats, which `check` refuses, so the fill value is not a - member here: the repair reads a float, or a pair for a complex dtype. + field here: the repair reads a float, or a pair for a complex dtype. """ zarr_format: Literal[2] @@ -410,7 +410,7 @@ def _non_finite_fill_value( ) -> Mapping[str, object]: """`document` with a non-finite float fill value, or complex fill value component, written by its name.""" array, problems = check( - _members(document, ("zarr_format", "dtype")), ZarrV2NonFiniteFillValueArrayMetadataJSON + _fields(document, ("zarr_format", "dtype")), ZarrV2NonFiniteFillValueArrayMetadataJSON ) if array is None or len(problems) != 0 or "fill_value" not in document: return document diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py index a1f3d5c5e9..c91b7a3b1c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py @@ -1,4 +1,4 @@ -"""Helpers that the v2 and v3 readers share: checking the keys of a document, refining its members and attributes, and reading and writing store JSON.""" +"""Helpers that the v2 and v3 readers share: checking the keys of a document, refining its fields and attributes, and reading and writing store JSON.""" from __future__ import annotations @@ -19,7 +19,11 @@ within, ) from zarr_metadata._typed_json import typeddict_keys -from zarr_metadata.definition._definition import RefusedField, UnclaimedField, is_field +from zarr_metadata.definition._definition import ( + RefusedDeclaration, + UnclaimedDeclaration, + is_declaration, +) from zarr_metadata.definition._scope import Conflict, claim_key from zarr_metadata.v3._definition import ExtraFieldDefinition @@ -28,7 +32,7 @@ from zarr_metadata._common import JSONValue from zarr_metadata._typed_json import Loc - from zarr_metadata.definition._definition import ResolvedField + from zarr_metadata.definition._definition import ResolvedDeclaration def missing_keys( @@ -44,7 +48,7 @@ def missing_keys( def unexpected_keys( allowed: frozenset[str], doc: Mapping[object, object] ) -> tuple[ValidationProblem, ...]: - """One problem per member outside a closed document's declared shape. The kind is `unknown_key`, as a closed TypedDict's checker reports it, so a caller who tolerates a member another writer added can tell it apart from a wrong value.""" + """One problem per field outside a closed document's declared shape. The kind is `unknown_key`, as a closed TypedDict's checker reports it, so a caller who tolerates a field another writer added can tell it apart from a wrong value.""" problems: list[ValidationProblem] = [] for key in doc: if not isinstance(key, str): @@ -68,20 +72,20 @@ def check_literal( return () -def other_members( +def other_fields( doc: Mapping[object, object], standard_keys: frozenset[str], *, additional_reserved_keys: frozenset[str] = frozenset(), at: Loc = (), ) -> tuple[dict[str, JSONValue], tuple[ValidationProblem, ...]]: - """Each member outside `standard_keys`, refined as user data, and every problem found in them. A member that is not JSON is left out of the result. + """Each field outside `standard_keys`, refined as user data, and every problem found in them. A field that is not JSON is left out of the result. - The spec does not interpret such a member, so like attributes, and - like a v3 member that nothing claims, it may hold `NaN`, `Infinity` + The spec does not interpret such a field, so like attributes, and + like a v3 field that nothing claims, it may hold `NaN`, `Infinity` and `-Infinity`. """ - members: dict[str, JSONValue] = {} + fields: dict[str, JSONValue] = {} problems: list[ValidationProblem] = [] reserved_keys = standard_keys | additional_reserved_keys for key, value in doc.items(): @@ -95,84 +99,83 @@ def other_members( refined, found = refine_user_data(value, (*at, key)) problems.extend(within(found, at)) if len(found) == 0: - members[key] = refined - return members, tuple(problems) + fields[key] = refined + return fields, tuple(problems) -def member_changes( - members: Mapping[str, object], update: type, spec_members: frozenset[str] +def field_changes( + fields: Mapping[str, object], update: type, spec_fields: frozenset[str] ) -> dict[str, object]: - """The document members that `members`, given to a model's `update` or `create_default`, set: each member that `update` (the model's update TypedDict) declares, and each member in its `extra_fields`. A value of `UNSET` removes its member. - - Raises `TypeError` for a member that `update` does not declare, and - for a member of `extra_fields` that the spec defines (one of - `spec_members`) or that `update` takes by name. So a misspelled - member is never written as a member the spec does not define, which a - reader that must understand every such member would refuse. Also - raises `TypeError` for a field given where a member's JSON goes. + """The document fields that `fields`, given to a model's `update` or `create_default`, set: each field that `update` (the model's update TypedDict) declares, and each field in its `extra_fields`. A value of `UNSET` removes its field. + + Raises `TypeError` for a field that `update` does not declare, and + for a field of `extra_fields` that the spec defines (one of + `spec_fields`) or that `update` takes by name. So a misspelled + field is never written as a field the spec does not define, which a + reader that must understand every such field would refuse. Also + raises `TypeError` for a declaration given where a field's JSON goes. """ - declared = typeddict_keys(update).members - unknown = [key for key in members if key not in declared] + declared = typeddict_keys(update).fields + unknown = [key for key in fields if key not in declared] if len(unknown) != 0: names = ", ".join(map(repr, unknown)) msg = ( - f"unknown member {names}: give each member the spec defines by its name, and any " - "other member in extra_fields" + f"unknown field {names}: give each field the spec defines by its name, and any " + "other field in extra_fields" if "extra_fields" in declared - else f"unknown member {names}" + else f"unknown field {names}" ) raise TypeError(msg) - changes = {key: value for key, value in members.items() if key != "extra_fields"} + changes = {key: value for key, value in fields.items() if key != "extra_fields"} for key, value in changes.items(): - _refuse_fields(key, value) - extra = members.get("extra_fields", {}) + _refuse_declarations(key, value) + extra = fields.get("extra_fields", {}) if not is_object(extra): - msg = f"extra_fields maps members to their values, got {extra!r}" + msg = f"extra_fields maps fields to their values, got {extra!r}" raise TypeError(msg) for key, value in extra.items(): if not isinstance(key, str): - msg = ( - f"extra_fields maps each member's key, a string, to its value, got the key {key!r}" - ) + msg = f"extra_fields maps each field's key, a string, to its value, got the key {key!r}" raise TypeError(msg) - # `extra_fields` itself is the mapping, so a member of that name is - # given in it like any other member the spec does not define. + # `extra_fields` itself is the mapping, so a field of that name is + # given in it like any other field the spec does not define. if key in declared and key != "extra_fields": - msg = f"{key!r} is a member that update takes by name; give it by name, not in extra_fields" + msg = f"{key!r} is a field that update takes by name; give it by name, not in extra_fields" raise TypeError(msg) - if key in spec_members: - msg = f"{key!r} is a member the spec defines, which extra_fields does not hold" + if key in spec_fields: + msg = f"{key!r} is a field the spec defines, which extra_fields does not hold" raise TypeError(msg) - _refuse_fields(key, value) + _refuse_declarations(key, value) changes[key] = value return changes -def unclaimed_fields_of( - fields: Iterable[tuple[Loc, ResolvedField[Any]]], waived: Iterable[str] -) -> dict[Loc, UnclaimedField]: - """Each of `fields` (a reading's fields, at every depth) that nothing in scope claims, by its location, except one inside an extra member whose key is in `waived`: a member that says `"must_understand": false`, which a reader may ignore whole, whether or not a definition claims it.""" +def unclaimed_declarations_of( + declarations: Iterable[tuple[Loc, ResolvedDeclaration[Any]]], waived: Iterable[str] +) -> dict[Loc, UnclaimedDeclaration]: + """Each of `declarations` (a reading's declarations, at every depth) that nothing in scope claims, by its location, except one inside an extra field whose key is in `waived`: a field that says `"must_understand": false`, which a reader may ignore whole, whether or not a definition claims it.""" ignored = frozenset(waived) return { - loc: field - for loc, field in fields - if isinstance(field, UnclaimedField) and not (len(loc) != 0 and loc[0] in ignored) + loc: declaration + for loc, declaration in declarations + if isinstance(declaration, UnclaimedDeclaration) + and not (len(loc) != 0 and loc[0] in ignored) } -def _refuse_fields(key: str, value: object) -> None: - """Raise `TypeError` when `value`, given as the member at `key`, is a field or holds fields, rather than the JSON a document writes, with the JSON to give instead.""" +def _refuse_declarations(key: str, value: object) -> None: + """Raise `TypeError` when `value`, given as the field at `key`, is a declaration or holds declarations, rather than the JSON a document writes, with the JSON to give instead.""" for item in value if is_array(value) else (value,): - if not is_field(item): + if not is_declaration(item): continue - if isinstance(item, RefusedField): - advice = "a refused field has no JSON to write; give the member as JSON" + if isinstance(item, RefusedDeclaration): + advice = "a refused declaration has no JSON to write; give the field as JSON" elif issubclass(item.read_as, ExtraFieldDefinition): - # An extra field is read as the member alone, `{key: value}`. - advice = f"give the member's value, field.to_json()[{item.name!r}]" + # An extra field is read as the field alone, `{key: value}`. + advice = f"give the field's value, declaration.to_json()[{item.name!r}]" else: - advice = "give the field as JSON, field.to_json()" - msg = f"{key}: a member is given as a document writes it, got a field; {advice}" + advice = "give the declaration as JSON, declaration.to_json()" + msg = f"{key}: a field is given as a document writes it, got a declaration; {advice}" raise TypeError(msg) @@ -317,16 +320,21 @@ def dump_store_json(value: object, *, indent: int | str | None = None) -> bytes: def located_conflicts( - fields: Iterable[tuple[Loc, ResolvedField[Any]]], conflicts: Sequence[Conflict] + declarations: Iterable[tuple[Loc, ResolvedDeclaration[Any]]], conflicts: Sequence[Conflict] ) -> tuple[Conflict, ...]: - """Locate each of `conflicts` (found against a reading's claims) in the document, as a problem is located. A conflict appears once for each place in `fields` where its name occurs.""" + """Locate each of `conflicts` (found against a reading's claims) in the document, as a problem is located. A conflict appears once for each place in `declarations` where its name occurs.""" located: list[Conflict] = [] - placed = list(fields) + placed = list(declarations) for conflict in conflicts: - places = [(loc, field) for loc, field in placed if claim_key(field) == conflict.key] + places = [ + (loc, declaration) + for loc, declaration in placed + if claim_key(declaration) == conflict.key + ] if len(places) == 0: located.append(conflict) located.extend( - dataclasses.replace(conflict, loc=loc, written=field.name) for loc, field in places + dataclasses.replace(conflict, loc=loc, written=declaration.name) + for loc, declaration in places ) return tuple(located) diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index f369f2beec..648598b2db 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -8,7 +8,7 @@ freely with non-pydantic code (equality, isinstance, nesting). Validation delegates to the library: a raw document routes through `from_json` (the single source of truth for validation and normalization, so pydantic's -field-level coercion can never bypass it). A field type reads the fields +field-level coercion can never bypass it). A field type reads the declarations of its document in the scope given by pydantic's validation context (pydantic passes this context to every validator). The scope is chosen as follows: @@ -247,7 +247,7 @@ def coerce(value: object) -> _M: def _refined(cls: type[_M], value: _M, refine: Callable[[_M, _C], _M], context: _C) -> _M: - """`value`, an instance of `cls`, read again in `context` with `refine` (its `refined_in`), or a `pydantic_core.ValidationError` with one line error for each problem the scope finds, as for a document, and one `scope_conflict` line error for each conflict or lost meaning, at the field it is in.""" + """`value`, an instance of `cls`, read again in `context` with `refine` (its `refined_in`), or a `pydantic_core.ValidationError` with one line error for each problem the scope finds, as for a document, and one `scope_conflict` line error for each conflict or lost meaning, at the declaration it is in.""" try: return refine(value, context) except MetadataValidationError as error: diff --git a/packages/zarr-metadata/src/zarr_metadata/typed_json.py b/packages/zarr-metadata/src/zarr_metadata/typed_json.py index 30ed95f695..9af43126db 100644 --- a/packages/zarr-metadata/src/zarr_metadata/typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/typed_json.py @@ -71,7 +71,7 @@ user data such as attributes), a `Literal`, `tuple[T, ...]` and `tuple[T1, T2]`, a union, a TypedDict, `Mapping[str, V]`, a `NewType` and a type alias. A TypedDict that -contains anything else raises `TypeError`, naming the member and the +contains anything else raises `TypeError`, naming the field and the TypedDict that contains it. The same happens for a TypedDict the spec itself forbids, such as one with a key that is both `Required` and `NotRequired`, and for a generic TypedDict, because no runtime records diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_context.py b/packages/zarr-metadata/src/zarr_metadata/v2/_context.py index b13aa90620..09094c3882 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/_context.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_context.py @@ -1,4 +1,4 @@ -"""The scope a Zarr v2 document is read in: one slot for each kind of v2 field.""" +"""The scope a Zarr v2 document is read in: one slot for each kind of v2 declaration.""" from __future__ import annotations @@ -35,10 +35,10 @@ from zarr_metadata.definition._definition import D, Definition ZarrV2Kind: TypeAlias = ZarrV2DataTypeDefinition[Any] | ZarrV2CodecDefinition[Any] -"""A definition of a kind of v2 field.""" +"""A definition of a kind of v2 declaration.""" _K = TypeVar("_K", bound=ZarrV2Kind) -"""A kind of v2 field, so a type checker refuses a kind of the other format.""" +"""A kind of v2 declaration, so a type checker refuses a kind of the other format.""" _KINDS: Final[frozenset[type[Definition[Any]]]] = frozenset( get_origin(member) for member in get_args(ZarrV2Kind) @@ -46,7 +46,7 @@ def is_v2_kind(kind: type[Definition[Any]]) -> TypeIs[type[ZarrV2Kind]]: - """Whether `kind` is a kind of v2 field: one of the kinds in `ZarrV2Kind`.""" + """Whether `kind` is a kind of v2 declaration: one of the kinds in `ZarrV2Kind`.""" return kind in _KINDS @@ -73,7 +73,7 @@ class ZarrV2ContextSlots(TypedDict, total=False, closed=True): class ZarrV2Context: """A scope that reads Zarr v2 documents. Every v2 reader takes one. - It implements `Context`, and has one slot for each kind of field a v2 + It implements `Context`, and has one slot for each kind of declaration a v2 document holds, as `ZarrV2ContextSlots` declares them: `data_types`, read from `dtype`, and `codecs`, read from `compressor` and `filters`. Each slot is a read-only mapping from name to definition. The diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_field_json.py b/packages/zarr-metadata/src/zarr_metadata/v2/_declaration_json.py similarity index 100% rename from packages/zarr-metadata/src/zarr_metadata/v2/_field_json.py rename to packages/zarr-metadata/src/zarr_metadata/v2/_declaration_json.py diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py index 789c8bbd86..ad92c92b6c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py @@ -1,4 +1,4 @@ -"""The kinds of field in a Zarr v2 array document: a dtype, and a codec in `compressor` or `filters`. +"""The kinds of declaration in a Zarr v2 array document: a dtype, and a codec in `compressor` or `filters`. A v2 dtype is written as a NumPy typestr, which is a byte order, a type code and a size, such as ` int | None: "unit": "a unit", "scale_factor": "a unit", } -"""How a message names the part of a typestr that each member of its configuration comes from.""" +"""How a message names the part of a typestr that each field of its configuration comes from.""" def typestr_problem(name: str, at: Loc) -> ValidationProblem | None: @@ -153,7 +153,7 @@ class ZarrV2DataTypeDefinition(WithFillValue[C], kind=True, format=2): """ label: ClassVar[str] = "v2 data type" - field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV2DataTypeField,) + declaration_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV2DataTypeDeclaration,) def _refusal(self) -> str | None: if parse_typestr(self.name) is not None: @@ -242,7 +242,7 @@ def _in_configuration(cls, loc: Loc, place: Loc) -> Loc: # `_named_configuration` gives a v2 dtype a configuration only as a # struct's `{_RECORDS: records}`, and every other dtype's # configuration is encoded in its name, which is located at the - # field. So a place is the configuration itself, `()`, or starts + # declaration. So a place is the configuration itself, `()`, or starts # with `_RECORDS`, which the document does not write: it writes the # records as the dtype itself. return (*loc, *place[1:]) @@ -257,7 +257,7 @@ class ZarrV2CodecDefinition(Definition[C], kind=True, format=2): """A v2 codec: a numcodecs id, and the TypedDict that describes its parameters. A document writes `{"id": name, **parameters}`. The configuration of - the definition is the parameters, read at the location of the field + the definition is the parameters, read at the location of the declaration itself. """ @@ -325,8 +325,8 @@ def _name_loc(cls, loc: Loc) -> Loc: "STRUCT_NAME", "TYPESTR_PATTERN", "ZarrV2CodecDefinition", + "ZarrV2DataTypeDeclaration", "ZarrV2DataTypeDefinition", - "ZarrV2DataTypeField", "item_bytes", "parse_typestr", "typestr_problem", diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index f4bd7803a7..6ea1dca028 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -7,7 +7,7 @@ from typing_extensions import TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v2._field_json import ZarrV2CodecMetadata, ZarrV2DataTypeMetadata +from zarr_metadata.v2._declaration_json import ZarrV2CodecMetadata, ZarrV2DataTypeMetadata ZarrV2ArrayOrder = Literal["C", "F"] """Literal type of permitted values for the `order` field of v2 array metadata. @@ -69,7 +69,7 @@ class ZarrV2ArrayMetadataJSON(TypedDict, extra_items=JSONValue): The type is open: other keys "SHOULD NOT be present ... and SHOULD be ignored by implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), and each - other member is a JSON value. + other field is a JSON value. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec/__init__.py index 246647167f..caa51c794e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec/__init__.py @@ -10,8 +10,8 @@ from typing import Any, Final +from zarr_metadata.v2._declaration_json import ZarrV2CodecMetadata from zarr_metadata.v2._definition import ZarrV2CodecDefinition -from zarr_metadata.v2._field_json import ZarrV2CodecMetadata from zarr_metadata.v2.codec.checksum import ( ADLER32_CODEC_V2, CRC32_CODEC_V2, diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/__init__.py index 05e7ebfcf4..b6c54b904f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/__init__.py @@ -8,8 +8,8 @@ from typing import Any, Final +from zarr_metadata.v2._declaration_json import ZarrV2DataTypeMetadata from zarr_metadata.v2._definition import ZarrV2DataTypeDefinition -from zarr_metadata.v2._field_json import ZarrV2DataTypeMetadata from zarr_metadata.v2.data_type.fixed_width import ( BYTES_DATA_TYPE_V2, STR_DATA_TYPE_V2, diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/struct.py index 305c5a3460..9147cee464 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/struct.py @@ -1,4 +1,4 @@ -"""The v2 `struct` type: an array of field records. The type of each record is a field of its own.""" +"""The v2 `struct` type: an array of field records. The type of each record is a declaration of its own.""" from __future__ import annotations @@ -9,11 +9,11 @@ from typing_extensions import ReadOnly, TypedDict from zarr_metadata._json import ValidationProblem -from zarr_metadata.definition._definition import AcceptedField +from zarr_metadata.definition._definition import AcceptedDeclaration from zarr_metadata.v2._definition import ( STRUCT_NAME, + ZarrV2DataTypeDeclaration, ZarrV2DataTypeDefinition, - ZarrV2DataTypeField, item_bytes, ) from zarr_metadata.v2.data_type._rules import sized_base64_rules @@ -25,10 +25,10 @@ from zarr_metadata.definition._definition import Nested ZarrV2StructRecord = ( - tuple[str, ZarrV2DataTypeField, tuple[Annotated[int, Ge(0)], ...]] - | tuple[str, ZarrV2DataTypeField] + tuple[str, ZarrV2DataTypeDeclaration, tuple[Annotated[int, Ge(0)], ...]] + | tuple[str, ZarrV2DataTypeDeclaration] ) -"""A field record: `[name, dtype]` or `[name, dtype, shape]`. The dtype is a nested field (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L152-L174).""" +"""A field record: `[name, dtype]` or `[name, dtype, shape]`. The dtype is a nested declaration (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L152-L174).""" class ZarrV2StructConfiguration(TypedDict, closed=True): @@ -65,10 +65,10 @@ def _record_size(configuration: ZarrV2StructConfiguration, nested: Nested) -> in """Return the number of bytes that one record of `configuration` takes. Each field takes its item size in bytes times the product of its subarray shape, and a nested struct takes the size of its own record. Return None when the size of a field is not known (as for `|O`) or its dtype was not read.""" total = 0 for index, record in enumerate(configuration["fields"]): - field = nested.get(("fields", index, 1)) - if not isinstance(field, AcceptedField): + declaration = nested.get(("fields", index, 1)) + if not isinstance(declaration, AcceptedDeclaration): return None - size = _item_size(field) + size = _item_size(declaration) if size is None: return None shape = record[2] if len(record) == 3 else () @@ -76,11 +76,13 @@ def _record_size(configuration: ZarrV2StructConfiguration, nested: Nested) -> in return total -def _item_size(field: AcceptedField[Any]) -> int | None: - """Return the number of bytes that one item of the dtype read as `field` takes, or None when that is not known. A typestr gives the size of its type, which for `U` counts code points of four bytes each.""" - if field.name == STRUCT_NAME or "fields" in field.configuration: - return _record_size(cast("ZarrV2StructConfiguration", field.configuration), field.nested) - return item_bytes(field.name) +def _item_size(declaration: AcceptedDeclaration[Any]) -> int | None: + """Return the number of bytes that one item of the dtype read as `declaration` takes, or None when that is not known. A typestr gives the size of its type, which for `U` counts code points of four bytes each.""" + if declaration.name == STRUCT_NAME or "fields" in declaration.configuration: + return _record_size( + cast("ZarrV2StructConfiguration", declaration.configuration), declaration.nested + ) + return item_bytes(declaration.name) STRUCT_DATA_TYPE_V2: Final = ZarrV2DataTypeDefinition( diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/definition.py b/packages/zarr-metadata/src/zarr_metadata/v2/definition.py index 8cf4990992..fb81ad4d54 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/definition.py @@ -1,16 +1,16 @@ -"""Read Zarr v2 fields against their definitions. This is the public module for v2 reading. +"""Read Zarr v2 declarations against their definitions. This is the public module for v2 reading. -A v2 array document has two kinds of field: its `dtype`, and the codecs +A v2 array document has two kinds of declaration: its `dtype`, and the codecs in `compressor` and `filters`. Zarr v2 has no extension registry, but it still has a scope: the data types that zarr-python 2.x writes (one definition per NumPy family), and the codecs that numcodecs 0.16 configures (one per id that this package models). `CORE_V2` is that -scope. To read one field in it, call `zarr_metadata.definition.resolve` -with the field's kind, as for a v3 field: +scope. To read one declaration in it, call `zarr_metadata.definition.resolve` +with the declaration's kind, as for a v3 declaration: `resolve(" TypeIs[type[ZarrV3Kind]]: - """Whether `kind` is a kind of v3 field: one of the kinds in `ZarrV3Kind`.""" + """Whether `kind` is a kind of v3 declaration: one of the kinds in `ZarrV3Kind`.""" return kind in _KINDS @@ -95,23 +95,23 @@ class ZarrV3ContextSlots(TypedDict, total=False, closed=True): array_extra_fields: ( Iterable[ArrayExtraFieldDefinition[Any]] | Mapping[str, ArrayExtraFieldDefinition[Any]] ) - """Members of an array document that the spec does not define, by key.""" + """Fields of an array document that the spec does not define, by key.""" group_extra_fields: ( Iterable[GroupExtraFieldDefinition[Any]] | Mapping[str, GroupExtraFieldDefinition[Any]] ) - """Members of a group document that the spec does not define, by key.""" + """Fields of a group document that the spec does not define, by key.""" @final class ZarrV3Context: """A scope that reads Zarr v3 documents. Every v3 reader takes one. - It implements `Context`, and has one slot for each kind of field a v3 + It implements `Context`, and has one slot for each kind of declaration a v3 document holds, as `ZarrV3ContextSlots` declares them. The slots for - the spec's extension points are named like the document member that - holds such fields: `data_types`, `chunk_grids`, `chunk_key_encodings`, + the spec's extension points are named like the document field that + holds such declarations: `data_types`, `chunk_grids`, `chunk_key_encodings`, `codecs` and `storage_transformers`. `array_extra_fields` and - `group_extra_fields` hold the definitions of members the spec does not + `group_extra_fields` hold the definitions of fields the spec does not define, by key. Each slot is a read-only mapping from the name each definition is filed under (`r*` for raw bits) to the definition. The constructor and `update` take, for each slot, an iterable of @@ -180,12 +180,12 @@ def storage_transformers(self) -> Mapping[str, StorageTransformerDefinition[Any] @property def array_extra_fields(self) -> Mapping[str, ArrayExtraFieldDefinition[Any]]: - """The definitions of members of an array document that the spec does not define, by key.""" + """The definitions of fields of an array document that the spec does not define, by key.""" return self.definitions_of(ArrayExtraFieldDefinition) @property def group_extra_fields(self) -> Mapping[str, GroupExtraFieldDefinition[Any]]: - """The definitions of members of a group document that the spec does not define, by key.""" + """The definitions of fields of a group document that the spec does not define, by key.""" return self.definitions_of(GroupExtraFieldDefinition) def definitions_of(self, kind: type[_K]) -> Mapping[str, _K]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_core.py b/packages/zarr-metadata/src/zarr_metadata/v3/_core.py index 5a3f52f4e4..d28fa6a4cb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_core.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_core.py @@ -5,7 +5,7 @@ optional. `CORE_AND_EXTENSIONS` adds the extensions that `zarr-extensions` registers and this package defines. A name in neither scope is not rejected, because that is what keeps the format open. Instead it is read -as an `UnclaimedField` and not checked. +as an `UnclaimedDeclaration` and not checked. """ from __future__ import annotations diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index 708e7b3c5d..4ddd0b213b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -3,9 +3,9 @@ The kinds at the spec's five extension points are `DataTypeDefinition`, `ChunkGridDefinition`, `ChunkKeyEncodingDefinition`, `CodecDefinition` and `StorageTransformerDefinition`. `ArrayExtraFieldDefinition` and -`GroupExtraFieldDefinition` read members the spec does not define. This module also holds the chunk that a +`GroupExtraFieldDefinition` read fields the spec does not define. This module also holds the chunk that a codec receives, how a data type's values are stored, the raw bits name -`r*`, and the JSON Schema of a v3 field. The reading itself is in +`r*`, and the JSON Schema of a v3 declaration. The reading itself is in `zarr_metadata.definition`, which both formats share. """ @@ -49,26 +49,26 @@ typeddict_keys, ) from zarr_metadata.definition._definition import ( - AcceptedField, + AcceptedDeclaration, C, Definition, Nested, Problems, - ResolvedField, + ResolvedDeclaration, WithFillValue, asked, - is_field, + is_declaration, no_rules, read_only, ruled, ) from zarr_metadata.v3._common import ( - ZarrV3ChunkGridField, - ZarrV3ChunkKeyEncodingField, - ZarrV3CodecField, - ZarrV3DataTypeField, - ZarrV3StaticCodecField, - ZarrV3StorageTransformerField, + ZarrV3ChunkGridDeclaration, + ZarrV3ChunkKeyEncodingDeclaration, + ZarrV3CodecDeclaration, + ZarrV3DataTypeDeclaration, + ZarrV3StaticCodecDeclaration, + ZarrV3StorageTransformerDeclaration, ) from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON @@ -157,11 +157,11 @@ class DataTypeDefinition(WithFillValue[C], kind=True, format=3): `fill_value` is the JSON shape of a fill value, such as `Int8FillValue`. It is an annotation that the checker reads the same - way it reads a configuration's members, including any range. + way it reads a configuration's fields, including any range. `fill_value_rules` yields the spec violations in a fill value of that shape that the type cannot express, such as a hex string of the wrong width. - The rules receive the configuration, the nested fields as the scope + The rules receive the configuration, the nested declarations as the scope read them (a struct's field types), and the typed fill value. A data type that does not declare its fill value accepts any JSON. @@ -179,7 +179,7 @@ class DataTypeDefinition(WithFillValue[C], kind=True, format=3): The `bytes` codec uses this to decide whether the data type it receives needs an `endian` (it does for multi-byte numbers). A struct's storage depends on its fields, so `storage` receives the - nested fields as the scope read them. A data type that does not + nested declarations as the scope read them. A data type that does not declare it leaves its storage unknown. A data type named the way a document writes raw bits of one size (such @@ -188,7 +188,7 @@ class DataTypeDefinition(WithFillValue[C], kind=True, format=3): """ label: ClassVar[str] = "data type" - field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3DataTypeField,) + declaration_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3DataTypeDeclaration,) @classmethod def _filed_under(cls, name: str) -> tuple[str | None, dict[str, JSONValue] | None]: @@ -213,7 +213,7 @@ def _envelope_json(cls, name: str, configuration: Mapping[str, JSONValue]) -> JS return name storage: Callable[[C, Nested], StorageClass | None] = unknown_storage - """How the values are stored, given the configuration and the nested fields; None when unknown.""" + """How the values are stored, given the configuration and the nested declarations; None when unknown.""" def _refusal(self) -> str | None: if RAW_BYTES_NAME_PATTERN.fullmatch(self.name) is not None: @@ -233,7 +233,7 @@ class ChunkGridDefinition(Definition[C], kind=True, format=3): `shape_rules` yields the spec violations in a grid with this configuration over an array of a given shape, such as a dimension with no chunk length, or chunks that do not cover the dimension. It receives - the configuration, the nested fields as the scope read them, and the + the configuration, the nested declarations as the scope read them, and the shape, and it locates its problems in the configuration. A grid that does not declare shape rules fits every shape. @@ -246,7 +246,7 @@ class ChunkGridDefinition(Definition[C], kind=True, format=3): """ label: ClassVar[str] = "chunk grid" - field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3ChunkGridField,) + declaration_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3ChunkGridDeclaration,) shape_rules: Callable[[C, Nested, tuple[int, ...]], Iterable[ValidationProblem]] = no_rules """The spec violations in this grid over an array of a given shape, located in the configuration.""" @@ -259,7 +259,7 @@ class ChunkKeyEncodingDefinition(Definition[C], kind=True, format=3): """A chunk key encoding.""" label: ClassVar[str] = "chunk key encoding" - field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3ChunkKeyEncodingField,) + declaration_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3ChunkKeyEncodingDeclaration,) CodecKind = Literal["array_array", "array_bytes", "bytes_bytes"] @@ -289,7 +289,7 @@ class CodecDefinition(Definition[C], kind=True, format=3): A codec that receives an array (array -> array, array -> bytes) declares the spec violations for a given input `Chunk` with - `chunk_rules`. It receives the configuration, the nested fields as the + `chunk_rules`. It receives the configuration, the nested declarations as the scope read them, and the chunk, and it locates its problems in the configuration. For example, a `bytes` codec without `endian` that receives a multi-byte data type is a problem. An array -> array codec @@ -310,7 +310,7 @@ class CodecDefinition(Definition[C], kind=True, format=3): A codec that contains its own pipelines declares what each one receives with `pipelines`. Given the codec's input chunk, it returns the chunk that the first codec of each pipeline receives, keyed by the - configuration member that holds the pipeline. For example, a shard's + configuration field that holds the pipeline. For example, a shard's inner codecs receive its inner chunks, and its index codecs receive the shard index. Like the transition, it is called whatever the chunk rules found, and must return only what is true in either case. A @@ -320,7 +320,10 @@ class CodecDefinition(Definition[C], kind=True, format=3): """ label: ClassVar[str] = "codec" - field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3CodecField, ZarrV3StaticCodecField) + declaration_aliases: ClassVar[tuple[TypeAliasType, ...]] = ( + ZarrV3CodecDeclaration, + ZarrV3StaticCodecDeclaration, + ) kind: CodecKind size: CodecSize @@ -329,7 +332,7 @@ class CodecDefinition(Definition[C], kind=True, format=3): transition: Callable[[C, Nested, Chunk], Chunk] = unknown_chunk """The chunk the next codec receives, given the chunk this array -> array codec receives.""" pipelines: Callable[[C, Nested, Chunk], Mapping[str, Chunk]] = no_pipelines - """The chunk each contained pipeline receives, keyed by the configuration member that holds the pipeline.""" + """The chunk each contained pipeline receives, keyed by the configuration field that holds the pipeline.""" def _refusal(self) -> str | None: kind: object = self.kind @@ -338,17 +341,19 @@ def _refusal(self) -> str | None: size: object = self.size if size not in get_args(CodecSize): return f"{self.name!r}: size is one of {get_args(CodecSize)!r}, got {size!r}" - defaults = {member.name: member.default for member in dataclasses.fields(CodecDefinition)} - for member in _UNASKED[self.kind]: - if getattr(self, member) is not defaults[member]: - return f"{self.name!r}: {member}, which no codec of kind {kind!r} is asked" + defaults = { + attribute.name: attribute.default for attribute in dataclasses.fields(CodecDefinition) + } + for attribute in _UNASKED[self.kind]: + if getattr(self, attribute) is not defaults[attribute]: + return f"{self.name!r}: {attribute}, which no codec of kind {kind!r} is asked" return None @classmethod def _alias_problems( cls, alias: TypeAliasType, definition: Definition[Any], loc: Loc ) -> Problems: - """A codec of dynamic size in a member annotated with `ZarrV3StaticCodecField`, which admits only codecs of static size, is a problem at the field.""" + """A codec of dynamic size in a field annotated with `ZarrV3StaticCodecDeclaration`, which admits only codecs of static size, is a problem at the declaration.""" if alias not in STATIC_SIZE_ALIASES: return () if not isinstance(definition, CodecDefinition) or definition.size == "static": @@ -366,43 +371,45 @@ class StorageTransformerDefinition(Definition[C], kind=True, format=3): """A storage transformer.""" label: ClassVar[str] = "storage transformer" - field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3StorageTransformerField,) + declaration_aliases: ClassVar[tuple[TypeAliasType, ...]] = ( + ZarrV3StorageTransformerDeclaration, + ) @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class ExtraFieldDefinition(Definition[C]): - """A member of a v3 document that the spec does not define, an extra field: its name is the member's key, and its configuration is the member's value. + """A field of a v3 document that the spec does not define, an extra field: its name is the field's key, and its configuration is the field's value. This is not a kind. `ArrayExtraFieldDefinition` and `GroupExtraFieldDefinition` are, because one key may mean different things in an array document and in a group document. An extra field is not one of the spec's extension points, and no registry names it, so - any key may name one. As a field it is written as the member alone, - `{key: value}`, and it is located at the member's key. A definition - reads a member whose value is an object. A member that no definition + any key may name one. As a declaration it is written as the field alone, + `{key: value}`, and it is located at the field's key. A definition + reads a field whose value is an object. A field that no definition in scope claims is not read: a reader keeps it as it is written, and lists it in `must_understand_fields` unless it says `"must_understand": false`. - `finite` says whether every number in the member must be finite, as - RFC 8259 JSON requires. A member that holds user data sets it to + `finite` says whether every number in the field must be finite, as + RFC 8259 JSON requires. A field that holds user data sets it to False: consolidated metadata holds node documents, whose attributes - may hold `NaN`, `Infinity` and `-Infinity`. A member that nothing + may hold `NaN`, `Infinity` and `-Infinity`. A field that nothing claims is not read, so like attributes it may hold any number. - The spec lets every extension member carry `"must_understand": false`, - which tells a reader that does not understand the member that it may + The spec lets every extension field carry `"must_understand": false`, + which tells a reader that does not understand the field that it may ignore it. So the configuration declares `must_understand`, and a - definition whose configuration does not is refused. A member the spec + definition whose configuration does not is refused. A field the spec defines, such as an array's `shape`, is never an extra field, and a definition named after one is refused too. """ finite: bool = True - """Whether every number in the member must be finite.""" + """Whether every number in the field must be finite.""" - spec_members: ClassVar[frozenset[str]] = frozenset() - """The members the spec defines in a document this kind reads extra fields of, which are never extra fields.""" + spec_fields: ClassVar[frozenset[str]] = frozenset() + """The fields the spec defines in a document this kind reads extra fields of, which are never extra fields.""" def _finite_numbers(self) -> bool: return self.finite @@ -416,14 +423,14 @@ def _refusal(self) -> str | None: try: takes = _takes_the_waiver(self.configuration) except TypeError as error: - # Name the member, as the checker names a member it cannot read. + # Name the field, as the checker names a field it cannot read. keys = typeddict_keys(self.configuration) - member = "must_understand" if "must_understand" in keys.members else "extra_items" - return f"{self.name!r}: {self.configuration.__name__}.{member}: {error}" + field = "must_understand" if "must_understand" in keys.fields else "extra_items" + return f"{self.name!r}: {self.configuration.__name__}.{field}: {error}" if not takes: return ( f"{self.name!r}: expected a configuration that takes must_understand as a " - "boolean that may be false, which the spec lets every extension member carry; " + "boolean that may be false, which the spec lets every extension field carry; " f"{self.configuration.__name__} does not" ) return None @@ -432,9 +439,9 @@ def _refusal(self) -> str | None: def _name_problem(cls, name: str, at: Loc) -> ValidationProblem | None: # No registry names extra fields, so any key the spec does not # define may name one. - if name in cls.spec_members: + if name in cls.spec_fields: return ValidationProblem( - at, f"{name!r} is a member the spec defines, not an extra field", "invalid_value" + at, f"{name!r} is a field the spec defines, not an extra field", "invalid_value" ) return None @@ -442,17 +449,17 @@ def _name_problem(cls, name: str, at: Loc) -> ValidationProblem | None: def _named_configuration( cls, value: object ) -> tuple[str | None, Mapping[str, object] | None, Problems]: - # A member whose value is not an object has no configuration. Only - # a definition that claims the member reads it, and `_refined` + # A field whose value is not an object has no configuration. Only + # a definition that claims the field reads it, and `_refined` # refuses such a value then. if not is_object(value) or len(value) != 1: return None, None, () - ((name, member),) = value.items() + ((name, configuration),) = value.items() if not isinstance(name, str): return None, None, () - if not is_object(member): + if not is_object(configuration): return name, None, () - return name, {key: item for key, item in member.items() if isinstance(key, str)}, () + return name, {key: item for key, item in configuration.items() if isinstance(key, str)}, () @classmethod def _envelope_problems(cls, value: object) -> Problems: @@ -460,7 +467,7 @@ def _envelope_problems(cls, value: object) -> Problems: if name is None: return ( ValidationProblem( - (), "expected a member, as an object with one string key", "invalid_type" + (), "expected a field, as an object with one string key", "invalid_type" ), ) bad = cls._name_problem(name, ()) @@ -470,9 +477,9 @@ def _envelope_problems(cls, value: object) -> Problems: def _refined( cls, data: object, loc: Loc, definition: Definition[Any] | None ) -> tuple[JSONValue | None, Problems]: - # The member's value is at the member's location, `loc`. A member + # The field's value is at the field's location, `loc`. A field # nothing claims is not read, so like attributes it is user data. - # A claimed member is user data when its definition says so, and + # A claimed field is user data when its definition says so, and # its value must be an object. name = cls._named_configuration(data)[0] if name is None or not is_object(data): @@ -495,7 +502,7 @@ def _envelope_json(cls, name: str, configuration: Mapping[str, JSONValue]) -> JS def _unclaimed_json( cls, json: JSONValue, name: str, configuration: Mapping[str, JSONValue] ) -> JSONValue: - # A member no definition claims is written as it is, whatever its + # A field no definition claims is written as it is, whatever its # value; only a definition reads it as an object. return json @@ -511,7 +518,7 @@ def _name_loc(cls, loc: Loc) -> Loc: def _takes_the_waiver(configuration: type) -> bool: """Whether `configuration`, a TypedDict, takes the spec's `"must_understand": false`: the type it gives `must_understand`, declared or as another key, admits `false` and only booleans.""" keys = typeddict_keys(configuration) - declared = keys.members.get("must_understand") + declared = keys.fields.get("must_understand") booleans = _booleans(keys.extra_items if declared is None else declared[0], frozenset()) return booleans is not None and False in booleans @@ -546,26 +553,26 @@ def _booleans(annotation: object, seen: frozenset[object]) -> frozenset[bool] | @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class ArrayExtraFieldDefinition(ExtraFieldDefinition[C], kind=True, format=3): - """An extra field of a v3 array document: a member the spec does not define, read by its key.""" + """An extra field of a v3 array document: a field the spec does not define, read by its key.""" label: ClassVar[str] = "array extra field" - spec_members: ClassVar[frozenset[str]] = frozenset( - typeddict_keys(ZarrV3ArrayMetadataJSON).members + spec_fields: ClassVar[frozenset[str]] = frozenset( + typeddict_keys(ZarrV3ArrayMetadataJSON).fields ) @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class GroupExtraFieldDefinition(ExtraFieldDefinition[C], kind=True, format=3): - """An extra field of a v3 group document: a member the spec does not define, read by its key. `consolidated_metadata` is one.""" + """An extra field of a v3 group document: a field the spec does not define, read by its key. `consolidated_metadata` is one.""" label: ClassVar[str] = "group extra field" - spec_members: ClassVar[frozenset[str]] = frozenset( - typeddict_keys(ZarrV3GroupMetadataJSON).members + spec_fields: ClassVar[frozenset[str]] = frozenset( + typeddict_keys(ZarrV3GroupMetadataJSON).fields ) -STATIC_SIZE_ALIASES: Final[frozenset[TypeAliasType]] = frozenset({ZarrV3StaticCodecField}) -"""The field aliases whose codec must be of static size.""" +STATIC_SIZE_ALIASES: Final[frozenset[TypeAliasType]] = frozenset({ZarrV3StaticCodecDeclaration}) +"""The declaration aliases whose codec must be of static size.""" Lengths: TypeAlias = tuple[frozenset[int] | None, ...] @@ -578,7 +585,7 @@ class Chunk: Unknown parts are None. This includes the lengths along an axis that the grid does not declare, and every part of the chunk output by a - codec that does not declare its output. A data type field that the + codec that does not declare its output. A data type declaration that the scope did not read is kept as written, and it also says nothing about the values. A codec's chunk rules check only what is known, so a chunk with nothing known about it, `Chunk()`, never causes a problem. @@ -586,8 +593,8 @@ class Chunk: lengths: Lengths | None = None """For each axis, the chunk lengths along it; None when even the number of axes is unknown.""" - data_type: ResolvedField[DataTypeDefinition[Any]] | None = None - """The data type field of the values, as a scope read it. None when no field declares the data type: a document with no data type (its reading holds `UNSET`) passes the pipeline a chunk of unknown type.""" + data_type: ResolvedDeclaration[DataTypeDefinition[Any]] | None = None + """The data type declaration of the values, as a scope read it. None when no field declares the data type: a document with no data type (its reading holds `UNSET`) passes the pipeline a chunk of unknown type.""" def __post_init__(self) -> None: lengths = cast("object", self.lengths) @@ -595,8 +602,8 @@ def __post_init__(self) -> None: msg = f"a chunk's lengths are a frozenset of integers or None per axis, got {lengths!r}" raise TypeError(msg) data_type = cast("object", self.data_type) - if data_type is not None and not _is_data_type_field(data_type): - msg = f"a chunk's data type is a data type field a scope read, got {data_type!r}" + if data_type is not None and not _is_data_type_declaration(data_type): + msg = f"a chunk's data type is a data type declaration a scope read, got {data_type!r}" raise TypeError(msg) @property @@ -605,9 +612,9 @@ def rank(self) -> int | None: return None if self.lengths is None else len(self.lengths) -def _is_data_type_field(value: object) -> bool: - """Whether `value` is a field that a scope read as a data type.""" - return is_field(value) and value.read_as is DataTypeDefinition +def _is_data_type_declaration(value: object) -> bool: + """Whether `value` is a declaration that a scope read as a data type.""" + return is_declaration(value) and value.read_as is DataTypeDefinition def _is_lengths(value: object) -> TypeGuard[Lengths]: @@ -625,8 +632,8 @@ def _is_lengths(value: object) -> TypeGuard[Lengths]: return True -def storage_of(data_type: ResolvedField[DataTypeDefinition[Any]]) -> StorageClass | None: - """How the values of `data_type` (a data type field a scope read) are stored; None when unknown. +def storage_of(data_type: ResolvedDeclaration[DataTypeDefinition[Any]]) -> StorageClass | None: + """How the values of `data_type` (a data type declaration a scope read) are stored; None when unknown. The storage is unknown when the scope did not read the data type, or its definition does not declare storage. The definition's `storage` is @@ -634,7 +641,7 @@ def storage_of(data_type: ResolvedField[DataTypeDefinition[Any]]) -> StorageClas class. If it raises an error, a note is added that says which data type's storage raised it. """ - if not isinstance(data_type, AcceptedField): + if not isinstance(data_type, AcceptedDeclaration): return None definition, configuration = data_type.definition, data_type.configuration found = asked( @@ -652,26 +659,26 @@ def storage_of(data_type: ResolvedField[DataTypeDefinition[Any]]) -> StorageClas def chunk_grid_lengths( - chunk_grid: ResolvedField[ChunkGridDefinition[Any]], shape: tuple[int, ...], loc: Loc = () + chunk_grid: ResolvedDeclaration[ChunkGridDefinition[Any]], shape: tuple[int, ...], loc: Loc = () ) -> tuple[Lengths, Problems]: - """The chunk lengths of `chunk_grid` (a chunk grid field a scope read) along each axis of an array of `shape`, and the problems with the grid for that shape. + """The chunk lengths of `chunk_grid` (a chunk grid declaration a scope read) along each axis of an array of `shape`, and the problems with the grid for that shape. A chunk has an extent "for each dimension of the array" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L277-L281), so the lengths have an entry for each dimension of `shape`, with None where they are unknown. The grid's shape rules check it first, and locate their problems in the configuration, under `loc` (the location - of the field). For example, a regular grid with chunk lengths for two + of the declaration). For example, a regular grid with chunk lengths for two dimensions over an array with three dimensions is a problem. Only a grid that fits the shape returns its lengths. A grid that the scope did not read (it is out of scope or invalid) is not checked and returns no lengths. The result of a grid's `chunk_lengths` is checked: lengths of the wrong type raise `TypeError`, and lengths with a different number of axes than `shape` raise `ValueError`. Both - indicate a bug in the definition, not in the field. + indicate a bug in the definition, not in the declaration. """ unknown: Lengths = (None,) * len(shape) - if not isinstance(chunk_grid, AcceptedField): + if not isinstance(chunk_grid, AcceptedDeclaration): return unknown, () definition, configuration = chunk_grid.definition, chunk_grid.configuration at = (*loc, "configuration") diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py b/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py index 420266eec4..899bf761c1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py @@ -1,4 +1,4 @@ -"""The JSON Schema of a Zarr v3 field and of a whole v3 `zarr.json`, as a scope reads them.""" +"""The JSON Schema of a Zarr v3 declaration and of a whole v3 `zarr.json`, as a scope reads them.""" from __future__ import annotations @@ -11,7 +11,7 @@ Definition, EmptyConfiguration, as_kind, - field_kind, + declaration_kind, ) from zarr_metadata.definition._envelope import EXTENSION_NAME_SCHEMA_PATTERN from zarr_metadata.v3._context import ZarrV3Context, ZarrV3Kind, is_v3_kind @@ -34,9 +34,9 @@ from zarr_metadata._typed_json import JSONSchema, SchemaLeaf -def kind_field(kind: type[Definition[Any]]) -> object: - """The field alias for a member that holds any field of `kind`: the first alias the kind declares that does not restrict the field. For example, `ZarrV3CodecField` instead of `ZarrV3StaticCodecField`.""" - return next(alias for alias in kind.field_aliases if alias not in STATIC_SIZE_ALIASES) +def kind_declaration(kind: type[Definition[Any]]) -> object: + """The declaration alias for a field that holds any declaration of `kind`: the first alias the kind declares that does not restrict the declaration. For example, `ZarrV3CodecDeclaration` instead of `ZarrV3StaticCodecDeclaration`.""" + return next(alias for alias in kind.declaration_aliases if alias not in STATIC_SIZE_ALIASES) _RAW_BYTES_SCHEMA_PATTERN: Final = f"^{RAW_BYTES_NAME_PATTERN.pattern}(?![\\s\\S])" @@ -49,63 +49,63 @@ def kind_field(kind: type[Definition[Any]]) -> object: """ -def field_json_schema_v3( +def declaration_json_schema_v3( kind: type[ZarrV3Kind], *, context: ZarrV3Context | None = None ) -> JSONSchema: - """The JSON Schema of one metadata field read as `kind` in `context` (`CORE_AND_EXTENSIONS` when none is given): everything `resolve` checks, except the rules. A scope of another format raises `TypeError`, as it does for every v3 reader. An extra field is written as the member alone, `{key: value}`. + """The JSON Schema of one declaration read as `kind` in `context` (`CORE_AND_EXTENSIONS` when none is given): everything `resolve` checks, except the rules. A scope of another format raises `TypeError`, as it does for every v3 reader. An extra field is written as the field alone, `{key: value}`. - The schema accepts a field that one of the definitions in scope reads + The schema accepts a declaration that one of the definitions in scope reads (its name, its configuration as the TypedDict declares it, an optional `must_understand` of `true`, and its bare name when it needs no configuration). It also accepts a name that no definition claims, with - any configuration, which is what keeps the format open. A field inside + any configuration, which is what keeps the format open. A declaration inside a configuration is written the same way, in the same scope, and a - member that accepts only codecs of static size accepts only those. The + field that accepts only codecs of static size accepts only those. The output is JSON Schema draft 2020-12, as `json_schema` writes it. The - nested fields and the configuration of each definition are in - `$defs`, under the name of the field alias or TypedDict. Constraints + nested declarations and the configuration of each definition are in + `$defs`, under the name of the declaration alias or TypedDict. Constraints that only a rule enforces (a blosc `typesize` checked against its - `shuffle`) are not in the schema, so a field it accepts may still have + `shuffle`) are not in the schema, so a declaration it accepts may still have a problem. """ asked = as_kind(kind) if not is_v3_kind(asked): msg = ( f"{asked.__name__} is a kind of another format; the JSON Schema writer writes " - "Zarr v3 fields only" + "Zarr v3 declarations only" ) raise TypeError(msg) context = scoped(context, CORE_AND_EXTENSIONS) - schemas = Schemas(field_schemas(context)) + schemas = Schemas(declaration_schemas(context)) if issubclass(asked, (ArrayExtraFieldDefinition, GroupExtraFieldDefinition)): return schemas.document(_extra_field_schema(asked, context, schemas)) - return schemas.document(schemas.of(kind_field(asked))) + return schemas.document(schemas.of(kind_declaration(asked))) -def field_schemas(context: ZarrV3Context) -> SchemaLeaf: - """The schema leaf that `field_json_schema_v3` uses. It writes each field alias as a field of its kind, as `context` reads it.""" +def declaration_schemas(context: ZarrV3Context) -> SchemaLeaf: + """The schema leaf that `declaration_json_schema_v3` uses. It writes each declaration alias as a declaration of its kind, as `context` reads it.""" def leaf(annotation: object, schemas: Schemas) -> JSONSchema | None: - kind = field_kind(annotation) + kind = declaration_kind(annotation) if kind is None or not is_alias(annotation): return None alias = annotation static = annotation in STATIC_SIZE_ALIASES return schemas.defined( - alias, alias.__name__, lambda: _field_schema(kind, static, context, schemas) + alias, alias.__name__, lambda: _declaration_schema(kind, static, context, schemas) ) return leaf -def _field_schema( +def _declaration_schema( kind: type[Definition[Any]], static: bool, context: ZarrV3Context, schemas: Schemas ) -> JSONSchema: - """The schema of a field of `kind` as `context` reads it: a field that a definition in scope reads, or a field that none of them claims. + """The schema of a declaration of `kind` as `context` reads it: a declaration that a definition in scope reads, or a declaration that none of them claims. - `kind` is always a v3 kind. The v3 document TypedDicts hold v3 fields - only, a v3 definition whose configuration holds a field of the other - format is refused when it is built, and `field_json_schema_v3` refuses + `kind` is always a v3 kind. The v3 document TypedDicts hold v3 declarations + only, a v3 definition whose configuration holds a declaration of the other + format is refused when it is built, and `declaration_json_schema_v3` refuses a kind of the other format before it writes anything. The check below only narrows `kind` for the type checker. """ @@ -127,7 +127,7 @@ def _extra_field_schema( context: ZarrV3Context, schemas: Schemas, ) -> JSONSchema: - """The schema of an extra field of `kind` as `context` reads it, written as the member alone, `{key: value}`: a member whose key a definition in scope claims, with the value its configuration declares, or a member with any other key that the spec does not define, and any JSON value.""" + """The schema of an extra field of `kind` as `context` reads it, written as the field alone, `{key: value}`: a field whose key a definition in scope claims, with the value its configuration declares, or a field with any other key that the spec does not define, and any JSON value.""" claimed = list(context.definitions_of(kind).values()) branches: list[JSONValue] = [ { @@ -139,8 +139,8 @@ def _extra_field_schema( for definition in claimed ] unclaimed: JSONSchema = {"type": "object", "minProperties": 1, "maxProperties": 1} - # Neither a claimed key nor a member the spec defines is an unclaimed member. - others = sorted({definition.name for definition in claimed} | kind.spec_members) + # Neither a claimed key nor a field the spec defines is an unclaimed field. + others = sorted({definition.name for definition in claimed} | kind.spec_fields) unclaimed["propertyNames"] = {"not": {"enum": [*others]}} branches.append(unclaimed) return {"anyOf": branches} @@ -154,7 +154,7 @@ def written_name(definition: Definition[Any]) -> JSONSchema: def _read_by(definition: Definition[Any], schemas: Schemas) -> list[JSONValue]: - """The schemas of the fields `definition` reads: an object with its name and configuration, and its bare name when it needs no configuration. + """The schemas of the declarations `definition` reads: an object with its name and configuration, and its bare name when it needs no configuration. The raw bits name encodes the configuration, so the configuration written next to it must be empty. @@ -178,7 +178,7 @@ def _read_by(definition: Definition[Any], schemas: Schemas) -> list[JSONValue]: def _unclaimed(table: Mapping[str, Definition[Any]]) -> list[JSONValue]: - """The schemas of the fields that no definition in `table` claims: a name that none of them uses, either bare or with any configuration.""" + """The schemas of the declarations that no definition in `table` claims: a name that none of them uses, either bare or with any configuration.""" claimed: list[JSONValue] = [written_name(definition) for definition in table.values()] name: JSONSchema = {"type": "string", "pattern": EXTENSION_NAME_SCHEMA_PATTERN} if len(claimed) != 0: @@ -202,7 +202,7 @@ def node_metadata_json_schema_v3(*, context: ZarrV3Context | None = None) -> JSO Use it in an editor that validates a `zarr.json` while it is written, or in a validator written in another language. The schema is JSON Schema draft 2020-12, in the form `json_schema` writes. Each - extension point is a field as the scope `context` reads it: either a + extension point is a declaration as the scope `context` reads it: either a name that a definition in scope claims, with the configuration that definition declares, or a name that no definition claims. When the document names a data type in scope, the fill value must have the @@ -215,8 +215,8 @@ def node_metadata_json_schema_v3(*, context: ZarrV3Context | None = None) -> JSO is in `$defs` under the name of its TypedDict. `ZarrV3ArrayMetadataJSON` describes only an array. - A JSON Schema describes each member on its own. It does not contain - the rules that relate members to each other: one dimension name per + A JSON Schema describes each field on its own. It does not contain + the rules that relate fields to each other: one dimension name per dimension of the shape, a chunk grid that fits the shape, codecs in pipeline order with each codec checked against the chunk it receives, and the checks in a definition's `rules`. So a document @@ -234,8 +234,8 @@ def node_metadata_json_schema_v3(*, context: ZarrV3Context | None = None) -> JSO def _documents(context: ZarrV3Context) -> SchemaLeaf: - """The schema leaf for documents read in `context`: an array's fill value constrained by its data type, each field as `context` reads it, and each extra field that a definition in `context` claims.""" - fields = field_schemas(context) + """The schema leaf for documents read in `context`: an array's fill value constrained by its data type, each declaration as `context` reads it, and each extra field that a definition in `context` claims.""" + declarations = declaration_schemas(context) def leaf(annotation: object, schemas: Schemas) -> JSONSchema | None: if annotation is ZarrV3ArrayMetadataJSON: @@ -250,7 +250,7 @@ def leaf(annotation: object, schemas: Schemas) -> JSONSchema | None: schemas.object_of(ZarrV3GroupMetadataJSON), context.group_extra_fields, schemas ), ) - return fields(annotation, schemas) + return declarations(annotation, schemas) return leaf @@ -258,21 +258,21 @@ def leaf(annotation: object, schemas: Schemas) -> JSONSchema | None: def _with_extra_fields( schema: JSONSchema, definitions: Mapping[str, ExtraFieldDefinition[Any]], schemas: Schemas ) -> JSONSchema: - """`schema`, a node document's schema, with a property for each extra field in `definitions`: the member's value, as the configuration TypedDict of the definition that claims its key declares it.""" + """`schema`, a node document's schema, with a property for each extra field in `definitions`: the field's value, as the configuration TypedDict of the definition that claims its key declares it.""" if len(definitions) == 0: return schema - properties = _member_object(schema, "properties") + properties = _field_object(schema, "properties") extra: dict[str, JSONValue] = { key: schemas.of(definition.configuration) for key, definition in definitions.items() } return {**schema, "properties": {**properties, **extra}} -def _member_object(schema: Mapping[str, JSONValue], key: str) -> Mapping[str, JSONValue]: - """The object at member `key` of `schema`, or an empty one when the member is absent. Raises `TypeError` for a member that is not an object, which is a bug in the schema this module built.""" +def _field_object(schema: Mapping[str, JSONValue], key: str) -> Mapping[str, JSONValue]: + """The object at field `key` of `schema`, or an empty one when the field is absent. Raises `TypeError` for a field that is not an object, which is a bug in the schema this module built.""" found = schema.get(key, {}) if not isinstance(found, Mapping): - msg = f"the schema member {key!r} is an object, got {found!r}" + msg = f"the schema field {key!r} is an object, got {found!r}" raise TypeError(msg) return found @@ -285,8 +285,8 @@ def _array(context: ZarrV3Context, schemas: Schemas) -> JSONSchema: empty `codecs`. """ schema = schemas.object_of(ZarrV3ArrayMetadataJSON) - properties = _member_object(schema, "properties") - codecs = _member_object(properties, "codecs") + properties = _field_object(schema, "properties") + codecs = _field_object(properties, "codecs") schema = {**schema, "properties": {**properties, "codecs": {**codecs, "minItems": 1}}} schema = _with_extra_fields(schema, context.array_extra_fields, schemas) held: list[JSONValue] = [] @@ -307,8 +307,8 @@ def _array(context: ZarrV3Context, schemas: Schemas) -> JSONSchema: __all__ = [ - "field_json_schema_v3", - "field_schemas", + "declaration_json_schema_v3", + "declaration_schemas", "node_metadata_json_schema_v3", "written_name", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py index d0c9eb82eb..e1432ea9ab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py @@ -35,8 +35,8 @@ from zarr_metadata._json import ValidationProblem, is_object, is_tuple, with_input from zarr_metadata.definition._definition import ( - AcceptedField, - ResolvedField, + AcceptedDeclaration, + ResolvedDeclaration, asked, in_document, read_only, @@ -67,7 +67,7 @@ def _no_stages() -> Mapping[str, tuple[Stage, ...]]: class Stage: """One codec of a pipeline, and the chunk it receives.""" - codec: ResolvedField[CodecDefinition[Any]] + codec: ResolvedDeclaration[CodecDefinition[Any]] """The codec, as the scope read it.""" incoming: Chunk | None """The chunk the codec receives. @@ -77,7 +77,7 @@ class Stage: when its input is not known to be an array. """ inner: Mapping[str, tuple[Stage, ...]] = dataclasses.field(default_factory=_no_stages) - """The pipelines the codec contains, keyed by the configuration member that holds each one. Each pipeline pairs each codec with the chunk it receives.""" + """The pipelines the codec contains, keyed by the configuration field that holds each one. Each pipeline pairs each codec with the chunk it receives.""" def __post_init__(self) -> None: # Make this read-only, like everything else a reading returns. @@ -103,15 +103,15 @@ def __reduce__(self) -> tuple[type[Stage], tuple[object, ...]]: def read_pipeline( - codecs: Sequence[ResolvedField[CodecDefinition[Any]]], chunk: Chunk, loc: Loc = () + codecs: Sequence[ResolvedDeclaration[CodecDefinition[Any]]], chunk: Chunk, loc: Loc = () ) -> tuple[tuple[Stage, ...], Problems]: - """Read `codecs` (codec fields read by a context) as a pipeline that receives `chunk`. Return each codec with the chunk it receives, and the problems found. + """Read `codecs` (codec declarations read by a context) as a pipeline that receives `chunk`. Return each codec with the chunk it receives, and the problems found. First the order is checked: array -> array codecs, then one array -> bytes codec, then bytes -> bytes codecs. Then each codec is processed in turn, with the chunk the previous codec passed on. Its chunk rules check it, and each problem is located where the document - writes it: a member of its configuration in the configuration (or at + writes it: a field of its configuration in the configuration (or at the configuration itself, missing, for a codec that writes none), and a problem with the codec as a whole at the codec, as `_codec_problems` says. Each pipeline it contains is read the same way, at its location @@ -138,7 +138,7 @@ def read_pipeline( incoming = Chunk() if handed is None else handed at = (*loc, index, "configuration") inner = _no_stages() - if isinstance(codec, AcceptedField): + if isinstance(codec, AcceptedDeclaration): configuration, nested = codec.configuration, codec.nested found = _chunk_problems(definition, configuration, nested, incoming, at) problems.extend(_codec_problems(codec, (*loc, index), at, found)) @@ -147,7 +147,7 @@ def read_pipeline( stages.append(Stage(codec, incoming, inner)) if definition.kind == "array_bytes": handed = None - elif not isinstance(codec, AcceptedField): + elif not isinstance(codec, AcceptedDeclaration): handed = Chunk() else: handed = _handed_on(definition, codec.configuration, codec.nested, incoming, at) @@ -157,7 +157,7 @@ def read_pipeline( def _order_problems( - codecs: Sequence[ResolvedField[CodecDefinition[Any]]], loc: Loc + codecs: Sequence[ResolvedDeclaration[CodecDefinition[Any]]], loc: Loc ) -> Iterator[ValidationProblem]: """Array -> array codecs, then one array -> bytes codec, then bytes -> bytes codecs. @@ -215,7 +215,7 @@ def _chunk_problems( def _codec_problems( - codec: ResolvedField[CodecDefinition[Any]], here: Loc, at: Loc, found: Problems + codec: ResolvedDeclaration[CodecDefinition[Any]], here: Loc, at: Loc, found: Problems ) -> Problems: """`found`, the problems the chunk rules find in `codec` at `here`, located under `at`, its configuration, each moved to where the document writes it. @@ -243,11 +243,11 @@ def _inner_pipelines( ) -> tuple[Mapping[str, tuple[Stage, ...]], Problems]: """Read each pipeline contained in a codec that receives `chunk`, starting from the chunk that the codec's `pipelines` returns. - A pipeline's codecs are the fields in the configuration member that + A pipeline's codecs are the declarations in the configuration field that holds it, as the scope read them. Its problems are located at that - member, under `at`. `pipelines` is the extension author's code, so its - result is checked: it must map configuration members to chunks, and - each member must be a list of codecs. + field, under `at`. `pipelines` is the extension author's code, so its + result is checked: it must map configuration fields to chunks, and + each field must be a list of codecs. """ given = asked( definition, @@ -257,22 +257,22 @@ def _inner_pipelines( ) if not _is_pipelines(given): msg = ( - f"{definition.name!r}: its pipelines give a mapping of members to chunks, got {given!r}" + f"{definition.name!r}: its pipelines give a mapping of fields to chunks, got {given!r}" ) raise TypeError(msg) read: dict[str, tuple[Stage, ...]] = {} problems: list[ValidationProblem] = [] - for member, handed in given.items(): - codecs = _held(definition, configuration, nested, member) - read[member], found = read_pipeline(codecs, handed, (*at, member)) + for field, handed in given.items(): + codecs = _held(definition, configuration, nested, field) + read[field], found = read_pipeline(codecs, handed, (*at, field)) problems.extend(found) return read, tuple(problems) def _is_pipelines(value: object) -> TypeGuard[Mapping[str, Chunk]]: - """Whether `value` maps members of a configuration to chunks.""" + """Whether `value` maps fields of a configuration to chunks.""" return is_object(value) and all( - isinstance(member, str) and isinstance(chunk, Chunk) for member, chunk in value.items() + isinstance(field, str) and isinstance(chunk, Chunk) for field, chunk in value.items() ) @@ -280,20 +280,20 @@ def _held( definition: CodecDefinition[Any], configuration: Mapping[str, object], nested: Nested, - member: str, -) -> tuple[ResolvedField[CodecDefinition[Any]], ...]: - """The codecs in configuration member `member`, as the scope read them. + field: str, +) -> tuple[ResolvedDeclaration[CodecDefinition[Any]], ...]: + """The codecs in configuration field `field`, as the scope read them. - If the member holds anything other than a list of fields read as + If the field holds anything other than a list of declarations read as codecs, this raises `TypeError`, because the definition that names the - member has a bug. + field has a bug. """ - entries: object = configuration.get(member) - places = [(member, index) for index in range(len(entries))] if is_tuple(entries) else None + entries: object = configuration.get(field) + places = [(field, index) for index in range(len(entries))] if is_tuple(entries) else None if places is None or not all( place in nested and nested[place].read_as is CodecDefinition for place in places ): - msg = f"{definition.name!r}: its pipelines name {member!r}, which holds no list of codecs" + msg = f"{definition.name!r}: its pipelines name {field!r}, which holds no list of codecs" raise TypeError(msg) return tuple(nested[place] for place in places) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/array.py b/packages/zarr-metadata/src/zarr_metadata/v3/array.py index 6c815d40fd..27cb2427b1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/array.py @@ -8,12 +8,12 @@ from zarr_metadata._common import JSONValue from zarr_metadata.v3._common import ( - ZarrV3ChunkGridField, - ZarrV3ChunkKeyEncodingField, - ZarrV3CodecField, - ZarrV3DataTypeField, + ZarrV3ChunkGridDeclaration, + ZarrV3ChunkKeyEncodingDeclaration, + ZarrV3CodecDeclaration, + ZarrV3DataTypeDeclaration, ZarrV3ExtensionField, - ZarrV3StorageTransformerField, + ZarrV3StorageTransformerDeclaration, ) @@ -22,24 +22,24 @@ class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): Zarr v3 array metadata document (the `zarr.json` content for an array). Extra keys may contain arbitrary JSON values. Each extension point is - annotated with the field alias of its kind. For example, `data_type` is - a `ZarrV3DataTypeField`, and each entry of `codecs` is a - `ZarrV3CodecField`. The alias describes the JSON of the metadata field, - and it tells a scope which kind to read the field as. + annotated with the declaration alias of its kind. For example, `data_type` is + a `ZarrV3DataTypeDeclaration`, and each entry of `codecs` is a + `ZarrV3CodecDeclaration`. The alias describes the JSON of the declaration, + and it tells a scope which kind to read the declaration as. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#array-metadata """ zarr_format: Literal[3] node_type: Literal["array"] - data_type: ZarrV3DataTypeField + data_type: ZarrV3DataTypeDeclaration shape: tuple[Annotated[int, Ge(0)], ...] - chunk_grid: ZarrV3ChunkGridField - chunk_key_encoding: ZarrV3ChunkKeyEncodingField + chunk_grid: ZarrV3ChunkGridDeclaration + chunk_key_encoding: ZarrV3ChunkKeyEncodingDeclaration fill_value: JSONValue - codecs: tuple[ZarrV3CodecField, ...] + codecs: tuple[ZarrV3CodecDeclaration, ...] attributes: NotRequired[Mapping[str, JSONValue]] - storage_transformers: NotRequired[tuple[ZarrV3StorageTransformerField, ...]] + storage_transformers: NotRequired[tuple[ZarrV3StorageTransformerDeclaration, ...]] dimension_names: NotRequired[tuple[str | None, ...]] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py index 569a298d7b..2f99ca9b7d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py @@ -9,9 +9,9 @@ codec that has a configuration, `CodecConfiguration`. For the JSON of any codec entry (one of the entries in array metadata's -`codecs`), import `ZarrV3MetadataFieldJSON` from `zarr_metadata.v3`. A -configuration member that holds a codec, such as sharding's inner -pipelines, is annotated `ZarrV3CodecField` from +`codecs`), import `ZarrV3DeclarationJSON` from `zarr_metadata.v3`. A +configuration field that holds a codec, such as sharding's inner +pipelines, is annotated `ZarrV3CodecDeclaration` from `zarr_metadata.v3.definition` instead, so that the reader reads it as a codec. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 51aadb5f22..cbea5eae9b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -66,7 +66,7 @@ class BloscCodecObject(TypedDict, closed=True): The spec requires `typesize` "unless `shuffle` is `"noshuffle"`, in which case the value is ignored", so this is the one value that changes whether -another member is required. +another field is required. """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 8094321161..118d09cf77 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -12,11 +12,11 @@ from zarr_metadata._common import JSONValue from zarr_metadata._json import ValidationProblem, shown from zarr_metadata.definition._definition import ( - AcceptedField, + AcceptedDeclaration, Nested, fill_value_problems, ) -from zarr_metadata.v3._common import ZarrV3DataTypeField +from zarr_metadata.v3._common import ZarrV3DataTypeDeclaration from zarr_metadata.v3._definition import ( Chunk, CodecDefinition, @@ -84,7 +84,7 @@ class CastValueCodecConfiguration(TypedDict, closed=True): bare-string primitive name or a `{name, configuration}` envelope. """ - data_type: ZarrV3DataTypeField + data_type: ZarrV3DataTypeDeclaration rounding: NotRequired[CastRoundingMode] out_of_range: NotRequired[CastOutOfRangeMode] scalar_map: NotRequired[ScalarMap] @@ -150,7 +150,7 @@ def _rules( not model real numbers, that is the only problem reported for it. """ target = nested.get(("data_type",)) - if not isinstance(target, AcceptedField): + if not isinstance(target, AcceptedDeclaration): return name, written = target.definition.name, target.name if name in _NO_REAL_NUMBERS: @@ -182,7 +182,7 @@ def _chunk_rules( data type. """ source = chunk.data_type - if not isinstance(source, AcceptedField): + if not isinstance(source, AcceptedDeclaration): return if source.definition.name in _NO_REAL_NUMBERS: yield ValidationProblem( @@ -201,8 +201,8 @@ def _transition(configuration: CastValueCodecConfiguration, nested: Nested, chun It "converts (casts) the values of the input array to a new data type ... and it leaves all other array properties intact" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L3). - The output data type is the data type field as the scope read it. If - the scope did not read that field, the output data type is unknown. + The output data type is the data type declaration as the scope read it. If + the scope did not read that declaration, the output data type is unknown. """ return Chunk(chunk.lengths, nested.get(("data_type",))) @@ -218,7 +218,7 @@ def _transition(configuration: CastValueCodecConfiguration, nested: Nested, chun ) """The `cast_value` codec. -The target data type is a nested field. It is read in the same scope as +The target data type is a nested declaration. It is read in the same scope as the codec, and it becomes the data type of the output chunk. """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 5816d229f6..88e8967030 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -12,7 +12,7 @@ from zarr_metadata._common import JSONValue from zarr_metadata._json import ValidationProblem, shown from zarr_metadata.definition._definition import ( - AcceptedField, + AcceptedDeclaration, Nested, fill_value_problems, ) @@ -98,7 +98,7 @@ def _chunk_rules( The rules, not the chunk rules, reject a null. """ source = chunk.data_type - if not isinstance(source, AcceptedField): + if not isinstance(source, AcceptedDeclaration): return if source.definition.name in NOT_NUMBERS: yield ValidationProblem( @@ -107,10 +107,10 @@ def _chunk_rules( "invalid_value", ) return - for member in ("offset", "scale"): - value = configuration.get(member) + for field in ("offset", "scale"): + value = configuration.get(field) if value is not None: - yield from fill_value_problems(source, value, (member,)) + yield from fill_value_problems(source, value, (field,)) def _transition( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 17e702e213..125759efbf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -12,12 +12,12 @@ from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import ( - AcceptedField, + AcceptedDeclaration, Nested, ) from zarr_metadata.v3._common import ( - ZarrV3CodecField, - ZarrV3StaticCodecField, + ZarrV3CodecDeclaration, + ZarrV3StaticCodecDeclaration, ) from zarr_metadata.v3._definition import ( Chunk, @@ -60,8 +60,8 @@ class ShardingIndexedCodecConfiguration(TypedDict, closed=True): """ chunk_shape: tuple[Annotated[int, Ge(1)], ...] - codecs: tuple[ZarrV3CodecField, ...] - index_codecs: tuple[ZarrV3StaticCodecField, ...] + codecs: tuple[ZarrV3CodecDeclaration, ...] + index_codecs: tuple[ZarrV3StaticCodecDeclaration, ...] index_location: NotRequired[ShardingIndexLocation] @@ -80,7 +80,7 @@ class ShardingIndexedCodecObject(TypedDict, closed=True): `index_codecs`), so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L129-L138 (`chunk_shape`) - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L141-L155 (`codecs` and `index_codecs`, the members the spec marks required) + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L141-L155 (`codecs` and `index_codecs`, the fields the spec marks required) https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ @@ -122,7 +122,7 @@ def _chunk_rules( ) -_UINT64: Final = AcceptedField( +_UINT64: Final = AcceptedDeclaration( json=UINT64_DATA_TYPE_NAME, name=UINT64_DATA_TYPE_NAME, definition=UINT64_DATA_TYPE, @@ -200,7 +200,7 @@ def _canonical( ) """The `sharding_indexed` codec. -Its two pipelines are nested fields. Each codec in them is read in the +Its two pipelines are nested declarations. Each codec in them is read in the same scope as the shard, and each pipeline is read as a pipeline: the inner codecs receive the inner chunks, and the index codecs receive the shard index. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index a3aa01afb9..fed05f5676 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -8,7 +8,7 @@ This is a known non-core interoperability extension. The core spec fixes only its `{kind, must_understand, metadata}` envelope, which `ZarrV3ConsolidatedMetadataEnvelopeJSON` types with any JSON as the -entries. `CONSOLIDATED_METADATA_EXTRA_FIELD` reads a group's member as +entries. `CONSOLIDATED_METADATA_EXTRA_FIELD` reads a group's field as that envelope, and `CORE` holds it. `ZarrV3ConsolidatedMetadataJSON` types the entries as the reference implementation writes them. """ @@ -26,7 +26,7 @@ class ZarrV3ConsolidatedMetadataEnvelopeJSON(TypedDict, closed=True): """ - The `consolidated_metadata` member of a v3 group document, as the core spec fixes it. + The `consolidated_metadata` field of a v3 group document, as the core spec fixes it. The spec allows this one additional field on a group, for historical reasons, and fixes its envelope: `kind` is `"inline"`, @@ -83,7 +83,7 @@ class ZarrV3ConsolidatedMetadataJSON(TypedDict, closed=True): finite=False, ) ) -"""A group's `consolidated_metadata` member, read as the envelope the core spec fixes. The documents it lists are not read: they may hold the non-finite numbers that node attributes may, and `ZarrV3ConsolidatedMetadata` reads them.""" +"""A group's `consolidated_metadata` field, read as the envelope the core spec fixes. The documents it lists are not read: they may hold the non-finite numbers that node attributes may, and `ZarrV3ConsolidatedMetadata` reads them.""" __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 7c8367349d..862ae0fea2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -114,7 +114,7 @@ def _storage(configuration: RawBytesConfiguration, nested: Nested) -> StorageCla """Raw bits, `r*`: a single data type whose name carries its size. `r16` reads as `r*` with `{"bits": 16}`. A size that is not a positive -multiple of 8 is a problem of the field that names it. The simplest +multiple of 8 is a problem of the declaration that names it. The simplest spelling writes the size back into the name in decimal, so `r008` becomes `r8`. """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index ac974198d4..b5842afb7e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -16,7 +16,7 @@ fill_value_problems, spelled_canonically, ) -from zarr_metadata.v3._common import ZarrV3DataTypeField +from zarr_metadata.v3._common import ZarrV3DataTypeDeclaration from zarr_metadata.v3._definition import ( DataTypeDefinition, StorageClass, @@ -44,7 +44,7 @@ class StructField(TypedDict, closed=True): """ name: ReadOnly[str] - data_type: ReadOnly[ZarrV3DataTypeField] + data_type: ReadOnly[ZarrV3DataTypeDeclaration] class StructConfiguration(TypedDict, closed=True): @@ -84,8 +84,8 @@ def _rules(configuration: StructConfiguration, nested: Nested) -> Iterator[Valid if len(fields) == 0: yield ValidationProblem(("fields",), "expected at least one struct field", "invalid_value") seen: dict[str, int] = {} - for index, member in enumerate(fields): - name = member["name"] + for index, field in enumerate(fields): + name = field["name"] if name == "": yield ValidationProblem( ("fields", index, "name"), "expected a non-empty field name", "invalid_value" @@ -118,7 +118,7 @@ def _fill_value_rules( does not hold a reading of that field type, the fill value of that field is not checked. """ - names = [member["name"] for member in configuration["fields"]] + names = [field["name"] for field in configuration["fields"]] declared = set(names) for index, name in enumerate(names): if name not in value: @@ -144,8 +144,8 @@ def _fill_value_canonical( field is kept as written. """ spelled: dict[str, JSONValue] = {} - for index, member in enumerate(configuration["fields"]): - name = member["name"] + for index, field in enumerate(configuration["fields"]): + name = field["name"] field_type = nested.get(("fields", index, "data_type")) held = value[name] spelled[name] = held if field_type is None else spelled_canonically(field_type, held) @@ -188,7 +188,7 @@ def _storage(configuration: StructConfiguration, nested: Nested) -> StorageClass fill_value_canonical=_fill_value_canonical, storage=_storage, ) -"""The `struct` data type: a record of named fields. The type of each field is a nested field.""" +"""The `struct` data type: a record of named fields. The type of each field is a nested declaration.""" __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py index 90cb1e466e..7813b3f8e5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py @@ -5,16 +5,16 @@ is determined by the input size, or `"dynamic"` if it depends on the values), `DataTypeDefinition`, `ChunkGridDefinition`, `ChunkKeyEncodingDefinition`, `StorageTransformerDefinition`, and, for -the members of a document that the spec does not define, +the fields of a document that the spec does not define, `ArrayExtraFieldDefinition` and `GroupExtraFieldDefinition`. What a -definition is and how a field is read are in `zarr_metadata.definition`, +definition is and how a declaration is read are in `zarr_metadata.definition`, which both formats share. `ZarrV3Context` is the v3 scope, with one slot for each kind: `data_types`, `chunk_grids`, `chunk_key_encodings`, `codecs`, `storage_transformers`, `array_extra_fields` and `group_extra_fields`. `CORE` and `CORE_AND_EXTENSIONS` are the v3 scopes this package provides. -A scope reads whole documents as well as fields. +A scope reads whole documents as well as declarations. `read_array_metadata_v3(document, context=CONTEXT)`, from `zarr_metadata.model`, reads each extension point of a v3 array document through the definitions in `CONTEXT`, and so do the model's `from_json` and @@ -23,27 +23,27 @@ the shape, by the grid's definition. The codecs are checked as a pipeline, each by its definition against the chunk it receives. -A member that holds another metadata field is annotated with the field -alias of its kind (a shard's `codecs: tuple[ZarrV3CodecField, ...]`) and -is read in the same scope as its parent field. A problem with the nested -field belongs to that field and is reported at its location. The parent -field is still read, as a document containing the nested field would be. -A member that accepts only codecs of static size is annotated -`ZarrV3StaticCodecField`. A shard's `index_codecs` uses this, because a +A field that holds another declaration is annotated with the declaration +alias of its kind (a shard's `codecs: tuple[ZarrV3CodecDeclaration, ...]`) and +is read in the same scope as its parent declaration. A problem with the nested +declaration belongs to that declaration and is reported at its location. The parent +declaration is still read, as a document containing the nested declaration would be. +A field that accepts only codecs of static size is annotated +`ZarrV3StaticCodecDeclaration`. A shard's `index_codecs` uses this, because a reader locates the index from a size it knows before reading it. A codec -of dynamic size in such a member is a problem at its location, when the -field is read in a scope. A name that nothing claims is not checked, and +of dynamic size in such a field is a problem at its location, when the +declaration is read in a scope. A name that nothing claims is not checked, and its size is unknown, like everything else about it. -`ZarrV3MetadataFieldJSON` describes the same JSON, but it is checked -only as JSON, so a definition rejects a member typed with it. An +`ZarrV3DeclarationJSON` describes the same JSON, but it is checked +only as JSON, so a definition rejects a field typed with it. An extension with nothing to configure uses `EmptyConfiguration`, and is written with its name only. A data type also declares how its values are stored. `storage` is a -function of the configuration and the nested fields that returns a +function of the configuration and the nested declarations that returns a `StorageClass`: single bytes, several bytes at a time, or each value in as many bytes as it needs. A struct's storage depends on its fields. -`storage_of(data_type)` calls it for a data type field the scope read. +`storage_of(data_type)` calls it for a data type declaration the scope read. The `bytes` codec requires an `endian` for multi-byte numbers, and a struct rejects a field whose values vary in size. A data type that does not declare storage leaves it unknown. @@ -61,18 +61,18 @@ A codec is checked against its input. The array passes its first codec a `Chunk`: the lengths of the grid's chunks along each of the array's -dimensions, and the array's data type field, with None for unknown +dimensions, and the array's data type declaration, with None for unknown parts. A codec that receives an array declares the spec violations for a given chunk with `chunk_rules`, located in its configuration (for example, a `transpose` whose `order` has the wrong number of axes). An array -> array codec declares what it passes to the next codec with `transition`, whatever its chunk rules found. For example, `transpose` -permutes the axes. A reading reads the codec fields as a pipeline. It +permutes the axes. A reading reads the codec declarations as a pipeline. It checks their order (array -> array codecs, one array -> bytes codec, bytes -> bytes codecs), then checks each codec against the chunk it receives, and records a `Stage` for each codec with that chunk. A codec that contains its own pipelines declares what each one receives with -`pipelines`, keyed by the configuration member that holds each +`pipelines`, keyed by the configuration field that holds each pipeline. For example, a shard's inner codecs receive its inner chunks, and its index codecs receive the shard index. Each pipeline is read the same way, and its stages are stored as the codec's `Stage.inner`. @@ -89,17 +89,17 @@ `r*`. `r*` itself is notation, and a document that uses it as a name refers to nothing in any scope. -**JSON Schema.** `zarr_metadata.json_schema` writes a v3 field, or a whole +**JSON Schema.** `zarr_metadata.json_schema` writes a v3 declaration, or a whole `zarr.json`, as a JSON Schema. """ from zarr_metadata.v3._common import ( - ZarrV3ChunkGridField, - ZarrV3ChunkKeyEncodingField, - ZarrV3CodecField, - ZarrV3DataTypeField, - ZarrV3StaticCodecField, - ZarrV3StorageTransformerField, + ZarrV3ChunkGridDeclaration, + ZarrV3ChunkKeyEncodingDeclaration, + ZarrV3CodecDeclaration, + ZarrV3DataTypeDeclaration, + ZarrV3StaticCodecDeclaration, + ZarrV3StorageTransformerDeclaration, ) from zarr_metadata.v3._context import ZarrV3Context, ZarrV3ContextSlots, ZarrV3Kind from zarr_metadata.v3._core import CORE, CORE_AND_EXTENSIONS @@ -138,14 +138,14 @@ "Stage", "StorageClass", "StorageTransformerDefinition", - "ZarrV3ChunkGridField", - "ZarrV3ChunkKeyEncodingField", - "ZarrV3CodecField", + "ZarrV3ChunkGridDeclaration", + "ZarrV3ChunkKeyEncodingDeclaration", + "ZarrV3CodecDeclaration", "ZarrV3Context", "ZarrV3ContextSlots", - "ZarrV3DataTypeField", + "ZarrV3DataTypeDeclaration", "ZarrV3Kind", - "ZarrV3StaticCodecField", - "ZarrV3StorageTransformerField", + "ZarrV3StaticCodecDeclaration", + "ZarrV3StorageTransformerDeclaration", "storage_of", ] diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index ab5e7c046b..d09cb2599e 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -23,14 +23,14 @@ validate_json, ) from zarr_metadata.definition import ( - AcceptedField, - UnclaimedField, + AcceptedDeclaration, + UnclaimedDeclaration, canonical_of, configuration_of, resolve, with_problems, ) -from zarr_metadata.definition._definition import fields_of +from zarr_metadata.definition._definition import declarations_of from zarr_metadata.model import ( UNSET, ZarrV2ArrayMetadata, @@ -233,7 +233,7 @@ def test_v3_a_data_type_with_nothing_to_configure_is_written_by_its_bare_name( "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], } model = ZarrV3ArrayMetadata.from_json(document) - # The document is written back as it was written. The field on its own + # The document is written back as it was written. The declaration on its own # is written by its bare name, which every reader accepts. assert model.to_json()["data_type"] == data_type assert model.data_type.to_json() == "int32" @@ -297,7 +297,7 @@ def test_v3_no_storage_transformers_omitted() -> None: def _written_extra_fields(model: ZarrV3ArrayMetadata) -> dict[str, object]: - """Each extra field of `model`, by key, as the document writes the member's value.""" + """Each extra field of `model`, by key, as the document writes the field's value.""" return {key: cast("Any", extra.to_json())[key] for key, extra in model.extra_fields.items()} @@ -323,15 +323,15 @@ def test_v3_to_key_value_is_valid_json_under_zarr_json() -> None: # --- V3 standard-key sets -------------------------------------------------- -def test_spec_members_are_the_members_the_spec_defines() -> None: - """`ArrayExtraFieldDefinition.spec_members` holds every member of `ZarrV3ArrayMetadataJSON`, required or not, and nothing else: any other member of an array document is an extra field.""" - members = ArrayExtraFieldDefinition.spec_members +def test_spec_fields_are_the_fields_the_spec_defines() -> None: + """`ArrayExtraFieldDefinition.spec_fields` holds every field of `ZarrV3ArrayMetadataJSON`, required or not, and nothing else: any other field of an array document is an extra field.""" + fields = ArrayExtraFieldDefinition.spec_fields assert ( - members + fields == ZarrV3ArrayMetadataJSON.__required_keys__ | ZarrV3ArrayMetadataJSON.__optional_keys__ ) - assert {"zarr_format", "node_type", "shape", "codecs", "dimension_names"} <= members - assert "my_ext" not in members + assert {"zarr_format", "node_type", "shape", "codecs", "dimension_names"} <= fields + assert "my_ext" not in fields # --- create_default -------------------------------------------------------- @@ -360,7 +360,7 @@ def test_v3_create_default_applies_overrides() -> None: @pytest.mark.parametrize( - ("model", "members"), + ("model", "fields"), [ (ZarrV3ArrayMetadata, {"attributes": {"a": 1}, "fill_value": 3}), (ZarrV3ArrayMetadata, {"dimension_names": UNSET, "attributes": UNSET}), @@ -393,12 +393,12 @@ def test_v3_create_default_applies_overrides() -> None: "v2-group-unset", ], ) -def test_create_default_takes_the_members_update_takes( - model: object, members: dict[str, Any] +def test_create_default_takes_the_fields_update_takes( + model: object, fields: dict[str, Any] ) -> None: - """Every model's `create_default` takes the same keyword members as its `update`, including `UNSET` to drop a member and, for a v3 group, a `ZarrV3ConsolidatedMetadata` as `consolidated_metadata`. Applying them at creation gives the same model as applying them afterwards.""" + """Every model's `create_default` takes the same keyword fields as its `update`, including `UNSET` to drop a field and, for a v3 group, a `ZarrV3ConsolidatedMetadata` as `consolidated_metadata`. Applying them at creation gives the same model as applying them afterwards.""" kind = cast("Any", model) - assert kind.create_default(**members) == kind.create_default().update(**members) + assert kind.create_default(**fields) == kind.create_default().update(**fields) def test_v2_create_default_is_valid_empty_array() -> None: @@ -464,14 +464,14 @@ def test_update_no_args_returns_equal_model( # V3-only update tests — kept direct (extra_fields is v3-specific) -def test_update_can_add_an_extension_member() -> None: - """update can add a member the spec does not define, which the model holds in extra_fields.""" +def test_update_can_add_an_extension_field() -> None: + """update can add a field the spec does not define, which the model holds in extra_fields.""" base = ZarrV3ArrayMetadata.create_default() updated = base.update(extra_fields={"my_ext": {"must_understand": False}}) assert _written_extra_fields(updated) == {"my_ext": {"must_understand": False}} -def test_update_reads_every_member_in_the_models_own_scope() -> None: +def test_update_reads_every_field_in_the_models_own_scope() -> None: """`update` reads the new document in the scope the model was read in, so a name that scope leaves unclaimed stays unclaimed. Use `with_context` to read it in another scope. `zstd` is an extension, so `CORE` leaves it unclaimed and unchecked. @@ -479,34 +479,34 @@ def test_update_reads_every_member_in_the_models_own_scope() -> None: little: ZarrV3NamedConfigJSON = {"name": "bytes", "configuration": {"endian": "little"}} zstd: ZarrV3NamedConfigJSON = {"name": "zstd", "configuration": {"level": 3, "checksum": False}} base = ZarrV3ArrayMetadata.create_default(context=CORE, codecs=(little, zstd)) - assert isinstance(base.codecs[1], UnclaimedField) + assert isinstance(base.codecs[1], UnclaimedDeclaration) kept = base.update(attributes={"k": 1}) assert kept.codecs[1] == base.codecs[1] assert kept.context == CORE given = base.with_context(CORE_AND_EXTENSIONS).update(codecs=(little, zstd)) - assert isinstance(given.codecs[1], AcceptedField) + assert isinstance(given.codecs[1], AcceptedDeclaration) -def test_update_leaves_out_a_member_given_as_unset() -> None: - """`update` removes each member that a document may omit when it is given as `UNSET`: `dimension_names`, `attributes`, and a member the spec does not define.""" +def test_update_leaves_out_a_field_given_as_unset() -> None: + """`update` removes each field that a document may omit when it is given as `UNSET`: `dimension_names`, `attributes`, and a field the spec does not define.""" base = ZarrV3ArrayMetadata.create_default( shape=(2,), dimension_names=("x",), attributes={"a": 1}, extra_fields={"my_ext": {"must_understand": False}}, ) - for updated, member in ( + for updated, field in ( (base.update(dimension_names=UNSET), "dimension_names"), (base.update(attributes=UNSET), "attributes"), (base.update(extra_fields={"my_ext": UNSET}), "my_ext"), ): written = updated.to_json() - assert member not in written - assert {key: value for key, value in base.to_json().items() if key != member} == written + assert field not in written + assert {key: value for key, value in base.to_json().items() if key != field} == written @pytest.mark.parametrize( - "members", + "fields", [ {"data_type": {"name": "uint8"}}, {"codecs": ("bytes",)}, @@ -536,16 +536,16 @@ def test_update_leaves_out_a_member_given_as_unset() -> None: "codec-verbose", "empty-configuration", "unclaimed", - "a-field-a-codec-holds-bare", + "a-declaration-a-codec-holds-bare", ], ) def test_a_model_is_what_its_document_says_not_how_it_is_spelled( - members: ZarrV3ArrayMetadataUpdate, + fields: ZarrV3ArrayMetadataUpdate, ) -> None: """A model read back from its own document equals itself, and it equals the model read from any other spelling of the same document.""" - model = ZarrV3ArrayMetadata.create_default(**members) + model = ZarrV3ArrayMetadata.create_default(**fields) assert ZarrV3ArrayMetadata.from_json(model.to_json()) == model - written = {**model.to_json(), **members} + written = {**model.to_json(), **fields} assert ZarrV3ArrayMetadata.from_json(written) == model @@ -574,7 +574,7 @@ def test_a_model_is_what_its_document_says_not_how_it_is_spelled( def test_two_models_are_one_array_when_their_fill_values_are_one_value( data_type: object, left: object, right: object, same: bool ) -> None: - """Two models are equal when their fill values are the same value, however each is spelled, just as two fields are equal when they read the same.""" + """Two models are equal when their fill values are the same value, however each is spelled, just as two declarations are equal when they read the same.""" codecs = [{"name": "vlen-bytes"} if data_type == "bytes" else _LITTLE] document = {**ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json(), "codecs": codecs} models = [ @@ -600,7 +600,7 @@ def test_two_models_are_one_array_when_their_fill_values_are_one_value( @pytest.mark.parametrize( - ("member", "left", "right"), + ("field", "left", "right"), [ ( "codecs", @@ -620,12 +620,12 @@ def test_two_models_are_one_array_when_their_fill_values_are_one_value( ], ids=["blosc-typesize-noshuffle", "zstd-checksum-false", "default-separator"], ) -def test_two_models_are_one_array_when_their_fields_read_the_same( - member: str, left: object, right: object +def test_two_models_are_one_array_when_their_declarations_read_the_same( + field: str, left: object, right: object ) -> None: - """The spec's equivalences, which each definition's `canonical` normalizes, hold for a model's `==`, even though `to_json` writes each field as given.""" + """The spec's equivalences, which each definition's `canonical` normalizes, hold for a model's `==`, even though `to_json` writes each declaration as given.""" document = ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json() - models = [ZarrV3ArrayMetadata.from_json({**document, member: value}) for value in (left, right)] + models = [ZarrV3ArrayMetadata.from_json({**document, field: value}) for value in (left, right)] assert models[0] == models[1] assert hash(models[0]) == hash(models[1]) assert models[0].to_json() != models[1].to_json() @@ -697,8 +697,8 @@ def test_a_model_pickles_and_copies_with_its_definitions() -> None: assert configuration_of(again.codecs[1], GZIP_CODEC) == {"level": 1} -def test_update_replaces_a_member_rather_than_merging_into_it() -> None: - """update replaces each member it is given whole, and keeps the others.""" +def test_update_replaces_a_field_rather_than_merging_into_it() -> None: + """update replaces each field it is given whole, and keeps the others.""" base = ZarrV3ArrayMetadata.create_default( extra_fields={"a": {"must_understand": False, "x": 1}, "b": {"must_understand": False}} ) @@ -803,12 +803,12 @@ def test_v3_from_json_defaults_for_omitted_optionals() -> None: def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: - """V3 from_json routes unknown top-level keys into extra_fields. No definition in the default scope claims `my_ext`, so it is an `UnclaimedField`, written as the document wrote it.""" + """V3 from_json routes unknown top-level keys into extra_fields. No definition in the default scope claims `my_ext`, so it is an `UnclaimedDeclaration`, written as the document wrote it.""" doc = ZarrV3ArrayMetadata.create_default( extra_fields={"my_ext": {"must_understand": False}} ).to_json() model = ZarrV3ArrayMetadata.from_json(doc) - assert isinstance(model.extra_fields["my_ext"], UnclaimedField) + assert isinstance(model.extra_fields["my_ext"], UnclaimedDeclaration) assert _written_extra_fields(model) == {"my_ext": {"must_understand": False}} @@ -995,13 +995,13 @@ def test_v3_parser_accepts_bare_string_data_type() -> None: @pytest.mark.parametrize("name", ["bytes", "acme.codec", "urn:example:codec"]) -def test_metadata_field_accepts_a_name_as_the_spec_names_one(name: str) -> None: +def test_declaration_accepts_a_name_as_the_spec_names_one(name: str) -> None: """The envelope check accepts a name in a form the spec allows for an extension name, whether or not a definition in scope claims it.""" assert resolve({"name": name}, CodecDefinition, CORE_AND_EXTENSIONS)[1] == () @pytest.mark.parametrize("value", [0, 1, "false", None]) -def test_metadata_field_must_understand_must_be_boolean(value: object) -> None: +def test_declaration_must_understand_must_be_boolean(value: object) -> None: """must_understand is a JSON boolean, not a truthy scalar.""" _, problems = resolve( {"name": "acme.x", "must_understand": value}, CodecDefinition, CORE_AND_EXTENSIONS @@ -1011,7 +1011,7 @@ def test_metadata_field_must_understand_must_be_boolean(value: object) -> None: ] -def test_metadata_field_rejects_unknown_envelope_member() -> None: +def test_declaration_rejects_unknown_envelope_field() -> None: """Unknown envelope keys cannot be silently discarded during normalization.""" _, problems = resolve({"name": "acme.x", "typo": 1}, CodecDefinition, CORE_AND_EXTENSIONS) assert [(problem.loc, problem.kind) for problem in problems] == [(("typo",), "unknown_key")] @@ -1116,8 +1116,8 @@ def test_v2_from_key_value_rejects_zarray_attributes() -> None: ] -def test_v2_from_key_value_keeps_zarray_extra_members() -> None: - """Other raw `.zarray` members "SHOULD be ignored by implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92). They are not checked, and they are kept as written.""" +def test_v2_from_key_value_keeps_zarray_extra_fields() -> None: + """Other raw `.zarray` fields "SHOULD be ignored by implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92). They are not checked, and they are kept as written.""" doc: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) doc.pop("attributes", None) doc["vendor_extension"] = {} @@ -1237,13 +1237,13 @@ def test_json_type_guard_rejects_abstract_sequence() -> None: assert parse_json(range(3)) == (0, 1, 2) -def test_a_field_read_from_abstract_containers_writes_canonical_containers() -> None: - """`resolve` reads a field given as abstract containers, and the field writes canonical containers at every nesting level.""" +def test_a_declaration_read_from_abstract_containers_writes_canonical_containers() -> None: + """`resolve` reads a declaration given as abstract containers, and the declaration writes canonical containers at every nesting level.""" value = UserDict({"name": "example", "configuration": UserDict({"values": range(2)})}) - field, problems = resolve(value, CodecDefinition, CORE_AND_EXTENSIONS) - assert isinstance(field, UnclaimedField) - written = field.to_json() + declaration, problems = resolve(value, CodecDefinition, CORE_AND_EXTENSIONS) + assert isinstance(declaration, UnclaimedDeclaration) + written = declaration.to_json() assert problems == () assert isinstance(written, dict) @@ -1258,7 +1258,7 @@ def test_validate_json_reports_json_in_message() -> None: assert "JSON" in problems[0].message -METADATA_FIELD_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ +DECLARATION_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ Expect("bytes", frozenset(), id="bare-string"), Expect({"name": "acme.x", "configuration": {"a": 1}}, frozenset(), id="named-config"), Expect({"name": "bytes"}, frozenset(), id="name-only"), @@ -1278,11 +1278,11 @@ def test_validate_json_reports_json_in_message() -> None: ] -@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) -def test_resolve_reports_the_problems_of_a_metadata_field( +@pytest.mark.parametrize("case", DECLARATION_VALIDATE_CASES, ids=lambda c: c.id) +def test_resolve_reports_the_problems_of_a_declaration( case: Expect[object, frozenset[tuple[str | int, ...]]], ) -> None: - """`resolve` reports the problems of a metadata-field value, read as a codec.""" + """`resolve` reports the problems of a declaration value, read as a codec.""" _, problems = resolve(case.input, CodecDefinition, CORE_AND_EXTENSIONS) assert (problems == ()) is (case.output == frozenset()) assert {p.loc for p in problems} >= case.output @@ -1484,13 +1484,13 @@ def test_metadata_validation_error_holds_problems() -> None: problems = [ ValidationProblem(loc=("shape",), message="missing required key", kind="missing_key"), ValidationProblem( - loc=("data_type",), message="expected a metadata field", kind="invalid_type" + loc=("data_type",), message="expected a declaration", kind="invalid_type" ), ] err = MetadataValidationError(problems) assert err.problems == tuple(problems) assert "shape: missing required key" in str(err) - assert "data_type: expected a metadata field" in str(err) + assert "data_type: expected a declaration" in str(err) def test_the_error_pickles_and_copies_as_its_problems() -> None: @@ -1733,7 +1733,7 @@ def test_array_zarr_format_rejects_float( assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_type")] -def test_array_v2_keeps_an_unknown_document_member() -> None: +def test_array_v2_keeps_an_unknown_document_field() -> None: """Other .zarray keys "SHOULD NOT be present ... and SHOULD be ignored": they are tolerated, not checked, and kept as written in `extra_fields`. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92 @@ -1747,7 +1747,7 @@ def test_array_v2_keeps_an_unknown_document_member() -> None: @pytest.mark.parametrize( - ("member", "kind"), + ("field", "kind"), [ (object(), "invalid_type"), ({1, 2}, "invalid_type"), @@ -1755,17 +1755,17 @@ def test_array_v2_keeps_an_unknown_document_member() -> None: ], ids=["object", "set", "bytes"], ) -def test_error_array_v2_unknown_member_that_is_not_json(member: object, kind: str) -> None: - """Ignored does not mean unchecked: an unknown member must be a JSON value, as in v3.""" - doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"unexpected": member} +def test_error_array_v2_unknown_field_that_is_not_json(field: object, kind: str) -> None: + """Ignored does not mean unchecked: an unknown field must be a JSON value, as in v3.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"unexpected": field} assert [(p.loc, p.kind) for p in read_array_metadata_v2(doc).problems] == [ (("unexpected",), kind) ] -def test_an_unknown_member_may_hold_a_non_finite_number_in_either_format() -> None: - """A member the spec does not define is not read, so like attributes it may hold `NaN` and the infinities, in a v2 `.zarray` as in a v3 document, read alone or in consolidated metadata.""" +def test_an_unknown_field_may_hold_a_non_finite_number_in_either_format() -> None: + """A field the spec does not define is not read, so like attributes it may hold `NaN` and the infinities, in a v2 `.zarray` as in a v3 document, read alone or in consolidated metadata.""" v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"unexpected": math.nan} v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"unexpected": math.nan} zmetadata = {"zarr_consolidated_format": 1, "metadata": {"a/.zarray": v2}} @@ -1808,7 +1808,7 @@ def test_from_key_value_rejects_non_standard_json_constant() -> None: @pytest.mark.parametrize( - ("members", "problems"), + ("fields", "problems"), [ ({"fill_value": math.nan}, [(("fill_value",), "invalid_value")]), # The default fill value, 0, is not a boolean. A data type requires @@ -1836,16 +1836,16 @@ def test_from_key_value_rejects_non_standard_json_constant() -> None: ], ) def test_error_create_default_refuses_a_document_with_a_problem( - members: ZarrV3ArrayMetadataUpdate, problems: list[tuple[tuple[str | int, ...], str]] + fields: ZarrV3ArrayMetadataUpdate, problems: list[tuple[tuple[str | int, ...], str]] ) -> None: """A model is built by a read, so a document that the read refuses produces no model, and nothing is written.""" with pytest.raises(MetadataValidationError) as raised: - ZarrV3ArrayMetadata.create_default(**members) + ZarrV3ArrayMetadata.create_default(**fields) assert [(p.loc, p.kind) for p in raised.value.problems] == problems @pytest.mark.parametrize( - ("members", "problems"), + ("fields", "problems"), [ ({"fill_value": 300}, [(("fill_value",), "invalid_value")]), # A shape requires a grid that fits it. @@ -1854,15 +1854,15 @@ def test_error_create_default_refuses_a_document_with_a_problem( ids=["fill-value-out-of-range", "shape-without-its-grid"], ) def test_error_update_refuses_a_document_with_a_problem( - members: ZarrV3ArrayMetadataUpdate, problems: list[tuple[tuple[str | int, ...], str]] + fields: ZarrV3ArrayMetadataUpdate, problems: list[tuple[tuple[str | int, ...], str]] ) -> None: base = ZarrV3ArrayMetadata.create_default(shape=(4,)) with pytest.raises(MetadataValidationError) as raised: - base.update(**members) + base.update(**fields) assert [(p.loc, p.kind) for p in raised.value.problems] == problems -def test_error_update_refuses_to_leave_out_a_member_a_document_holds() -> None: +def test_error_update_refuses_to_leave_out_a_field_a_document_holds() -> None: base = ZarrV3ArrayMetadata.create_default(shape=(4,)) with pytest.raises(MetadataValidationError) as raised: base.update(shape=UNSET) # pyright: ignore[reportArgumentType] @@ -1899,7 +1899,7 @@ def test_a_codec_that_is_not_json_is_placed_by_the_definition_that_claims_its_na def test_a_shard_nested_as_deep_as_a_reader_walks_is_read_and_written() -> None: - """Every walker of fields uses more than one stack frame per shard, so the deepest nesting the cap allows is where the interpreter's recursion limit would show. One shard deeper is reported as the depth problem.""" + """Every walker of declarations uses more than one stack frame per shard, so the deepest nesting the cap allows is where the interpreter's recursion limit would show. One shard deeper is reported as the depth problem.""" little = {"name": "bytes", "configuration": {"endian": "little"}} def nested(shards: int) -> list[object]: @@ -1933,7 +1933,7 @@ def nested(shards: int) -> list[object]: (shard,) = model.codecs assert json_text(canonical_of(shard, ())) == json_text(cast("JSONValue", codecs[0])) # Each level has a shard and its index codec, plus the innermost codec. - assert len(list(with_problems(fields_of(shard), ()))) == 2 * deepest + 1 + assert len(list(with_problems(declarations_of(shard), ()))) == 2 * deepest + 1 problems = read_array_metadata_v3({**document, "codecs": nested(deepest + 1)}).problems assert {(problem.kind, len(problem.loc), problem.message) for problem in problems} == { ("invalid_value", JSON_DEPTH, f"nested deeper than the {JSON_DEPTH} levels a reader walks") @@ -1957,15 +1957,15 @@ def test_a_model_is_written_as_deep_as_it_is_read() -> None: assert copy.deepcopy(model) == model -def test_a_field_s_configuration_member_is_counted_from_the_field_s_root() -> None: - # `resolve` reads a field on its own, from the field's root. A member +def test_a_declaration_s_configuration_field_is_counted_from_the_declaration_s_root() -> None: + # `resolve` reads a declaration on its own, from the declaration's root. A field # of its configuration is two levels down, and the cap counts from the - # root, not from the member. + # root, not from the field. nested: dict[str, object] = {} for _ in range(JSON_DEPTH - 2): nested = {"x": nested} - field = {"name": "acme.x", "configuration": {"y": nested}} - _, problems = resolve(field, CodecDefinition, CORE_AND_EXTENSIONS) + declaration = {"name": "acme.x", "configuration": {"y": nested}} + _, problems = resolve(declaration, CodecDefinition, CORE_AND_EXTENSIONS) assert [(len(p.loc), p.kind) for p in problems] == [(JSON_DEPTH, "invalid_value")] assert problems[0].loc[:2] == ("configuration", "y") shallower = {"name": "acme.x", "configuration": {"y": nested["x"]}} @@ -2370,7 +2370,7 @@ def test_dimension_names_absent_and_all_null_are_distinct() -> None: @pytest.mark.parametrize( - ("read", "document", "member", "value", "kind"), + ("read", "document", "field", "value", "kind"), [ (read_array_metadata_v3, ZarrV3ArrayMetadata, "zarr_format", "3", "invalid_type"), (read_array_metadata_v3, ZarrV3ArrayMetadata, "zarr_format", 2, "invalid_value"), @@ -2387,17 +2387,17 @@ def test_dimension_names_absent_and_all_null_are_distinct() -> None: ), ], ) -def test_error_a_member_outside_the_values_it_takes( +def test_error_a_field_outside_the_values_it_takes( read: Callable[[object], ZarrV2ArrayMetadataReading | ZarrV3ArrayMetadataReading], document: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], - member: str, + field: str, value: object, kind: str, ) -> None: # A value is the wrong type when none of the allowed values has its JSON # type (a string where a number belongs). Otherwise it is the wrong value. - written = {**document.create_default().to_json(), member: value} - assert [(p.loc, p.kind) for p in read(written).problems] == [((member,), kind)] + written = {**document.create_default().to_json(), field: value} + assert [(p.loc, p.kind) for p in read(written).problems] == [((field,), kind)] def test_error_a_document_nested_deeper_than_a_reader_walks_is_a_problem() -> None: @@ -2419,12 +2419,12 @@ def test_error_a_document_nested_deeper_than_a_reader_walks_is_a_problem() -> No ZarrV3ArrayMetadata.from_json(document) -def test_error_a_field_object_in_a_document_is_not_json() -> None: - # An `AcceptedField` built by hand, with a configuration its definition +def test_error_a_declaration_object_in_a_document_is_not_json() -> None: + # An `AcceptedDeclaration` built by hand, with a configuration its definition # refuses, and inserted into a document is refused as a non-JSON value. - # So nothing built by hand passes as read. A model holds its own fields + # So nothing built by hand passes as read. A model holds its own declarations # as read. - smuggled = AcceptedField( + smuggled = AcceptedDeclaration( json="gzip", name="gzip", definition=GZIP_CODEC, configuration={"level": 99, "window": 1} ) document = { @@ -2436,8 +2436,8 @@ def test_error_a_field_object_in_a_document_is_not_json() -> None: ] with pytest.raises(MetadataValidationError): ZarrV3ArrayMetadata.from_json(document) - # `update` refuses a field among the members it is given with a - # `TypeError`, as `test_error_update_takes_fields_as_json` shows. + # `update` refuses a declaration among the fields it is given with a + # `TypeError`, as `test_error_update_takes_declarations_as_json` shows. model = ZarrV3ArrayMetadata.create_default(shape=(2,)) with pytest.raises(TypeError, match="to_json"): model.update(codecs=cast("Any", (model.codecs[0].to_json(), smuggled))) diff --git a/packages/zarr-metadata/tests/model/test_consolidating.py b/packages/zarr-metadata/tests/model/test_consolidating.py index 850888d223..c4d4e5089f 100644 --- a/packages/zarr-metadata/tests/model/test_consolidating.py +++ b/packages/zarr-metadata/tests/model/test_consolidating.py @@ -82,7 +82,7 @@ def test_a_model_is_accepted_as_a_consolidated_entry_when_it_refines_into_the_sc beside: dict[str, Any], gained: bool, ) -> None: - """A node model can be given wherever consolidated metadata lists a document. The model is accepted in two cases. If the scope of the consolidated metadata reads every claim of the model identically, the model's reading is kept and not read again. If that scope claims something that the model's scope left unclaimed, the model's document is read again in that scope. In both cases the consolidated metadata holds the model in its own scope, equals the one built from the documents, and a group it is given to writes plain JSON. A group model's fields are its extra fields: read in a scope that claims nothing, its `consolidated_metadata` is unclaimed, and the scope of the consolidated metadata gains it. A listed group's own listing is listed flat next to it, as the convention asks.""" + """A node model can be given wherever consolidated metadata lists a document. The model is accepted in two cases. If the scope of the consolidated metadata reads every claim of the model identically, the model's reading is kept and not read again. If that scope claims something that the model's scope left unclaimed, the model's document is read again in that scope. In both cases the consolidated metadata holds the model in its own scope, equals the one built from the documents, and a group it is given to writes plain JSON. A group model's declarations are its extra fields: read in a scope that claims nothing, its `consolidated_metadata` is unclaimed, and the scope of the consolidated metadata gains it. A listed group's own listing is listed flat next to it, as the convention asks.""" held = _consolidated(context, a=child, **beside) node = held.metadata["a"] assert node.context is held.context @@ -156,7 +156,7 @@ def test_error_a_gain_that_surfaces_a_problem_is_reported_at_the_problem() -> No def test_the_reader_sees_models_as_the_constructor_does() -> None: - """Given a member that lists models, `read_consolidated_metadata_v3` reads them the same way the constructor does, and holds the adopted model.""" + """Given a field that lists models, `read_consolidated_metadata_v3` reads them the same way the constructor does, and holds the adopted model.""" reading = read_consolidated_metadata_v3( {**INLINE, "metadata": {"a": ZarrV3ArrayMetadata(ARRAY)}} ) @@ -166,15 +166,15 @@ def test_the_reader_sees_models_as_the_constructor_does() -> None: def test_error_the_reader_reports_a_conflicting_model_as_the_constructor_does() -> None: - """Given a member that lists a model which conflicts with the scope, `read_consolidated_metadata_v3` reports the conflict at the entry's path, as the constructor refuses it.""" - member = {**INLINE, "metadata": {"a": ZarrV3ArrayMetadata(ARRAY, context=PRIVATE)}} - reading = read_consolidated_metadata_v3(member) + """Given a field that lists a model which conflicts with the scope, `read_consolidated_metadata_v3` reports the conflict at the entry's path, as the constructor refuses it.""" + field = {**INLINE, "metadata": {"a": ZarrV3ArrayMetadata(ARRAY, context=PRIVATE)}} + reading = read_consolidated_metadata_v3(field) assert [problem.loc for problem in reading.problems] == [("metadata", "a", "codecs", 0)] assert reading.metadata is None def test_a_group_writes_the_consolidated_metadata_it_is_given_in_its_own_scope() -> None: - """A `ZarrV3ConsolidatedMetadata` given to `update` is written as its document. It does not bring its own scope into the group: the group's `context` stays its own, and reading the written member in that scope gives the models in it.""" + """A `ZarrV3ConsolidatedMetadata` given to `update` is written as its document. It does not bring its own scope into the group: the group's `context` stays its own, and reading the written field in that scope gives the models in it.""" held = _consolidated(CORE_AND_EXTENSIONS, a=ZarrV3ArrayMetadata(ARRAY)) group = ZarrV3GroupMetadata.create_default(context=CORE, attributes={"t": 1}) updated = group.update(consolidated_metadata=held) @@ -224,7 +224,7 @@ def test_error_a_group_refuses_consolidated_metadata_that_would_lose_a_meaning() def test_a_listed_group_s_listing_is_read_only_where_the_scope_claims_it( context: ZarrV3Context, found: list[tuple[str | int, ...]] ) -> None: - """A listed group's own `consolidated_metadata` is a listing only in a scope that claims the member. In a scope that leaves it unclaimed, the listed group keeps it as written, so the reader does not read it either, whatever it holds.""" + """A listed group's own `consolidated_metadata` is a listing only in a scope that claims the field. In a scope that leaves it unclaimed, the listed group keeps it as written, so the reader does not read it either, whatever it holds.""" group = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": {"garbage": 1}} reading = read_consolidated_metadata_v3({**INLINE, "metadata": {"g": group}}, context=context) assert [problem.loc for problem in reading.problems if problem.kind == "unknown_key"] == found @@ -253,7 +253,7 @@ def test_error_a_group_s_listing_with_a_problem_is_located_in_the_group() -> Non @pytest.mark.parametrize("listing", [5, None], ids=["a-number", "null"]) def test_error_a_group_s_listing_that_is_not_an_object(listing: object) -> None: - """In a scope that leaves `consolidated_metadata` unclaimed, a group holding one that is not an object, a number or null, opens. The group still writes a listing, so `consolidated()` refuses it with an `invalid_type` problem located at the member, rather than answering that the group lists none.""" + """In a scope that leaves `consolidated_metadata` unclaimed, a group holding one that is not an object, a number or null, opens. The group still writes a listing, so `consolidated()` refuses it with an `invalid_type` problem located at the field, rather than answering that the group lists none.""" group = ZarrV3GroupMetadata( {"zarr_format": 3, "node_type": "group", "consolidated_metadata": listing}, context=ZarrV3Context(), @@ -291,12 +291,12 @@ def test_error_a_model_listed_too_deep_is_refused_where_it_sits() -> None: for _ in range(JSON_DEPTH - 3): deep = [deep] child = ZarrV3ArrayMetadata({**ARRAY, "attributes": {"x": deep}}) - member = {**INLINE, "metadata": {"a": child}} - reading = read_consolidated_metadata_v3(member) + field = {**INLINE, "metadata": {"a": child}} + reading = read_consolidated_metadata_v3(field) assert [problem.loc[:3] for problem in reading.problems] == [("metadata", "a", "attributes")] assert reading.metadata is None with pytest.raises(MetadataValidationError): - ZarrV3ConsolidatedMetadata(member) + ZarrV3ConsolidatedMetadata(field) assert [ problem.loc[:4] for problem in read_group_metadata_v3(_group_document(a=child.to_json())).problems @@ -343,11 +343,11 @@ def test_error_a_conflict_between_definitions_alike_says_they_differ() -> None: def test_the_input_types_are_types_a_signature_can_hold() -> None: """`ZarrV3NodeMetadataInput` and `ZarrV3ConsolidatedMetadataInput` resolve to types, not strings, when a caller's `get_type_hints` reads them as annotations at run time.""" - def take(entry: ZarrV3NodeMetadataInput, member: ZarrV3ConsolidatedMetadataInput) -> None: + def take(entry: ZarrV3NodeMetadataInput, field: ZarrV3ConsolidatedMetadataInput) -> None: pass hints = get_type_hints(take) - assert {"entry", "member"} <= set(hints) + assert {"entry", "field"} <= set(hints) assert not isinstance(hints["entry"], str) diff --git a/packages/zarr-metadata/tests/model/test_construction.py b/packages/zarr-metadata/tests/model/test_construction.py index e3c15a6d2b..4cb7967547 100644 --- a/packages/zarr-metadata/tests/model/test_construction.py +++ b/packages/zarr-metadata/tests/model/test_construction.py @@ -17,7 +17,7 @@ import pytest -from zarr_metadata.definition import RefusedField, resolve +from zarr_metadata.definition import RefusedDeclaration, resolve from zarr_metadata.model import ( UNSET, ZarrV2ArrayMetadata, @@ -72,7 +72,7 @@ def _rebuilt(model: object, changes: dict[str, object]) -> Any: # noqa: ANN401 - the tests give the model as object - """Rebuild the v2 model `model` from its document, with `changes` replacing members, in the model's own scope. This is what `update` does, and it also works for the consolidated model.""" + """Rebuild the v2 model `model` from its document, with `changes` replacing fields, in the model's own scope. This is what `update` does, and it also works for the consolidated model.""" held = cast("Any", model) return type(held)({**held.to_json(), **changes}, context=held.context) @@ -115,7 +115,7 @@ def test_a_v2_model_is_built_as_a_read_builds_it(model: object) -> None: ], ids=["array-lists", "array-mapping", "group-mapping"], ) -def test_a_v3_model_holds_its_members_as_a_read_refines_them( +def test_a_v3_model_holds_its_fields_as_a_read_refines_them( model: object, changes: dict[str, object] ) -> None: """A model holds arrays as tuples and objects as dicts, as a read holds them. So a model updated with other container types equals the model a read builds, and it round-trips through its store.""" @@ -132,7 +132,7 @@ def test_a_v3_model_holds_its_members_as_a_read_refines_them( ], ids=["v2-sequences", "v2-consolidated-mapping"], ) -def test_a_v2_model_holds_its_members_as_a_read_refines_them( +def test_a_v2_model_holds_its_fields_as_a_read_refines_them( model: object, changes: dict[str, object] ) -> None: """A v2 model holds arrays as tuples and objects as dicts, as a read holds them. So a v2 model rebuilt with other container types equals the model a read builds.""" @@ -142,29 +142,29 @@ def test_a_v2_model_holds_its_members_as_a_read_refines_them( @pytest.mark.parametrize( - ("model", "member"), + ("model", "field"), [(ARRAY, "acme.x"), (GROUP, "attributes")], ids=["array-extra-field", "group-attributes"], ) def test_a_v3_model_shares_no_container_with_what_it_was_built_of( - model: object, member: str + model: object, field: str ) -> None: """A model holds copies of the containers passed to `update`. Changing those containers afterwards does not change the model.""" held: dict[str, object] = {"must_understand": False, "z": {"y": 1}} - changes = {"attributes": held} if member == "attributes" else {"extra_fields": {member: held}} + changes = {"attributes": held} if field == "attributes" else {"extra_fields": {field: held}} built = cast("Any", model).update(**changes) held["w"] = math.nan cast("dict[str, object]", held["z"])["y"] = math.nan expected = {"must_understand": False, "z": {"y": 1}} - if member == "attributes": + if field == "attributes": assert built.attributes == expected else: - assert built.extra_fields[member].to_json() == {member: expected} + assert built.extra_fields[field].to_json() == {field: expected} assert type(built).from_key_value(built.to_key_value()) == built @pytest.mark.parametrize( - ("model", "member"), + ("model", "field"), [ (V2_ARRAY, "attributes"), (V2_GROUP, "attributes"), @@ -173,14 +173,14 @@ def test_a_v3_model_shares_no_container_with_what_it_was_built_of( ids=["v2-array-attributes", "v2-group-attributes", "v2-consolidated"], ) def test_a_v2_model_shares_no_container_with_what_it_was_built_of( - model: object, member: str + model: object, field: str ) -> None: """A v2 model holds copies of the containers it is built from: changing them afterwards changes nothing in the document the model writes.""" held: dict[str, object] = {"acme.x": {"must_understand": False}} - built = _rebuilt(model, {member: held}) + built = _rebuilt(model, {field: held}) held["acme.y"] = math.nan cast("dict[str, object]", held["acme.x"])["z"] = math.nan - assert built.to_json()[member] == {"acme.x": {"must_understand": False}} + assert built.to_json()[field] == {"acme.x": {"must_understand": False}} assert type(built).from_key_value(built.to_key_value()) == built @@ -204,8 +204,8 @@ def counted( assert values == [3] changed = model.update(fill_value=4) assert values == [3, 4] - member: dict[str, Any] = {**_INLINE, "metadata": {"a": changed.to_json()}} - group = ZarrV3GroupMetadata.create_default(context=context, consolidated_metadata=member) + field: dict[str, Any] = {**_INLINE, "metadata": {"a": changed.to_json()}} + group = ZarrV3GroupMetadata.create_default(context=context, consolidated_metadata=field) assert values == [3, 4] group.to_key_value() assert values == [3, 4] @@ -261,10 +261,10 @@ def test_error_a_v3_model_updated_into_an_invalid_one_is_refused_at_the_change( [ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ZarrV2ArrayMetadata], ids=["v3-array", "v3-group", "v2-array"], ) -def test_a_member_named_extra_fields_is_set_and_removed_through_extra_fields( +def test_a_field_named_extra_fields_is_set_and_removed_through_extra_fields( kind: Any, # noqa: ANN401 - a model class ) -> None: - """A document may hold a member that is itself named `extra_fields`. The spec does not define it, so `update` and `create_default` take it in the `extra_fields` mapping, as any other such member.""" + """A document may hold a field that is itself named `extra_fields`. The spec does not define it, so `update` and `create_default` take it in the `extra_fields` mapping, as any other such field.""" added = kind.create_default(extra_fields={"extra_fields": {"must_understand": False}}) assert added.to_json()["extra_fields"] == {"must_understand": False} assert "extra_fields" not in added.update(extra_fields={"extra_fields": UNSET}).to_json() @@ -275,10 +275,10 @@ def test_a_member_named_extra_fields_is_set_and_removed_through_extra_fields( [ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ZarrV2ArrayMetadata], ids=["v3-array", "v3-group", "v2-array"], ) -def test_members_the_spec_does_not_define_are_set_and_removed_through_extra_fields( +def test_fields_the_spec_does_not_define_are_set_and_removed_through_extra_fields( kind: Any, # noqa: ANN401 - a model class ) -> None: - """`create_default` and `update` take the members the spec does not define in `extra_fields`. Each member given there is set, each given as `UNSET` is removed, and the other members are kept.""" + """`create_default` and `update` take the fields the spec does not define in `extra_fields`. Each field given there is set, each given as `UNSET` is removed, and the other fields are kept.""" added = kind.create_default( extra_fields={"acme.flag": 1, "acme.other": {"must_understand": False}} ) @@ -293,8 +293,8 @@ def test_members_the_spec_does_not_define_are_set_and_removed_through_extra_fiel [ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ZarrV2ArrayMetadata, ZarrV2GroupMetadata], ids=["v3-array", "v3-group", "v2-array", "v2-group"], ) -def test_error_a_member_update_does_not_take_is_refused(kind: Any) -> None: # noqa: ANN401 - a model class - """A misspelled member is a `TypeError` from `create_default` and from `update`. It is never written as a member the spec does not define, which a reader that must understand every member would refuse.""" +def test_error_a_field_update_does_not_take_is_refused(kind: Any) -> None: # noqa: ANN401 - a model class + """A misspelled field is a `TypeError` from `create_default` and from `update`. It is never written as a field the spec does not define, which a reader that must understand every field would refuse.""" with pytest.raises(TypeError, match="'attribute'"): kind.create_default(attribute={}) with pytest.raises(TypeError, match="'attribute'"): @@ -302,7 +302,7 @@ def test_error_a_member_update_does_not_take_is_refused(kind: Any) -> None: # n @pytest.mark.parametrize( - ("kind", "member"), + ("kind", "field"), [ (ZarrV3ArrayMetadata, "shape"), (ZarrV3GroupMetadata, "node_type"), @@ -310,43 +310,43 @@ def test_error_a_member_update_does_not_take_is_refused(kind: Any) -> None: # n ], ids=["v3-array", "v3-group", "v2-array"], ) -def test_error_extra_fields_refuses_a_member_the_spec_defines(kind: Any, member: str) -> None: # noqa: ANN401 - a model class - """`extra_fields` holds only the members the spec does not define. A member the spec defines is given by its own name, so giving it in `extra_fields` is a `TypeError`.""" - with pytest.raises(TypeError, match=repr(member)): - kind.create_default(extra_fields={member: 1}) +def test_error_extra_fields_refuses_a_field_the_spec_defines(kind: Any, field: str) -> None: # noqa: ANN401 - a model class + """`extra_fields` holds only the fields the spec does not define. A field the spec defines is given by its own name, so giving it in `extra_fields` is a `TypeError`.""" + with pytest.raises(TypeError, match=repr(field)): + kind.create_default(extra_fields={field: 1}) -def test_error_update_takes_fields_as_json() -> None: - """`update` takes each member as a document writes it. A field the model read, such as each of `model.codecs`, is refused with a `TypeError` that says to write it with `to_json()`, rather than reported as a value that is not JSON.""" +def test_error_update_takes_declarations_as_json() -> None: + """`update` takes each field as a document writes it. A declaration the model read, such as each of `model.codecs`, is refused with a `TypeError` that says to write it with `to_json()`, rather than reported as a value that is not JSON.""" with pytest.raises(TypeError, match=r"codecs: .*to_json\(\)"): ARRAY.update(codecs=ARRAY.codecs) # pyright: ignore[reportArgumentType] -def test_error_update_takes_no_refused_field() -> None: - """A field refused by its definition holds JSON its definition does not read, so it has no JSON for `update` to write: given in a member, it is refused with a `TypeError` that says to give the member as JSON.""" +def test_error_update_takes_no_refused_declaration() -> None: + """A declaration refused by its definition holds JSON its definition does not read, so it has no JSON for `update` to write: given in a field, it is refused with a `TypeError` that says to give the field as JSON.""" refused, _ = resolve({"name": "gzip", "configuration": {"level": 99}}, CodecDefinition, CORE) - assert isinstance(refused, RefusedField) - with pytest.raises(TypeError, match="a refused field has no JSON"): + assert isinstance(refused, RefusedDeclaration) + with pytest.raises(TypeError, match="a refused declaration has no JSON"): ARRAY.update(codecs=(ARRAY.to_json()["codecs"][0], refused)) # pyright: ignore[reportArgumentType] def test_error_extra_fields_takes_values_as_json() -> None: - """A field given as the value of a member in `extra_fields`, or an extra field given as a member's value, is refused with a `TypeError` that names `to_json()[key]`, which is the member's value. Writing a whole field there would nest the member inside itself and lose its waiver.""" + """A declaration given as the value of a field in `extra_fields`, or an extra field given as a field's value, is refused with a `TypeError` that names `to_json()[key]`, which is the field's value. Writing a whole declaration there would nest the field inside itself and lose its waiver.""" source = ZarrV3ArrayMetadata.create_default(extra_fields={"foo": {"must_understand": False}}) with pytest.raises(TypeError, match=r"to_json\(\)\['foo'\]"): ARRAY.update(extra_fields=dict(source.extra_fields)) # pyright: ignore[reportArgumentType] - member = GROUP.extra_fields["consolidated_metadata"] + field = GROUP.extra_fields["consolidated_metadata"] with pytest.raises(TypeError, match=r"to_json\(\)\['consolidated_metadata'\]"): - GROUP.update(consolidated_metadata=member) # pyright: ignore[reportArgumentType] - # The advice follows the field's kind, not the key it was given under. + GROUP.update(consolidated_metadata=field) # pyright: ignore[reportArgumentType] + # The advice follows the declaration's kind, not the key it was given under. with pytest.raises(TypeError, match=r"to_json\(\)\['foo'\]"): ARRAY.update(extra_fields={"bar": source.extra_fields["foo"]}) # pyright: ignore[reportArgumentType] with pytest.raises(TypeError, match=r"to_json\(\)$"): ARRAY.update(extra_fields={"bytes": ARRAY.codecs[0]}) # pyright: ignore[reportArgumentType] -def test_error_extra_fields_refuses_a_member_update_takes_by_name() -> None: - """A member that `update` takes by name, such as a group's `consolidated_metadata`, is refused in `extra_fields`, so one member is never given twice, and a listing given by name is never replaced unchecked.""" +def test_error_extra_fields_refuses_a_field_update_takes_by_name() -> None: + """A field that `update` takes by name, such as a group's `consolidated_metadata`, is refused in `extra_fields`, so one field is never given twice, and a listing given by name is never replaced unchecked.""" with pytest.raises(TypeError, match="give it by name"): GROUP.update(extra_fields={"consolidated_metadata": UNSET}) @@ -361,8 +361,8 @@ def test_error_extra_fields_that_is_not_a_mapping( kind: Any, # noqa: ANN401 - a model class extra_fields: object, ) -> None: - """`extra_fields` maps members to their values, so `update` refuses `UNSET` or a list of pairs given in its place with a `TypeError`, rather than removing every member or reading the pairs as members.""" - with pytest.raises(TypeError, match="extra_fields maps members to their values"): + """`extra_fields` maps fields to their values, so `update` refuses `UNSET` or a list of pairs given in its place with a `TypeError`, rather than removing every field or reading the pairs as fields.""" + with pytest.raises(TypeError, match="extra_fields maps fields to their values"): kind.create_default().update(extra_fields=extra_fields) @@ -372,19 +372,19 @@ def test_error_extra_fields_that_is_not_a_mapping( ids=["v3-array", "v3-group", "v2-array"], ) def test_error_extra_fields_with_a_key_that_is_not_a_string(kind: Any) -> None: # noqa: ANN401 - a model class - """A document's members are keyed by strings, so `update` refuses an `extra_fields` with any other key with a `TypeError` that names the key.""" - with pytest.raises(TypeError, match="extra_fields maps each member's key, a string"): + """A document's fields are keyed by strings, so `update` refuses an `extra_fields` with any other key with a `TypeError` that names the key.""" + with pytest.raises(TypeError, match="extra_fields maps each field's key, a string"): kind.create_default().update(extra_fields={1: 1}) def test_error_a_v2_group_takes_no_extra_fields() -> None: - """A v2 group's `.zgroup` holds only `zarr_format`, so its `update` takes no `extra_fields`, and refuses it as an unknown member.""" - with pytest.raises(TypeError, match="unknown member 'extra_fields'"): + """A v2 group's `.zgroup` holds only `zarr_format`, so its `update` takes no `extra_fields`, and refuses it as an unknown field.""" + with pytest.raises(TypeError, match="unknown field 'extra_fields'"): V2_GROUP.update(extra_fields={"acme": 1}) # pyright: ignore[reportCallIssue] def test_error_consolidated_metadata_of_documents_at_bad_paths_is_refused() -> None: - """The consolidated member, read on its own, refuses documents at paths that no node can have. It also reports a missing parent group, as the group's read does.""" + """The consolidated field, read on its own, refuses documents at paths that no node can have. It also reports a missing parent group, as the group's read does.""" documents = { "x": ARRAY.to_json(), "x/a": ARRAY.to_json(), @@ -440,7 +440,7 @@ def test_error_v2_create_default_refuses_chunks_its_default_shape_does_not_take( def test_error_consolidated_metadata_paths_are_strings() -> None: - """A `metadata` member with a key that is not a string is reported as a problem of the member, as the read reports it, and does not raise `TypeError`.""" + """A `metadata` field with a key that is not a string is reported as a problem of the field, as the read reports it, and does not raise `TypeError`.""" with pytest.raises(MetadataValidationError) as raised: ZarrV3ConsolidatedMetadata({**_INLINE, "metadata": {1: ARRAY.to_json()}}) assert [(found.loc, found.kind) for found in raised.value.problems] == [ diff --git a/packages/zarr-metadata/tests/model/test_extension_points.py b/packages/zarr-metadata/tests/model/test_extension_points.py index 063338e160..9a3a605cce 100644 --- a/packages/zarr-metadata/tests/model/test_extension_points.py +++ b/packages/zarr-metadata/tests/model/test_extension_points.py @@ -3,7 +3,7 @@ The model checks a document's structure itself. The configuration of a codec, data type, chunk grid or chunk key encoding is checked by its definition. The definition is found in a scope (`CORE_AND_EXTENSIONS` -unless a caller passes another), and problems are reported at the field's +unless a caller passes another), and problems are reported at the declaration's location in the document. A name that no definition in the scope claims is not checked. """ @@ -91,7 +91,7 @@ class LenientGzipConfiguration(TypedDict, closed=True): def test_a_document_reads_through_the_definitions_in_its_scope( document: dict[str, Any], context: ZarrV3Context ) -> None: - # Definitions in the scope check their fields. Fields with no definition + # Definitions in the scope check their declarations. Declarations with no definition # in the scope are left to the reader. The model reads in the same scope # as the reader, and its output reads back in that scope as the same # model. @@ -152,7 +152,7 @@ def test_a_document_reads_through_the_definitions_in_its_scope( def test_error_a_configuration_its_definition_refuses( fields: dict[str, Any], loc: tuple[str | int, ...], kind: str ) -> None: - # The problem is located at the field's position in the document. The + # The problem is located at the declaration's position in the document. The # reader reports it, and the model refuses the document. document = _document(**fields) assert [ @@ -163,18 +163,18 @@ def test_error_a_configuration_its_definition_refuses( def test_error_an_array_in_consolidated_metadata() -> None: - """A codec configuration that the codec's definition refuses, in an array listed in consolidated metadata, is reported at its location in the member.""" - member = {"kind": "inline", "must_understand": False, "metadata": {"a": BAD_GZIP_ARRAY}} + """A codec configuration that the codec's definition refuses, in an array listed in consolidated metadata, is reported at its location in the field.""" + field = {"kind": "inline", "must_understand": False, "metadata": {"a": BAD_GZIP_ARRAY}} assert [ - (problem.loc, problem.kind) for problem in read_consolidated_metadata_v3(member).problems + (problem.loc, problem.kind) for problem in read_consolidated_metadata_v3(field).problems ] == [(("metadata", "a", "codecs", 1, "configuration", "level"), "invalid_value")] def test_an_array_in_consolidated_metadata_reads_in_the_scope_it_is_read_in() -> None: - """An array listed in consolidated metadata is read in the scope the member is read in: a lenient `gzip` accepts the level the core `gzip` refuses, and the model round-trips in that scope.""" - member = {"kind": "inline", "must_understand": False, "metadata": {"a": BAD_GZIP_ARRAY}} + """An array listed in consolidated metadata is read in the scope the field is read in: a lenient `gzip` accepts the level the core `gzip` refuses, and the model round-trips in that scope.""" + field = {"kind": "inline", "must_understand": False, "metadata": {"a": BAD_GZIP_ARRAY}} lenient = CORE.update(codecs={**CORE.codecs, "gzip": LENIENT_GZIP}) - reading = read_consolidated_metadata_v3(member, context=lenient) + reading = read_consolidated_metadata_v3(field, context=lenient) assert reading.problems == () assert reading.metadata is not None assert ( @@ -194,7 +194,7 @@ def test_error_an_envelope_is_judged_once() -> None: ] == [(("codecs", 1, "must_understand"), "invalid_value")] -def test_error_a_field_that_is_not_json_is_not_read() -> None: +def test_error_a_declaration_that_is_not_json_is_not_read() -> None: # The JSON check runs first. The NaN is reported, and the configuration # next to it is not read. document = _document( diff --git a/packages/zarr-metadata/tests/model/test_extra_fields.py b/packages/zarr-metadata/tests/model/test_extra_fields.py index 86684ce928..d34f5bc43b 100644 --- a/packages/zarr-metadata/tests/model/test_extra_fields.py +++ b/packages/zarr-metadata/tests/model/test_extra_fields.py @@ -1,4 +1,4 @@ -"""Members of a v3 document that the spec does not define: extra fields, read by the definitions a scope holds for them.""" +"""Fields of a v3 document that the spec does not define: extra fields, read by the definitions a scope holds for them.""" from __future__ import annotations @@ -12,13 +12,13 @@ from typing_extensions import TypeAliasType, TypedDict from zarr_metadata.definition import ( - AcceptedField, - RefusedField, - UnclaimedField, + AcceptedDeclaration, + RefusedDeclaration, + UnclaimedDeclaration, canonical_of, resolve, ) -from zarr_metadata.json_schema import field_json_schema_v3 +from zarr_metadata.json_schema import declaration_json_schema_v3 from zarr_metadata.model import ZarrV3ArrayMetadata, ZarrV3GroupMetadata from zarr_metadata.problems import MetadataValidationError from zarr_metadata.typed_json import UserJSONValue @@ -31,25 +31,25 @@ class AcmeTilingConfiguration(TypedDict, closed=True): - """The value of an `acme.tiling` member: a tile edge, and the waiver that lets other readers skip it.""" + """The value of an `acme.tiling` field: a tile edge, and the waiver that lets other readers skip it.""" edge: Annotated[int, Ge(1)] must_understand: bool ACME_TILING = ArrayExtraFieldDefinition(name="acme.tiling", configuration=AcmeTilingConfiguration) -"""A reader's own definition of the `acme.tiling` member of an array.""" +"""A reader's own definition of the `acme.tiling` field of an array.""" class AcmeNoteConfiguration(TypedDict, closed=True): - """The value of an `acme.note` member: any JSON, and the waiver.""" + """The value of an `acme.note` field: any JSON, and the waiver.""" note: UserJSONValue must_understand: NotRequired[bool] ACME_NOTE = ArrayExtraFieldDefinition(name="acme.note", configuration=AcmeNoteConfiguration) -"""A reader's own definition of the `acme.note` member of an array, whose numbers must be finite.""" +"""A reader's own definition of the `acme.note` field of an array, whose numbers must be finite.""" CONTEXT = CORE.update( array_extra_fields=[*CORE.array_extra_fields.values(), ACME_TILING, ACME_NOTE] @@ -102,18 +102,18 @@ def test_a_scope_reads_the_extra_fields_it_claims( claimed: set[str], must_understand: dict[str, object], ) -> None: - """Each member the spec does not define is an extra field. A member whose key a definition in scope claims is an `AcceptedField`, and a reader understands it. Any other member is an `UnclaimedField`, kept as written, and `must_understand_fields` lists it unless it says `"must_understand": false`. Nothing reads an unclaimed member, so like attributes it may hold any number, finite or not. A definition of a group's extra field does not read a member of an array, and `CORE` reads a group's `consolidated_metadata`. The document is written back as it was read.""" + """Each field the spec does not define is an extra field. A field whose key a definition in scope claims is an `AcceptedDeclaration`, and a reader understands it. Any other field is an `UnclaimedDeclaration`, kept as written, and `must_understand_fields` lists it unless it says `"must_understand": false`. Nothing reads an unclaimed field, so like attributes it may hold any number, finite or not. A definition of a group's extra field does not read a field of an array, and `CORE` reads a group's `consolidated_metadata`. The document is written back as it was read.""" model: ZarrV3ArrayMetadata | ZarrV3GroupMetadata if document["node_type"] == "array": model = ZarrV3ArrayMetadata(document, context=context) else: model = ZarrV3GroupMetadata(document, context=context) accepted = { - key for key, extra in model.extra_fields.items() if isinstance(extra, AcceptedField) + key for key, extra in model.extra_fields.items() if isinstance(extra, AcceptedDeclaration) } assert accepted == claimed assert all( - isinstance(extra, UnclaimedField) + isinstance(extra, UnclaimedDeclaration) for key, extra in model.extra_fields.items() if key not in claimed ) @@ -139,7 +139,7 @@ def _same_json(a: object, b: object) -> bool: ids=["numbers", "strings", "null", "arrays", "objects", "empty-object-or-null"], ) def test_unclaimed_extra_fields_compare_as_written(first: object, second: object) -> None: - """Nothing reads an unclaimed member, so documents that write it differently are different documents: they compare unequal, and neither refines the other, whatever JSON the member holds.""" + """Nothing reads an unclaimed field, so documents that write it differently are different documents: they compare unequal, and neither refines the other, whatever JSON the field holds.""" a = ZarrV3ArrayMetadata({**ARRAY, "acme.flag": first}) b = ZarrV3ArrayMetadata({**ARRAY, "acme.flag": second}) assert a != b @@ -148,33 +148,33 @@ def test_unclaimed_extra_fields_compare_as_written(first: object, second: object assert a == ZarrV3ArrayMetadata({**ARRAY, "acme.flag": first}) -def test_an_extra_field_is_a_field_of_the_reading() -> None: - """A claimed extra field takes part in what the model's reading holds: it is a field at its key, read by the definition of its kind and name, and `resolve` reads it as the member alone, `{key: value}`. The JSON Schema of the kind accepts the member, and, like `resolve`, refuses a member the spec defines.""" +def test_an_extra_field_is_a_declaration_of_the_reading() -> None: + """A claimed extra field takes part in what the model's reading holds: it is a declaration at its key, read by the definition of its kind and name, and `resolve` reads it as the field alone, `{key: value}`. The JSON Schema of the kind accepts the field, and, like `resolve`, refuses a field the spec defines.""" model = ZarrV3ArrayMetadata({**ARRAY, "acme.tiling": TILING}, context=CONTEXT) - assert dict(model.reading.fields())[("acme.tiling",)] == model.extra_fields["acme.tiling"] + assert dict(model.reading.declarations())[("acme.tiling",)] == model.extra_fields["acme.tiling"] tiling = model.extra_fields["acme.tiling"] - assert isinstance(tiling, AcceptedField) + assert isinstance(tiling, AcceptedDeclaration) assert tiling.definition is ACME_TILING read, problems = resolve({"acme.tiling": TILING}, ArrayExtraFieldDefinition, CONTEXT) - assert isinstance(read, AcceptedField) + assert isinstance(read, AcceptedDeclaration) assert problems == () - schema = field_json_schema_v3(ArrayExtraFieldDefinition, context=CONTEXT) + schema = declaration_json_schema_v3(ArrayExtraFieldDefinition, context=CONTEXT) assert Draft202012Validator(schema).is_valid({"acme.tiling": TILING}) assert not Draft202012Validator(schema).is_valid({"acme.tiling": {"edge": 0}}) - # A member the spec defines is not an extra field, as `resolve` says. + # A field the spec defines is not an extra field, as `resolve` says. assert not Draft202012Validator(schema).is_valid({"shape": 5}) assert resolve({"shape": 5}, ArrayExtraFieldDefinition, CONTEXT)[1] != () def test_the_simplest_spelling_of_an_extra_field_is_plain_json() -> None: - """`canonical_of` returns plain JSON for an extra field, with no read-only views inside, so `json.dumps` writes it. `CORE` claims a group's `consolidated_metadata`, whose configuration holds objects with no nested field.""" + """`canonical_of` returns plain JSON for an extra field, with no read-only views inside, so `json.dumps` writes it. `CORE` claims a group's `consolidated_metadata`, whose configuration holds objects with no nested declaration.""" model = ZarrV3GroupMetadata({**GROUP, "consolidated_metadata": INLINE}, context=CORE) simplest = canonical_of(model.extra_fields["consolidated_metadata"], ()) assert json.loads(json.dumps(simplest)) == {"consolidated_metadata": INLINE} def test_error_a_claimed_extra_field_that_is_not_an_object() -> None: - """A definition reads a member whose value is an object. Any other value of a claimed member is a problem at the member's key, as `CORE` refuses a `consolidated_metadata` of null.""" + """A definition reads a field whose value is an object. Any other value of a claimed field is a problem at the field's key, as `CORE` refuses a `consolidated_metadata` of null.""" with pytest.raises(MetadataValidationError) as raised: ZarrV3GroupMetadata({**GROUP, "consolidated_metadata": None}, context=CORE) assert [(p.loc, p.kind) for p in raised.value.problems] == [ @@ -183,7 +183,7 @@ def test_error_a_claimed_extra_field_that_is_not_an_object() -> None: def test_error_a_claimed_extra_field_with_a_bad_configuration() -> None: - """A claimed member's value is checked against its definition's configuration, and each problem is located in the member, under its key.""" + """A claimed field's value is checked against its definition's configuration, and each problem is located in the field, under its key.""" with pytest.raises(MetadataValidationError) as raised: ZarrV3ArrayMetadata( {**ARRAY, "acme.tiling": {"edge": 0, "must_understand": True}}, context=CONTEXT @@ -211,27 +211,27 @@ def test_error_a_non_finite_number_in_an_extra_field_that_holds_no_user_data() - (GroupExtraFieldDefinition, "node_type"), ], ) -def test_error_an_extra_field_named_after_a_member_the_spec_defines(kind: Any, name: str) -> None: # noqa: ANN401 - either extra-field kind - """A member the spec defines is never an extra field, so a definition named after one would never read anything: it is refused when it is built, as a data type named like raw bits is.""" +def test_error_an_extra_field_named_after_a_field_the_spec_defines(kind: Any, name: str) -> None: # noqa: ANN401 - either extra-field kind + """A field the spec defines is never an extra field, so a definition named after one would never read anything: it is refused when it is built, as a data type named like raw bits is.""" with pytest.raises(TypeError, match="the spec defines"): kind(name=name, configuration=AcmeTilingConfiguration) @pytest.mark.parametrize( - ("kind", "member"), + ("kind", "field"), [ (ArrayExtraFieldDefinition, {"shape": (4,)}), (GroupExtraFieldDefinition, {"node_type": "group"}), ], ids=["array-shape", "group-node-type"], ) -def test_error_resolve_a_member_the_spec_defines_as_an_extra_field( +def test_error_resolve_a_field_the_spec_defines_as_an_extra_field( kind: Any, # noqa: ANN401 - either extra-field kind - member: dict[str, Any], + field: dict[str, Any], ) -> None: - """`resolve` reads a member the spec defines, read as an extra field, as a `RefusedField` with one `invalid_value` problem that says the spec defines it, for an array and for a group.""" - read, problems = resolve(member, kind, CORE) - assert isinstance(read, RefusedField) + """`resolve` reads a field the spec defines, read as an extra field, as a `RefusedDeclaration` with one `invalid_value` problem that says the spec defines it, for an array and for a group.""" + read, problems = resolve(field, kind, CORE) + assert isinstance(read, RefusedDeclaration) assert [(p.loc, p.kind) for p in problems] == [((), "invalid_value")] assert "the spec defines" in problems[0].message @@ -308,13 +308,13 @@ class _ExtraItemsOfAnUndefinedAlias(TypedDict, extra_items=_UndefinedWaiver): r"'acme.x': _ExtraItemsOfAnUndefinedAlias.extra_items: _UndefinedWaiver: name '_Missing'", ), ], - ids=["member", "extra-items"], + ids=["field", "extra-items"], ) def test_error_an_extra_field_waiver_typed_by_an_alias_that_does_not_resolve( configuration: Any, # noqa: ANN401 - a TypedDict match: str, ) -> None: - """An alias that names what is not defined is refused when the definition is built, with a `TypeError` that names the definition, the TypedDict and where the alias sits, as for any other member the checker cannot read. The waiver may be typed by the `must_understand` member, or, when the TypedDict declares no such member, by its `extra_items`.""" + """An alias that names what is not defined is refused when the definition is built, with a `TypeError` that names the definition, the TypedDict and where the alias sits, as for any other field the checker cannot read. The waiver may be typed by the `must_understand` field, or, when the TypedDict declares no such field, by its `extra_items`.""" with pytest.raises(TypeError, match=match): ArrayExtraFieldDefinition(name="acme.x", configuration=configuration) @@ -352,7 +352,7 @@ def test_an_extra_field_configuration_takes_the_waiver(configuration: Any) -> No ], ) def test_error_an_extra_field_configuration_that_refuses_the_waiver(configuration: Any) -> None: # noqa: ANN401 - a TypedDict - """The spec lets every extension member carry `"must_understand": false`, so the configuration of an extra field takes it: the type it gives `must_understand` admits `false`, and only booleans. A definition whose configuration refuses the waiver, or takes values that are not booleans, or types the waiver with an annotation that is not a type, such as a list, is refused when it is built, rather than misreading the documents that carry it.""" + """The spec lets every extension field carry `"must_understand": false`, so the configuration of an extra field takes it: the type it gives `must_understand` admits `false`, and only booleans. A definition whose configuration refuses the waiver, or takes values that are not booleans, or types the waiver with an annotation that is not a type, such as a list, is refused when it is built, rather than misreading the documents that carry it.""" with pytest.raises( TypeError, match="expected a configuration that takes must_understand as a boolean" ): @@ -360,7 +360,7 @@ def test_error_an_extra_field_configuration_that_refuses_the_waiver(configuratio def test_error_finite_is_a_boolean() -> None: - """`finite` decides whether a member may hold non-finite numbers, so it must be a boolean: anything else is refused when the definition is built.""" + """`finite` decides whether a field may hold non-finite numbers, so it must be a boolean: anything else is refused when the definition is built.""" with pytest.raises(TypeError, match="finite"): ArrayExtraFieldDefinition( name="acme.tiling", diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 4594ba1f4e..221e1ec7b1 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -15,7 +15,7 @@ from zarr_metadata.definition import ( EmptyConfiguration, Nested, - UnclaimedField, + UnclaimedDeclaration, resolve, ) from zarr_metadata.model import ( @@ -39,7 +39,7 @@ read_group_metadata_v3, read_node_metadata_v3, ) -from zarr_metadata.model._group_v3 import GroupMembersV3 +from zarr_metadata.model._group_v3 import GroupFieldsV3 from zarr_metadata.problems import MetadataValidationError, ValidationProblem from zarr_metadata.v2.group import ( ZarrV2GroupMetadataJSON, @@ -147,8 +147,8 @@ def _v2_consolidated_problems(document: object) -> list[tuple[tuple[str | int, . return [(p.loc, p.kind) for p in raised.value.problems] -def _v3_field_problems(field: object) -> list[tuple[tuple[str | int, ...], str]]: - return [(p.loc, p.kind) for p in resolve(field, CodecDefinition, CORE_AND_EXTENSIONS)[1]] +def _v3_declaration_problems(declaration: object) -> list[tuple[tuple[str | int, ...], str]]: + return [(p.loc, p.kind) for p in resolve(declaration, CodecDefinition, CORE_AND_EXTENSIONS)[1]] @pytest.mark.parametrize( @@ -176,17 +176,21 @@ def _v3_field_problems(field: object) -> list[tuple[tuple[str | int, ...], str]] [(("consolidated_metadata", "x"), "unknown_key")], ), ( - lambda: _v3_field_problems({"name": "gzip", "configuration": {"level": 1}, "x": 1}), + lambda: _v3_declaration_problems( + {"name": "gzip", "configuration": {"level": 1}, "x": 1} + ), [(("x",), "unknown_key")], ), ( - lambda: _v3_field_problems({"name": "gzip", "configuration": {"level": 1, "x": 1}}), + lambda: _v3_declaration_problems( + {"name": "gzip", "configuration": {"level": 1, "x": 1}} + ), [(("configuration", "x"), "unknown_key")], ), ], - ids=["v2-group", "v2-consolidated", "v3-consolidated", "v3-field", "v3-configuration"], + ids=["v2-group", "v2-consolidated", "v3-consolidated", "v3-declaration", "v3-configuration"], ) -def test_a_member_a_closed_object_does_not_declare_is_an_unknown_key( +def test_a_field_a_closed_object_does_not_declare_is_an_unknown_key( problems: Callable[[], list[tuple[tuple[str | int, ...], str]]], expected: list[tuple[tuple[str | int, ...], str]], ) -> None: @@ -304,8 +308,8 @@ def test_a_chain_of_consolidated_groups_is_bounded_by_the_levels_a_reader_walks( depth = f"nested deeper than the {JSON_DEPTH} levels a reader walks" deepest = (JSON_DEPTH - 1) // 3 assert read_group_metadata_v3(_chain_of_groups(deepest)).problems == () - member = _chain_of_groups(deepest)["consolidated_metadata"] - problems = read_consolidated_metadata_v3(member).problems + field = _chain_of_groups(deepest)["consolidated_metadata"] + problems = read_consolidated_metadata_v3(field).problems assert all(len(problem.loc) < JSON_DEPTH - 1 for problem in problems) assert {problem.message for problem in problems} == { 'expected a node the group lists, got "/a/a", which "/a" lists alone' @@ -317,7 +321,7 @@ def test_a_chain_of_consolidated_groups_is_bounded_by_the_levels_a_reader_walks( for problem in read_group_metadata_v3(chain).problems ] == [(JSON_DEPTH, depth)] problems = read_consolidated_metadata_v3(chain["consolidated_metadata"]).problems - # Located in the member, one level below the group document. + # Located in the field, one level below the group document. assert [ (len(problem.loc) + 1, problem.message) for problem in problems @@ -335,7 +339,7 @@ def nested(levels: int) -> dict[str, object]: value = {"x": value} return value - # A document is three levels down. Its attribute, or a member of an + # A document is three levels down. Its attribute, or a field of an # extra field, is two more levels down. group = { "zarr_format": 3, @@ -354,15 +358,15 @@ def nested(levels: int) -> dict[str, object]: } assert read_node_metadata_v3(group).problems == () assert read_node_metadata_v3(array).problems == () - member = {"kind": "inline", "must_understand": False, "metadata": {"g": group, "h": array}} - document = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": member} + field = {"kind": "inline", "must_understand": False, "metadata": {"g": group, "h": array}} + document = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": field} assert [ (len(problem.loc), problem.message) for problem in read_group_metadata_v3(document).problems ] == [(JSON_DEPTH, depth)] * 4 - # Located in the member, one level below the group document. + # Located in the field, one level below the group document. assert [ (problem.loc[:4], len(problem.loc) + 1) - for problem in read_consolidated_metadata_v3(member).problems + for problem in read_consolidated_metadata_v3(field).problems ] == [ (("metadata", "g", "acme_extra", "a"), JSON_DEPTH), (("metadata", "g", "attributes", "a"), JSON_DEPTH), @@ -372,7 +376,7 @@ def nested(levels: int) -> dict[str, object]: def test_a_document_a_level_before_the_cap_holds_its_scalars_and_nothing_else() -> None: - """Each container member is past the cap. It is checked at its location before it is walked, as `refine_json` of the whole document would check it. The scalars are read.""" + """Each container field is past the cap. It is checked at its location before it is walked, as `refine_json` of the whole document would check it. The scalars are read.""" depth = f"nested deeper than the {JSON_DEPTH} levels a reader walks" def a_level_before_the_cap(document: dict[str, object]) -> dict[str, object]: @@ -408,13 +412,13 @@ def a_level_before_the_cap(document: dict[str, object]) -> dict[str, object]: def test_a_listing_key_too_long_to_write_is_shown_by_its_size( interpreter_writes_4300_digits: None, ) -> None: - member = {"kind": "inline", "must_understand": False, "metadata": {10**5000: 1}} - (problem,) = read_consolidated_metadata_v3(member).problems + field = {"kind": "inline", "must_understand": False, "metadata": {10**5000: 1}} + (problem,) = read_consolidated_metadata_v3(field).problems assert problem.message == f"non-string key an integer of {(10**5000).bit_length()} bits" def test_a_group_reads_the_documents_it_holds_from_where_they_sit() -> None: - """A child that is valid on its own, nested to the last level a reader walks, is three levels deeper when a group holds it as a document. The group's read refuses it at the cap, as the validator does. The consolidated member, read on its own, also refuses it, counting from its location under the group's key.""" + """A child that is valid on its own, nested to the last level a reader walks, is three levels deeper when a group holds it as a document. The group's read refuses it at the cap, as the validator does. The consolidated field, read on its own, also refuses it, counting from its location under the group's key.""" # On its own, the innermost object is at the last level a reader walks. # As a document held by a group, it is three levels deeper. nested: dict[str, object] = {} @@ -472,8 +476,8 @@ def test_group_v2_key_value_split() -> None: @pytest.mark.parametrize( ("extra_key", "kind"), [("attributes", "invalid_value"), ("vendor_extension", "unknown_key")] ) -def test_v2_group_from_key_value_rejects_zgroup_extra_members(extra_key: str, kind: str) -> None: - """Raw `.zgroup` documents reject every non-spec member. `attributes` is not unknown, but belongs in `.zattrs`.""" +def test_v2_group_from_key_value_rejects_zgroup_extra_fields(extra_key: str, kind: str) -> None: + """Raw `.zgroup` documents reject every non-spec field. `attributes` is not unknown, but belongs in `.zattrs`.""" doc: dict[str, object] = {"zarr_format": 2, extra_key: {}} with pytest.raises(MetadataValidationError) as exc_info: @@ -540,8 +544,8 @@ def test_group_update_keys_match_settable_model_fields() -> None: assert set(ZarrV2GroupMetadataUpdate.__annotations__) == {"attributes"} -def test_update_takes_every_member_of_the_document_it_may_change() -> None: - """Each v3 model's `update` takes every member of its document except `zarr_format` and `node_type`, which it cannot change, and `extra_fields`, which holds the members the spec does not define. A group's `update` also declares `consolidated_metadata`, which the document type leaves to its extra items, because `update` accepts node models there.""" +def test_update_takes_every_field_of_the_document_it_may_change() -> None: + """Each v3 model's `update` takes every field of its document except `zarr_format` and `node_type`, which it cannot change, and `extra_fields`, which holds the fields the spec does not define. A group's `update` also declares `consolidated_metadata`, which the document type leaves to its extra items, because `update` accepts node models there.""" fixed = {"zarr_format", "node_type"} for update, document, declared in ( (ZarrV3ArrayMetadataUpdate, ZarrV3ArrayMetadataJSON, {"extra_fields"}), @@ -558,7 +562,7 @@ def test_update_takes_every_member_of_the_document_it_may_change() -> None: def test_consolidated_v3_roundtrip() -> None: - """A v3 group with inline consolidated metadata round-trips, holding the member as written. Read on its own, the member holds an array model and a group model.""" + """A v3 group with inline consolidated metadata round-trips, holding the field as written. Read on its own, the field holds an array model and a group model.""" child = ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json() doc = { "zarr_format": 3, @@ -577,7 +581,7 @@ def test_consolidated_v3_roundtrip() -> None: assert isinstance(held.metadata["g"], ZarrV3GroupMetadata) -def test_error_consolidated_v3_must_understand_is_not_a_member_to_set() -> None: +def test_error_consolidated_v3_must_understand_is_not_a_field_to_set() -> None: """`must_understand` is always `False`, as `kind` is always `"inline"`, so the constructor does not take it as an argument.""" with pytest.raises(TypeError, match="must_understand"): cast("Any", ZarrV3ConsolidatedMetadata)(must_understand=True, metadata={}) @@ -611,22 +615,22 @@ def test_consolidated_v3_not_a_mapping() -> None: GZIP_99 = {"name": "gzip", "configuration": {"level": 99}} A: tuple[str, ...] = ("metadata", "a") -"""The location, in a `consolidated_metadata` member, of the document at path `a`.""" +"""The location, in a `consolidated_metadata` field, of the document at path `a`.""" -def _array(**members: object) -> dict[str, object]: - return {**ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json(), **members} +def _array(**fields: object) -> dict[str, object]: + return {**ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json(), **fields} def _inline(**documents: object) -> dict[str, object]: return {"kind": "inline", "must_understand": False, "metadata": documents} -def _group(**members: object) -> dict[str, object]: - return {"zarr_format": 3, "node_type": "group", **members} +def _group(**fields: object) -> dict[str, object]: + return {"zarr_format": 3, "node_type": "group", **fields} -def test_error_a_consolidated_envelope_reports_the_members_it_lacks_in_its_order() -> None: +def test_error_a_consolidated_envelope_reports_the_fields_it_lacks_in_its_order() -> None: document = _group(consolidated_metadata={"kind": "inline"}) assert [(p.loc, p.kind) for p in read_group_metadata_v3(document).problems] == [ (("consolidated_metadata", "must_understand"), "missing_key"), @@ -640,7 +644,7 @@ def test_error_a_consolidated_envelope_reports_the_members_it_lacks_in_its_order def test_group_update_keeps_the_documents_it_holds() -> None: - """`update` does not read the consolidated documents again: a child read in an extended scope keeps each field as it was read, and the group keeps its scope.""" + """`update` does not read the consolidated documents again: a child read in an extended scope keeps each declaration as it was read, and the group keeps its scope.""" context = CORE_AND_EXTENSIONS.update(codecs=[*CORE_AND_EXTENSIONS.codecs.values(), ACME_X]) child = ZarrV3ArrayMetadata.create_default( context=context, shape=(4,), codecs=(LITTLE, {"name": "acme.x"}) @@ -656,61 +660,64 @@ def test_group_update_keeps_the_documents_it_holds() -> None: def test_group_update_writes_the_consolidated_metadata_it_is_given() -> None: - """`update` takes consolidated metadata as a document writes it, or as a `ZarrV3ConsolidatedMetadata`, whose document it writes; the group holds the same member either way. `UNSET` removes it. `zstd` is an extension, so `CORE` leaves it unclaimed, and the member read in the group's scope holds it as unclaimed.""" + """`update` takes consolidated metadata as a document writes it, or as a `ZarrV3ConsolidatedMetadata`, whose document it writes; the group holds the same field either way. `UNSET` removes it. `zstd` is an extension, so `CORE` leaves it unclaimed, and the field read in the group's scope holds it as unclaimed.""" zstd = {"name": "zstd", "configuration": {"level": 3, "checksum": False}} - member = cast("Any", _inline(a=_array(codecs=[LITTLE, zstd]))) - held = ZarrV3ConsolidatedMetadata(member, context=CORE) + field = cast("Any", _inline(a=_array(codecs=[LITTLE, zstd]))) + held = ZarrV3ConsolidatedMetadata(field, context=CORE) child = held.metadata["a"] assert isinstance(child, ZarrV3ArrayMetadata) - assert isinstance(child.codecs[1], UnclaimedField) + assert isinstance(child.codecs[1], UnclaimedDeclaration) group = ZarrV3GroupMetadata.create_default(context=CORE) updated = group.update(consolidated_metadata=held) assert updated.consolidated_metadata == held.to_json() - assert updated == group.update(consolidated_metadata=member) + assert updated == group.update(consolidated_metadata=field) assert ZarrV3ConsolidatedMetadata(updated.consolidated_metadata, context=CORE) == held removed = updated.update(consolidated_metadata=UNSET) assert removed.consolidated_metadata is UNSET -def _fields_of_an_array(*at: str | int) -> list[tuple[str | int, ...]]: - """Return the locations of a default array's fields, under `at`.""" +def _declarations_of_an_array(*at: str | int) -> list[tuple[str | int, ...]]: + """Return the locations of a default array's declarations, under `at`.""" points = ("data_type", "chunk_grid", "chunk_key_encoding") return [*((*at, point) for point in points), (*at, "codecs", 0)] @pytest.mark.parametrize( - ("member", "paths", "locs"), + ("field", "paths", "locs"), [ - (_inline(a=_array(), g=_group()), ["a", "g"], _fields_of_an_array(*A)), + (_inline(a=_array(), g=_group()), ["a", "g"], _declarations_of_an_array(*A)), # Every node below the group, including one below another group, at its path. ( _inline(g=_group(), **{"g/b": _array()}), ["g", "g/b"], - _fields_of_an_array("metadata", "g/b"), + _declarations_of_an_array("metadata", "g/b"), ), # A group's consolidated metadata inside another group's, listing a node # the outer group also lists. The listed group's own listing is # checked against the outer one, and its documents are not models - # of the outer member. The listed group's member is its extra field, + # of the outer field. The listed group's field is its extra field, # which `CORE` reads as an envelope. ( _inline(g=_group(consolidated_metadata=_inline(b=_array())), **{"g/b": _array()}), ["g", "g/b"], - [("metadata", "g", "consolidated_metadata"), *_fields_of_an_array("metadata", "g/b")], + [ + ("metadata", "g", "consolidated_metadata"), + *_declarations_of_an_array("metadata", "g/b"), + ], ), ], ids=["an-array-and-a-group", "paths", "nested"], ) def test_consolidated_metadata_reads_each_document_it_lists( - member: dict[str, object], paths: list[str], locs: list[tuple[str | int, ...]] + field: dict[str, object], paths: list[str], locs: list[tuple[str | int, ...]] ) -> None: - """`read_consolidated_metadata_v3` reads each document a member lists as the node its `node_type` declares, and the reading's fields are each listed array's fields, located in the member. The model equals the one the constructor builds.""" - reading = read_consolidated_metadata_v3(member) + """`read_consolidated_metadata_v3` reads each document a field lists as the node its `node_type` declares, and the reading's declarations are each listed array's declarations, located in the field. The model equals the one the constructor builds.""" + reading = read_consolidated_metadata_v3(field) assert reading.problems == () assert reading.metadata is not None assert list(reading.metadata.metadata) == paths - assert [loc for loc, _ in reading.fields()] == locs - assert reading.metadata == ZarrV3ConsolidatedMetadata(member) + assert [loc for loc, _ in reading.declarations()] == locs + assert reading.metadata == ZarrV3ConsolidatedMetadata(field) @pytest.mark.parametrize( @@ -731,9 +738,9 @@ def test_error_a_document_in_consolidated_metadata_is_at_a_node_s_path_below_the ) -> None: # A path is node names joined by "/", as the reference implementation # stores it. The group's own path, "/", and this path make the node's path. - member = _inline(**{path: _group()}) + field = _inline(**{path: _group()}) message = f"expected the path of a node below the group, got {json.dumps(path)}, which {fault}" - assert read_consolidated_metadata_v3(member).problems == ( + assert read_consolidated_metadata_v3(field).problems == ( ValidationProblem(("metadata", path), message, "invalid_value"), ) @@ -752,9 +759,9 @@ def test_error_no_document_in_consolidated_metadata_is_below_an_array( ) -> None: # "Group nodes may have children but array nodes may not." The message # names each node by its path in the hierarchy below the group. - member = _inline(**documents) + field = _inline(**documents) message = f'expected a node below a group, got "/{path}", below the array "/a"' - assert read_consolidated_metadata_v3(member).problems == ( + assert read_consolidated_metadata_v3(field).problems == ( ValidationProblem(("metadata", path), message, "invalid_value"), ) @@ -762,8 +769,8 @@ def test_error_no_document_in_consolidated_metadata_is_below_an_array( def test_a_document_of_no_node_type_is_taken_as_a_group_s() -> None: # The document's own problem is reported at its location, and the nodes # below it are not refused because of it. - member = _inline(a={"zarr_format": 3, "node_type": "x"}, **{"a/b": _array()}) - assert [(p.loc, p.kind) for p in read_consolidated_metadata_v3(member).problems] == [ + field = _inline(a={"zarr_format": 3, "node_type": "x"}, **{"a/b": _array()}) + assert [(p.loc, p.kind) for p in read_consolidated_metadata_v3(field).problems] == [ ((*A, "node_type"), "invalid_value") ] @@ -771,8 +778,8 @@ def test_a_document_of_no_node_type_is_taken_as_a_group_s() -> None: def test_error_consolidated_metadata_holds_the_group_holding_each_document() -> None: # The nearest missing group above a node is reported once, with a count # of the missing groups above it, up to the group that holds the documents. - member = _inline(**{"a/b/c": _array(), "a/b/d": _array()}) - assert [(p.loc, p.message, p.kind) for p in read_consolidated_metadata_v3(member).problems] == [ + field = _inline(**{"a/b/c": _array(), "a/b/d": _array()}) + assert [(p.loc, p.message, p.kind) for p in read_consolidated_metadata_v3(field).problems] == [ ( ("metadata", "a/b"), 'missing the group holding "/a/b/c", and 1 group above it', @@ -790,7 +797,7 @@ def test_error_a_consolidated_metadata_key_that_is_not_a_string() -> None: NESTED = ("metadata", "g", "consolidated_metadata", "metadata") -"""The location, in the outer member, of the own listing of the group at `g`.""" +"""The location, in the outer field, of the own listing of the group at `g`.""" @pytest.mark.parametrize( @@ -844,15 +851,15 @@ def test_a_listed_group_s_own_listing_lists_what_the_group_lists( flat: dict[str, object], expected: list[tuple[tuple[str, ...], str, str]], ) -> None: - """A group listed in consolidated metadata may list nodes of its own, and each must be listed by the outer member too, with the same node type. A group holding the member does not read the listings, so it reads without problems.""" - member = _inline(g=_group(consolidated_metadata=_inline(**listed)), **flat) - problems = read_consolidated_metadata_v3(member).problems + """A group listed in consolidated metadata may list nodes of its own, and each must be listed by the outer field too, with the same node type. A group holding the field does not read the listings, so it reads without problems.""" + field = _inline(g=_group(consolidated_metadata=_inline(**listed)), **flat) + problems = read_consolidated_metadata_v3(field).problems assert [(p.loc, p.message, p.kind) for p in problems] == expected - assert read_group_metadata_v3(_group(consolidated_metadata=member)).problems == () + assert read_group_metadata_v3(_group(consolidated_metadata=field)).problems == () def test_each_document_its_consolidated_metadata_holds_is_read_once() -> None: - """A group does not read the documents its consolidated metadata lists. Reading the member reads each of them once.""" + """A group does not read the documents its consolidated metadata lists. Reading the field reads each of them once.""" reads: list[GzipCodecConfiguration] = [] def counted( @@ -868,14 +875,14 @@ def counted( } ) array = _array(codecs=[LITTLE, {"name": "gzip", "configuration": {"level": 5}}]) - member = _inline(a=array) + field = _inline(a=array) assert ( - read_group_metadata_v3(_group(consolidated_metadata=member), context=context).problems == () + read_group_metadata_v3(_group(consolidated_metadata=field), context=context).problems == () ) assert reads == [] - assert read_consolidated_metadata_v3(member, context=context).problems == () + assert read_consolidated_metadata_v3(field, context=context).problems == () assert reads == [{"level": 5}] - ZarrV3ConsolidatedMetadata(member, context=context) + ZarrV3ConsolidatedMetadata(field, context=context) assert len(reads) == 2 @@ -894,7 +901,7 @@ def test_error_a_group_document_with_a_problem_reads_as_no_model() -> None: @pytest.mark.parametrize( - ("member", "problems"), + ("field", "problems"), [ ( _inline(a=_array(codecs=[LITTLE, GZIP_99]), b=_array()), @@ -906,13 +913,13 @@ def test_error_a_group_document_with_a_problem_reads_as_no_model() -> None: ids=["a-document", "no-node-type"], ) def test_error_consolidated_metadata_with_a_problem_reads_as_no_model( - member: dict[str, object], problems: list[tuple[tuple[str | int, ...], str]] + field: dict[str, object], problems: list[tuple[tuple[str | int, ...], str]] ) -> None: - """A problem in a listed document is a problem of the member, located at the document's path, and the reading holds no model. The group holding the member has no problem.""" - reading = read_consolidated_metadata_v3(member) + """A problem in a listed document is a problem of the field, located at the document's path, and the reading holds no model. The group holding the field has no problem.""" + reading = read_consolidated_metadata_v3(field) assert [(p.loc, p.kind) for p in reading.problems] == problems assert reading.metadata is None - assert read_group_metadata_v3(_group(consolidated_metadata=member)).problems == () + assert read_group_metadata_v3(_group(consolidated_metadata=field)).problems == () # --- read_node_metadata_v3 ------------------------------------------------- @@ -952,7 +959,7 @@ def test_error_a_node_type_the_spec_does_not_define(node_type: object, kind: str assert isinstance(read, ZarrV3UnknownNodeMetadataReading) assert [(p.loc, p.kind) for p in read.problems] == [(("node_type",), kind)] assert read.metadata is None - assert list(read.fields()) == [] + assert list(read.declarations()) == [] def test_error_a_document_without_a_node_type() -> None: @@ -1233,8 +1240,8 @@ def test_group_v3_valid_consolidated_passes_the_reader(listed: dict[str, object] assert ZarrV3GroupMetadata(doc).to_json() == arrays_to_tuples(doc) -def test_v3_consolidated_rejects_unknown_envelope_member() -> None: - """The inline consolidated envelope is closed and never drops accepted members.""" +def test_v3_consolidated_rejects_unknown_envelope_field() -> None: + """The inline consolidated envelope is closed and never drops accepted fields.""" doc = { "kind": "inline", "must_understand": False, @@ -1246,8 +1253,8 @@ def test_v3_consolidated_rejects_unknown_envelope_member() -> None: ZarrV3ConsolidatedMetadata.from_json(doc) -def test_v2_consolidated_rejects_unknown_document_member() -> None: - """The v2 consolidated document is closed and never drops accepted members.""" +def test_v2_consolidated_rejects_unknown_document_field() -> None: + """The v2 consolidated document is closed and never drops accepted fields.""" doc = {"zarr_consolidated_format": 1, "metadata": {}, "unexpected": 1} with pytest.raises(MetadataValidationError, match="unexpected"): @@ -1257,7 +1264,7 @@ def test_v2_consolidated_rejects_unknown_document_member() -> None: @pytest.mark.parametrize("key", [1, None], ids=["int", "none"]) def test_v2_consolidated_rejects_non_string_document_key(key: object) -> None: """A non-string key is reported as a problem at the document root. It is not - reported as a member at a location that looks like an index, and it does + reported as a field at a location that looks like an index, and it does not raise `TypeError`.""" doc = {"zarr_consolidated_format": 1, "metadata": {}, key: "x"} @@ -1379,7 +1386,7 @@ def test_a_group_writes_no_empty_attributes() -> None: def test_error_a_listed_group_s_own_listing_holds_the_documents_the_group_lists() -> None: - """Each node has one document. A group listed in consolidated metadata may list a node that the outer member also lists, but only with the same document (as read) that the outer member holds at the joined key. A document that reads differently is a problem at the nested entry. The reader, the constructor, and models given as entries all report it the same way.""" + """Each node has one document. A group listed in consolidated metadata may list a node that the outer field also lists, but only with the same document (as read) that the outer field holds at the joined key. A document that reads differently is a problem at the nested entry. The reader, the constructor, and models given as entries all report it the same way.""" same = _array(shape=(4,), attributes={"a": 1}) other = _array(shape=(9,), attributes={"a": 1}) agreed = _inline(b=_group(consolidated_metadata=_inline(c=same)), **{"b/c": same}) @@ -1436,23 +1443,23 @@ def test_error_a_listed_document_with_only_an_unknown_key_is_still_compared() -> def test_error_a_model_inside_consolidated_metadata_is_not_json() -> None: - """A group holds its consolidated metadata as JSON, so a node model inside the member is a value that is not JSON. `ZarrV3ConsolidatedMetadata` is the place that takes node models as entries.""" - member = _inline(a=ZarrV3ArrayMetadata.create_default()) - problems = read_group_metadata_v3(_group(consolidated_metadata=member)).problems + """A group holds its consolidated metadata as JSON, so a node model inside the field is a value that is not JSON. `ZarrV3ConsolidatedMetadata` is the place that takes node models as entries.""" + field = _inline(a=ZarrV3ArrayMetadata.create_default()) + problems = read_group_metadata_v3(_group(consolidated_metadata=field)).problems assert [(p.loc, p.kind) for p in problems] == [(("consolidated_metadata", *A), "invalid_type")] with pytest.raises(MetadataValidationError): - ZarrV3GroupMetadata.create_default(consolidated_metadata=cast("Any", member)) + ZarrV3GroupMetadata.create_default(consolidated_metadata=cast("Any", field)) -def test_error_a_model_is_built_only_of_members_a_read_found_nothing_wrong_with() -> None: - """The members of a group read with a problem hold no document. Building a model from them raises `TypeError` instead of producing an empty model.""" +def test_error_a_model_is_built_only_of_fields_a_read_found_nothing_wrong_with() -> None: + """The fields of a group read with a problem hold no document. Building a model from them raises `TypeError` instead of producing an empty model.""" from zarr_metadata.model._group_v3 import _document_of with pytest.raises(TypeError, match="nothing wrong with"): - _document_of(GroupMembersV3(None, {}, UNSET, None)) + _document_of(GroupFieldsV3(None, {}, UNSET, None)) -def test_error_the_unset_sentinel_at_consolidated_metadata_is_not_an_absent_member() -> None: +def test_error_the_unset_sentinel_at_consolidated_metadata_is_not_an_absent_field() -> None: """A reader returns `UNSET` for a key that a document does not hold. If `UNSET` is written as the value of `consolidated_metadata`, it is treated like any other value that is not an object, and it is never written back as null.""" document = { "zarr_format": 3, @@ -1467,15 +1474,15 @@ def test_error_the_unset_sentinel_at_consolidated_metadata_is_not_an_absent_memb ZarrV3GroupMetadata(document) -def test_a_consolidated_model_given_as_the_member_keeps_the_order_its_document_writes() -> None: - """A `ZarrV3ConsolidatedMetadata` given to `update` is written as its document, in the document's key order, so the group writes it the same way the member wrote it.""" +def test_a_consolidated_model_given_as_the_field_keeps_the_order_its_document_writes() -> None: + """A `ZarrV3ConsolidatedMetadata` given to `update` is written as its document, in the document's key order, so the group writes it the same way the field wrote it.""" array = ZarrV3ArrayMetadata.create_default(shape=(2,)) - member = ZarrV3ConsolidatedMetadata( + field = ZarrV3ConsolidatedMetadata( {"metadata": {"a": array.to_json()}, "kind": "inline", "must_understand": False} ) - group = ZarrV3GroupMetadata.create_default().update(consolidated_metadata=member) + group = ZarrV3GroupMetadata.create_default().update(consolidated_metadata=field) written = cast("dict[str, Any]", group.to_json())["consolidated_metadata"] assert list(written) == ["metadata", "kind", "must_understand"] assert group == ZarrV3GroupMetadata( - {"zarr_format": 3, "node_type": "group", "consolidated_metadata": member.to_json()} + {"zarr_format": 3, "node_type": "group", "consolidated_metadata": field.to_json()} ) diff --git a/packages/zarr-metadata/tests/model/test_pair.py b/packages/zarr-metadata/tests/model/test_pair.py index 0f1b06341e..e23a6ecb27 100644 --- a/packages/zarr-metadata/tests/model/test_pair.py +++ b/packages/zarr-metadata/tests/model/test_pair.py @@ -13,11 +13,11 @@ from zarr_metadata._json import refine_json from zarr_metadata.definition import ( - AcceptedField, + AcceptedDeclaration, EmptyConfiguration, Nested, ScopeConflictError, - UnclaimedField, + UnclaimedDeclaration, ) from zarr_metadata.model import ( UNSET, @@ -97,15 +97,15 @@ def test_a_model_is_its_document_read_in_its_scope( def test_to_json_keeps_the_spelling_the_document_was_written_in( document: dict[str, Any], key: str, written: object ) -> None: - """`to_json` writes each field as the document wrote it, not in a reader's preferred spelling, because the model is the document.""" + """`to_json` writes each declaration as the document wrote it, not in a reader's preferred spelling, because the model is the document.""" assert ZarrV3ArrayMetadata(document).to_json()[key] == written def test_properties_are_what_the_reading_holds() -> None: - """The typed members (fields as the scope read them, shape, fill value, attributes) are read-only views of the reading.""" + """The typed attributes (declarations as the scope read them, shape, fill value, attributes) are read-only views of the reading.""" model = ZarrV3ArrayMetadata({**ARRAY, "attributes": {"a": [1]}, "acme": 1}) - assert isinstance(model.data_type, AcceptedField) - assert isinstance(model.codecs[0], AcceptedField) + assert isinstance(model.data_type, AcceptedDeclaration) + assert isinstance(model.codecs[0], AcceptedDeclaration) assert model.shape == (4,) assert model.fill_value == 0 assert model.dimension_names is UNSET @@ -117,7 +117,9 @@ def test_properties_are_what_the_reading_holds() -> None: model.attributes["b"] = 1 # pyright: ignore[reportIndexIssue] with pytest.raises(AttributeError): model.shape = (5,) # pyright: ignore[reportAttributeAccessIssue] - assert isinstance(ZarrV3ArrayMetadata(ARRAY, context=ZarrV3Context()).data_type, UnclaimedField) + assert isinstance( + ZarrV3ArrayMetadata(ARRAY, context=ZarrV3Context()).data_type, UnclaimedDeclaration + ) def test_a_reading_without_problems_builds_the_model_without_reading_again() -> None: @@ -151,7 +153,7 @@ def test_error_a_document_with_a_problem_is_refused_at_construction() -> None: def test_models_are_equal_by_what_their_documents_mean( left: ZarrV3ArrayMetadata, right: ZarrV3ArrayMetadata, equal: bool ) -> None: - """Two models are equal when every field reads the same with the same definition and the rest is the same JSON. Spelling and unused definitions do not matter. A private definition under a core name does matter, and so does a name that one model reads and the other leaves unclaimed.""" + """Two models are equal when every declaration reads the same with the same definition and the rest is the same JSON. Spelling and unused definitions do not matter. A private definition under a core name does matter, and so does a name that one model reads and the other leaves unclaimed.""" assert (left == right) is equal if equal: assert hash(left) == hash(right) @@ -214,7 +216,7 @@ def test_error_a_model_whose_scope_does_not_pickle_says_so() -> None: @pytest.mark.parametrize( - ("model", "members", "expected"), + ("model", "fields", "expected"), [ ( ZarrV3ArrayMetadata(ARRAY), @@ -246,19 +248,19 @@ def test_error_a_model_whose_scope_does_not_pickle_says_so() -> None: ), ), ], - ids=["set", "unset", "update-keeps-scope", "members-together"], + ids=["set", "unset", "update-keeps-scope", "fields-together"], ) -def test_update_reads_new_members_in_the_models_own_scope( - model: ZarrV3ArrayMetadata, members: dict[str, Any], expected: ZarrV3ArrayMetadata +def test_update_reads_new_fields_in_the_models_own_scope( + model: ZarrV3ArrayMetadata, fields: dict[str, Any], expected: ZarrV3ArrayMetadata ) -> None: - """`update` replaces the document's members with the given JSON members, removes a member given as `UNSET`, and reads the result in the model's own scope. So a model read in a private scope stays in that scope.""" - updated = model.update(**members) + """`update` replaces the document's fields with the given JSON fields, removes a field given as `UNSET`, and reads the result in the model's own scope. So a model read in a private scope stays in that scope.""" + updated = model.update(**fields) assert updated == expected assert updated.context == model.context def test_error_update_refuses_a_document_with_a_problem() -> None: - """`update` raises `MetadataValidationError` when the members make an invalid document, as the constructor does. Here the invalid document has two dimension names for one dimension.""" + """`update` raises `MetadataValidationError` when the fields make an invalid document, as the constructor does. Here the invalid document has two dimension names for one dimension.""" with pytest.raises(MetadataValidationError): ZarrV3ArrayMetadata(ARRAY).update(dimension_names=("x", "y")) @@ -310,7 +312,7 @@ def test_with_context_reads_the_document_in_any_scope() -> None: private = model.with_context(PRIVATE) assert private.context == PRIVATE assert private.codecs[0].definition == MY_BYTES - assert isinstance(model.with_context(ZarrV3Context()).codecs[0], UnclaimedField) + assert isinstance(model.with_context(ZarrV3Context()).codecs[0], UnclaimedDeclaration) assert model.with_context(None).context == CORE_AND_EXTENSIONS @@ -353,16 +355,16 @@ def test_error_with_context_refuses_a_document_the_scope_reads_with_a_problem() "loss", "equal", "conflict", - "other-members-differ", + "other-fields-differ", "fill-value-spelled-by-the-informed-side", - "gain-of-a-field-holding-fields", + "gain-of-a-declaration-holding-declarations", "fill-value-the-gained-definition-refuses", ], ) def test_refines_orders_models_by_information( upper: ZarrV3ArrayMetadata, lower: ZarrV3ArrayMetadata, expected: bool ) -> None: - """A model refines another when every field refines its counterpart, and every other member is the same. Fields held inside a field count too, so gaining a shard's inner fields is a gain. The fill value is compared as the more informed data type spells it. A fill value that this data type's definition refuses is not a refinement, and does not raise an error.""" + """A model refines another when every declaration refines its counterpart, and every other field is the same. Declarations held inside a declaration count too, so gaining a shard's inner declarations is a gain. The fill value is compared as the more informed data type spells it. A fill value that this data type's definition refuses is not a refinement, and does not raise an error.""" assert upper.refines(lower) is expected @@ -388,27 +390,27 @@ def test_refines_orders_models_by_information( } -MEMBER: dict[str, Any] = CONSOLIDATED["consolidated_metadata"] +FIELD: dict[str, Any] = CONSOLIDATED["consolidated_metadata"] def test_a_group_holds_its_consolidated_metadata_as_written() -> None: - """A group model holds its `consolidated_metadata` member as written, refined and read-only, and does not read the documents it lists. `ZarrV3ConsolidatedMetadata` reads them: each becomes a model in its scope, and writes its document as written.""" + """A group model holds its `consolidated_metadata` field as written, refined and read-only, and does not read the documents it lists. `ZarrV3ConsolidatedMetadata` reads them: each becomes a model in its scope, and writes its document as written.""" group = ZarrV3GroupMetadata(CONSOLIDATED, context=CORE) - member = group.consolidated_metadata - assert member == refine_json(MEMBER)[0] + field = group.consolidated_metadata + assert field == refine_json(FIELD)[0] with pytest.raises(TypeError): - operator.setitem(cast("Any", member)["metadata"], "a", ARRAY) + operator.setitem(cast("Any", field)["metadata"], "a", ARRAY) assert group.to_json() == refine_json(CONSOLIDATED)[0] assert ZarrV3GroupMetadata(GROUP).consolidated_metadata is UNSET - held = ZarrV3ConsolidatedMetadata(member, context=group.context) + held = ZarrV3ConsolidatedMetadata(field, context=group.context) assert set(held.metadata) == {"a", "b", "b/c"} for node in held.metadata.values(): assert node.context is held.context assert held.metadata["b/c"].to_json()["data_type"] == {"name": "uint8"} -def test_consolidated_metadata_reads_each_nested_field_once() -> None: - """Reading a group calls no definition's rules for the documents its consolidated metadata lists. Reading the consolidated metadata calls them once per field, and builds the models from that read.""" +def test_consolidated_metadata_reads_each_nested_declaration_once() -> None: + """Reading a group calls no definition's rules for the documents its consolidated metadata lists. Reading the consolidated metadata calls them once per declaration, and builds the models from that read.""" calls: list[int] = [] def counted(configuration: object, nested: Nested) -> Iterator[ValidationProblem]: @@ -420,13 +422,13 @@ def counted(configuration: object, nested: Nested) -> Iterator[ValidationProblem ) ZarrV3GroupMetadata(CONSOLIDATED, context=context) assert calls == [] - ZarrV3ConsolidatedMetadata(MEMBER, context=context) + ZarrV3ConsolidatedMetadata(FIELD, context=context) assert len(calls) == 3 # `a`, `b/c`, and `c` in `b`'s own listing, each one bytes codec def _listing(a: dict[str, Any]) -> dict[str, Any]: - """`MEMBER` with `a` listed at `a`.""" - return {**MEMBER, "metadata": {**MEMBER["metadata"], "a": a}} + """`FIELD` with `a` listed at `a`.""" + return {**FIELD, "metadata": {**FIELD["metadata"], "a": a}} @pytest.mark.parametrize( @@ -445,13 +447,13 @@ def _listing(a: dict[str, Any]) -> dict[str, Any]: False, ), ( - ZarrV3ConsolidatedMetadata(MEMBER), - ZarrV3ConsolidatedMetadata(MEMBER, context=CORE), + ZarrV3ConsolidatedMetadata(FIELD), + ZarrV3ConsolidatedMetadata(FIELD, context=CORE), True, ), ( - ZarrV3ConsolidatedMetadata(MEMBER), - ZarrV3ConsolidatedMetadata(MEMBER, context=PRIVATE), + ZarrV3ConsolidatedMetadata(FIELD), + ZarrV3ConsolidatedMetadata(FIELD, context=PRIVATE), False, ), ( @@ -475,7 +477,7 @@ def test_groups_and_consolidated_metadata_are_equal_by_what_they_read( right: ZarrV3GroupMetadata | ZarrV3ConsolidatedMetadata, equal: bool, ) -> None: - """A group is compared by its attributes, consolidated metadata and extra fields as JSON text, because it reads no field: no scope changes it, and a document listed in another spelling makes another group. Consolidated metadata is compared by what each document it lists means in its scope. Equal models have equal hashes.""" + """A group is compared by its attributes, consolidated metadata and extra fields as JSON text, because it reads no declaration: no scope changes it, and a document listed in another spelling makes another group. Consolidated metadata is compared by what each document it lists means in its scope. Equal models have equal hashes.""" assert (left == right) is equal if equal: assert hash(left) == hash(right) @@ -490,12 +492,12 @@ def test_a_group_round_trips_through_its_document_in_its_scope(document: dict[st def test_group_update_with_context_and_refined_in_behave_as_the_arrays_do() -> None: - """`update` reads in the group's scope and keeps the consolidated metadata unless a new one is given. A group's fields are its extra fields: in a scope that claims nothing, `consolidated_metadata` is unclaimed, and `refined_in` and `with_context` move the group to a scope that reads it as the envelope, which is a gain.""" + """`update` reads in the group's scope and keeps the consolidated metadata unless a new one is given. A group's declarations are its extra fields: in a scope that claims nothing, `consolidated_metadata` is unclaimed, and `refined_in` and `with_context` move the group to a scope that reads it as the envelope, which is a gain.""" group = ZarrV3GroupMetadata(CONSOLIDATED, context=ZarrV3Context()) assert group.update(attributes={"g": 2}).consolidated_metadata == group.consolidated_metadata - assert isinstance(group.extra_fields["consolidated_metadata"], UnclaimedField) + assert isinstance(group.extra_fields["consolidated_metadata"], UnclaimedDeclaration) for moved in (group.refined_in(CORE), group.with_context(PRIVATE)): - assert isinstance(moved.extra_fields["consolidated_metadata"], AcceptedField) + assert isinstance(moved.extra_fields["consolidated_metadata"], AcceptedDeclaration) assert moved.consolidated_metadata == group.consolidated_metadata assert moved.refines(group) assert not group.refines(moved) @@ -504,7 +506,7 @@ def test_group_update_with_context_and_refined_in_behave_as_the_arrays_do() -> N def test_error_group_refined_in_refuses_a_scope_that_loses_an_extra_field() -> None: - """A group read in `CORE` reads its `consolidated_metadata` as the envelope. A scope that leaves the member unclaimed would lose that reading, so `refined_in` raises `ScopeConflictError`, located at the member.""" + """A group read in `CORE` reads its `consolidated_metadata` as the envelope. A scope that leaves the field unclaimed would lose that reading, so `refined_in` raises `ScopeConflictError`, located at the field.""" group = ZarrV3GroupMetadata(CONSOLIDATED, context=CORE) with pytest.raises(ScopeConflictError) as raised: group.refined_in(ZarrV3Context()) @@ -513,13 +515,13 @@ def test_error_group_refined_in_refuses_a_scope_that_loses_an_extra_field() -> N def test_consolidated_metadata_with_context_and_refined_in_behave_as_the_arrays_do() -> None: """`refined_in` moves every document up the order. `with_context` reads every document in another scope.""" - held = ZarrV3ConsolidatedMetadata(MEMBER, context=ZarrV3Context()) + held = ZarrV3ConsolidatedMetadata(FIELD, context=ZarrV3Context()) refined = held.refined_in(CORE) assert refined.refines(held) assert refined.context == CORE array = refined.metadata["a"] assert isinstance(array, ZarrV3ArrayMetadata) - assert isinstance(array.codecs[0], AcceptedField) + assert isinstance(array.codecs[0], AcceptedDeclaration) private_array = held.with_context(PRIVATE).metadata["a"] assert isinstance(private_array, ZarrV3ArrayMetadata) assert private_array.codecs[0].definition == MY_BYTES @@ -527,7 +529,7 @@ def test_consolidated_metadata_with_context_and_refined_in_behave_as_the_arrays_ def test_error_consolidated_metadata_refined_in_refuses_a_conflict() -> None: """`refined_in` raises `ScopeConflictError` for a scope that reads a name of a listed document by another definition, and locates each conflict under that document's path.""" - held = ZarrV3ConsolidatedMetadata(MEMBER, context=CORE) + held = ZarrV3ConsolidatedMetadata(FIELD, context=CORE) with pytest.raises(ScopeConflictError) as raised: held.refined_in(PRIVATE) assert {conflict.loc for conflict in raised.value.conflicts} == { @@ -537,13 +539,13 @@ def test_error_consolidated_metadata_refined_in_refuses_a_conflict() -> None: def test_consolidated_metadata_reads_on_its_own() -> None: - """`ZarrV3ConsolidatedMetadata(member, context)` reads a member as written, or as a group holds it, with each document as a model in that scope. `read_consolidated_metadata_v3` returns the reading of the same model.""" - held = ZarrV3ConsolidatedMetadata(MEMBER, context=CORE) + """`ZarrV3ConsolidatedMetadata(field, context)` reads a field as written, or as a group holds it, with each document as a model in that scope. `read_consolidated_metadata_v3` returns the reading of the same model.""" + held = ZarrV3ConsolidatedMetadata(FIELD, context=CORE) group = ZarrV3GroupMetadata(CONSOLIDATED, context=CORE) assert held == ZarrV3ConsolidatedMetadata(group.consolidated_metadata, context=CORE) - assert held.to_json() == refine_json(MEMBER)[0] - assert ZarrV3ConsolidatedMetadata.from_json(MEMBER) == ZarrV3ConsolidatedMetadata(MEMBER) - reading = read_consolidated_metadata_v3(MEMBER, context=CORE) + assert held.to_json() == refine_json(FIELD)[0] + assert ZarrV3ConsolidatedMetadata.from_json(FIELD) == ZarrV3ConsolidatedMetadata(FIELD) + reading = read_consolidated_metadata_v3(FIELD, context=CORE) assert reading.problems == () assert reading.metadata == held assert held.reading.metadata is held @@ -551,17 +553,17 @@ def test_consolidated_metadata_reads_on_its_own() -> None: def test_error_consolidated_metadata_with_a_problem_is_refused_at_the_documents_path() -> None: """A problem in a listed document is reported as a problem of the consolidated metadata, located under `metadata.`. `read_consolidated_metadata_v3` returns the same problems and no model.""" - member = {**MEMBER, "metadata": {"a": {**ARRAY, "shape": [-1]}}} + field = {**FIELD, "metadata": {"a": {**ARRAY, "shape": [-1]}}} with pytest.raises(MetadataValidationError) as raised: - ZarrV3ConsolidatedMetadata(member) + ZarrV3ConsolidatedMetadata(field) assert [problem.loc for problem in raised.value.problems] == [("metadata", "a", "shape")] - reading = read_consolidated_metadata_v3(member) + reading = read_consolidated_metadata_v3(field) assert reading.problems == raised.value.problems assert reading.metadata is None -def test_the_held_field_machinery_is_gone() -> None: - """A v3 model is built only by reading its document. So nothing in the package accepts fields that were already read, or re-reads a model in an empty scope. `NO_SCOPE`, `overlapping` and `held` are removed.""" +def test_the_held_declaration_machinery_is_gone() -> None: + """A v3 model is built only by reading its document. So nothing in the package accepts declarations that were already read, or re-reads a model in an empty scope. `NO_SCOPE`, `overlapping` and `held` are removed.""" import inspect import zarr_metadata.model._array_v3 as validation @@ -580,7 +582,7 @@ def test_error_refined_in_refuses_a_gain_that_surfaces_a_problem() -> None: def test_a_scope_conflict_says_where_each_conflict_sits() -> None: - """`refined_in` reports each conflict with the field's location in the document, the same way a problem is located. For example, a loss of `bytes` is reported at `codecs.0`, including inside the shard.""" + """`refined_in` reports each conflict with the declaration's location in the document, the same way a problem is located. For example, a loss of `bytes` is reported at `codecs.0`, including inside the shard.""" model = ZarrV3ArrayMetadata(SHARDED, context=CORE) with pytest.raises(ScopeConflictError) as raised: model.refined_in(ZarrV3Context()) @@ -598,8 +600,8 @@ def test_a_scope_conflict_says_where_each_conflict_sits() -> None: @pytest.mark.parametrize( "build", [ - lambda: ZarrV3ConsolidatedMetadata(MEMBER, context=CORE_AND_EXTENSIONS), - lambda: read_consolidated_metadata_v3(MEMBER, context=CORE_AND_EXTENSIONS).metadata, + lambda: ZarrV3ConsolidatedMetadata(FIELD, context=CORE_AND_EXTENSIONS), + lambda: read_consolidated_metadata_v3(FIELD, context=CORE_AND_EXTENSIONS).metadata, ], ids=["constructor", "reader"], ) @@ -627,7 +629,7 @@ def test_with_context_moves_every_listed_model_without_reading_again( def test_consolidated_metadata_refines_nothing_of_another_type() -> None: """`refines` on consolidated metadata returns False for an object that is not consolidated metadata, as it does for arrays and groups, rather than raising.""" - held = ZarrV3ConsolidatedMetadata(MEMBER) + held = ZarrV3ConsolidatedMetadata(FIELD) assert held.refines(cast("Any", ZarrV3GroupMetadata(GROUP))) is False @@ -675,10 +677,10 @@ def test_a_models_views_cannot_be_changed_in_place( read_group_metadata_v3(GROUP), read_group_metadata_v3(CONSOLIDATED), read_group_metadata_v3({**GROUP, "attributes": 5}), - read_consolidated_metadata_v3(MEMBER), - read_consolidated_metadata_v3({**MEMBER, "kind": "sidecar"}), + read_consolidated_metadata_v3(FIELD), + read_consolidated_metadata_v3({**FIELD, "kind": "sidecar"}), ], - ids=["group", "consolidated", "group-problem", "member", "member-problem"], + ids=["group", "consolidated", "group-problem", "field", "field-problem"], ) def test_a_group_or_consolidated_reading_pickles_and_copies( reading: ZarrV3GroupMetadataReading | ZarrV3ConsolidatedMetadataReading, @@ -700,12 +702,12 @@ def test_an_array_reading_pickles_through_its_model() -> None: assert again.metadata.reading is again -def test_error_a_models_fields_cannot_be_changed_in_place() -> None: - """The fields a model returns (a codec's configuration, the fields a shard holds) are read-only, so `codecs ==` and `refines` cannot become inconsistent with `==`.""" +def test_error_a_models_declarations_cannot_be_changed_in_place() -> None: + """The declarations a model returns (a codec's configuration, the declarations a shard holds) are read-only, so `codecs ==` and `refines` cannot become inconsistent with `==`.""" model = ZarrV3ArrayMetadata(ARRAY) same = ZarrV3ArrayMetadata(ARRAY) codec = model.codecs[0] - assert isinstance(codec, AcceptedField) + assert isinstance(codec, AcceptedDeclaration) with pytest.raises(TypeError): codec.configuration["endian"] = "big" # pyright: ignore[reportIndexIssue] assert model.codecs == same.codecs diff --git a/packages/zarr-metadata/tests/model/test_pair_v2.py b/packages/zarr-metadata/tests/model/test_pair_v2.py index 400ecf9c6a..47b7e481a7 100644 --- a/packages/zarr-metadata/tests/model/test_pair_v2.py +++ b/packages/zarr-metadata/tests/model/test_pair_v2.py @@ -12,10 +12,10 @@ import pytest from zarr_metadata.definition import ( - AcceptedField, + AcceptedDeclaration, EmptyConfiguration, ScopeConflictError, - UnclaimedField, + UnclaimedDeclaration, ) from zarr_metadata.model import ( UNSET, @@ -53,7 +53,7 @@ def _document(changes: dict[str, Any]) -> dict[str, Any]: - """Return `ARRAY` with `changes` applied, leaving out any member given as `UNSET`.""" + """Return `ARRAY` with `changes` applied, leaving out any field given as `UNSET`.""" return {key: value for key, value in {**ARRAY, **changes}.items() if value is not UNSET} @@ -72,16 +72,16 @@ def test_a_model_is_its_document_read_in_its_scope() -> None: def test_properties_are_what_the_reading_holds() -> None: - """`dtype`, `compressor` and `filters` are the fields as the scope read them, or None where the document has `null`. `shape`, `chunks`, `fill_value`, `order`, `dimension_separator`, `attributes` and `extra_fields` are the members as the read refined them, and they are read-only. `zarr_format` and `node_type` are 2 and `"array"`, as a v3 array model has 3 and `"array"`.""" + """`dtype`, `compressor` and `filters` are the declarations as the scope read them, or None where the document has `null`. `shape`, `chunks`, `fill_value`, `order`, `dimension_separator`, `attributes` and `extra_fields` are the fields as the read refined them, and they are read-only. `zarr_format` and `node_type` are 2 and `"array"`, as a v3 array model has 3 and `"array"`.""" model = ZarrV2ArrayMetadata({**ARRAY, "filters": [{"id": "x"}], "extra": [1]}) - assert isinstance(model.dtype, AcceptedField) + assert isinstance(model.dtype, AcceptedDeclaration) assert model.dtype.definition is FLOAT_DATA_TYPE_V2 - assert isinstance(model.compressor, AcceptedField) + assert isinstance(model.compressor, AcceptedDeclaration) assert model.compressor.definition is ZLIB_CODEC_V2 assert model.filters is not None - assert isinstance(model.filters[0], UnclaimedField) - members = (model.shape, model.chunks, model.fill_value, model.order, model.dimension_separator) - assert members == ((4,), (2,), 0, "C", ".") + assert isinstance(model.filters[0], UnclaimedDeclaration) + fields = (model.shape, model.chunks, model.fill_value, model.order, model.dimension_separator) + assert fields == ((4,), (2,), 0, "C", ".") assert model.attributes == {"a": (1, 2)} assert model.extra_fields == {"extra": (1,)} assert (model.zarr_format, model.node_type) == (2, "array") @@ -185,10 +185,10 @@ def test_a_model_round_trips_through_its_document_pickle_and_copy( ], ids=["shape", "dtype", "unset-attributes", "null-compressor", "extra", "unset-extra"], ) -def test_update_reads_new_members_in_the_models_own_scope( +def test_update_reads_new_fields_in_the_models_own_scope( changes: dict[str, Any], expect: dict[str, Any], absent: tuple[str, ...] ) -> None: - """`update` replaces the document's members with the given JSON members, takes a member the spec does not define in `extra_fields`, leaves out any member given as `UNSET`, and reads the result in the model's own scope. The original model is unchanged.""" + """`update` replaces the document's fields with the given JSON fields, takes a field the spec does not define in `extra_fields`, leaves out any field given as `UNSET`, and reads the result in the model's own scope. The original model is unchanged.""" model = ZarrV2ArrayMetadata({**ARRAY, "extra": 0}, PRIVATE) changed = model.update(**changes) assert changed.context is PRIVATE @@ -219,9 +219,9 @@ def test_error_update_refuses_a_document_with_a_problem(changes: dict[str, Any], def test_with_context_and_refined_in_read_the_document_in_another_scope() -> None: """`with_context` reads the document in any scope, and losing information is allowed. `refined_in` only moves up the order. A scope that claims what the current scope left unclaimed is a gain. A scope that reads a name with another definition, or with no definition, raises a `ScopeConflictError` that gives the name's location. A gain that reveals a problem raises a `MetadataValidationError`.""" unclaimed = ZarrV2ArrayMetadata({**ARRAY, "compressor": {"id": "zlib"}}, SMALL) - assert isinstance(unclaimed.compressor, UnclaimedField) + assert isinstance(unclaimed.compressor, UnclaimedDeclaration) gained = unclaimed.refined_in(PRIVATE) - assert isinstance(gained.compressor, AcceptedField) + assert isinstance(gained.compressor, AcceptedDeclaration) assert gained.compressor.definition is BARE_ZLIB assert unclaimed.refines(unclaimed) assert gained.refines(unclaimed) @@ -230,7 +230,7 @@ def test_with_context_and_refined_in_read_the_document_in_another_scope() -> Non gained.refined_in(CORE_V2) assert [c.loc for c in conflict.value.conflicts] == [("compressor",)] lost = gained.with_context(SMALL) - assert isinstance(lost.compressor, UnclaimedField) + assert isinstance(lost.compressor, UnclaimedDeclaration) assert lost == unclaimed assert gained.with_context(PRIVATE) == gained with pytest.raises(MetadataValidationError): @@ -240,7 +240,7 @@ def test_with_context_and_refined_in_read_the_document_in_another_scope() -> Non def test_refines_orders_models_by_information() -> None: - """`refines` is true when each field refines its counterpart and every other member is the same, with the fill value compared as the more informed dtype spells it. A fill value that this dtype refuses is not a refinement. A value of another type refines nothing.""" + """`refines` is true when each declaration refines its counterpart and every other field is the same, with the fill value compared as the more informed dtype spells it. A fill value that this dtype refuses is not a refinement. A value of another type refines nothing.""" core = ZarrV2ArrayMetadata(ARRAY) assert core.refines(ZarrV2ArrayMetadata({**ARRAY, "fill_value": 0.0})) filled = ZarrV2ArrayMetadata({**ARRAY, "compressor": {"id": "zlib", "level": 1}}) @@ -295,7 +295,7 @@ def test_a_group_is_its_document_read_in_its_scope( def test_a_group_updates_moves_scope_and_refines_as_the_arrays_do() -> None: - """`update` reads the new attributes in the model's own scope, and `UNSET` removes them. `with_context` and `refined_in` read the document in another scope and never conflict, since a group holds no field. Two groups are equal when their attributes are written the same way, and `refines` is that equality. No `.zattrs` counts as empty attributes, so a group with none and a group with an empty one are equal and refine each other.""" + """`update` reads the new attributes in the model's own scope, and `UNSET` removes them. `with_context` and `refined_in` read the document in another scope and never conflict, since a group holds no declaration. Two groups are equal when their attributes are written the same way, and `refines` is that equality. No `.zattrs` counts as empty attributes, so a group with none and a group with an empty one are equal and refine each other.""" model = ZarrV2GroupMetadata(GROUP) assert model.update(attributes={"h": 2}).attributes == {"h": 2} removed = model.update(attributes=UNSET) @@ -316,7 +316,7 @@ def test_a_group_updates_moves_scope_and_refines_as_the_arrays_do() -> None: def test_error_a_group_document_with_a_problem_is_refused_at_construction() -> None: - """A group document with a problem -- a member the spec does not define, an attribute key that is not a string -- is refused with every problem.""" + """A group document with a problem -- a field the spec does not define, an attribute key that is not a string -- is refused with every problem.""" with pytest.raises(MetadataValidationError) as raised: ZarrV2GroupMetadata({"zarr_format": 2, "attributes": {1: "a"}, "extra": 1}) assert sorted((p.loc, p.kind) for p in raised.value.problems) == [ @@ -400,10 +400,10 @@ def test_consolidated_metadata_is_equal_by_its_nodes_and_moves_scope_with_them() assert empty.refines(ZarrV2ConsolidatedMetadata(CONSOLIDATED)) small = ZarrV2ConsolidatedMetadata(CONSOLIDATED, SMALL) assert isinstance(small.metadata["a"], ZarrV2ArrayMetadata) - assert isinstance(small.metadata["a"].compressor, UnclaimedField) + assert isinstance(small.metadata["a"].compressor, UnclaimedDeclaration) gained = small.refined_in(PRIVATE) assert isinstance(gained.metadata["a"], ZarrV2ArrayMetadata) - assert isinstance(gained.metadata["a"].compressor, AcceptedField) + assert isinstance(gained.metadata["a"].compressor, AcceptedDeclaration) assert gained.refines(small) assert not small.refines(gained) with pytest.raises(ScopeConflictError) as conflict: diff --git a/packages/zarr-metadata/tests/model/test_problem_invariants.py b/packages/zarr-metadata/tests/model/test_problem_invariants.py index b9d185207e..5882c721b5 100644 --- a/packages/zarr-metadata/tests/model/test_problem_invariants.py +++ b/packages/zarr-metadata/tests/model/test_problem_invariants.py @@ -2,7 +2,7 @@ Each test starts from valid documents of every kind a reader reads, and changes them in a few places: a value replaced with any JSON value or with -a name a document might write, a member removed, a member added, a key +a name a document might write, a field removed, a field added, a key renamed. Whatever the reader makes of the result, its problems must be located in the document it was given, and none may be reported twice. A tool places a diagnostic at a problem's `loc`, so a location the document @@ -11,7 +11,7 @@ The other tests pin one rule each, where an earlier release broke it: a typestr is one token, so each of its problems is a wrong value of the dtype, never an unknown or missing key; a float for an integer `Literal` -is the wrong type wherever the `Literal` sits; a member nothing claims +is the wrong type wherever the `Literal` sits; a field nothing claims may hold any number in both formats; and both formats refuse the same node paths. """ @@ -30,7 +30,7 @@ from zarr_metadata import UNSET from zarr_metadata._json import ValidationProblem, value_at -from zarr_metadata.definition import RefusedField +from zarr_metadata.definition import RefusedDeclaration from zarr_metadata.model import ( read_array_metadata_v2, read_array_metadata_v3, @@ -337,7 +337,7 @@ def test_a_typestr_s_problems_are_wrong_values_of_the_dtype(typestr: str, nested assert found <= {(at, "invalid_value")} # The typestr is refused exactly when it has a problem. A type code the # spec does not list is unclaimed, which is no problem. - assert isinstance(dict(reading.fields())[at], RefusedField) == (len(found) != 0) + assert isinstance(dict(reading.declarations())[at], RefusedDeclaration) == (len(found) != 0) _INTEGER_LITERALS: dict[str, tuple[Reader, dict[str, Any], tuple[str, ...]]] = { @@ -394,14 +394,14 @@ def test_a_float_for_an_integer_literal_is_the_wrong_type(which: str, value: flo @_PROPERTY -@given(member=_ANY_NUMBER) -def test_a_member_nothing_claims_may_hold_any_number_in_either_format(member: object) -> None: - """A member the spec does not define is not read, so like attributes it may hold `NaN` and the infinities, in a v2 `.zarray`, in a `.zmetadata` entry, and in a v3 document alike.""" - v2 = {**_V2_ARRAY, "unexpected": member} - v3 = {**_V3_ARRAY, "unexpected": {"must_understand": False, "value": member}} +@given(field=_ANY_NUMBER) +def test_a_field_nothing_claims_may_hold_any_number_in_either_format(field: object) -> None: + """A field the spec does not define is not read, so like attributes it may hold `NaN` and the infinities, in a v2 `.zarray`, in a `.zmetadata` entry, and in a v3 document alike.""" + v2 = {**_V2_ARRAY, "unexpected": field} + v3 = {**_V3_ARRAY, "unexpected": {"must_understand": False, "value": field}} event( "non-finite" - if any(isinstance(x, float) and not math.isfinite(x) for x in _leaves(member)) + if any(isinstance(x, float) and not math.isfinite(x) for x in _leaves(field)) else "finite" ) assert read_array_metadata_v2(v2).problems == () diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index 9f10a5c177..dfee076cf7 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -15,8 +15,8 @@ - `InstanceOf` makes pydantic's core schema an is-instance check (no field introspection), - validation goes through `from_json` (the single source of truth for what - a well-formed document is, including normalization: bare-string metadata - fields, arrays-to-tuples), + a well-formed document is, including normalization: bare-string + declarations, arrays-to-tuples), - serialization goes through `to_json` (the canonical document form). `MetadataValidationError` subclasses `ValueError`, so pydantic converts a @@ -93,7 +93,7 @@ class ArrayManifest(BaseModel): def test_raw_document_is_validated_into_a_model() -> None: """A raw metadata document on a pydantic field is parsed by from_json, - with the library's normalization applied (tuples, canonical field form).""" + with the library's normalization applied (tuples, canonical declaration form).""" manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) assert isinstance(manifest.metadata, ZarrV3ArrayMetadata) assert manifest.metadata.shape == (10,) @@ -159,7 +159,7 @@ def test_native_dataclass_introspection_is_not_supported() -> None: path needs its divergences documented again.""" from zarr_metadata.model import UNSET from zarr_metadata.typed_json import JSONValue - from zarr_metadata.v3 import ZarrV3ExtensionField, ZarrV3MetadataFieldJSON + from zarr_metadata.v3 import ZarrV3DeclarationJSON, ZarrV3ExtensionField from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON def build_and_use() -> None: @@ -169,7 +169,7 @@ def build_and_use() -> None: _types_namespace={ "JSONValue": JSONValue, "ZarrV3ExtensionField": ZarrV3ExtensionField, - "ZarrV3MetadataFieldJSON": ZarrV3MetadataFieldJSON, + "ZarrV3DeclarationJSON": ZarrV3DeclarationJSON, "ZarrV3ArrayMetadataJSON": ZarrV3ArrayMetadataJSON, "UNSET": UNSET, }, diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 1458f8be7a..362551d287 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -18,7 +18,7 @@ from pydantic.json_schema import JsonSchemaMode import zarr_metadata.pydantic as zmp -from zarr_metadata.definition import AcceptedField, UnclaimedField +from zarr_metadata.definition import AcceptedDeclaration, UnclaimedDeclaration from zarr_metadata.json_schema import json_schema, node_metadata_json_schema_v3 from zarr_metadata.model import ( ZarrV2ArrayMetadata, @@ -221,7 +221,7 @@ def test_v3_array_schema_rejects_false_at_every_extension_point(field: str) -> N _assert_runtime_and_schema_reject(zmp.ZarrV3ArrayMetadataField, doc) -def test_metadata_field_schema_rejects_unknown_members() -> None: +def test_declaration_schema_rejects_unknown_fields() -> None: """Named-configuration envelopes are closed in both runtime and schema validation.""" doc = json.loads(json.dumps(V3_ARRAY_DOC)) doc["codecs"] = [{"name": "bytes", "unexpected": 1}] @@ -236,7 +236,7 @@ def test_metadata_field_schema_rejects_unknown_members() -> None: (zmp.ZarrV2ConsolidatedMetadataField, V2_CONSOLIDATED_DOC), ], ) -def test_v2_schema_rejects_unknown_document_members( +def test_v2_schema_rejects_unknown_document_fields( field_type: object, source: dict[str, object] ) -> None: """Closed v2 merged documents expose their runtime boundary in JSON Schema.""" @@ -246,8 +246,8 @@ def test_v2_schema_rejects_unknown_document_members( _assert_runtime_and_schema_reject(field_type, doc) -def test_v2_array_schema_allows_unknown_document_members() -> None: - """The v2 array document is open ("SHOULD be ignored", https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92) in both runtime and schema validation. An unknown member is kept as written.""" +def test_v2_array_schema_allows_unknown_document_fields() -> None: + """The v2 array document is open ("SHOULD be ignored", https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92) in both runtime and schema validation. An unknown field is kept as written.""" doc = json.loads(json.dumps(V2_ARRAY_DOC)) doc["unexpected"] = 1 adapter = TypeAdapter(zmp.ZarrV2ArrayMetadataField) @@ -325,7 +325,7 @@ def test_a_v3_field_type_reads_in_the_scope_the_validation_context_holds( ) -> None: """The field type reads in the scope from the validation context, which pydantic passes to every validator. `zstd` is an extension, so `CORE` leaves it unclaimed.""" model = TypeAdapter(zmp.ZarrV3ArrayMetadataField).validate_python(_WITH_ZSTD, context=context) - assert isinstance(model.codecs[1], AcceptedField) is read + assert isinstance(model.codecs[1], AcceptedDeclaration) is read class _Bound(BaseModel): @@ -339,7 +339,7 @@ def test_a_field_bound_to_a_scope_reads_in_that_scope() -> None: for context in (None, CORE_AND_EXTENSIONS, {zmp.CONTEXT_KEY: CORE_AND_EXTENSIONS}): read = _Bound.model_validate({"metadata": _WITH_ZSTD}, context=context).metadata assert read.context == CORE - assert isinstance(read.codecs[1], UnclaimedField) + assert isinstance(read.codecs[1], UnclaimedDeclaration) held = ZarrV3ArrayMetadata(_WITH_ZSTD, context=CORE) assert _Bound(metadata=held).metadata == held assert _Bound(metadata=held).model_dump()["metadata"] == held.to_json() @@ -439,7 +439,7 @@ def test_error_an_instance_a_scope_finds_problems_in_reports_each_one(validate: @pytest.mark.parametrize("validate", _READ_IN_SCOPE) def test_error_an_instance_a_scope_reads_differently_reports_each_conflict(validate: Any) -> None: # noqa: ANN401 - a validating function - """An instance read where `zstd` is claimed would lose that meaning in `CORE`. pydantic gets one `scope_conflict` line error per conflict, at the field it is in, with that field as written as its input, and a message that names the codec.""" + """An instance read where `zstd` is claimed would lose that meaning in `CORE`. pydantic gets one `scope_conflict` line error per conflict, at the declaration it is in, with that declaration as written as its input, and a message that names the codec.""" instance = ZarrV3ArrayMetadata(_WITH_ZSTD, context=CORE_AND_EXTENSIONS) with pytest.raises(ValidationError) as raised: validate(instance, CORE) @@ -520,7 +520,7 @@ def test_error_a_consolidated_instance_a_scope_reads_differently_reports_each_co found: object, named: str, ) -> None: - """A consolidated instance read again in a scope that claims less would lose the meaning of a listed node's field, through a validation context or a field bound with `InScope`. pydantic gets one `scope_conflict` line error, located at the field inside the listed node, a node's path for v3 and its file for v2, with that field as written as its input, and a message that names the codec.""" + """A consolidated instance read again in a scope that claims less would lose the meaning of a listed node's declaration, through a validation context or a field bound with `InScope`. pydantic gets one `scope_conflict` line error, located at the declaration inside the listed node, a node's path for v3 and its file for v2, with that declaration as written as its input, and a message that names the codec.""" with pytest.raises(ValidationError) as raised: validate(instance) (line,) = raised.value.errors() @@ -608,10 +608,10 @@ def test_a_message_holding_a_ctx_placeholder_is_reported_as_it_is() -> None: assert error.get("ctx") == {"expected": (3,), "message": 'expected 3, got "{expected}"'} -def test_a_ctx_member_named_message_yields_to_a_message_holding_a_placeholder() -> None: +def test_a_ctx_key_named_message_yields_to_a_message_holding_a_placeholder() -> None: # The `message` key that carries the message goes last in the ctx, so no # ctx key after it is substituted into the carried text. When no message - # needs to be carried, a ctx member named `message` is kept. + # needs to be carried, a ctx key named `message` is kept. ctx = {"message": "theirs", "expected": 3} problem = ValidationProblem(("a",), 'got "{expected}"', "invalid_value", ctx=ctx) error = ValidationError.from_exception_data("T", [zmp._line_error(problem, {"a": 1})]).errors()[ @@ -627,10 +627,10 @@ def test_a_ctx_member_named_message_yields_to_a_message_holding_a_placeholder() def test_a_line_error_for_what_a_problem_could_not_hold_reports_what_sits_there() -> None: # A problem holds no input when the value is not JSON that a reader walks - # (here, an `AcceptedField` built by hand and placed among the codecs). The + # (here, an `AcceptedDeclaration` built by hand and placed among the codecs). The # line error then reports that object. For a missing key, the line error # reports the object that lacks the key. - smuggled = AcceptedField( + smuggled = AcceptedDeclaration( json="gzip", name="gzip", definition=GZIP_CODEC, configuration={"level": 1} ) doc = { @@ -670,14 +670,14 @@ def test_a_line_error_for_a_document_past_the_cap_renders() -> None: assert json.loads(raised.value.json())[0]["type"] == "invalid_value" -@pytest.mark.parametrize("field", ["Foo/bar", "", "r*", {"name": "Int8"}]) -def test_the_pydantic_schema_names_an_extension_as_the_reader_does(field: object) -> None: +@pytest.mark.parametrize("declaration", ["Foo/bar", "", "r*", {"name": "Int8"}]) +def test_the_pydantic_schema_names_an_extension_as_the_reader_does(declaration: object) -> None: # The schema that pydantic generates checks a name the same way the reader does. schema = TypeAdapter(zmp.ZarrV3ArrayMetadataField).json_schema() # Use arrays as lists, as JSON parses them, because `jsonschema` does not accept a tuple. document: dict[str, Any] = json.loads(json.dumps(V3_ARRAY_DOC)) assert Draft202012Validator(schema).is_valid(document) - named: dict[str, Any] = {**document, "data_type": field} + named: dict[str, Any] = {**document, "data_type": declaration} assert not Draft202012Validator(schema).is_valid(named) with pytest.raises(ValidationError): TypeAdapter(zmp.ZarrV3ArrayMetadataField).validate_python(named) @@ -685,7 +685,7 @@ def test_the_pydantic_schema_names_an_extension_as_the_reader_does(field: object def test_v2_field_types_read_in_the_validation_contexts_scope() -> None: """The v2 field types read a document in the scope that the validation context holds. The context is either a `Context` itself or holds one in its `zarr_metadata_context_v2` item. When it holds none, they read in `CORE_V2`. The v3 field types work the same way.""" - from zarr_metadata.definition import UnclaimedField + from zarr_metadata.definition import UnclaimedDeclaration from zarr_metadata.v2.data_type.scalar import UINT_DATA_TYPE_V2 from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context @@ -695,7 +695,7 @@ def test_v2_field_types_read_in_the_validation_contexts_scope() -> None: small = ZarrV2Context(data_types=[UINT_DATA_TYPE_V2]) read = adapter.validate_python({**doc, "compressor": {"id": "zlib"}}, context=small) assert read.context is small - assert isinstance(read.compressor, UnclaimedField) + assert isinstance(read.compressor, UnclaimedDeclaration) held = adapter.validate_python(doc, context={"zarr_metadata_context_v2": small}) assert held.context is small group = TypeAdapter(zmp.ZarrV2GroupMetadataField).validate_python(V2_GROUP_DOC, context=small) @@ -704,7 +704,7 @@ def test_v2_field_types_read_in_the_validation_contexts_scope() -> None: def test_each_format_reads_in_its_own_context_key() -> None: """A mapping context gives each format's scope under its own key: `zarr_metadata_context` for v3 and `zarr_metadata_context_v2` for v2. So a v3 scope given for the v3 fields leaves the v2 fields in `CORE_V2`. A bare `Context` is the scope for every field type.""" - from zarr_metadata.definition import UnclaimedField + from zarr_metadata.definition import UnclaimedDeclaration from zarr_metadata.v2.data_type.scalar import UINT_DATA_TYPE_V2 from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS @@ -722,7 +722,7 @@ def test_each_format_reads_in_its_own_context_key() -> None: assert own.context is small bare = adapter.validate_python({**doc, "compressor": {"id": "zlib"}}, context=small) assert bare.context is small - assert isinstance(bare.compressor, UnclaimedField) + assert isinstance(bare.compressor, UnclaimedDeclaration) assert zmp.CONTEXT_KEY_V2 == "zarr_metadata_context_v2" diff --git a/packages/zarr-metadata/tests/model/test_read_array_metadata.py b/packages/zarr-metadata/tests/model/test_read_array_metadata.py index 921aa19aae..c43f935881 100644 --- a/packages/zarr-metadata/tests/model/test_read_array_metadata.py +++ b/packages/zarr-metadata/tests/model/test_read_array_metadata.py @@ -1,7 +1,7 @@ -"""Tests for reading a v3 array document: each field as a scope read it, and its codecs as a pipeline. +"""Tests for reading a v3 array document: each declaration as a scope read it, and its codecs as a pipeline. `read_array_metadata_v3` returns everything that one read of a document -finds: each field with its location, the kind it was read as, and the +finds: each declaration with its location, the kind it was read as, and the scope's result for it; the chunk the codecs receive; each codec with the chunk it receives; every problem; and the document's model when there are no problems. @@ -15,14 +15,14 @@ from zarr_metadata._json import arrays_to_tuples from zarr_metadata.definition import ( - AcceptedField, + AcceptedDeclaration, Definition, - RefusedField, - UnclaimedField, + RefusedDeclaration, + UnclaimedDeclaration, resolve, with_problems, ) -from zarr_metadata.definition._definition import fields_of +from zarr_metadata.definition._definition import declarations_of from zarr_metadata.model import ( UNSET, ZarrV3ArrayMetadata, @@ -44,7 +44,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Iterator - from zarr_metadata.definition import ResolvedField + from zarr_metadata.definition import ResolvedDeclaration from zarr_metadata.typed_json import Loc from zarr_metadata.v3.definition import Lengths @@ -52,9 +52,9 @@ ZSTD = {"name": "zstd", "configuration": {"level": 1}} POINTS: list[tuple[Loc, type[Definition[Any]], type]] = [ - (("data_type",), DataTypeDefinition, AcceptedField), - (("chunk_grid",), ChunkGridDefinition, AcceptedField), - (("chunk_key_encoding",), ChunkKeyEncodingDefinition, AcceptedField), + (("data_type",), DataTypeDefinition, AcceptedDeclaration), + (("chunk_grid",), ChunkGridDefinition, AcceptedDeclaration), + (("chunk_key_encoding",), ChunkKeyEncodingDefinition, AcceptedDeclaration), ] """The single-valued extension points of a default document, each read at its location.""" @@ -65,7 +65,9 @@ def _document(shape: tuple[int, ...] = (4,), **fields: object) -> dict[str, Any] return cast("dict[str, Any]", arrays_to_tuples(document)) -def _codec(loc: Loc, variant: type = AcceptedField) -> tuple[Loc, type[Definition[Any]], type]: +def _codec( + loc: Loc, variant: type = AcceptedDeclaration +) -> tuple[Loc, type[Definition[Any]], type]: return (loc, CodecDefinition, variant) @@ -80,7 +82,7 @@ def _shard(chunk_shape: list[int], codecs: list[object] | None = None, **more: o @pytest.mark.parametrize( - ("document", "context", "fields", "handed"), + ("document", "context", "declarations", "handed"), [ (_document(), CORE_AND_EXTENSIONS, [*POINTS, _codec(("codecs", 0))], [(frozenset({4}),)]), # Each codec receives the chunk that the previous codec outputs. @@ -92,7 +94,7 @@ def _shard(chunk_shape: list[int], codecs: list[object] | None = None, **more: o [*POINTS, _codec(("codecs", 0)), _codec(("codecs", 1))], [(frozenset({4}), frozenset({6})), (frozenset({6}), frozenset({4}))], ), - # The fields a shard holds come after it, at their locations in its + # The declarations a shard holds come after it, at their locations in its # configuration. In the core spec's scope, an extension inside the # shard is unclaimed at its location. ( @@ -106,13 +108,13 @@ def _shard(chunk_shape: list[int], codecs: list[object] | None = None, **more: o *POINTS, _codec(("codecs", 0)), _codec(("codecs", 0, "configuration", "codecs", 0)), - _codec(("codecs", 0, "configuration", "codecs", 1), UnclaimedField), + _codec(("codecs", 0, "configuration", "codecs", 1), UnclaimedDeclaration), _codec(("codecs", 0, "configuration", "index_codecs", 0)), _codec(("codecs", 0, "configuration", "index_codecs", 1)), ], [(frozenset({8}), frozenset({8}))], ), - # A shard within a shard: each field comes first, then the fields it holds. + # A shard within a shard: each declaration comes first, then the declarations it holds. ( _document( (16, 16), @@ -132,7 +134,7 @@ def _shard(chunk_shape: list[int], codecs: list[object] | None = None, **more: o ], [(frozenset({8}), frozenset({8}))], ), - # A shard that its check refuses still keeps the fields it read inside it. + # A shard that its check refuses still keeps the declarations it read inside it. ( _document( (16, 16), @@ -142,9 +144,9 @@ def _shard(chunk_shape: list[int], codecs: list[object] | None = None, **more: o CORE, [ *POINTS, - _codec(("codecs", 0), RefusedField), + _codec(("codecs", 0), RefusedDeclaration), _codec(("codecs", 0, "configuration", "codecs", 0)), - _codec(("codecs", 0, "configuration", "codecs", 1), UnclaimedField), + _codec(("codecs", 0, "configuration", "codecs", 1), UnclaimedDeclaration), _codec(("codecs", 0, "configuration", "index_codecs", 0)), ], [(frozenset({8}), frozenset({8}))], @@ -171,19 +173,19 @@ def _shard(chunk_shape: list[int], codecs: list[object] | None = None, **more: o ( ("data_type", "configuration", "fields", 0, "data_type"), DataTypeDefinition, - AcceptedField, + AcceptedDeclaration, ), ( ("data_type", "configuration", "fields", 1, "data_type"), DataTypeDefinition, - UnclaimedField, + UnclaimedDeclaration, ), *POINTS[1:], _codec(("codecs", 0)), ], [(frozenset({4}),)], ), - # A field that its definition refuses, or that nothing claims, is + # A declaration that its definition refuses, or that nothing claims, is # read as the kind of its position. An unclaimed chunk grid gives no # lengths, and a codec after the array -> bytes codec receives bytes. ( @@ -195,11 +197,11 @@ def _shard(chunk_shape: list[int], codecs: list[object] | None = None, **more: o CORE_AND_EXTENSIONS, [ POINTS[0], - (("chunk_grid",), ChunkGridDefinition, UnclaimedField), + (("chunk_grid",), ChunkGridDefinition, UnclaimedDeclaration), POINTS[2], _codec(("codecs", 0)), - _codec(("codecs", 1), RefusedField), - (("storage_transformers", 0), StorageTransformerDefinition, UnclaimedField), + _codec(("codecs", 1), RefusedDeclaration), + (("storage_transformers", 0), StorageTransformerDefinition, UnclaimedDeclaration), ], [(None,), None], ), @@ -208,7 +210,7 @@ def _shard(chunk_shape: list[int], codecs: list[object] | None = None, **more: o _document(data_type=float("nan")), CORE_AND_EXTENSIONS, [ - (("data_type",), DataTypeDefinition, RefusedField), + (("data_type",), DataTypeDefinition, RefusedDeclaration), *POINTS[1:], _codec(("codecs", 0)), ], @@ -222,20 +224,22 @@ def _shard(chunk_shape: list[int], codecs: list[object] | None = None, **more: o "a-shard-within-a-shard", "a-shard-its-check-refuses", "a-struct-s-field-types", - "fields-read-as-nothing", + "declarations-read-as-nothing", "a-data-type-that-is-not-json", ], ) -def test_a_document_reads_as_each_field_where_it_sits_and_its_codecs_as_a_pipeline( +def test_a_document_reads_as_each_declaration_where_it_sits_and_its_codecs_as_a_pipeline( document: dict[str, Any], context: ZarrV3Context, - fields: list[tuple[Loc, type[Definition[Any]], type]], + declarations: list[tuple[Loc, type[Definition[Any]], type]], handed: list[Lengths | None], ) -> None: reading = read_array_metadata_v3(document, context=context) # A model exists only for a document with no problem. assert (reading.metadata is None) is (reading.problems != ()) - assert [(loc, field.read_as, type(field)) for loc, field in reading.fields()] == fields + assert [ + (loc, declaration.read_as, type(declaration)) for loc, declaration in reading.declarations() + ] == declarations # The chunk uses None for a property that nothing declares. The reading # uses `UNSET` for a key that the document lacks. if reading.data_type is UNSET: @@ -257,30 +261,30 @@ def test_a_document_reads_as_each_field_where_it_sits_and_its_codecs_as_a_pipeli {"name": "transpose", "configuration": {"order": [1, 0]}}, ], ) -"""An array document with a problem in each place where a field can have one, plus one problem outside any field.""" +"""An array document with a problem in each place where a declaration can have one, plus one problem outside any declaration.""" -def _array_fields( +def _array_declarations( document: object, -) -> Iterator[tuple[Loc, ResolvedField[Any], tuple[ValidationProblem, ...]]]: +) -> Iterator[tuple[Loc, ResolvedDeclaration[Any], tuple[ValidationProblem, ...]]]: reading = read_array_metadata_v3(document) - return with_problems(reading.fields(), reading.problems) + return with_problems(reading.declarations(), reading.problems) -def _field_fields( +def _codec_declarations( document: object, -) -> Iterator[tuple[Loc, ResolvedField[Any], tuple[ValidationProblem, ...]]]: +) -> Iterator[tuple[Loc, ResolvedDeclaration[Any], tuple[ValidationProblem, ...]]]: resolved, problems = resolve( cast("dict[str, Any]", document)["codecs"][0], CodecDefinition, CORE_AND_EXTENSIONS, ("codecs", 0), ) - return with_problems(fields_of(resolved, ("codecs", 0)), problems) + return with_problems(declarations_of(resolved, ("codecs", 0)), problems) _INNER_LEVEL = ("codecs", 0, "configuration", "codecs", 1, "configuration", "level") -_EACH_FIELD_S = { +_EACH_DECLARATION_S = { ("data_type",): [], ("chunk_grid",): [("chunk_grid", "configuration", "chunk_shape")], ("chunk_key_encoding",): [], @@ -291,39 +295,43 @@ def _field_fields( ("codecs", 1): [], ("codecs", 2): [("codecs", 2)], } -"""Each field of `_WITH_PROBLEMS`, with the location of each of its problems.""" +"""Each declaration of `_WITH_PROBLEMS`, with the location of each of its problems.""" @pytest.mark.parametrize( - ("fields", "expected"), + ("declarations", "expected"), [ - (_array_fields, _EACH_FIELD_S), + (_array_declarations, _EACH_DECLARATION_S), ( - _field_fields, - {loc: problems for loc, problems in _EACH_FIELD_S.items() if loc[:2] == ("codecs", 0)}, + _codec_declarations, + { + loc: problems + for loc, problems in _EACH_DECLARATION_S.items() + if loc[:2] == ("codecs", 0) + }, ), ], - ids=["an-array", "one-field"], + ids=["an-array", "one-declaration"], ) -def test_each_field_comes_with_the_problems_located_in_it( - fields: Callable[ - [object], Iterator[tuple[Loc, ResolvedField[Any], tuple[ValidationProblem, ...]]] +def test_each_declaration_comes_with_the_problems_located_in_it( + declarations: Callable[ + [object], Iterator[tuple[Loc, ResolvedDeclaration[Any], tuple[ValidationProblem, ...]]] ], expected: dict[Loc, list[Loc]], ) -> None: - # Each field gets the problems found while reading it, plus the problems + # Each declaration gets the problems found while reading it, plus the problems # the document check found at its location: a transpose out of the # pipeline's order, at its own position, and a chunk grid that does not - # match the shape. A field also gets the problems of the fields it + # match the shape. A declaration also gets the problems of the declarations it # holds, so a shard with a bad inner codec has that problem too. A - # problem outside any field (the fill value) belongs to no field. + # problem outside any declaration (the fill value) belongs to no declaration. assert { - loc: [p.loc for p in problems] for loc, _, problems in fields(_WITH_PROBLEMS) + loc: [p.loc for p in problems] for loc, _, problems in declarations(_WITH_PROBLEMS) } == expected -def test_a_document_with_no_problem_reads_as_its_model_holding_the_fields_read() -> None: - # The model holds the fields the read produced, not a second reading of them. +def test_a_document_with_no_problem_reads_as_its_model_holding_the_declarations_read() -> None: + # The model holds the declarations the read produced, not a second reading of them. document = _document(codecs=[{"name": "transpose", "configuration": {"order": [0]}}, LITTLE]) reading = read_array_metadata_v3(document) model = reading.metadata @@ -342,31 +350,31 @@ def test_error_a_value_that_is_not_a_mapping_reads_as_nothing() -> None: reading = read_array_metadata_v3(["not", "a", "document"]) not_an_object = ValidationProblem((), "expected an object", "invalid_type") assert reading == ZarrV3ArrayMetadataReading(problems=(not_an_object,)) - assert list(reading.fields()) == [] + assert list(reading.declarations()) == [] @pytest.mark.parametrize( - ("member", "attribute"), + ("field", "attribute"), [("codecs", "pipeline"), ("storage_transformers", "storage_transformers")], ) -def test_error_a_list_of_fields_that_is_not_a_list_reads_as_empty( - member: str, attribute: str +def test_error_a_list_of_declarations_that_is_not_a_list_reads_as_empty( + field: str, attribute: str ) -> None: document = _document() - document[member] = "bytes" + document[field] = "bytes" reading = read_array_metadata_v3(document) assert getattr(reading, attribute) == () - assert [(p.loc, p.kind) for p in reading.problems] == [((member,), "invalid_type")] + assert [(p.loc, p.kind) for p in reading.problems] == [((field,), "invalid_type")] -@pytest.mark.parametrize("member", ["data_type", "chunk_grid", "chunk_key_encoding"]) -def test_error_a_document_without_a_field_reads_it_as_unset(member: str) -> None: +@pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) +def test_error_a_document_without_a_declaration_reads_it_as_unset(field: str) -> None: document = _document() - del document[member] + del document[field] reading = read_array_metadata_v3(document) - assert getattr(reading, member) is UNSET - assert (member,) not in [loc for loc, _ in reading.fields()] - assert ValidationProblem((member,), "missing required key", "missing_key") in reading.problems + assert getattr(reading, field) is UNSET + assert (field,) not in [loc for loc, _ in reading.declarations()] + assert ValidationProblem((field,), "missing required key", "missing_key") in reading.problems def test_error_a_document_without_a_shape_hands_its_codecs_chunks_of_no_known_rank() -> None: diff --git a/packages/zarr-metadata/tests/model/test_read_array_metadata_v2.py b/packages/zarr-metadata/tests/model/test_read_array_metadata_v2.py index afbd7fe0da..133365fe1c 100644 --- a/packages/zarr-metadata/tests/model/test_read_array_metadata_v2.py +++ b/packages/zarr-metadata/tests/model/test_read_array_metadata_v2.py @@ -1,4 +1,4 @@ -"""Tests for reading a v2 array document once. A reading holds each field as the scope read it, every problem, and the model.""" +"""Tests for reading a v2 array document once. A reading holds each declaration as the scope read it, every problem, and the model.""" from __future__ import annotations @@ -7,10 +7,10 @@ import pytest from zarr_metadata.definition import ( - AcceptedField, + AcceptedDeclaration, EmptyConfiguration, - RefusedField, - UnclaimedField, + RefusedDeclaration, + UnclaimedDeclaration, ) from zarr_metadata.model import ( UNSET, @@ -34,7 +34,7 @@ @pytest.mark.parametrize( ("changes", "context", "dtype", "compressor", "filters", "locs"), [ - ({}, None, AcceptedField, None, None, [("dtype",)]), + ({}, None, AcceptedDeclaration, None, None, [("dtype",)]), ( { "dtype": " None: - """`read_array_metadata_v2` reads the document once in the given scope (`CORE_V2` by default). Each of `dtype`, `compressor` and `filters` is an `AcceptedField`, an `UnclaimedField`, or None, depending on what the document contains. `fields()` returns them in document order, with a struct's record types after the struct. A valid document produces no problems.""" + """`read_array_metadata_v2` reads the document once in the given scope (`CORE_V2` by default). Each of `dtype`, `compressor` and `filters` is an `AcceptedDeclaration`, an `UnclaimedDeclaration`, or None, depending on what the document contains. `declarations()` returns them in document order, with a struct's record types after the struct. A valid document produces no problems.""" reading = read_array_metadata_v2({**BASE, **changes}, context=context) assert reading.problems == () assert type(reading.dtype) is dtype @@ -83,7 +83,7 @@ def test_a_reading_holds_each_field_as_the_scope_read_it( assert reading.filters is not UNSET found = None if reading.filters is None else tuple(type(f) for f in reading.filters) assert found == filters - assert [loc for loc, _ in reading.fields()] == locs + assert [loc for loc, _ in reading.declarations()] == locs @pytest.mark.parametrize( @@ -102,16 +102,16 @@ def test_a_reading_holds_each_field_as_the_scope_read_it( def test_error_a_reading_reports_every_problem( value: object, problems: list[tuple[Loc, str]] ) -> None: - """Reading an invalid document reports every problem. A refused field is held as a `RefusedField`, and the reading has no model.""" + """Reading an invalid document reports every problem. A refused declaration is held as a `RefusedDeclaration`, and the reading has no model.""" reading = read_array_metadata_v2(value) assert [(p.loc, p.kind) for p in reading.problems] == problems assert reading.metadata is None if isinstance(value, dict) and "dtype" in problems[0][0]: - assert isinstance(reading.dtype, RefusedField) + assert isinstance(reading.dtype, RefusedDeclaration) def test_error_filters_that_are_not_an_array_are_not_read_as_null() -> None: - """A `filters` member that is neither null nor an array is a problem, and the reading holds no filters for it: `UNSET`, not None, because None means that the document writes `null`.""" + """A `filters` field that is neither null nor an array is a problem, and the reading holds no filters for it: `UNSET`, not None, because None means that the document writes `null`.""" reading = read_array_metadata_v2({**BASE, "filters": {"id": "crc32"}}) assert [(p.loc, p.kind) for p in reading.problems] == [(("filters",), "invalid_type")] assert reading.filters is UNSET @@ -147,7 +147,7 @@ def __eq__(self, other: object) -> bool: [("order", _EqualTo("C")), ("dimension_separator", _EqualTo("."))], ids=["order", "dimension_separator"], ) -def test_error_a_v2_member_that_only_compares_equal_to_an_allowed_value_is_refused( +def test_error_a_v2_field_that_only_compares_equal_to_an_allowed_value_is_refused( key: str, value: object ) -> None: """`order` and `dimension_separator` are checked after refinement. A value that is not a string, but compares equal to an allowed string, is outside the allowed set. It is reported as the wrong type, and no model accepts it.""" diff --git a/packages/zarr-metadata/tests/model/test_read_node_metadata_v2.py b/packages/zarr-metadata/tests/model/test_read_node_metadata_v2.py index f183b659c8..eafda48aeb 100644 --- a/packages/zarr-metadata/tests/model/test_read_node_metadata_v2.py +++ b/packages/zarr-metadata/tests/model/test_read_node_metadata_v2.py @@ -46,14 +46,14 @@ def _store(**files: object) -> dict[str, bytes]: def test_read_group_metadata_v2_reads_a_group() -> None: - """`read_group_metadata_v2` returns a reading with no problems, whose model equals the group read from the same document, and the model's `reading` is that reading. A group has no fields for a scope to read.""" + """`read_group_metadata_v2` returns a reading with no problems, whose model equals the group read from the same document, and the model's `reading` is that reading. A group has no declarations for a scope to read.""" document = {"zarr_format": 2, "attributes": {"a": 1}} reading = read_group_metadata_v2(document) assert reading.problems == () assert reading.metadata == ZarrV2GroupMetadata(document) assert reading.metadata is not None assert reading.metadata.reading == reading - assert list(reading.fields()) == [] + assert list(reading.declarations()) == [] def test_error_read_group_metadata_v2_reports_the_problems_of_a_bad_group() -> None: @@ -129,7 +129,7 @@ def test_error_files_with_both_node_files_are_no_node() -> None: def test_error_a_node_file_that_holds_attributes_is_refused_alike_in_a_store_and_a_zmetadata( name: str, document: dict[str, Any] ) -> None: - """A `.zarray` or `.zgroup` that holds `attributes` itself is refused, with a message that says the attributes belong in `.zattrs`. `read_node_metadata_v2` and `read_consolidated_metadata_v2` report the same problem at the member in the file and in its entry. The kind is `invalid_value` for both files: the member is taken by the attributes, not unknown, and a caller who tolerates an unknown key would lose the attributes it holds.""" + """A `.zarray` or `.zgroup` that holds `attributes` itself is refused, with a message that says the attributes belong in `.zattrs`. `read_node_metadata_v2` and `read_consolidated_metadata_v2` report the same problem at the field in the file and in its entry. The kind is `invalid_value` for both files: the field is taken by the attributes, not unknown, and a caller who tolerates an unknown key would lose the attributes it holds.""" held = {**document, "attributes": {}} from_files = read_node_metadata_v2({name: held}).problems from_entries = read_consolidated_metadata_v2( @@ -219,19 +219,25 @@ def test_error_a_store_file_that_is_not_json_raises_when_building_a_node() -> No assert [(p.loc, p.kind) for p in raised.value.problems] == [((".zarray",), "invalid_json")] -def test_a_v2_node_reading_locates_its_fields_in_the_files_it_read() -> None: - """A node's reading is of its files, so each field is under `.zarray`, as each problem is, and `with_problems` pairs a field with the problems in it. The model reads the merged document, so its own reading locates the same field at `dtype`. A pickled node reading keeps where its fields are.""" +def test_a_v2_node_reading_locates_its_declarations_in_the_files_it_read() -> None: + """A node's reading is of its files, so each declaration is under `.zarray`, as each problem is, and `with_problems` pairs a declaration with the problems in it. The model reads the merged document, so its own reading locates the same declaration at `dtype`. A pickled node reading keeps where its declarations are.""" struct = {**ZARRAY, "dtype": [["a", " None: def test_read_consolidated_metadata_v2_reads_a_zmetadata() -> None: - """`read_consolidated_metadata_v2` returns a reading with no problems whose model equals the one the constructor builds, and the model's `reading` is that reading. `fields()` gives each node's fields located in the `.zmetadata`.""" + """`read_consolidated_metadata_v2` returns a reading with no problems whose model equals the one the constructor builds, and the model's `reading` is that reading. `declarations()` gives each node's declarations located in the `.zmetadata`.""" reading = read_consolidated_metadata_v2(CONSOLIDATED) assert reading.problems == () assert reading.metadata == ZarrV2ConsolidatedMetadata(CONSOLIDATED) assert reading.metadata is not None assert reading.metadata.reading == reading - assert [loc for loc, _ in reading.fields()] == [("metadata", "a/.zarray", "dtype")] + assert [loc for loc, _ in reading.declarations()] == [("metadata", "a/.zarray", "dtype")] @pytest.mark.parametrize( @@ -281,13 +287,13 @@ def test_read_consolidated_metadata_v2_reads_a_zmetadata() -> None: ], ids=["path", "nested-path", "root"], ) -def test_consolidated_fields_are_located_at_the_keys_the_document_writes( +def test_consolidated_declarations_are_located_at_the_keys_the_document_writes( entries: dict[str, Any], key: str ) -> None: - """`fields()` locates each node's fields under the key that the `.zmetadata` writes for the node's `.zarray`, so each location leads into the document.""" + """`declarations()` locates each node's declarations under the key that the `.zmetadata` writes for the node's `.zarray`, so each location leads into the document.""" reading = read_consolidated_metadata_v2({"zarr_consolidated_format": 1, "metadata": entries}) assert reading.problems == () - assert [loc for loc, _ in reading.fields()] == [("metadata", key, "dtype")] + assert [loc for loc, _ in reading.declarations()] == [("metadata", key, "dtype")] def test_error_read_consolidated_metadata_v2_reports_problems_instead_of_raising() -> None: diff --git a/packages/zarr-metadata/tests/model/test_repair.py b/packages/zarr-metadata/tests/model/test_repair.py index 18d7639040..a810069b94 100644 --- a/packages/zarr-metadata/tests/model/test_repair.py +++ b/packages/zarr-metadata/tests/model/test_repair.py @@ -382,7 +382,7 @@ def test_read_repaired_v2_node_reports_what_no_repair_applies_to() -> None: ], ), (ZMETADATA_CLEAN, ZMETADATA_CLEAN, []), - # The root's .zgroup: no zarr-python version writes the member there. + # The root's .zgroup: no zarr-python version writes the field there. ( {"zarr_consolidated_format": 1, "metadata": {".zgroup": ZGROUP_FROM_ZARR3}}, {"zarr_consolidated_format": 1, "metadata": {".zgroup": ZGROUP_FROM_ZARR3}}, @@ -418,7 +418,7 @@ def test_read_repaired_v2_node_reports_what_no_repair_applies_to() -> None: def test_repair_v2_undoes_the_consolidated_metadata_zarr_3_writes_into_a_zgroup_entry( value: object, repaired: object, repairs: list[tuple[tuple[str | int, ...], str]] ) -> None: - """zarr-python 3.x writes a `consolidated_metadata` member into each non-root `.zgroup` entry of a `.zmetadata`, but the v2 group document does not allow that member. `repair_consolidated_metadata_v2` removes it and reports its location. Anything else is left unchanged, and the input document is not modified.""" + """zarr-python 3.x writes a `consolidated_metadata` field into each non-root `.zgroup` entry of a `.zmetadata`, but the v2 group document does not allow that field. `repair_consolidated_metadata_v2` removes it and reports its location. Anything else is left unchanged, and the input document is not modified.""" before = copy.deepcopy(value) document, made = repair_consolidated_metadata_v2(value) assert document == repaired @@ -429,7 +429,7 @@ def test_repair_v2_undoes_the_consolidated_metadata_zarr_3_writes_into_a_zgroup_ def test_read_repaired_consolidated_v2_reads_what_the_strict_read_refuses() -> None: - """The strict `ZarrV2ConsolidatedMetadata` refuses the `.zmetadata` that zarr-python 3.x writes, reporting the problem at the entry's member. `read_repaired_consolidated_metadata_v2` returns the strict reading of the repaired document, as `read_consolidated_metadata_v2` reads it, and the repairs, as the node repair readers do. A problem that no repair fixes is in that reading, which then holds no model.""" + """The strict `ZarrV2ConsolidatedMetadata` refuses the `.zmetadata` that zarr-python 3.x writes, reporting the problem at the entry's field. `read_repaired_consolidated_metadata_v2` returns the strict reading of the repaired document, as `read_consolidated_metadata_v2` reads it, and the repairs, as the node repair readers do. A problem that no repair fixes is in that reading, which then holds no model.""" with pytest.raises(MetadataValidationError) as raised: ZarrV2ConsolidatedMetadata(ZMETADATA_FROM_ZARR3) assert [p.loc for p in raised.value.problems] == [ diff --git a/packages/zarr-metadata/tests/model/test_scope_threading.py b/packages/zarr-metadata/tests/model/test_scope_threading.py index 1a8192d185..fb5689a555 100644 --- a/packages/zarr-metadata/tests/model/test_scope_threading.py +++ b/packages/zarr-metadata/tests/model/test_scope_threading.py @@ -116,7 +116,7 @@ def test_every_entry_point_reads_in_the_scope_it_is_given(read: Callable[..., ob @pytest.mark.parametrize("read", GROUP_ENTRY_POINTS.values(), ids=GROUP_ENTRY_POINTS.keys()) def test_a_group_reads_the_same_in_every_scope(read: Callable[..., object]) -> None: - """A group document has no fields, and a group does not read the documents its consolidated metadata lists. So each group entry point accepts a group that lists `BAD`, in every scope.""" + """A group document has no declarations, and a group does not read the documents its consolidated metadata lists. So each group entry point accepts a group that lists `BAD`, in every scope.""" accepted = _accepts(read) assert accepted(EMPTY) is True assert accepted(CORE_AND_EXTENSIONS) is True diff --git a/packages/zarr-metadata/tests/model/test_scope_threading_v2.py b/packages/zarr-metadata/tests/model/test_scope_threading_v2.py index 677889ec50..1ccb619685 100644 --- a/packages/zarr-metadata/tests/model/test_scope_threading_v2.py +++ b/packages/zarr-metadata/tests/model/test_scope_threading_v2.py @@ -36,7 +36,7 @@ def _accepts(read: Callable[..., object]) -> Callable[[Context | None], bool]: - """Return whether `read`, given `context`, finds nothing wrong. Success is a model, a reading with a model, or a field that was read or left unclaimed.""" + """Return whether `read`, given `context`, finds nothing wrong. Success is a model, a reading with a model, or a declaration that was read or left unclaimed.""" def accepted(context: Context | None) -> bool: try: @@ -44,7 +44,7 @@ def accepted(context: Context | None) -> bool: except MetadataValidationError: return False if isinstance(found, tuple) and len(found) == 2 and isinstance(found[1], tuple): - return len(found[1]) == 0 # a resolved field and its problems + return len(found[1]) == 0 # a resolved declaration and its problems reading = getattr(found, "reading", found) metadata = getattr(reading, "metadata", found) return metadata is not None diff --git a/packages/zarr-metadata/tests/model/test_store_json.py b/packages/zarr-metadata/tests/model/test_store_json.py index 411e215613..6df8208115 100644 --- a/packages/zarr-metadata/tests/model/test_store_json.py +++ b/packages/zarr-metadata/tests/model/test_store_json.py @@ -202,10 +202,10 @@ def test_attributes_holding_non_finite_numbers_round_trip_through_the_store( @pytest.mark.parametrize( - ("key", "member"), + ("key", "field"), [ ("extension", {"must_understand": False, "x": math.nan}), - # Only a v3 group holds node documents inline. In an array, a member + # Only a v3 group holds node documents inline. In an array, a field # with that name is an extra field that nothing in scope claims. ( "consolidated_metadata", @@ -214,11 +214,11 @@ def test_attributes_holding_non_finite_numbers_round_trip_through_the_store( ], ids=["extension-field", "consolidated-in-an-array"], ) -def test_an_unclaimed_member_holding_non_finite_numbers_round_trips_through_the_store( - key: str, member: dict[str, object] +def test_an_unclaimed_field_holding_non_finite_numbers_round_trips_through_the_store( + key: str, field: dict[str, object] ) -> None: - """Nothing reads a member that no definition in scope claims, so like attributes it may hold `NaN` and the infinities: the model reads it, and writes it back as bare tokens.""" - model = ZarrV3ArrayMetadata.from_key_value(_stored("zarr.json", {**_ARRAY, key: member})) + """Nothing reads a field that no definition in scope claims, so like attributes it may hold `NaN` and the infinities: the model reads it, and writes it back as bare tokens.""" + model = ZarrV3ArrayMetadata.from_key_value(_stored("zarr.json", {**_ARRAY, key: field})) written = model.to_key_value() assert b"NaN" in written["zarr.json"] assert ZarrV3ArrayMetadata.from_key_value(written) == model diff --git a/packages/zarr-metadata/tests/model/test_unclaimed_fields.py b/packages/zarr-metadata/tests/model/test_unclaimed_declarations.py similarity index 71% rename from packages/zarr-metadata/tests/model/test_unclaimed_fields.py rename to packages/zarr-metadata/tests/model/test_unclaimed_declarations.py index cef80f6c70..a9d76d3c47 100644 --- a/packages/zarr-metadata/tests/model/test_unclaimed_fields.py +++ b/packages/zarr-metadata/tests/model/test_unclaimed_declarations.py @@ -1,4 +1,4 @@ -"""`unclaimed_fields`: every field of a node that nothing in its scope claims, at any depth, which is what a reader must understand before it opens the node.""" +"""`unclaimed_declarations`: every declaration of a node that nothing in its scope claims, at any depth, which is what a reader must understand before it opens the node.""" from __future__ import annotations @@ -7,7 +7,7 @@ import pytest from typing_extensions import TypedDict -from zarr_metadata.definition import UnclaimedField +from zarr_metadata.definition import UnclaimedDeclaration from zarr_metadata.model import ( ZarrV2ArrayMetadata, ZarrV2GroupMetadata, @@ -20,7 +20,7 @@ from zarr_metadata.v3.definition import ( ArrayExtraFieldDefinition, GroupExtraFieldDefinition, - ZarrV3CodecField, + ZarrV3CodecDeclaration, ) ARRAY: dict[str, Any] = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) @@ -30,9 +30,9 @@ class _MirrorConfiguration(TypedDict, closed=True): - """The value of an `acme.mirror` member: a codec, and the waiver.""" + """The value of an `acme.mirror` field: a codec, and the waiver.""" - codec: ZarrV3CodecField + codec: ZarrV3CodecDeclaration must_understand: NotRequired[bool] @@ -42,7 +42,7 @@ class _MirrorConfiguration(TypedDict, closed=True): ArrayExtraFieldDefinition(name="acme.mirror", configuration=_MirrorConfiguration), ] ) -"""`CORE`, with a definition of an `acme.mirror` member that holds a codec.""" +"""`CORE`, with a definition of an `acme.mirror` field that holds a codec.""" GROUP_MIRRORING = CORE.update( group_extra_fields=[ @@ -50,7 +50,7 @@ class _MirrorConfiguration(TypedDict, closed=True): GroupExtraFieldDefinition(name="acme.mirror", configuration=_MirrorConfiguration), ] ) -"""`CORE`, with a definition of an `acme.mirror` member of a group that holds a codec.""" +"""`CORE`, with a definition of an `acme.mirror` field of a group that holds a codec.""" SHARD = { "name": "sharding_indexed", @@ -107,25 +107,25 @@ class _MirrorConfiguration(TypedDict, closed=True): "all-claimed", "a-codec", "a-codec-inside-a-shard", - "an-extra-member", - "a-waived-extra-member", - "a-group-member", + "an-extra-field", + "a-waived-extra-field", + "a-group-field", "a-v2-codec", "a-v2-group", - "a-codec-inside-a-claimed-member", - "a-codec-inside-a-claimed-member-that-waives", - "a-codec-inside-a-claimed-group-member", - "a-codec-inside-a-claimed-group-member-that-waives", + "a-codec-inside-a-claimed-field", + "a-codec-inside-a-claimed-field-that-waives", + "a-codec-inside-a-claimed-group-field", + "a-codec-inside-a-claimed-group-field-that-waives", ], ) -def test_unclaimed_fields_are_what_a_reader_must_understand( +def test_unclaimed_declarations_are_what_a_reader_must_understand( model: Any, # noqa: ANN401 - a model of either format unclaimed: set[tuple[str | int, ...]], ) -> None: - """`unclaimed_fields` maps the location of each field nothing in scope claims to the field, at any depth: a codec inside a shard is located inside the shard's configuration. An extra member that says `"must_understand": false` is left out with every field inside it, claimed or not, as the spec lets a reader ignore it. So a reader can open a node exactly when it is empty.""" - found = model.unclaimed_fields + """`unclaimed_declarations` maps the location of each declaration nothing in scope claims to the declaration, at any depth: a codec inside a shard is located inside the shard's configuration. An extra field that says `"must_understand": false` is left out with every declaration inside it, claimed or not, as the spec lets a reader ignore it. So a reader can open a node exactly when it is empty.""" + found = model.unclaimed_declarations assert set(found) == unclaimed - assert all(isinstance(field, UnclaimedField) for field in found.values()) + assert all(isinstance(declaration, UnclaimedDeclaration) for declaration in found.values()) @pytest.mark.parametrize( @@ -134,7 +134,7 @@ def test_unclaimed_fields_are_what_a_reader_must_understand( ids=["array", "group"], ) @pytest.mark.parametrize( - ("member", "wider", "narrower"), + ("field", "wider", "narrower"), [ ( {"codec": {"name": "gzip", "configuration": {"level": 1}}}, @@ -150,19 +150,19 @@ def test_a_model_that_refines_another_has_no_more_to_understand( kind: Any, # noqa: ANN401 - a v3 node model class document: dict[str, Any], context: Any, # noqa: ANN401 - a v3 scope - member: dict[str, Any], + field: dict[str, Any], wider: set[tuple[str | int, ...]], narrower: set[tuple[str | int, ...]], ) -> None: - """A scope that claims more of a document reads a model that refines the model of a scope that claims less, and its `unclaimed_fields` lie at or inside those of the model it refines, for an array and for a group. Here `CORE` leaves an unwaived `acme.mirror` member unclaimed as a whole; the scope that claims the member leaves at most the codec inside it, which lies inside the member, and a waived member is left out by both.""" - claimed_less = kind({**document, "acme.mirror": member}, context=CORE) - claimed_more = kind({**document, "acme.mirror": member}, context=context) + """A scope that claims more of a document reads a model that refines the model of a scope that claims less, and its `unclaimed_declarations` lie at or inside those of the model it refines, for an array and for a group. Here `CORE` leaves an unwaived `acme.mirror` field unclaimed as a whole; the scope that claims the field leaves at most the codec inside it, which lies inside the field, and a waived field is left out by both.""" + claimed_less = kind({**document, "acme.mirror": field}, context=CORE) + claimed_more = kind({**document, "acme.mirror": field}, context=context) assert claimed_more.refines(claimed_less) - assert set(claimed_less.unclaimed_fields) == wider - assert set(claimed_more.unclaimed_fields) == narrower + assert set(claimed_less.unclaimed_declarations) == wider + assert set(claimed_more.unclaimed_declarations) == narrower assert all( - any(loc[: len(outer)] == outer for outer in claimed_less.unclaimed_fields) - for loc in claimed_more.unclaimed_fields + any(loc[: len(outer)] == outer for outer in claimed_less.unclaimed_declarations) + for loc in claimed_more.unclaimed_declarations ) diff --git a/packages/zarr-metadata/tests/test_contexts.py b/packages/zarr-metadata/tests/test_contexts.py index ef8f408727..63d7a051a6 100644 --- a/packages/zarr-metadata/tests/test_contexts.py +++ b/packages/zarr-metadata/tests/test_contexts.py @@ -11,7 +11,7 @@ import pytest from typing_extensions import TypedDict -from zarr_metadata.definition import Context, ScopeConflictError, UnclaimedField, resolve +from zarr_metadata.definition import Context, ScopeConflictError, UnclaimedDeclaration, resolve from zarr_metadata.definition._context import slot_kinds from zarr_metadata.model import ( ZarrV2ArrayMetadata, @@ -54,7 +54,7 @@ def test_the_scope_of_each_format_behaves_alike( gzip: Any, # noqa: ANN401 - either gzip definition zarr_format: int, ) -> None: - """Both scope classes implement `Context` in the same way: the format they read, `claimant`, `definitions_of`, a view per slot, `update` replacing a slot, `union`, equality and hashing by what each slot files, a short repr, and pickling and copying. A field is read in either one with `resolve`.""" + """Both scope classes implement `Context` in the same way: the format they read, `claimant`, `definitions_of`, a view per slot, `update` replacing a slot, `union`, equality and hashing by what each slot files, a short repr, and pickling and copying. A declaration is read in either one with `resolve`.""" built = cls(codecs=[gzip]) assert built.zarr_format == zarr_format assert built.claimant(kind, "gzip") is gzip @@ -71,8 +71,8 @@ def test_the_scope_of_each_format_behaves_alike( assert pickle.loads(pickle.dumps(core)) == core assert copy.copy(core) is core assert copy.deepcopy(core) is core - field, _ = resolve({"id": "acme"} if zarr_format == 2 else "acme", kind, cls()) - assert isinstance(field, UnclaimedField) + declaration, _ = resolve({"id": "acme"} if zarr_format == 2 else "acme", kind, cls()) + assert isinstance(declaration, UnclaimedDeclaration) @pytest.mark.parametrize(("cls", "core", "kind", "gzip", "zarr_format"), FORMATS) diff --git a/packages/zarr-metadata/tests/test_json_schema.py b/packages/zarr-metadata/tests/test_json_schema.py index 7145af7e0c..8203f8691c 100644 --- a/packages/zarr-metadata/tests/test_json_schema.py +++ b/packages/zarr-metadata/tests/test_json_schema.py @@ -1,4 +1,4 @@ -"""JSON Schemas for what the package reads: a TypedDict, a field in a scope, and a `zarr.json`. +"""JSON Schemas for what the package reads: a TypedDict, a declaration in a scope, and a `zarr.json`. Each schema is tested against the reader it is generated from. The schema accepts every value the reader finds no problem with. The schema also @@ -28,7 +28,7 @@ resolve, ) from zarr_metadata.json_schema import ( - field_json_schema_v3, + declaration_json_schema_v3, json_schema, node_metadata_json_schema_v3, ) @@ -98,12 +98,12 @@ def _holding(annotation: object) -> type: Both = _closed("Both", {"first": Inner, "second": _closed("Inner", {"y": str})}) -def _held(member: dict[str, Any], defs: dict[str, Any] | None = None) -> dict[str, Any]: - """The schema of `_holding(annotation)`, where `member` is the schema of `annotation`.""" +def _held(field: dict[str, Any], defs: dict[str, Any] | None = None) -> dict[str, Any]: + """The schema of `_holding(annotation)`, where `field` is the schema of `annotation`.""" schema: dict[str, Any] = { "$schema": DIALECT, "type": "object", - "properties": {"value": member}, + "properties": {"value": field}, "required": ["value"], "additionalProperties": False, } @@ -413,7 +413,7 @@ def test_bounds_in_layers_are_written_as_check_holds_a_value_to_them( def _places(value: object, path: tuple[str | int, ...] = ()) -> list[tuple[str | int, ...]]: - """Every place in `value`: itself, and each member or entry inside it, depth first.""" + """Every place in `value`: itself, and each field or entry inside it, depth first.""" found = [path] if isinstance(value, dict): for key, entry in cast("dict[str, object]", value).items(): @@ -449,7 +449,7 @@ def _put(value: object, path: tuple[str | int, ...], new: object) -> object: @st.composite def _changed(draw: st.DrawFn, value: object) -> object: - """`value` changed in one or two places: an item replaced, dropped or added, a string extended by one character, or a field renamed.""" + """`value` changed in one or two places: an item replaced, dropped or added, a string extended by one character, or a declaration renamed.""" for _ in range(draw(st.integers(1, 2))): path = draw(st.sampled_from(_places(value))) here = _at(value, path) @@ -457,8 +457,8 @@ def _changed(draw: st.DrawFn, value: object) -> object: if how == "drop" and len(path) != 0: value = _put(value, path, _GONE) elif how == "add" and isinstance(here, dict): - members = cast("dict[str, object]", here) - value = _put(value, path, {**members, draw(st.sampled_from(_KEYS)): draw(_VALUES)}) + fields = cast("dict[str, object]", here) + value = _put(value, path, {**fields, draw(st.sampled_from(_KEYS)): draw(_VALUES)}) elif how == "add" and isinstance(here, list): value = _put(value, path, [*cast("list[object]", here), draw(_VALUES)]) elif how == "extend" and isinstance(here, str): @@ -473,31 +473,31 @@ def _changed(draw: st.DrawFn, value: object) -> object: _CONTEXTS = (CORE, CORE_AND_EXTENSIONS, ZarrV3Context()) -_FIELD_BASES: tuple[tuple[str, object], ...] = ( +_DECLARATION_BASES: tuple[tuple[str, object], ...] = ( *CASES, # Names that nothing claims, so nothing checks their configuration. ("codecs:acme.x", {"name": "acme.x", "configuration": {"x": [1]}}), ("data_type:acme.t", {"name": "acme.t", "configuration": {"bits": 8}}), ("chunk_grid:acme.g", {"name": "acme.g", "configuration": {"chunk_shape": [0]}}), ) -_FIELD_SCHEMAS: dict[tuple[type[ZarrV3Kind], int], Draft202012Validator] = {} +_DECLARATION_SCHEMAS: dict[tuple[type[ZarrV3Kind], int], Draft202012Validator] = {} -def _field_validator(kind: type[ZarrV3Kind], context: int) -> Draft202012Validator: - if (kind, context) not in _FIELD_SCHEMAS: - schema = field_json_schema_v3(kind, context=_CONTEXTS[context]) - _FIELD_SCHEMAS[(kind, context)] = Draft202012Validator(schema) - return _FIELD_SCHEMAS[(kind, context)] +def _declaration_validator(kind: type[ZarrV3Kind], context: int) -> Draft202012Validator: + if (kind, context) not in _DECLARATION_SCHEMAS: + schema = declaration_json_schema_v3(kind, context=_CONTEXTS[context]) + _DECLARATION_SCHEMAS[(kind, context)] = Draft202012Validator(schema) + return _DECLARATION_SCHEMAS[(kind, context)] -# --- a field in a scope ----------------------------------------------------- +# --- a declaration in a scope ----------------------------------------------- _INDEXED = {"chunk_shape": [2], "codecs": ["bytes"]} -# Each field, the kind and scope it is read in, whether the schema accepts +# Each declaration, the kind and scope it is read in, whether the schema accepts # it, and whether the scope reads it without a problem. Where the two # differ, a rule found a problem that the schema cannot express. -FIELDS: list[tuple[str, object, type[ZarrV3Kind], ZarrV3Context, bool, bool]] = [ +DECLARATIONS: list[tuple[str, object, type[ZarrV3Kind], ZarrV3Context, bool, bool]] = [ ("object", {"name": "gzip", "configuration": {"level": 5}}, CodecDefinition, CORE, True, True), ( "out-of-bounds", @@ -534,7 +534,7 @@ def _field_validator(kind: type[ZarrV3Kind], context: int) -> Draft202012Validat False, ), ( - "stray-member", + "stray-field", {"name": "gzip", "configuration": {"level": 5}, "version": 2}, CodecDefinition, CORE, @@ -697,51 +697,53 @@ def _field_validator(kind: type[ZarrV3Kind], context: int) -> Draft202012Validat @pytest.mark.parametrize( - ("field", "kind", "context", "accepted", "clean"), - [case[1:] for case in FIELDS], - ids=[case[0] for case in FIELDS], + ("declaration", "kind", "context", "accepted", "clean"), + [case[1:] for case in DECLARATIONS], + ids=[case[0] for case in DECLARATIONS], ) -def test_field_json_schema_says_what_the_scope_reads( - field: object, kind: type[ZarrV3Kind], context: ZarrV3Context, accepted: bool, clean: bool +def test_declaration_json_schema_says_what_the_scope_reads( + declaration: object, kind: type[ZarrV3Kind], context: ZarrV3Context, accepted: bool, clean: bool ) -> None: - schema = field_json_schema_v3(kind, context=context) + schema = declaration_json_schema_v3(kind, context=context) Draft202012Validator.check_schema(schema) - assert Draft202012Validator(schema).is_valid(cast("Any", field)) == accepted - assert (resolve(field, kind, context)[1] == ()) == clean + assert Draft202012Validator(schema).is_valid(cast("Any", declaration)) == accepted + assert (resolve(declaration, kind, context)[1] == ()) == clean @pytest.mark.parametrize( - ("key", "field"), CASES, ids=[f"{key}:{index}" for index, (key, _) in enumerate(CASES)] + ("key", "declaration"), CASES, ids=[f"{key}:{index}" for index, (key, _) in enumerate(CASES)] ) -def test_every_example_field_is_one_its_schema_accepts(key: str, field: object) -> None: - schema = field_json_schema_v3(KINDS[key.split(":")[0]], context=CORE_AND_EXTENSIONS) - assert Draft202012Validator(schema).is_valid(cast("Any", field)) +def test_every_example_declaration_is_one_its_schema_accepts(key: str, declaration: object) -> None: + schema = declaration_json_schema_v3(KINDS[key.split(":")[0]], context=CORE_AND_EXTENSIONS) + assert Draft202012Validator(schema).is_valid(cast("Any", declaration)) @_PROPERTY @given(st.data()) -def test_a_field_read_without_a_problem_is_one_its_schema_accepts(data: st.DataObject) -> None: +def test_a_declaration_read_without_a_problem_is_one_its_schema_accepts( + data: st.DataObject, +) -> None: # Each example is changed in one place and read in one of three scopes. # If the scope still reads it without a problem, the schema accepts it. - key, example = data.draw(st.sampled_from(_FIELD_BASES), label="example") + key, example = data.draw(st.sampled_from(_DECLARATION_BASES), label="example") kind = KINDS[key.split(":")[0]] context = data.draw(st.sampled_from(range(len(_CONTEXTS))), label="context") - field = data.draw(_changed(json.loads(json.dumps(example))), label="field") - read = resolve(field, kind, _CONTEXTS[context])[1] == () + declaration = data.draw(_changed(json.loads(json.dumps(example))), label="declaration") + read = resolve(declaration, kind, _CONTEXTS[context])[1] == () event("read without a problem" if read else "a problem") if read: - assert _field_validator(kind, context).is_valid(cast("Any", field)) + assert _declaration_validator(kind, context).is_valid(cast("Any", declaration)) -def test_error_field_json_schema_v3_refuses_a_scope_of_another_format() -> None: - """A v2 scope cannot read a v3 field, so `field_json_schema_v3` raises `TypeError` for one, as every v3 reader does.""" +def test_error_declaration_json_schema_v3_refuses_a_scope_of_another_format() -> None: + """A v2 scope cannot read a v3 declaration, so `declaration_json_schema_v3` raises `TypeError` for one, as every v3 reader does.""" with pytest.raises(TypeError, match="v3 document is read in a scope of that format"): - field_json_schema_v3(CodecDefinition, context=CORE_V2) # pyright: ignore[reportArgumentType] + declaration_json_schema_v3(CodecDefinition, context=CORE_V2) # pyright: ignore[reportArgumentType] -def test_field_json_schema_refuses_what_is_not_a_kind() -> None: +def test_declaration_json_schema_refuses_what_is_not_a_kind() -> None: with pytest.raises(TypeError, match="is not a kind of metadata"): - field_json_schema_v3(Definition, context=CORE) # pyright: ignore[reportArgumentType] + declaration_json_schema_v3(Definition, context=CORE) # pyright: ignore[reportArgumentType] # --- a zarr.json ------------------------------------------------------------ @@ -817,7 +819,7 @@ def _consolidated(**metadata: object) -> dict[str, Any]: False, ), ("an-extension-field", {**ARRAY, "acme": {"must_understand": False}}, True, True), - # Constraints across several members are checked by the rules, not the schema. + # Constraints across several fields are checked by the rules, not the schema. ("names-for-another-rank", {**ARRAY, "dimension_names": ["x"]}, True, False), ( "codecs-out-of-order", @@ -919,7 +921,7 @@ def test_every_schema_is_a_json_schema_written_the_same_each_time(context: ZarrV ChunkKeyEncodingDefinition, StorageTransformerDefinition, ): - Draft202012Validator.check_schema(field_json_schema_v3(kind, context=context)) + Draft202012Validator.check_schema(declaration_json_schema_v3(kind, context=context)) @pytest.mark.parametrize( @@ -955,16 +957,16 @@ def test_the_stricter_of_two_bounds_of_one_keyword_is_kept( assert json_schema(_holding(_annotated[Bounded, outer])) == _held(expected) -def test_a_stray_member_beside_an_unclaimed_name_is_refused_by_the_schema() -> None: +def test_a_stray_field_beside_an_unclaimed_name_is_refused_by_the_schema() -> None: # The schema rejects it, as the reader does: the envelope of an # unclaimed name holds only the name and its configuration. - field = {"name": "zfpy", "version": 2} - assert [(p.loc, p.kind) for p in resolve(field, CodecDefinition, CORE)[1]] == [ + declaration = {"name": "zfpy", "version": 2} + assert [(p.loc, p.kind) for p in resolve(declaration, CodecDefinition, CORE)[1]] == [ (("version",), "unknown_key") ] - assert not Draft202012Validator(field_json_schema_v3(CodecDefinition, context=CORE)).is_valid( - field - ) + assert not Draft202012Validator( + declaration_json_schema_v3(CodecDefinition, context=CORE) + ).is_valid(declaration) @pytest.mark.parametrize( @@ -999,11 +1001,13 @@ def test_a_stray_member_beside_an_unclaimed_name_is_refused_by_the_schema() -> N def test_the_schema_names_an_extension_as_the_reader_does(name: str) -> None: # A name matches the spec's regex or is a URI. The schema's `pattern` # accepts exactly the names that `well_named` accepts. - field = {"name": name} - _, problems = resolve(field, CodecDefinition, CORE) + declaration = {"name": name} + _, problems = resolve(declaration, CodecDefinition, CORE) accepted = len(problems) == 0 assert ( - Draft202012Validator(field_json_schema_v3(CodecDefinition, context=CORE)).is_valid(field) + Draft202012Validator(declaration_json_schema_v3(CodecDefinition, context=CORE)).is_valid( + declaration + ) is accepted ) assert accepted is ( diff --git a/packages/zarr-metadata/tests/test_problem_data.py b/packages/zarr-metadata/tests/test_problem_data.py index c2ecafacb6..de25f9e52b 100644 --- a/packages/zarr-metadata/tests/test_problem_data.py +++ b/packages/zarr-metadata/tests/test_problem_data.py @@ -275,7 +275,7 @@ def _raised(read: Callable[[], object]) -> Sequence[ValidationProblem]: "resolve", "fill-value-problems", "validate-json", - "resolve-a-field-that-is-not-json", + "resolve-a-declaration-that-is-not-json", "validate-node-name", "validate-node-path", "read-array", @@ -292,7 +292,7 @@ def test_every_problem_a_reader_finds_holds_what_the_value_handed_in_holds_there read: Callable[[object], Sequence[ValidationProblem]], value: object ) -> None: # Each problem holds part of the input value itself, not of a copy the - # reader made along the way. This holds for problems in a field, a + # reader made along the way. This holds for problems in a declaration, a # pipeline, a chunk grid checked against the shape, and a document held # by a group. A missing key holds no input, and neither does a value # that is not JSON, such as a store's bytes. @@ -311,9 +311,9 @@ class WideBits(TypedDict, closed=True): bits: Annotated[int, Le(16)] -def test_a_problem_with_what_a_name_carries_is_the_field_s() -> None: +def test_a_problem_with_what_a_name_carries_is_the_declaration_s() -> None: # `r24` carries `{"bits": 24}`, which this scope's raw-bits definition - # refuses. The problem is located at the field, because the name is what + # refuses. The problem is located at the declaration, because the name is what # the document wrote there. The bound on `bits` is not a bound on the # name, so the problem's `ctx` is empty. context = CORE_AND_EXTENSIONS.update( diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index e0095aec31..81b09fe8d4 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -41,7 +41,7 @@ def _group_rank(s: str) -> int: "ZarrV3ConsolidatedMetadataJSON", "ZarrV3ConsolidatedMetadataEnvelopeJSON", "ZarrV3NamedConfigJSON", - "ZarrV3MetadataFieldJSON", + "ZarrV3DeclarationJSON", "JSONValue", # Category A' — metadata models (in-memory dataclasses over the documents) "ZarrV2ArrayMetadata", @@ -273,10 +273,6 @@ def test_all_is_grouped_and_unique() -> None: # `CodecDefinition`, which holds a codec's name, configuration TypedDict # and rules. "Definition", - # A configuration member that holds a metadata field of that kind: for - # example, `ZarrV3CodecField`, read in the same scope as the field that - # contains it. - "Field", ) _EXTENSION_NAME = re.compile(r"^(?:[A-Z][a-z0-9]*)+?(?:" + "|".join(_EXTENSION_ROLES) + r")$") @@ -306,15 +302,15 @@ def test_all_is_grouped_and_unique() -> None: "Nested", "NodeName", "NodePath", - # The result of reading a field in a scope: `AcceptedField` (read by the - # definition that claims its name), `UnclaimedField`, or `RefusedField`. - # `ResolvedField` is the union of the three. - "AcceptedField", - "RefusedField", - "ResolvedField", + # The result of reading a declaration in a scope: `AcceptedDeclaration` (read by the + # definition that claims its name), `UnclaimedDeclaration`, or `RefusedDeclaration`. + # `ResolvedDeclaration` is the union of the three. + "AcceptedDeclaration", + "RefusedDeclaration", + "ResolvedDeclaration", "Stage", "StorageClass", - "UnclaimedField", + "UnclaimedDeclaration", "CastOutOfRangeMode", "CastRoundingMode", "Endianness", diff --git a/packages/zarr-metadata/tests/test_typed_json.py b/packages/zarr-metadata/tests/test_typed_json.py index 4aded20145..08b54ed7f0 100644 --- a/packages/zarr-metadata/tests/test_typed_json.py +++ b/packages/zarr-metadata/tests/test_typed_json.py @@ -1,8 +1,8 @@ """The checker: parsers compiled from annotations for the shapes that JSON values take. -These tests use the checker on its own, without any metadata field. A +These tests use the checker on its own, without any declaration. A leaf stands in for whatever a caller adds (the definition layer adds the -shape of a member that holds another metadata field), and `no_leaf` adds +shape of a field that holds another declaration), and `no_leaf` adds nothing. """ @@ -643,13 +643,13 @@ def _under_both_typeddicts(declarations: list[ParameterSet]) -> list[ParameterSe @pytest.mark.parametrize( - ("source", "members", "required", "extra_items", "declared", "typeddicts"), + ("source", "fields", "required", "extra_items", "declared", "typeddicts"), _under_both_typeddicts(_DECLARATIONS), ) @pytest.mark.parametrize("postponed", [False, True], ids=["evaluated", "postponed"]) def test_typeddict_keys_follow_the_typing_spec( source: str, - members: set[str], + fields: set[str], required: set[str], extra_items: str, declared: bool, @@ -665,7 +665,7 @@ def test_typeddict_keys_follow_the_typing_spec( # runtime does not record that. module = _declared(source, postponed=postponed, monkeypatch=monkeypatch, typeddicts=typeddicts) keys = typeddict_keys(module.T) - assert set(keys.members) == members + assert set(keys.fields) == fields assert keys.required == required assert keys.extra_items == eval(extra_items, vars(module)) assert keys.declared is declared @@ -1079,9 +1079,9 @@ def test_a_key_s_type_keeps_the_metadata_its_annotation_carries() -> None: # The metadata is kept, inner layer first, as `Annotated` flattens # nested layers. A note on `object` leaves the type open. keys = typeddict_keys(Bounded) - assert keys.members["level"] == (Digit, True) - assert keys.members["shape"] == (tuple[Annotated[int, Ge(1)], ...], False) - assert keys.members["name"] == ( + assert keys.fields["level"] == (Digit, True) + assert keys.fields["shape"] == (tuple[Annotated[int, Ge(1)], ...], False) + assert keys.fields["name"] == ( Annotated[str, "the inner note", Doc("the outer note")], False, ) @@ -1144,19 +1144,19 @@ def test_error_a_constraint_the_checker_does_not_read(constraint: object) -> Non parser(Annotated[str, constraint], no_leaf) -class _ConstrainedMember(TypedDict, closed=True): +class _ConstrainedField(TypedDict, closed=True): a: Annotated[int, MultipleOf(2)] -class _HoldsAConstrainedMember(TypedDict, closed=True): - inner: _ConstrainedMember +class _HoldsAConstrainedField(TypedDict, closed=True): + inner: _ConstrainedField class _NotAJSONShape(TypedDict, closed=True): a: set[int] -class _HoldsAMemberNotAJSONShape(TypedDict, closed=True): +class _HoldsAFieldNotAJSONShape(TypedDict, closed=True): inner: _NotAJSONShape @@ -1173,12 +1173,12 @@ class _HoldsAnUnresolvableTypedDict(TypedDict, closed=True): inner: _Unresolvable -class _ConstraintOnAReadableMember(TypedDict, closed=True): +class _ConstraintOnAReadableField(TypedDict, closed=True): inner: Annotated[_Readable, MultipleOf(2)] -class _ConstraintOnAConstrainedMember(TypedDict, closed=True): - inner: Annotated[_ConstrainedMember, MultipleOf(2)] +class _ConstraintOnAConstrainedField(TypedDict, closed=True): + inner: Annotated[_ConstrainedField, MultipleOf(2)] class _SelfReferentialWithAConstraint(TypedDict, closed=True): @@ -1257,11 +1257,11 @@ class _HoldsAnOptionalNotAJSONShape(TypedDict, closed=True): inner: _NotAJSONShape | None -class _HoldsATupleOfAConstrainedMember(TypedDict, closed=True): - inner: tuple[_ConstrainedMember, ...] +class _HoldsATupleOfAConstrainedField(TypedDict, closed=True): + inner: tuple[_ConstrainedField, ...] -class _ConstraintOnANotAJSONShapeMember(TypedDict, closed=True): +class _ConstraintOnANotAJSONShapeField(TypedDict, closed=True): inner: Annotated[_NotAJSONShape, MultipleOf(2)] @@ -1311,17 +1311,17 @@ class _SelfReferentialReadable(TypedDict, closed=True): ("typeddict", "unread"), [ ( - _ConstrainedMember, + _ConstrainedField, Unread( - "_ConstrainedMember.a: MultipleOf(multiple_of=2) is not a constraint the checker " + "_ConstrainedField.a: MultipleOf(multiple_of=2) is not a constraint the checker " "reads, which are Gt, Ge, Lt and Le", constraint=True, ), ), ( - _HoldsAConstrainedMember, + _HoldsAConstrainedField, Unread( - "_HoldsAConstrainedMember.inner: _ConstrainedMember.a: MultipleOf(multiple_of=2) is " + "_HoldsAConstrainedField.inner: _ConstrainedField.a: MultipleOf(multiple_of=2) is " "not a constraint the checker reads, which are Gt, Ge, Lt and Le", constraint=True, ), @@ -1335,9 +1335,9 @@ class _SelfReferentialReadable(TypedDict, closed=True): ), ), ( - _HoldsAMemberNotAJSONShape, + _HoldsAFieldNotAJSONShape, Unread( - "_HoldsAMemberNotAJSONShape.inner: _NotAJSONShape: a is not a shape JSON takes; " + "_HoldsAFieldNotAJSONShape.inner: _NotAJSONShape: a is not a shape JSON takes; " "write it as one -- a number, string, boolean, null, array, object, or an alias of " "one", constraint=False, @@ -1352,17 +1352,17 @@ class _SelfReferentialReadable(TypedDict, closed=True): ), ), ( - _ConstraintOnAReadableMember, + _ConstraintOnAReadableField, Unread( - "_ConstraintOnAReadableMember.inner: MultipleOf(multiple_of=2) is not a " + "_ConstraintOnAReadableField.inner: MultipleOf(multiple_of=2) is not a " "constraint the checker reads, which are Gt, Ge, Lt and Le", constraint=True, ), ), ( - _ConstraintOnAConstrainedMember, + _ConstraintOnAConstrainedField, Unread( - "_ConstraintOnAConstrainedMember.inner: _ConstrainedMember.a: " + "_ConstraintOnAConstrainedField.inner: _ConstrainedField.a: " "MultipleOf(multiple_of=2) is not a constraint the checker reads, which are Gt, " "Ge, Lt and Le", constraint=True, @@ -1459,18 +1459,18 @@ class _SelfReferentialReadable(TypedDict, closed=True): ), ), ( - _HoldsATupleOfAConstrainedMember, + _HoldsATupleOfAConstrainedField, Unread( - "_HoldsATupleOfAConstrainedMember.inner: _ConstrainedMember.a: " + "_HoldsATupleOfAConstrainedField.inner: _ConstrainedField.a: " "MultipleOf(multiple_of=2) is not a constraint the checker reads, which are Gt, " "Ge, Lt and Le", constraint=True, ), ), ( - _ConstraintOnANotAJSONShapeMember, + _ConstraintOnANotAJSONShapeField, Unread( - "_ConstraintOnANotAJSONShapeMember.inner: _NotAJSONShape: a is not a shape JSON " + "_ConstraintOnANotAJSONShapeField.inner: _NotAJSONShape: a is not a shape JSON " "takes; write it as one -- a number, string, boolean, null, array, object, or an " "alias of one", constraint=False, @@ -1525,24 +1525,24 @@ class _SelfReferentialReadable(TypedDict, closed=True): "not-a-json-shape", "not-a-json-shape-in-a-nested-typeddict", "a-nested-typeddict-that-cannot-be-read", - "a-constraint-on-a-member-that-is-a-typeddict", - "a-constraint-on-the-member-and-inside-its-typeddict", + "a-constraint-on-a-field-that-is-a-typeddict", + "a-constraint-on-the-field-and-inside-its-typeddict", "self-referential-with-a-constraint", "self-referential-not-a-json-shape", "self-referential-with-a-constraint-on-the-reference", "a-cycle-of-two-typeddicts", - "a-constraint-after-a-member-that-leads-back", - "not-a-json-shape-after-a-member-that-leads-back", - "a-constraint-on-a-member-that-leads-back", - "a-constraint-on-a-second-member-that-leads-back", - "a-documented-member-that-leads-back-before-a-constrained-one", - "a-constraint-after-a-member-that-leads-back-optionally", + "a-constraint-after-a-field-that-leads-back", + "not-a-json-shape-after-a-field-that-leads-back", + "a-constraint-on-a-field-that-leads-back", + "a-constraint-on-a-second-field-that-leads-back", + "a-documented-field-that-leads-back-before-a-constrained-one", + "a-constraint-after-a-field-that-leads-back-optionally", "a-typeddict-in-a-union", "a-typeddict-in-a-tuple", - "a-constraint-on-a-member-whose-typeddict-is-not-a-json-shape", + "a-constraint-on-a-field-whose-typeddict-is-not-a-json-shape", "self-referential-through-an-alias", "a-bound-on-a-tuple-that-leads-back", - "a-bound-on-a-tuple-that-leads-back-before-a-member-not-a-json-shape", + "a-bound-on-a-tuple-that-leads-back-before-a-field-not-a-json-shape", "self-referential-through-a-newtype", "an-alias-that-does-not-resolve", "self-referential-and-readable", @@ -1552,7 +1552,7 @@ class _SelfReferentialReadable(TypedDict, closed=True): def test_unread_in_names_the_part_without_a_parser_and_whether_it_is_a_constraint( typeddict: type, unread: Unread | None ) -> None: - """`unread_in` names the first part of a TypedDict that the checker has no parser for, down to the TypedDict that contains it, also through a union or a container, and says whether it is a constraint the checker does not read, so a definition can advise checking it in its `rules`. A member that is not a JSON shape is not such a constraint. A nested TypedDict that cannot be read at all is named with the reason. When both a member and the TypedDict it holds have such a part, the one inside is named first, as the checker meets it first. A TypedDict that holds itself, directly or through another, is named where its own members are unread, and a member that leads back, also through a union, is named only when nothing else is, unless metadata of its own, such as a constraint but not a `Doc`, is unread. A TypedDict the checker reads whole has no unread part.""" + """`unread_in` names the first part of a TypedDict that the checker has no parser for, down to the TypedDict that contains it, also through a union or a container, and says whether it is a constraint the checker does not read, so a definition can advise checking it in its `rules`. A field that is not a JSON shape is not such a constraint. A nested TypedDict that cannot be read at all is named with the reason. When both a field and the TypedDict it holds have such a part, the one inside is named first, as the checker meets it first. A TypedDict that holds itself, directly or through another, is named where its own fields are unread, and a field that leads back, also through a union, is named only when nothing else is, unless metadata of its own, such as a constraint but not a `Doc`, is unread. A TypedDict the checker reads whole has no unread part.""" assert unread_in(typeddict, no_leaf) == unread @@ -1803,4 +1803,4 @@ class _Three(TypedDict, closed=True): def test_metadata_of_three_annotated_layers_is_kept_in_order() -> None: - assert typeddict_keys(_Three).members["x"] == (Annotated[int, "inner", "middle", "outer"], True) + assert typeddict_keys(_Three).fields["x"] == (Annotated[int, "inner", "middle", "outer"], True) diff --git a/packages/zarr-metadata/tests/test_typed_json_properties.py b/packages/zarr-metadata/tests/test_typed_json_properties.py index b21ed65096..d875b4ec8a 100644 --- a/packages/zarr-metadata/tests/test_typed_json_properties.py +++ b/packages/zarr-metadata/tests/test_typed_json_properties.py @@ -23,13 +23,14 @@ from __future__ import annotations import copy +import dataclasses import itertools import json import sys import types import typing from collections.abc import Mapping -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Annotated, Any, Literal, NewType, NotRequired, Required, TypeAlias, Union, cast import pytest @@ -84,7 +85,7 @@ class Named: @dataclass(frozen=True) class Itself: - """The TypedDict that contains the member: a class that contains itself.""" + """The TypedDict that contains the field: a class that contains itself.""" @dataclass(frozen=True) @@ -97,7 +98,7 @@ class ExtraItems: @dataclass(frozen=True) -class Member: +class Field: key: str spec: Spec wrappers: tuple[str, ...] @@ -109,7 +110,7 @@ class Layer: """One class in a TypedDict's ancestry: its own keys and keywords, and how its module writes its annotations.""" total: bool - members: tuple[Member, ...] + fields: tuple[Field, ...] says: Says postponed: bool = False """In a mixed build, whether the class's annotations are left as strings.""" @@ -128,7 +129,7 @@ class Object: Spec: TypeAlias = Leaf | OneOf | ArrayOf | Fixed | AnyOf | MappingOf | Named | Itself | Object -def members_of(spec: Object) -> dict[str, tuple[Spec, bool]]: +def fields_of(spec: Object) -> dict[str, tuple[Spec, bool]]: """Each key the TypedDict declares, its spec, and whether it is required, by the typing spec. A `Required` or `NotRequired` qualifier decides; otherwise the `total` @@ -136,15 +137,15 @@ def members_of(spec: Object) -> dict[str, tuple[Spec, bool]]: """ found: dict[str, tuple[Spec, bool]] = {} for layer in spec.layers: - for member in layer.members: - found[member.key] = (member.spec, _required(member, layer.total)) + for field in layer.fields: + found[field.key] = (field.spec, _required(field, layer.total)) return found -def _required(member: Member, total: bool) -> bool: - if "Required" in member.wrappers: +def _required(field: Field, total: bool) -> bool: + if "Required" in field.wrappers: return True - if "NotRequired" in member.wrappers: + if "NotRequired" in field.wrappers: return False return total @@ -197,14 +198,14 @@ def conforms(value: object, spec: Spec, itself: Object | None = None) -> bool: return conforms(value, itself, itself) if not isinstance(value, dict): return False - members = members_of(spec) - for key, (member, required) in members.items(): + fields = fields_of(spec) + for key, (field, required) in fields.items(): if key in value: - if not conforms(value[key], member, spec): + if not conforms(value[key], field, spec): return False elif required: return False - others = [key for key in value if key not in members] + others = [key for key in value if key not in fields] said = says_of(spec) if said == "closed": return len(others) == 0 @@ -262,7 +263,7 @@ class Build: """A spec built from source text, with each class in its own module and the names it uses imported there.""" mode: Mode - source: list[str] = field(default_factory=list) + source: list[str] = dataclasses.field(default_factory=list) """The source text of what was built, for the notes of a failing example.""" def annotation(self, spec: Spec) -> object: @@ -306,9 +307,9 @@ def text(self, spec: Spec, itself: str | None, home: types.ModuleType, *, quoted name, built = self.typeddict(spec) return self._named(name, built, home, quoted=quoted) - def member(self, member: Member, itself: str, home: types.ModuleType, *, quoted: bool) -> str: - text = self.text(member.spec, itself, home, quoted=quoted) - for wrapper in member.wrappers: + def field(self, field: Field, itself: str, home: types.ModuleType, *, quoted: bool) -> str: + text = self.text(field.spec, itself, home, quoted=quoted) + for wrapper in field.wrappers: text = f"Annotated[{text}, 'meta']" if wrapper == "Annotated" else f"{wrapper}[{text}]" return text @@ -323,8 +324,8 @@ def typeddict(self, spec: Object) -> tuple[str, object]: postponed = self.mode == "postponed" or (self.mode == "mixed" and layer.postponed) quoted = not postponed and layer.quoted texts = { - member.key: self.member(member, names[-1], home, quoted=quoted) - for member in layer.members + field.key: self.field(field, names[-1], home, quoted=quoted) + for field in layer.fields } annotations = ( texts if postponed else {key: _evaluated(text, home) for key, text in texts.items()} @@ -393,7 +394,7 @@ def _objects(draw: st.DrawFn, inner: st.SearchStrategy[Spec]) -> Object: owners = [draw(st.integers(0, count - 1)) for _ in keys] layers: list[Layer] = [] said: Says = "unsaid" - declared: dict[str, tuple[Member, bool]] = {} + declared: dict[str, tuple[Field, bool]] = {} for index in range(count): # A subclass of a closed TypedDict, or of one with extra items, is # limited to what the typing spec allows: it may repeat the same @@ -404,26 +405,26 @@ def _objects(draw: st.DrawFn, inner: st.SearchStrategy[Spec]) -> Object: choices.append(ExtraItems(draw(inner))) says = draw(st.sampled_from(choices)) total = draw(st.booleans()) - members = [ - Member(key, draw(inner), draw(_wrappers())) + fields = [ + Field(key, draw(inner), draw(_wrappers())) for key, owner in zip(keys, owners, strict=True) if owner == index and not restricted ] if not restricted: - members += draw(_redeclared(declared, total)) + fields += draw(_redeclared(declared, total)) if index == count - 1 and not restricted and draw(st.booleans()): - members.append(Member("self", ArrayOf(Itself()), draw(_wrappers()))) + fields.append(Field("self", ArrayOf(Itself()), draw(_wrappers()))) postponed, quoted = draw(st.booleans()), draw(st.booleans()) - layers.append(Layer(total, tuple(members), says, postponed, quoted)) - declared.update({member.key: (member, _required(member, total)) for member in members}) + layers.append(Layer(total, tuple(fields), says, postponed, quoted)) + declared.update({field.key: (field, _required(field, total)) for field in fields}) said = says if says != "unsaid" else said speaks = any(layer.says != "unsaid" for layer in layers) # From 3.13, `typing.TypedDict` rejects a read-only key that redeclares a # key it thinks is mutable, and it cannot see `ReadOnly` in a postponed # base. So it would not build some classes the spec allows, and no key is # declared twice when it is used. - redeclares = sum(len(layer.members) for layer in layers) > len( - {member.key for layer in layers for member in layer.members} + redeclares = sum(len(layer.fields) for layer in layers) > len( + {field.key for layer in layers for field in layer.fields} ) stdlib = (_TYPING_TAKES_PEP_728 or not speaks) and not redeclares and draw(st.booleans()) return Object(tuple(layers), stdlib) @@ -431,16 +432,14 @@ def _objects(draw: st.DrawFn, inner: st.SearchStrategy[Spec]) -> Object: @st.composite def _redeclared( - draw: st.DrawFn, declared: dict[str, tuple[Member, bool]], total: bool -) -> list[Member]: + draw: st.DrawFn, declared: dict[str, tuple[Field, bool]], total: bool +) -> list[Field]: """No key, or one read-only key declared by a base, narrowed as the typing spec allows a subclass to. Its type may narrow (a union to some of its branches), and a key that was not required may become required. The reverse is not allowed. """ - candidates = sorted( - key for key, (member, _) in declared.items() if "ReadOnly" in member.wrappers - ) + candidates = sorted(key for key, (field, _) in declared.items() if "ReadOnly" in field.wrappers) if len(candidates) == 0 or not draw(st.booleans()): return [] key = draw(st.sampled_from(candidates)) @@ -451,9 +450,9 @@ def _redeclared( branches = tuple(spec.branches[index] for index in kept) spec = branches[0] if len(branches) == 1 else AnyOf(branches) wrappers = draw(_wrappers()) - if was_required and not _required(Member(key, spec, wrappers), total): + if was_required and not _required(Field(key, spec, wrappers), total): wrappers = ("Required", *(wrapper for wrapper in wrappers if wrapper != "NotRequired")) - return [Member(key, spec, wrappers)] + return [Field(key, spec, wrappers)] def specs(depth: int) -> st.SearchStrategy[Spec]: @@ -520,13 +519,13 @@ def conforming(spec: Spec, depth: int, itself: Object | None = None) -> st.Searc @st.composite def _object_values(draw: st.DrawFn, spec: Object, depth: int) -> dict[str, object]: - members = members_of(spec) + fields = fields_of(spec) value: dict[str, object] = {} - for key, (member, required) in members.items(): + for key, (field, required) in fields.items(): if required or draw(st.booleans()): - value[key] = draw(conforming(member, depth - 1, spec)) + value[key] = draw(conforming(field, depth - 1, spec)) said = says_of(spec) - others = st.text(max_size=3).filter(lambda key: key not in members) + others = st.text(max_size=3).filter(lambda key: key not in fields) if said in ("unsaid", "open"): value.update(draw(st.dictionaries(others, json_values(depth - 1), max_size=2))) elif isinstance(said, ExtraItems): @@ -727,7 +726,7 @@ def test_typeddict_keys_follow_the_typing_spec(data: st.DataObject) -> None: # not state it. The result follows the typing spec, whether the # annotations are evaluated or not. spec = data.draw(_objects(specs(_DEPTH - 1)), label="spec") - members = members_of(spec) + fields = fields_of(spec) said = says_of(spec) event(f"{len(spec.layers)} classes") event(f"says {said if isinstance(said, str) else 'extra_items'}") @@ -735,8 +734,8 @@ def test_typeddict_keys_follow_the_typing_spec(data: st.DataObject) -> None: build = Build(mode) keys = typeddict_keys(cast("type", build.annotation(spec))) note(build.text_of()) - assert set(keys.members) == set(members) - assert keys.required == {key for key, (_, required) in members.items() if required} + assert set(keys.fields) == set(fields) + assert keys.required == {key for key, (_, required) in fields.items() if required} assert keys.closed == (said == "closed") assert keys.open == (said in ("unsaid", "open")) assert keys.declared == (said != "unsaid") @@ -752,7 +751,7 @@ def test_a_key_a_closed_typeddict_does_not_declare_is_reported_and_left_out( spec = data.draw( _objects(specs(_DEPTH - 1)).filter(lambda drawn: says_of(drawn) == "closed"), label="spec" ) - declared = members_of(spec) + declared = fields_of(spec) value = data.draw(_object_values(spec, _DEPTH), label="value") key = data.draw(st.text(max_size=3).filter(lambda drawn: drawn not in declared), label="key") build = Build(data.draw(st.sampled_from(_MODES), label="mode")) @@ -786,9 +785,9 @@ def _hides_a_qualifier(spec: Object) -> bool: return any( layer.postponed and any( - ("NotRequired" in member.wrappers and layer.total) - or ("Required" in member.wrappers and not layer.total) - for member in layer.members + ("NotRequired" in field.wrappers and layer.total) + or ("Required" in field.wrappers and not layer.total) + for field in layer.fields ) for layer in spec.layers ) @@ -796,9 +795,9 @@ def _hides_a_qualifier(spec: Object) -> bool: _REACHES = { "a-key-redeclared": lambda spec: any( - member.key in {earlier.key for layer in spec.layers[:index] for earlier in layer.members} + field.key in {earlier.key for layer in spec.layers[:index] for earlier in layer.fields} for index, later in enumerate(spec.layers) - for member in later.members + for field in later.fields ), "openness-inherited": lambda spec: ( spec.layers[-1].says == "unsaid" and any(layer.says != "unsaid" for layer in spec.layers) @@ -811,12 +810,12 @@ def _hides_a_qualifier(spec: Object) -> bool: "a-quoted-name-a-subclass-inherits": lambda spec: ( len(spec.layers) > 1 and any( - layer.quoted and any(_nests_a_name(member.spec) for member in layer.members) + layer.quoted and any(_nests_a_name(field.spec) for field in layer.fields) for layer in spec.layers[:-1] ) ), "a-class-that-holds-itself": lambda spec: any( - member.key == "self" for layer in spec.layers for member in layer.members + field.key == "self" for layer in spec.layers for field in layer.fields ), } diff --git a/packages/zarr-metadata/tests/v2/test_codecs.py b/packages/zarr-metadata/tests/v2/test_codecs.py index 8405842773..0db277f03c 100644 --- a/packages/zarr-metadata/tests/v2/test_codecs.py +++ b/packages/zarr-metadata/tests/v2/test_codecs.py @@ -8,9 +8,9 @@ import pytest from zarr_metadata.definition import ( - AcceptedField, - RefusedField, - UnclaimedField, + AcceptedDeclaration, + RefusedDeclaration, + UnclaimedDeclaration, canonical_of, resolve, ) @@ -97,7 +97,9 @@ "adler32": ({"id": "adler32"},), "fletcher32": ({"id": "fletcher32"},), } -CASES = [(name, field) for name, fields in EXAMPLES.items() for field in fields] +CASES = [ + (name, declaration) for name, declarations in EXAMPLES.items() for declaration in declarations +] def test_every_codec_has_an_example() -> None: @@ -106,17 +108,17 @@ def test_every_codec_has_an_example() -> None: @pytest.mark.parametrize( - ("name", "field"), CASES, ids=[f"{n}:{i}" for i, (n, _) in enumerate(CASES)] + ("name", "declaration"), CASES, ids=[f"{n}:{i}" for i, (n, _) in enumerate(CASES)] ) def test_every_example_reads_and_is_written_back_as_it_was( - name: str, field: ZarrV2CodecMetadata + name: str, declaration: ZarrV2CodecMetadata ) -> None: """A numcodecs configuration is read by the definition its `id` names, with no problem, and is written back unchanged: the `id` next to the parameters. Its canonical form reads as the same codec.""" - resolved, problems = resolve(field, ZarrV2CodecDefinition, CONTEXT) - assert isinstance(resolved, AcceptedField) + resolved, problems = resolve(declaration, ZarrV2CodecDefinition, CONTEXT) + assert isinstance(resolved, AcceptedDeclaration) assert resolved.definition.name == name assert problems == () - assert resolved.to_json() == field + assert resolved.to_json() == declaration assert resolve(canonical_of(resolved, problems), ZarrV2CodecDefinition, CONTEXT)[0] == resolved @@ -131,7 +133,7 @@ def test_every_example_reads_and_is_written_back_as_it_was( @pytest.mark.parametrize( - ("field", "canonical"), + ("declaration", "canonical"), [ ({"id": "zlib"}, {"id": "zlib", "level": 1}), ({"id": "gzip"}, {"id": "gzip", "level": 1}), @@ -203,10 +205,10 @@ def test_every_example_reads_and_is_written_back_as_it_was( ], ) def test_the_canonical_form_fills_in_defaults_and_writes_each_value_one_way( - field: dict[str, Any], canonical: dict[str, Any] + declaration: dict[str, Any], canonical: dict[str, Any] ) -> None: """The canonical form of a codec fills in each parameter that numcodecs gives a default, where the configuration omits it or writes null, as `get_config()` writes it. A blosc `shuffle` of `true` or `false` is 1 or 0, which numcodecs reads it as, and an `astype` left out is the `dtype`. It writes a dtype parameter as NumPy spells it (` None: - """A codec id that no definition in scope claims is read as `UnclaimedField`. Its parameters are kept and not checked, as for an unclaimed v3 extension.""" - resolved, problems = resolve(field, ZarrV2CodecDefinition, CONTEXT) - assert isinstance(resolved, UnclaimedField) +def test_an_id_the_package_does_not_model_is_unclaimed(declaration: dict[str, Any]) -> None: + """A codec id that no definition in scope claims is read as `UnclaimedDeclaration`. Its parameters are kept and not checked, as for an unclaimed v3 extension.""" + resolved, problems = resolve(declaration, ZarrV2CodecDefinition, CONTEXT) + assert isinstance(resolved, UnclaimedDeclaration) assert problems == () - assert json.dumps(resolved.to_json()) == json.dumps(field) + assert json.dumps(resolved.to_json()) == json.dumps(declaration) @pytest.mark.parametrize( - ("field", "at", "kind"), + ("declaration", "at", "kind"), [ ({"id": "zlib", "level": 10}, ("c", "level"), "invalid_value"), ({"id": "gzip", "level": -2}, ("c", "level"), "invalid_value"), @@ -317,9 +319,11 @@ def test_an_id_the_package_does_not_model_is_unclaimed(field: dict[str, Any]) -> ], ) def test_error_a_parameter_outside_what_numcodecs_takes_is_a_problem( - field: dict[str, Any], at: Loc, kind: str + declaration: dict[str, Any], at: Loc, kind: str ) -> None: - """Each of these is reported at the parameter's location inside the field: a parameter out of the range numcodecs reads, a parameter of the wrong type, a parameter that is missing when numcodecs has no default for it, a dtype parameter that an array's dtype could not be, and a key that no codec declares.""" - resolved, problems = resolve(field, ZarrV2CodecDefinition, CONTEXT, ("c",)) + """Each of these is reported at the parameter's location inside the declaration: a parameter out of the range numcodecs reads, a parameter of the wrong type, a parameter that is missing when numcodecs has no default for it, a dtype parameter that an array's dtype could not be, and a key that no codec declares.""" + resolved, problems = resolve(declaration, ZarrV2CodecDefinition, CONTEXT, ("c",)) assert [(p.loc, p.kind) for p in problems] == [(at, kind)] - assert isinstance(resolved, AcceptedField if kind == "unknown_key" else RefusedField) + assert isinstance( + resolved, AcceptedDeclaration if kind == "unknown_key" else RefusedDeclaration + ) diff --git a/packages/zarr-metadata/tests/v2/test_data_types.py b/packages/zarr-metadata/tests/v2/test_data_types.py index 29778ebfa0..5d852ee472 100644 --- a/packages/zarr-metadata/tests/v2/test_data_types.py +++ b/packages/zarr-metadata/tests/v2/test_data_types.py @@ -9,15 +9,15 @@ from zarr_metadata import UNSET from zarr_metadata._json import value_at from zarr_metadata.definition import ( - AcceptedField, - RefusedField, - UnclaimedField, + AcceptedDeclaration, + RefusedDeclaration, + UnclaimedDeclaration, canonical_fill_value, canonical_of, fill_value_problems, resolve, ) -from zarr_metadata.definition._definition import fields_of +from zarr_metadata.definition._definition import declarations_of from zarr_metadata.v2.data_type import DATA_TYPES_V2 from zarr_metadata.v2.definition import ZarrV2Context, ZarrV2DataTypeDefinition @@ -77,7 +77,7 @@ def _read(value: object) -> tuple[type, list[tuple[Loc, str]]]: def test_every_family_reads_its_typestrs(value: object, family: str, canonical: object) -> None: """Each typestr of a family is read by the family's definition. Its canonical spelling is the typestr NumPy writes: `|` for a type of one byte or with no byte order, and `us` for a microsecond unit. A records array is read as `struct`, and the type of each record is read too. A records array may be empty.""" resolved, problems = resolve(value, ZarrV2DataTypeDefinition, CONTEXT) - assert isinstance(resolved, AcceptedField) + assert isinstance(resolved, AcceptedDeclaration) assert resolved.definition.name == family assert problems == () assert canonical_of(resolved, problems) == canonical @@ -85,9 +85,9 @@ def test_every_family_reads_its_typestrs(value: object, family: str, canonical: @pytest.mark.parametrize("value", [" None: - """A typestr whose type code is not listed in the v2 spec is filed under its own name, which no definition in scope claims. It is read as `UnclaimedField` with no problem, and written back unchanged.""" + """A typestr whose type code is not listed in the v2 spec is filed under its own name, which no definition in scope claims. It is read as `UnclaimedDeclaration` with no problem, and written back unchanged.""" resolved, problems = resolve(value, ZarrV2DataTypeDefinition, CONTEXT) - assert isinstance(resolved, UnclaimedField) + assert isinstance(resolved, UnclaimedDeclaration) assert problems == () assert resolved.to_json() == value @@ -148,9 +148,9 @@ def test_a_type_code_the_spec_does_not_list_is_unclaimed(value: str) -> None: ], ) def test_error_a_typestr_the_family_does_not_take_is_a_problem(value: object, at: Loc) -> None: - """A typestr with a size, byte order or unit that its family does not accept is refused with one `invalid_value` problem, located at the field. A struct's problems are located at the record and the position inside it, where the document writes them. If a record's type is refused, the problem is reported there and the struct is still read.""" + """A typestr with a size, byte order or unit that its family does not accept is refused with one `invalid_value` problem, located at the declaration. A struct's problems are located at the record and the position inside it, where the document writes them. If a record's type is refused, the problem is reported there and the struct is still read.""" kind, problems = _read(value) - assert kind is RefusedField or len(at) > 1 + assert kind is RefusedDeclaration or len(at) > 1 assert next(loc for loc, _ in problems) == at if isinstance(value, str): # A typestr is one token: one wrong value, never a key a reader may skip. @@ -274,11 +274,11 @@ def test_error_a_struct_record_s_problem_holds_the_value_the_document_writes_the def test_a_struct_record_type_sits_where_the_document_writes_it() -> None: - """`fields_of` places a record's type where the document writes it, `("dtype", 0, 1)`: under neither a `configuration` key nor a `fields` key, which a v2 document does not have.""" + """`declarations_of` places a record's type where the document writes it, `("dtype", 0, 1)`: under neither a `configuration` key nor a `fields` key, which a v2 document does not have.""" resolved, _ = resolve( [["a", " None: assert pickle.loads(pickle.dumps(CORE_V2)) == CORE_V2 -def test_resolve_reads_one_v2_field_in_a_v2_scope() -> None: - """`resolve` reads a v2 dtype or codec with its kind in a v2 scope, prefixing every problem with `loc`, as it reads a v3 field.""" +def test_resolve_reads_one_v2_declaration_in_a_v2_scope() -> None: + """`resolve` reads a v2 dtype or codec with its kind in a v2 scope, prefixing every problem with `loc`, as it reads a v3 declaration.""" dtype, problems = resolve(" None: - """A dtype field is either a string, which is its name, or an array of field records, which is a `struct` whose configuration is the records. Any other value has no name, and so has the string `struct`: it is not a typestr, and only records read as a struct.""" + """A dtype declaration is either a string, which is its name, or an array of field records, which is a `struct` whose configuration is the records. Any other value has no name, and so has the string `struct`: it is not a typestr, and only records read as a struct.""" assert ZarrV2DataTypeDefinition._named_configuration(value) == (name, configuration, ()) @@ -115,7 +115,7 @@ def test_a_v2_dtype_is_a_string_or_records( def test_a_v2_dtype_envelope_is_a_typestr_or_records( value: object, problems: list[tuple[Loc, str]] ) -> None: - """The envelope of a dtype field is valid when it is a typestr or an array of records. A string that is neither, `struct` included, is reported as an invalid value, and the message names the typestr form. Any other value is reported as an invalid type.""" + """The envelope of a dtype declaration is valid when it is a typestr or an array of records. A string that is neither, `struct` included, is reported as an invalid value, and the message names the typestr form. Any other value is reported as an invalid type.""" found = ZarrV2DataTypeDefinition._envelope_problems(value) assert [(p.loc, p.kind) for p in found] == problems if value == "float32": @@ -123,7 +123,7 @@ def test_a_v2_dtype_envelope_is_a_typestr_or_records( def test_a_v2_dtype_writes_back_as_a_string_or_records() -> None: - """`envelope_json` writes a dtype as its string, and a struct as its records. The name is located at the field, and so is each record: a place under the struct's `fields` is located without `fields`.""" + """`envelope_json` writes a dtype as its string, and a struct as its records. The name is located at the declaration, and so is each record: a place under the struct's `fields` is located without `fields`.""" assert ZarrV2DataTypeDefinition._envelope_json(" None: - """A codec field is an object. Its `id` is its name and its other members are its parameters. A value that is not an object, or has no string `id`, has no name, and the envelope problems say why.""" + """A codec declaration is an object. Its `id` is its name and its other fields are its parameters. A value that is not an object, or has no string `id`, has no name, and the envelope problems say why.""" assert ZarrV2CodecDefinition._named_configuration(value) == (name, configuration, ()) assert [(p.loc, p.kind) for p in ZarrV2CodecDefinition._envelope_problems(value)] == problems def test_a_v2_codec_writes_back_with_its_id_beside_its_parameters() -> None: - """`envelope_json` writes a codec as `{"id": name, **parameters}`. The parameters are located at the field and the name is located under `id`.""" + """`envelope_json` writes a codec as `{"id": name, **parameters}`. The parameters are located at the declaration and the name is located under `id`.""" assert ZarrV2CodecDefinition._envelope_json("zlib", {"level": 1}) == {"id": "zlib", "level": 1} assert ZarrV2CodecDefinition._in_configuration(("compressor",), ("level",)) == ( "compressor", diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py index 90f3f809ac..8a9c057182 100644 --- a/packages/zarr-metadata/tests/v3/test_definitions.py +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -1,4 +1,4 @@ -"""Reading one metadata field against its definition: type-checked, checked by rules, and read in a scope.""" +"""Reading one declaration against its definition: type-checked, checked by rules, and read in a scope.""" from __future__ import annotations @@ -23,13 +23,13 @@ ) from zarr_metadata._json import refine_json from zarr_metadata.definition import ( - AcceptedField, + AcceptedDeclaration, Context, Definition, EmptyConfiguration, Nested, - RefusedField, - UnclaimedField, + RefusedDeclaration, + UnclaimedDeclaration, canonical_of, configuration_of, resolve, @@ -43,8 +43,8 @@ from zarr_metadata.typed_json import ( check, ) -from zarr_metadata.v2.definition import CORE_V2, ZarrV2CodecDefinition, ZarrV2DataTypeField -from zarr_metadata.v3 import ZarrV3MetadataFieldJSON +from zarr_metadata.v2.definition import CORE_V2, ZarrV2CodecDefinition, ZarrV2DataTypeDeclaration +from zarr_metadata.v3 import ZarrV3DeclarationJSON from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID from zarr_metadata.v3.codec.gzip import GZIP_CODEC, GzipCodecConfiguration @@ -58,7 +58,7 @@ CodecDefinition, DataTypeDefinition, StorageTransformerDefinition, - ZarrV3CodecField, + ZarrV3CodecDeclaration, ZarrV3Context, ) @@ -69,13 +69,13 @@ class AcmeStackConfiguration(TypedDict, closed=True): """A third-party container: a codec that holds a pipeline of codecs.""" - codecs: tuple[ZarrV3CodecField, ...] + codecs: tuple[ZarrV3CodecDeclaration, ...] def acme_stack_rules( configuration: AcmeStackConfiguration, nested: Nested ) -> Iterator[ValidationProblem]: - # A rule may read a nested field's name, because rules run only on a + # A rule may read a nested declaration's name, because rules run only on a # configuration whose nested envelopes are valid. for index, codec in enumerate(configuration["codecs"]): if (codec if isinstance(codec, str) else codec["name"]) == "acme.stack": @@ -85,7 +85,11 @@ def acme_stack_rules( # The rule also uses what the scope read: a codec of another kind is # refused, and a codec that nothing in scope claims is not checked. inner = nested.get(("codecs", index)) - definition = inner.definition if isinstance(inner, (AcceptedField, RefusedField)) else None + definition = ( + inner.definition + if isinstance(inner, (AcceptedDeclaration, RefusedDeclaration)) + else None + ) if isinstance(definition, CodecDefinition) and definition.kind != "bytes_bytes": yield ValidationProblem( ("codecs", index), "a stack holds bytes -> bytes codecs", "invalid_value" @@ -119,7 +123,7 @@ class AcmeTreeConfiguration(TypedDict, closed=True): class ByCodec(TypedDict, closed=True): - codec: ZarrV3CodecField + codec: ZarrV3CodecDeclaration weight: int @@ -129,12 +133,12 @@ class Raw(TypedDict, closed=True): class AcmeFallbackConfiguration(TypedDict, closed=True): - """A union whose branches disagree on whether `codec` is a metadata field.""" + """A union whose branches disagree on whether `codec` is a declaration.""" fallback: ByCodec | Raw -class AcmeRoutesConfiguration(TypedDict, extra_items=ZarrV3CodecField): +class AcmeRoutesConfiguration(TypedDict, extra_items=ZarrV3CodecDeclaration): """Every key is a route, and holds the codec for that route.""" @@ -216,8 +220,10 @@ def acme_paired_rules( ) -INT8: Final = AcceptedField(json="int8", name="int8", definition=INT8_DATA_TYPE, configuration={}) -"""An `int8` field as a scope reads it: read by its definition, with no nested fields.""" +INT8: Final = AcceptedDeclaration( + json="int8", name="int8", definition=INT8_DATA_TYPE, configuration={} +) +"""An `int8` declaration as a scope reads it: read by its definition, with no nested declarations.""" def _locs(problems: tuple[ValidationProblem, ...]) -> list[tuple[tuple[str | int, ...], str]]: @@ -230,7 +236,7 @@ def test_core_is_a_subset_of_core_and_extensions() -> None: assert CORE.definitions_of(kind).items() <= CORE_AND_EXTENSIONS.definitions_of(kind).items() -def test_a_read_field_keeps_the_fields_it_read_inside() -> None: +def test_a_read_declaration_keeps_the_declarations_it_read_inside() -> None: # Keyed by location in the configuration, as the scope read them. shard = { "name": "sharding_indexed", @@ -241,7 +247,7 @@ def test_a_read_field_keeps_the_fields_it_read_inside() -> None: }, } resolved, _ = resolve(shard, CodecDefinition, CORE_AND_EXTENSIONS) - assert isinstance(resolved, AcceptedField) + assert isinstance(resolved, AcceptedDeclaration) assert {loc: inner.json for loc, inner in resolved.nested.items()} == { ("codecs", 0): "bytes", ("index_codecs", 0): "bytes", @@ -249,22 +255,24 @@ def test_a_read_field_keeps_the_fields_it_read_inside() -> None: } cast_value = {"name": "cast_value", "configuration": {"data_type": "int8"}} resolved, _ = resolve(cast_value, CodecDefinition, CORE_AND_EXTENSIONS) - assert isinstance(resolved, AcceptedField) + assert isinstance(resolved, AcceptedDeclaration) assert resolved.nested[("data_type",)] == INT8 - # A field with no nested fields has nothing inside. A field that nothing - # in scope claims holds no fields at all. A field refused by its check - # or rules keeps the nested fields it read. + # A declaration with no nested declarations has nothing inside. A declaration that nothing + # in scope claims holds no declarations at all. A declaration refused by its check + # or rules keeps the nested declarations it read. assert resolve("int8", DataTypeDefinition, CORE)[0] == INT8 unclaimed = {"name": "acme.cast", "configuration": {"data_type": "int8"}} - assert isinstance(resolve(unclaimed, CodecDefinition, CORE_AND_EXTENSIONS)[0], UnclaimedField) + assert isinstance( + resolve(unclaimed, CodecDefinition, CORE_AND_EXTENSIONS)[0], UnclaimedDeclaration + ) refused = {"name": "cast_value", "configuration": {"data_type": "int8", "rounding": 1}} resolved, _ = resolve(refused, CodecDefinition, CORE_AND_EXTENSIONS) - assert isinstance(resolved, RefusedField) + assert isinstance(resolved, RefusedDeclaration) assert resolved.nested[("data_type",)] == INT8 @pytest.mark.parametrize( - ("field", "kind", "name"), + ("declaration", "kind", "name"), [ ("int8", DataTypeDefinition, "int8"), ({"name": "gzip", "configuration": {"level": 1}}, CodecDefinition, "gzip"), @@ -274,12 +282,12 @@ def test_a_read_field_keeps_the_fields_it_read_inside() -> None: ({"configuration": {}}, CodecDefinition, None), (5, CodecDefinition, None), ], - ids=["bare-name", "object", "raw-bits", "out-of-scope", "no-name", "not-a-field"], + ids=["bare-name", "object", "raw-bits", "out-of-scope", "no-name", "not-a-declaration"], ) -def test_a_reading_says_the_name_the_field_was_written_with( - field: JSONValue, kind: type[Definition[Any]], name: str | None +def test_a_reading_says_the_name_the_declaration_was_written_with( + declaration: JSONValue, kind: type[Definition[Any]], name: str | None ) -> None: - assert resolve(field, kind, CORE_AND_EXTENSIONS)[0].name == name + assert resolve(declaration, kind, CORE_AND_EXTENSIONS)[0].name == name KINDLESS: Final = Definition(name="acme.kindless", configuration=EmptyConfiguration) @@ -292,18 +300,18 @@ def test_a_reading_says_the_name_the_field_was_written_with( (INT8, DataTypeDefinition), # A name read by a definition filed under a different name, as raw bits are. ( - AcceptedField( + AcceptedDeclaration( json="r16", name="r16", definition=RAW_BYTES_DATA_TYPE, configuration={"bits": 16} ), DataTypeDefinition, ), # Type arguments are dropped, as `resolve` drops them. ( - UnclaimedField(json="acme.t", name="acme.t", read_as=DataTypeDefinition[Any]), + UnclaimedDeclaration(json="acme.t", name="acme.t", read_as=DataTypeDefinition[Any]), DataTypeDefinition, ), ( - RefusedField( + RefusedDeclaration( json="int8", name="int8", read_as=DataTypeDefinition[Any], @@ -311,45 +319,46 @@ def test_a_reading_says_the_name_the_field_was_written_with( ), DataTypeDefinition, ), - # Not a field, so it has no name and nothing claims it. - (RefusedField(json=5, name=None, read_as=CodecDefinition), CodecDefinition), + # Not a declaration, so it has no name and nothing claims it. + (RefusedDeclaration(json=5, name=None, read_as=CodecDefinition), CodecDefinition), ], ids=["read", "raw-bits", "unclaimed", "refused", "refused-nameless"], ) -def test_a_field_built_by_hand_is_of_the_kind_it_says( - built: AcceptedField[Any] | UnclaimedField | RefusedField[Any], read_as: type[Definition[Any]] +def test_a_declaration_built_by_hand_is_of_the_kind_it_says( + built: AcceptedDeclaration[Any] | UnclaimedDeclaration | RefusedDeclaration[Any], + read_as: type[Definition[Any]], ) -> None: assert built.read_as is read_as class _HoldsAV2DataType(TypedDict, closed=True): - """A v3 configuration with a member that holds a v2 data type field.""" + """A v3 configuration with a field that holds a v2 data type declaration.""" - dt: ZarrV2DataTypeField + dt: ZarrV2DataTypeDeclaration class _HoldsAV3Codec(TypedDict, closed=True): - """A v2 configuration with a member that holds a v3 codec field.""" + """A v2 configuration with a field that holds a v3 codec declaration.""" - inner: ZarrV3CodecField + inner: ZarrV3CodecDeclaration class _HoldsV2DataTypes(TypedDict, closed=True): - """A v3 configuration with a member that holds an array of v2 data type fields.""" + """A v3 configuration with a field that holds an array of v2 data type declarations.""" - dts: tuple[ZarrV2DataTypeField, ...] + dts: tuple[ZarrV2DataTypeDeclaration, ...] class _HoldsANestedV2DataType(TypedDict, closed=True): - """A v3 configuration with a member that holds an object with a v2 data type field.""" + """A v3 configuration with a field that holds an object with a v2 data type declaration.""" nested: _HoldsAV2DataType class _HoldsV3Codecs(TypedDict, closed=True): - """A v2 configuration with a member that holds an array of v3 codec fields.""" + """A v2 configuration with a field that holds an array of v3 codec declarations.""" - inner: tuple[ZarrV3CodecField, ...] + inner: tuple[ZarrV3CodecDeclaration, ...] @pytest.mark.parametrize( @@ -359,17 +368,17 @@ class _HoldsV3Codecs(TypedDict, closed=True): lambda: CodecDefinition( name="acme.mix", configuration=_HoldsAV2DataType, kind="array_array", size="static" ), - "ZarrV2DataTypeField", + "ZarrV2DataTypeDeclaration", ), ( lambda: ZarrV2CodecDefinition(name="acme_mix", configuration=_HoldsAV3Codec), - "ZarrV3CodecField", + "ZarrV3CodecDeclaration", ), ( lambda: CodecDefinition( name="acme.mix", configuration=_HoldsV2DataTypes, kind="array_array", size="static" ), - "ZarrV2DataTypeField", + "ZarrV2DataTypeDeclaration", ), ( lambda: CodecDefinition( @@ -378,11 +387,11 @@ class _HoldsV3Codecs(TypedDict, closed=True): kind="array_array", size="static", ), - "ZarrV2DataTypeField", + "ZarrV2DataTypeDeclaration", ), ( lambda: ZarrV2CodecDefinition(name="acme_mix", configuration=_HoldsV3Codecs), - "ZarrV3CodecField", + "ZarrV3CodecDeclaration", ), ], ids=[ @@ -393,20 +402,20 @@ class _HoldsV3Codecs(TypedDict, closed=True): "v2-holding-an-array-of-v3", ], ) -def test_error_a_configuration_holding_a_field_of_the_other_format( +def test_error_a_configuration_holding_a_declaration_of_the_other_format( build: Callable[[], object], named: str ) -> None: - """A definition reads the fields of its own format only: a document of one format never holds a field of the other. A configuration with a member that holds the other format's field alias, directly, as the items of an array, or inside a nested TypedDict, is refused when the definition is built, with a `TypeError` that names the alias, so no read of it fails part way.""" + """A definition reads the declarations of its own format only: a document of one format never holds a declaration of the other. A configuration with a field that holds the other format's declaration alias, directly, as the items of an array, or inside a nested TypedDict, is refused when the definition is built, with a `TypeError` that names the alias, so no read of it fails part way.""" with pytest.raises(TypeError, match=named): build() @pytest.mark.parametrize("definition", [None, KINDLESS], ids=["none", "kindless"]) -def test_error_a_field_read_by_hand_by_what_is_not_a_definition_of_a_kind( +def test_error_a_declaration_read_by_hand_by_what_is_not_a_definition_of_a_kind( definition: Definition[Any] | None, ) -> None: - with pytest.raises(TypeError, match="a field read is read by a definition of a kind"): - AcceptedField( + with pytest.raises(TypeError, match="a declaration read is read by a definition of a kind"): + AcceptedDeclaration( json="acme.kindless", name="acme.kindless", definition=definition, # pyright: ignore[reportArgumentType] @@ -415,53 +424,59 @@ def test_error_a_field_read_by_hand_by_what_is_not_a_definition_of_a_kind( @pytest.mark.parametrize("name", ["int16", "r16"]) -def test_error_a_field_read_by_hand_by_a_definition_filed_under_another_name(name: str) -> None: +def test_error_a_declaration_read_by_hand_by_a_definition_filed_under_another_name( + name: str, +) -> None: with pytest.raises( - TypeError, match=f"a field named '{name}' is read by the definition filed under it" + TypeError, match=f"a declaration named '{name}' is read by the definition filed under it" ): - AcceptedField(json=name, name=name, definition=INT8_DATA_TYPE, configuration={}) + AcceptedDeclaration(json=name, name=name, definition=INT8_DATA_TYPE, configuration={}) -def test_error_a_field_refused_by_hand_by_a_definition_of_another_kind() -> None: +def test_error_a_declaration_refused_by_hand_by_a_definition_of_another_kind() -> None: with pytest.raises( TypeError, match="read as a DataTypeDefinition is read by one, got CodecDefinition" ): - RefusedField(json="gzip", name="gzip", read_as=DataTypeDefinition, definition=GZIP_CODEC) + RefusedDeclaration( + json="gzip", name="gzip", read_as=DataTypeDefinition, definition=GZIP_CODEC + ) @pytest.mark.parametrize("name", ["int16", None]) -def test_error_a_field_refused_by_hand_by_a_definition_filed_under_another_name( +def test_error_a_declaration_refused_by_hand_by_a_definition_filed_under_another_name( name: str | None, ) -> None: with pytest.raises( - TypeError, match=f"a field named {name!r} is read by the definition filed under it" + TypeError, match=f"a declaration named {name!r} is read by the definition filed under it" ): - RefusedField(json=name, name=name, read_as=DataTypeDefinition, definition=INT8_DATA_TYPE) + RefusedDeclaration( + json=name, name=name, read_as=DataTypeDefinition, definition=INT8_DATA_TYPE + ) @pytest.mark.parametrize( "build", [ - lambda: UnclaimedField(json="acme.t", name="acme.t", read_as=Definition), - lambda: RefusedField(json="acme.t", name="acme.t", read_as=Definition), + lambda: UnclaimedDeclaration(json="acme.t", name="acme.t", read_as=Definition), + lambda: RefusedDeclaration(json="acme.t", name="acme.t", read_as=Definition), ], ids=["unclaimed", "refused"], ) -def test_error_a_field_built_by_hand_of_no_kind(build: Callable[[], object]) -> None: +def test_error_a_declaration_built_by_hand_of_no_kind(build: Callable[[], object]) -> None: with pytest.raises(TypeError, match="is not a kind of metadata"): build() -def test_error_a_field_nothing_claims_built_by_hand_without_a_name() -> None: - with pytest.raises(TypeError, match="a field nothing in scope claims is named"): - UnclaimedField(json=5, name=None, read_as=CodecDefinition) # pyright: ignore[reportArgumentType] +def test_error_a_declaration_nothing_claims_built_by_hand_without_a_name() -> None: + with pytest.raises(TypeError, match="a declaration nothing in scope claims is named"): + UnclaimedDeclaration(json=5, name=None, read_as=CodecDefinition) # pyright: ignore[reportArgumentType] @pytest.mark.parametrize( - ("field", "kind", "written"), + ("declaration", "kind", "written"), [ # A data type with nothing to configure is written as its bare name, - # however it was written; every other field is written as an object. + # however it was written; every other declaration is written as an object. ("int8", DataTypeDefinition, "int8"), ( {"name": "int8", "configuration": {}, "must_understand": True}, @@ -475,7 +490,7 @@ def test_error_a_field_nothing_claims_built_by_hand_without_a_name() -> None: ChunkGridDefinition, {"name": "regular", "configuration": {"chunk_shape": (2, 3)}}, ), - # Each field held in a configuration is written the same way. + # Each declaration held in a configuration is written the same way. ( { "name": "sharding_indexed", @@ -500,8 +515,8 @@ def test_error_a_field_nothing_claims_built_by_hand_without_a_name() -> None: CodecDefinition, {"name": "cast_value", "configuration": {"data_type": "int8"}}, ), - # Inside it, an unclaimed field is written the same way, and a - # refused field is written as it was. + # Inside it, an unclaimed declaration is written the same way, and a + # refused declaration is written as it was. ( { "name": "acme.stack", @@ -551,49 +566,49 @@ def test_error_a_field_nothing_claims_built_by_hand_without_a_name() -> None: "unclaimed-data-type", ], ) -def test_a_field_is_written_as_every_reader_takes_it( - field: JSONValue, kind: type[Definition[Any]], written: JSONValue +def test_a_declaration_is_written_as_every_reader_takes_it( + declaration: JSONValue, kind: type[Definition[Any]], written: JSONValue ) -> None: - resolved, _ = resolve(field, kind, CONTEXT) - assert isinstance(resolved, (AcceptedField, UnclaimedField)) + resolved, _ = resolve(declaration, kind, CONTEXT) + assert isinstance(resolved, (AcceptedDeclaration, UnclaimedDeclaration)) assert resolved.to_json() == written @pytest.mark.parametrize( - ("field", "kind", "variant", "configuration", "problems"), + ("declaration", "kind", "variant", "configuration", "problems"), [ ( {"name": "gzip", "configuration": {"level": 5}}, CodecDefinition, - AcceptedField, + AcceptedDeclaration, {"level": 5}, [], ), - ("crc32c", CodecDefinition, AcceptedField, {}, []), - ({"name": "crc32c"}, CodecDefinition, AcceptedField, {}, []), - ({"name": "crc32c", "configuration": {}}, CodecDefinition, AcceptedField, {}, []), - ({"name": "crc32c", "must_understand": True}, CodecDefinition, AcceptedField, {}, []), - ("bytes", CodecDefinition, AcceptedField, {}, []), + ("crc32c", CodecDefinition, AcceptedDeclaration, {}, []), + ({"name": "crc32c"}, CodecDefinition, AcceptedDeclaration, {}, []), + ({"name": "crc32c", "configuration": {}}, CodecDefinition, AcceptedDeclaration, {}, []), + ({"name": "crc32c", "must_understand": True}, CodecDefinition, AcceptedDeclaration, {}, []), + ("bytes", CodecDefinition, AcceptedDeclaration, {}, []), ( {"name": "bytes", "configuration": {"endian": "big"}}, CodecDefinition, - AcceptedField, + AcceptedDeclaration, {"endian": "big"}, [], ), ( {"name": "regular", "configuration": {"chunk_shape": [2, 3]}}, ChunkGridDefinition, - AcceptedField, + AcceptedDeclaration, {"chunk_shape": (2, 3)}, [], ), # An unknown key is not fatal: it is reported and left out of the - # configuration, and the field is still read. + # configuration, and the declaration is still read. ( {"name": "gzip", "configuration": {"level": 5, "extra": 1}}, CodecDefinition, - AcceptedField, + AcceptedDeclaration, {"level": 5}, [(("configuration", "extra"), "unknown_key")], ), @@ -601,19 +616,19 @@ def test_a_field_is_written_as_every_reader_takes_it( ( {"name": "acme.bounded", "configuration": {"level": 5, "windw": 10}}, CodecDefinition, - AcceptedField, + AcceptedDeclaration, {"level": 5}, [(("configuration", "windw"), "unknown_key")], ), # A key that is not required, written as a string under postponed # annotations, and every key of a `total=False` TypedDict. - ("acme.level", CodecDefinition, AcceptedField, {}, []), - ("acme.total", CodecDefinition, AcceptedField, {}, []), + ("acme.level", CodecDefinition, AcceptedDeclaration, {}, []), + ("acme.total", CodecDefinition, AcceptedDeclaration, {}, []), # A kind with type arguments is the same kind. ( {"name": "gzip", "configuration": {"level": 5}}, CodecDefinition[Any], - AcceptedField, + AcceptedDeclaration, {"level": 5}, [], ), @@ -623,11 +638,11 @@ def test_a_field_is_written_as_every_reader_takes_it( "configuration": {"label": "a", "children": [{"label": "b", "children": []}]}, }, CodecDefinition, - AcceptedField, + AcceptedDeclaration, {"label": "a", "children": ({"label": "b", "children": ()},)}, [], ), - # Only the union branch that matched has its nested fields read. Here + # Only the union branch that matched has its nested declarations read. Here # `codec` is plain JSON, not a gzip with a missing configuration. ( { @@ -635,27 +650,33 @@ def test_a_field_is_written_as_every_reader_takes_it( "configuration": {"fallback": {"codec": {"name": "gzip"}, "note": "x"}}, }, CodecDefinition, - AcceptedField, + AcceptedDeclaration, {"fallback": {"codec": {"name": "gzip"}, "note": "x"}}, [], ), - # `extra_items` is a field alias: every other key holds a codec, + # `extra_items` is a declaration alias: every other key holds a codec, # stored as a document writes it. ( {"name": "acme.routes", "configuration": {"fast": "crc32c"}}, CodecDefinition, - AcceptedField, + AcceptedDeclaration, {"fast": {"name": "crc32c"}}, [], ), # Nothing in scope claims it, so it is not checked and not refused. - ({"name": "zfpy", "configuration": {"x": 1}}, CodecDefinition, UnclaimedField, None, []), - # A nested field is read in the same scope and stored as a document - # writes it. A nested field that is out of scope is not checked. + ( + {"name": "zfpy", "configuration": {"x": 1}}, + CodecDefinition, + UnclaimedDeclaration, + None, + [], + ), + # A nested declaration is read in the same scope and stored as a document + # writes it. A nested declaration that is out of scope is not checked. ( {"name": "acme.stack", "configuration": {"codecs": ["crc32c", "zfpy"]}}, CodecDefinition, - AcceptedField, + AcceptedDeclaration, {"codecs": ({"name": "crc32c"}, {"name": "zfpy"})}, [], ), @@ -676,22 +697,22 @@ def test_a_field_is_written_as_every_reader_takes_it( "kind-with-type-arguments", "recursive", "union-branch-that-read", - "extra-items-of-fields", + "extra-items-of-declarations", "out-of-scope", "nested", ], ) -def test_a_field_is_read_in_scope( - field: object, +def test_a_declaration_is_read_in_scope( + declaration: object, kind: type[Definition[Any]], variant: type, configuration: dict[str, object] | None, problems: list[tuple[tuple[str | int, ...], str]], ) -> None: - resolved, found = resolve(field, kind, CONTEXT) + resolved, found = resolve(declaration, kind, CONTEXT) assert type(resolved) is variant assert ( - resolved.configuration if isinstance(resolved, AcceptedField) else None + resolved.configuration if isinstance(resolved, AcceptedDeclaration) else None ) == configuration assert _locs(found) == problems @@ -700,29 +721,29 @@ def test_error_a_rule_refuses_a_value() -> None: resolved, found = resolve( {"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CONTEXT ) - assert isinstance(resolved, RefusedField) + assert isinstance(resolved, RefusedDeclaration) assert resolved.definition is GZIP_CODEC assert _locs(found) == [(("configuration", "level"), "invalid_value")] -def test_error_a_member_of_the_wrong_type_is_not_asked_of_the_rules() -> None: +def test_error_a_field_of_the_wrong_type_is_not_asked_of_the_rules() -> None: resolved, found = resolve( {"name": "gzip", "configuration": {"level": "5"}}, CodecDefinition, CONTEXT ) - assert isinstance(resolved, RefusedField) + assert isinstance(resolved, RefusedDeclaration) assert _locs(found) == [(("configuration", "level"), "invalid_type")] def test_error_a_required_configuration_is_missing() -> None: resolved, found = resolve("gzip", CodecDefinition, CONTEXT) - assert isinstance(resolved, RefusedField) + assert isinstance(resolved, RefusedDeclaration) assert _locs(found) == [(("configuration",), "missing_key")] def test_error_the_configuration_is_not_an_object() -> None: - # The result is a RefusedField, still claimed by the definition its name refers to. + # The result is a RefusedDeclaration, still claimed by the definition its name refers to. resolved, found = resolve({"name": "gzip", "configuration": 5}, CodecDefinition, CONTEXT) - assert resolved == RefusedField( + assert resolved == RefusedDeclaration( json={"name": "gzip", "configuration": 5}, name="gzip", read_as=CodecDefinition, @@ -732,17 +753,17 @@ def test_error_the_configuration_is_not_an_object() -> None: def test_error_must_understand_false_is_refused() -> None: - # The envelope's problem is reported with the field. The configuration + # The envelope's problem is reported with the declaration. The configuration # was read, so a later layer can still check the codec. resolved, found = resolve( {"name": "crc32c", "must_understand": False}, CodecDefinition, CONTEXT ) - assert isinstance(resolved, AcceptedField) + assert isinstance(resolved, AcceptedDeclaration) assert _locs(found) == [(("must_understand",), "invalid_value")] @pytest.mark.parametrize( - ("member", "kind"), + ("field", "kind"), [ ("data_type", DataTypeDefinition), ("chunk_grid", ChunkGridDefinition), @@ -752,21 +773,21 @@ def test_error_must_understand_false_is_refused() -> None: ], ) def test_error_must_understand_false_is_refused_wherever_the_model_refuses_it( - member: str, kind: type[Definition[Any]] + field: str, kind: type[Definition[Any]] ) -> None: - # One interpretation of the spec, applied by two readers: reading a field + # One interpretation of the spec, applied by two readers: reading a declaration # here, and the model's validator. If only one of them changes, this fails. - field: dict[str, JSONValue] = {"name": "acme.example", "must_understand": False} - listed = member in ("codecs", "storage_transformers") + declaration: dict[str, JSONValue] = {"name": "acme.example", "must_understand": False} + listed = field in ("codecs", "storage_transformers") document: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) - document[member] = [field] if listed else field - at = (member, 0) if listed else (member,) + document[field] = [declaration] if listed else declaration + at = (field, 0) if listed else (field,) by_model = [ (problem.loc[len(at) :], problem.kind) for problem in read_array_metadata_v3(document).problems if problem.loc[: len(at)] == at ] - assert _locs(resolve(field, kind, CORE_AND_EXTENSIONS)[1]) == by_model + assert _locs(resolve(declaration, kind, CORE_AND_EXTENSIONS)[1]) == by_model assert by_model == [(("must_understand",), "invalid_value")] @@ -820,13 +841,13 @@ def rules(configuration: GzipCodecConfiguration, nested: Nested) -> None: ] -def test_error_null_is_not_a_field() -> None: +def test_error_null_is_not_a_declaration() -> None: # JSON null is valid JSON. Refining it gives `None` with no problem, - # which is not a usable result. Read as a field or checked as a + # which is not a usable result. Read as a declaration or checked as a # configuration, it is a value of the wrong type. resolved, found = resolve(None, CodecDefinition, CONTEXT) assert (resolved, _locs(found)) == ( - RefusedField(json=None, name=None, read_as=CodecDefinition), + RefusedDeclaration(json=None, name=None, read_as=CodecDefinition), [((), "invalid_type")], ) configuration, found = GZIP_CODEC.read_configuration(None) @@ -839,15 +860,15 @@ def test_error_a_value_that_is_not_json() -> None: resolved, found = resolve( {"name": "gzip", "configuration": {"level": math.nan}}, CodecDefinition, CONTEXT ) - assert resolved == RefusedField( + assert resolved == RefusedDeclaration( json=UNSET, name="gzip", read_as=CodecDefinition, definition=GZIP_CODEC ) assert _locs(found) == [(("configuration", "level"), "invalid_value")] -def test_error_a_value_that_is_not_a_field() -> None: +def test_error_a_value_that_is_not_a_declaration() -> None: resolved, found = resolve(5, CodecDefinition, CONTEXT) - assert resolved == RefusedField(json=5, name=None, read_as=CodecDefinition) + assert resolved == RefusedDeclaration(json=5, name=None, read_as=CodecDefinition) assert len(found) == 1 @@ -855,33 +876,33 @@ def test_error_a_regular_grid_extent_is_negative() -> None: resolved, found = resolve( {"name": "regular", "configuration": {"chunk_shape": [2, -1]}}, ChunkGridDefinition, CONTEXT ) - assert isinstance(resolved, RefusedField) + assert isinstance(resolved, RefusedDeclaration) assert _locs(found) == [(("configuration", "chunk_shape", 1), "invalid_value")] -def test_error_a_nested_field_is_judged_where_it_sits() -> None: - # The problem belongs to the nested field. The field that holds it is +def test_error_a_nested_declaration_is_judged_where_it_sits() -> None: + # The problem belongs to the nested declaration. The declaration that holds it is # still read, as a document holding it would be. - field = { + declaration = { "name": "acme.stack", "configuration": {"codecs": ["crc32c", {"name": "gzip", "configuration": {"level": 12}}]}, } - resolved, found = resolve(field, CodecDefinition, CONTEXT) - assert isinstance(resolved, AcceptedField) - assert isinstance(resolved.nested[("codecs", 1)], RefusedField) + resolved, found = resolve(declaration, CodecDefinition, CONTEXT) + assert isinstance(resolved, AcceptedDeclaration) + assert isinstance(resolved.nested[("codecs", 1)], RefusedDeclaration) assert _locs(found) == [ (("configuration", "codecs", 1, "configuration", "level"), "invalid_value") ] -def test_error_a_nested_field_in_extra_items_is_judged_where_it_sits() -> None: - field = { +def test_error_a_nested_declaration_in_extra_items_is_judged_where_it_sits() -> None: + declaration = { "name": "acme.routes", "configuration": {"slow": {"name": "gzip", "configuration": {"level": 12}}}, } - resolved, found = resolve(field, CodecDefinition, CONTEXT) - assert isinstance(resolved, AcceptedField) - assert isinstance(resolved.nested[("slow",)], RefusedField) + resolved, found = resolve(declaration, CodecDefinition, CONTEXT) + assert isinstance(resolved, AcceptedDeclaration) + assert isinstance(resolved.nested[("slow",)], RefusedDeclaration) assert _locs(found) == [(("configuration", "slow", "configuration", "level"), "invalid_value")] @@ -892,7 +913,7 @@ def test_error_a_nested_envelope_s_problem_is_its_own_and_the_rules_are_asked() resolved, found = resolve( {"name": "acme.stack", "configuration": {"codecs": [unread]}}, CodecDefinition, CONTEXT ) - assert isinstance(resolved, AcceptedField) + assert isinstance(resolved, AcceptedDeclaration) assert _locs(found) == [(("configuration", "codecs", 0, "must_understand"), "invalid_value")] # The stack's rules still run. One rule refuses the array -> bytes # codec it holds, which is a problem of the stack itself. @@ -901,28 +922,28 @@ def test_error_a_nested_envelope_s_problem_is_its_own_and_the_rules_are_asked() CodecDefinition, CONTEXT, ) - assert isinstance(resolved, RefusedField) + assert isinstance(resolved, RefusedDeclaration) assert _locs(found) == [ (("configuration", "codecs", 1), "invalid_value"), (("configuration", "codecs", 0, "must_understand"), "invalid_value"), ] -def test_error_a_container_rule_is_not_asked_of_a_malformed_nested_field() -> None: - # The stack's rule reads each nested field's name. A nested field with no +def test_error_a_container_rule_is_not_asked_of_a_malformed_nested_declaration() -> None: + # The stack's rule reads each nested declaration's name. A nested declaration with no # name is reported at its location, and the rule does not run, as it # would not in `read_configuration`. - field = {"name": "acme.stack", "configuration": {"codecs": [{"configuration": {}}]}} - resolved, found = resolve(field, CodecDefinition, CONTEXT) - assert isinstance(resolved, RefusedField) + declaration = {"name": "acme.stack", "configuration": {"codecs": [{"configuration": {}}]}} + resolved, found = resolve(declaration, CodecDefinition, CONTEXT) + assert isinstance(resolved, RefusedDeclaration) assert _locs(found) == [(("configuration", "codecs", 0, "name"), "missing_key")] - assert ACME_STACK.read_configuration(field["configuration"])[0] is None + assert ACME_STACK.read_configuration(declaration["configuration"])[0] is None def test_error_a_rule_about_the_whole_configuration_lands_on_it() -> None: # A rule reports locations relative to the configuration. An empty # location is the configuration, whether it is checked alone or read in - # a field. + # a declaration. _, judged = ACME_PAIRED.read_configuration({"first": 1}) _, read = resolve( {"name": "acme.paired", "configuration": {"first": 1}}, CodecDefinition, CONTEXT @@ -931,27 +952,29 @@ def test_error_a_rule_about_the_whole_configuration_lands_on_it() -> None: assert _locs(read) == [(("configuration",), "invalid_value")] -def test_error_a_rule_reads_the_fields_the_configuration_holds_as_the_scope_read_them() -> None: +def test_error_a_rule_reads_the_declarations_the_configuration_holds_as_the_scope_read_them() -> ( + None +): # `bytes` is an array -> bytes codec, which the stack's rule refuses # based on what the scope read. `read_configuration` reads in no scope, # so its rule sees no nested readings. - field = {"name": "acme.stack", "configuration": {"codecs": ["crc32c", "bytes"]}} - resolved, found = resolve(field, CodecDefinition, CONTEXT) - assert isinstance(resolved, RefusedField) + declaration = {"name": "acme.stack", "configuration": {"codecs": ["crc32c", "bytes"]}} + resolved, found = resolve(declaration, CodecDefinition, CONTEXT) + assert isinstance(resolved, RefusedDeclaration) assert _locs(found) == [(("configuration", "codecs", 1), "invalid_value")] - assert ACME_STACK.read_configuration(field["configuration"])[1] == () + assert ACME_STACK.read_configuration(declaration["configuration"])[1] == () -def test_error_a_nested_member_is_not_a_field() -> None: +def test_error_a_nested_field_is_not_a_declaration() -> None: resolved, found = resolve( {"name": "acme.stack", "configuration": {"codecs": [5]}}, CodecDefinition, CONTEXT ) - assert isinstance(resolved, RefusedField) + assert isinstance(resolved, RefusedDeclaration) assert _locs(found) == [(("configuration", "codecs", 0), "invalid_type")] def test_check_needs_nothing_but_the_value_and_a_typeddict() -> None: - # The first step checks JSON against a TypedDict. A nested field is only + # The first step checks JSON against a TypedDict. A nested declaration is only # an envelope here, so no scope is needed and no name is checked. assert check({"level": 5}, GzipCodecConfiguration) == ({"level": 5}, ()) stack, problems = check({"codecs": ["gzip", {"name": "anything"}]}, AcmeStackConfiguration) @@ -986,7 +1009,7 @@ def test_error_check_holds_an_array_document_s_shape_to_its_bound() -> None: def test_configuration_of_types_what_its_definition_read() -> None: - """`configuration_of(field, definition)` returns, as a plain `dict`, the configuration of a field that `definition` read and accepted. It returns `None` for a field another definition read, for a field nothing claims, and for a field that `definition` claims but refused, whose configuration it did not read.""" + """`configuration_of(declaration, definition)` returns, as a plain `dict`, the configuration of a declaration that `definition` read and accepted. It returns `None` for a declaration another definition read, for a declaration nothing claims, and for a declaration that `definition` claims but refused, whose configuration it did not read.""" resolved, _ = resolve({"name": "gzip", "configuration": {"level": 5}}, CodecDefinition, CONTEXT) assert configuration_of(resolved, GZIP_CODEC) == {"level": 5} assert type(configuration_of(resolved, GZIP_CODEC)) is dict @@ -994,10 +1017,10 @@ def test_configuration_of_types_what_its_definition_read() -> None: unclaimed, _ = resolve( {"name": "acme.x", "configuration": {"level": 1}}, CodecDefinition, CONTEXT ) - assert isinstance(unclaimed, UnclaimedField) + assert isinstance(unclaimed, UnclaimedDeclaration) assert configuration_of(unclaimed, GZIP_CODEC) is None refused, _ = resolve({"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CONTEXT) - assert isinstance(refused, RefusedField) + assert isinstance(refused, RefusedDeclaration) assert refused.definition is GZIP_CODEC assert configuration_of(refused, GZIP_CODEC) is None @@ -1064,8 +1087,8 @@ class _HugeFormat(_ClaimsWithNoFormat): ], ) def test_error_resolve_refuses_what_is_not_a_scope(context: object) -> None: - """`resolve` reads a field in a scope, which it takes with no default. `None`, which a model reader would take, or any other value that is not a scope raises `TypeError` naming it, rather than failing part way.""" - with pytest.raises(TypeError, match="a field is read in a scope"): + """`resolve` reads a declaration in a scope, which it takes with no default. `None`, which a model reader would take, or any other value that is not a scope raises `TypeError` naming it, rather than failing part way.""" + with pytest.raises(TypeError, match="a declaration is read in a scope"): resolve({"name": "bytes"}, CodecDefinition, context) # pyright: ignore[reportArgumentType] @@ -1083,10 +1106,10 @@ def test_error_check_judges_a_nested_envelope() -> None: assert [found.loc for found in problems] == [("codecs", 0, "configuration")] -def test_read_configuration_leaves_out_a_member_a_nested_field_s_envelope_does_not_declare() -> ( +def test_read_configuration_leaves_out_a_field_a_nested_declaration_s_envelope_does_not_declare() -> ( None ): - # The member is reported as an unknown key and left out, as the checker + # The field is reported as an unknown key and left out, as the checker # leaves out a key that a closed TypedDict does not declare. The # configuration is still returned. typed, problems = ACME_STACK.read_configuration({"codecs": [{"name": "crc32c", "x": 1}]}) @@ -1094,7 +1117,9 @@ def test_read_configuration_leaves_out_a_member_a_nested_field_s_envelope_does_n assert _locs(problems) == [(("codecs", 0, "x"), "unknown_key")] -def test_error_read_configuration_refuses_a_nested_field_that_need_not_be_understood() -> None: +def test_error_read_configuration_refuses_a_nested_declaration_that_need_not_be_understood() -> ( + None +): # A `must_understand` of false is not an unknown key, so the # configuration is not returned. typed, problems = ACME_STACK.read_configuration( @@ -1190,7 +1215,7 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: ), ("int8", "uint8", DataTypeDefinition, False), # The spec's equivalences, which each definition's `canonical` - # normalizes: a member at its default, a run-length encoding, and + # normalizes: a field at its default, a run-length encoding, and # leading zeros in a size. ( {"name": "blosc", "configuration": {**NOSHUFFLE}}, @@ -1260,7 +1285,7 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: "no-or-empty-configuration", "must-understand-true-or-absent", "unclaimed-bare-or-object", - "a-field-it-holds-bare-or-object", + "a-declaration-it-holds-bare-or-object", "another-configuration", "unclaimed-another-configuration", "another-name", @@ -1275,24 +1300,24 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: "raw-bits-leading-zeros", ], ) -def test_two_fields_are_equal_when_they_read_the_same( +def test_two_declarations_are_equal_when_they_read_the_same( one: JSONValue, other: JSONValue, kind: type[Definition[Any]], equal: bool ) -> None: - """However each field was spelled, equal fields have the same simplest spelling and the same hash, though each is written back as it was read.""" + """However each declaration was spelled, equal declarations have the same simplest spelling and the same hash, though each is written back as it was read.""" first, _ = resolve(one, kind, CORE_AND_EXTENSIONS) second, _ = resolve(other, kind, CORE_AND_EXTENSIONS) - assert isinstance(first, (AcceptedField, UnclaimedField)) - assert isinstance(second, (AcceptedField, UnclaimedField)) + assert isinstance(first, (AcceptedDeclaration, UnclaimedDeclaration)) + assert isinstance(second, (AcceptedDeclaration, UnclaimedDeclaration)) assert (first == second) is equal assert (canonical_of(first, ()) == canonical_of(second, ())) is equal if equal: assert hash(first) == hash(second) -def test_a_field_copied_or_pickled_is_read_by_a_definition_equal_to_its_own() -> None: - field, _ = resolve({"name": "gzip", "configuration": {"level": 1}}, CodecDefinition, CORE) - for again in (pickle.loads(pickle.dumps(field)), copy.deepcopy(field)): - assert again == field +def test_a_declaration_copied_or_pickled_is_read_by_a_definition_equal_to_its_own() -> None: + declaration, _ = resolve({"name": "gzip", "configuration": {"level": 1}}, CodecDefinition, CORE) + for again in (pickle.loads(pickle.dumps(declaration)), copy.deepcopy(declaration)): + assert again == declaration assert configuration_of(again, GZIP_CODEC) == {"level": 1} @@ -1311,7 +1336,7 @@ class Loose(TypedDict): class AcmePlainConfiguration(TypedDict, closed=True): - inner: ZarrV3MetadataFieldJSON + inner: ZarrV3DeclarationJSON class AcmeDecimal(TypedDict, closed=True): @@ -1332,7 +1357,7 @@ def test_error_a_definition_configuration_is_a_typeddict() -> None: CodecDefinition(name="acme.bad", configuration=dict, kind="bytes_bytes", size="dynamic") -def test_error_a_definition_member_is_a_shape_json_takes() -> None: +def test_error_a_definition_field_is_a_shape_json_takes() -> None: with pytest.raises(TypeError, match="Unreadable: members is not a shape JSON takes"): CodecDefinition( name="acme.bad", configuration=Unreadable, kind="bytes_bytes", size="dynamic" @@ -1345,9 +1370,9 @@ def test_error_a_configuration_says_what_its_other_keys_are() -> None: CodecDefinition(name="acme.loose", configuration=Loose, kind="bytes_bytes", size="dynamic") -def test_error_a_member_typed_as_plain_field_json_is_refused() -> None: +def test_error_a_field_typed_as_plain_declaration_json_is_refused() -> None: # It would be checked only as JSON, and its name would never be matched to a definition. - with pytest.raises(TypeError, match="annotate it with the field alias of its kind"): + with pytest.raises(TypeError, match="annotate it with the declaration alias of its kind"): CodecDefinition( name="acme.plain", configuration=AcmePlainConfiguration, @@ -1426,13 +1451,15 @@ def test_error_a_data_type_fill_value_no_checker_reads() -> None: DataTypeDefinition(name="acme.set", configuration=EmptyConfiguration, fill_value=set[int]) -def test_error_a_data_type_fill_value_holding_a_metadata_field() -> None: - # A fill value is a value of the data type, which no scope reads as a field. +def test_error_a_data_type_fill_value_holding_a_declaration() -> None: + # A fill value is a value of the data type, which no scope reads as a declaration. with pytest.raises( - TypeError, match="'acme.f': fill_value: ZarrV3CodecField holds a metadata field" + TypeError, match="'acme.f': fill_value: ZarrV3CodecDeclaration holds a declaration" ): DataTypeDefinition( - name="acme.f", configuration=EmptyConfiguration, fill_value=tuple[ZarrV3CodecField, ...] + name="acme.f", + configuration=EmptyConfiguration, + fill_value=tuple[ZarrV3CodecDeclaration, ...], ) @@ -1460,14 +1487,14 @@ def test_error_a_function_member_that_is_not_a_function( @pytest.mark.parametrize("kind", [Definition, AcmeCodecDefinition]) -def test_error_a_field_is_read_as_a_kind_of_metadata(kind: type[Definition[Any]]) -> None: - # Nothing is filed under either kind, so the field would never be checked. +def test_error_a_declaration_is_read_as_a_kind_of_metadata(kind: type[Definition[Any]]) -> None: + # Nothing is filed under either kind, so the declaration would never be checked. with pytest.raises(TypeError, match="is not a kind of metadata"): resolve({"name": "gzip", "configuration": {"level": 99}}, kind, CONTEXT) def test_error_a_scope_refuses_a_definition_of_no_kind() -> None: - """A definition that is no kind of v3 field has no slot, and a slot refuses it, at run time as a type checker does.""" + """A definition that is no kind of v3 declaration has no slot, and a slot refuses it, at run time as a type checker does.""" kindless = Definition(name="acme.kindless", configuration=EmptyConfiguration) with pytest.raises(TypeError, match="a definition of no kind"): ZarrV3Context(codecs=[kindless]) # pyright: ignore[reportArgumentType] @@ -1507,24 +1534,24 @@ def test_error_a_scope_refuses_a_definition_of_no_kind() -> None: ) def test_an_extension_is_named_as_the_spec_names_one(name: str, valid: bool) -> None: """An extension name matches the spec's regex `^[a-z][a-z0-9-_.]+$`, or is a URI, which earlier versions of the spec required. Any other name is refused before any definition is consulted.""" - for field, at in ((name, ()), ({"name": name}, ("name",))): - resolved, problems = resolve(field, CodecDefinition, CORE) + for declaration, at in ((name, ()), ({"name": name}, ("name",))): + resolved, problems = resolve(declaration, CodecDefinition, CORE) if valid: - assert not isinstance(resolved, RefusedField) + assert not isinstance(resolved, RefusedDeclaration) assert problems == () else: - assert isinstance(resolved, RefusedField) + assert isinstance(resolved, RefusedDeclaration) assert [(p.loc, p.kind) for p in problems] == [(at, "invalid_value")] assert "expected an extension name" in problems[0].message -def test_error_a_bad_name_is_a_problem_when_the_field_is_not_json_too() -> None: +def test_error_a_bad_name_is_a_problem_when_the_declaration_is_not_json_too() -> None: # On both paths the name is refused before any definition is consulted. - # A field whose configuration is not JSON reports its name problem as + # A declaration whose configuration is not JSON reports its name problem as # the JSON path does, and no definition is asked to claim it. - field = {"name": "Acme", "configuration": {"x": float("nan")}} - resolved, problems = resolve(field, CodecDefinition, CORE) - assert isinstance(resolved, RefusedField) + declaration = {"name": "Acme", "configuration": {"x": float("nan")}} + resolved, problems = resolve(declaration, CodecDefinition, CORE) + assert isinstance(resolved, RefusedDeclaration) assert resolved.definition is None assert [(p.loc, p.kind) for p in problems] == [ (("name",), "invalid_value"), @@ -1541,14 +1568,14 @@ def test_error_a_definition_is_named_as_the_spec_names_an_extension(name: str) - ) -def test_error_a_field_built_by_hand_is_named_as_the_spec_names_one() -> None: +def test_error_a_declaration_built_by_hand_is_named_as_the_spec_names_one() -> None: with pytest.raises(TypeError, match="named as a document names a"): - UnclaimedField(json="Int8", name="Int8", read_as=CodecDefinition) + UnclaimedDeclaration(json="Int8", name="Int8", read_as=CodecDefinition) -def test_error_a_field_without_a_name_is_missing_one() -> None: +def test_error_a_declaration_without_a_name_is_missing_one() -> None: resolved, problems = resolve({"configuration": {}}, CodecDefinition, CORE) - assert isinstance(resolved, RefusedField) + assert isinstance(resolved, RefusedDeclaration) assert [(p.loc, p.kind, p.message) for p in problems] == [ (("name",), "missing_key", "missing required key") ] @@ -1560,13 +1587,13 @@ def test_raw_bits_of_more_than_a_hundred_digits_are_no_size_but_a_name(digits: i # hundred digits. Such a name is an extension name, which nothing in # scope claims. resolved, problems = resolve("r" + "1" * digits, DataTypeDefinition, CORE_AND_EXTENSIONS) - assert type(resolved) is UnclaimedField + assert type(resolved) is UnclaimedDeclaration assert problems == () def test_error_a_rule_that_writes_to_the_configuration_fails_there() -> None: - # A definition's functions receive a read-only view of the field's own - # configuration, so a function cannot change what a field holds. + # A definition's functions receive a read-only view of the declaration's own + # configuration, so a function cannot change what a declaration holds. def writes( configuration: GzipCodecConfiguration, nested: Nested ) -> Iterator[ValidationProblem]: @@ -1619,11 +1646,11 @@ def refuses(configuration: GzipCodecConfiguration) -> GzipCodecConfiguration: assert raised.value.__notes__ == ["raised by the canonical of 'gzip'"] -def test_error_a_field_s_configuration_and_nested_fields_are_read_only() -> None: - """A field's `json`, `configuration` and `nested` attributes are read-only at every level. So a field cannot be changed into a state where its key, `==` and `refines` disagree.""" +def test_error_a_declaration_s_configuration_and_nested_declarations_are_read_only() -> None: + """A declaration's `json`, `configuration` and `nested` attributes are read-only at every level. So a declaration cannot be changed into a state where its key, `==` and `refines` disagree.""" whole = {**SHARD, "index_codecs": [LE, {"name": "crc32c"}]} shard, _ = resolve({"name": "sharding_indexed", "configuration": whole}, CodecDefinition, CORE) - assert isinstance(shard, AcceptedField) + assert isinstance(shard, AcceptedDeclaration) with pytest.raises(TypeError): shard.configuration["index_location"] = "start" # pyright: ignore[reportIndexIssue] codecs = shard.configuration["codecs"] @@ -1638,7 +1665,7 @@ def test_error_a_field_s_configuration_and_nested_fields_are_read_only() -> None with pytest.raises(TypeError): shard.json["name"] = "gzip" # pyright: ignore[reportIndexIssue] unclaimed, _ = resolve({"name": "acme.x", "configuration": {"a": [1]}}, CodecDefinition, CORE) - assert isinstance(unclaimed, UnclaimedField) + assert isinstance(unclaimed, UnclaimedDeclaration) with pytest.raises(TypeError): unclaimed.configuration["a"] = 2 # pyright: ignore[reportIndexIssue] refused, _ = resolve( @@ -1646,15 +1673,15 @@ def test_error_a_field_s_configuration_and_nested_fields_are_read_only() -> None CodecDefinition, CORE, ) - assert isinstance(refused, RefusedField) + assert isinstance(refused, RefusedDeclaration) with pytest.raises(TypeError): del refused.nested[("codecs", 0)] # pyright: ignore[reportIndexIssue] -def test_a_field_holding_fields_is_copied_and_pickled_whole() -> None: - """A field pickles and deep-copies together with the fields it holds, and stays equal to itself, whether it is accepted, refused or unclaimed.""" +def test_a_declaration_holding_declarations_is_copied_and_pickled_whole() -> None: + """A declaration pickles and deep-copies together with the declarations it holds, and stays equal to itself, whether it is accepted, refused or unclaimed.""" whole = {**SHARD, "index_codecs": [LE, {"name": "crc32c"}]} - fields = [ + declarations = [ resolve({"name": "sharding_indexed", "configuration": whole}, CodecDefinition, CORE)[0], resolve({"name": "acme.x", "configuration": {"a": [1]}}, CodecDefinition, CORE)[0], resolve( @@ -1663,15 +1690,15 @@ def test_a_field_holding_fields_is_copied_and_pickled_whole() -> None: CORE, )[0], ] - for field in fields: - for again in (pickle.loads(pickle.dumps(field)), copy.deepcopy(field)): - assert again == field - assert type(again) is type(field) - assert again.nested == field.nested + for declaration in declarations: + for again in (pickle.loads(pickle.dumps(declaration)), copy.deepcopy(declaration)): + assert again == declaration + assert type(again) is type(declaration) + assert again.nested == declaration.nested @pytest.mark.parametrize( - "field", + "declaration", [ "crc32c", {"name": "gzip", "configuration": {"level": 1}}, @@ -1693,50 +1720,48 @@ def test_a_field_holding_fields_is_copied_and_pickled_whole() -> None: "nameless", ], ) -def test_a_field_its_caller_refined_reads_as_resolve_reads_it(field: object) -> None: - """`resolve_refined` is `resolve` for a caller that has already refined the field. It gives the same field and the same problems, at the same locations, with the same `input`.""" - refined, problems = refine_json(field, ("codecs", 0)) +def test_a_declaration_its_caller_refined_reads_as_resolve_reads_it(declaration: object) -> None: + """`resolve_refined` is `resolve` for a caller that has already refined the declaration. It gives the same declaration and the same problems, at the same locations, with the same `input`.""" + refined, problems = refine_json(declaration, ("codecs", 0)) assert problems == () - expected = resolve(field, CodecDefinition, CORE, ("codecs", 0)) - found = resolve_refined(field, refined, problems, CodecDefinition, CORE, ("codecs", 0)) + expected = resolve(declaration, CodecDefinition, CORE, ("codecs", 0)) + found = resolve_refined(declaration, refined, problems, CodecDefinition, CORE, ("codecs", 0)) assert found == expected assert [(p.loc, p.kind, p.input) for p in found[1]] == [ (p.loc, p.kind, p.input) for p in expected[1] ] -def test_error_a_field_its_caller_found_not_to_be_json_is_refused_unread() -> None: - """The problems `refine_json` found become the field's problems. The field is refused as not JSON, and its name still identifies the definition that claims it, as with `resolve`.""" - field = {"name": "gzip", "configuration": {"level": {1, 2}}} - refined, problems = refine_json(field, ("codecs", 0)) +def test_error_a_declaration_its_caller_found_not_to_be_json_is_refused_unread() -> None: + """The problems `refine_json` found become the declaration's problems. The declaration is refused as not JSON, and its name still identifies the definition that claims it, as with `resolve`.""" + declaration = {"name": "gzip", "configuration": {"level": {1, 2}}} + refined, problems = refine_json(declaration, ("codecs", 0)) assert refined is None resolved, found = resolve_refined( - field, refined, problems, CodecDefinition, CORE, ("codecs", 0) + declaration, refined, problems, CodecDefinition, CORE, ("codecs", 0) ) - assert resolved == RefusedField( + assert resolved == RefusedDeclaration( json=UNSET, name="gzip", read_as=CodecDefinition, definition=GZIP_CODEC ) assert [(p.loc, p.kind) for p in found] == [ (("codecs", 0, "configuration", "level"), "invalid_type") ] - assert (resolved, found) == resolve(field, CodecDefinition, CORE, ("codecs", 0)) + assert (resolved, found) == resolve(declaration, CodecDefinition, CORE, ("codecs", 0)) -def test_error_a_refined_field_is_not_read_in_a_scope_of_the_other_format() -> None: +def test_error_a_refined_declaration_is_not_read_in_a_scope_of_the_other_format() -> None: with pytest.raises(TypeError, match="got a v2 scope"): resolve_refined("gzip", "gzip", (), CodecDefinition, CORE_V2) -V2_IN_A_V3_SCOPE = ( - "a v2 codec is a field of a Zarr v2 document, read in a scope of that format, got a v3 scope" -) +V2_IN_A_V3_SCOPE = "a v2 codec is a declaration of a Zarr v2 document, read in a scope of that format, got a v3 scope" V3_IN_A_V2_SCOPE = ( - "a codec is a field of a Zarr v3 document, read in a scope of that format, got a v2 scope" + "a codec is a declaration of a Zarr v3 document, read in a scope of that format, got a v2 scope" ) @pytest.mark.parametrize( - ("field", "kind", "context", "message"), + ("declaration", "kind", "context", "message"), [ ({"id": "gzip"}, ZarrV2CodecDefinition, CORE, V2_IN_A_V3_SCOPE), (5, ZarrV2CodecDefinition, CORE, V2_IN_A_V3_SCOPE), @@ -1745,9 +1770,9 @@ def test_error_a_refined_field_is_not_read_in_a_scope_of_the_other_format() -> N ], ids=["v2-named", "v2-nameless", "v3-named", "v3-nameless"], ) -def test_error_resolve_refuses_a_kind_of_the_other_format_whatever_the_field( - field: object, kind: type[Definition[Any]], context: Context, message: str +def test_error_resolve_refuses_a_kind_of_the_other_format_whatever_the_declaration( + declaration: object, kind: type[Definition[Any]], context: Context, message: str ) -> None: - """`resolve` with a kind of one format and a scope of the other raises one `TypeError`, before it reads the field. So the message is the same for a field with a name that a scope could look up and for a value with no name.""" + """`resolve` with a kind of one format and a scope of the other raises one `TypeError`, before it reads the declaration. So the message is the same for a declaration with a name that a scope could look up and for a value with no name.""" with pytest.raises(TypeError, match=f"^{re.escape(message)}$"): - resolve(field, kind, context) + resolve(declaration, kind, context) diff --git a/packages/zarr-metadata/tests/v3/test_every_definition.py b/packages/zarr-metadata/tests/v3/test_every_definition.py index f881aa5b3c..057b97cd03 100644 --- a/packages/zarr-metadata/tests/v3/test_every_definition.py +++ b/packages/zarr-metadata/tests/v3/test_every_definition.py @@ -16,9 +16,9 @@ from zarr_metadata import ValidationProblem from zarr_metadata._json import value_at from zarr_metadata.definition import ( - AcceptedField, + AcceptedDeclaration, Definition, - RefusedField, + RefusedDeclaration, canonical_of, configuration_of, fill_value_problems, @@ -47,7 +47,7 @@ "group_extra_field": GroupExtraFieldDefinition, } -# Fields that read without problems, per extension: at least one per definition in scope. +# Declarations that read without problems, per extension: at least one per definition in scope. EXAMPLES: dict[str, tuple[object, ...]] = { "codecs:blosc": ( { @@ -156,17 +156,19 @@ ), } -CASES = [(key, field) for key, fields in EXAMPLES.items() for field in fields] +CASES = [ + (key, declaration) for key, declarations in EXAMPLES.items() for declaration in declarations +] -def _read(key: str, field: object) -> tuple[type, list[tuple[tuple[str | int, ...], str]]]: - """The type of the result of reading `field` in the scope (`AcceptedField`, `UnclaimedField` or `RefusedField`), and the location of each problem.""" - resolved, problems = resolve(field, KINDS[key.split(":")[0]], CORE_AND_EXTENSIONS) +def _read(key: str, declaration: object) -> tuple[type, list[tuple[tuple[str | int, ...], str]]]: + """The type of the result of reading `declaration` in the scope (`AcceptedDeclaration`, `UnclaimedDeclaration` or `RefusedDeclaration`), and the location of each problem.""" + resolved, problems = resolve(declaration, KINDS[key.split(":")[0]], CORE_AND_EXTENSIONS) return type(resolved), [(found.loc, found.kind) for found in problems] -def _problems(key: str, field: object) -> list[tuple[tuple[str | int, ...], str]]: - return _read(key, field)[1] +def _problems(key: str, declaration: object) -> list[tuple[tuple[str | int, ...], str]]: + return _read(key, declaration)[1] def test_a_scope_s_definitions_pickle() -> None: @@ -195,23 +197,26 @@ def test_every_definition_in_scope_has_an_example() -> None: @pytest.mark.parametrize( - ("key", "field"), CASES, ids=[f"{k}:{i}" for i, (k, _) in enumerate(CASES)] + ("key", "declaration"), CASES, ids=[f"{k}:{i}" for i, (k, _) in enumerate(CASES)] ) -def test_every_example_reads_and_its_simplest_spelling_is_stable(key: str, field: object) -> None: - # The field is an AcceptedField with no problems. Its simplest spelling +def test_every_example_reads_and_its_simplest_spelling_is_stable( + key: str, declaration: object +) -> None: + # The declaration is an AcceptedDeclaration with no problems. Its simplest spelling # reads the same way, and is its own simplest spelling. - assert _read(key, field) == (AcceptedField, []) + assert _read(key, declaration) == (AcceptedDeclaration, []) kind = KINDS[key.split(":")[0]] - simplest, problems = canonicalize(field, kind, CORE_AND_EXTENSIONS) + simplest, problems = canonicalize(declaration, kind, CORE_AND_EXTENSIONS) assert problems == () assert simplest is not None assert canonicalize(simplest, kind, CORE_AND_EXTENSIONS) == (simplest, ()) - # A field that was already read gives the same spelling as its JSON, without reading it again. - assert canonical_of(*resolve(field, kind, CORE_AND_EXTENSIONS)) == simplest + # A declaration that was already read gives the same spelling as its JSON, + # without reading it again. + assert canonical_of(*resolve(declaration, kind, CORE_AND_EXTENSIONS)) == simplest @pytest.mark.parametrize( - ("field", "simplest"), + ("declaration", "simplest"), [ # `noshuffle` ignores `typesize`, so the simplest spelling drops it. ( @@ -241,7 +246,7 @@ def test_every_example_reads_and_its_simplest_spelling_is_stable(key: str, field # But a data type with nothing configured is written as its bare name, # as core data types are written. ({"name": "int8", "configuration": {}, "must_understand": True}, "int8"), - # Each nested field is written in its own simplest spelling. + # Each nested declaration is written in its own simplest spelling. ( { "name": "sharding_indexed", @@ -284,7 +289,7 @@ def test_every_example_reads_and_its_simplest_spelling_is_stable(key: str, field "configuration": {"kind": "inline", "chunk_shapes": (((32, 3), 16), 8)}, }, ), - # A member set to its spec default is left out: a key encoding's + # A field set to its spec default is left out: a key encoding's # separator, a shard's index location, and zstd's checksum. ({"name": "default", "configuration": {"separator": "/"}}, {"name": "default"}), ({"name": "v2", "configuration": {"separator": "."}}, {"name": "v2"}), @@ -331,20 +336,20 @@ def test_every_example_reads_and_its_simplest_spelling_is_stable(key: str, field "zstd-checksum", ], ) -def test_the_simplest_spelling(field: dict[str, Any], simplest: object) -> None: +def test_the_simplest_spelling(declaration: dict[str, Any], simplest: object) -> None: kinds = { "rectilinear": ChunkGridDefinition, "int8": DataTypeDefinition, "default": ChunkKeyEncodingDefinition, "v2": ChunkKeyEncodingDefinition, } - kind = kinds.get(field["name"], CodecDefinition) - assert canonicalize(field, kind, CORE_AND_EXTENSIONS) == (simplest, ()) - assert canonical_of(*resolve(field, kind, CORE_AND_EXTENSIONS)) == simplest + kind = kinds.get(declaration["name"], CodecDefinition) + assert canonicalize(declaration, kind, CORE_AND_EXTENSIONS) == (simplest, ()) + assert canonical_of(*resolve(declaration, kind, CORE_AND_EXTENSIONS)) == simplest @pytest.mark.parametrize( - ("field", "bits", "simplest"), + ("declaration", "bits", "simplest"), [ ("r16", 16, "r16"), ({"name": "r8"}, 8, "r8"), @@ -355,15 +360,15 @@ def test_the_simplest_spelling(field: dict[str, Any], simplest: object) -> None: ids=["bare", "object", "empty-configuration", "leading-zeros"], ) def test_raw_bits_read_as_r_star_with_the_size_their_name_carries( - field: object, bits: int, simplest: str + declaration: object, bits: int, simplest: str ) -> None: - resolved, problems = resolve(field, DataTypeDefinition, CORE_AND_EXTENSIONS) + resolved, problems = resolve(declaration, DataTypeDefinition, CORE_AND_EXTENSIONS) assert problems == () - assert isinstance(resolved, AcceptedField) + assert isinstance(resolved, AcceptedDeclaration) assert resolved.definition is RAW_BYTES_DATA_TYPE - assert resolved.json == field + assert resolved.json == declaration assert configuration_of(resolved, RAW_BYTES_DATA_TYPE) == {"bits": bits} - assert canonicalize(field, DataTypeDefinition, CORE_AND_EXTENSIONS) == (simplest, ()) + assert canonicalize(declaration, DataTypeDefinition, CORE_AND_EXTENSIONS) == (simplest, ()) def test_a_reader_reads_raw_bits_its_own_way_by_defining_r_star() -> None: @@ -374,7 +379,7 @@ def test_a_reader_reads_raw_bits_its_own_way_by_defining_r_star() -> None: context = CORE_AND_EXTENSIONS.update(data_types={**CORE_AND_EXTENSIONS.data_types, "r*": mine}) resolved, problems = resolve("r12", DataTypeDefinition, context) assert problems == () - assert isinstance(resolved, AcceptedField) + assert isinstance(resolved, AcceptedDeclaration) assert resolved.definition is mine # The package's own definition does not accept 12 bits, and refuses them. again, _ = resolve( @@ -382,25 +387,25 @@ def test_a_reader_reads_raw_bits_its_own_way_by_defining_r_star() -> None: DataTypeDefinition, context.update(data_types={**context.data_types, "r*": RAW_BYTES_DATA_TYPE}), ) - assert isinstance(again, RefusedField) + assert isinstance(again, RefusedDeclaration) assert again.definition is RAW_BYTES_DATA_TYPE @pytest.mark.parametrize( - ("field", "at"), + ("declaration", "at"), [("r*", ()), ({"name": "r*", "configuration": {"bits": 16}}, ("name",))], ids=["bare", "object"], ) -def test_error_r_star_is_notation_and_no_name(field: object, at: tuple[str, ...]) -> None: +def test_error_r_star_is_notation_and_no_name(declaration: object, at: tuple[str, ...]) -> None: # `r*` is how the specification's table writes raw bits, not a name a # document can use: `*` is not allowed in an extension name. - resolved, problems = resolve(field, DataTypeDefinition, CORE_AND_EXTENSIONS) - assert type(resolved) is RefusedField + resolved, problems = resolve(declaration, DataTypeDefinition, CORE_AND_EXTENSIONS) + assert type(resolved) is RefusedDeclaration assert [(p.loc, p.kind) for p in problems] == [(at, "invalid_value")] @pytest.mark.parametrize( - ("field", "kind", "simplest"), + ("declaration", "kind", "simplest"), [ ({"name": "zfpy"}, CodecDefinition, {"name": "zfpy"}), # Only its own definition can decide its simplest form, so its @@ -420,10 +425,10 @@ def test_error_r_star_is_notation_and_no_name(field: object, at: tuple[str, ...] ], ids=["object", "configured", "bare-codec", "spelled-out", "data-type"], ) -def test_an_unclaimed_field_keeps_its_configuration_in_its_kind_s_envelope( - field: object, kind: type[Definition[Any]], simplest: object +def test_an_unclaimed_declaration_keeps_its_configuration_in_its_kind_s_envelope( + declaration: object, kind: type[Definition[Any]], simplest: object ) -> None: - assert canonicalize(field, kind, CORE) == (simplest, ()) + assert canonicalize(declaration, kind, CORE) == (simplest, ()) def _shard(codecs: list[object], index_codecs: list[object]) -> dict[str, object]: @@ -432,7 +437,7 @@ def _shard(codecs: list[object], index_codecs: list[object]) -> dict[str, object @pytest.mark.parametrize( - ("field", "kind", "found"), + ("declaration", "kind", "found"), [ ( {"name": "gzip", "configuration": {"level": 12}}, @@ -481,26 +486,26 @@ def _shard(codecs: list[object], index_codecs: list[object]) -> dict[str, object "must-understand-false", "stray", "null", - "holding-a-refused-field", + "holding-a-refused-declaration", "a-codec-of-dynamic-size-where-one-of-static-size-goes", ], ) -def test_error_a_field_with_a_problem_has_no_simplest_spelling( - field: dict[str, Any] | None, kind: type[Definition[Any]], found: list[object] +def test_error_a_declaration_with_a_problem_has_no_simplest_spelling( + declaration: dict[str, Any] | None, kind: type[Definition[Any]], found: list[object] ) -> None: # What the author wrote is kept. A simpler spelling would drop the - # unknown key, the stray member or the `must_understand`, or would - # write a value that is not valid. A field that was already read, + # unknown key, the stray field or the `must_understand`, or would + # write a value that is not valid. A declaration that was already read, # together with its problems, has no simplest spelling either. - simplest, problems = canonicalize(field, kind, CORE_AND_EXTENSIONS) + simplest, problems = canonicalize(declaration, kind, CORE_AND_EXTENSIONS) assert simplest is None assert [(problem.loc, problem.kind) for problem in problems] == found - assert canonical_of(*resolve(field, kind, CORE_AND_EXTENSIONS)) is None + assert canonical_of(*resolve(declaration, kind, CORE_AND_EXTENSIONS)) is None def test_error_a_canonical_that_does_not_hold_is_the_definitions_fault() -> None: # The output of `canonical` is checked again. A level the rules refuse - # is a bug in the definition, not in the field it was asked about. + # is a bug in the definition, not in the declaration it was asked about. lying = dataclasses.replace( GZIP_CODEC, name="acme.lying", canonical=lambda configuration: {"level": -7} ) @@ -584,7 +589,7 @@ def test_error_sharding_inner_chunk_extent_is_zero() -> None: @pytest.mark.parametrize( - ("kind", "field", "loc", "ctx"), + ("kind", "declaration", "loc", "ctx"), [ ( CodecDefinition, @@ -663,17 +668,17 @@ def test_error_sharding_inner_chunk_extent_is_zero() -> None: "numpy-time-scale-factor", ], ) -def test_a_bound_is_its_member_s_type_and_its_problem_holds_it( +def test_a_bound_is_its_field_s_type_and_its_problem_holds_it( kind: type[Definition[Any]], - field: dict[str, Any], + declaration: dict[str, Any], loc: tuple[str | int, ...], ctx: dict[str, int], ) -> None: # The bound is declared on the TypedDict, as pydantic reads a bound, # not in a rule. The problem holds the bound and the value found. - _, problems = resolve(field, kind, CORE_AND_EXTENSIONS) + _, problems = resolve(declaration, kind, CORE_AND_EXTENSIONS) assert [(p.loc, p.kind, p.input, dict(p.ctx)) for p in problems] == [ - (loc, "invalid_value", value_at(field, loc), ctx) + (loc, "invalid_value", value_at(declaration, loc), ctx) ] @@ -880,15 +885,15 @@ def test_error_struct_field_type_is_judged_where_it_sits() -> None: def test_error_a_raw_bytes_width_is_not_a_positive_multiple_of_8(name: str) -> None: # Raw bits of a size the spec does not allow are reported, not left # unchecked. The document wrote a name, so the problem is located at - # the field. + # the declaration. assert _problems("data_type:r*", name) == [((), "invalid_value")] def test_error_a_configuration_written_beside_a_raw_bits_name() -> None: # The name carries the configuration, so a configuration written next - # to it may hold nothing: each member is a key that nothing declares. + # to it may hold nothing: each field is a key that nothing declares. assert _read("data_type:r*", {"name": "r16", "configuration": {"bits": 16}}) == ( - AcceptedField, + AcceptedDeclaration, [(("configuration", "bits"), "unknown_key")], ) @@ -901,7 +906,7 @@ def test_error_a_chunk_key_separator_is_not_one_the_encoding_takes() -> None: def test_a_rule_is_a_function_over_the_typeddict() -> None: # The rules are plain functions over JSON. A caller with one - # configuration can call them without a scope or a field. + # configuration can call them without a scope or a declaration. from zarr_metadata.v3.codec.blosc import BLOSC_CODEC _, problems = BLOSC_CODEC.read_configuration({**BLOSC, "clevel": 10}) diff --git a/packages/zarr-metadata/tests/v3/test_fill_values.py b/packages/zarr-metadata/tests/v3/test_fill_values.py index 27436365b3..b66c8e0f94 100644 --- a/packages/zarr-metadata/tests/v3/test_fill_values.py +++ b/packages/zarr-metadata/tests/v3/test_fill_values.py @@ -2,7 +2,7 @@ Each data type's definition declares the JSON shape of its fill value and the rules for a value of that shape. `fill_value_problems` checks a fill -value against a data type field that a scope read. The v3 array +value against a data type declaration that a scope read. The v3 array validators check a document's `fill_value` against its `data_type`. """ @@ -24,7 +24,7 @@ ) from zarr_metadata._json import JSON_DEPTH, value_at from zarr_metadata.definition import ( - AcceptedField, + AcceptedDeclaration, EmptyConfiguration, Nested, canonical_fill_value, @@ -280,8 +280,8 @@ def test_error_an_array_document_s_fill_value_its_data_type_refuses() -> None: assert [(p.loc, p.kind) for p in read_array_metadata_v3(document).problems] == [ (("fill_value",), "invalid_value") ] - member = {"kind": "inline", "must_understand": False, "metadata": {"a": document}} - assert [(p.loc, p.kind) for p in read_consolidated_metadata_v3(member).problems] == [ + field = {"kind": "inline", "must_understand": False, "metadata": {"a": document}} + assert [(p.loc, p.kind) for p in read_consolidated_metadata_v3(field).problems] == [ (("metadata", "a", "fill_value"), "invalid_value") ] @@ -320,7 +320,7 @@ def deep(levels: int) -> dict[str, object]: def test_a_struct_read_without_its_field_types_leaves_its_fields_unjudged() -> None: # A reading built by hand, with no reading of the field types. configuration = {"fields": ({"name": "a", "data_type": "int8"},)} - struct = AcceptedField( + struct = AcceptedDeclaration( json=STRUCT, name="struct", definition=STRUCT_DATA_TYPE, configuration=configuration ) assert fill_value_problems(struct, {"a": 300}) == () @@ -334,10 +334,10 @@ def _alike(left: object, right: object) -> bool: return json.dumps(left, sort_keys=True) == json.dumps(right, sort_keys=True) -def _read(data_type: JSONValue) -> AcceptedField[DataTypeDefinition[Any]]: +def _read(data_type: JSONValue) -> AcceptedDeclaration[DataTypeDefinition[Any]]: resolved, found = resolve(data_type, DataTypeDefinition, CORE_AND_EXTENSIONS) assert found == () - assert isinstance(resolved, AcceptedField) + assert isinstance(resolved, AcceptedDeclaration) return resolved diff --git a/packages/zarr-metadata/tests/v3/test_grid_shapes.py b/packages/zarr-metadata/tests/v3/test_grid_shapes.py index 5d79ea655e..b71543715c 100644 --- a/packages/zarr-metadata/tests/v3/test_grid_shapes.py +++ b/packages/zarr-metadata/tests/v3/test_grid_shapes.py @@ -3,7 +3,7 @@ Each grid's definition declares two things: what the spec disallows for a grid with its configuration over an array of a given shape, and, for a shape the grid fits, the lengths its chunks take along each axis. -`chunk_grid_lengths` reports both for a grid field that a scope read. The +`chunk_grid_lengths` reports both for a grid declaration that a scope read. The v3 array validators check a document's `chunk_grid` against its `shape`. """ @@ -111,7 +111,7 @@ def test_a_grid_that_says_nothing_of_the_shape_fits_every_one_its_lengths_unknow def test_error_a_grid_whose_chunk_lengths_have_another_number_of_axes() -> None: - # This is a bug in the definition, not in the field, so it raises. + # This is a bug in the definition, not in the declaration, so it raises. flat = ChunkGridDefinition( name="acme.grid", configuration=EmptyConfiguration, diff --git a/packages/zarr-metadata/tests/v3/test_kinds.py b/packages/zarr-metadata/tests/v3/test_kinds.py index 37fafd05f2..9b9935a945 100644 --- a/packages/zarr-metadata/tests/v3/test_kinds.py +++ b/packages/zarr-metadata/tests/v3/test_kinds.py @@ -8,15 +8,15 @@ import pytest from zarr_metadata.definition import ( - AcceptedField, + AcceptedDeclaration, Definition, EmptyConfiguration, - UnclaimedField, + UnclaimedDeclaration, resolve, ) from zarr_metadata.definition._definition import as_kind, kind_of from zarr_metadata.definition._scope import kind_name -from zarr_metadata.json_schema import field_json_schema_v3 +from zarr_metadata.json_schema import declaration_json_schema_v3 from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context, ZarrV2DataTypeDefinition from zarr_metadata.v3.definition import ( CORE, @@ -28,7 +28,7 @@ @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class MyCodec(CodecDefinition[Any]): - """A codec definition with an extra member of its own. It is still a codec.""" + """A codec definition with an extra attribute of its own. It is still a codec.""" note: str = "" @@ -44,7 +44,7 @@ def test_a_subclass_of_a_kind_is_a_definition_of_that_kind() -> None: assert kind_of(mine) is CodecDefinition context = ZarrV3Context(codecs=[mine]) assert context.claimant(CodecDefinition, "mine") is mine - assert isinstance(resolve({"name": "mine"}, CodecDefinition, context)[0], AcceptedField) + assert isinstance(resolve({"name": "mine"}, CodecDefinition, context)[0], AcceptedDeclaration) def test_a_kind_is_named_by_its_class_and_its_label() -> None: @@ -65,14 +65,14 @@ def test_error_a_class_that_declares_no_kind_is_of_none() -> None: as_kind(NoKind) -def test_error_the_json_schema_writer_writes_v3_fields_only() -> None: - """`field_json_schema_v3` writes only the v3 envelope. For a kind of another format, it raises a `TypeError` that names this limit instead of writing a wrong schema.""" +def test_error_the_json_schema_writer_writes_v3_declarations_only() -> None: + """`declaration_json_schema_v3` writes only the v3 envelope. For a kind of another format, it raises a `TypeError` that names this limit instead of writing a wrong schema.""" with pytest.raises(TypeError, match="Zarr v3"): - field_json_schema_v3(ZarrV2DataTypeDefinition, context=CORE_V2) # pyright: ignore[reportArgumentType] + declaration_json_schema_v3(ZarrV2DataTypeDefinition, context=CORE_V2) # pyright: ignore[reportArgumentType] -def test_a_v3_value_that_is_no_field_is_shown_in_the_message() -> None: - """A nested value that is not a metadata field at all is reported with the value in the message, as it was before the kind's envelope checked it.""" +def test_a_v3_value_that_is_no_declaration_is_shown_in_the_message() -> None: + """A nested value that is not a declaration at all is reported with the value in the message, as it was before the kind's envelope checked it.""" _, problems = resolve( {"name": "sharding_indexed", "configuration": {"chunk_shape": [1], "codecs": [3]}}, CodecDefinition, @@ -82,7 +82,7 @@ def test_a_v3_value_that_is_no_field_is_shown_in_the_message() -> None: def test_error_a_kind_declares_its_format() -> None: - """A kind declares which Zarr format its fields belong to, `format=2` or `format=3`, in the class header next to `kind=True`. Declaring a kind without a format, or with any other format, raises `TypeError` at class creation.""" + """A kind declares which Zarr format its declarations belong to, `format=2` or `format=3`, in the class header next to `kind=True`. Declaring a kind without a format, or with any other format, raises `TypeError` at class creation.""" with pytest.raises(TypeError, match="format"): type("NoFormat", (Definition,), {}, kind=True) with pytest.raises(TypeError, match="format"): @@ -111,14 +111,14 @@ def test_error_a_scope_files_one_format() -> None: CORE.union(CORE_V2) # pyright: ignore[reportArgumentType] -def test_error_a_field_is_read_in_a_scope_of_its_format() -> None: - """`resolve` raises `TypeError` for a scope whose format differs from the kind's. In an empty scope of its format, it reads a field as unclaimed.""" +def test_error_a_declaration_is_read_in_a_scope_of_its_format() -> None: + """`resolve` raises `TypeError` for a scope whose format differs from the kind's. In an empty scope of its format, it reads a declaration as unclaimed.""" with pytest.raises(TypeError, match="format"): resolve(" None: diff --git a/packages/zarr-metadata/tests/v3/test_pipelines.py b/packages/zarr-metadata/tests/v3/test_pipelines.py index 6323be02a1..15c7345b91 100644 --- a/packages/zarr-metadata/tests/v3/test_pipelines.py +++ b/packages/zarr-metadata/tests/v3/test_pipelines.py @@ -16,7 +16,7 @@ from zarr_metadata.definition import ( EmptyConfiguration, Nested, - ResolvedField, + ResolvedDeclaration, resolve, ) from zarr_metadata.model import ( @@ -31,8 +31,8 @@ Chunk, CodecDefinition, DataTypeDefinition, - ZarrV3CodecField, - ZarrV3DataTypeField, + ZarrV3CodecDeclaration, + ZarrV3DataTypeDeclaration, ) if TYPE_CHECKING: @@ -82,12 +82,12 @@ def _transpose(*order: int) -> JSONValue: return {"name": "transpose", "configuration": {"order": list(order)}} -def _cast(data_type: JSONValue, **members: JSONValue) -> JSONValue: - return {"name": "cast_value", "configuration": {"data_type": data_type, **members}} +def _cast(data_type: JSONValue, **fields: JSONValue) -> JSONValue: + return {"name": "cast_value", "configuration": {"data_type": data_type, **fields}} -def _scale(**members: JSONValue) -> JSONValue: - return {"name": "scale_offset", "configuration": members} +def _scale(**fields: JSONValue) -> JSONValue: + return {"name": "scale_offset", "configuration": fields} def _of(data_type: JSONValue) -> Chunk: @@ -177,7 +177,7 @@ def _lengths(*axes: set[int]) -> tuple[frozenset[int], ...]: [_of("float32"), _of("uint8")], ), # A cast to a data type that nothing in scope claims outputs that - # field, which says nothing about the values. + # declaration, which says nothing about the values. ([_cast("acme.decimal"), "bytes"], _of("float32"), [_of("float32"), _of("acme.decimal")]), # After a codec that nothing in scope claims, a cast still declares the # data type it outputs. An arithmetic codec that receives an unknown @@ -292,7 +292,7 @@ def test_error_a_bytes_codec_without_endian_handed_values_of_several_bytes( def test_error_a_bytes_codec_s_problems_are_located_where_its_document_writes( codec: JSONValue, no_endian: tuple[str | int, ...] ) -> None: - """A codec's chunk rules are located in its document. A missing `endian` is a member of the configuration the codec writes, or the configuration itself when it writes none, as for a codec that requires one. A codec handed values it cannot encode is the codec's problem, however it is spelled.""" + """A codec's chunk rules are located in its document. A missing `endian` is a field of the configuration the codec writes, or the configuration itself when it writes none, as for a codec that requires one. A codec handed values it cannot encode is the codec's problem, however it is spelled.""" assert _problems([codec], _of("int16")) == [(no_endian, "missing_key")] assert _problems([codec], _of("string")) == [((0,), "invalid_value")] @@ -347,7 +347,7 @@ def test_error_a_cast_handed_complex_numbers() -> None: @pytest.mark.parametrize( - ("members", "data_type", "found"), + ("fields", "data_type", "found"), [ ({"offset": 0.5}, "int16", [((0, "configuration", "offset"), "invalid_type")]), ({"scale": 70000}, "int16", [((0, "configuration", "scale"), "invalid_value")]), @@ -355,9 +355,9 @@ def test_error_a_cast_handed_complex_numbers() -> None: ], ) def test_error_a_scale_or_offset_that_is_not_a_value_of_the_data_type_it_is_handed( - members: dict[str, JSONValue], data_type: str, found: list[tuple[tuple[str | int, ...], str]] + fields: dict[str, JSONValue], data_type: str, found: list[tuple[tuple[str | int, ...], str]] ) -> None: - assert _problems([_scale(**members), LITTLE], _of(data_type)) == found + assert _problems([_scale(**fields), LITTLE], _of(data_type)) == found def test_error_a_transition_that_gives_something_else() -> None: @@ -401,7 +401,7 @@ def transition(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) [ ((4, 2), None, "a chunk's lengths are"), (None, "float32", "a chunk's data type is"), - # A field read as a codec, though nothing claims it. + # A declaration read as a codec, though nothing claims it. (None, resolve("acme.t", CodecDefinition, CONTEXT)[0], "a chunk's data type is"), ], ) @@ -430,7 +430,7 @@ def transition(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) @pytest.mark.parametrize( - ("kind", "member"), + ("kind", "attribute"), [ ("bytes_bytes", "chunk_rules"), ("bytes_bytes", "transition"), @@ -438,11 +438,11 @@ def transition(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) ("array_bytes", "transition"), ], ) -def test_error_a_codec_hook_nothing_would_ask(kind: str, member: str) -> None: +def test_error_a_codec_hook_nothing_would_ask(kind: str, attribute: str) -> None: # A bytes -> bytes codec receives bytes, and only an array -> array # codec outputs a chunk, so these hooks would never be called. - hook = {member: lambda configuration, nested, chunk: ()} - with pytest.raises(TypeError, match=f"'acme.x': {member}, "): + hook = {attribute: lambda configuration, nested, chunk: ()} + with pytest.raises(TypeError, match=f"'acme.x': {attribute}, "): CodecDefinition( name="acme.x", configuration=EmptyConfiguration, @@ -486,13 +486,13 @@ def test_error_an_array_document_s_codecs_out_of_order_or_not_fitting_its_chunks class AcmeHolderConfiguration(TypedDict, closed=True): - codecs: tuple[ZarrV3CodecField, ...] - types: tuple[ZarrV3DataTypeField, ...] + codecs: tuple[ZarrV3CodecDeclaration, ...] + types: tuple[ZarrV3DataTypeDeclaration, ...] def _holder( pipelines: object, types: JSONValue = ("uint8",) -) -> ResolvedField[CodecDefinition[Any]]: +) -> ResolvedDeclaration[CodecDefinition[Any]]: """A codec that holds a pipeline of codecs and a list of data types, `types`, and whose `pipelines` hook is `pipelines`.""" holder = CodecDefinition( name="acme.holder", @@ -501,20 +501,20 @@ def _holder( size="dynamic", pipelines=pipelines, # pyright: ignore[reportArgumentType] ) - field = {"name": "acme.holder", "configuration": {"codecs": [LITTLE], "types": types}} + declaration = {"name": "acme.holder", "configuration": {"codecs": [LITTLE], "types": types}} return resolve( - field, CodecDefinition, CONTEXT.update(codecs=[*CONTEXT.codecs.values(), holder]) + declaration, CodecDefinition, CONTEXT.update(codecs=[*CONTEXT.codecs.values(), holder]) )[0] @pytest.mark.parametrize("given", ["a chunk", {"codecs": "a chunk"}, {("codecs",): Chunk()}]) def test_error_pipelines_that_give_something_else(given: object) -> None: - with pytest.raises(TypeError, match="'acme.holder': its pipelines give a mapping of members"): + with pytest.raises(TypeError, match="'acme.holder': its pipelines give a mapping of fields"): read_pipeline([_holder(lambda configuration, nested, chunk: given)], CHUNK) @pytest.mark.parametrize( - ("member", "types"), + ("field", "types"), [ ("nowhere", ("uint8",)), ("types", ("uint8",)), @@ -522,11 +522,9 @@ def test_error_pipelines_that_give_something_else(given: object) -> None: ("types", ("acme.t",)), ], ) -def test_error_pipelines_that_name_a_member_holding_no_codecs( - member: str, types: JSONValue -) -> None: - holder = _holder(lambda configuration, nested, chunk: {member: Chunk()}, types) - with pytest.raises(TypeError, match=f"its pipelines name {member!r}, which holds no list"): +def test_error_pipelines_that_name_a_field_holding_no_codecs(field: str, types: JSONValue) -> None: + holder = _holder(lambda configuration, nested, chunk: {field: Chunk()}, types) + with pytest.raises(TypeError, match=f"its pipelines name {field!r}, which holds no list"): read_pipeline([holder], CHUNK) diff --git a/packages/zarr-metadata/tests/v3/test_rule_locations.py b/packages/zarr-metadata/tests/v3/test_rule_locations.py index 585a70ae61..a33bb892fb 100644 --- a/packages/zarr-metadata/tests/v3/test_rule_locations.py +++ b/packages/zarr-metadata/tests/v3/test_rule_locations.py @@ -8,7 +8,7 @@ import pytest from zarr_metadata import ValidationProblem -from zarr_metadata.definition import RefusedField, resolve +from zarr_metadata.definition import RefusedDeclaration, resolve from zarr_metadata.v2.definition import CORE_V2, ZarrV2DataTypeDefinition from zarr_metadata.v3.codec.bytes import BYTES_CODEC from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE @@ -46,7 +46,7 @@ def _wants_an_endian( def test_error_a_rule_of_a_name_that_encodes_its_configuration_is_a_wrong_value_of_the_name( loc: Loc, ) -> None: - """A name such as `r16` or ` None: - """A codec that writes no configuration has an empty one the document does not hold. A member its rules find missing is the configuration itself, missing, and a problem with the configuration as a whole is the codec's.""" + """A codec that writes no configuration has an empty one the document does not hold. A field its rules find missing is the configuration itself, missing, and a problem with the configuration as a whole is the codec's.""" context = CORE.update( codecs={**CORE.codecs, "bytes": dataclasses.replace(BYTES_CODEC, rules=_wants_an_endian)} ) diff --git a/packages/zarr-metadata/tests/v3/test_scope.py b/packages/zarr-metadata/tests/v3/test_scope.py index a99f1ac2df..2400033adf 100644 --- a/packages/zarr-metadata/tests/v3/test_scope.py +++ b/packages/zarr-metadata/tests/v3/test_scope.py @@ -13,13 +13,13 @@ Conflict, Definition, EmptyConfiguration, - RefusedField, - ResolvedField, + RefusedDeclaration, + ResolvedDeclaration, ScopeConflictError, resolve, ) from zarr_metadata.definition._context import disagreements -from zarr_metadata.definition._definition import fields_of +from zarr_metadata.definition._definition import declarations_of from zarr_metadata.definition._scope import Disagreements, claims_of, kind_name, refines from zarr_metadata.v3.codec.bytes import BYTES_CODEC from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC @@ -61,7 +61,9 @@ """ -def _read(data: object, kind: type[Definition[Any]], context: ZarrV3Context) -> ResolvedField[Any]: +def _read( + data: object, kind: type[Definition[Any]], context: ZarrV3Context +) -> ResolvedDeclaration[Any]: return resolve(data, kind, context)[0] @@ -112,7 +114,7 @@ def test_error_a_scope_conflict_says_each_disagreement() -> None: @pytest.mark.parametrize( - ("field", "claims"), + ("declaration", "claims"), [ (_read("gzip", CodecDefinition, CORE), {(CodecDefinition, "gzip"): GZIP_CODEC}), (_read("zstd", CodecDefinition, CORE), {(CodecDefinition, "zstd"): None}), @@ -132,25 +134,27 @@ def test_error_a_scope_conflict_says_each_disagreement() -> None: _read({"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE), {(CodecDefinition, "gzip"): GZIP_CODEC}, ), - (RefusedField(json=3, name=None, read_as=CodecDefinition), {}), + (RefusedDeclaration(json=3, name=None, read_as=CodecDefinition), {}), ], ids=["read", "unclaimed", "raw-bits", "nested", "refused-claimed", "refused-nameless"], ) def test_claims_of_says_what_a_reading_claimed_of_each_name( - field: ResolvedField[Any], claims: dict[object, object] + declaration: ResolvedDeclaration[Any], claims: dict[object, object] ) -> None: - """A reading's claims map each name written by the field, and by the fields it holds, to the definition that read it. Each name is keyed as the scope files it (raw bits under `r*`), with None where nothing claimed the name. A field refused by a definition still claims that definition. A field with no name claims nothing.""" - assert claims_of(fields_of(field)) == claims + """A reading's claims map each name written by the declaration, and by the declarations it holds, to the definition that read it. Each name is keyed as the scope files it (raw bits under `r*`), with None where nothing claimed the name. A declaration refused by a definition still claims that definition. A declaration with no name claims nothing.""" + assert claims_of(declarations_of(declaration)) == claims def test_error_claims_of_refuses_one_name_read_two_ways() -> None: - """Fields read in two scopes that give one name two different definitions have no single set of claims. `claims_of` raises a `ScopeConflictError` that names the key.""" - fields = [ - *fields_of(_read("gzip", CodecDefinition, CORE), ("codecs", 0)), - *fields_of(_read("gzip", CodecDefinition, ZarrV3Context(codecs=[MY_GZIP])), ("codecs", 1)), + """Declarations read in two scopes that give one name two different definitions have no single set of claims. `claims_of` raises a `ScopeConflictError` that names the key.""" + declarations = [ + *declarations_of(_read("gzip", CodecDefinition, CORE), ("codecs", 0)), + *declarations_of( + _read("gzip", CodecDefinition, ZarrV3Context(codecs=[MY_GZIP])), ("codecs", 1) + ), ] with pytest.raises(ScopeConflictError) as raised: - claims_of(fields) + claims_of(declarations) (conflict,) = raised.value.conflicts assert (conflict.key, conflict.claimed, conflict.found, conflict.loc) == ( (CodecDefinition, "gzip"), @@ -160,8 +164,8 @@ def test_error_claims_of_refuses_one_name_read_two_ways() -> None: ) -GZIP_FIELD = {"name": "gzip", "configuration": {"level": 5}} -ZSTD_FIELD = {"name": "zstd", "configuration": {"level": 3}} +GZIP_DECLARATION = {"name": "gzip", "configuration": {"level": 5}} +ZSTD_DECLARATION = {"name": "zstd", "configuration": {"level": 3}} BLOSC = {"cname": "zstd", "shuffle": "noshuffle", "blocksize": 0} BLOSC_LEVEL_ONE = {"name": "blosc", "configuration": {**BLOSC, "clevel": 1}} BLOSC_LEVEL_TRUE = {"name": "blosc", "configuration": {**BLOSC, "clevel": True}} @@ -179,34 +183,38 @@ def test_error_claims_of_refuses_one_name_read_two_ways() -> None: "name": "sharding_indexed", "configuration": { "chunk_shape": [1], - "codecs": ["bytes", ZSTD_FIELD], + "codecs": ["bytes", ZSTD_DECLARATION], "index_codecs": [{"name": "bytes", "configuration": {"endian": "little"}}, "crc32c"], }, } @pytest.mark.parametrize( - ("field", "other", "expected"), + ("declaration", "other", "expected"), [ - (_read(GZIP_FIELD, CodecDefinition, CORE), _read(GZIP_FIELD, CodecDefinition, CORE), True), + ( + _read(GZIP_DECLARATION, CodecDefinition, CORE), + _read(GZIP_DECLARATION, CodecDefinition, CORE), + True, + ), ( _read({"name": "crc32c"}, CodecDefinition, CORE), _read("crc32c", CodecDefinition, CORE), True, ), ( - _read(ZSTD_FIELD, CodecDefinition, CORE_AND_EXTENSIONS), - _read(ZSTD_FIELD, CodecDefinition, CORE), + _read(ZSTD_DECLARATION, CodecDefinition, CORE_AND_EXTENSIONS), + _read(ZSTD_DECLARATION, CodecDefinition, CORE), True, ), ( - _read(ZSTD_FIELD, CodecDefinition, CORE), - _read(ZSTD_FIELD, CodecDefinition, CORE_AND_EXTENSIONS), + _read(ZSTD_DECLARATION, CodecDefinition, CORE), + _read(ZSTD_DECLARATION, CodecDefinition, CORE_AND_EXTENSIONS), False, ), ( - _read(GZIP_FIELD, CodecDefinition, CORE), - _read(GZIP_FIELD, CodecDefinition, ZarrV3Context(codecs=[MY_GZIP])), + _read(GZIP_DECLARATION, CodecDefinition, CORE), + _read(GZIP_DECLARATION, CodecDefinition, ZarrV3Context(codecs=[MY_GZIP])), False, ), ( @@ -259,22 +267,22 @@ def test_error_claims_of_refuses_one_name_read_two_ways() -> None: "unclaimed-differ", "gain-over-another-spelling", "no-gain-over-true-for-one", - "holding-a-refused-field", + "holding-a-refused-declaration", ], ) def test_refines_orders_readings_by_information( - field: ResolvedField[Any], other: ResolvedField[Any], expected: bool + declaration: ResolvedDeclaration[Any], other: ResolvedDeclaration[Any], expected: bool ) -> None: - """`field` refines `other` when it reads the same where both read, and gains where `other` left a name unclaimed, including in the fields it holds. A loss, a conflict, a refused field, or two unclaimed fields written differently do not refine.""" - assert refines(field, other) is expected + """`declaration` refines `other` when it reads the same where both read, and gains where `other` left a name unclaimed, including in the declarations it holds. A loss, a conflict, a refused declaration, or two unclaimed declarations written differently do not refine.""" + assert refines(declaration, other) is expected DOCUMENTS = [ - GZIP_FIELD, + GZIP_DECLARATION, "crc32c", {"name": "crc32c"}, {"name": "crc32c", "configuration": {}}, - ZSTD_FIELD, + ZSTD_DECLARATION, NESTED_ZSTD, BLOSC_LEVEL_ONE, SHARD_HOLDING_REFUSED, @@ -287,9 +295,9 @@ def test_refines_orders_readings_by_information( @given(st.sampled_from(READINGS), st.sampled_from(READINGS), st.sampled_from(READINGS)) def test_refines_is_a_partial_order_whose_bottom_is_equality( - a: ResolvedField[Any], b: ResolvedField[Any], c: ResolvedField[Any] + a: ResolvedDeclaration[Any], b: ResolvedDeclaration[Any], c: ResolvedDeclaration[Any] ) -> None: - """Over readings of documents in several scopes and spellings, `refines` is reflexive and transitive. Two fields that refine each other are equal, and equal fields are refined by the same fields.""" + """Over readings of documents in several scopes and spellings, `refines` is reflexive and transitive. Two declarations that refine each other are equal, and equal declarations are refined by the same declarations.""" assert refines(a, a) if refines(a, b) and refines(b, c): assert refines(a, c) @@ -393,7 +401,7 @@ def test_a_kind_is_named_in_words(kind: type[Definition[Any]], said: str) -> Non def test_a_gain_is_judged_by_what_the_definition_reads_not_by_spelling() -> None: - """An accepted field refines an unclaimed one when its definition, reading the JSON of the unclaimed field, produces the accepted field. Two spellings of one configuration count as the same gain, so `refines` is transitive through `==`. A spelling that the definition reads as something else is not a gain.""" + """An accepted declaration refines an unclaimed one when its definition, reading the JSON of the unclaimed declaration, produces the accepted declaration. Two spellings of one configuration count as the same gain, so `refines` is transitive through `==`. A spelling that the definition reads as something else is not a gain.""" from zarr_metadata.definition import ( resolve, ) @@ -450,7 +458,7 @@ def test_a_scope_conflict_error_pickles_and_copies_with_its_conflicts() -> None: assert again.__notes__ == ["seen in a union"] -def test_a_conflict_names_the_field_as_the_document_writes_it_and_tells_the_definitions_apart() -> ( +def test_a_conflict_names_the_declaration_as_the_document_writes_it_and_tells_the_definitions_apart() -> ( None ): """A conflict found in a document reports the name the document writes, `r16`, not the name its definition is filed under, `r*`. It tells two definitions of one name apart by the configuration each declares, as the messages for consolidated entries do.""" diff --git a/packages/zarr-metadata/tests/v3/test_sharding.py b/packages/zarr-metadata/tests/v3/test_sharding.py index c74b173c50..7b6b981992 100644 --- a/packages/zarr-metadata/tests/v3/test_sharding.py +++ b/packages/zarr-metadata/tests/v3/test_sharding.py @@ -14,7 +14,7 @@ from zarr_metadata import JSONValue from zarr_metadata.definition import ( - ResolvedField, + ResolvedDeclaration, resolve, ) from zarr_metadata.model import ( @@ -41,7 +41,7 @@ INDEX: list[JSONValue] = [LITTLE, "crc32c"] -def _dt(name: JSONValue) -> ResolvedField[DataTypeDefinition[Any]]: +def _dt(name: JSONValue) -> ResolvedDeclaration[DataTypeDefinition[Any]]: return resolve(name, DataTypeDefinition, CORE_AND_EXTENSIONS)[0] @@ -50,7 +50,7 @@ def _dt(name: JSONValue) -> ResolvedField[DataTypeDefinition[Any]]: def _chunk( - *axes: set[int] | None, data_type: ResolvedField[DataTypeDefinition[Any]] = FLOAT32 + *axes: set[int] | None, data_type: ResolvedDeclaration[DataTypeDefinition[Any]] = FLOAT32 ) -> Chunk: return Chunk(tuple(None if axis is None else frozenset(axis) for axis in axes), data_type) @@ -96,9 +96,9 @@ def _handed(stages: tuple[Stage, ...], at: Where = ()) -> dict[Where, Chunk | No """The chunk each codec receives, for every pipeline held by the codecs, keyed by the codec's location.""" found: dict[Where, Chunk | None] = {} for index, stage in enumerate(stages): - for member, held in stage.inner.items(): - found |= {(*at, index, member, place): s.incoming for place, s in enumerate(held)} - found |= _handed(held, (*at, index, member)) + for field, held in stage.inner.items(): + found |= {(*at, index, field, place): s.incoming for place, s in enumerate(held)} + found |= _handed(held, (*at, index, field)) return found diff --git a/packages/zarr-metadata/tests/v3/test_storage.py b/packages/zarr-metadata/tests/v3/test_storage.py index fadcf36049..a4f1d98c8c 100644 --- a/packages/zarr-metadata/tests/v3/test_storage.py +++ b/packages/zarr-metadata/tests/v3/test_storage.py @@ -1,7 +1,7 @@ """How a data type's values are stored: in single bytes, in several bytes each, or each in as many bytes as it needs. Each data type's definition declares its storage. `storage_of` returns -the storage of a data type field that a scope read. The `bytes` codec +the storage of a data type declaration that a scope read. The `bytes` codec asks for the storage of the data type it receives, and a struct asks for the storage of each of its fields. """ From c31062e798ed46de7fb72839c97657bfe374808f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 1 Nov 2026 10:54:03 +0100 Subject: [PATCH 05/21] docs(zarr-metadata): say which fields hold declarations and complete the rename's changelog Review fixes for the field/declaration rename: an extra field is a declaration written as the field alone, the array models' plain fields are the ones that hold no declaration, must_understand_fields covers extra fields only, and the context guide matches definitions to declarations. The fragment drops four names that were never public and adds spec_fields, the declaration wording of messages, and the renamed JSON Schema $defs keys. Assisted-by: ClaudeCode:claude-opus-5-5 --- packages/zarr-metadata/README.md | 9 +++++---- .../zarr-metadata/changes/+field-terminology.removal.md | 5 +++-- packages/zarr-metadata/docs/context.md | 2 +- packages/zarr-metadata/docs/index.md | 9 +++++---- .../src/zarr_metadata/definition/_definition.py | 6 +++--- .../zarr-metadata/src/zarr_metadata/model/_array_v2.py | 6 +++--- .../zarr-metadata/src/zarr_metadata/model/_array_v3.py | 6 +++--- .../zarr-metadata/src/zarr_metadata/model/_group_v3.py | 2 +- 8 files changed, 24 insertions(+), 21 deletions(-) diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index 2cdb843a7a..738276d8cc 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -272,12 +272,13 @@ string against those rules. A top-level field that the spec does not define is an extra field. A scope can hold a definition for one, by its key, in the `array_extra_fields` or `group_extra_fields` slot of `ZarrV3Context`. A -field whose key a definition claims is read by that definition, like any -other declaration. Any other field is kept as written, as an `UnclaimedDeclaration`. +field whose key a definition claims is read by that definition: an extra field is a declaration +written as the field alone, `{key: value}`. Any other extra field is kept as +written, as an `UnclaimedDeclaration`. The scope is what a reader recognizes, so the model's `must_understand_fields` lists each unclaimed field that does not say `"must_understand": false`: the spec requires a reader to fail to open a -node with any such field. It covers fields only. A data type, chunk +node with any such field. It covers extra fields only. A data type, chunk grid, chunk key encoding, codec or storage transformer that nothing in scope claims is an `UnclaimedDeclaration` too, also when it sits inside another declaration, such as a codec inside a shard. A model's `unclaimed_declarations` lists @@ -298,7 +299,7 @@ built. `to_json` returns the document as it was written, and Every typed attribute is a view of that read: each declaration as the scope read it (an `AcceptedDeclaration` or an `UnclaimedDeclaration`), and `shape`, `attributes` -and the other fields as the read refined them. These views are read-only +and the fields that hold no declaration as the read refined them. These views are read-only at every level: a list in the document is shown as a tuple, and an object as a read-only mapping. `to_json` returns plain containers. diff --git a/packages/zarr-metadata/changes/+field-terminology.removal.md b/packages/zarr-metadata/changes/+field-terminology.removal.md index 165fed9bf6..4f140db8b7 100644 --- a/packages/zarr-metadata/changes/+field-terminology.removal.md +++ b/packages/zarr-metadata/changes/+field-terminology.removal.md @@ -1,10 +1,11 @@ Rename the package's vocabulary to match zarrs and the v3 spec: a name/value pair in a JSON metadata object is now a *field* (it was a *member*), and a value at an extension point that a scope resolves is now a *declaration* (it was a *field*). The public names change with no aliases: - `AcceptedField`, `UnclaimedField`, `RefusedField` and `ResolvedField` are now `AcceptedDeclaration`, `UnclaimedDeclaration`, `RefusedDeclaration` and `ResolvedDeclaration`. -- `is_field`, `fields_of`, `field_key` and `field_kind` are now `is_declaration`, `declarations_of`, `declaration_key` and `declaration_kind`. - `ZarrV3DataTypeField`, `ZarrV3ChunkGridField`, `ZarrV3ChunkKeyEncodingField`, `ZarrV3CodecField`, `ZarrV3StaticCodecField`, `ZarrV3StorageTransformerField` and `ZarrV2DataTypeField` are now the same names ending in `Declaration`, and a kind's `field_aliases` is now `declaration_aliases`. - `ZarrV3MetadataFieldJSON` is now `ZarrV3DeclarationJSON`, and `field_json_schema_v3` is now `declaration_json_schema_v3`. - A reading's `fields()` is now `declarations()`, and a model's `unclaimed_fields` is now `unclaimed_declarations`. - The first parameter of `with_problems` is now `declarations` (it was `fields`), and the first parameter of `ZarrV3ConsolidatedMetadata` is now `value` (it was `member`). +- `ExtraFieldDefinition.spec_members` (and so `ArrayExtraFieldDefinition.spec_members` and `GroupExtraFieldDefinition.spec_members`) is now `spec_fields`. +- The JSON Schemas that `node_metadata_json_schema_v3` and `declaration_json_schema_v3` write name each alias under `$defs` by its new name, so a `$ref` to `#/$defs/ZarrV3CodecField` is now `#/$defs/ZarrV3CodecDeclaration`, and likewise for the other kinds. -Messages that said *member* now say *field*. `extra_fields`, `ExtraFieldDefinition`, `ZarrV3ExtensionField`, `StructField` and the pydantic `...MetadataField` types keep their names. +Messages that said *member* now say *field*, and messages that said *field* or *metadata field* for an extension-point value now say *declaration*, as in "expected a declaration (string or extension object)". `extra_fields`, `ExtraFieldDefinition`, `ZarrV3ExtensionField`, `StructField` and the pydantic `...MetadataField` types keep their names. diff --git a/packages/zarr-metadata/docs/context.md b/packages/zarr-metadata/docs/context.md index 13000b935b..38a51dd12b 100644 --- a/packages/zarr-metadata/docs/context.md +++ b/packages/zarr-metadata/docs/context.md @@ -6,7 +6,7 @@ Resolving metadata documents requires a context, a `Context`, that defines which The Zarr V3 spec defines the JSON structure of `zarr.json` for arrays and groups exactly, by defining the fields that must be present and the JSON types they must have. But the __interpretation__ of those fields as actual data types, codecs, chunk grids, etc is not strictly defined by the spec. Relating a JSON value like `"bytes"` to the `bytes` codec requires a mapping from names to codec types. -`zarr-metadata` makes this explicit by validating JSON documents with a __context object__. That context object contains definitions of codecs, data types, etc, which can be matched to fields in the JSON document. +`zarr-metadata` makes this explicit by validating JSON documents with a __context object__. That context object contains definitions of codecs, data types, etc, which can be matched to the declarations in the JSON document. ### Context example diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md index d6c9403518..06436863ac 100644 --- a/packages/zarr-metadata/docs/index.md +++ b/packages/zarr-metadata/docs/index.md @@ -274,12 +274,13 @@ string against those rules. A top-level field that the spec does not define is an extra field. A scope can hold a definition for one, by its key, in the `array_extra_fields` or `group_extra_fields` slot of `ZarrV3Context`. A -field whose key a definition claims is read by that definition, like any -other declaration. Any other field is kept as written, as an `UnclaimedDeclaration`. +field whose key a definition claims is read by that definition: an extra field is a declaration +written as the field alone, `{key: value}`. Any other extra field is kept as +written, as an `UnclaimedDeclaration`. The scope is what a reader recognizes, so the model's `must_understand_fields` lists each unclaimed field that does not say `"must_understand": false`: the spec requires a reader to fail to open a -node with any such field. It covers fields only. A data type, chunk +node with any such field. It covers extra fields only. A data type, chunk grid, chunk key encoding, codec or storage transformer that nothing in scope claims is an `UnclaimedDeclaration` too, also when it sits inside another declaration, such as a codec inside a shard. A model's `unclaimed_declarations` lists @@ -300,7 +301,7 @@ built. `to_json` returns the document as it was written, and Every typed attribute is a view of that read: each declaration as the scope read it (an `AcceptedDeclaration` or an `UnclaimedDeclaration`), and `shape`, `attributes` -and the other fields as the read refined them. These views are read-only +and the fields that hold no declaration as the read refined them. These views are read-only at every level: a list in the document is shown as a tuple, and an object as a read-only mapping. `to_json` returns plain containers. diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index bba9c1f157..16f75c29f2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -707,8 +707,8 @@ def _as_written(declaration: _NestedDeclaration) -> JSONValue: return declaration.json -def _declared(declaration: _NestedDeclaration) -> JSONValue: - """A nested declaration restored as it was written, except for envelope fields that are not declared. Those are reported and left out, in the same way the checker leaves out a key that a closed TypedDict does not declare.""" +def _envelope_keys_only(declaration: _NestedDeclaration) -> JSONValue: + """A nested declaration restored as it was written, except for the fields its envelope TypedDict does not declare. Those are reported and left out, in the same way the checker leaves out a key that a closed TypedDict does not declare.""" if not isinstance(declaration.json, Mapping): return declaration.json envelope = cast("Mapping[str, JSONValue]", declaration.json) @@ -833,7 +833,7 @@ def _configuration_checked( problems = (*found, *(problem for declaration in nested for problem in _envelope(declaration))) if not _usable(problems): return None, problems - return cast("T", _put_back(typed, _declared, nested)), problems + return cast("T", _put_back(typed, _envelope_keys_only, nested)), problems def named_configuration( diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index fa260fae0e..6038a9b4f6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -119,7 +119,7 @@ def _is_canonical_dtype_v2(value: object) -> bool: @dataclass(frozen=True, slots=True) class ArrayFieldsV2: - """The fields other than declarations of a v2 array document that a read found no problems with, refined as the model holds them.""" + """The fields that hold no declaration in a v2 array document that a read found no problems with, refined as the model holds them.""" shape: tuple[int, ...] chunks: tuple[int, ...] @@ -614,7 +614,7 @@ def _key_of(self) -> tuple[object, ...]: def _plain_key( self, dtype: AcceptedDeclaration[ZarrV2DataTypeDefinition[Any]] | UnclaimedDeclaration ) -> tuple[object, ...]: - """The fields other than declarations that `refines` compares. The fill value is in the spelling of `dtype`, the dtype of the more informed side.""" + """The fields that hold no declaration, which `refines` compares. The fill value is in the spelling of `dtype`, the dtype of the more informed side.""" fields = self._fields return ( fields.shape, @@ -756,7 +756,7 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadat return self.with_context(context) def refines(self, other: ZarrV2ArrayMetadata) -> bool: - """Whether this model holds everything `other` holds. Each declaration must refine its counterpart. A `null` compressor or `null` filters refine only a `null`. Every other field must be the same, with the fill value compared in the spelling of the more informed dtype. If that dtype rejects the other model's fill value, this is not a refinement.""" + """Whether this model holds everything `other` holds. Each declaration must refine its counterpart. A `null` compressor or `null` filters refine only a `null`. Every field that holds no declaration must be the same, with the fill value compared in the spelling of the more informed dtype. If that dtype rejects the other model's fill value, this is not a refinement.""" if type(other) is not type(self): return False if (self.compressor is None) != (other.compressor is None): diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 01a705c126..bc1fa56e61 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -211,7 +211,7 @@ def _array_reading(values: dict[str, Any]) -> ZarrV3ArrayMetadataReading: @dataclass(frozen=True, slots=True) class ArrayFieldsV3: - """The fields other than declarations of a v3 array document that a read found no problems with, refined as the model holds them.""" + """The fields that hold no declaration in a v3 array document that a read found no problems with, refined as the model holds them.""" shape: tuple[int, ...] fill_value: JSONValue @@ -579,7 +579,7 @@ def _key_of(self) -> tuple[object, ...]: def _plain_key( self, data_type: AcceptedDeclaration[DataTypeDefinition[Any]] | UnclaimedDeclaration ) -> tuple[object, ...]: - """The fields other than declarations that `refines` compares. The fill value is in the spelling of `data_type`, the data type of the more informed side.""" + """The fields that hold no declaration, which `refines` compares. The fill value is in the spelling of `data_type`, the data type of the more informed side.""" fields = self._fields return ( fields.shape, @@ -731,7 +731,7 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadat return self.with_context(context) def refines(self, other: ZarrV3ArrayMetadata) -> bool: - """Whether this model holds everything `other` holds. Each declaration must refine its counterpart, in the order `refines` defines for declarations (including the declarations nested in each declaration). Every other field must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement.""" + """Whether this model holds everything `other` holds. Each declaration must refine its counterpart, in the order `refines` defines for declarations (including the declarations nested in each declaration). Every field that holds no declaration must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement.""" if type(other) is not type(self): return False if len(self.codecs) != len(other.codecs) or len(self.storage_transformers) != len( diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index d95dd99079..66c8a2535a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -635,7 +635,7 @@ class ZarrV3GroupMetadataReading: """Each field the spec does not define, as the scope read it, by key: an `UnclaimedDeclaration` when no definition in scope claims the key.""" def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: - """Each declaration in the document, as the scope read it, with its location: the extra fields, each at its key.""" + """Each declaration in the document, as the scope read it, with its location: each extra field, as a declaration at its key.""" for key, extra in self.extra_fields.items(): yield from declarations_of(extra, (key,)) From 2b11adc8e6ade1ad412489f0f64d5b3a7e0d63f5 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 1 Nov 2026 17:04:19 +0100 Subject: [PATCH 06/21] docs(zarr-metadata): number the field/declaration fragment after its PR Assisted-by: ClaudeCode:claude-opus-5-5 --- .../changes/{+field-terminology.removal.md => 432.removal.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{+field-terminology.removal.md => 432.removal.md} (100%) diff --git a/packages/zarr-metadata/changes/+field-terminology.removal.md b/packages/zarr-metadata/changes/432.removal.md similarity index 100% rename from packages/zarr-metadata/changes/+field-terminology.removal.md rename to packages/zarr-metadata/changes/432.removal.md From 59f035707ff925dde56a5e98b2afe36daf6ffbd4 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 1 Nov 2026 17:16:13 +0100 Subject: [PATCH 07/21] docs(zarr-metadata): keep the field/declaration fragment unnumbered until it goes upstream towncrier links fragments to upstream zarr-python PRs, and upstream #432 is an unrelated merged PR; this change has only a fork PR so far. Assisted-by: ClaudeCode:claude-opus-5-5 --- .../changes/{432.removal.md => +field-terminology.removal.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{432.removal.md => +field-terminology.removal.md} (100%) diff --git a/packages/zarr-metadata/changes/432.removal.md b/packages/zarr-metadata/changes/+field-terminology.removal.md similarity index 100% rename from packages/zarr-metadata/changes/432.removal.md rename to packages/zarr-metadata/changes/+field-terminology.removal.md From bc177b3fa2142f184e1a5ee04084622616635a07 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Sun, 1 Nov 2026 18:40:38 +0100 Subject: [PATCH 08/21] feat(zarr-metadata)!: read what zarr-python writes in a ZARR_PYTHON_CONTEXT scope (#4583) * feat(zarr-metadata)!: read what zarr-python writes in a ZARR_PYTHON_CONTEXT scope Add definitions for the extensions zarr-python writes. zarr-extensions registers `fixed_length_utf32` and the `vlen-utf8` and `vlen-bytes` codecs, so they join CORE_AND_EXTENSIONS_CONTEXT. zarr-python's own `null_terminated_bytes` and `variable_length_bytes`, which no specification registers, are only in the new ZARR_PYTHON_CONTEXT, so the default scope stays spec-pure. Rename CORE to CORE_CONTEXT and CORE_AND_EXTENSIONS to CORE_AND_EXTENSIONS_CONTEXT, so that a scope's name says it is a context. Fixes d-v-b/zarr-python#423 Assisted-by: ClaudeCode:claude-opus-5-5 * docs(zarr-metadata): number the changelog fragments for PR 426 Assisted-by: ClaudeCode:claude-opus-5-5 * docs(zarr-metadata): number the changelog fragments for PR 4583 Assisted-by: ClaudeCode:claude-opus-5-5 * fix(zarr-metadata): keep the old scope names and tighten the string rules Keep `CORE` and `CORE_AND_EXTENSIONS` importable from `zarr_metadata.v3` and `zarr_metadata.v3.definition` as the same scopes with a DeprecationWarning, so a consumer such as pydantic-zarr keeps working across the rename. Check that `vlen-utf8` encodes `string` and `vlen-bytes` encodes `bytes` (or zarr-python's `variable_length_bytes`), as zarr-extensions says, instead of accepting any data type whose values vary in size. Do not count trailing null bytes against the size of a `null_terminated_bytes` fill value, since they are padding. Spell a `fixed_length_utf32` fill value without its trailing U+0000 padding, so `"a"` and `"a\u0000"` have one canonical spelling. Assisted-by: ClaudeCode:claude-opus-5-5 * feat(zarr-metadata): read zarr-python's raw_bytes and rename CORE_V2 Add zarr-python's `raw_bytes` data type (NumPy's `V`) to ZARR_PYTHON_CONTEXT, so the scope reads every data type zarr-python writes. Its names start with `ZarrPython`, since `RawBytes` already names the spec's raw bits. A fill value longer than `length_bytes` is a problem, because zarr-python cuts it short, and the canonical spelling pads it with null bytes, as zarr-python writes it. Rename CORE_V2 to CORE_V2_CONTEXT, so every scope's name ends in `_CONTEXT`. The old name keeps working with a DeprecationWarning, through the same `renamed_scope` helper as the v3 names, which moves to `zarr_metadata._renamed`. Share the base64 decoding and spelling of fill values in `_byte.py`, and say that the vlen codecs' `configuration` key is optional, since zarr-python writes an empty object. Assisted-by: ClaudeCode:claude-opus-5-5 * refactor(zarr-metadata): declare the v2 scope's old name once Move CORE_V2_CONTEXT and its table of old names into `v2/_core.py`, as `v3/_core.py` holds the v3 scopes, so `zarr_metadata.v2` and `zarr_metadata.v2.definition` share one table. Say why a `fixed_length_utf32` fill value counts its trailing U+0000 code points, as the spec requires, while `null_terminated_bytes` follows zarr-python and does not, and test a value that only its padding makes too long. Assisted-by: ClaudeCode:claude-opus-5-5 * fix(zarr-metadata): give main's strategies the renamed scopes and new definitions The Hypothesis strategies merged from main (#4575) imported `CORE`, `CORE_AND_EXTENSIONS` and `CORE_V2` from modules that now hold only the `_CONTEXT` names, and their coverage test requires a strategy for every definition in `CORE_AND_EXTENSIONS_CONTEXT`. Rename the imports, and add strategies for `fixed_length_utf32`, `vlen-utf8` and `vlen-bytes`; each vlen codec draws nothing for a data type it does not read. Assisted-by: ClaudeCode:claude-opus-5-5 * fix(zarr-metadata): draw zarr-python's own data types in the testing strategies `DEFAULT_STRATEGIES` holds a strategy for each definition the package defines, but `null_terminated_bytes`, `variable_length_bytes` and `raw_bytes` had none, so `fields()` raised `LookupError` for them. Draw each, and check the strategies against `ZARR_PYTHON_CONTEXT`, which holds every definition. Say that the documents in test_zarr_python_scope.py leave out the compressors zarr-python adds. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 * test(zarr-metadata): pin the registered extensions to CORE_AND_EXTENSIONS_CONTEXT The strategies are now checked against `ZARR_PYTHON_CONTEXT`, which no longer proves that `fixed_length_utf32`, `vlen-utf8` and `vlen-bytes` are in `CORE_AND_EXTENSIONS_CONTEXT`. Assert it directly. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 * refactor(zarr-metadata): let the vlen strategies ask their codec's rules Each vlen codec strategy named its compatible data types a second time; draw nothing for exactly the chunks the codec's own chunk rules refuse, so the two cannot drift. Draw `fixed_length_utf32` fill values with trailing U+0000 padding too, which the canonical spelling removes. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- packages/zarr-metadata/README.md | 18 +- .../zarr-metadata/changes/4583.feature.md | 1 + .../zarr-metadata/changes/4583.removal.md | 1 + packages/zarr-metadata/docs/api/index.md | 4 +- packages/zarr-metadata/docs/api/v3/codec.md | 4 + .../zarr-metadata/docs/api/v3/data_type.md | 8 + packages/zarr-metadata/docs/index.md | 18 +- .../zarr-metadata/docs/parsing-validation.md | 12 +- .../src/zarr_metadata/__init__.py | 24 ++ .../src/zarr_metadata/_renamed.py | 26 ++ .../src/zarr_metadata/definition/__init__.py | 12 +- .../zarr_metadata/definition/_definition.py | 2 +- .../src/zarr_metadata/json_schema.py | 2 +- .../src/zarr_metadata/model/__init__.py | 2 +- .../src/zarr_metadata/model/_array_v2.py | 18 +- .../src/zarr_metadata/model/_array_v3.py | 12 +- .../src/zarr_metadata/model/_group_v2.py | 30 +-- .../src/zarr_metadata/model/_group_v3.py | 38 +-- .../src/zarr_metadata/model/_repair.py | 10 +- .../src/zarr_metadata/pydantic.py | 30 +-- .../src/zarr_metadata/testing/_codec.py | 17 +- .../src/zarr_metadata/testing/_data_type.py | 56 +++++ .../src/zarr_metadata/testing/strategies.py | 4 +- .../src/zarr_metadata/v2/__init__.py | 18 +- .../src/zarr_metadata/v2/_core.py | 17 ++ .../src/zarr_metadata/v2/definition.py | 24 +- .../src/zarr_metadata/v3/__init__.py | 38 ++- .../src/zarr_metadata/v3/_core.py | 60 ++++- .../src/zarr_metadata/v3/_json_schema.py | 10 +- .../src/zarr_metadata/v3/codec/__init__.py | 2 +- .../src/zarr_metadata/v3/codec/_vlen.py | 27 +++ .../src/zarr_metadata/v3/codec/vlen_bytes.py | 77 ++++++ .../src/zarr_metadata/v3/codec/vlen_utf8.py | 71 ++++++ .../src/zarr_metadata/v3/consolidated.py | 2 +- .../zarr_metadata/v3/data_type/__init__.py | 9 +- .../src/zarr_metadata/v3/data_type/_byte.py | 35 ++- .../src/zarr_metadata/v3/data_type/bytes.py | 16 +- .../v3/data_type/fixed_length_utf32.py | 124 ++++++++++ .../v3/data_type/null_terminated_bytes.py | 121 ++++++++++ .../v3/data_type/variable_length_bytes.py | 62 +++++ .../v3/data_type/zarr_python_raw_bytes.py | 123 ++++++++++ .../src/zarr_metadata/v3/definition.py | 27 ++- .../zarr-metadata/tests/model/test_array.py | 38 +-- .../tests/model/test_consolidating.py | 69 +++--- .../tests/model/test_construction.py | 20 +- .../tests/model/test_extension_points.py | 23 +- .../tests/model/test_extra_fields.py | 31 ++- .../zarr-metadata/tests/model/test_group.py | 26 +- .../zarr-metadata/tests/model/test_pair.py | 88 +++---- .../zarr-metadata/tests/model/test_pair_v2.py | 18 +- .../tests/model/test_pydantic_module.py | 101 ++++---- .../tests/model/test_read_array_metadata.py | 27 ++- .../model/test_read_array_metadata_v2.py | 6 +- .../tests/model/test_scope_threading.py | 18 +- .../tests/model/test_scope_threading_v2.py | 16 +- .../tests/model/test_unclaimed_fields.py | 38 +-- packages/zarr-metadata/tests/test_contexts.py | 17 +- .../zarr-metadata/tests/test_json_schema.py | 103 ++++---- .../zarr-metadata/tests/test_problem_data.py | 14 +- .../zarr-metadata/tests/test_public_api.py | 13 + .../tests/test_renamed_scopes.py | 45 ++++ .../tests/testing/test_strategies.py | 30 +-- .../zarr-metadata/tests/v2/test_definition.py | 30 +-- .../tests/v3/test_definitions.py | 124 ++++++---- .../tests/v3/test_every_definition.py | 69 ++++-- .../tests/v3/test_fill_values.py | 26 +- .../tests/v3/test_grid_shapes.py | 18 +- packages/zarr-metadata/tests/v3/test_kinds.py | 20 +- .../zarr-metadata/tests/v3/test_pipelines.py | 12 +- .../tests/v3/test_rule_locations.py | 24 +- packages/zarr-metadata/tests/v3/test_scope.py | 127 +++++----- .../zarr-metadata/tests/v3/test_sharding.py | 6 +- .../zarr-metadata/tests/v3/test_storage.py | 16 +- .../tests/v3/test_zarr_python_scope.py | 226 ++++++++++++++++++ 74 files changed, 1987 insertions(+), 664 deletions(-) create mode 100644 packages/zarr-metadata/changes/4583.feature.md create mode 100644 packages/zarr-metadata/changes/4583.removal.md create mode 100644 packages/zarr-metadata/src/zarr_metadata/_renamed.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v2/_core.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/codec/_vlen.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_bytes.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_utf8.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py create mode 100644 packages/zarr-metadata/tests/test_renamed_scopes.py create mode 100644 packages/zarr-metadata/tests/v3/test_zarr_python_scope.py diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index 3c5513d1dd..12583e0c96 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -41,7 +41,7 @@ closely model the content of the Zarr specifications, such as: `TypedDict` with its rules, and the scope that decides which definitions read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every definition and scope of their format, such as `BYTES_CODEC` and - `CORE_AND_EXTENSIONS`. + `CORE_AND_EXTENSIONS_CONTEXT`. - **JSON type checking** ([`zarr_metadata.typed_json`](https://zarr-metadata.readthedocs.io/en/latest/api/typed_json/)): `check` checks a JSON value against any `TypedDict` and reports every problem with its location. @@ -104,7 +104,9 @@ of context-free consistency rules, e.g.: In a v3 document, validation also checks each extension point: the data type, the chunk grid, the chunk key encoding, each codec, and each storage transformer. Each one is checked by the definition that claims its name in -a scope. The scope is `CORE_AND_EXTENSIONS` unless a `context` is passed. +a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. +`ZARR_PYTHON_CONTEXT` adds the data types that zarr-python writes but no +specification registers. If the definition rejects a configuration, the reader rejects it too, and a configuration key the definition does not declare is reported as `unknown_key`. If no definition in the scope claims a name, the reader @@ -232,13 +234,13 @@ each field: ```python test="true" session="readme" from zarr_metadata.model import read_array_metadata_v3 from zarr_metadata.definition import canonical_of, with_problems -from zarr_metadata.v3.definition import CORE +from zarr_metadata.v3.definition import CORE_CONTEXT reading = read_array_metadata_v3(raw) beyond_core = [ loc for loc, field in reading.fields() - if field.name is not None and CORE.claimant(field.read_as, field.name) is None + if field.name is not None and CORE_CONTEXT.claimant(field.read_as, field.name) is None ] simplest = { loc: canonical_of(field, problems) @@ -283,7 +285,7 @@ scope claims is an `UnclaimedField` too, also when it sits inside another field, such as a codec inside a shard. A model's `unclaimed_fields` lists every field nothing in scope claims, at any depth, by its location, and leaves out the members that waive `must_understand`: a reader can open -the node when it is empty. `CORE` holds +the node when it is empty. `CORE_CONTEXT` holds `CONSOLIDATED_METADATA_EXTRA_FIELD`, which reads a group's `consolidated_metadata` as the envelope the spec fixes. A group's `consolidated()` reads the documents that member lists, in the group's @@ -291,7 +293,7 @@ own scope. A v3 model consists of its document and the scope it was read in. `ZarrV3ArrayMetadata(document, context=None)` reads the document in the -scope (`CORE_AND_EXTENSIONS` when none is given) and raises +scope (`CORE_AND_EXTENSIONS_CONTEXT` when none is given) and raises `MetadataValidationError` with every problem, so an invalid model is never built. `to_json` returns the document as it was written, and `to_key_value` writes that same document to store bytes. @@ -322,7 +324,7 @@ all of them with `ZarrV3ConsolidatedMetadata`, or with `read_consolidated_metadata_v3`, which returns problems instead of raising them. A document with a problem then fails only when it is read. Pass `context=group.context` to read the documents in the group's own -scope; with no `context`, they are read in `CORE_AND_EXTENSIONS`. +scope; with no `context`, they are read in `CORE_AND_EXTENSIONS_CONTEXT`. To write consolidated metadata, build a `ZarrV3ConsolidatedMetadata` and pass it to the group's `update`. The group reads the listed models in its @@ -332,7 +334,7 @@ models as entries. Such an entry is accepted when its claims refine into the model's scope, and refused at its path otherwise. To consolidate children that were read in several scopes, consolidate them in the `union` of those scopes. Every reader takes `context=None` for the -default scope, `CORE_AND_EXTENSIONS`. +default scope, `CORE_AND_EXTENSIONS_CONTEXT`. Two models are equal when they represent the same document, even if each is spelled differently. Values the package interprets (each field, and the diff --git a/packages/zarr-metadata/changes/4583.feature.md b/packages/zarr-metadata/changes/4583.feature.md new file mode 100644 index 0000000000..cbbe44f174 --- /dev/null +++ b/packages/zarr-metadata/changes/4583.feature.md @@ -0,0 +1 @@ +Add the `ZARR_PYTHON_CONTEXT` scope, which reads what zarr-python writes: `CORE_AND_EXTENSIONS_CONTEXT` plus zarr-python's own data types `null_terminated_bytes` (NumPy's `S`), `raw_bytes` (NumPy's `V`) and `variable_length_bytes`, which no specification registers. `CORE_AND_EXTENSIONS_CONTEXT` now also reads the `fixed_length_utf32` data type (NumPy's `U`) and the `vlen-utf8` and `vlen-bytes` codecs that zarr-extensions registers, so the strings zarr-python writes are checked instead of read as unclaimed. diff --git a/packages/zarr-metadata/changes/4583.removal.md b/packages/zarr-metadata/changes/4583.removal.md new file mode 100644 index 0000000000..9faa94ee31 --- /dev/null +++ b/packages/zarr-metadata/changes/4583.removal.md @@ -0,0 +1 @@ +Rename the scopes `CORE`, `CORE_AND_EXTENSIONS` and `CORE_V2` to `CORE_CONTEXT`, `CORE_AND_EXTENSIONS_CONTEXT` and `CORE_V2_CONTEXT`, so that a scope's name says it is a context. The old names, imported from `zarr_metadata.v3`, `zarr_metadata.v3.definition`, `zarr_metadata.v2` or `zarr_metadata.v2.definition`, still refer to the same scopes with a `DeprecationWarning`, and will be removed in a later release. diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md index 4a23c5c2fb..e95a186284 100644 --- a/packages/zarr-metadata/docs/api/index.md +++ b/packages/zarr-metadata/docs/api/index.md @@ -22,7 +22,7 @@ The package is organized to mirror the structure of the Zarr specifications: - [`zarr_metadata.v2`](v2.md) — `TypedDict` shapes for Zarr v2 documents (`.zarray`, `.zgroup`, `.zattrs`, `.zmetadata`), and `zarr_metadata.v2.definition`: the v2 kinds of metadata, the scope - `CORE_V2`, and the readers of one `dtype` or codec. Each dtype family's + `CORE_V2_CONTEXT`, and the readers of one `dtype` or codec. Each dtype family's definition is in `zarr_metadata.v2.data_type`, and each codec's in `zarr_metadata.v2.codec` - [`zarr_metadata.v3`](v3/index.md) — `TypedDict` shapes for Zarr v3 @@ -35,7 +35,7 @@ The package is organized to mirror the structure of the Zarr specifications: scope type `Context`. Its module docstring is the guide - [`zarr_metadata.v3.definition`](v3/definition.md) — the five v3 kinds of metadata (`CodecDefinition`, `DataTypeDefinition`, ...), the codec - pipeline types, and the v3 scopes `CORE` and `CORE_AND_EXTENSIONS` + pipeline types, and the v3 scopes `CORE_CONTEXT` and `CORE_AND_EXTENSIONS_CONTEXT` The document types, models, and spec vocabulary — including the store keys — are re-exported at the top level, so diff --git a/packages/zarr-metadata/docs/api/v3/codec.md b/packages/zarr-metadata/docs/api/v3/codec.md index cb96d2c7d5..ae5171acd0 100644 --- a/packages/zarr-metadata/docs/api/v3/codec.md +++ b/packages/zarr-metadata/docs/api/v3/codec.md @@ -22,4 +22,8 @@ title: codec ::: zarr_metadata.v3.codec.transpose +::: zarr_metadata.v3.codec.vlen_bytes + +::: zarr_metadata.v3.codec.vlen_utf8 + ::: zarr_metadata.v3.codec.zstd diff --git a/packages/zarr-metadata/docs/api/v3/data_type.md b/packages/zarr-metadata/docs/api/v3/data_type.md index 3f8444625a..2bdfe0ad2c 100644 --- a/packages/zarr-metadata/docs/api/v3/data_type.md +++ b/packages/zarr-metadata/docs/api/v3/data_type.md @@ -40,8 +40,16 @@ title: data_type ::: zarr_metadata.v3.data_type.string +::: zarr_metadata.v3.data_type.fixed_length_utf32 + ::: zarr_metadata.v3.data_type.numpy_datetime64 ::: zarr_metadata.v3.data_type.numpy_timedelta64 ::: zarr_metadata.v3.data_type.struct + +::: zarr_metadata.v3.data_type.null_terminated_bytes + +::: zarr_metadata.v3.data_type.variable_length_bytes + +::: zarr_metadata.v3.data_type.zarr_python_raw_bytes diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md index b3936156d0..6b238b7017 100644 --- a/packages/zarr-metadata/docs/index.md +++ b/packages/zarr-metadata/docs/index.md @@ -39,7 +39,7 @@ closely model the content of the Zarr specifications, such as: `TypedDict` with its rules, and the scope that decides which definitions read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every definition and scope of their format, such as `BYTES_CODEC` and - `CORE_AND_EXTENSIONS`. + `CORE_AND_EXTENSIONS_CONTEXT`. - **JSON type checking** ([`zarr_metadata.typed_json`](api/typed_json.md)): `check` checks a JSON value against any `TypedDict` and reports every problem with its location. @@ -102,7 +102,9 @@ of context-free consistency rules, e.g.: In a v3 document, validation also checks each extension point: the data type, the chunk grid, the chunk key encoding, each codec, and each storage transformer. Each one is checked by the definition that claims its name in -a scope. The scope is `CORE_AND_EXTENSIONS` unless a `context` is passed. +a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. +`ZARR_PYTHON_CONTEXT` adds the data types that zarr-python writes but no +specification registers. If the definition rejects a configuration, the reader rejects it too, and a configuration key the definition does not declare is reported as `unknown_key`. If no definition in the scope claims a name, the reader @@ -234,13 +236,13 @@ each field: ```python test="true" session="index" from zarr_metadata.model import read_array_metadata_v3 from zarr_metadata.definition import canonical_of, with_problems -from zarr_metadata.v3.definition import CORE +from zarr_metadata.v3.definition import CORE_CONTEXT reading = read_array_metadata_v3(raw) beyond_core = [ loc for loc, field in reading.fields() - if field.name is not None and CORE.claimant(field.read_as, field.name) is None + if field.name is not None and CORE_CONTEXT.claimant(field.read_as, field.name) is None ] simplest = { loc: canonical_of(field, problems) @@ -285,7 +287,7 @@ scope claims is an `UnclaimedField` too, also when it sits inside another field, such as a codec inside a shard. A model's `unclaimed_fields` lists every field nothing in scope claims, at any depth, by its location, and leaves out the members that waive `must_understand`: a reader can open -the node when it is empty. `CORE` holds +the node when it is empty. `CORE_CONTEXT` holds `CONSOLIDATED_METADATA_EXTRA_FIELD`, which reads a group's `consolidated_metadata` as the envelope the spec fixes. A group's `consolidated()` reads the documents that member lists, in the group's @@ -293,7 +295,7 @@ own scope. A v3 model consists of its document and the scope it was read in. `ZarrV3ArrayMetadata(document, context=None)` reads the document in the -scope (`CORE_AND_EXTENSIONS` when none is given) and raises +scope (`CORE_AND_EXTENSIONS_CONTEXT` when none is given) and raises `MetadataValidationError` with every problem, so an invalid model is never built. `to_json` returns the document as it was written, and `to_key_value` writes that same document to store bytes. @@ -324,7 +326,7 @@ all of them with `ZarrV3ConsolidatedMetadata`, or with `read_consolidated_metadata_v3`, which returns problems instead of raising them. A document with a problem then fails only when it is read. Pass `context=group.context` to read the documents in the group's own -scope; with no `context`, they are read in `CORE_AND_EXTENSIONS`. +scope; with no `context`, they are read in `CORE_AND_EXTENSIONS_CONTEXT`. To write consolidated metadata, build a `ZarrV3ConsolidatedMetadata` and pass it to the group's `update`. The group reads the listed models in its @@ -334,7 +336,7 @@ models as entries. Such an entry is accepted when its claims refine into the model's scope, and refused at its path otherwise. To consolidate children that were read in several scopes, consolidate them in the `union` of those scopes. Every reader takes `context=None` for the -default scope, `CORE_AND_EXTENSIONS`. +default scope, `CORE_AND_EXTENSIONS_CONTEXT`. Two models are equal when they represent the same document, even if each is spelled differently. Values the package interprets (each field, and the diff --git a/packages/zarr-metadata/docs/parsing-validation.md b/packages/zarr-metadata/docs/parsing-validation.md index adfe8e51c2..a4122d87ea 100644 --- a/packages/zarr-metadata/docs/parsing-validation.md +++ b/packages/zarr-metadata/docs/parsing-validation.md @@ -161,7 +161,7 @@ This procedure yields one of two values: `zarr-metadata` does not run each stage as a separate function. Instead, a single read of the document collects the problems from every stage, so a caller gets the full list of problems at once. -The **registry** from the resolve-and-validate stage is called a **scope** in this library. For Zarr V3 it is a [`ZarrV3Context`](api/v3/definition.md) object, with one slot for each kind of field (`data_types`, `codecs`, `chunk_grids`, ...), and every reader takes a `context` argument. The default scope is `CORE_AND_EXTENSIONS`, which holds the data types, codecs, chunk grids, etc. defined in the Zarr V3 spec and in [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/). `CORE` holds only the definitions from the Zarr V3 spec. +The **registry** from the resolve-and-validate stage is called a **scope** in this library. For Zarr V3 it is a [`ZarrV3Context`](api/v3/definition.md) object, with one slot for each kind of field (`data_types`, `codecs`, `chunk_grids`, ...), and every reader takes a `context` argument. The default scope is `CORE_AND_EXTENSIONS_CONTEXT`, which holds the data types, codecs, chunk grids, etc. defined in the Zarr V3 spec and in [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/). `CORE_CONTEXT` holds only the definitions from the Zarr V3 spec. `ZARR_PYTHON_CONTEXT` adds to `CORE_AND_EXTENSIONS_CONTEXT` the data types that zarr-python writes but no specification registers (`null_terminated_bytes` for NumPy's `S`, `raw_bytes` for NumPy's `V`, and `variable_length_bytes`), so a strict check of what zarr-python writes passes it. `read_array_metadata_v3` reads a JSON document and returns a "reading". `reading.problems` is a tuple of the problems, and `reading.metadata` is the validated model, or `None` if there are problems. Each problem has a `loc`, which is its position in the document as a tuple, a `kind`, like `missing_key` or `invalid_value`, and a human-readable `message`: @@ -193,10 +193,10 @@ To read one field on its own, such as a codec, use `resolve` from `zarr_metadata ```python test="true" session="parsing-validation" from zarr_metadata.definition import resolve -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition field, problems = resolve( - {"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE_AND_EXTENSIONS + {"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT ) for problem in problems: print(problem.loc, problem.kind) @@ -214,17 +214,17 @@ To make an unknown data type or codec known, add its definition to the scope's s ```python test="true" session="parsing-validation" from zarr_metadata.definition import EmptyConfiguration -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition # The definition of the codec that was unknown: here, a codec named # `acme.codec` that takes no configuration. MY_CODEC = CodecDefinition( name="acme.codec", configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" ) -context = CORE_AND_EXTENSIONS.update(codecs=[*CORE_AND_EXTENSIONS.codecs.values(), MY_CODEC]) +context = CORE_AND_EXTENSIONS_CONTEXT.update(codecs=[*CORE_AND_EXTENSIONS_CONTEXT.codecs.values(), MY_CODEC]) reading = read_array_metadata_v3(raw, context=context) ``` -`update` replaces each slot it is given and keeps the others. A slot can also be given as a mapping from each definition's name to it, so `CORE_AND_EXTENSIONS.update(codecs={**CORE_AND_EXTENSIONS.codecs, "gzip": MY_GZIP})` replaces the gzip definition with your own. +`update` replaces each slot it is given and keeps the others. A slot can also be given as a mapping from each definition's name to it, so `CORE_AND_EXTENSIONS_CONTEXT.update(codecs={**CORE_AND_EXTENSIONS_CONTEXT.codecs, "gzip": MY_GZIP})` replaces the gzip definition with your own. [^1]: The one exception is the `r*` data type, which is identified by a templated string with no `configuration` object. This mixes the identifier with the configuration in the same JSON value, which is not good design. So this library normalizes `r*` into the same form as all the other metadata objects: the field `"r16"` keeps its name, and is read by the `r*` definition with the configuration `{"bits": 16}`. diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index bc57d90fd0..5e8b1ffcba 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -136,6 +136,16 @@ TransposeCodecMetadata, TransposeCodecName, ) +from zarr_metadata.v3.codec.vlen_bytes import ( + VLEN_BYTES_CODEC_NAME, + VlenBytesCodecMetadata, + VlenBytesCodecName, +) +from zarr_metadata.v3.codec.vlen_utf8 import ( + VLEN_UTF8_CODEC_NAME, + VlenUtf8CodecMetadata, + VlenUtf8CodecName, +) from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME, ZstdCodecMetadata, ZstdCodecName from zarr_metadata.v3.consolidated import ( ZARR_V3_CONSOLIDATED_METADATA_KEY, @@ -162,6 +172,11 @@ Complex128DataTypeName, Complex128FillValue, ) +from zarr_metadata.v3.data_type.fixed_length_utf32 import ( + FIXED_LENGTH_UTF32_DATA_TYPE_NAME, + FixedLengthUtf32DataTypeName, + FixedLengthUtf32FillValue, +) from zarr_metadata.v3.data_type.float16 import ( FLOAT16_DATA_TYPE_NAME, Float16DataTypeName, @@ -265,6 +280,7 @@ "DEFAULT_CHUNK_KEY_ENCODING_NAME", "DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR", "ENDIANNESS", + "FIXED_LENGTH_UTF32_DATA_TYPE_NAME", "FLOAT16_DATA_TYPE_NAME", "FLOAT32_DATA_TYPE_NAME", "FLOAT64_DATA_TYPE_NAME", @@ -291,6 +307,8 @@ "UNSET", "V2_CHUNK_KEY_ENCODING_NAME", "V2_CHUNK_KEY_ENCODING_SEPARATOR", + "VLEN_BYTES_CODEC_NAME", + "VLEN_UTF8_CODEC_NAME", "ZARR_V2_ARRAY_DIMENSION_SEPARATOR", "ZARR_V2_ARRAY_METADATA_STORE_KEY", "ZARR_V2_ARRAY_ORDER", @@ -325,6 +343,8 @@ "DefaultChunkKeyEncodingName", "DefaultChunkKeyEncodingSeparator", "Endianness", + "FixedLengthUtf32DataTypeName", + "FixedLengthUtf32FillValue", "Float16DataTypeName", "Float16FillValue", "Float32DataTypeName", @@ -378,6 +398,10 @@ "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", "ValidationProblem", + "VlenBytesCodecMetadata", + "VlenBytesCodecName", + "VlenUtf8CodecMetadata", + "VlenUtf8CodecName", "ZarrV2ArrayDimensionSeparator", "ZarrV2ArrayMetadata", "ZarrV2ArrayMetadataJSON", diff --git a/packages/zarr-metadata/src/zarr_metadata/_renamed.py b/packages/zarr-metadata/src/zarr_metadata/_renamed.py new file mode 100644 index 0000000000..d0267a70db --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/_renamed.py @@ -0,0 +1,26 @@ +"""Scopes this package renamed, which keep their old names for now with a `DeprecationWarning`.""" + +import warnings +from collections.abc import Mapping + + +def renamed_scope(module: str, name: str, renames: Mapping[str, tuple[str, object]]) -> object: + """The scope that `name`, an old name in `renames`, refers to, with a `DeprecationWarning`. + + `renames` maps each old name to the scope's name now and the scope. A + module that exported the old names calls this from its `__getattr__`, + so an import of an old name keeps working. Raises `AttributeError`, as + attribute lookup does, when `name` is not an old name. + """ + if name not in renames: + raise AttributeError(f"module {module!r} has no attribute {name!r}") + new, scope = renames[name] + warnings.warn( + f"`{name}` is deprecated and will be removed; use `{new}`, the same scope", + DeprecationWarning, + stacklevel=3, + ) + return scope + + +__all__ = ["renamed_scope"] diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index 6dd9c86173..782807f4a8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -40,7 +40,7 @@ configuration, checks the envelope of each nested field (an extra member, a `must_understand` of `false`), and then runs the rules, for one configuration: `GZIP_CODEC.read_configuration({"level": 12})`. -3. `resolve(field, CodecDefinition, CORE_AND_EXTENSIONS)` reads a whole +3. `resolve(field, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT)` reads a whole field in a scope. It checks the envelope, maps the name to a definition, checks the configuration, and reads each nested field the same way. It returns the result of the reading and every problem. The @@ -70,17 +70,17 @@ from zarr_metadata.definition import resolve from zarr_metadata.v3.codec.gzip import GZIP_CODEC - from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition + from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition resolved, problems = resolve({"name": "gzip", "configuration": {"level": 12}}, - CodecDefinition, CORE_AND_EXTENSIONS) + CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT) resolved # RefusedField(..., definition=CodecDefinition(name='gzip'), ...) problems[0].loc # ('configuration', 'level') problems[0].input # 12 dict(problems[0].ctx) # {'ge': 0, 'le': 9} resolved, problems = resolve({"name": "gzip", "configuration": {"level": 5}}, - CodecDefinition, CORE_AND_EXTENSIONS) + CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT) resolved.definition is GZIP_CODEC # True resolved.configuration # {'level': 5} @@ -124,7 +124,7 @@ from zarr_metadata import ValidationProblem from zarr_metadata.definition import Nested - from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition + from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition class AcmeLz4Configuration(TypedDict, closed=True): @@ -149,7 +149,7 @@ def acme_lz4_rules( size="dynamic", rules=acme_lz4_rules, ) - CONTEXT = CORE_AND_EXTENSIONS.update(codecs=[*CORE_AND_EXTENSIONS.codecs.values(), ACME_LZ4]) + CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.update(codecs=[*CORE_AND_EXTENSIONS_CONTEXT.codecs.values(), ACME_LZ4]) The TypedDict declares how keys it does not declare are handled. With `closed=True`, such a key is a problem, as above. With `extra_items=`, diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index 5d50f8e605..0c30ffd16b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1374,7 +1374,7 @@ def resolve( else raises `TypeError`. """ if not _is_scope(context): - msg = f"a field is read in a scope, such as CORE_AND_EXTENSIONS, got {described(context)}" + msg = f"a field is read in a scope, such as CORE_AND_EXTENSIONS_CONTEXT, got {described(context)}" raise TypeError(msg) asked = as_kind(kind) _check_format(asked, context) diff --git a/packages/zarr-metadata/src/zarr_metadata/json_schema.py b/packages/zarr-metadata/src/zarr_metadata/json_schema.py index 9b7443a8f4..ea80ac0242 100644 --- a/packages/zarr-metadata/src/zarr_metadata/json_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/json_schema.py @@ -30,7 +30,7 @@ **A whole `zarr.json`.** `node_metadata_json_schema_v3(context=None)` writes a whole v3 `zarr.json`, array or group, as the scope reads it -(`CORE_AND_EXTENSIONS` when none is given). Its fields are written +(`CORE_AND_EXTENSIONS_CONTEXT` when none is given). Its fields are written as the scope reads them. For each definition, the schema accepts its field (its name, its configuration as its TypedDict declares it, including bounds, a `must_understand` of `true`, and its bare name when diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 98776c7c5d..ab6006ca2a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -9,7 +9,7 @@ `read_consolidated_metadata_v2`. A reader checks a document's JSON structure. In a v3 document it also reads each extension point (codecs, chunk grids, data types, ...) through the definition that claims its -name in a scope. The scope is `CORE_AND_EXTENSIONS` unless a `context` +name in a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. A v3 reader checks the fill value against the data type the document names, the chunk grid against the shape, and the codecs as a pipeline, each codec against the chunk it receives. Each reader does one diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index 7768e68210..a4336ea0a5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -70,7 +70,7 @@ from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY from zarr_metadata.v2.codec import ZarrV2CodecMetadata from zarr_metadata.v2.definition import ( - CORE_V2, + CORE_V2_CONTEXT, ZarrV2CodecDefinition, ZarrV2DataTypeDefinition, ) @@ -437,7 +437,7 @@ def in_files( def read_array_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2ArrayMetadataReading: - """Read `value`, a v2 array document, in `context` (`CORE_V2` when none is given), whatever it holds. + """Read `value`, a v2 array document, in `context` (`CORE_V2_CONTEXT` when none is given), whatever it holds. The result holds everything the read finds: the dtype, the compressor and each filter as the scope read them (an `AcceptedField` read by @@ -447,7 +447,7 @@ def read_array_metadata_v2( problem. One that the scope does not claim is not a problem. `fill_value` is checked against the dtype the scope read. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) reading, members = read_array_v2(value, context) if members is None: return reading @@ -510,7 +510,7 @@ class ZarrV2ArrayMetadata(Keyed): node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) reading, members = read_array_v2(document, context) if members is None: raise MetadataValidationError(reading.problems) @@ -731,7 +731,7 @@ def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetad in `context`. The existing reading is reused when `context` reads every claim identically. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) if disagreements(context, claims_of(self._reading.fields())).agrees: return self._of(self._document, context, self._reading, self._members) return type(self)(self._document, context=context) @@ -745,7 +745,7 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadat in the document. Raises `MetadataValidationError` when a definition that `context` claims rejects what was written. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) found = disagreements(context, claims_of(self._reading.fields())) if len(found.conflicts) != 0: raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) @@ -807,7 +807,9 @@ def create_default( if lengths is not None: given["chunks"] = lengths if "dtype" in given and "fill_value" not in given: - dtype, _ = resolve(given["dtype"], ZarrV2DataTypeDefinition, scoped(context, CORE_V2)) + dtype, _ = resolve( + given["dtype"], ZarrV2DataTypeDefinition, scoped(context, CORE_V2_CONTEXT) + ) if len(fill_value_problems(dtype, 0)) != 0: given["fill_value"] = None merged = {key: value for key, value in {**document, **given}.items() if value is not UNSET} @@ -836,7 +838,7 @@ def from_key_value( valid, with every problem the read finds. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) files, problems = decoded_node_files_v2( mapping, (ZARR_V2_ARRAY_METADATA_STORE_KEY, ZARR_V2_ATTRIBUTES_STORE_KEY) ) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 53d12254bc..58fb7e30c5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -62,7 +62,7 @@ # A runtime import: `get_type_hints` resolves the public signatures. from zarr_metadata.v3._context import ZarrV3Context # noqa: TC001 -from zarr_metadata.v3._core import CORE_AND_EXTENSIONS +from zarr_metadata.v3._core import CORE_AND_EXTENSIONS_CONTEXT from zarr_metadata.v3._definition import ( ArrayExtraFieldDefinition, Chunk, @@ -499,7 +499,7 @@ class ZarrV3ArrayMetadata(Keyed): node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_array_v3(document, context) if members is None: raise MetadataValidationError(reading.problems) @@ -696,7 +696,7 @@ def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetad in `context`. The existing reading is reused when `context` reads every claim identically. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) if disagreements(context, claims_of(self._reading.fields())).agrees: return self._of(self._document, context, self._reading, self._members) return type(self)(self._document, context=context) @@ -713,7 +713,7 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadat gain can expose a problem. `with_context` reads the document in any scope. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) found = disagreements(context, claims_of(self._reading.fields())) if len(found.conflicts) != 0: raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) @@ -811,7 +811,7 @@ def from_key_value( are not JSON, or the document is not valid. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) return cls(load_store_json(mapping, ZARR_V3_ARRAY_METADATA_STORE_KEY), context=context) @@ -888,7 +888,7 @@ def read_array_metadata_v3( spec's fields), iterate over the reading's `fields()`. A value that is not an object has no fields. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_array_v3(value, context) if members is None: return reading diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py index eb290a6cb7..1a0cdc5e48 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py @@ -63,7 +63,7 @@ from zarr_metadata.v2.array import ZARR_V2_ARRAY_METADATA_STORE_KEY from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY from zarr_metadata.v2.consolidated import ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY -from zarr_metadata.v2.definition import CORE_V2 +from zarr_metadata.v2.definition import CORE_V2_CONTEXT from zarr_metadata.v2.group import ZARR_V2_GROUP_METADATA_STORE_KEY, ZarrV2GroupMetadataJSON if TYPE_CHECKING: @@ -136,8 +136,8 @@ def __reduce__(self) -> str | tuple[object, ...]: def read_group_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2GroupMetadataReading: - """Read `value`, a v2 group document in the merged form (`.zgroup` plus `attributes` from `.zattrs`), in `context` (`CORE_V2` when none is given), whatever it holds: every problem, and the model when there are none. A group has no fields for a scope to read; `context` is taken for consistency with every other v2 reader.""" - context = scoped(context, CORE_V2) + """Read `value`, a v2 group document in the merged form (`.zgroup` plus `attributes` from `.zattrs`), in `context` (`CORE_V2_CONTEXT` when none is given), whatever it holds: every problem, and the model when there are none. A group has no fields for a scope to read; `context` is taken for consistency with every other v2 reader.""" + context = scoped(context, CORE_V2_CONTEXT) problems = _group_problems_v2(value) if len(problems) != 0: return ZarrV2GroupMetadataReading(problems=problems) @@ -173,7 +173,7 @@ def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: def read_node_metadata_v2( files: Mapping[str, object], *, context: ZarrV2Context | None = None ) -> ZarrV2NodeMetadataReading: - """Read the v2 node described by `files`, in `context` (`CORE_V2` when none is given), as the node type its files declare. + """Read the v2 node described by `files`, in `context` (`CORE_V2_CONTEXT` when none is given), as the node type its files declare. `files` maps each of the node's files to its decoded JSON: `.zarray` or `.zgroup`, and `.zattrs` when there is one. They can come from a @@ -194,7 +194,7 @@ def read_node_metadata_v2( v2 counterpart of `read_node_metadata_v3`, which reads one `zarr.json` and its `node_type`. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) is_array = ZARR_V2_ARRAY_METADATA_STORE_KEY in files is_group = ZARR_V2_GROUP_METADATA_STORE_KEY in files if is_array and is_group: @@ -234,7 +234,7 @@ def node_metadata_from_key_value_v2( mapping: Mapping[StoreKey, bytes], *, context: ZarrV2Context | None = None ) -> ZarrV2NodeMetadata: """Return the model of the v2 node whose files are in `mapping`, a store's keys and bytes: each file is decoded, then read as `read_node_metadata_v2` reads it. Raises `MetadataValidationError` with every problem, including a file whose bytes are not JSON, and `TypeError` for a scope of another format before any bytes are decoded.""" - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) files, problems = decoded_node_files_v2(mapping) if len(problems) != 0: raise MetadataValidationError(problems) @@ -278,7 +278,7 @@ class ZarrV2GroupMetadata(Keyed): node_type: Final[Literal["group"]] = "group" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) problems = _group_problems_v2(document) if len(problems) != 0: raise MetadataValidationError(problems) @@ -368,7 +368,7 @@ def update(self, **members: Unpack[ZarrV2GroupMetadataUpdate]) -> ZarrV2GroupMet def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: """Return this document read in `context`: the same group, holding that scope.""" - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) return self._of(self._document, context) def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: @@ -406,7 +406,7 @@ def from_key_value( valid, with every problem the read finds. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) files, problems = decoded_node_files_v2( mapping, (ZARR_V2_GROUP_METADATA_STORE_KEY, ZARR_V2_ATTRIBUTES_STORE_KEY) ) @@ -449,8 +449,8 @@ def __reduce__(self) -> str | tuple[object, ...]: def read_consolidated_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2ConsolidatedMetadataReading: - """Read `value`, a v2 `.zmetadata` document, in `context` (`CORE_V2` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV2ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" - context = scoped(context, CORE_V2) + """Read `value`, a v2 `.zmetadata` document, in `context` (`CORE_V2_CONTEXT` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV2ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" + context = scoped(context, CORE_V2_CONTEXT) document, nodes, files, problems = _read_consolidated_v2(value, context) if len(problems) != 0: return ZarrV2ConsolidatedMetadataReading(problems=problems) @@ -487,7 +487,7 @@ class ZarrV2ConsolidatedMetadata(Keyed): zarr_consolidated_format: Final = 1 def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) read, nodes, files, problems = _read_consolidated_v2(document, context) if len(problems) != 0: raise MetadataValidationError(problems) @@ -576,7 +576,7 @@ def __reduce__(self) -> tuple[type[ZarrV2ConsolidatedMetadata], tuple[object, Za def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: """Return this document with every node read in `context`, whatever that changes. Raises `MetadataValidationError` when a node has a problem in `context`.""" - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) return type(self)(self._document, context=context) def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: @@ -586,7 +586,7 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2Consolidated node's entry. Raises `MetadataValidationError` when a gain exposes a problem. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) conflicts: list[Conflict] = [] for path, node in self._metadata.items(): if not isinstance(node, ZarrV2ArrayMetadata): @@ -633,7 +633,7 @@ def from_key_value( are not JSON, or the document is not valid. Raises `TypeError` for a scope of another format before the bytes are decoded. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) return cls( load_store_json(mapping, ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY), context=context ) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index 155d8e8019..a189856678 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -80,7 +80,7 @@ # A runtime import: `get_type_hints` resolves the public signatures. from zarr_metadata.v3._context import ZarrV3Context # noqa: TC001 -from zarr_metadata.v3._core import CORE_AND_EXTENSIONS +from zarr_metadata.v3._core import CORE_AND_EXTENSIONS_CONTEXT from zarr_metadata.v3._definition import GroupExtraFieldDefinition from zarr_metadata.v3._hierarchy import hierarchy_problems, path_faults from zarr_metadata.v3.consolidated import ( @@ -169,7 +169,7 @@ class ZarrV3GroupMetadata(Keyed): the document as written, refined, and `context` is the scope. `attributes` and `consolidated_metadata` are views of the values the read refined. Each member the spec does not define is an extra field, - in `extra_fields`, as the scope read it. `CORE` reads a + in `extra_fields`, as the scope read it. `CORE_CONTEXT` reads a `consolidated_metadata` member as the envelope the core spec fixes, and no definition reads the documents the member lists: they follow a convention of the reference implementation, not the core spec. To @@ -186,7 +186,7 @@ class ZarrV3GroupMetadata(Keyed): node_type: Final[Literal["group"]] = "group" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_group_v3(document, context) if members is None or len(reading.problems) != 0: raise MetadataValidationError(reading.problems) @@ -281,7 +281,7 @@ def attributes(self) -> Mapping[str, JSONValue]: def extra_fields( self, ) -> Mapping[str, AcceptedField[GroupExtraFieldDefinition[Any]] | UnclaimedField]: - """Each member the spec does not define, by key, as the scope read it: an `AcceptedField` when a definition in scope claims the key (as `CORE` claims `consolidated_metadata`), and an `UnclaimedField` otherwise.""" + """Each member the spec does not define, by key, as the scope read it: an `AcceptedField` when a definition in scope claims the key (as `CORE_CONTEXT` claims `consolidated_metadata`), and an `UnclaimedField` otherwise.""" return self._shown[3] @property @@ -365,7 +365,7 @@ def update(self, **members: Unpack[ZarrV3GroupMetadataUpdate]) -> ZarrV3GroupMet def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadata: """Return this document read in `context`, whatever that changes. Raises `MetadataValidationError` when the document has a problem in `context`. The existing reading is reused when `context` reads every claim identically.""" - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) if disagreements(context, claims_of(self._reading.fields())).agrees: return self._of(self._document, context, self._reading, self._members) return type(self)(self._document, context=context) @@ -382,7 +382,7 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadat in `context`, and `ZarrV3ConsolidatedMetadata.refined_in` checks that they read the same there. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) found = disagreements(context, claims_of(self._reading.fields())) if len(found.conflicts) != 0: raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) @@ -409,7 +409,7 @@ def create_default( **overrides: Unpack[ZarrV3GroupMetadataUpdate], ) -> ZarrV3GroupMetadata: """Return a group with no attributes, with any document members in `overrides` applied, read in `context`. `overrides` takes what `update` takes, including `UNSET` to drop a member and a `ZarrV3ConsolidatedMetadata` as `consolidated_metadata`. Raises `MetadataValidationError` when the resulting document has a problem.""" - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) changes = member_changes( overrides, ZarrV3GroupMetadataUpdate, GroupExtraFieldDefinition.spec_members ) @@ -426,7 +426,7 @@ def from_json( Raises `MetadataValidationError` with every problem the read finds. A `consolidated_metadata` of `null`, which zarr-python 3.0 and 3.1 wrote, is a value the document wrote that is not an object. In a - scope that claims the member, as `CORE` does, this is a problem, + scope that claims the member, as `CORE_CONTEXT` does, this is a problem, because the spec requires an object. Use `read_repaired_node_metadata_v3` to read such a store. A member the spec does not define is kept in `extra_fields`. @@ -443,7 +443,7 @@ def from_key_value( are not JSON, or the document is not valid. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) return cls(load_store_json(mapping, ZARR_V3_GROUP_METADATA_STORE_KEY), context=context) @@ -495,7 +495,7 @@ class ZarrV3ConsolidatedMetadata(Keyed): must_understand: Final = False def __init__(self, member: object, context: ZarrV3Context | None = None) -> None: - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) # The member is always under a group's key, even when read on its # own, so nesting depth is counted from there, as in the group. readings, members, document, problems = _read_consolidated_v3( @@ -561,7 +561,7 @@ def __reduce__(self) -> tuple[type[ZarrV3ConsolidatedMetadata], tuple[object, Za def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ConsolidatedMetadata: """Return this member with every document read in `context`, whatever that changes. Raises `MetadataValidationError` when a document has a problem in `context`. The existing readings are reused when `context` reads every claim of every document identically.""" - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) if all( disagreements(context, claims_of(node.reading.fields())).agrees for node in self._metadata.values() @@ -577,7 +577,7 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3Consolidated document's path. Raises `MetadataValidationError` when a gain exposes a problem. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) conflicts = [ dataclasses.replace( conflict, loc=("metadata", path, *(() if conflict.loc is None else conflict.loc)) @@ -681,8 +681,8 @@ def __reduce__(self) -> str | tuple[object, ...]: def read_consolidated_metadata_v3( value: object, *, context: ZarrV3Context | None = None ) -> ZarrV3ConsolidatedMetadataReading: - """Read `value`, a group's `consolidated_metadata` member, in `context` (`CORE_AND_EXTENSIONS` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV3ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" - context = scoped(context, CORE_AND_EXTENSIONS) + """Read `value`, a group's `consolidated_metadata` member, in `context` (`CORE_AND_EXTENSIONS_CONTEXT` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV3ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) readings, members, document, problems = _read_consolidated_v3( value, context, (ZARR_V3_CONSOLIDATED_METADATA_KEY,) ) @@ -737,7 +737,7 @@ def read_node_metadata_v3( read `node_type` from unvalidated JSON, and a document in another format is reported as not v3. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) node_type, problems = _node_type(value) if node_type == "array": return read_array_metadata_v3(value, context=context) @@ -761,7 +761,7 @@ def node_metadata_from_json_v3( `MetadataValidationError` with every problem `read_node_metadata_v3` finds, including a `node_type` that is neither array nor group. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading = read_node_metadata_v3(data, context=context) if reading.metadata is None: raise MetadataValidationError(reading.problems) @@ -776,7 +776,7 @@ def node_metadata_from_key_value_v3( Raises `MetadataValidationError` when the key is missing, its bytes are not JSON, or the document is not a valid array or group. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) # Array documents and group documents are both stored at `zarr.json`. document = load_store_json(mapping, ZARR_V3_GROUP_METADATA_STORE_KEY) return node_metadata_from_json_v3(document, context=context) @@ -853,13 +853,13 @@ def read_group_metadata_v3( reader must understand each one that does not declare `must_understand: false`; the model reports these as `must_understand_fields`. A `consolidated_metadata` member, if - present and claimed by the scope, as `CORE` claims it, is checked + present and claimed by the scope, as `CORE_CONTEXT` claims it, is checked against the envelope the core spec fixes. The documents it lists are not read; `read_consolidated_metadata_v3` reads them. This is the form of `ZarrV3GroupMetadata(value, context)` that returns problems instead of raising them. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_group_v3(value, context) if members is None or members.document is None: return reading diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_repair.py b/packages/zarr-metadata/src/zarr_metadata/model/_repair.py index d16663b1fe..679a584fc8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_repair.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_repair.py @@ -45,9 +45,9 @@ ) from zarr_metadata.model._group_v3 import ZarrV3NodeMetadataReading, read_node_metadata_v3 from zarr_metadata.v2.array import ZARR_V2_ARRAY_METADATA_STORE_KEY -from zarr_metadata.v2.definition import CORE_V2 +from zarr_metadata.v2.definition import CORE_V2_CONTEXT from zarr_metadata.v2.group import ZARR_V2_GROUP_METADATA_STORE_KEY -from zarr_metadata.v3._core import CORE_AND_EXTENSIONS +from zarr_metadata.v3._core import CORE_AND_EXTENSIONS_CONTEXT from zarr_metadata.v3.consolidated import ZARR_V3_CONSOLIDATED_METADATA_KEY if TYPE_CHECKING: @@ -278,7 +278,7 @@ def read_repaired_node_metadata_v3( Content that no repair applies to is read as it is, and reported the same way `read_node_metadata_v3` reports it. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) repaired, repairs = repair_node_metadata_v3(value) return ZarrV3RepairedNodeMetadataReading( read_node_metadata_v3(repaired, context=context), repairs @@ -482,7 +482,7 @@ def read_repaired_node_metadata_v2( Content that no repair applies to is read as it is, and reported the same way `read_node_metadata_v2` reports it. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) repaired, repairs = repair_node_metadata_v2(files) return ZarrV2RepairedNodeMetadataReading( read_node_metadata_v2(repaired, context=context), repairs @@ -510,7 +510,7 @@ def read_repaired_consolidated_metadata_v2( read as it is, and reported the same way `read_consolidated_metadata_v2` reports it. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) repaired, repairs = repair_consolidated_metadata_v2(value) return ZarrV2RepairedConsolidatedMetadataReading( read_consolidated_metadata_v2(repaired, context=context), repairs diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index f369f2beec..84ded7390e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -20,7 +20,7 @@ `"zarr_metadata_context_v2"` item. This lets a model with fields of both formats give a scope for each format. - If no scope is given for a format, its field types read in the default - scope for that format, `CORE_AND_EXTENSIONS` or `CORE_V2`. + scope for that format, `CORE_AND_EXTENSIONS_CONTEXT` or `CORE_V2_CONTEXT`. A mapping key that starts like these keys but is not one of them raises `TypeError`, so a misspelled key is not read as no scope. To fix a field's @@ -128,8 +128,8 @@ class ArrayManifest(BaseModel): from zarr_metadata.definition._context import scoped from zarr_metadata.definition._definition import Context from zarr_metadata.definition._scope import ScopeConflictError -from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, ZarrV3Context +from zarr_metadata.v2.definition import CORE_V2_CONTEXT, ZarrV2Context +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, ZarrV3Context if TYPE_CHECKING: from collections.abc import Callable, Mapping @@ -321,7 +321,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json, _model.ZarrV3ArrayMetadata.refined_in, - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, CONTEXT_KEY, ), json_schema_input_type=_ZarrV3ArrayMetadataSchema, @@ -337,7 +337,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json, _model.ZarrV2ArrayMetadata.refined_in, - CORE_V2, + CORE_V2_CONTEXT, CONTEXT_KEY_V2, ), json_schema_input_type=_ZarrV2ArrayMetadataSchema, @@ -353,7 +353,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json, _model.ZarrV3GroupMetadata.refined_in, - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, CONTEXT_KEY, ), json_schema_input_type=_ZarrV3GroupMetadataSchema, @@ -369,7 +369,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json, _model.ZarrV2GroupMetadata.refined_in, - CORE_V2, + CORE_V2_CONTEXT, CONTEXT_KEY_V2, ), json_schema_input_type=_ZarrV2GroupMetadataSchema, @@ -385,7 +385,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV3ConsolidatedMetadata, _model.ZarrV3ConsolidatedMetadata.from_json, _model.ZarrV3ConsolidatedMetadata.refined_in, - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, CONTEXT_KEY, ), json_schema_input_type=_ZarrV3ConsolidatedMetadataSchema, @@ -404,7 +404,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV2ConsolidatedMetadata, _model.ZarrV2ConsolidatedMetadata.from_json, _model.ZarrV2ConsolidatedMetadata.refined_in, - CORE_V2, + CORE_V2_CONTEXT, CONTEXT_KEY_V2, ), json_schema_input_type=_ZarrV2ConsolidatedMetadataSchema, @@ -419,15 +419,15 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _FIELD_PARTS: Final[Mapping[type[Any], tuple[ZarrV3Context | ZarrV2Context, object]]] = ( MappingProxyType( { - _model.ZarrV3ArrayMetadata: (CORE_AND_EXTENSIONS, _ZarrV3ArrayMetadataSchema), - _model.ZarrV2ArrayMetadata: (CORE_V2, _ZarrV2ArrayMetadataSchema), - _model.ZarrV3GroupMetadata: (CORE_AND_EXTENSIONS, _ZarrV3GroupMetadataSchema), - _model.ZarrV2GroupMetadata: (CORE_V2, _ZarrV2GroupMetadataSchema), + _model.ZarrV3ArrayMetadata: (CORE_AND_EXTENSIONS_CONTEXT, _ZarrV3ArrayMetadataSchema), + _model.ZarrV2ArrayMetadata: (CORE_V2_CONTEXT, _ZarrV2ArrayMetadataSchema), + _model.ZarrV3GroupMetadata: (CORE_AND_EXTENSIONS_CONTEXT, _ZarrV3GroupMetadataSchema), + _model.ZarrV2GroupMetadata: (CORE_V2_CONTEXT, _ZarrV2GroupMetadataSchema), _model.ZarrV3ConsolidatedMetadata: ( - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, _ZarrV3ConsolidatedMetadataSchema, ), - _model.ZarrV2ConsolidatedMetadata: (CORE_V2, _ZarrV2ConsolidatedMetadataSchema), + _model.ZarrV2ConsolidatedMetadata: (CORE_V2_CONTEXT, _ZarrV2ConsolidatedMetadataSchema), } ) ) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py index caf7b71a05..e966cd9087 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -12,7 +12,7 @@ from hypothesis import strategies as st -from zarr_metadata.definition._definition import AcceptedField +from zarr_metadata.definition._definition import AcceptedField, EmptyConfiguration from zarr_metadata.testing._data_type import ( COMPLEX_WIDTHS, FLOAT_WIDTHS, @@ -30,6 +30,8 @@ from zarr_metadata.v3.codec.gzip import GZIP_CODEC from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC +from zarr_metadata.v3.codec.vlen_bytes import VLEN_BYTES_CODEC +from zarr_metadata.v3.codec.vlen_utf8 import VLEN_UTF8_CODEC from zarr_metadata.v3.codec.zstd import ZSTD_CODEC, ZSTD_MAX_LEVEL, ZSTD_MIN_LEVEL from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE @@ -155,6 +157,17 @@ def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | return st.fixed_dictionaries({}, optional={"offset": offset, "scale": scale}) +def _vlen(definition: CodecDefinition[Any]) -> CodecStrategy: + """A codec with nothing to configure that takes only some data types: none for a chunk of another, which its own chunk rules say.""" + return CodecStrategy( + configurations=lambda chunk: ( + None + if any(definition.chunk_rules(EmptyConfiguration(), {}, chunk)) + else EMPTY_CONFIGURATIONS + ) + ) + + def _unchunked(configurations: st.SearchStrategy[Mapping[str, JSONValue]]) -> CodecStrategy: """A codec whose configurations do not depend on the chunk it is handed.""" return CodecStrategy(configurations=lambda _chunk: configurations) @@ -167,6 +180,8 @@ def _unchunked(configurations: st.SearchStrategy[Mapping[str, JSONValue]]) -> Co GZIP_CODEC: _unchunked(st.fixed_dictionaries({"level": st.integers(0, 9)})), SCALE_OFFSET_CODEC: CodecStrategy(configurations=_scale_offset), TRANSPOSE_CODEC: CodecStrategy(configurations=_transpose), + VLEN_BYTES_CODEC: _vlen(VLEN_BYTES_CODEC), + VLEN_UTF8_CODEC: _vlen(VLEN_UTF8_CODEC), ZSTD_CODEC: _unchunked( st.fixed_dictionaries( {"level": st.integers(ZSTD_MIN_LEVEL, ZSTD_MAX_LEVEL)}, diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py index dd06df28a0..b592b2d979 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py @@ -21,6 +21,10 @@ from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE +from zarr_metadata.v3.data_type.fixed_length_utf32 import ( + FIXED_LENGTH_UTF32_DATA_TYPE, + FixedLengthUtf32Configuration, +) from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE @@ -28,6 +32,10 @@ from zarr_metadata.v3.data_type.int16 import INT16_DATA_TYPE from zarr_metadata.v3.data_type.int32 import INT32_DATA_TYPE from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE +from zarr_metadata.v3.data_type.null_terminated_bytes import ( + NULL_TERMINATED_BYTES_DATA_TYPE, + NullTerminatedBytesConfiguration, +) from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE, RawBytesConfiguration @@ -36,6 +44,11 @@ from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE +from zarr_metadata.v3.data_type.variable_length_bytes import VARIABLE_LENGTH_BYTES_DATA_TYPE +from zarr_metadata.v3.data_type.zarr_python_raw_bytes import ( + ZARR_PYTHON_RAW_BYTES_DATA_TYPE, + ZarrPythonRawBytesConfiguration, +) if TYPE_CHECKING: from collections.abc import Mapping @@ -148,9 +161,33 @@ def _complex_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: ).map(_base64) """A `bytes` fill value: byte values, or the bytes in standard-alphabet base64.""" +_LENGTH_BYTES: Final = st.integers(1, _MAX_BYTES) +"""The `length_bytes` of zarr-python's fixed-length byte types.""" + + +def _base64_fill_values( + configuration: Mapping[str, JSONValue], +) -> st.SearchStrategy[JSONValue]: + """At most `length_bytes` bytes in standard-alphabet base64, as zarr-python's fixed-length byte types take them.""" + length_bytes = cast("ZarrPythonRawBytesConfiguration", configuration)["length_bytes"] + return st.binary(max_size=length_bytes).map(_base64) + + _TEXT: Final = st.text(st.characters(exclude_categories=("Cs",)), max_size=16) """Text a JSON document can hold: no lone surrogate, which is no Unicode scalar value.""" +_MAX_CODE_POINTS: Final = 16 +"""The most code points a drawn `fixed_length_utf32` value holds.""" + + +def _fixed_length_utf32_fill_values( + configuration: Mapping[str, JSONValue], +) -> st.SearchStrategy[JSONValue]: + """Text of at most one code point for each 4 bytes of the size, some of them trailing `U+0000` padding.""" + capacity = cast("FixedLengthUtf32Configuration", configuration)["length_bytes"] // 4 + return st.text(st.characters(exclude_categories=("Cs",)) | st.just("\x00"), max_size=capacity) + + _NUMPY_TIME: Final = DataTypeStrategy( configurations=st.fixed_dictionaries( { @@ -184,6 +221,25 @@ def _complex_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: ), BYTES_DATA_TYPE: _fill_values_of(_BYTES_FILL_VALUES), STRING_DATA_TYPE: _fill_values_of(_TEXT), + NULL_TERMINATED_BYTES_DATA_TYPE: DataTypeStrategy( + configurations=_LENGTH_BYTES.map( + lambda length_bytes: NullTerminatedBytesConfiguration(length_bytes=length_bytes) + ), + fill_values=_base64_fill_values, + ), + VARIABLE_LENGTH_BYTES_DATA_TYPE: _fill_values_of(st.binary(max_size=_MAX_BYTES).map(_base64)), + ZARR_PYTHON_RAW_BYTES_DATA_TYPE: DataTypeStrategy( + configurations=_LENGTH_BYTES.map( + lambda length_bytes: ZarrPythonRawBytesConfiguration(length_bytes=length_bytes) + ), + fill_values=_base64_fill_values, + ), + FIXED_LENGTH_UTF32_DATA_TYPE: DataTypeStrategy( + configurations=st.integers(1, _MAX_CODE_POINTS).map( + lambda count: FixedLengthUtf32Configuration(length_bytes=4 * count) + ), + fill_values=_fixed_length_utf32_fill_values, + ), NUMPY_DATETIME64_DATA_TYPE: _NUMPY_TIME, NUMPY_TIMEDELTA64_DATA_TYPE: _NUMPY_TIME, } diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py index fd13c43839..8491c9c171 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py @@ -60,7 +60,7 @@ def test_my_reader_reads_every_blosc_codec(field): Strategies, ) from zarr_metadata.v3._context import ZarrV3Context -from zarr_metadata.v3._core import CORE +from zarr_metadata.v3._core import CORE_CONTEXT from zarr_metadata.v3._definition import Chunk, DataTypeDefinition if TYPE_CHECKING: @@ -181,7 +181,7 @@ def _written_bare(definition: Definition[Any]) -> bool: """ if not isinstance(definition, DataTypeDefinition): return False - return CORE.claimant(DataTypeDefinition, definition.name) is definition + return CORE_CONTEXT.claimant(DataTypeDefinition, definition.name) is definition @st.composite diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py index 741ff5cfb5..af6dda6c22 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py @@ -6,12 +6,16 @@ and `data_type`, and the v2 kinds and scope are in `definition`. This module also exports every v2 definition (`GZIP_CODEC_V2`, -`FLOAT_DATA_TYPE_V2`, ...), the scope `CORE_V2`, and the scope class +`FLOAT_DATA_TYPE_V2`, ...), the scope `CORE_V2_CONTEXT`, and the scope class `ZarrV2Context`, so that one import builds or updates a scope. Each is the same object as in its own submodule, where a definition's configuration types and constants are. """ +from typing import TYPE_CHECKING + +from zarr_metadata._renamed import renamed_scope +from zarr_metadata.v2._core import RENAMED_SCOPES_V2 from zarr_metadata.v2.codec.checksum import ( ADLER32_CODEC_V2, CRC32_CODEC_V2, @@ -59,7 +63,7 @@ DATETIME64_DATA_TYPE_V2, TIMEDELTA64_DATA_TYPE_V2, ) -from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context +from zarr_metadata.v2.definition import CORE_V2_CONTEXT, ZarrV2Context __all__ = [ "ADLER32_CODEC_V2", @@ -70,7 +74,7 @@ "BYTES_DATA_TYPE_V2", "BZ2_CODEC_V2", "COMPLEX_DATA_TYPE_V2", - "CORE_V2", + "CORE_V2_CONTEXT", "CRC32C_CODEC_V2", "CRC32_CODEC_V2", "DATETIME64_DATA_TYPE_V2", @@ -98,3 +102,11 @@ "ZSTD_CODEC_V2", "ZarrV2Context", ] + + +if not TYPE_CHECKING: + # Hidden from type checkers, which would otherwise read every missing + # name of this module as a scope. + def __getattr__(name: str) -> object: + """`CORE_V2`, the scope's name before it ended in `_CONTEXT`, with a `DeprecationWarning`.""" + return renamed_scope(__name__, name, RENAMED_SCOPES_V2) diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_core.py b/packages/zarr-metadata/src/zarr_metadata/v2/_core.py new file mode 100644 index 0000000000..0daaa6c344 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_core.py @@ -0,0 +1,17 @@ +"""The scope this package provides for Zarr v2.""" + +from __future__ import annotations + +from typing import Final + +from zarr_metadata.v2._context import ZarrV2Context +from zarr_metadata.v2.codec import CODECS_V2 +from zarr_metadata.v2.data_type import DATA_TYPES_V2 + +CORE_V2_CONTEXT: Final = ZarrV2Context(data_types=DATA_TYPES_V2, codecs=CODECS_V2) +"""The data types that zarr-python 2.x writes and the codecs that numcodecs 0.16 configures.""" + +RENAMED_SCOPES_V2: Final = {"CORE_V2": ("CORE_V2_CONTEXT", CORE_V2_CONTEXT)} +"""The v2 scope's name before it ended in `_CONTEXT`, with the scope's name now and the scope.""" + +__all__ = ["CORE_V2_CONTEXT", "RENAMED_SCOPES_V2"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/definition.py b/packages/zarr-metadata/src/zarr_metadata/v2/definition.py index 8cf4990992..d0da681c58 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/definition.py @@ -4,10 +4,10 @@ in `compressor` and `filters`. Zarr v2 has no extension registry, but it still has a scope: the data types that zarr-python 2.x writes (one definition per NumPy family), and the codecs that numcodecs 0.16 -configures (one per id that this package models). `CORE_V2` is that +configures (one per id that this package models). `CORE_V2_CONTEXT` is that scope. To read one field in it, call `zarr_metadata.definition.resolve` with the field's kind, as for a v3 field: -`resolve("