diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a4141d7356..13145a9251 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,7 +34,7 @@ jobs: strategy: matrix: - python-version: ['3.12', '3.13', '3.14'] + python-version: ['3.12', '3.13', '3.14', '3.15'] dependency-set: ["minimal", "optional"] os: ["ubuntu-latest"] include: @@ -101,7 +101,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.12', "3.14"] + python-version: ['3.12', "3.14", "3.15"] dependency-set: ["upstream", "min_deps"] exclude: - python-version: "3.14" @@ -201,6 +201,32 @@ jobs: run: | HATCH_ENV=test.py3.13-minimal just benchmark + examples: + name: Example scripts + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 # required for hatch version discovery when uv builds zarr + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.13' + cache: 'pip' + - name: Install uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + - name: Install Hatch + run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 + - name: Run Examples + run: | + HATCH_ENV=test.py3.13-minimal just test-examples + test-complete: name: Test complete @@ -209,7 +235,8 @@ jobs: test, test-upstream-and-min-deps, doctests, - benchmarks + benchmarks, + examples ] if: always() runs-on: ubuntu-latest diff --git a/Justfile b/Justfile index b3f187e231..df96003eef 100644 --- a/Justfile +++ b/Justfile @@ -64,6 +64,10 @@ hypothesis *args: doctest *args: hatch run doctest:pytest tests/test_docs.py -v "$@" +# Run each example script against the local checkout, in its PEP 723 environment +test-examples *args: + hatch run {{ quote(hatch_env) }}:pytest --run-examples tests/test_examples.py -v "$@" + # Run the benchmark suite benchmark *args: hatch run {{ quote(hatch_env) }}:pytest --benchmark-enable tests/benchmarks "$@" diff --git a/changes/4560.bugfix.md b/changes/4560.bugfix.md new file mode 100644 index 0000000000..71f396e1b8 --- /dev/null +++ b/changes/4560.bugfix.md @@ -0,0 +1,4 @@ +Numcodecs filters such as `Delta` and `FixedScaleOffset` now pass chunks of +their declared shape to the next codec when writing. Before, the next codec +received a flattened 1-D chunk, so chaining a filter with `TransposeCodec` or +a 2-D image codec failed or lost data. diff --git a/examples/README.md b/examples/README.md index a6b5fa2179..fa664d2c58 100644 --- a/examples/README.md +++ b/examples/README.md @@ -19,7 +19,7 @@ examples/ To add a new example: 1. Create a new subdirectory: `examples/my_example/` -2. Add your Python code: `examples/my_example/my_example.py` +2. Add your Python code: `examples/my_example/my_example.py`. Start it with a [PEP 723](https://peps.python.org/pep-0723/) header that lists its dependencies, including `zarr`. `just test-examples` runs every example against your local checkout, and CI runs it on every pull request. 3. Create documentation: `examples/my_example/README.md` 4. Create a documentation page at `docs/user-guide/examples/my_example.md`. The documentation page should simply link to the `README.md` and the source code, e.g.: diff --git a/examples/codec_pipeline_performance/codec_pipeline_performance.py b/examples/codec_pipeline_performance/codec_pipeline_performance.py index b821f3e6a7..d146c51f4a 100644 --- a/examples/codec_pipeline_performance/codec_pipeline_performance.py +++ b/examples/codec_pipeline_performance/codec_pipeline_performance.py @@ -1,5 +1,5 @@ # /// script -# requires-python = ">=3.11" +# requires-python = ">=3.12" # dependencies = [ # "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", # "numpy", diff --git a/examples/sharding_coalescing/sharding_coalescing.py b/examples/sharding_coalescing/sharding_coalescing.py index 5da62ed806..66838d2255 100644 --- a/examples/sharding_coalescing/sharding_coalescing.py +++ b/examples/sharding_coalescing/sharding_coalescing.py @@ -1,5 +1,5 @@ # /// script -# requires-python = ">=3.11" +# requires-python = ">=3.12" # dependencies = [ # "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", # "numpy", diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index 3c5513d1dd..a01ed26842 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -41,14 +41,14 @@ closely model the content of the Zarr specifications, such as: `TypedDict` with its rules, and the scope that decides which definitions read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every definition and scope of their format, such as `BYTES_CODEC` and - `CORE_AND_EXTENSIONS`. + `CORE_AND_EXTENSIONS_CONTEXT`. - **JSON type checking** ([`zarr_metadata.typed_json`](https://zarr-metadata.readthedocs.io/en/latest/api/typed_json/)): `check` checks a JSON value against any `TypedDict` and reports every problem with its location. - **Problems** ([`zarr_metadata.problems`](https://zarr-metadata.readthedocs.io/en/latest/api/problems/)): the problem values every reader returns, and the error that carries them. - **JSON Schema** ([`zarr_metadata.json_schema`](https://zarr-metadata.readthedocs.io/en/latest/api/json_schema/)): the - JSON Schema of a `TypedDict`, a v3 field, or a whole `zarr.json`. + JSON Schema of a `TypedDict`, a v3 declaration, or a whole `zarr.json`. - **Optional Pydantic integration** ([`zarr_metadata.pydantic`](https://zarr-metadata.readthedocs.io/en/latest/api/pydantic/), requires Pydantic 2.13 or newer): each model as a Pydantic field type that validates raw documents through the same strict parser. @@ -104,7 +104,9 @@ of context-free consistency rules, e.g.: In a v3 document, validation also checks each extension point: the data type, the chunk grid, the chunk key encoding, each codec, and each storage transformer. Each one is checked by the definition that claims its name in -a scope. The scope is `CORE_AND_EXTENSIONS` unless a `context` is passed. +a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. +`ZARR_PYTHON_CONTEXT` adds the data types that zarr-python writes but no +specification registers. If the definition rejects a configuration, the reader rejects it too, and a configuration key the definition does not declare is reported as `unknown_key`. If no definition in the scope claims a name, the reader @@ -211,38 +213,39 @@ with one of them is refused: `read_array_metadata_v3` reads a document once and returns everything the read found: -- each field as the scope read it: an `AcceptedField` when a definition - claims its name, an `UnclaimedField` when none does, or a - `RefusedField`, together with the field's location and the kind it was +- each declaration as the scope read it: an `AcceptedDeclaration` when a definition + claims its name, an `UnclaimedDeclaration` when none does, or a + `RefusedDeclaration`, together with the declaration's location and the kind it was read as; - each codec, with the chunk it receives; - every problem; - the model, when there are no problems. `from_json` returns that model, or raises the problems. A consumer can -apply its own policy by walking the fields, without reading anything -twice. `with_problems` pairs each field with its problems: those located -in the field and in the fields it contains. This matches how zod's +apply its own policy by walking the declarations, without reading anything +twice. `with_problems` pairs each declaration with its problems: those located +in the declaration and in the declarations it contains. This matches how zod's `flattenError` groups issues. `canonical_of` returns the simplest -spelling of a field that has no problems, without reading it again. For -example, the code below finds the fields that go beyond the core spec (a -field with no name is already a problem), and the simplest spelling of -each field: +spelling of a declaration that has no problems, without reading it again. For +example, the code below finds the declarations that go beyond the core spec (a +declaration with no name is already a problem), and the simplest spelling of +each declaration: ```python test="true" session="readme" from zarr_metadata.model import read_array_metadata_v3 from zarr_metadata.definition import canonical_of, with_problems -from zarr_metadata.v3.definition import CORE +from zarr_metadata.v3.definition import CORE_CONTEXT reading = read_array_metadata_v3(raw) beyond_core = [ loc - for loc, field in reading.fields() - if field.name is not None and CORE.claimant(field.read_as, field.name) is None + for loc, declaration in reading.declarations() + if declaration.name is not None + and CORE_CONTEXT.claimant(declaration.read_as, declaration.name) is None ] simplest = { - loc: canonical_of(field, problems) - for loc, field, problems in with_problems(reading.fields(), reading.problems) + loc: canonical_of(declaration, problems) + for loc, declaration, problems in with_problems(reading.declarations(), reading.problems) } metadata = reading.metadata # None when reading.problems is not empty ``` @@ -272,18 +275,19 @@ string against those rules. A top-level member that the spec does not define is an extra field. A scope can hold a definition for one, by its key, in the `array_extra_fields` or `group_extra_fields` slot of `ZarrV3Context`. A -member whose key a definition claims is read by that definition, like any -other field. Any other member is kept as written, as an `UnclaimedField`. +member whose key a definition claims is read by that definition: an extra field is a declaration +written as the member alone, `{key: value}`. Any other extra field is kept as +written, as an `UnclaimedDeclaration`. The scope is what a reader recognizes, so the model's `must_understand_fields` lists each unclaimed member that does not say `"must_understand": false`: the spec requires a reader to fail to open a -node with any such member. It covers members only. A data type, chunk +node with any such member. It covers extra fields only. A data type, chunk grid, chunk key encoding, codec or storage transformer that nothing in -scope claims is an `UnclaimedField` too, also when it sits inside another -field, such as a codec inside a shard. A model's `unclaimed_fields` lists -every field nothing in scope claims, at any depth, by its location, and +scope claims is an `UnclaimedDeclaration` too, also when it sits inside another +declaration, such as a codec inside a shard. A model's `unclaimed_declarations` lists +every declaration nothing in scope claims, at any depth, by its location, and leaves out the members that waive `must_understand`: a reader can open -the node when it is empty. `CORE` holds +the node when it is empty. `CORE_CONTEXT` holds `CONSOLIDATED_METADATA_EXTRA_FIELD`, which reads a group's `consolidated_metadata` as the envelope the spec fixes. A group's `consolidated()` reads the documents that member lists, in the group's @@ -291,14 +295,14 @@ own scope. A v3 model consists of its document and the scope it was read in. `ZarrV3ArrayMetadata(document, context=None)` reads the document in the -scope (`CORE_AND_EXTENSIONS` when none is given) and raises +scope (`CORE_AND_EXTENSIONS_CONTEXT` when none is given) and raises `MetadataValidationError` with every problem, so an invalid model is never built. `to_json` returns the document as it was written, and `to_key_value` writes that same document to store bytes. -Every typed member is a view of that read: each field as the scope read -it (an `AcceptedField` or an `UnclaimedField`), and `shape`, `attributes` -and the other members as the read refined them. These views are read-only +Every typed attribute is a view of that read: each declaration as the scope read +it (an `AcceptedDeclaration` or an `UnclaimedDeclaration`), and `shape`, `attributes` +and the members that hold no declaration as the read refined them. These views are read-only at every level: a list in the document is shown as a tuple, and an object as a read-only mapping. `to_json` returns plain containers. @@ -322,7 +326,7 @@ all of them with `ZarrV3ConsolidatedMetadata`, or with `read_consolidated_metadata_v3`, which returns problems instead of raising them. A document with a problem then fails only when it is read. Pass `context=group.context` to read the documents in the group's own -scope; with no `context`, they are read in `CORE_AND_EXTENSIONS`. +scope; with no `context`, they are read in `CORE_AND_EXTENSIONS_CONTEXT`. To write consolidated metadata, build a `ZarrV3ConsolidatedMetadata` and pass it to the group's `update`. The group reads the listed models in its @@ -332,10 +336,10 @@ models as entries. Such an entry is accepted when its claims refine into the model's scope, and refused at its path otherwise. To consolidate children that were read in several scopes, consolidate them in the `union` of those scopes. Every reader takes `context=None` for the -default scope, `CORE_AND_EXTENSIONS`. +default scope, `CORE_AND_EXTENSIONS_CONTEXT`. Two models are equal when they represent the same document, even if each -is spelled differently. Values the package interprets (each field, and the +is spelled differently. Values the package interprets (each declaration, and the fill value against its data type) are compared by their canonical spelling, as returned by `canonical_of` and `canonical_fill_value`. For example, `"NaN"` and `"0x7fc00000"` are the same `float32` fill value, @@ -355,7 +359,7 @@ the same data. The v3 spec defines its blosc codec, and says `noshuffle` ignores `typesize`. A v2 node with no `.zattrs` equals one with an empty `.zattrs`, as a v3 document with and without an empty `attributes` are equal. Values the package does -not interpret (attributes, extra fields, and the configuration of a field +not interpret (attributes, extra fields, and the configuration of a declaration that nothing in the scope claims) are compared as JSON text. JSON text distinguishes `true` from `1` and `-0.0` from `0.0`, and treats `NaN` as equal to itself. Equal models have equal hashes, but may write different @@ -365,11 +369,11 @@ model holds nothing that can be changed in place. `node_metadata_json_schema_v3` writes the rules the readers apply as a JSON Schema (draft 2020-12). Use it in an editor that checks a `zarr.json` while you write it, or with a validator in another language. Each -extension point is described as a field as its scope reads it: a claimed +extension point is described as a declaration as its scope reads it: a claimed configuration follows its definition's TypedDict, including bounds, and a name that nothing in the scope claims may have any configuration. The fill value must be a valid fill value for the data type the document names. -`field_json_schema_v3(kind, *, context=None)` writes the schema of one field, and +`declaration_json_schema_v3(kind, *, context=None)` writes the schema of one declaration, and `json_schema` writes the schema of any TypedDict as `check` reads it. All three are in `zarr_metadata.json_schema`. A schema describes each member on its own, not the rules that relate members to each other, so a document the schema accepts may still diff --git a/packages/zarr-metadata/changes/4574.bugfix.1.md b/packages/zarr-metadata/changes/4574.bugfix.1.md index cd474dc82b..0d387acd14 100644 --- a/packages/zarr-metadata/changes/4574.bugfix.1.md +++ b/packages/zarr-metadata/changes/4574.bugfix.1.md @@ -2,6 +2,6 @@ writes it. A record's problem was located under a `fields` key the document does not have, at `("dtype", "fields", 0, 1)`; it is now at `("dtype", 0, 1)`, and carries the value found there as its `input`. - `fields()` places each record's type there too. A definition's + `declarations()` places each record's type there too. A definition's functions still find the record types in `nested` at `("fields", 0, 1)`, their place in the configuration. diff --git a/packages/zarr-metadata/changes/4574.bugfix.7.md b/packages/zarr-metadata/changes/4574.bugfix.7.md index 386052625e..4d1227150c 100644 --- a/packages/zarr-metadata/changes/4574.bugfix.7.md +++ b/packages/zarr-metadata/changes/4574.bugfix.7.md @@ -1,4 +1,4 @@ -A definition's rules are located where the document writes the field. A +A definition's rules are located where the document writes the declaration. A `bytes` codec written as `"bytes"` or `{"name": "bytes"}`, with values of several bytes, reports its missing `endian` at `codecs.N.configuration`, the configuration it does not write, as a diff --git a/packages/zarr-metadata/changes/4574.bugfix.9.md b/packages/zarr-metadata/changes/4574.bugfix.9.md index ec6aa4385c..e51f5dc30f 100644 --- a/packages/zarr-metadata/changes/4574.bugfix.9.md +++ b/packages/zarr-metadata/changes/4574.bugfix.9.md @@ -5,8 +5,8 @@ problems in the merged document, so a `.zarray` that wrote `attributes` beside a `.zattrs` that was not an object gave two problems at `("attributes",)`, a location in neither file. An array node's reading - locates its fields there too: its new `at` is `(".zarray",)`, so - `fields()` gives `(".zarray", "dtype")` and `with_problems` pairs each - field with its problems. Its model reads the merged document, so the - model's own `reading` locates the field at `("dtype",)`, and no longer + locates its declarations there too: its new `at` is `(".zarray",)`, so + `declarations()` gives `(".zarray", "dtype")` and `with_problems` pairs each + declaration with its problems. Its model reads the merged document, so the + model's own `reading` locates the declaration at `("dtype",)`, and no longer equals the node reading. diff --git a/packages/zarr-metadata/changes/4575.feature.md b/packages/zarr-metadata/changes/4575.feature.md new file mode 100644 index 0000000000..1753597304 --- /dev/null +++ b/packages/zarr-metadata/changes/4575.feature.md @@ -0,0 +1 @@ +Added `zarr_metadata.testing.strategies`, Hypothesis strategies that draw valid Zarr v3 metadata declarations, behind a new `testing` extra (`pip install zarr-metadata[testing]`). There is one strategy for every codec, data type, chunk grid and chunk key encoding the package defines that holds no other declaration. `declarations` draws every envelope spelling the spec allows, or only the canonical one with `spellings="canonical"`. Codecs can be drawn for the `Chunk` they are handed, chunk grids for an array shape, and fill values for a data type. The strategies are a mapping keyed by definition, so an extension author can add strategies for their own definitions. diff --git a/packages/zarr-metadata/changes/4583.feature.md b/packages/zarr-metadata/changes/4583.feature.md new file mode 100644 index 0000000000..cbbe44f174 --- /dev/null +++ b/packages/zarr-metadata/changes/4583.feature.md @@ -0,0 +1 @@ +Add the `ZARR_PYTHON_CONTEXT` scope, which reads what zarr-python writes: `CORE_AND_EXTENSIONS_CONTEXT` plus zarr-python's own data types `null_terminated_bytes` (NumPy's `S`), `raw_bytes` (NumPy's `V`) and `variable_length_bytes`, which no specification registers. `CORE_AND_EXTENSIONS_CONTEXT` now also reads the `fixed_length_utf32` data type (NumPy's `U`) and the `vlen-utf8` and `vlen-bytes` codecs that zarr-extensions registers, so the strings zarr-python writes are checked instead of read as unclaimed. diff --git a/packages/zarr-metadata/changes/4583.removal.md b/packages/zarr-metadata/changes/4583.removal.md new file mode 100644 index 0000000000..9faa94ee31 --- /dev/null +++ b/packages/zarr-metadata/changes/4583.removal.md @@ -0,0 +1 @@ +Rename the scopes `CORE`, `CORE_AND_EXTENSIONS` and `CORE_V2` to `CORE_CONTEXT`, `CORE_AND_EXTENSIONS_CONTEXT` and `CORE_V2_CONTEXT`, so that a scope's name says it is a context. The old names, imported from `zarr_metadata.v3`, `zarr_metadata.v3.definition`, `zarr_metadata.v2` or `zarr_metadata.v2.definition`, still refer to the same scopes with a `DeprecationWarning`, and will be removed in a later release. diff --git a/packages/zarr-metadata/changes/4588.feature.md b/packages/zarr-metadata/changes/4588.feature.md new file mode 100644 index 0000000000..e066923667 --- /dev/null +++ b/packages/zarr-metadata/changes/4588.feature.md @@ -0,0 +1 @@ +`zarr_metadata.v3.data_type`, `zarr_metadata.v3.codec`, `zarr_metadata.v3.chunk_grid` and `zarr_metadata.v3.chunk_key_encoding` re-export every name their submodules export, so `from zarr_metadata.v3.data_type import BoolDataTypeName, Uint8DataTypeName` works again, as it did before 0.6.0. Each name is the same object as in its submodule, which stays its documented home. diff --git a/packages/zarr-metadata/changes/4590.doc.md b/packages/zarr-metadata/changes/4590.doc.md new file mode 100644 index 0000000000..16b7e4ce76 --- /dev/null +++ b/packages/zarr-metadata/changes/4590.doc.md @@ -0,0 +1 @@ +Adds a docstring and an API reference entry for every public object. diff --git a/packages/zarr-metadata/changes/4592.removal.md b/packages/zarr-metadata/changes/4592.removal.md new file mode 100644 index 0000000000..5080a931ea --- /dev/null +++ b/packages/zarr-metadata/changes/4592.removal.md @@ -0,0 +1,10 @@ +Rename the word for a value at an extension point: it is now a *declaration* (it was a *field*). A name/value pair in a JSON metadata object stays a *member*. The package already said *field* in the JSON sense in `extra_fields`, `ExtraFieldDefinition`, `ZarrV3ExtensionField` and `must_understand_fields`, after the spec's "additional fields", so one word meant two things. The public names change with no aliases: + +- `AcceptedField`, `UnclaimedField`, `RefusedField` and `ResolvedField` are now `AcceptedDeclaration`, `UnclaimedDeclaration`, `RefusedDeclaration` and `ResolvedDeclaration`. +- `ZarrV3DataTypeField`, `ZarrV3ChunkGridField`, `ZarrV3ChunkKeyEncodingField`, `ZarrV3CodecField`, `ZarrV3StaticCodecField`, `ZarrV3StorageTransformerField` and `ZarrV2DataTypeField` are now the same names ending in `Declaration`, and a kind's `field_aliases` is now `declaration_aliases`. +- `ZarrV3MetadataFieldJSON` is now `ZarrV3DeclarationJSON`, and `field_json_schema_v3` is now `declaration_json_schema_v3`. +- A reading's `fields()` is now `declarations()`, and a model's `unclaimed_fields` is now `unclaimed_declarations`. +- The first parameter of `with_problems` is now `declarations` (it was `fields`). +- The JSON Schemas that `node_metadata_json_schema_v3` and `declaration_json_schema_v3` write name each alias under `$defs` by its new name, so a `$ref` to `#/$defs/ZarrV3CodecField` is now `#/$defs/ZarrV3CodecDeclaration`, and likewise for the other kinds. + +Messages that said *field* or *metadata field* for an extension-point value now say *declaration*, as in "expected a declaration (string or extension object)". Prose that said *extra member* or *extension member* now says *extra field* and *extension field*, after the names. `extra_fields`, `ExtraFieldDefinition`, `ZarrV3ExtensionField`, `StructField` and the pydantic `...MetadataField` types keep their names. diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md index 4a23c5c2fb..a3e7be2665 100644 --- a/packages/zarr-metadata/docs/api/index.md +++ b/packages/zarr-metadata/docs/api/index.md @@ -7,7 +7,7 @@ title: API reference The package is organized to mirror the structure of the Zarr specifications: - [`zarr_metadata.json_schema`](json_schema.md) — JSON Schema output for - any `TypedDict`, one v3 field, or a whole v3 `zarr.json` + any `TypedDict`, one v3 declaration, or a whole v3 `zarr.json` - [`zarr_metadata.model`](model.md) — document models (each one a document and the scope it was read in), the readers that return problems instead of raising them, and the `UNSET` sentinel @@ -22,7 +22,7 @@ The package is organized to mirror the structure of the Zarr specifications: - [`zarr_metadata.v2`](v2.md) — `TypedDict` shapes for Zarr v2 documents (`.zarray`, `.zgroup`, `.zattrs`, `.zmetadata`), and `zarr_metadata.v2.definition`: the v2 kinds of metadata, the scope - `CORE_V2`, and the readers of one `dtype` or codec. Each dtype family's + `CORE_V2_CONTEXT`, and the readers of one `dtype` or codec. Each dtype family's definition is in `zarr_metadata.v2.data_type`, and each codec's in `zarr_metadata.v2.codec` - [`zarr_metadata.v3`](v3/index.md) — `TypedDict` shapes for Zarr v3 @@ -30,12 +30,12 @@ The package is organized to mirror the structure of the Zarr specifications: [chunk key encodings](v3/chunk_key_encoding.md), [codecs](v3/codec.md), and [data types](v3/data_type.md) - [`zarr_metadata.definition`](definition.md) — what both formats share - for reading a field: `Definition`, `resolve`, the three results of a - reading (`AcceptedField`, `UnclaimedField`, `RefusedField`), and the + for reading a declaration: `Definition`, `resolve`, the three results of a + reading (`AcceptedDeclaration`, `UnclaimedDeclaration`, `RefusedDeclaration`), and the scope type `Context`. Its module docstring is the guide - [`zarr_metadata.v3.definition`](v3/definition.md) — the five v3 kinds of metadata (`CodecDefinition`, `DataTypeDefinition`, ...), the codec - pipeline types, and the v3 scopes `CORE` and `CORE_AND_EXTENSIONS` + pipeline types, and the v3 scopes `CORE_CONTEXT` and `CORE_AND_EXTENSIONS_CONTEXT` The document types, models, and spec vocabulary — including the store keys — are re-exported at the top level, so @@ -45,9 +45,13 @@ The model layer's readers are imported from [`zarr_metadata.model`](model.md) directly, and the problem types every reader reports from [`zarr_metadata.problems`](problems.md). -## Common types +## Top-level names Two cross-cutting types are also exported from the top-level `zarr_metadata` namespace: `JSONValue`, documented in [`zarr_metadata.typed_json`](typed_json.md), and `ZarrV3NamedConfigJSON`, documented in [`zarr_metadata.v3`](v3/index.md). + +The package version is exported only from the top level: + +::: zarr_metadata.__version__ diff --git a/packages/zarr-metadata/docs/api/testing.md b/packages/zarr-metadata/docs/api/testing.md new file mode 100644 index 0000000000..ebf7dcd31e --- /dev/null +++ b/packages/zarr-metadata/docs/api/testing.md @@ -0,0 +1,10 @@ +--- +title: testing.strategies +--- + +::: zarr_metadata.testing + +::: zarr_metadata.testing.strategies + options: + inherited_members: false + show_if_no_docstring: false diff --git a/packages/zarr-metadata/docs/api/v3/codec.md b/packages/zarr-metadata/docs/api/v3/codec.md index cb96d2c7d5..ae5171acd0 100644 --- a/packages/zarr-metadata/docs/api/v3/codec.md +++ b/packages/zarr-metadata/docs/api/v3/codec.md @@ -22,4 +22,8 @@ title: codec ::: zarr_metadata.v3.codec.transpose +::: zarr_metadata.v3.codec.vlen_bytes + +::: zarr_metadata.v3.codec.vlen_utf8 + ::: zarr_metadata.v3.codec.zstd diff --git a/packages/zarr-metadata/docs/api/v3/data_type.md b/packages/zarr-metadata/docs/api/v3/data_type.md index 3f8444625a..2bdfe0ad2c 100644 --- a/packages/zarr-metadata/docs/api/v3/data_type.md +++ b/packages/zarr-metadata/docs/api/v3/data_type.md @@ -40,8 +40,16 @@ title: data_type ::: zarr_metadata.v3.data_type.string +::: zarr_metadata.v3.data_type.fixed_length_utf32 + ::: zarr_metadata.v3.data_type.numpy_datetime64 ::: zarr_metadata.v3.data_type.numpy_timedelta64 ::: zarr_metadata.v3.data_type.struct + +::: zarr_metadata.v3.data_type.null_terminated_bytes + +::: zarr_metadata.v3.data_type.variable_length_bytes + +::: zarr_metadata.v3.data_type.zarr_python_raw_bytes diff --git a/packages/zarr-metadata/docs/api/v3/index.md b/packages/zarr-metadata/docs/api/v3/index.md index 91ed9f49bc..ca57e973b8 100644 --- a/packages/zarr-metadata/docs/api/v3/index.md +++ b/packages/zarr-metadata/docs/api/v3/index.md @@ -6,7 +6,7 @@ title: v3 options: members: false -::: zarr_metadata.v3.ZarrV3MetadataFieldJSON +::: zarr_metadata.v3.ZarrV3DeclarationJSON ::: zarr_metadata.v3.ZarrV3NamedConfigJSON diff --git a/packages/zarr-metadata/docs/context.md b/packages/zarr-metadata/docs/context.md index 4786effd33..38a51dd12b 100644 --- a/packages/zarr-metadata/docs/context.md +++ b/packages/zarr-metadata/docs/context.md @@ -6,7 +6,7 @@ Resolving metadata documents requires a context, a `Context`, that defines which The Zarr V3 spec defines the JSON structure of `zarr.json` for arrays and groups exactly, by defining the fields that must be present and the JSON types they must have. But the __interpretation__ of those fields as actual data types, codecs, chunk grids, etc is not strictly defined by the spec. Relating a JSON value like `"bytes"` to the `bytes` codec requires a mapping from names to codec types. -`zarr-metadata` makes this explicit by validating JSON documents with a __context object__. That context object contains definitions of codecs, data types, etc, which can be matched to fields in the JSON document. +`zarr-metadata` makes this explicit by validating JSON documents with a __context object__. That context object contains definitions of codecs, data types, etc, which can be matched to the declarations in the JSON document. ### Context example @@ -17,7 +17,7 @@ from zarr_metadata import ( MetadataValidationError, ZarrV3ArrayMetadata, ZarrV3ArrayMetadataJSON, - ZarrV3MetadataFieldJSON, + ZarrV3DeclarationJSON, ) from zarr_metadata.model import read_array_metadata_v3 from zarr_metadata.v3 import ( @@ -29,7 +29,7 @@ from zarr_metadata.v3 import ( ZarrV3Context, ) -codecs: tuple[ZarrV3MetadataFieldJSON, ...] = ( +codecs: tuple[ZarrV3DeclarationJSON, ...] = ( {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "gzip", "configuration": {"level": 99}}, {"name": "unknown"}, @@ -65,7 +65,7 @@ context = ZarrV3Context( # document is valid in this context. small = ZarrV3ArrayMetadata(document, context=context) print(small.codecs[1]) -# UnclaimedField(json=mappingproxy({'name': 'gzip', 'configuration': mappingproxy({'level': 99})}), name='gzip', read_as=, configuration=mappingproxy({'level': 99})) +# UnclaimedDeclaration(json=mappingproxy({'name': 'gzip', 'configuration': mappingproxy({'level': 99})}), name='gzip', read_as=, configuration=mappingproxy({'level': 99})) ``` In this example, our context object only contained one codec definition, for the `bytes` codec. But the array metadata document contained three codecs, named `"bytes"`, `"gzip"` and `"unknown"`. @@ -82,12 +82,12 @@ Now read the same document in the new context. `read_array_metadata_v3` returns ```python test="true" session="context" reading = read_array_metadata_v3(document, context=new_context) -for loc, field in reading.fields(): +for loc, declaration in reading.declarations(): if loc[0] == "codecs": - print(loc, type(field).__name__) -# ('codecs', 0) AcceptedField -# ('codecs', 1) RefusedField -# ('codecs', 2) UnclaimedField + print(loc, type(declaration).__name__) +# ('codecs', 0) AcceptedDeclaration +# ('codecs', 1) RefusedDeclaration +# ('codecs', 2) UnclaimedDeclaration for problem in reading.problems: print(problem.loc, problem.kind, problem.message) @@ -99,9 +99,9 @@ print(reading.metadata) These are the three scenarios: -- `bytes` is parsed to an `AcceptedField`. A definition in the context claims the name `bytes`, and the configuration passes its checks. -- The malformed `gzip` is parsed to a `RefusedField`. A definition claims the name `gzip`, but the configuration fails one of its checks: a `level` of 99 is outside the range gzip allows. The read reports the problem, with its location in the document, and builds no model, so `reading.metadata` is `None`. Building the model directly raises the same problem as a `MetadataValidationError`. -- The `unknown` codec is parsed to an `UnclaimedField`. Nothing in the context claims the name, so it is kept as written and not checked. This is not a problem: the spec lets a document name codecs a reader does not know. +- `bytes` is parsed to an `AcceptedDeclaration`. A definition in the context claims the name `bytes`, and the configuration passes its checks. +- The malformed `gzip` is parsed to a `RefusedDeclaration`. A definition claims the name `gzip`, but the configuration fails one of its checks: a `level` of 99 is outside the range gzip allows. The read reports the problem, with its location in the document, and builds no model, so `reading.metadata` is `None`. Building the model directly raises the same problem as a `MetadataValidationError`. +- The `unknown` codec is parsed to an `UnclaimedDeclaration`. Nothing in the context claims the name, so it is kept as written and not checked. This is not a problem: the spec lets a document name codecs a reader does not know. So one JSON document can be valid in one context and invalid in another. This is why a model is a document _and_ the context it was read in: the context defines the interpretation of the document. A model keeps its context, and moving it to another context reads the document again. Our `small` model was valid in its original context, but it is not valid in the new one: @@ -113,16 +113,16 @@ except MetadataValidationError as error: # [('codecs', 1, 'configuration', 'level')] ``` -An unclaimed field is never an error, but a reader that does not understand a codec cannot decode the array's chunks. `unclaimed_fields` lists every field that nothing in the context claims, so a reader can check that it understands everything before it opens the array. With a valid `gzip` level, the document reads in the new context, and only `unknown` is left: +An unclaimed declaration is never an error, but a reader that does not understand a codec cannot decode the array's chunks. `unclaimed_declarations` lists every declaration that nothing in the context claims, so a reader can check that it understands everything before it opens the array. With a valid `gzip` level, the document reads in the new context, and only `unknown` is left: ```python test="true" session="context" fixed_codecs = (codecs[0], {"name": "gzip", "configuration": {"level": 1}}, codecs[2]) fixed = ZarrV3ArrayMetadata({**document, "codecs": fixed_codecs}, context=new_context) -print(list(fixed.unclaimed_fields)) +print(list(fixed.unclaimed_declarations)) # [('codecs', 2)] # In the small context, gzip is unclaimed too. -print(list(small.unclaimed_fields)) +print(list(small.unclaimed_declarations)) # [('codecs', 1), ('codecs', 2)] ``` diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md index b3936156d0..b139dfd307 100644 --- a/packages/zarr-metadata/docs/index.md +++ b/packages/zarr-metadata/docs/index.md @@ -39,14 +39,14 @@ closely model the content of the Zarr specifications, such as: `TypedDict` with its rules, and the scope that decides which definitions read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every definition and scope of their format, such as `BYTES_CODEC` and - `CORE_AND_EXTENSIONS`. + `CORE_AND_EXTENSIONS_CONTEXT`. - **JSON type checking** ([`zarr_metadata.typed_json`](api/typed_json.md)): `check` checks a JSON value against any `TypedDict` and reports every problem with its location. - **Problems** ([`zarr_metadata.problems`](api/problems.md)): the problem values every reader returns, and the error that carries them. - **JSON Schema** ([`zarr_metadata.json_schema`](api/json_schema.md)): the - JSON Schema of a `TypedDict`, a v3 field, or a whole `zarr.json`. + JSON Schema of a `TypedDict`, a v3 declaration, or a whole `zarr.json`. - **Optional Pydantic integration** ([`zarr_metadata.pydantic`](api/pydantic.md), requires Pydantic 2.13 or newer): each model as a Pydantic field type that validates raw documents through the same strict parser. @@ -102,7 +102,9 @@ of context-free consistency rules, e.g.: In a v3 document, validation also checks each extension point: the data type, the chunk grid, the chunk key encoding, each codec, and each storage transformer. Each one is checked by the definition that claims its name in -a scope. The scope is `CORE_AND_EXTENSIONS` unless a `context` is passed. +a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. +`ZARR_PYTHON_CONTEXT` adds the data types that zarr-python writes but no +specification registers. If the definition rejects a configuration, the reader rejects it too, and a configuration key the definition does not declare is reported as `unknown_key`. If no definition in the scope claims a name, the reader @@ -213,38 +215,38 @@ with one of them is refused: `read_array_metadata_v3` reads a document once and returns everything the read found: -- each field as the scope read it: an `AcceptedField` when a definition - claims its name, an `UnclaimedField` when none does, or a - `RefusedField`, together with the field's location and the kind it was +- each declaration as the scope read it: an `AcceptedDeclaration` when a definition + claims its name, an `UnclaimedDeclaration` when none does, or a + `RefusedDeclaration`, together with the declaration's location and the kind it was read as; - each codec, with the chunk it receives; - every problem; - the model, when there are no problems. `from_json` returns that model, or raises the problems. A consumer can -apply its own policy by walking the fields, without reading anything -twice. `with_problems` pairs each field with its problems: those located -in the field and in the fields it contains. This matches how zod's +apply its own policy by walking the declarations, without reading anything +twice. `with_problems` pairs each declaration with its problems: those located +in the declaration and in the declarations it contains. This matches how zod's `flattenError` groups issues. `canonical_of` returns the simplest -spelling of a field that has no problems, without reading it again. For -example, the code below finds the fields that go beyond the core spec (a -field with no name is already a problem), and the simplest spelling of -each field: +spelling of a declaration that has no problems, without reading it again. For +example, the code below finds the declarations that go beyond the core spec (a +declaration with no name is already a problem), and the simplest spelling of +each declaration: ```python test="true" session="index" from zarr_metadata.model import read_array_metadata_v3 from zarr_metadata.definition import canonical_of, with_problems -from zarr_metadata.v3.definition import CORE +from zarr_metadata.v3.definition import CORE_CONTEXT reading = read_array_metadata_v3(raw) beyond_core = [ loc - for loc, field in reading.fields() - if field.name is not None and CORE.claimant(field.read_as, field.name) is None + for loc, declaration in reading.declarations() + if declaration.name is not None and CORE_CONTEXT.claimant(declaration.read_as, declaration.name) is None ] simplest = { - loc: canonical_of(field, problems) - for loc, field, problems in with_problems(reading.fields(), reading.problems) + loc: canonical_of(declaration, problems) + for loc, declaration, problems in with_problems(reading.declarations(), reading.problems) } metadata = reading.metadata # None when reading.problems is not empty ``` @@ -274,18 +276,19 @@ string against those rules. A top-level member that the spec does not define is an extra field. A scope can hold a definition for one, by its key, in the `array_extra_fields` or `group_extra_fields` slot of `ZarrV3Context`. A -member whose key a definition claims is read by that definition, like any -other field. Any other member is kept as written, as an `UnclaimedField`. +member whose key a definition claims is read by that definition: an extra field is a declaration +written as the member alone, `{key: value}`. Any other extra field is kept as +written, as an `UnclaimedDeclaration`. The scope is what a reader recognizes, so the model's `must_understand_fields` lists each unclaimed member that does not say `"must_understand": false`: the spec requires a reader to fail to open a -node with any such member. It covers members only. A data type, chunk +node with any such member. It covers extra fields only. A data type, chunk grid, chunk key encoding, codec or storage transformer that nothing in -scope claims is an `UnclaimedField` too, also when it sits inside another -field, such as a codec inside a shard. A model's `unclaimed_fields` lists -every field nothing in scope claims, at any depth, by its location, and +scope claims is an `UnclaimedDeclaration` too, also when it sits inside another +declaration, such as a codec inside a shard. A model's `unclaimed_declarations` lists +every declaration nothing in scope claims, at any depth, by its location, and leaves out the members that waive `must_understand`: a reader can open -the node when it is empty. `CORE` holds +the node when it is empty. `CORE_CONTEXT` holds `CONSOLIDATED_METADATA_EXTRA_FIELD`, which reads a group's `consolidated_metadata` as the envelope the spec fixes. A group's `consolidated()` reads the documents that member lists, in the group's @@ -293,14 +296,14 @@ own scope. A v3 model consists of its document and the scope it was read in. `ZarrV3ArrayMetadata(document, context=None)` reads the document in the -scope (`CORE_AND_EXTENSIONS` when none is given) and raises +scope (`CORE_AND_EXTENSIONS_CONTEXT` when none is given) and raises `MetadataValidationError` with every problem, so an invalid model is never built. `to_json` returns the document as it was written, and `to_key_value` writes that same document to store bytes. -Every typed member is a view of that read: each field as the scope read -it (an `AcceptedField` or an `UnclaimedField`), and `shape`, `attributes` -and the other members as the read refined them. These views are read-only +Every typed attribute is a view of that read: each declaration as the scope read +it (an `AcceptedDeclaration` or an `UnclaimedDeclaration`), and `shape`, `attributes` +and the members that hold no declaration as the read refined them. These views are read-only at every level: a list in the document is shown as a tuple, and an object as a read-only mapping. `to_json` returns plain containers. @@ -324,7 +327,7 @@ all of them with `ZarrV3ConsolidatedMetadata`, or with `read_consolidated_metadata_v3`, which returns problems instead of raising them. A document with a problem then fails only when it is read. Pass `context=group.context` to read the documents in the group's own -scope; with no `context`, they are read in `CORE_AND_EXTENSIONS`. +scope; with no `context`, they are read in `CORE_AND_EXTENSIONS_CONTEXT`. To write consolidated metadata, build a `ZarrV3ConsolidatedMetadata` and pass it to the group's `update`. The group reads the listed models in its @@ -334,10 +337,10 @@ models as entries. Such an entry is accepted when its claims refine into the model's scope, and refused at its path otherwise. To consolidate children that were read in several scopes, consolidate them in the `union` of those scopes. Every reader takes `context=None` for the -default scope, `CORE_AND_EXTENSIONS`. +default scope, `CORE_AND_EXTENSIONS_CONTEXT`. Two models are equal when they represent the same document, even if each -is spelled differently. Values the package interprets (each field, and the +is spelled differently. Values the package interprets (each declaration, and the fill value against its data type) are compared by their canonical spelling, as returned by `canonical_of` and `canonical_fill_value`. For example, `"NaN"` and `"0x7fc00000"` are the same `float32` fill value, @@ -357,7 +360,7 @@ the same data. The v3 spec defines its blosc codec, and says `noshuffle` ignores `typesize`. A v2 node with no `.zattrs` equals one with an empty `.zattrs`, as a v3 document with and without an empty `attributes` are equal. Values the package does -not interpret (attributes, extra fields, and the configuration of a field +not interpret (attributes, extra fields, and the configuration of a declaration that nothing in the scope claims) are compared as JSON text. JSON text distinguishes `true` from `1` and `-0.0` from `0.0`, and treats `NaN` as equal to itself. Equal models have equal hashes, but may write different @@ -367,11 +370,11 @@ model holds nothing that can be changed in place. `node_metadata_json_schema_v3` writes the rules the readers apply as a JSON Schema (draft 2020-12). Use it in an editor that checks a `zarr.json` while you write it, or with a validator in another language. Each -extension point is described as a field as its scope reads it: a claimed +extension point is described as a declaration as its scope reads it: a claimed configuration follows its definition's TypedDict, including bounds, and a name that nothing in the scope claims may have any configuration. The fill value must be a valid fill value for the data type the document names. -`field_json_schema_v3(kind, *, context=None)` writes the schema of one field, and +`declaration_json_schema_v3(kind, *, context=None)` writes the schema of one declaration, and `json_schema` writes the schema of any TypedDict as `check` reads it. All three are in `zarr_metadata.json_schema`. A schema describes each member on its own, not the rules that relate members to each other, so a document the schema accepts may still diff --git a/packages/zarr-metadata/docs/parsing-validation.md b/packages/zarr-metadata/docs/parsing-validation.md index adfe8e51c2..1be626d5ee 100644 --- a/packages/zarr-metadata/docs/parsing-validation.md +++ b/packages/zarr-metadata/docs/parsing-validation.md @@ -66,7 +66,7 @@ This procedure yields one of two values: 1. A Zarr-metadata-shaped JSON object with consistent `shape` and `dimension_names` fields 2. A problem localized to the `dimension_names` field. Strictly speaking, we can't say which of the two fields is wrong. But we treat `shape` as the reference for the number of axes in the array, so the problem goes on the field that disagrees with it. `shape` is required, and other fields like `chunk_grid` depend on it; `dimension_names` is optional, and only names the axes. This metadata has to be repaired; there's no other way to handle it. -### Resolving and validating each field +### Resolving and validating each declaration At this stage we check that each metadata object names a known type, given a registry of data types, codec types, etc., and that its configuration is valid for that type. @@ -85,16 +85,16 @@ For example, the string `"float64"` should resolve to the 64-bit floating point We say "should" because that's what the Zarr V3 spec says about this. There is not a hard requirement that all implementations resolve the string `"float64"` to the 64-bit floating point data type. Someone could write their own (deviant) Zarr implementation that gives the name `"float64"` to a boolean data type. The mapping from JSON identifier (a string) to metadata type (a definition) is held by the registry that entered scope at this stage of validation. -For each field, we do two steps: +For each declaration, we do two steps: -1. **Resolve** the name. We look up the name of the field in the registry. +1. **Resolve** the name. We look up the name of the declaration in the registry. 2. **Validate** the configuration. If we found a match, we must then ensure that the configuration of the JSON value works for the type we resolved. -What if a name isn't in the registry? This is **not** a problem with the document. The document might be perfectly valid, and written by a program with a bigger registry than ours. An unknown name tells us something about our registry, not about the document. So we don't report it as a problem. Instead we mark the field as **unknown**, skip the second step, and leave it to the program that reads the document to decide what to do with it. A program that only needs to list the arrays in a hierarchy might be fine with an unknown codec. A program that decodes chunks is not, and it can try again with a more comprehensive registry. +What if a name isn't in the registry? This is **not** a problem with the document. The document might be perfectly valid, and written by a program with a bigger registry than ours. An unknown name tells us something about our registry, not about the document. So we don't report it as a problem. Instead we mark the declaration as **unknown**, skip the second step, and leave it to the program that reads the document to decide what to do with it. A program that only needs to list the arrays in a hierarchy might be fine with an unknown codec. A program that decodes chunks is not, and it can try again with a more comprehensive registry. If the name is in the registry, we validate the configuration. Suppose our metadata JSON had the following `chunk_grid` field: `"chunk_grid": {"name": "regular", "configuration": "error"}`. If we resolved the name of the chunk grid (`"regular"`) to the [regular chunk grid](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/chunk-grids/regular-grid/index.html) defined in the core spec, then we must do additional checks on the structure of the `configuration` field before we can interpret this chunk grid. In this case, the spec requires that the `configuration` field be a JSON object with a `chunk_shape` field, which is a JSON array of positive integers. Our `"configuration": "error"` field has the wrong type, so this is a problem localized to `/chunk_grid/configuration`. -Validating a configuration might require resolving more fields. Consider this example: +Validating a configuration might require resolving more declarations. Consider this example: ```json { @@ -115,7 +115,7 @@ The name `"sharding_indexed"` should resolve to the [codec with the same name](h Repeating these steps, recursively if necessary, for all metadata objects, we yield one of two values: -1. A Zarr metadata object where each field is either unknown, or resolved to a type and internally consistent with the definition of that type +1. A Zarr metadata object where each declaration is either unknown, or resolved to a type and internally consistent with the definition of that type 2. An enumeration of localized validation problems. ### Consistency checks between fields @@ -150,18 +150,18 @@ Each codec in the pipeline is also handed a value produced by everything that ca To check these rules, we walk the pipeline in order, tracking the chunk that each codec is handed. For codecs that hold pipelines of their own, like `sharding_indexed`, we walk those inner pipelines too. -What about fields marked as unknown? We know nothing about them, and so we can't check anything that depends on them. If the data type is unknown, we can't check the fill value, or whether the `bytes` codec needs an `endian`. If a codec is unknown, we can't say what kind of codec it is, or what it hands on to the next codec. In these cases we skip the check rather than guess. An unknown field limits what we can say about the rest of the document. +What about declarations marked as unknown? We know nothing about them, and so we can't check anything that depends on them. If the data type is unknown, we can't check the fill value, or whether the `bytes` codec needs an `endian`. If a codec is unknown, we can't say what kind of codec it is, or what it hands on to the next codec. In these cases we skip the check rather than guess. An unknown declaration limits what we can say about the rest of the document. This procedure yields one of two values: -1. A Zarr metadata object where every known field is valid on its own and consistent with every other known field. If no field is unknown, this is the metadata that a program can use to decode chunks. If some field is unknown, the program needs a registry that knows it before it can decode chunks. +1. A Zarr metadata object where every known declaration is valid on its own and consistent with every other known declaration. If no declaration is unknown, this is the metadata that a program can use to decode chunks. If some declaration is unknown, the program needs a registry that knows it before it can decode chunks. 2. An enumeration of localized consistency problems, e.g. "`/chunk_grid/configuration/chunk_shape`: expected one chunk length per dimension of `shape`". ## How `zarr-metadata` implements these stages `zarr-metadata` does not run each stage as a separate function. Instead, a single read of the document collects the problems from every stage, so a caller gets the full list of problems at once. -The **registry** from the resolve-and-validate stage is called a **scope** in this library. For Zarr V3 it is a [`ZarrV3Context`](api/v3/definition.md) object, with one slot for each kind of field (`data_types`, `codecs`, `chunk_grids`, ...), and every reader takes a `context` argument. The default scope is `CORE_AND_EXTENSIONS`, which holds the data types, codecs, chunk grids, etc. defined in the Zarr V3 spec and in [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/). `CORE` holds only the definitions from the Zarr V3 spec. +The **registry** from the resolve-and-validate stage is called a **scope** in this library. For Zarr V3 it is a [`ZarrV3Context`](api/v3/definition.md) object, with one slot for each kind of declaration (`data_types`, `codecs`, `chunk_grids`, ...), and every reader takes a `context` argument. The default scope is `CORE_AND_EXTENSIONS_CONTEXT`, which holds the data types, codecs, chunk grids, etc. defined in the Zarr V3 spec and in [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/). `CORE_CONTEXT` holds only the definitions from the Zarr V3 spec. `ZARR_PYTHON_CONTEXT` adds to `CORE_AND_EXTENSIONS_CONTEXT` the data types that zarr-python writes but no specification registers (`null_terminated_bytes` for NumPy's `S`, `raw_bytes` for NumPy's `V`, and `variable_length_bytes`), so a strict check of what zarr-python writes passes it. `read_array_metadata_v3` reads a JSON document and returns a "reading". `reading.problems` is a tuple of the problems, and `reading.metadata` is the validated model, or `None` if there are problems. Each problem has a `loc`, which is its position in the document as a tuple, a `kind`, like `missing_key` or `invalid_value`, and a human-readable `message`: @@ -189,24 +189,24 @@ for problem in reading.problems: If you only want the model, `ZarrV3ArrayMetadata.from_json(raw)` returns it, or raises `MetadataValidationError` with every problem in `.problems`. -To read one field on its own, such as a codec, use `resolve` from `zarr_metadata.definition`. It takes the field, the kind of field, and the scope, and returns the field as read and its problems: +To read one declaration on its own, such as a codec, use `resolve` from `zarr_metadata.definition`. It takes the declaration, the kind of declaration, and the scope, and returns the declaration as read and its problems: ```python test="true" session="parsing-validation" from zarr_metadata.definition import resolve -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition -field, problems = resolve( - {"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE_AND_EXTENSIONS +declaration, problems = resolve( + {"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT ) for problem in problems: print(problem.loc, problem.kind) # ('configuration', 'level') invalid_value ``` -A field with an unknown name is called **unclaimed**, because no definition in the scope claims that name. It is not reported as a problem: it is read as an `UnclaimedField`, and the checks that depend on it are skipped, as described above. So an empty `reading.problems` does not mean that every field is known. A program that decodes chunks should also check that no field is unclaimed. A model's `unclaimed_fields` lists each one, at any depth, by its location, and leaves out an extra member that says `"must_understand": false`, with everything in it, since a reader may ignore it. A reading with problems has no model, and a node with problems cannot be opened either: +A declaration with an unknown name is called **unclaimed**, because no definition in the scope claims that name. It is not reported as a problem: it is read as an `UnclaimedDeclaration`, and the checks that depend on it are skipped, as described above. So an empty `reading.problems` does not mean that every declaration is known. A program that decodes chunks should also check that no declaration is unclaimed. A model's `unclaimed_declarations` lists each one, at any depth, by its location, and leaves out an extra field that says `"must_understand": false`, with everything in it, since a reader may ignore it. A reading with problems has no model, and a node with problems cannot be opened either: ```python test="true" session="parsing-validation" -unclaimed = None if reading.metadata is None else reading.metadata.unclaimed_fields +unclaimed = None if reading.metadata is None else reading.metadata.unclaimed_declarations can_open = unclaimed is not None and len(unclaimed) == 0 ``` @@ -214,17 +214,17 @@ To make an unknown data type or codec known, add its definition to the scope's s ```python test="true" session="parsing-validation" from zarr_metadata.definition import EmptyConfiguration -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition # The definition of the codec that was unknown: here, a codec named # `acme.codec` that takes no configuration. MY_CODEC = CodecDefinition( name="acme.codec", configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" ) -context = CORE_AND_EXTENSIONS.update(codecs=[*CORE_AND_EXTENSIONS.codecs.values(), MY_CODEC]) +context = CORE_AND_EXTENSIONS_CONTEXT.update(codecs=[*CORE_AND_EXTENSIONS_CONTEXT.codecs.values(), MY_CODEC]) reading = read_array_metadata_v3(raw, context=context) ``` -`update` replaces each slot it is given and keeps the others. A slot can also be given as a mapping from each definition's name to it, so `CORE_AND_EXTENSIONS.update(codecs={**CORE_AND_EXTENSIONS.codecs, "gzip": MY_GZIP})` replaces the gzip definition with your own. +`update` replaces each slot it is given and keeps the others. A slot can also be given as a mapping from each definition's name to it, so `CORE_AND_EXTENSIONS_CONTEXT.update(codecs={**CORE_AND_EXTENSIONS_CONTEXT.codecs, "gzip": MY_GZIP})` replaces the gzip definition with your own. -[^1]: The one exception is the `r*` data type, which is identified by a templated string with no `configuration` object. This mixes the identifier with the configuration in the same JSON value, which is not good design. So this library normalizes `r*` into the same form as all the other metadata objects: the field `"r16"` keeps its name, and is read by the `r*` definition with the configuration `{"bits": 16}`. +[^1]: The one exception is the `r*` data type, which is identified by a templated string with no `configuration` object. This mixes the identifier with the configuration in the same JSON value, which is not good design. So this library normalizes `r*` into the same form as all the other metadata objects: the declaration `"r16"` keeps its name, and is read by the `r*` definition with the configuration `{"bits": 16}`. diff --git a/packages/zarr-metadata/mkdocs.yml b/packages/zarr-metadata/mkdocs.yml index 10464e3cf0..b19c0932d2 100644 --- a/packages/zarr-metadata/mkdocs.yml +++ b/packages/zarr-metadata/mkdocs.yml @@ -28,6 +28,7 @@ nav: - ' zarr_metadata.model': api/model.md - ' zarr_metadata.problems': api/problems.md - ' zarr_metadata.pydantic': api/pydantic.md + - ' zarr_metadata.testing.strategies': api/testing.md - ' zarr_metadata.typed_json': api/typed_json.md - ' zarr_metadata.v2': api/v2.md - ' zarr_metadata.v3': diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index abd93bc729..f7690d914c 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -43,6 +43,11 @@ dependencies = [ "typing_extensions>=4.16", ] +[project.optional-dependencies] +# `zarr_metadata.testing.strategies`. 6.85: first release with +# `characters(exclude_categories=...)`. +testing = ["hypothesis>=6.85"] + [project.urls] Homepage = "https://github.com/zarr-developers/zarr-python" Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata" @@ -57,6 +62,7 @@ test = [ "jsonschema", "hypothesis", "pytest-examples>=0.0.18", + "griffelib", ] docs = [ # Pins match the zarr-python docs environment in the repo-root diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index bc57d90fd0..e4a24e083e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -7,7 +7,7 @@ - `zarr_metadata.v3` and `zarr_metadata.v2`: the JSON types of each document, extension and data type; - `zarr_metadata.model`: the document models and their readers; -- `zarr_metadata.definition`: definitions, scopes and the field reader +- `zarr_metadata.definition`: definitions, scopes and the declaration reader that both formats share; - `zarr_metadata.problems`: the problems every reader reports; - `zarr_metadata.typed_json`: `check`, the type checker for JSON; @@ -62,7 +62,7 @@ ZarrV2GroupMetadataStoreKey, ZarrV2ZGroupJSON, ) -from zarr_metadata.v3 import ZarrV3ExtensionField, ZarrV3MetadataFieldJSON, ZarrV3NamedConfigJSON +from zarr_metadata.v3 import ZarrV3DeclarationJSON, ZarrV3ExtensionField, ZarrV3NamedConfigJSON from zarr_metadata.v3.array import ( ZARR_V3_ARRAY_METADATA_STORE_KEY, ZarrV3ArrayMetadataJSON, @@ -136,6 +136,16 @@ TransposeCodecMetadata, TransposeCodecName, ) +from zarr_metadata.v3.codec.vlen_bytes import ( + VLEN_BYTES_CODEC_NAME, + VlenBytesCodecMetadata, + VlenBytesCodecName, +) +from zarr_metadata.v3.codec.vlen_utf8 import ( + VLEN_UTF8_CODEC_NAME, + VlenUtf8CodecMetadata, + VlenUtf8CodecName, +) from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME, ZstdCodecMetadata, ZstdCodecName from zarr_metadata.v3.consolidated import ( ZARR_V3_CONSOLIDATED_METADATA_KEY, @@ -162,6 +172,11 @@ Complex128DataTypeName, Complex128FillValue, ) +from zarr_metadata.v3.data_type.fixed_length_utf32 import ( + FIXED_LENGTH_UTF32_DATA_TYPE_NAME, + FixedLengthUtf32DataTypeName, + FixedLengthUtf32FillValue, +) from zarr_metadata.v3.data_type.float16 import ( FLOAT16_DATA_TYPE_NAME, Float16DataTypeName, @@ -247,6 +262,7 @@ ) __version__ = version("zarr-metadata") +"""The installed version of the `zarr-metadata` distribution.""" __all__ = [ @@ -265,6 +281,7 @@ "DEFAULT_CHUNK_KEY_ENCODING_NAME", "DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR", "ENDIANNESS", + "FIXED_LENGTH_UTF32_DATA_TYPE_NAME", "FLOAT16_DATA_TYPE_NAME", "FLOAT32_DATA_TYPE_NAME", "FLOAT64_DATA_TYPE_NAME", @@ -291,6 +308,8 @@ "UNSET", "V2_CHUNK_KEY_ENCODING_NAME", "V2_CHUNK_KEY_ENCODING_SEPARATOR", + "VLEN_BYTES_CODEC_NAME", + "VLEN_UTF8_CODEC_NAME", "ZARR_V2_ARRAY_DIMENSION_SEPARATOR", "ZARR_V2_ARRAY_METADATA_STORE_KEY", "ZARR_V2_ARRAY_ORDER", @@ -325,6 +344,8 @@ "DefaultChunkKeyEncodingName", "DefaultChunkKeyEncodingSeparator", "Endianness", + "FixedLengthUtf32DataTypeName", + "FixedLengthUtf32FillValue", "Float16DataTypeName", "Float16FillValue", "Float32DataTypeName", @@ -378,6 +399,10 @@ "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", "ValidationProblem", + "VlenBytesCodecMetadata", + "VlenBytesCodecName", + "VlenUtf8CodecMetadata", + "VlenUtf8CodecName", "ZarrV2ArrayDimensionSeparator", "ZarrV2ArrayMetadata", "ZarrV2ArrayMetadataJSON", @@ -405,12 +430,12 @@ "ZarrV3ConsolidatedMetadataEnvelopeJSON", "ZarrV3ConsolidatedMetadataInput", "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3DeclarationJSON", "ZarrV3ExtensionField", "ZarrV3GroupMetadata", "ZarrV3GroupMetadataJSON", "ZarrV3GroupMetadataStoreKey", "ZarrV3GroupMetadataUpdate", - "ZarrV3MetadataFieldJSON", "ZarrV3NamedConfigJSON", "ZarrV3NodeMetadataInput", "ZstdCodecMetadata", diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index e866f907b4..a431c53880 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -2,8 +2,8 @@ JSON types that the shared reading code needs. `JSONValue` is the type of any JSON value. `ZarrV3NamedConfigJSON` is the -`{name, configuration, must_understand}` envelope of a v3 field. It is v3's, -but it lives here because the shared field reader in +`{name, configuration, must_understand}` envelope of a v3 declaration. It is v3's, +but it lives here because the shared declaration reader in `zarr_metadata.definition` uses the envelope as its default. Public homes: `zarr_metadata.typed_json` for `JSONValue`, `zarr_metadata.v3` for the envelope. @@ -46,7 +46,7 @@ class ZarrV3NamedConfigJSON(TypedDict, closed=True): """ - Externally-tagged union member for a metadata field. + Externally-tagged union member for a declaration. The optional `configuration` mapping holds arbitrary JSON-encodable values. `must_understand` is implicitly true when absent. @@ -54,7 +54,7 @@ class ZarrV3NamedConfigJSON(TypedDict, closed=True): `name` and `configuration` are `ReadOnly` (PEP 705) so that concrete entity types — `BloscCodecObject`, `RegularChunkGridObject`, and the rest — are assignable to this type, and therefore to - `ZarrV3MetadataFieldJSON`. Without `ReadOnly` both items are invariant, + `ZarrV3DeclarationJSON`. Without `ReadOnly` both items are invariant, so a concrete `name: Literal["blosc"]` does not satisfy `name: str`, and a required `configuration` does not satisfy a `NotRequired` one. That made the package's own codec types unusable in the very fields they @@ -63,7 +63,7 @@ class ZarrV3NamedConfigJSON(TypedDict, closed=True): requires the narrowed type to be assignable to the input type. `must_understand` stays writable: nothing needs to narrow it, and - keeping it mutable lets writers set it on an already-constructed field. + keeping it mutable lets writers set it on an already-constructed declaration. The type is `closed` (PEP 728): the spec's named-configuration envelope has exactly these three members, and closing it is also what makes this @@ -73,5 +73,13 @@ class ZarrV3NamedConfigJSON(TypedDict, closed=True): """ name: ReadOnly[str] + """The extension's name.""" configuration: NotRequired[ReadOnly[Mapping[str, JSONValue]]] + """The extension's parameters. May be absent.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this extension must refuse the document. + + Absent means `true`. The spec does not permit `false` for a data type, chunk + grid or chunk key encoding + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581). + """ diff --git a/packages/zarr-metadata/src/zarr_metadata/_json.py b/packages/zarr-metadata/src/zarr_metadata/_json.py index bc18f39d6f..69829d32f4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_json.py @@ -130,14 +130,23 @@ class ValidationProblem: """ loc: tuple[str | int, ...] + """The path from the root of what was judged to the offending value. + + Keys and indices, e.g. `("codecs", 0, "name")` in a document; an empty + `loc` refers to that root. + """ message: str + """The human-readable description of what is wrong.""" kind: ProblemKind + """The kind of problem, for programmatic dispatch.""" input: JSONValue | UNSET = dataclasses.field( default=UNSET, kw_only=True, compare=False, repr=False ) + """The JSON found at `loc`, or `UNSET` when nothing is there or it is not JSON.""" ctx: Mapping[str, JSONValue] = dataclasses.field( default_factory=_no_ctx, kw_only=True, compare=False, repr=False ) + """What was expected, e.g. bounds or a closed set of values; empty when only a type was.""" def __post_init__(self) -> None: # Enforce the annotations at runtime. A rule written without a type @@ -215,6 +224,7 @@ class MetadataValidationError(ValueError): """ problems: tuple[ValidationProblem, ...] + """Every problem found, in the order found.""" def __init__(self, problems: Sequence[ValidationProblem]) -> None: self.problems = tuple(problems) diff --git a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py index 1b1d7bbda5..c7b2074ea2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py @@ -55,11 +55,9 @@ class ZarrV3MandatoryNamedConfigJSON(TypedDict, closed=True): must_understand: NotRequired[Literal[True]] -ZarrV3MetadataFieldJSON = ExtensionName | ZarrV3NamedConfigJSON -ZarrV3MandatoryMetadataFieldJSON = ExtensionName | ZarrV3MandatoryNamedConfigJSON -ZarrV3CodecPipelineJSON = Annotated[ - tuple[ZarrV3MandatoryMetadataFieldJSON, ...], Field(min_length=1) -] +ZarrV3DeclarationJSON = ExtensionName | ZarrV3NamedConfigJSON +ZarrV3MandatoryDeclarationJSON = ExtensionName | ZarrV3MandatoryNamedConfigJSON +ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MandatoryDeclarationJSON, ...], Field(min_length=1)] ZarrV2FilterPipelineJSON = tuple[ZarrV2CodecMetadata, ...] @@ -68,14 +66,14 @@ class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue): zarr_format: Literal[3] node_type: Literal["array"] - data_type: ZarrV3MandatoryMetadataFieldJSON + data_type: ZarrV3MandatoryDeclarationJSON shape: tuple[NonNegativeInt, ...] - chunk_grid: ZarrV3MandatoryMetadataFieldJSON - chunk_key_encoding: ZarrV3MandatoryMetadataFieldJSON + chunk_grid: ZarrV3MandatoryDeclarationJSON + chunk_key_encoding: ZarrV3MandatoryDeclarationJSON fill_value: JSONValue codecs: ZarrV3CodecPipelineJSON attributes: NotRequired[Mapping[str, JSONValue]] - storage_transformers: NotRequired[tuple[ZarrV3MandatoryMetadataFieldJSON, ...]] + storage_transformers: NotRequired[tuple[ZarrV3MandatoryDeclarationJSON, ...]] dimension_names: NotRequired[tuple[str | None, ...]] @@ -146,6 +144,6 @@ class ZarrV2ConsolidatedMetadataJSON(TypedDict, closed=True): "ZarrV3ArrayMetadataJSON", "ZarrV3ConsolidatedMetadataEnvelopeJSON", "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3DeclarationJSON", "ZarrV3GroupMetadataJSON", - "ZarrV3MetadataFieldJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/_renamed.py b/packages/zarr-metadata/src/zarr_metadata/_renamed.py new file mode 100644 index 0000000000..d0267a70db --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/_renamed.py @@ -0,0 +1,26 @@ +"""Scopes this package renamed, which keep their old names for now with a `DeprecationWarning`.""" + +import warnings +from collections.abc import Mapping + + +def renamed_scope(module: str, name: str, renames: Mapping[str, tuple[str, object]]) -> object: + """The scope that `name`, an old name in `renames`, refers to, with a `DeprecationWarning`. + + `renames` maps each old name to the scope's name now and the scope. A + module that exported the old names calls this from its `__getattr__`, + so an import of an old name keeps working. Raises `AttributeError`, as + attribute lookup does, when `name` is not an old name. + """ + if name not in renames: + raise AttributeError(f"module {module!r} has no attribute {name!r}") + new, scope = renames[name] + warnings.warn( + f"`{name}` is deprecated and will be removed; use `{new}`, the same scope", + DeprecationWarning, + stacklevel=3, + ) + return scope + + +__all__ = ["renamed_scope"] diff --git a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py index 280bae918a..e9574b46c4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py @@ -23,7 +23,7 @@ `check` is the full entry point for a caller that has a JSON value and a TypedDict. The public module `zarr_metadata.typed_json` exports it. -This module does not know what a metadata field is. A caller with its own +This module does not know what a declaration is. A caller with its own shapes passes a `leaf`. The parser asks the leaf first for every annotation at every depth, and uses the parser the leaf returns without changes. Parsers are compiled once per annotation and are pure functions @@ -259,8 +259,21 @@ class TypedDictKeys: """ members: Mapping[str, tuple[object, bool]] + """Every key it declares, its bases' included, with its value's type and whether it is required. + + The type has its qualifiers and `Annotated` metadata peeled. + """ extra_items: object + """What any other key may hold. + + `Never` when the TypedDict is closed, `object` when it is open, and the + `extra_items` type otherwise. + """ declared: bool + """Whether the TypedDict or a base said what other keys may hold, rather than defaulting. + + It is said with `closed=` or `extra_items=`. A TypedDict that says nothing is open. + """ @property def required(self) -> frozenset[str]: diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index 6dd9c86173..c271fd7077 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -1,14 +1,14 @@ -"""Reading metadata fields against their definitions, in either Zarr format. +"""Reading declarations against their definitions, in either Zarr format. Every extension point of a Zarr document (a v3 codec, data type, chunk grid, chunk key encoding or storage transformer; a v2 dtype or numcodecs -codec) is a metadata field: a name, and a configuration whose JSON the +codec) is a declaration: a name, and a configuration whose JSON the extension defines. A **definition** declares what that JSON is and what it may contain. A definition is a value, not a class to subclass. It has: - `name`, the name the metadata uses; - `configuration`, the TypedDict for the configuration's JSON. This is - the single declaration of the JSON: the type checker is compiled from + the single source of the JSON: the type checker is compiled from it, and a checked configuration has it as its static type. It is interpreted as the typing spec defines a TypedDict: `total`, `Required`, `NotRequired`, `closed` and `extra_items` mean what they @@ -24,7 +24,7 @@ in `zarr_metadata.v3.definition`, and the v2 kinds are in `zarr_metadata.v2.definition`. This module holds what both formats share: `Definition` and `WithFillValue`, which the kinds derive from; `resolve`, -which reads a field in a scope; the three results of a reading; and +which reads a declaration in a scope; the three results of a reading; and `Context`, the protocol of a scope. **Reading JSON.** There are three steps. Each feeds the next, and each @@ -34,62 +34,62 @@ type-checks JSON against a TypedDict and needs nothing else. It returns a value of the TypedDict or None, and every problem, each with a location. The value contains only what the TypedDict admits. A - member typed with a field alias is checked as metadata field JSON. + member typed with a declaration alias is checked as declaration JSON. 2. `definition.read_configuration(configuration)` returns the configuration as that definition reads it. It type-checks the - configuration, checks the envelope of each nested field (an extra + configuration, checks the envelope of each nested declaration (an extra member, a `must_understand` of `false`), and then runs the rules, for one configuration: `GZIP_CODEC.read_configuration({"level": 12})`. -3. `resolve(field, CodecDefinition, CORE_AND_EXTENSIONS)` reads a whole - field in a scope. It checks the envelope, maps the name to a - definition, checks the configuration, and reads each nested field the +3. `resolve(declaration, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT)` reads a whole + declaration in a scope. It checks the envelope, maps the name to a + definition, checks the configuration, and reads each nested declaration the same way. It returns the result of the reading and every problem. The - result is one of three types. `AcceptedField` is a field read by the + result is one of three types. `AcceptedDeclaration` is a declaration read by the definition that claims its name, with the configuration it checked - and allowed, and its nested fields, each read the same way. - `UnclaimedField` is a name that nothing in scope claims. It is not - checked, which is what keeps the format open. `RefusedField` is a - field with problems that explain why it was rejected. Each type has the - field's JSON, the `name` it is written with, the kind it was read as + and allowed, and its nested declarations, each read the same way. + `UnclaimedDeclaration` is a name that nothing in scope claims. It is not + checked, which is what keeps the format open. `RefusedDeclaration` is a + declaration with problems that explain why it was rejected. Each type has the + declaration's JSON, the `name` it is written with, the kind it was read as (`read_as`), the `definition` that claims its name (None for - `UnclaimedField`), and its nested fields as the scope read them - (`nested`). The two types a model can hold, `AcceptedField` and - `UnclaimedField`, have a `configuration` and `to_json()`, which - returns the field as a document writes it. Two of them are equal when - they read the same, however each was spelled. `ResolvedField` is the + `UnclaimedDeclaration`), and its nested declarations as the scope read them + (`nested`). The two types a model can hold, `AcceptedDeclaration` and + `UnclaimedDeclaration`, have a `configuration` and `to_json()`, which + returns the declaration as a document writes it. Two of them are equal when + they read the same, however each was spelled. `ResolvedDeclaration` is the union of the three, for use with `match`. - `configuration_of(field, GZIP_CODEC)` returns the configuration of a - field that `GZIP_CODEC` accepted, typed as its TypedDict - (`GzipCodecConfiguration`), and None for any other field, including - one that `GZIP_CODEC` claimed and refused. A field is read as one of + `configuration_of(declaration, GZIP_CODEC)` returns the configuration of a + declaration that `GZIP_CODEC` accepted, typed as its TypedDict + (`GzipCodecConfiguration`), and None for any other declaration, including + one that `GZIP_CODEC` claimed and refused. A declaration is read as one of the kinds of its format, with or without type arguments. - `resolve(field, Definition, context)` raises `TypeError`, because + `resolve(declaration, Definition, context)` raises `TypeError`, because nothing is filed under `Definition`. A whole v3 array document is read by `read_array_metadata_v3`, in `zarr_metadata.model`. Its - reading returns each field with its location in the document. + reading returns each declaration with its location in the document. from zarr_metadata.definition import resolve from zarr_metadata.v3.codec.gzip import GZIP_CODEC - from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition + from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition resolved, problems = resolve({"name": "gzip", "configuration": {"level": 12}}, - CodecDefinition, CORE_AND_EXTENSIONS) - resolved # RefusedField(..., definition=CodecDefinition(name='gzip'), ...) + CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT) + resolved # RefusedDeclaration(..., definition=CodecDefinition(name='gzip'), ...) problems[0].loc # ('configuration', 'level') problems[0].input # 12 dict(problems[0].ctx) # {'ge': 0, 'le': 9} resolved, problems = resolve({"name": "gzip", "configuration": {"level": 5}}, - CodecDefinition, CORE_AND_EXTENSIONS) + CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT) resolved.definition is GZIP_CODEC # True resolved.configuration # {'level': 5} Problems are values, not exceptions: `ValidationProblem(loc, message, kind)`, with `kind` one of `invalid_type`, `invalid_value`, -`missing_key`, `unknown_key` and `invalid_json`. A field with an +`missing_key`, `unknown_key` and `invalid_json`. A declaration with an unknown key is still read: the key is reported, and the configuration is checked without it. So a consumer that tolerates unknown keys can -filter problems by kind and use the result. The field is valid only +filter problems by kind and use the result. The declaration is valid only when there are no problems at all. Each problem stores the content of its message as data, as pydantic's errors and zod's issues do: `input` is the value found at `loc` (the `12` above), and `ctx` is what was @@ -103,15 +103,15 @@ because `closed` and `extra_items` come from PEP 728, and `typing.TypedDict` does not accept them on the Python versions this package supports. The rules receive the configuration and its nested -fields as the scope read them. A field that is read stores its nested -fields as `AcceptedField.nested`, a `Nested` mapping keyed by location, +declarations as the scope read them. A declaration that is read stores its nested +declarations as `AcceptedDeclaration.nested`, a `Nested` mapping keyed by location, so a struct's rules can access its field types. `read_configuration` -reads in no scope, so it passes no nested fields. A rule's message +reads in no scope, so it passes no nested declarations. A rule's message should show a value as JSON, as the package's own messages do, with `zarr_metadata.problems.shown`: `null`, `[1, 2]`, `"C"`. A rule reports the location of a problem. The value at that location becomes the problem's `input` automatically. Define each function at module top level. A model holds -the definitions that read its fields, so the model can be pickled, and +the definitions that read its declarations, so the model can be pickled, and compares equal after loading, only if its definitions can. A lambda or a closure cannot be pickled, and a `functools.partial` can be pickled but does not compare equal to its loaded copy. @@ -124,7 +124,7 @@ from zarr_metadata import ValidationProblem from zarr_metadata.definition import Nested - from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition + from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition class AcmeLz4Configuration(TypedDict, closed=True): @@ -149,7 +149,7 @@ def acme_lz4_rules( size="dynamic", rules=acme_lz4_rules, ) - CONTEXT = CORE_AND_EXTENSIONS.update(codecs=[*CORE_AND_EXTENSIONS.codecs.values(), ACME_LZ4]) + CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.update(codecs=[*CORE_AND_EXTENSIONS_CONTEXT.codecs.values(), ACME_LZ4]) The TypedDict declares how keys it does not declare are handled. With `closed=True`, such a key is a problem, as above. With `extra_items=`, @@ -180,11 +180,11 @@ def acme_lz4_rules( function that yields the spec violations in a fill value of that shape that the type cannot express, such as a hex string of the wrong width, or a number of byte values that does not match the size. The rules -receive the configuration, the nested fields as the scope read them, +receive the configuration, the nested declarations as the scope read them, and the typed fill value, so a struct checks each field's fill value against that field's own type. `fill_value_problems(data_type, value)` checks a fill value against a -data type field the scope read. If nothing in scope claims the data +data type declaration the scope read. If nothing in scope claims the data type, the fill value is not checked. A data type that does not declare its fill value accepts any JSON. A fill value can have more than one spelling (`"NaN"` and `"0x7fc00000"` @@ -199,16 +199,16 @@ def acme_lz4_rules( or `UNSET` for a fill value with a problem. A data type that does not declare `fill_value_canonical` keeps each value as written. -**The simplest spelling.** A field without problems has a simplest -equivalent spelling, and fields are compared by it. Each nested field is +**The simplest spelling.** A declaration without problems has a simplest +equivalent spelling, and declarations are compared by it. Each nested declaration is written in its own simplest spelling, then the definition's `canonical` is applied (blosc drops a `typesize` that `noshuffle` ignores, and a rectilinear grid run-length encodes its chunk shapes). The envelope uses the shortest form every reader accepts: a data type with nothing to -configure is its bare name, and any other field is an object, +configure is its bare name, and any other declaration is an object, `{"name": ...}`, because a Zarr v3.0 reader does not accept a short-hand name in `codecs`. Raw bits write their size back into the name, in -decimal, so `r008` becomes `r8`. A field with any problem, including an +decimal, so `r008` becomes `r8`. A declaration with any problem, including an unknown key, has no simplest spelling, because a simpler spelling would erase what its author wrote. The result of `canonical` is checked again, and an invalid result raises `ValueError`, which indicates a bug in the @@ -222,8 +222,8 @@ def acme_lz4_rules( Zarr format a scope reads, and `claimant`, which says which definition of a kind reads a name. That is all `resolve` needs, so any object that provides these two can be a scope. A scope has one slot for each kind of -field its format has, named like the document member that holds such -fields: `ZarrV3Context(codecs=[...], data_types=[...])` builds one, and +declaration its format has, named like the document member that holds such +declarations: `ZarrV3Context(codecs=[...], data_types=[...])` builds one, and `context.codecs["gzip"]` reads one slot. A type checker rejects a definition in the wrong slot, and a scope of one format given to a reader of the other. The constructor and the reader also reject them at @@ -245,11 +245,11 @@ def acme_lz4_rules( `TypeError` that says what is wrong: a `configuration` that is not a TypedDict, that does not declare how undeclared keys are handled, or that has a member no checker reads (the error names the TypedDict that -contains it); a `name` that is not a string; a member declared as a +contains it); a `name` that is not a string; an attribute declared as a function (`rules`, `canonical`, `fill_value_rules`, `storage`, `shape_rules`, `chunk_lengths`, `chunk_rules`, `transition`, `pipelines`) that is not a function; a data type's `fill_value` that no -checker reads, or that contains a metadata field, which a value of the +checker reads, or that contains a declaration, which a value of the data type never does; a codec `kind` that is not one of the three, or a `size` that is not `"static"` or `"dynamic"`; a function that is never called for a codec of its kind (chunk rules or pipelines of a @@ -262,14 +262,14 @@ def acme_lz4_rules( """ from zarr_metadata.definition._definition import ( - AcceptedField, + AcceptedDeclaration, Context, Definition, EmptyConfiguration, Nested, - RefusedField, - ResolvedField, - UnclaimedField, + RefusedDeclaration, + ResolvedDeclaration, + UnclaimedDeclaration, WithFillValue, canonical_fill_value, canonical_of, @@ -285,17 +285,17 @@ def acme_lz4_rules( ) __all__ = [ - "AcceptedField", + "AcceptedDeclaration", "ClaimKey", "Conflict", "Context", "Definition", "EmptyConfiguration", "Nested", - "RefusedField", - "ResolvedField", + "RefusedDeclaration", + "ResolvedDeclaration", "ScopeConflictError", - "UnclaimedField", + "UnclaimedDeclaration", "WithFillValue", "canonical_fill_value", "canonical_of", diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_context.py b/packages/zarr-metadata/src/zarr_metadata/definition/_context.py index 44a889dd23..5f3b0772bb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_context.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_context.py @@ -1,8 +1,8 @@ """How a format's scope files its definitions. -No global registry files definitions. Reading a field means +No global registry files definitions. Reading a declaration means deciding which definition its name refers to. That decision depends on -the *scope* the field is read in, not on the definitions themselves. A +the *scope* the declaration is read in, not on the definitions themselves. A scope is a value built from definitions. `Context` is the protocol a reader needs from one. `ZarrV3Context` and `ZarrV2Context`, in each format's `definition` module, are the scopes of each format. They are diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index 5d50f8e605..9e3d746d24 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1,4 +1,4 @@ -"""Reading one metadata field against its definition. +"""Reading one declaration against its definition. There are three steps. Each step feeds the next and needs more input than the step before: @@ -7,9 +7,9 @@ checker compiled from the TypedDict's annotations over the JSON. It checks the shape member by member, locates every problem, and returns a value of the TypedDict. A key the TypedDict does not declare is - reported and left out. A member that holds another metadata field (a - shard's codecs, a cast's data type) is annotated with a field alias - such as `ZarrV3CodecField`, and is checked as an envelope: a name, or + reported and left out. A member that holds another declaration (a + shard's codecs, a cast's data type) is annotated with a declaration alias + such as `ZarrV3CodecDeclaration`, and is checked as an envelope: a name, or a named configuration. Which definition the name refers to is a question for a scope, so the check does not need one. 2. The rules need the definition. They are plain functions over the @@ -17,12 +17,12 @@ bound, or members that depend on each other). `Definition.read_configuration` runs the check and then the rules, for a caller that has one configuration. -3. The reading needs a scope. `resolve` maps the field's name to a +3. The reading needs a scope. `resolve` maps the declaration's name to a definition through a `Context`, checks the configuration, and reads - each nested field found by the check, in the same scope. + each nested declaration found by the check, in the same scope. A definition is a value, not a class to subclass. It holds the name the -metadata uses, the TypedDict that is the single declaration of the +metadata uses, the TypedDict that is the single source of the configuration's JSON, and its rules. It checks itself when it is built. The kind of metadata it defines is its type (`CodecDefinition`, `ChunkGridDefinition`), and a scope files it by that type. Building a @@ -31,7 +31,7 @@ # The hooks of `Definition` (`_named_configuration`, `_envelope_problems` and # the rest) are private: a kind overrides them, and no user calls them. The -# functions in this module that read a field call them, from outside the +# functions in this module that read a declaration call them, from outside the # class body, which pyright reports as private use. This module is where # reading is implemented, so it is allowed here, and nowhere else. # pyright: reportPrivateUsage=false @@ -96,7 +96,7 @@ Any configuration is a `Mapping[str, JSONValue]`. That is the default when a kind is written without a type argument: -`resolve(field, CodecDefinition, context)` is typed as +`resolve(declaration, CodecDefinition, context)` is typed as `CodecDefinition[Mapping[str, JSONValue]]`, with no unknown types. """ @@ -126,18 +126,18 @@ def fill_value_as_written(configuration: object, nested: object, value: T) -> T: class EmptyConfiguration(TypedDict, closed=True): - """The configuration of a definition with nothing to configure. Its field is written with its name only.""" + """The configuration of a definition with nothing to configure. Its declaration is written with its name only.""" -_FIELD_KINDS: Final[dict[TypeAliasType, type[Definition[Any]]]] = {} -"""Maps each field alias to the kind that a member annotated with it is read as. Each kind adds its aliases when its class is built.""" +_DECLARATION_KINDS: Final[dict[TypeAliasType, type[Definition[Any]]]] = {} +"""Maps each declaration alias to the kind that a member annotated with it is read as. Each kind adds its aliases when its class is built.""" @dataclass(frozen=True, kw_only=True, slots=True) class Definition(Generic[C]): """One extension's metadata as JSON: its name, its configuration TypedDict, and its rules. - `configuration` is the TypedDict, and so it is the single declaration + `configuration` is the TypedDict, and so it is the single source of the JSON. The checker is compiled from it, it is the static type of a checked configuration, and a document's author writes to it. It is interpreted as the typing spec defines a TypedDict: `total`, @@ -149,8 +149,8 @@ class Definition(Generic[C]): the spec it finds in a configuration of that type. It receives only a configuration that has passed the check: it contains only what the TypedDict admits, each member is within the bounds its type declares, - and the nested fields are passed as the scope read them (for example, a - struct's field types). The nested fields are empty when no scope read + and the nested declarations are passed as the scope read them (for example, a + struct's field types). The nested declarations are empty when no scope read the configuration. `read_configuration` runs both steps, for a caller that has JSON. @@ -175,34 +175,34 @@ class Definition(Generic[C]): a kind is a definition of that kind, and does not declare `kind`. """ _format: ClassVar[Literal[2, 3] | None] = None - """The Zarr format of the documents that contain fields of this kind. It is declared with the kind and inherited by its definitions, and it sets the format a scope reads.""" + """The Zarr format of the documents that contain declarations of this kind. It is declared with the kind and inherited by its definitions, and it sets the format a scope reads.""" label: ClassVar[str] = "definition" """The name of the kind in error messages, for example "codec".""" - field_aliases: ClassVar[tuple[TypeAliasType, ...]] = () - """The field aliases used to annotate a configuration member that holds a field of this kind. For a codec: `ZarrV3CodecField` and `ZarrV3StaticCodecField`.""" + declaration_aliases: ClassVar[tuple[TypeAliasType, ...]] = () + """The declaration aliases used to annotate a configuration member that holds a declaration of this kind. For a codec: `ZarrV3CodecDeclaration` and `ZarrV3StaticCodecDeclaration`.""" name: str """The name the metadata uses. A scope files the definition under this name.""" configuration: type[C] """The configuration TypedDict.""" rules: Callable[[C, Nested], Iterable[ValidationProblem]] = no_rules - """The spec violations in a well-typed configuration and its nested fields, located in the configuration.""" + """The spec violations in a well-typed configuration and its nested declarations, located in the configuration.""" canonical: Callable[[C], C] = unchanged """A well-typed, allowed configuration in its simplest equivalent spelling. This handles only the definition's own members. `canonicalize` puts - each nested field in its own canonical form, because it knows the + each nested declaration in its own canonical form, because it knows the location of each one. """ @classmethod def _name_problem(cls, name: str, at: Loc) -> ValidationProblem | None: - """The problem to report at `at` when no document of the format may use `name` for a field of this kind; None when it may. For Zarr v3, a name may be used when it is a valid extension name under the spec.""" + """The problem to report at `at` when no document of the format may use `name` for a declaration of this kind; None when it may. For Zarr v3, a name may be used when it is a valid extension name under the spec.""" return name_problem(name, at) @classmethod def _well_named(cls, name: str) -> bool: - """Whether a document of the format may use `name` for a field of this kind, as decided by `_name_problem`.""" + """Whether a document of the format may use `name` for a declaration of this kind, as decided by `_name_problem`.""" return cls._name_problem(name, ()) is None @classmethod @@ -225,17 +225,17 @@ def _carrying_name(self, configuration: Mapping[str, JSONValue]) -> str | None: def _named_configuration( cls, value: object ) -> tuple[str | None, Mapping[str, object] | None, Problems]: - """Split `value` (a field as a document of the format writes it) into `(name, configuration, problems)`, as the module-level `named_configuration` does for a v3 field.""" + """Split `value` (a declaration as a document of the format writes it) into `(name, configuration, problems)`, as the module-level `named_configuration` does for a v3 declaration.""" return named_configuration(value) @classmethod def _envelope_problems(cls, value: object) -> Problems: - """Every reason `value` is not a valid envelope for a field of this kind in this format. The contents of the configuration are not checked.""" + """Every reason `value` is not a valid envelope for a declaration of this kind in this format. The contents of the configuration are not checked.""" return envelope_problems(value, allow_must_understand_false=False) @classmethod def _envelope_json(cls, name: str, configuration: Mapping[str, JSONValue]) -> JSONValue: - """A field with `name` and `configuration`, as a document of the format writes it, in the shortest form every reader accepts. For v3 this is an object.""" + """A declaration with `name` and `configuration`, as a document of the format writes it, in the shortest form every reader accepts. For v3 this is an object.""" if len(configuration) != 0: return {"name": name, "configuration": configuration} return {"name": name} @@ -244,14 +244,14 @@ def _envelope_json(cls, name: str, configuration: Mapping[str, JSONValue]) -> JS def _unclaimed_json( cls, json: JSONValue, name: str, configuration: Mapping[str, JSONValue] ) -> JSONValue: - """A field of this kind that no definition claims, as a document writes it: `json` is the field as written. By default the name and configuration go in the envelope every reader accepts, as for a field a definition read.""" + """A declaration of this kind that no definition claims, as a document writes it: `json` is the declaration as written. By default the name and configuration go in the envelope every reader accepts, as for a declaration a definition read.""" return cls._envelope_json(name, configuration) @classmethod def _refined( cls, data: object, loc: Loc, definition: Definition[Any] | None ) -> tuple[JSONValue | None, Problems]: - """`data`, a field of this kind at `loc`, refined to JSON before it is read, with every reason it is not JSON. `definition` is the definition that claims the field, if any. By default every number must be finite, as RFC 8259 requires.""" + """`data`, a declaration of this kind at `loc`, refined to JSON before it is read, with every reason it is not JSON. `definition` is the definition that claims the declaration, if any. By default every number must be finite, as RFC 8259 requires.""" return refine_json(data, loc) @classmethod @@ -264,22 +264,22 @@ def _undeclared_parts(cls, family: str, name: str, keys: Sequence[str]) -> str: @classmethod def _in_configuration(cls, loc: Loc, place: Loc) -> Loc: - """The location in the document of `place`, a location in the configuration of a field at `loc`: under `configuration` for v3, or under the field itself for a format that writes the parameters next to the name. `place` is `()` for the configuration itself.""" + """The location in the document of `place`, a location in the configuration of a declaration at `loc`: under `configuration` for v3, or under the declaration itself for a format that writes the parameters next to the name. `place` is `()` for the configuration itself.""" return (*loc, "configuration", *place) @classmethod def _name_loc(cls, loc: Loc) -> Loc: - """The location of the name of a field at `loc` that is written as an object: under `name` for v3.""" + """The location of the name of a declaration at `loc` that is written as an object: under `name` for v3.""" return (*loc, "name") @classmethod def _alias_problems( cls, alias: TypeAliasType, definition: Definition[Any], loc: Loc ) -> Problems: - """The problems with using `definition` in a member annotated with `alias`, one of this kind's `field_aliases`. Empty by default. + """The problems with using `definition` in a member annotated with `alias`, one of this kind's `declaration_aliases`. Empty by default. A kind with an alias that admits only some of its definitions - overrides this. For example, `ZarrV3StaticCodecField` admits only + overrides this. For example, `ZarrV3StaticCodecDeclaration` admits only codecs of static size. """ return () @@ -287,7 +287,7 @@ def _alias_problems( def __init_subclass__( cls, *, kind: bool = False, format: Literal[2, 3] | None = None, **kwargs: object ) -> None: - """Register a subclass. `kind=True` declares a kind, with the Zarr `format` of the documents that contain its fields. This marks the class but not its subclasses, as `typing.Protocol` and SQLAlchemy's `__abstract__` do.""" + """Register a subclass. `kind=True` declares a kind, with the Zarr `format` of the documents that contain its declarations. This marks the class but not its subclasses, as `typing.Protocol` and SQLAlchemy's `__abstract__` do.""" # Use the explicit form, not bare `super()`. A dataclass with slots # is rebuilt, and the cell that bare `super()` reads refers to the # discarded class. @@ -299,7 +299,7 @@ def __init_subclass__( if kind: if format not in (2, 3): msg = ( - f"{cls.__name__}: a kind declares the Zarr format its fields belong to, " + f"{cls.__name__}: a kind declares the Zarr format its declarations belong to, " f"format=2 or format=3, got {format!r}" ) raise TypeError(msg) @@ -308,8 +308,8 @@ def __init_subclass__( elif format is not None: msg = f"{cls.__name__}: only a kind declares a format; a definition of a kind has its kind's" raise TypeError(msg) - for alias in cls.__dict__.get("field_aliases", ()): - _FIELD_KINDS[alias] = cls + for alias in cls.__dict__.get("declaration_aliases", ()): + _DECLARATION_KINDS[alias] = cls def __post_init__(self) -> None: refusal = _malformed(self) or self._refusal() @@ -329,7 +329,7 @@ def __repr__(self) -> str: return f"{type(self).__name__}(name={self.name!r})" def _refusal(self) -> str | None: - """A description of what is wrong with the members a kind adds; None if nothing is wrong, or if the kind adds no members.""" + """A description of what is wrong with the attributes a kind adds; None if nothing is wrong, or if the kind adds no attributes.""" return None def _finite_numbers(self) -> bool: @@ -342,10 +342,10 @@ def requires_configuration(self) -> bool: return len(typeddict_keys(self.configuration).required) != 0 def _check_configuration(self, value: object, loc: Loc = ()) -> tuple[C | None, Problems]: - """Type-check `value` as this definition's configuration, and check the envelope of each nested field. + """Type-check `value` as this definition's configuration, and check the envelope of each nested declaration. `zarr_metadata.typed_json.check` does only the type check. This - method also checks the envelope of each metadata field that a + method also checks the envelope of each declaration that a member holds. """ configuration, problems = _configuration_checked( @@ -357,13 +357,13 @@ def read_configuration(self, value: object, loc: Loc = ()) -> tuple[C | None, Pr """Type-check `value`, then check it with the rules. Return the configuration if it is valid, and every problem. The rules run only on a configuration that passed the type check, - including its bounds, and whose nested fields are well formed. Such + including its bounds, and whose nested declarations are well formed. Such a configuration contains only what its TypedDict admits. So a rule never receives a member of the wrong type, a member out of bounds, or a member the type does not allow. No scope reads the nested - fields here, so the rules see none of them as read. A rule about a - nested field (for example, a struct field whose type has values of - varying size) finds nothing to check. `resolve` reads the field in a + declarations here, so the rules see none of them as read. A rule about a + nested declaration (for example, a struct field whose type has values of + varying size) finds nothing to check. `resolve` reads the declaration in a scope and runs every rule. """ configuration, problems = self._check_configuration(value, loc) @@ -381,10 +381,10 @@ def _malformed(definition: Definition[Any]) -> str | None: name = cast("object", definition.name) if not isinstance(name, str): return f"a definition's name is a string, got {name!r}" - for member in _function_members(type(definition)): - value = getattr(definition, member) + for attribute in _function_attributes(type(definition)): + value = getattr(definition, attribute) if not callable(value): - return f"{name!r}: {member} is a function, got {value!r}" + return f"{name!r}: {attribute} is a function, got {value!r}" kind = type(definition) filed, _ = kind._filed_under(name) bad = None if filed is None else kind._name_problem(name, ()) @@ -397,11 +397,11 @@ def _malformed(definition: Definition[Any]) -> str | None: @functools.cache -def _function_members(kind: type[Definition[Any]]) -> tuple[str, ...]: - """The members `kind` declares as functions: each member whose annotation is a `Callable`.""" +def _function_attributes(kind: type[Definition[Any]]) -> tuple[str, ...]: + """The attributes `kind` declares as functions: each attribute whose annotation is a `Callable`.""" return tuple( - member - for member, annotation in get_type_hints(kind).items() + attribute + for attribute, annotation in get_type_hints(kind).items() if get_origin(annotation) is Callable ) @@ -430,7 +430,7 @@ class WithFillValue(Definition[C]): checker reads it the same way it reads a configuration's members, including any range. `fill_value_rules` yields the spec violations in a fill value of that shape that the type cannot express. It receives - the configuration, the nested fields as the scope read them, and the + the configuration, the nested declarations as the scope read them, and the typed fill value. `fill_value_canonical` writes a fill value without problems in the single canonical spelling of its value. So two fill values are the same value of the type exactly when their canonical @@ -454,8 +454,8 @@ def _refusal(self) -> str | None: @functools.cache def _fill_value_parser(annotation: object) -> Parser: - """The checker for a fill value's JSON shape, compiled once. Raises `TypeError` naming any part that no checker reads, or any metadata field in it.""" - return parser(annotation, _no_field) + """The checker for a fill value's JSON shape, compiled once. Raises `TypeError` naming any part that no checker reads, or any declaration in it.""" + return parser(annotation, _no_declaration) def kind_of(definition: object) -> type[Definition[Any]] | None: @@ -470,7 +470,7 @@ def _kind_of_class(cls: type[object]) -> type[Definition[Any]] | None: def format_of(kind: type[Definition[Any]]) -> Literal[2, 3]: - """The Zarr format of the documents that contain fields of `kind`, as declared by the kind. Raises `TypeError` for a class that is not a kind.""" + """The Zarr format of the documents that contain declarations of `kind`, as declared by the kind. Raises `TypeError` for a class that is not a kind.""" found = as_kind(kind)._format if found is None: # a kind cannot be declared without one msg = f"{kind!r} declares no format" @@ -495,10 +495,10 @@ def _kind_named(kind: object) -> type[Definition[Any]] | None: def as_kind(kind: object) -> type[Definition[Any]]: """The kind of metadata that `kind` refers to, with type arguments removed. Raises `TypeError` if it refers to no kind. - A scope files definitions by kind, so a field is read as a kind + A scope files definitions by kind, so a declaration is read as a kind (`CodecDefinition`, or `CodecDefinition[Any]`). It is never read as the base `Definition` or as a class that declares no kind, because - nothing is filed under those, and a field read as one would never be + nothing is filed under those, and a declaration read as one would never be checked. """ try: @@ -509,40 +509,44 @@ def as_kind(kind: object) -> type[Definition[Any]]: if found is not None: return found msg = ( - f"{kind!r} is not a kind of metadata; read a field as one of the kinds of a format, " + f"{kind!r} is not a kind of metadata; read a declaration as one of the kinds of a format, " "such as CodecDefinition" ) raise TypeError(msg) -def field_kind(annotation: object) -> type[Definition[Any]] | None: - """The kind of metadata field held by a member annotated with `annotation` (`CodecDefinition` for `ZarrV3CodecField`), or None if the member holds no field.""" +def declaration_kind(annotation: object) -> type[Definition[Any]] | None: + """The kind of declaration held by a member annotated with `annotation` (`CodecDefinition` for `ZarrV3CodecDeclaration`), or None if the member holds no declaration.""" if not is_alias(annotation): return None - return _FIELD_KINDS.get(annotation) + return _DECLARATION_KINDS.get(annotation) class Context(Protocol): """A scope: the definitions in scope while metadata of one Zarr format is read. - This is what reading a field needs from a scope: the Zarr format of the + This is what reading a declaration needs from a scope: the Zarr format of the documents it reads, and which definition of a kind claims a name. `ZarrV3Context` and `ZarrV2Context` implement it, each with one slot - per kind of field its format has, and so does any other object that - provides these two. `resolve` reads a field in any `Context`. + per kind of declaration its format has, and so does any other object that + provides these two. `resolve` reads a declaration in any `Context`. """ @property - def zarr_format(self) -> Literal[2, 3]: ... + def zarr_format(self) -> Literal[2, 3]: + """The Zarr format of the documents this scope reads.""" + ... - def claimant(self, kind: type[D], name: str) -> D | None: ... + def claimant(self, kind: type[D], name: str) -> D | None: + """The definition of `kind` in scope that reads `name` (a name as a document uses it), or None if there is none.""" + ... @dataclass(frozen=True, slots=True) -class _NestedField: - """A metadata field that the check found inside a configuration: its location, its kind, and its JSON. +class _NestedDeclaration: + """A declaration that the check found inside a configuration: its location, its kind, and its JSON. - The checker returns this in place of the JSON wherever a field alias + The checker returns this in place of the JSON wherever a declaration alias appears. `_checked` collects each one and puts its JSON back. It is carried in the value instead of being recorded separately, so a union branch that did not match leaves nothing behind. @@ -555,41 +559,41 @@ class _NestedField: """Whether the member that holds it accepts only codecs of static size.""" -def _field(annotation: object) -> Parser | None: - """The one checker shape that this module defines: a member that holds a metadata field. +def _declaration(annotation: object) -> Parser | None: + """The one checker shape that this module defines: a member that holds a declaration. The value is checked only as a bare name or an object, and passed on. - Whoever reads the field (`check` without a scope, `resolve` with one) + Whoever reads the declaration (`check` without a scope, `resolve` with one) checks the envelope and maps the name to a definition. """ - kind = field_kind(annotation) + kind = declaration_kind(annotation) if kind is None or not is_alias(annotation): return None alias = annotation def parse(value: object, loc: Loc) -> Parsed: # The value is refined JSON, which the checker types only as - # `object`. If it is not a field at all, the kind's envelope check + # `object`. If it is not a declaration at all, the kind's envelope check # reports that. - return _NestedField(loc, kind, cast("JSONValue", value), alias), () + return _NestedDeclaration(loc, kind, cast("JSONValue", value), alias), () return parse def _vetting(annotation: object) -> Parser | None: - """`_field`, but rejecting annotations that a definition's configuration cannot contain and still be read. + """`_declaration`, but rejecting annotations that a definition's configuration cannot contain and still be read. A member typed with the envelope's own TypedDict - (`ZarrV3MetadataFieldJSON`) is checked as plain JSON, so its name would + (`ZarrV3DeclarationJSON`) is checked as plain JSON, so its name would never be mapped to a definition and its configuration would never be checked. A TypedDict that does not declare how undeclared keys are handled is open, so an undeclared key would never be reported. """ if annotation is ZarrV3NamedConfigJSON: msg = ( - "a member typed ZarrV3MetadataFieldJSON checks as plain JSON, and is never read as " - "a field; annotate it with the field alias of its kind: " - + ", ".join(alias.__name__ for alias in _FIELD_KINDS) + "a member typed ZarrV3DeclarationJSON checks as plain JSON, and is never read as " + "a declaration; annotate it with the declaration alias of its kind: " + + ", ".join(alias.__name__ for alias in _DECLARATION_KINDS) ) raise TypeError(msg) if ( @@ -604,14 +608,14 @@ def _vetting(annotation: object) -> Parser | None: "on a typing_extensions.TypedDict" ) raise TypeError(msg) - return _field(annotation) + return _declaration(annotation) -def _no_field(annotation: object) -> Parser | None: - """A leaf checker that rejects a field alias. A fill value is a value of its data type and never contains a metadata field.""" - if is_alias(annotation) and field_kind(annotation) is not None: +def _no_declaration(annotation: object) -> Parser | None: + """A leaf checker that rejects a declaration alias. A fill value is a value of its data type and never contains a declaration.""" + if is_alias(annotation) and declaration_kind(annotation) is not None: name = annotation.__name__ - msg = f"{name} holds a metadata field, and a fill value is a value of its data type" + msg = f"{name} holds a declaration, and a fill value is a value of its data type" raise TypeError(msg) return None @@ -626,14 +630,14 @@ def _vet(configuration: type, zarr_format: int | None) -> None: raise TypeError(msg) def vetting(annotation: object) -> Parser | None: - # A document of one format never holds a field of the other, so a + # A document of one format never holds a declaration of the other, so a # definition never reads one. - kind = field_kind(annotation) + kind = declaration_kind(annotation) if kind is not None and zarr_format is not None and format_of(kind) != zarr_format: msg = ( - f"a member typed {getattr(annotation, '__name__', annotation)} holds a field of " + f"a member typed {getattr(annotation, '__name__', annotation)} holds a declaration of " f"a Zarr v{format_of(kind)} document, which a definition of a v{zarr_format} " - "field never reads" + "declaration never reads" ) raise TypeError(msg) return _vetting(annotation) @@ -649,7 +653,7 @@ def vetting(annotation: object) -> Parser | None: @dataclass(frozen=True, slots=True) class _Checker: - """A TypedDict's checker, compiled once, and whether a value of the TypedDict can contain a nested field.""" + """A TypedDict's checker, compiled once, and whether a value of the TypedDict can contain a nested declaration.""" parse: Parser nests: bool @@ -661,7 +665,7 @@ def _checker(shape: type) -> _Checker: met: list[object] = [] def leaf(annotation: object) -> Parser | None: - found = _field(annotation) + found = _declaration(annotation) if found is not None: met.append(annotation) return found @@ -671,28 +675,28 @@ def leaf(annotation: object) -> Parser | None: def _checked( shape: type, value: object, loc: Loc -) -> tuple[object, Problems, tuple[_NestedField, ...]]: - """Check `value` as `shape`. Return the typed value (with each nested field restored as written), every problem, and the nested fields in order.""" +) -> tuple[object, Problems, tuple[_NestedDeclaration, ...]]: + """Check `value` as `shape`. Return the typed value (with each nested declaration restored as written), every problem, and the nested declarations in order.""" typed, found, nested = _typed(shape, value, loc) return _put_back(typed, _as_written, nested), found, nested def _typed( shape: type, value: object, loc: Loc -) -> tuple[object, Problems, tuple[_NestedField, ...]]: - """Check `value` as `shape`, leaving each nested field where the checker found it. Return the typed value, every problem, and the nested fields in order.""" +) -> tuple[object, Problems, tuple[_NestedDeclaration, ...]]: + """Check `value` as `shape`, leaving each nested declaration where the checker found it. Return the typed value, every problem, and the nested declarations in order.""" checker = _checker(shape) typed, found = checker.parse(value, loc) if not checker.nests: return typed, found, () - nested: list[_NestedField] = [] + nested: list[_NestedDeclaration] = [] _collect(typed, nested) return typed, found, tuple(nested) -def _collect(value: object, nested: list[_NestedField]) -> None: - """Append to `nested` each nested field that the checker returned inside `value`, in order.""" - if isinstance(value, _NestedField): +def _collect(value: object, nested: list[_NestedDeclaration]) -> None: + """Append to `nested` each nested declaration that the checker returned inside `value`, in order.""" + if isinstance(value, _NestedDeclaration): nested.append(value) elif is_tuple(value): for entry in value: @@ -702,31 +706,33 @@ def _collect(value: object, nested: list[_NestedField]) -> None: _collect(entry, nested) -def _as_written(field: _NestedField) -> JSONValue: - """A nested field restored as it was written.""" - return field.json +def _as_written(declaration: _NestedDeclaration) -> JSONValue: + """A nested declaration restored as it was written.""" + return declaration.json -def _declared(field: _NestedField) -> JSONValue: - """A nested field restored as it was written, except for envelope members that are not declared. Those are reported and left out, in the same way the checker leaves out a key that a closed TypedDict does not declare.""" - if not isinstance(field.json, Mapping): - return field.json - envelope = cast("Mapping[str, JSONValue]", field.json) +def _envelope_keys_only(declaration: _NestedDeclaration) -> JSONValue: + """A nested declaration restored as it was written, except for the members its envelope TypedDict does not declare. Those are reported and left out, in the same way the checker leaves out a key that a closed TypedDict does not declare.""" + if not isinstance(declaration.json, Mapping): + return declaration.json + envelope = cast("Mapping[str, JSONValue]", declaration.json) return {key: value for key, value in envelope.items() if key in ENVELOPE_KEYS} def _put_back( - value: object, put: Callable[[_NestedField], JSONValue], nested: Sequence[_NestedField] + value: object, + put: Callable[[_NestedDeclaration], JSONValue], + nested: Sequence[_NestedDeclaration], ) -> object: - """Replace each of `nested` (the fields the checker returned inside `value`) with the result of `put`. Return `value` unchanged when there are none.""" + """Replace each of `nested` (the declarations the checker returned inside `value`) with the result of `put`. Return `value` unchanged when there are none.""" if len(nested) == 0: return value return _putting_back(value, put) -def _putting_back(value: object, put: Callable[[_NestedField], JSONValue]) -> object: - """`_put_back` for a value that contains nested fields: a recursive walk.""" - if isinstance(value, _NestedField): +def _putting_back(value: object, put: Callable[[_NestedDeclaration], JSONValue]) -> object: + """`_put_back` for a value that contains nested declarations: a recursive walk.""" + if isinstance(value, _NestedDeclaration): return put(value) if is_tuple(value): return tuple(_putting_back(entry, put) for entry in value) @@ -741,13 +747,13 @@ def _usable(problems: Sequence[ValidationProblem]) -> bool: def read_only(configuration: C) -> C: - """`configuration` in the form a definition's functions receive it: a read-only view of the field's own configuration, so a function that assigns a member fails. + """`configuration` in the form a definition's functions receive it: a read-only view of the declaration's own configuration, so a function that assigns a member fails. This function makes only the top-level members read-only. The value a member holds (for example, an object in a struct's `fields`) is passed as is. While a configuration is being read, that value can be mutable, and a function that writes into it changes the configuration. An - accepted field's configuration is already read-only at every level. + accepted declaration's configuration is already read-only at every level. """ return cast("C", MappingProxyType(cast("Mapping[str, JSONValue]", configuration))) @@ -803,18 +809,18 @@ def _located(prefix: Loc, problems: Iterable[ValidationProblem]) -> Problems: return tuple(dataclasses.replace(each, loc=(*prefix, *each.loc)) for each in found) -def _envelope(field: _NestedField) -> Problems: - """The problems with a nested field's envelope, located at the field.""" - return _located(field.loc, field.kind._envelope_problems(field.json)) +def _envelope(declaration: _NestedDeclaration) -> Problems: + """The problems with a nested declaration's envelope, located at the declaration.""" + return _located(declaration.loc, declaration.kind._envelope_problems(declaration.json)) def _configuration_checked( value: object, shape: type[T], loc: Loc = (), *, finite: bool = True ) -> tuple[T | None, Problems]: - """Check `value` as `shape`, as `typed_json.check` does, and check the envelope of each nested field. + """Check `value` as `shape`, as `typed_json.check` does, and check the envelope of each nested declaration. A definition's `read_configuration` starts with this step. A member - typed with a field alias holds a metadata field, and its envelope is + typed with a declaration alias holds a declaration, and its envelope is checked the same way as in a document. A `must_understand` of `false` is a problem of the configuration, and no value is returned. An extra member is an unknown key: it is reported and left out, and the value is @@ -828,19 +834,19 @@ def _configuration_checked( if len(problems) != 0: return None, problems typed, found, nested = _typed(shape, refined, loc) - problems = (*found, *(problem for field in nested for problem in _envelope(field))) + problems = (*found, *(problem for declaration in nested for problem in _envelope(declaration))) if not _usable(problems): return None, problems - return cast("T", _put_back(typed, _declared, nested)), problems + return cast("T", _put_back(typed, _envelope_keys_only, nested)), problems def named_configuration( value: object, ) -> tuple[str | None, Mapping[str, object] | None, Problems]: - """Split a metadata field into `(name, configuration, problems)`. + """Split a declaration into `(name, configuration, problems)`. - A field is a bare name, or an object that contains a name. A `None` - name means the value is not a metadata field at all. A `None` + A declaration is a bare name, or an object that contains a name. A `None` + name means the value is not a declaration at all. A `None` configuration means the bare spelling was used, or the key was left out. The only problem reported is a configuration that is present but not an object, at `("configuration",)`. @@ -866,80 +872,80 @@ def named_configuration( def _nothing_nested() -> Nested: - """The nested fields of a field that contains no read fields: an empty mapping.""" + """The nested declarations of a declaration that contains no read declarations: an empty mapping.""" return {} @dataclass(frozen=True, slots=True, kw_only=True) -class AcceptedField(Generic[D]): - """A field that a definition in scope read: the name it is written with, the definition, and the configuration the definition allowed. - - A problem with the envelope (an extra member, a `must_understand` of - `false`) is reported with the field, and the field is still read. The - same applies to a problem in a nested field, which belongs to that - nested field, as `nested` shows. Two fields are equal when they read - the same, however each was spelled, as compared by `field_key`. For - example, `"bytes"` and `{"name": "bytes"}` are one field. So are a +class AcceptedDeclaration(Generic[D]): + """A declaration that a definition in scope read: the name it is written with, the definition, and the configuration the definition allowed. + + A problem with the envelope (an extra envelope member, a `must_understand` of + `false`) is reported with the declaration, and the declaration is still read. The + same applies to a problem in a nested declaration, which belongs to that + nested declaration, as `nested` shows. Two declarations are equal when they read + the same, however each was spelled, as compared by `declaration_key`. For + example, `"bytes"` and `{"name": "bytes"}` are one declaration. So are a blosc with and without the `typesize` that `noshuffle` ignores, because - the definition's `canonical` removes it. Equal fields have equal + the definition's `canonical` removes it. Equal declarations have equal hashes. """ json: JSONValue - """The field as written, refined (arrays as tuples). It is not used for equality.""" + """The declaration as written, refined (arrays as tuples). It is not used for equality.""" name: str - """The name the field is written with, for example `"r16"`, even though its definition is filed under `r*`.""" + """The name the declaration is written with, for example `"r16"`, even though its definition is filed under `r*`.""" definition: D - """The definition that read the field.""" + """The definition that read the declaration.""" configuration: Mapping[str, JSONValue] """The configuration, type-checked and allowed by the rules. For raw bits, this is the configuration encoded in the name. - Each nested field is written as that field's `to_json` writes it, so + Each nested declaration is written as that declaration's `to_json` writes it, so the configuration shows what was read, however it was spelled. For example, a shard's `"crc32c"` and `{"name": "crc32c"}` are the same index codec. """ nested: Nested = dataclasses.field(default_factory=_nothing_nested) - """The fields in the configuration, each as the scope read it, keyed by location in the configuration. + """The declarations in the configuration, each as the scope read it, keyed by location in the configuration. For example, a struct's field types are at `("fields", 0, "data_type")` and a shard's codecs are at `("codecs", 0)`. A definition's functions - use this to inspect the fields inside its own configuration. + use this to inspect the declarations inside its own configuration. """ read_as: type[Definition[Any]] = dataclasses.field(init=False, repr=False) - """The kind of metadata the field was read as: the kind of its definition.""" + """The kind of metadata the declaration was read as: the kind of its definition.""" def __eq__(self, other: object) -> bool: - if not is_field(other): + if not is_declaration(other): return NotImplemented - return field_key(self) == field_key(other) + return declaration_key(self) == declaration_key(other) def __hash__(self) -> int: - return hash(field_key(self)) + return hash(declaration_key(self)) def __post_init__(self) -> None: - # Check the annotations at runtime. A field built by hand (an + # Check the annotations at runtime. A declaration built by hand (an # extension may do this) fails here, not later in a function that # trusts it. definition = cast("object", self.definition) kind = kind_of(definition) refusal = ( - f"a field read is read by a definition of a kind, got {definition!r}" + f"a declaration read is read by a definition of a kind, got {definition!r}" if kind is None else _misread(definition, kind, self.name) ) if refusal is not None: raise TypeError(refusal) object.__setattr__(self, "read_as", kind) - # Everything the field returns is read-only at every level, so a - # field cannot reach a state where its key, `==` and `refines` + # Everything the declaration returns is read-only at every level, so a + # declaration cannot reach a state where its key, `==` and `refines` # disagree. object.__setattr__(self, "json", frozen(self.json)) object.__setattr__(self, "configuration", frozen(self.configuration)) object.__setattr__(self, "nested", MappingProxyType(dict(self.nested))) - def __reduce__(self) -> tuple[Callable[..., AcceptedField[Any]], tuple[object, ...]]: - # Read-only views cannot be pickled, so pickle the field from its constructor arguments. + def __reduce__(self) -> tuple[Callable[..., AcceptedDeclaration[Any]], tuple[object, ...]]: + # Read-only views cannot be pickled, so pickle the declaration from its constructor arguments. return _accepted_field, ( copied(self.json), self.name, @@ -949,11 +955,11 @@ def __reduce__(self) -> tuple[Callable[..., AcceptedField[Any]], tuple[object, . ) def to_json(self) -> JSONValue: - """The field as a document writes it, in a form every reader accepts. The configuration is as read, and the result shares no objects with the field. + """The declaration as a document writes it, in a form every reader accepts. The configuration is as read, and the result shares no objects with the declaration. The envelope uses the shortest form every reader accepts. A data type with nothing to configure is written as its bare name, as core - data types have been written since Zarr v3.0. Any other field is an + data types have been written since Zarr v3.0. Any other declaration is an object, `{"name": ...}`, because a Zarr v3.0 reader does not accept a bare name in `codecs` (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L585-L592). @@ -964,31 +970,31 @@ def to_json(self) -> JSONValue: @dataclass(frozen=True, slots=True, kw_only=True) -class UnclaimedField: - """A field that nothing in scope claims: an extension that the scope does not check. Accepting such fields is what keeps the format open. +class UnclaimedDeclaration: + """A declaration that nothing in scope claims: an extension that the scope does not check. Accepting such declarations is what keeps the format open. - It is equal to another unclaimed field when both are of the same kind, + It is equal to another unclaimed declaration when both are of the same kind, have the same name, and a document writes the same JSON for both. - Nothing in scope interprets the field, so it is compared as JSON text, - as `field_key` defines. + Nothing in scope interprets the declaration, so it is compared as JSON text, + as `declaration_key` defines. """ json: JSONValue - """The field as written, refined (arrays as tuples). It is not used for equality.""" + """The declaration as written, refined (arrays as tuples). It is not used for equality.""" name: str """The name that nothing in scope claims.""" read_as: type[Definition[Any]] - """The kind of metadata the field was read as: the kind a claiming definition would have.""" + """The kind of metadata the declaration was read as: the kind a claiming definition would have.""" configuration: Mapping[str, JSONValue] = dataclasses.field(init=False) """The configuration as written, not checked; empty when no configuration is written.""" def __eq__(self, other: object) -> bool: - if not is_field(other): + if not is_declaration(other): return NotImplemented - return field_key(self) == field_key(other) + return declaration_key(self) == declaration_key(other) def __hash__(self) -> int: - return hash(field_key(self)) + return hash(declaration_key(self)) def __post_init__(self) -> None: # Check the annotations at runtime. Remove type arguments from @@ -997,7 +1003,7 @@ def __post_init__(self) -> None: name = cast("object", self.name) if not isinstance(name, str) or not self.read_as._well_named(name): msg = ( - "a field nothing in scope claims is named as a document names a " + "a declaration nothing in scope claims is named as a document names a " f"{self.read_as.label}, got {name!r}" ) raise TypeError(msg) @@ -1008,47 +1014,47 @@ def __post_init__(self) -> None: self, "configuration", frozen(cast("Mapping[str, JSONValue]", configuration)) ) - def __reduce__(self) -> tuple[Callable[..., UnclaimedField], tuple[object, ...]]: - # Read-only views cannot be pickled, so pickle the field from its constructor arguments. + def __reduce__(self) -> tuple[Callable[..., UnclaimedDeclaration], tuple[object, ...]]: + # Read-only views cannot be pickled, so pickle the declaration from its constructor arguments. return _unclaimed_field, (copied(self.json), self.name, self.read_as) @property def definition(self) -> None: - """The definition that read the field. Always None, because no definition read it.""" + """The definition that read the declaration. Always None, because no definition read it.""" return None @property def nested(self) -> Nested: - """The nested fields as the scope read them. Always empty, because nothing read the configuration.""" + """The nested declarations as the scope read them. Always empty, because nothing read the configuration.""" return _nothing_nested() def to_json(self) -> JSONValue: - """The field as a document writes it, sharing no objects with the field. The configuration is as written, in the envelope form every reader accepts, as `AcceptedField.to_json` writes it.""" + """The declaration as a document writes it, sharing no objects with the declaration. The configuration is as written, in the envelope form every reader accepts, as `AcceptedDeclaration.to_json` writes it.""" return copied(written_json(self)) @dataclass(frozen=True, slots=True, kw_only=True) -class RefusedField(Generic[D]): - """A field that could not be read: it is not a field at all, not JSON, or rejected by the definition that claims its name. Its problems explain why.""" +class RefusedDeclaration(Generic[D]): + """A declaration that could not be read: it is not a declaration at all, not JSON, or rejected by the definition that claims its name. Its problems explain why.""" json: JSONValue | UNSET - """The field as written, refined (arrays as tuples). `UNSET` when it is not JSON, which no document can contain.""" + """The declaration as written, refined (arrays as tuples). `UNSET` when it is not JSON, which no document can contain.""" name: str | None - """The name the field is written with; None when it has no name.""" + """The name the declaration is written with; None when it has no name.""" read_as: type[Definition[Any]] - """The kind of metadata the field was read as.""" + """The kind of metadata the declaration was read as.""" definition: D | None = None - """The definition that claims the name and rejected the field; None when nothing in scope claims it, or when the field has no name.""" + """The definition that claims the name and rejected the declaration; None when nothing in scope claims it, or when the declaration has no name.""" nested: Nested = dataclasses.field(default_factory=_nothing_nested) - """The nested fields, each as the scope read it; empty when the configuration was not checked against its TypedDict.""" + """The nested declarations, each as the scope read it; empty when the configuration was not checked against its TypedDict.""" def __eq__(self, other: object) -> bool: - if not is_field(other): + if not is_declaration(other): return NotImplemented - return field_key(self) == field_key(other) + return declaration_key(self) == declaration_key(other) def __hash__(self) -> int: - return hash(field_key(self)) + return hash(declaration_key(self)) def __post_init__(self) -> None: # Check the annotations at runtime. Remove type arguments from @@ -1063,8 +1069,8 @@ def __post_init__(self) -> None: object.__setattr__(self, "json", frozen(self.json)) object.__setattr__(self, "nested", MappingProxyType(dict(self.nested))) - def __reduce__(self) -> tuple[Callable[..., RefusedField[Any]], tuple[object, ...]]: - # Read-only views cannot be pickled, so pickle the field from its constructor arguments. + def __reduce__(self) -> tuple[Callable[..., RefusedDeclaration[Any]], tuple[object, ...]]: + # Read-only views cannot be pickled, so pickle the declaration from its constructor arguments. return _refused_field, ( UNSET if self.json is UNSET else copied(self.json), self.name, @@ -1074,28 +1080,34 @@ def __reduce__(self) -> tuple[Callable[..., RefusedField[Any]], tuple[object, .. ) -ResolvedField = TypeAliasType( - "ResolvedField", "AcceptedField[D] | UnclaimedField | RefusedField[D]", type_params=(D,) +ResolvedDeclaration = TypeAliasType( + "ResolvedDeclaration", + "AcceptedDeclaration[D] | UnclaimedDeclaration | RefusedDeclaration[D]", + type_params=(D,), ) -"""One metadata field as a scope read it: accepted by the definition that claims its name, unclaimed, or refused.""" +"""One declaration as a scope read it: accepted by the definition that claims its name, unclaimed, or refused.""" +# The three rebuild functions keep their `_field` names: a pickle looks them +# up by name, so renaming them would break existing pickles. def _accepted_field( json: JSONValue, name: str, definition: Definition[Any], configuration: Mapping[str, JSONValue], nested: Nested, -) -> AcceptedField[Any]: - """Rebuild an accepted field from its pickled arguments.""" - return AcceptedField( +) -> AcceptedDeclaration[Any]: + """Rebuild an accepted declaration from its pickled arguments.""" + return AcceptedDeclaration( json=json, name=name, definition=definition, configuration=configuration, nested=nested ) -def _unclaimed_field(json: JSONValue, name: str, read_as: type[Definition[Any]]) -> UnclaimedField: - """Rebuild an unclaimed field from its pickled arguments.""" - return UnclaimedField(json=json, name=name, read_as=read_as) +def _unclaimed_field( + json: JSONValue, name: str, read_as: type[Definition[Any]] +) -> UnclaimedDeclaration: + """Rebuild an unclaimed declaration from its pickled arguments.""" + return UnclaimedDeclaration(json=json, name=name, read_as=read_as) def _refused_field( @@ -1104,47 +1116,51 @@ def _refused_field( read_as: type[Definition[Any]], definition: Definition[Any] | None, nested: Nested, -) -> RefusedField[Any]: - """Rebuild a refused field from its pickled arguments.""" - return RefusedField(json=json, name=name, read_as=read_as, definition=definition, nested=nested) +) -> RefusedDeclaration[Any]: + """Rebuild a refused declaration from its pickled arguments.""" + return RefusedDeclaration( + json=json, name=name, read_as=read_as, definition=definition, nested=nested + ) -_FIELDS: Final = (AcceptedField, UnclaimedField, RefusedField) -"""The three possible results of reading a field in a scope.""" +_DECLARATIONS: Final = (AcceptedDeclaration, UnclaimedDeclaration, RefusedDeclaration) +"""The three possible results of reading a declaration in a scope.""" def _misread(definition: object, kind: type[Definition[Any]], name: object) -> str | None: - """A description of why `definition` cannot be the definition that read a field named `name` as a `kind`; None when it can.""" + """A description of why `definition` cannot be the definition that read a declaration named `name` as a `kind`; None when it can.""" if not isinstance(definition, kind): - return f"a field read as a {kind.__name__} is read by one, got {definition!r}" + return f"a declaration read as a {kind.__name__} is read by one, got {definition!r}" filed = definition.name if not isinstance(name, str) or filed_under(kind, name)[0] != filed: - return f"a field named {name!r} is read by the definition filed under it, got {filed!r}" + return ( + f"a declaration named {name!r} is read by the definition filed under it, got {filed!r}" + ) return None -def field_key(field: ResolvedField[Any]) -> tuple[object, ...]: - """The key that `==` and `hash` compare for a field. It represents what the field means, not how it was spelled. +def declaration_key(declaration: ResolvedDeclaration[Any]) -> tuple[object, ...]: + """The key that `==` and `hash` compare for a declaration. It represents what the declaration means, not how it was spelled. - An accepted field is compared by the definition that read it, by its + An accepted declaration is compared by the definition that read it, by its configuration in the definition's canonical spelling (as `canonical_of` writes it) as JSON text, and by the key of each nested - field. An unclaimed field is compared by its kind, its name and the + declaration. An unclaimed declaration is compared by its kind, its name and the JSON a document writes for it, as JSON text, because nothing interprets it. - A refused field is compared by what was written and by the definition - that rejected it. So two fields are equal when the parts a reader + A refused declaration is compared by what was written and by the definition + that rejected it. So two declarations are equal when the parts a reader understands read the same, and the parts no reader interprets are written the same way. """ - if isinstance(field, AcceptedField): - definition = cast("Definition[Any]", field.definition) - configuration: JSONValue = dict(field.configuration) - # Each nested field is compared by its own key, so its location is + if isinstance(declaration, AcceptedDeclaration): + definition = cast("Definition[Any]", declaration.definition) + configuration: JSONValue = dict(declaration.configuration) + # Each nested declaration is compared by its own key, so its location is # cleared before the definition's `canonical` sees the rest, in the - # same order as `_canonical_field`. `canonical` simplifies only the + # same order as `_canonical_declaration`. `canonical` simplifies only the # definition's own members. - for loc in field.nested: + for loc in declaration.nested: configuration = _replaced(configuration, loc, None) view = read_only(cast("Mapping[str, JSONValue]", configuration)) spelled = asked( @@ -1152,144 +1168,154 @@ def field_key(field: ResolvedField[Any]) -> tuple[object, ...]: "canonical", lambda: json_text(cast("JSONValue", dict(definition.canonical(view)))), ) - return ("read", definition, spelled, _nested_key(field.nested)) - if isinstance(field, UnclaimedField): - return ("unclaimed", field.read_as, field.name, json_text(written_json(field))) + return ("read", definition, spelled, _nested_key(declaration.nested)) + if isinstance(declaration, UnclaimedDeclaration): + return ( + "unclaimed", + declaration.read_as, + declaration.name, + json_text(written_json(declaration)), + ) return ( "refused", - field.read_as, - field.name, - field.definition, - UNSET if field.json is UNSET else json_text(field.json), - _nested_key(field.nested), + declaration.read_as, + declaration.name, + declaration.definition, + UNSET if declaration.json is UNSET else json_text(declaration.json), + _nested_key(declaration.nested), ) -def own_key(field: AcceptedField[Any]) -> tuple[object, ...]: - """The part of `field_key` for an accepted field that excludes nested fields: the definition and the canonical spelling of its own members.""" - return field_key(field)[:3] +def own_key(declaration: AcceptedDeclaration[Any]) -> tuple[object, ...]: + """The part of `declaration_key` for an accepted declaration that excludes nested declarations: the definition and the canonical spelling of its own members.""" + return declaration_key(declaration)[:3] def _nested_key(nested: Nested) -> tuple[tuple[Loc, tuple[object, ...]], ...]: - """The key of each nested field, paired with its location.""" - return tuple((loc, field_key(inner)) for loc, inner in nested.items()) + """The key of each nested declaration, paired with its location.""" + return tuple((loc, declaration_key(inner)) for loc, inner in nested.items()) -def document_json(field: ResolvedField[Any]) -> JSONValue | UNSET: - """A field as a document writes it, using the field's own values: as `to_json` writes it, or as originally written for a refused field. `UNSET` for a refused field that was not JSON. +def document_json(declaration: ResolvedDeclaration[Any]) -> JSONValue | UNSET: + """A declaration as a document writes it, using the declaration's own values: as `to_json` writes it, or as originally written for a refused declaration. `UNSET` for a refused declaration that was not JSON. A writer serializes this without changing it, so it is not copied. `to_json` is a copy of this. """ - if isinstance(field, RefusedField): - return field.json - return written_json(field) + if isinstance(declaration, RefusedDeclaration): + return declaration.json + return written_json(declaration) -def written_json(field: AcceptedField[Any] | UnclaimedField) -> JSONValue: - """A field that a scope accepted or left unclaimed, as a document writes it. This is `document_json` for a field that is JSON.""" - kind = field.read_as - if isinstance(field, UnclaimedField): - return kind._unclaimed_json(field.json, field.name, field.configuration) - if kind._filed_under(field.name)[1] is not None: - return kind._envelope_json(field.name, {}) - return kind._envelope_json(field.name, field.configuration) +def written_json(declaration: AcceptedDeclaration[Any] | UnclaimedDeclaration) -> JSONValue: + """A declaration that a scope accepted or left unclaimed, as a document writes it. This is `document_json` for a declaration that is JSON.""" + kind = declaration.read_as + if isinstance(declaration, UnclaimedDeclaration): + return kind._unclaimed_json(declaration.json, declaration.name, declaration.configuration) + if kind._filed_under(declaration.name)[1] is not None: + return kind._envelope_json(declaration.name, {}) + return kind._envelope_json(declaration.name, declaration.configuration) -Nested: TypeAlias = Mapping[Loc, ResolvedField[Any]] -"""The fields in a configuration, each as the scope read it, keyed by location in the configuration.""" +Nested: TypeAlias = Mapping[Loc, ResolvedDeclaration[Any]] +"""The declarations in a configuration, each as the scope read it, keyed by location in the configuration.""" -def is_field(value: object) -> TypeGuard[ResolvedField[Any]]: - """Whether `value` is a field that a scope read: `AcceptedField`, `UnclaimedField` or `RefusedField`.""" - return isinstance(value, _FIELDS) +def is_declaration(value: object) -> TypeGuard[ResolvedDeclaration[Any]]: + """Whether `value` is a declaration that a scope read: `AcceptedDeclaration`, `UnclaimedDeclaration` or `RefusedDeclaration`.""" + return isinstance(value, _DECLARATIONS) -def held(field: ResolvedField[D] | UNSET) -> AcceptedField[D] | UnclaimedField: - """Return `field`, which must come from a read that found no problems, as `AcceptedField` or `UnclaimedField`. Raises `TypeError` for a refused field or one never read, which such a read cannot produce.""" - if isinstance(field, (AcceptedField, UnclaimedField)): - return field - msg = f"expected a field a read found nothing wrong with, got {field!r}" +def held( + declaration: ResolvedDeclaration[D] | UNSET, +) -> AcceptedDeclaration[D] | UnclaimedDeclaration: + """Return `declaration`, which must come from a read that found no problems, as `AcceptedDeclaration` or `UnclaimedDeclaration`. Raises `TypeError` for a refused declaration or one never read, which such a read cannot produce.""" + if isinstance(declaration, (AcceptedDeclaration, UnclaimedDeclaration)): + return declaration + msg = f"expected a declaration a read found nothing wrong with, got {declaration!r}" raise TypeError(msg) -def fields_of( - resolved: ResolvedField[Any], loc: Loc = () -) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Yield `resolved` (a field a scope read) with its location, then each nested field recursively, each with its location. +def declarations_of( + resolved: ResolvedDeclaration[Any], loc: Loc = () +) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Yield `resolved` (a declaration a scope read) with its location, then each nested declaration recursively, each with its location. - `loc` is the location of `resolved`. A nested field is located where + `loc` is the location of `resolved`. A nested declaration is located where the document writes its place in the configuration, as the kind's `_in_configuration` says, the same way `resolve` locates problems. - Each field's nested fields are its `nested`. A field whose + Each declaration's nested declarations are its `nested`. A declaration whose configuration was not checked has none. """ yield loc, resolved for place, inner in resolved.nested.items(): - yield from fields_of(inner, resolved.read_as._in_configuration(loc, place)) + yield from declarations_of(inner, resolved.read_as._in_configuration(loc, place)) def with_problems( - fields: Iterable[tuple[Loc, ResolvedField[Any]]], problems: Sequence[ValidationProblem] -) -> Iterator[tuple[Loc, ResolvedField[Any], Problems]]: - """Yield each of `fields` with its location and the problems in `problems` located inside it, including inside its nested fields. - - `fields` and `problems` must come from the same read: a reading's - `fields()` and `problems`, or `fields_of` a field and the problems that - `resolve` returned with it. A field's problems include those found when + declarations: Iterable[tuple[Loc, ResolvedDeclaration[Any]]], + problems: Sequence[ValidationProblem], +) -> Iterator[tuple[Loc, ResolvedDeclaration[Any], Problems]]: + """Yield each of `declarations` with its location and the problems in `problems` located inside it, including inside its nested declarations. + + `declarations` and `problems` must come from the same read: a reading's + `declarations()` and `problems`, or `declarations_of` a declaration and the problems that + `resolve` returned with it. A declaration's problems include those found when it was read, and those the document found with it in its position (its place in the pipeline, the chunk it receives, the array's shape). So a - field with no problems is valid in that position, and `canonical_of` + declaration with no problems is valid in that position, and `canonical_of` can write it. This groups the problems at every depth, as zod's `treeifyError` groups issues: a problem with a shard's inner codec - belongs to the inner codec and to the shard. Each field comes before - its nested fields, in the order of `fields_of`, so the last field that - lists a problem is the innermost field containing it. A problem - outside every field (with the fill value or the shape) is not listed - under any field. + belongs to the inner codec and to the shard. Each declaration comes before + its nested declarations, in the order of `declarations_of`, so the last declaration that + lists a problem is the innermost declaration containing it. A problem + outside every declaration (with the fill value or the shape) is not listed + under any declaration. """ - located = list(fields) + located = list(declarations) held: dict[Loc, list[ValidationProblem]] = {loc: [] for loc, _ in located} for found in problems: for depth in range(len(found.loc) + 1): holder = held.get(found.loc[:depth]) if holder is not None: holder.append(found) - for loc, field in located: - yield loc, field, tuple(held[loc]) + for loc, declaration in located: + yield loc, declaration, tuple(held[loc]) -def configuration_of(resolved: ResolvedField[Any], definition: Definition[C]) -> C | None: +def configuration_of(resolved: ResolvedDeclaration[Any], definition: Definition[C]) -> C | None: """The configuration of `resolved`, typed as `definition` declares it, if `definition` accepted it. - `AcceptedField` stores its configuration as a read-only mapping. When - called with the definition that read the field, or one equal to it - (as with a pickled field), this returns a copy of it, a plain `dict` - typed as the TypedDict, which shares nothing with the field. Returns - None for any other field: one that another definition read, one that + `AcceptedDeclaration` stores its configuration as a read-only mapping. When + called with the definition that read the declaration, or one equal to it + (as with a pickled declaration), this returns a copy of it, a plain `dict` + typed as the TypedDict, which shares nothing with the declaration. Returns + None for any other declaration: one that another definition read, one that no definition claims, and one that `definition` claimed and refused. """ - if not isinstance(resolved, AcceptedField) or resolved.definition != definition: + if not isinstance(resolved, AcceptedDeclaration) or resolved.definition != definition: return None return cast("C", copied(resolved.configuration)) -def fill_value_problems(data_type: ResolvedField[F], value: object, loc: Loc = ()) -> Problems: - """The problems with `value` as a fill value of `data_type` (a data type field a scope read). +def fill_value_problems( + data_type: ResolvedDeclaration[F], value: object, loc: Loc = () +) -> Problems: + """The problems with `value` as a fill value of `data_type` (a data type declaration a scope read). `value` is refined to JSON first. If it is not JSON, that is reported first, whatever the data type. It is then checked against the JSON shape that the data type's definition declares, and by the fill value rules, in the same way `read_configuration` reads a configuration: a key the shape does not declare is reported and left out, and the rules - still check the rest. The rules see the nested fields of the + still check the rest. The rules see the nested declarations of the configuration as the scope read them, so a struct checks each field's fill value against that field's own type. If the scope did not read the data type (it is out of scope or invalid), a JSON fill value is not checked. `loc` prefixes every problem. """ refined, problems = refine_json(value, loc) - if len(problems) != 0 or not isinstance(data_type, AcceptedField): + if len(problems) != 0 or not isinstance(data_type, AcceptedDeclaration): return with_input(problems, value, loc) definition, configuration = data_type.definition, data_type.configuration typed, problems = _fill_value_parser(definition.fill_value)(refined, loc) @@ -1303,8 +1329,8 @@ def fill_value_problems(data_type: ResolvedField[F], value: object, loc: Loc = ( return with_input((*problems, *refused), value, loc) -def canonical_fill_value(data_type: ResolvedField[F], value: object) -> JSONValue | UNSET: - """`value` (a fill value of `data_type`, a data type field a scope read) in the single canonical spelling of its value; `UNSET` when it has a problem. +def canonical_fill_value(data_type: ResolvedDeclaration[F], value: object) -> JSONValue | UNSET: + """`value` (a fill value of `data_type`, a data type declaration a scope read) in the single canonical spelling of its value; `UNSET` when it has a problem. The spelling comes from the data type's `fill_value_canonical`. Two fill values of a data type are the same value exactly when their @@ -1312,7 +1338,7 @@ def canonical_fill_value(data_type: ResolvedField[F], value: object) -> JSONValu `json.dumps` writes it. `==` is not the same test, because it treats `-0.0` as equal to `0.0`. A fill value with a problem found by `fill_value_problems` has no canonical spelling, just as `canonical_of` - returns none for a field with a problem. The result is `UNSET` and not + returns none for a declaration with a problem. The result is `UNSET` and not `None`, because `None` is the JSON `null`, which is a valid fill value for a data type the scope did not read (such a data type keeps a fill value as written). @@ -1323,14 +1349,14 @@ def canonical_fill_value(data_type: ResolvedField[F], value: object) -> JSONValu return spelled_canonically(data_type, refined) -def spelled_canonically(data_type: ResolvedField[F], value: JSONValue) -> JSONValue: +def spelled_canonically(data_type: ResolvedDeclaration[F], value: JSONValue) -> JSONValue: """`value` (a fill value of `data_type` with no problems) in its canonical spelling, as `canonical_fill_value` returns it, without checking it again. The data type's `fill_value_canonical` is the extension author's code. Its result is checked to be JSON. If it raises an error, a note is added that says which data type's canonical spelling raised it. """ - if not isinstance(data_type, AcceptedField): + if not isinstance(data_type, AcceptedDeclaration): return value definition, configuration = data_type.definition, data_type.configuration spelled = asked( @@ -1353,28 +1379,28 @@ def spelled_canonically(data_type: ResolvedField[F], value: JSONValue) -> JSONVa def resolve( data: object, kind: type[D], context: Context, loc: Loc = () -) -> tuple[ResolvedField[D], Problems]: - """Read `data` (one metadata field) as a `kind` in `context`. Return the result of the reading, and every problem. +) -> tuple[ResolvedDeclaration[D], Problems]: + """Read `data` (one declaration) as a `kind` in `context`. Return the result of the reading, and every problem. - This runs all three steps for one field. `data` is refined to JSON and - its envelope is checked. An extra member, a `configuration` that is not + This runs all three steps for one declaration. `data` is refined to JSON and + its envelope is checked. An extra envelope member, a `configuration` that is not an object, and a `must_understand` that is not a boolean or is `false` are each a problem. The name is mapped to a definition in `context`. The configuration is checked against its TypedDict and by its rules. - Each nested field found by the check is read the same way, in the same - scope. A problem in a nested field belongs to that field and is - reported at its location, as with a document's fields. The result is - an `AcceptedField` read by the definition that claims the name; an - `UnclaimedField` when nothing in scope claims the name (an extension + Each nested declaration found by the check is read the same way, in the same + scope. A problem in a nested declaration belongs to that declaration and is + reported at its location, as with a document's declarations. The result is + an `AcceptedDeclaration` read by the definition that claims the name; an + `UnclaimedDeclaration` when nothing in scope claims the name (an extension with no model, not checked, which is what keeps the format open); or a - `RefusedField`, with the problems that explain why. `loc` prefixes + `RefusedDeclaration`, with the problems that explain why. `loc` prefixes every problem. `kind` must be one of the kinds of a format (for v3, those `ZarrV3Kind` names; for v2, those `ZarrV2Kind` names), with or without type arguments, and `context` a scope of that format. Anything else raises `TypeError`. """ if not _is_scope(context): - msg = f"a field is read in a scope, such as CORE_AND_EXTENSIONS, got {described(context)}" + msg = f"a declaration is read in a scope, such as CORE_AND_EXTENSIONS_CONTEXT, got {described(context)}" raise TypeError(msg) asked = as_kind(kind) _check_format(asked, context) @@ -1391,7 +1417,7 @@ def _check_format(kind: type[Definition[Any]], context: Context) -> None: scope_format = context.zarr_format if scope_format != format_of(kind): msg = ( - f"a {kind.label} is a field of a Zarr v{format_of(kind)} document, read in a scope " + f"a {kind.label} is a declaration of a Zarr v{format_of(kind)} document, read in a scope " f"of that format, got a v{scope_format} scope" ) raise TypeError(msg) @@ -1432,12 +1458,12 @@ def resolve_refined( kind: type[D], context: Context, loc: Loc = (), -) -> tuple[ResolvedField[D], Problems]: +) -> tuple[ResolvedDeclaration[D], Problems]: """`resolve` for `data` that the caller has already refined. `refined` and `problems` are the result of `refine_json(data, loc)`. This is for a reader that refines a document's members once and keeps the result (the document a model holds is built from it), instead of - refining each field again here. `problems` shows whether `data` is + refining each declaration again here. `problems` shows whether `data` is JSON. If it contains any problems, `refined` is not used. """ asked = as_kind(kind) @@ -1450,22 +1476,22 @@ def resolve_refined( name = asked._named_configuration(data)[0] bad = None if name is None else asked._name_problem(name, asked._name_loc(loc)) claimant = None if name is None or bad is not None else context.claimant(asked, name) - refused = RefusedField(json=UNSET, name=name, read_as=asked, definition=claimant) + refused = RefusedDeclaration(json=UNSET, name=name, read_as=asked, definition=claimant) found = problems if bad is None else (bad, *problems) - return cast("ResolvedField[D]", refused), with_input(found, data, loc) - resolved, found = _resolve_field(refined, asked, context, loc) - return cast("ResolvedField[D]", resolved), with_input(found, data, loc) + return cast("ResolvedDeclaration[D]", refused), with_input(found, data, loc) + resolved, found = _resolve_declaration(refined, asked, context, loc) + return cast("ResolvedDeclaration[D]", resolved), with_input(found, data, loc) -def _resolve_field( +def _resolve_declaration( data: JSONValue, kind: type[Definition[Any]], context: Context, loc: Loc -) -> tuple[ResolvedField[Definition[Any]], Problems]: - """Check the envelope of a refined field, then read the field. +) -> tuple[ResolvedDeclaration[Definition[Any]], Problems]: + """Check the envelope of a refined declaration, then read the declaration. The result of the reading depends only on the configuration. An extra member or a `must_understand` of `false` does not affect the - configuration, so it is reported alongside the field that was read, - and later layers can still check that field. + configuration, so it is reported alongside the declaration that was read, + and later layers can still check that declaration. """ envelope = _located(loc, kind._envelope_problems(data)) resolved, found = _read(data, kind, context, loc) @@ -1474,58 +1500,60 @@ def _resolve_field( def _read( data: JSONValue, kind: type[Definition[Any]], context: Context, loc: Loc -) -> tuple[ResolvedField[Definition[Any]], Problems]: +) -> tuple[ResolvedDeclaration[Definition[Any]], Problems]: name, given, malformed = kind._named_configuration(data) if name is None: - return RefusedField(json=data, name=None, read_as=kind), () + return RefusedDeclaration(json=data, name=None, read_as=kind), () if not kind._well_named(name): # The envelope check that every reader runs first reports this. No # definition is looked up for the name. - return RefusedField(json=data, name=name, read_as=kind), () + return RefusedDeclaration(json=data, name=name, read_as=kind), () definition = context.claimant(kind, name) if len(malformed) != 0: # The configuration is not an object, which the envelope problems # report. The name still determines which definition claims the - # field. - return RefusedField(json=data, name=name, read_as=kind, definition=definition), () + # declaration. + return RefusedDeclaration(json=data, name=name, read_as=kind, definition=definition), () if definition is None: - return UnclaimedField(json=data, name=name, read_as=kind), () + return UnclaimedDeclaration(json=data, name=name, read_as=kind), () _, carried = kind._filed_under(name) if carried is not None: return _read_carried(data, name, kind, definition, given, carried, loc) at = kind._in_configuration(loc, ()) if given is None and definition.requires_configuration: missing = problem(at, f"{name!r} requires a configuration", "missing_key") - return RefusedField(json=data, name=name, read_as=kind, definition=definition), missing + return RefusedDeclaration( + json=data, name=name, read_as=kind, definition=definition + ), missing typed, found, nested = _typed(definition.configuration, {} if given is None else given, at) - # Read the nested fields first, each one stack frame deeper, so the + # Read the nested declarations first, each one stack frame deeper, so the # rules see them as the scope read them. Each is put back as a document # writes it, so the configuration shows what was read, however each was # spelled. Their problems are reported after the problems from the # rules. - within: dict[Loc, ResolvedField[Any]] = {} + within: dict[Loc, ResolvedDeclaration[Any]] = {} written: dict[Loc, JSONValue] = {} inside: list[ValidationProblem] = [] - for field in nested: - place = field.loc[len(at) :] - placed = dataclasses.replace(field, loc=kind._in_configuration(loc, place)) + for declaration in nested: + place = declaration.loc[len(at) :] + placed = dataclasses.replace(declaration, loc=kind._in_configuration(loc, place)) inside.extend(_envelope(placed)) - inner, found_inside = _read(field.json, field.kind, context, placed.loc) + inner, found_inside = _read(declaration.json, declaration.kind, context, placed.loc) within[place] = inner - # The input was refined JSON, so the result is JSON, or a field + # The input was refined JSON, so the result is JSON, or a declaration # refused for its contents, never for not being JSON. - written[field.loc] = cast("JSONValue", document_json(inner)) + written[declaration.loc] = cast("JSONValue", document_json(inner)) inside.extend(found_inside) inside.extend(_sized(placed, inner)) - # The rules may look up a nested field by its name, so they run only - # when every nested field has a name. Any other problem with a nested - # field belongs to that field and is reported at its location, as with - # a document's fields. - sound = _usable(found) and all(_named(field) for field in nested) + # The rules may look up a nested declaration by its name, so they run only + # when every nested declaration has a name. Any other problem with a nested + # declaration belongs to that declaration and is reported at its location, as with + # a document's declarations. + sound = _usable(found) and all(_named(declaration) for declaration in nested) configuration = ( cast( "Mapping[str, JSONValue]", - _put_back(typed, lambda field: written[field.loc], nested), + _put_back(typed, lambda declaration: written[declaration.loc], nested), ) if sound else None @@ -1537,36 +1565,36 @@ def _read( ) placed = _in_document(kind, loc, given is not None, own) if configuration is None or not _usable(placed): - refused = RefusedField( + refused = RefusedDeclaration( json=data, name=name, read_as=kind, definition=definition, nested=within ) return refused, (*placed, *inside) - read = AcceptedField( + read = AcceptedDeclaration( json=data, name=name, definition=definition, configuration=configuration, nested=within ) return read, (*placed, *inside) def in_document( - field: ResolvedField[Any], loc: Loc, problems: Sequence[ValidationProblem] + declaration: ResolvedDeclaration[Any], loc: Loc, problems: Sequence[ValidationProblem] ) -> Problems: - """`problems`, which a definition's functions found in the configuration of `field` (a field at `loc`), located under the configuration as the kind locates it, each moved to where the document writes it, as `resolve` places a definition's rules.""" - written = field.read_as._named_configuration(field.json)[1] is not None - return _in_document(field.read_as, loc, written, problems) + """`problems`, which a definition's functions found in the configuration of `declaration` (a declaration at `loc`), located under the configuration as the kind locates it, each moved to where the document writes it, as `resolve` places a definition's rules.""" + written = declaration.read_as._named_configuration(declaration.json)[1] is not None + return _in_document(declaration.read_as, loc, written, problems) def _in_document( kind: type[Definition[Any]], loc: Loc, written: bool, problems: Sequence[ValidationProblem] ) -> Problems: - """`problems`, found in the configuration of a field of `kind` at `loc` and located under the configuration's location, each moved to where the document writes its place, as `kind._in_configuration` says. + """`problems`, found in the configuration of a declaration of `kind` at `loc` and located under the configuration's location, each moved to where the document writes its place, as `kind._in_configuration` says. - `written` says whether the document writes a configuration. A field + `written` says whether the document writes a configuration. A declaration whose configuration has a location of its own, and which writes none (a v3 codec written as its name alone, or with no `configuration`), has an empty configuration that the document does not hold. A member found missing from it is then the configuration itself, missing, as - for a field that requires a configuration, and a problem with the - configuration as a whole is the field's. + for a declaration that requires a configuration, and a problem with the + configuration as a whole is the declaration's. """ at = kind._in_configuration(loc, ()) absent = not written and at != loc @@ -1581,9 +1609,9 @@ def _in_document( return tuple(placed) -def _named(field: _NestedField) -> bool: - """Whether a nested field has a name, and its configuration (if any) is an object.""" - name, _, malformed = field.kind._named_configuration(field.json) +def _named(declaration: _NestedDeclaration) -> bool: + """Whether a nested declaration has a name, and its configuration (if any) is an object.""" + name, _, malformed = declaration.kind._named_configuration(declaration.json) return name is not None and len(malformed) == 0 @@ -1595,12 +1623,12 @@ def _read_carried( given: Mapping[str, object] | None, carried: Mapping[str, JSONValue], loc: Loc, -) -> tuple[ResolvedField[Definition[Any]], Problems]: - """Read a field whose name encodes its configuration (raw bits, such as `r16`) with the definition its name is filed under. +) -> tuple[ResolvedDeclaration[Definition[Any]], Problems]: + """Read a declaration whose name encodes its configuration (raw bits, such as `r16`) with the definition its name is filed under. The document wrote a name, so a problem with the configuration encoded in the name (such as a size that is not a positive multiple of 8) is a - problem of the field. A configuration written next to the name must be + problem of the declaration. A configuration written next to the name must be empty, so each member in it is an undeclared key. """ _, beside, _ = _checked( @@ -1617,7 +1645,7 @@ def _read_carried( undeclared = [key for key in carried if key not in members] configuration, judged = definition.read_configuration(declared) # A problem with the configuration encoded in the name belongs to the - # field. It is located at the field, where the name is, and the + # declaration. It is located at the declaration, where the name is, and the # `input` and `ctx` that described a configuration member do not apply # to the name, so they are cleared. Each is a wrong value of the name: # a part it lacks is not a missing key. @@ -1631,17 +1659,17 @@ def _read_carried( if len(undeclared) != 0: message = kind._undeclared_parts(definition.name, name, undeclared) problems = (*problems, ValidationProblem(loc, message, "invalid_value")) - # Any problem with the encoded configuration refuses the field. + # Any problem with the encoded configuration refuses the declaration. if configuration is None or not _usable(beside) or len(judged) != 0 or len(undeclared) != 0: - refused = RefusedField(json=data, name=name, read_as=kind, definition=definition) + refused = RefusedDeclaration(json=data, name=name, read_as=kind, definition=definition) return refused, problems - return AcceptedField( + return AcceptedDeclaration( json=data, name=name, definition=definition, configuration=configuration ), problems -def _sized(field: _NestedField, inner: ResolvedField[Any]) -> Problems: - """The problems with the definition that read `inner` in the member that holds `field`, as the kind of `field` reports them for its alias. +def _sized(declaration: _NestedDeclaration, inner: ResolvedDeclaration[Any]) -> Problems: + """The problems with the definition that read `inner` in the member that holds `declaration`, as the kind of `declaration` reports them for its alias. A name that nothing in scope claims is not checked. Nothing is known about it. @@ -1649,33 +1677,33 @@ def _sized(field: _NestedField, inner: ResolvedField[Any]) -> Problems: definition = inner.definition if definition is None: return () - return field.kind._alias_problems(field.alias, definition, field.loc) + return declaration.kind._alias_problems(declaration.alias, definition, declaration.loc) def canonicalize( data: object, kind: type[D], context: Context, loc: Loc = () ) -> tuple[JSONValue | None, Problems]: - """`data` (one metadata field) in its simplest equivalent spelling, and every problem. + """`data` (one declaration) in its simplest equivalent spelling, and every problem. - Only a field without problems has a simplest spelling. A simplest + Only a declaration without problems has a simplest spelling. A simplest spelling expresses what the author wrote in fewer words. A key the TypedDict does not declare, an extra envelope member, or a `must_understand` of `false` is something the author wrote that the - simplest spelling would erase. So a field with any problem returns - None, with its problems. Otherwise each nested field is written in its + simplest spelling would erase. So a declaration with any problem returns + None, with its problems. Otherwise each nested declaration is written in its own simplest spelling, and then the definition's `canonical` handles the rest. Its result is checked again, so a `canonical` that returns an invalid configuration raises `ValueError`, which indicates a bug in - the definition, not in the field. The envelope uses the shortest form + the definition, not in the declaration. The envelope uses the shortest form every reader accepts. A data type with nothing to configure is written as its bare name, as core data types have been written since Zarr - v3.0. Any other field is an object, `{"name": ...}`, because a Zarr + v3.0. Any other declaration is an object, `{"name": ...}`, because a Zarr v3.0 reader does not accept a short-hand name in `codecs` (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L585-L592). There is no `must_understand`, because leaving it out means `true`. A name that nothing in scope claims keeps the configuration it was written with, because only its own definition could decide how to - simplify it. For a field that a scope has already read, use + simplify it. For a declaration that a scope has already read, use `canonical_of` with its problems. """ resolved, problems = resolve(data, kind, context, loc) @@ -1683,17 +1711,17 @@ def canonicalize( def canonical_of( - resolved: ResolvedField[Any], problems: Sequence[ValidationProblem] + resolved: ResolvedDeclaration[Any], problems: Sequence[ValidationProblem] ) -> JSONValue | None: - """`resolved` (a field a scope read) in its simplest equivalent spelling, as `canonicalize` writes it; None when it has a problem. + """`resolved` (a declaration a scope read) in its simplest equivalent spelling, as `canonicalize` writes it; None when it has a problem. - `problems` are the field's problems, as returned by `resolve`, or as - `with_problems` returns them for each field of a reading. Only a field + `problems` are the declaration's problems, as returned by `resolve`, or as + `with_problems` returns them for each declaration of a reading. Only a declaration with no problems has a simplest spelling. A simpler spelling of a - field with a problem would erase what its author wrote (such as a key + declaration with a problem would erase what its author wrote (such as a key its TypedDict does not declare), or would write something invalid. The result is built from what the scope read, without reading the - field again, and is plain JSON that shares nothing with the field. + declaration again, and is plain JSON that shares nothing with the declaration. """ if len(problems) != 0: return None @@ -1701,17 +1729,17 @@ def canonical_of( return None if simplest is None else copied(simplest) -def _simplest(field: ResolvedField[Any]) -> JSONValue | None: - """A field in its simplest spelling; None when the field or one of its nested fields was refused, because a refused field has no simplest spelling.""" - if isinstance(field, AcceptedField): - return _canonical_field(field) - if isinstance(field, UnclaimedField): - return field.to_json() +def _simplest(declaration: ResolvedDeclaration[Any]) -> JSONValue | None: + """A declaration in its simplest spelling; None when the declaration or one of its nested declarations was refused, because a refused declaration has no simplest spelling.""" + if isinstance(declaration, AcceptedDeclaration): + return _canonical_declaration(declaration) + if isinstance(declaration, UnclaimedDeclaration): + return declaration.to_json() return None -def _canonical_field(resolved: AcceptedField[Any]) -> JSONValue | None: - """An accepted field in its simplest equivalent spelling: nested fields first, then its own members. None when a nested field was refused.""" +def _canonical_declaration(resolved: AcceptedDeclaration[Any]) -> JSONValue | None: + """An accepted declaration in its simplest equivalent spelling: nested declarations first, then its own members. None when a nested declaration was refused.""" definition, name = resolved.definition, resolved.name configuration: JSONValue = dict(resolved.configuration) for loc, inner in resolved.nested.items(): @@ -1755,16 +1783,16 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: __all__ = [ - "AcceptedField", + "AcceptedDeclaration", "C", "Context", "Definition", "EmptyConfiguration", "Nested", "Problems", - "RefusedField", - "ResolvedField", - "UnclaimedField", + "RefusedDeclaration", + "ResolvedDeclaration", + "UnclaimedDeclaration", "WithFillValue", "as_kind", "asked", @@ -1772,17 +1800,17 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "canonical_of", "canonicalize", "configuration_of", + "declaration_key", + "declaration_kind", + "declarations_of", "document_json", - "field_key", - "field_kind", - "fields_of", "filed_under", "fill_value_as_written", "fill_value_problems", "format_of", "held", "in_document", - "is_field", + "is_declaration", "kind_of", "named_configuration", "no_rules", diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_envelope.py b/packages/zarr-metadata/src/zarr_metadata/definition/_envelope.py index 01c9cdfc46..b47659068d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_envelope.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_envelope.py @@ -1,4 +1,4 @@ -"""The default field envelope: how a field is written as a name or as an object with `name`, `configuration` and `must_understand`, and which names are valid extension names. +"""The default declaration envelope: how a declaration is written as a name or as an object with `name`, `configuration` and `must_understand`, and which names are valid extension names. This is the envelope Zarr v3 uses. `Definition` uses it by default, and a kind of another format (the v2 kinds) overrides each method that uses it. @@ -11,7 +11,7 @@ from zarr_metadata._json import ValidationProblem, is_object, shown, shown_key ENVELOPE_KEYS: frozenset[str] = frozenset({"name", "configuration", "must_understand"}) -"""The keys a metadata field's envelope declares. Any other key is an unknown key.""" +"""The keys a declaration's envelope declares. Any other key is an unknown key.""" _EXTENSION_NAME: Final = re.compile(r"[a-z][a-z0-9_.-]+") @@ -61,12 +61,12 @@ def name_problem(name: str, at: tuple[str | int, ...]) -> ValidationProblem | No def envelope_problems( value: object, *, allow_must_understand_false: bool ) -> tuple[ValidationProblem, ...]: - """Every reason `value` is not a valid v3 metadata field envelope. The contents of the configuration are not checked. + """Every reason `value` is not a valid v3 declaration envelope. The contents of the configuration are not checked. The envelope is the part around the configuration: a `name` that is a valid extension name under the spec, a `configuration` that is an object with string keys, a boolean `must_understand`, and no other - keys. `resolve` calls this on a field it has already narrowed to JSON, + keys. `resolve` calls this on a declaration it has already narrowed to JSON, so the configuration is walked only once. """ if isinstance(value, str): @@ -76,29 +76,29 @@ def envelope_problems( return ( ValidationProblem( (), - f"expected a metadata field (string or extension object), got {shown(value)}", + f"expected a declaration (string or extension object), got {shown(value)}", "invalid_type", ), ) - field = value + declaration = value problems: list[ValidationProblem] = [] - for key in field: + for key in declaration: if not isinstance(key, str): problems.append( ValidationProblem( - (), f"non-string metadata field key {shown_key(key)}", "invalid_type" + (), f"non-string declaration key {shown_key(key)}", "invalid_type" ) ) elif key not in ENVELOPE_KEYS: problems.append(ValidationProblem((key,), f"unexpected key {key!r}", "unknown_key")) - if "name" not in field: + if "name" not in declaration: problems.append(ValidationProblem(("name",), "missing required key", "missing_key")) - elif not isinstance(field["name"], str): + elif not isinstance(declaration["name"], str): problems.append(ValidationProblem(("name",), "expected a string name", "invalid_type")) - elif (bad := name_problem(field["name"], ("name",))) is not None: + elif (bad := name_problem(declaration["name"], ("name",))) is not None: problems.append(bad) - if "configuration" in field: - configuration = field["configuration"] + if "configuration" in declaration: + configuration = declaration["configuration"] if not is_object(configuration): problems.append( ValidationProblem(("configuration",), "expected an object", "invalid_type") @@ -107,8 +107,8 @@ def envelope_problems( problems.append( ValidationProblem(("configuration",), "expected string keys", "invalid_type") ) - if "must_understand" in field: - must_understand = field["must_understand"] + if "must_understand" in declaration: + must_understand = declaration["must_understand"] if not isinstance(must_understand, bool): problems.append( ValidationProblem(("must_understand",), "expected a boolean", "invalid_type") diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py index 11ad7e1563..9a28bc2a32 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py @@ -20,13 +20,13 @@ from typing import TYPE_CHECKING, Any, Literal, TypeAlias, cast from zarr_metadata.definition._definition import ( - AcceptedField, + AcceptedDeclaration, Definition, - RefusedField, - UnclaimedField, + RefusedDeclaration, + UnclaimedDeclaration, as_kind, - field_key, - fields_of, + declaration_key, + declarations_of, filed_under, format_of, own_key, @@ -37,7 +37,7 @@ from collections.abc import Callable, Iterable, Sequence from zarr_metadata._typed_json import Loc - from zarr_metadata.definition._definition import D, ResolvedField + from zarr_metadata.definition._definition import D, ResolvedDeclaration ClaimKey: TypeAlias = tuple[type[Definition[Any]], str] """A kind and the name a definition is filed under. A scope's `claimant` looks up definitions by this key.""" @@ -51,9 +51,13 @@ class Conflict: """One place where two readings of a name disagree: the key, the definition one reading claimed, the definition the other found, and the location in the document when known.""" key: ClaimKey + """The kind, and the name the two definitions are filed under.""" claimed: Definition[Any] | None + """The definition one reading or scope gives the name; None when it leaves the name unclaimed.""" found: Definition[Any] | None + """The definition the other gives the name; None when it gives none, so the name would lose its meaning.""" loc: Loc | None = None + """Where the name occurs in the document; None when no document is available.""" written: str | None = None """The name the document uses at the conflict location, for example `r16` for a key of `r*`; None when no document is available.""" @@ -90,6 +94,7 @@ class ScopeConflictError(ValueError): def __init__(self, conflicts: Sequence[Conflict]) -> None: self.conflicts: tuple[Conflict, ...] = tuple(conflicts) + """Every conflict, in the order found; the message joins them.""" super().__init__("; ".join(str(conflict) for conflict in self.conflicts)) def __reduce__( @@ -101,31 +106,31 @@ def __reduce__( return type(self), (self.conflicts,), dict(self.__dict__) -def claim_key(field: ResolvedField[Any]) -> ClaimKey | None: - """The key `field` is claimed under: its kind and the name its definition is filed under (for example `r*` for `r16`). Returns None for a field with no name.""" - if field.name is None: +def claim_key(declaration: ResolvedDeclaration[Any]) -> ClaimKey | None: + """The key `declaration` is claimed under: its kind and the name its definition is filed under (for example `r*` for `r16`). Returns None for a declaration with no name.""" + if declaration.name is None: return None - filed, _ = filed_under(field.read_as, field.name) - return None if filed is None else (field.read_as, filed) + filed, _ = filed_under(declaration.read_as, declaration.name) + return None if filed is None else (declaration.read_as, filed) def claims_of( - fields: Iterable[tuple[Loc, ResolvedField[Any]]], + declarations: Iterable[tuple[Loc, ResolvedDeclaration[Any]]], ) -> dict[ClaimKey, Definition[Any] | None]: - """The claims that `fields` (each paired with its location) record for each name: the definition that read the name, or None if the name was unclaimed. + """The claims that `declarations` (each paired with its location) record for each name: the definition that read the name, or None if the name was unclaimed. - `fields` must come from one reading, as returned by `fields_of` or a - reading's `fields()`. If two fields claim the same name with different + `declarations` must come from one reading, as returned by `declarations_of` or a + reading's `declarations()`. If two declarations claim the same name with different definitions, this raises `ScopeConflictError`, because no single scope could have read them both. """ claims: dict[ClaimKey, Definition[Any] | None] = {} conflicts: list[Conflict] = [] - for loc, field in fields: - key = claim_key(field) + for loc, declaration in declarations: + key = claim_key(declaration) if key is None: continue - definition = field.definition + definition = declaration.definition if key in claims and claims[key] != definition: conflicts.append(Conflict(key, claims[key], definition, loc)) continue @@ -135,54 +140,54 @@ def claims_of( return claims -def refines(field: ResolvedField[Any], other: ResolvedField[Any]) -> bool: - """Whether `field` contains everything `other` contains: it reads the same where both read, and it may also read what `other` left unclaimed. +def refines(declaration: ResolvedDeclaration[Any], other: ResolvedDeclaration[Any]) -> bool: + """Whether `declaration` contains everything `other` contains: it reads the same where both read, and it may also read what `other` left unclaimed. This is the order in which one reading of a document refines another. - Suppose `other` is unclaimed and `field` is accepted. This counts as a - gain when re-reading the unclaimed field's JSON, with `field`'s - definition and the definitions of the fields nested in `field`, - produces `field`. Two spellings of one configuration count as one gain, - because `==` treats them as one field, so the order is transitive + Suppose `other` is unclaimed and `declaration` is accepted. This counts as a + gain when re-reading the unclaimed declaration's JSON, with `declaration`'s + definition and the definitions of the declarations nested in `declaration`, + produces `declaration`. Two spellings of one configuration count as one gain, + because `==` treats them as one declaration, so the order is transitive through equality. The reverse direction (accepted to unclaimed) is a loss. One name read by two different definitions is a conflict. A - refused field refines only itself. Two fields that refine each other + refused declaration refines only itself. Two declarations that refine each other are equal. """ - if isinstance(field, RefusedField) or isinstance(other, RefusedField): - return field == other - if isinstance(other, UnclaimedField): - if isinstance(field, UnclaimedField): - return field_key(field) == field_key(other) - return claim_key(field) == claim_key(other) and _gained(field, other) - if isinstance(field, UnclaimedField): + if isinstance(declaration, RefusedDeclaration) or isinstance(other, RefusedDeclaration): + return declaration == other + if isinstance(other, UnclaimedDeclaration): + if isinstance(declaration, UnclaimedDeclaration): + return declaration_key(declaration) == declaration_key(other) + return claim_key(declaration) == claim_key(other) and _gained(declaration, other) + if isinstance(declaration, UnclaimedDeclaration): return False - if field.definition != other.definition or own_key(field) != own_key(other): + if declaration.definition != other.definition or own_key(declaration) != own_key(other): return False - if set(field.nested) != set(other.nested): + if set(declaration.nested) != set(other.nested): return False - return all(refines(field.nested[loc], other.nested[loc]) for loc in field.nested) + return all(refines(declaration.nested[loc], other.nested[loc]) for loc in declaration.nested) -def _gained(field: AcceptedField[Any], other: UnclaimedField) -> bool: - """Whether reading `other` in the scope formed by `field`'s own claims produces `field`. This is the test for a gain.""" - again, _ = resolve(other.json, field.read_as, _Claimed.of(field)) - return again == field +def _gained(declaration: AcceptedDeclaration[Any], other: UnclaimedDeclaration) -> bool: + """Whether reading `other` in the scope formed by `declaration`'s own claims produces `declaration`. This is the test for a gain.""" + again, _ = resolve(other.json, declaration.read_as, _Claimed.of(declaration)) + return again == declaration @dataclass(frozen=True, slots=True) class _Claimed: - """The scope formed by a field's own claims: its definition and the definitions of its nested fields, keyed by kind and filed name. `_gained` re-reads a field in this scope.""" + """The scope formed by a declaration's own claims: its definition and the definitions of its nested declarations, keyed by kind and filed name. `_gained` re-reads a declaration in this scope.""" filed: Mapping[ClaimKey, Definition[Any]] zarr_format: Literal[2, 3] @classmethod - def of(cls, field: AcceptedField[Any]) -> _Claimed: - claimed = claims_of(fields_of(field)) + def of(cls, declaration: AcceptedDeclaration[Any]) -> _Claimed: + claimed = claims_of(declarations_of(declaration)) return cls( {key: definition for key, definition in claimed.items() if definition is not None}, - format_of(field.read_as), + format_of(declaration.read_as), ) def claimant(self, kind: type[D], name: str) -> D | None: diff --git a/packages/zarr-metadata/src/zarr_metadata/json_schema.py b/packages/zarr-metadata/src/zarr_metadata/json_schema.py index 9b7443a8f4..2296b78652 100644 --- a/packages/zarr-metadata/src/zarr_metadata/json_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/json_schema.py @@ -1,4 +1,4 @@ -"""JSON Schema output: a TypedDict, one Zarr v3 field, or a whole v3 `zarr.json`. +"""JSON Schema output: a TypedDict, one Zarr v3 declaration, or a whole v3 `zarr.json`. Each function writes JSON Schema draft 2020-12, for a validator in another language or for an editor that checks a document while it is @@ -23,34 +23,34 @@ # 'properties': {'level': {'type': 'integer', 'minimum': 0, 'maximum': 9}}, # 'required': ['level'], 'additionalProperties': False} -**One v3 field.** `field_json_schema_v3(kind, context=None)` writes one field -read as `kind` in the scope `context`: a field that a definition in +**One v3 declaration.** `declaration_json_schema_v3(kind, context=None)` writes one declaration +read as `kind` in the scope `context`: a declaration that a definition in scope reads, or a name that no definition claims, with any configuration. **A whole `zarr.json`.** `node_metadata_json_schema_v3(context=None)` writes a whole v3 `zarr.json`, array or group, as the scope reads it -(`CORE_AND_EXTENSIONS` when none is given). Its fields are written +(`CORE_AND_EXTENSIONS_CONTEXT` when none is given). Its declarations are written as the scope reads them. For each definition, the schema accepts its -field (its name, its configuration as its TypedDict declares it, +declaration (its name, its configuration as its TypedDict declares it, including bounds, a `must_understand` of `true`, and its bare name when it needs no configuration). It also accepts a name that nothing in scope -claims, with any configuration. A nested field is written in the same +claims, with any configuration. A nested declaration is written in the same scope, and a member that accepts only codecs of static size accepts -only those. The rules are not in the schema, so a field it accepts may -still have a problem. Any field that `resolve` reads without a problem +only those. The rules are not in the schema, so a declaration it accepts may +still have a problem. Any declaration that `resolve` reads without a problem is accepted by the schema, as JSON (arrays as lists, as a parser -returns them). Each configuration TypedDict and each field alias is +returns them). Each configuration TypedDict and each declaration alias is written once, in `$defs`, under its name. The fill value must match its data type's fill value schema. """ from zarr_metadata._typed_json import JSONSchema, json_schema -from zarr_metadata.v3._json_schema import field_json_schema_v3, node_metadata_json_schema_v3 +from zarr_metadata.v3._json_schema import declaration_json_schema_v3, node_metadata_json_schema_v3 __all__ = [ "JSONSchema", - "field_json_schema_v3", + "declaration_json_schema_v3", "json_schema", "node_metadata_json_schema_v3", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 98776c7c5d..0228861744 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -9,7 +9,7 @@ `read_consolidated_metadata_v2`. A reader checks a document's JSON structure. In a v3 document it also reads each extension point (codecs, chunk grids, data types, ...) through the definition that claims its -name in a scope. The scope is `CORE_AND_EXTENSIONS` unless a `context` +name in a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. A v3 reader checks the fill value against the data type the document names, the chunk grid against the shape, and the codecs as a pipeline, each codec against the chunk it receives. Each reader does one @@ -26,7 +26,7 @@ They undo each bug with `repair_node_metadata_v3` or `repair_node_metadata_v2` before the strict read, and report what they changed. `zarr_metadata.json_schema` writes a JSON Schema of what the v3 -readers read, except for the rules. One field on its own is read with +readers read, except for the rules. One declaration on its own is read with `zarr_metadata.definition.resolve`. The model constructors `from_json` and `from_key_value` raise diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index 7768e68210..ecaaaa621f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -29,11 +29,11 @@ from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._context import disagreements, scoped from zarr_metadata.definition._definition import ( - AcceptedField, - ResolvedField, - UnclaimedField, - field_key, - fields_of, + AcceptedDeclaration, + ResolvedDeclaration, + UnclaimedDeclaration, + declaration_key, + declarations_of, fill_value_problems, held, resolve, @@ -41,7 +41,7 @@ spelled_canonically, ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of -from zarr_metadata.definition._scope import refines as refines_field +from zarr_metadata.definition._scope import refines as refines_declaration from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, @@ -55,12 +55,12 @@ missing_keys, other_members, reading_of, - unclaimed_fields_of, + unclaimed_declarations_of, ) # A runtime import: `get_type_hints` resolves the public signatures. from zarr_metadata.v2._context import ZarrV2Context # noqa: TC001 -from zarr_metadata.v2._field_json import ZarrV2DataTypeMetadata +from zarr_metadata.v2._declaration_json import ZarrV2DataTypeMetadata from zarr_metadata.v2.array import ( ZARR_V2_ARRAY_METADATA_STORE_KEY, ZarrV2ArrayDimensionSeparator, @@ -70,7 +70,7 @@ from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY from zarr_metadata.v2.codec import ZarrV2CodecMetadata from zarr_metadata.v2.definition import ( - CORE_V2, + CORE_V2_CONTEXT, ZarrV2CodecDefinition, ZarrV2DataTypeDefinition, ) @@ -119,7 +119,7 @@ def _is_canonical_dtype_v2(value: object) -> bool: @dataclass(frozen=True, slots=True) class ArrayMembersV2: - """The members other than fields of a v2 array document that a read found no problems with, refined as the model holds them.""" + """The members that hold no declaration in a v2 array document that a read found no problems with, refined as the model holds them.""" shape: tuple[int, ...] chunks: tuple[int, ...] @@ -136,25 +136,25 @@ class ArrayMembersV2: class ZarrV2ArrayMetadataReading: """A v2 array document as a scope read it, whatever it holds: its dtype, compressor and filters, every problem, and the model when there are no problems. - A field the document does not hold is `UNSET`, and so is a `filters` + A declaration the document does not hold is `UNSET`, and so is a `filters` that is not an array, which holds no filters to read. A `compressor` - or `filters` written as `null` is None. Each problem and each field is + or `filters` written as `null` is None. Each problem and each declaration is located in what was read: the document, or the files of a node that `read_node_metadata_v2` reads, where `at` is `(".zarray",)`. """ - dtype: ResolvedField[ZarrV2DataTypeDefinition[Any]] | UNSET = UNSET + dtype: ResolvedDeclaration[ZarrV2DataTypeDefinition[Any]] | UNSET = UNSET """The dtype, as the scope read it.""" - compressor: ResolvedField[ZarrV2CodecDefinition[Any]] | UNSET | None = UNSET + compressor: ResolvedDeclaration[ZarrV2CodecDefinition[Any]] | UNSET | None = UNSET """The compressor, as the scope read it; None when written as `null`.""" - filters: tuple[ResolvedField[ZarrV2CodecDefinition[Any]], ...] | UNSET | None = UNSET + filters: tuple[ResolvedDeclaration[ZarrV2CodecDefinition[Any]], ...] | UNSET | None = UNSET """The filters, each as the scope read it; None when written as `null`.""" problems: tuple[ValidationProblem, ...] = () """Every reason the document is not valid.""" metadata: ZarrV2ArrayMetadata | None = None - """The document's model, which holds these fields, when there is no problem; None otherwise.""" + """The document's model, which holds these declarations, when there is no problem; None otherwise.""" at: Loc = () - """Where the document's fields are in what was read: `()` for a document read alone, `(".zarray",)` for the files of a node. A model's own reading is of its merged document, at `()`.""" + """Where the document's declarations are in what was read: `()` for a document read alone, `(".zarray",)` for the files of a node. A model's own reading is of its merged document, at `()`.""" def __reduce__(self) -> str | tuple[object, ...]: # Same as the v3 reading: pickle through the model, when there is one. @@ -162,19 +162,19 @@ def __reduce__(self) -> str | tuple[object, ...]: return (_reading_at, (self.metadata, self.at)) return object.__reduce__(self) - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Each field in the document, as the scope read it, with its location under `at`: the dtype, followed by a struct's record types, then the compressor, then each filter at its index.""" + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Each declaration in the document, as the scope read it, with its location under `at`: the dtype, followed by a struct's record types, then the compressor, then each filter at its index.""" if self.dtype is not UNSET: - yield from fields_of(self.dtype, (*self.at, "dtype")) + yield from declarations_of(self.dtype, (*self.at, "dtype")) if self.compressor is not UNSET and self.compressor is not None: - yield from fields_of(self.compressor, (*self.at, "compressor")) + yield from declarations_of(self.compressor, (*self.at, "compressor")) if self.filters is not UNSET and self.filters is not None: for index, entry in enumerate(self.filters): - yield from fields_of(entry, (*self.at, "filters", index)) + yield from declarations_of(entry, (*self.at, "filters", index)) def _reading_at(model: ZarrV2ArrayMetadata, at: Loc) -> ZarrV2ArrayMetadataReading: - """The reading that `model` holds, with its fields at `at`. Unpickling a reading that held a model rebuilds it with this function.""" + """The reading that `model` holds, with its declarations at `at`. Unpickling a reading that held a model rebuilds it with this function.""" return dataclasses.replace(cast("ZarrV2ArrayMetadataReading", reading_of(model)), at=at) @@ -226,7 +226,7 @@ def read_array_v2( "shape": shape, "chunks": chunks, } - dtype: ResolvedField[ZarrV2DataTypeDefinition[Any]] | UNSET = UNSET + dtype: ResolvedDeclaration[ZarrV2DataTypeDefinition[Any]] | UNSET = UNSET if "dtype" in doc: # A typestr is read by its family's definition, and a list of field # records as a struct. @@ -242,7 +242,7 @@ def read_array_v2( refined["order"] = order if len(found) != 0 or order not in ("C", "F"): problems.append(outside_of(("order",), doc["order"], ("C", "F"))) - compressor: ResolvedField[ZarrV2CodecDefinition[Any]] | UNSET | None = UNSET + compressor: ResolvedDeclaration[ZarrV2CodecDefinition[Any]] | UNSET | None = UNSET if "compressor" in doc: compressor = None refined["compressor"] = None @@ -257,7 +257,7 @@ def read_array_v2( ("compressor",), ) problems.extend(found) - filters: tuple[ResolvedField[ZarrV2CodecDefinition[Any]], ...] | UNSET | None = UNSET + filters: tuple[ResolvedDeclaration[ZarrV2CodecDefinition[Any]], ...] | UNSET | None = UNSET if "filters" in doc: entries = doc["filters"] if entries is None: @@ -276,7 +276,7 @@ def read_array_v2( else: # "A list of JSON objects providing codec configurations, or # null" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79): an empty list is a list. - read: list[ResolvedField[ZarrV2CodecDefinition[Any]]] = [] + read: list[ResolvedDeclaration[ZarrV2CodecDefinition[Any]]] = [] items: list[JSONValue] = [] for index, item in enumerate(entries): refined_item, found = refine_json(item, ("filters", index)) @@ -391,7 +391,7 @@ def node_document_v2( def read_array_node_v2( files: Mapping[str, object], context: ZarrV2Context ) -> ZarrV2ArrayMetadataReading: - """Read the v2 array node whose files are `files`, which hold its `.zarray`, as `read_node_metadata_v2` reads it: each problem and each field located in the file that holds it.""" + """Read the v2 array node whose files are `files`, which hold its `.zarray`, as `read_node_metadata_v2` reads it: each problem and each declaration located in the file that holds it.""" document, problems = node_document_v2( files, ZARR_V2_ARRAY_METADATA_STORE_KEY, ARRAY_FILE_ATTRIBUTES ) @@ -437,17 +437,17 @@ def in_files( def read_array_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2ArrayMetadataReading: - """Read `value`, a v2 array document, in `context` (`CORE_V2` when none is given), whatever it holds. + """Read `value`, a v2 array document, in `context` (`CORE_V2_CONTEXT` when none is given), whatever it holds. The result holds everything the read finds: the dtype, the compressor - and each filter as the scope read them (an `AcceptedField` read by - the definition that claims the typestr or id, an `UnclaimedField`, or - a `RefusedField`), every problem, and, when there are no problems, + and each filter as the scope read them (an `AcceptedDeclaration` read by + the definition that claims the typestr or id, an `UnclaimedDeclaration`, or + a `RefusedDeclaration`), every problem, and, when there are no problems, the document's model. A dtype or codec that the scope refuses is a problem. One that the scope does not claim is not a problem. `fill_value` is checked against the dtype the scope read. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) reading, members = read_array_v2(value, context) if members is None: return reading @@ -467,15 +467,34 @@ class ZarrV2ArrayMetadataUpdate(TypedDict, total=False, closed=True): """ shape: tuple[int, ...] + """The `shape` member: the length of each dimension.""" dtype: ZarrV2DataTypeMetadata + """The `dtype` member: a NumPy typestr, or a list of field records for a structured data type.""" chunks: tuple[int, ...] + """The `chunks` member: the length of each chunk along each dimension, one for each dimension of `shape`.""" fill_value: JSONValue + """The `fill_value` member: the value of an element in a chunk that was never written, or `None` for none.""" order: ZarrV2ArrayOrder + """The `order` member: the layout of elements within each chunk, `"C"` or `"F"`.""" compressor: ZarrV2CodecMetadata | None + """The `compressor` member: the configuration of the codec that compresses each chunk, or `None` for none.""" filters: tuple[ZarrV2CodecMetadata, ...] | None + """The `filters` member: the configuration of each codec applied to each chunk before the compressor, in order, or `None` for none.""" dimension_separator: ZarrV2ArrayDimensionSeparator | UNSET + """The `dimension_separator` member: the separator between a chunk's grid coordinates in its key, `"."` or `"/"`. + + `UNSET` removes it, which means `"."`. + """ attributes: Mapping[str, JSONValue] | UNSET + """The `attributes` member: user attributes, which a store holds in `.zattrs`. + + `UNSET` removes it, so the array has no `.zattrs`. + """ extra_fields: Mapping[str, JSONValue | UNSET] + """The members the spec does not define, by key: each one given is set, or removed when its value is `UNSET`, and any other is kept. + + A key the spec defines, or that the update takes by name, raises `TypeError`. + """ class ZarrV2ArrayMetadata(Keyed): @@ -489,9 +508,9 @@ class ZarrV2ArrayMetadata(Keyed): the `dimension_separator` property returns `"."`, and the model compares equal to one whose document writes `"."`, but `to_json` writes the document without the key. `dtype`, `compressor` and - each filter are views of the reading: an `AcceptedField` read by the + each filter are views of the reading: an `AcceptedDeclaration` read by the definition in scope that claims the typestr or id, or an - `UnclaimedField`. `attributes` is empty when no `.zattrs` exists, as + `UnclaimedDeclaration`. `attributes` is empty when no `.zattrs` exists, as in a v3 model. The document still tells the two apart: `to_json` and `to_key_value` write `attributes` and `.zattrs` only when a `.zattrs` exists, even an empty one. A member the spec does not define is kept @@ -507,10 +526,15 @@ class ZarrV2ArrayMetadata(Keyed): __slots__ = ("_context", "_document", "_members", "_reading", "_shown") zarr_format: Final[Literal[2]] = 2 # noqa: PYI064 - a bare Final does not narrow a union of models + """The Zarr format version, `2`. A class constant: comparing it narrows a union of models.""" node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above + """The kind of node, `"array"`. A class constant: comparing it narrows a union of models. + + A v2 document does not write this member: its `.zarray` key says the node is an array. + """ def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) reading, members = read_array_v2(document, context) if members is None: raise MetadataValidationError(reading.problems) @@ -589,7 +613,7 @@ def __repr__(self) -> str: def _key_of(self) -> tuple[object, ...]: """The key `==` and `hash` compare for a v2 array model: what its document means. - The key holds each field's `field_key`, and the fill value as JSON + The key holds each declaration's `declaration_key`, and the fill value as JSON text, in its canonical spelling when a definition in scope read the dtype. Every other member is included as it is, with JSON members as text. No `.zattrs` counts as empty attributes, since both mean @@ -602,17 +626,19 @@ def _key_of(self) -> tuple[object, ...]: members.order, members.dimension_separator, self._fill_value_key(), - field_key(self.dtype), - None if self.compressor is None else field_key(self.compressor), - None if self.filters is None else tuple(field_key(entry) for entry in self.filters), + declaration_key(self.dtype), + None if self.compressor is None else declaration_key(self.compressor), + None + if self.filters is None + else tuple(declaration_key(entry) for entry in self.filters), json_text({} if members.attributes is UNSET else members.attributes), json_text(members.extra_fields), ) def _plain_key( - self, dtype: AcceptedField[ZarrV2DataTypeDefinition[Any]] | UnclaimedField + self, dtype: AcceptedDeclaration[ZarrV2DataTypeDefinition[Any]] | UnclaimedDeclaration ) -> tuple[object, ...]: - """The members other than fields that `refines` compares. The fill value is in the spelling of `dtype`, the dtype of the more informed side.""" + """The members that hold no declaration, which `refines` compares. The fill value is in the spelling of `dtype`, the dtype of the more informed side.""" members = self._members return ( members.shape, @@ -627,7 +653,7 @@ def _plain_key( def _fill_value_key(self) -> str: """The fill value as `==` compares it, as JSON text: in its canonical spelling when a definition in scope read the dtype, and as written otherwise.""" fill_value = self._members.fill_value - if isinstance(self.dtype, AcceptedField): + if isinstance(self.dtype, AcceptedDeclaration): return json_text(spelled_canonically(self.dtype, fill_value)) return json_text(fill_value) @@ -662,15 +688,15 @@ def dimension_separator(self) -> ZarrV2ArrayDimensionSeparator: return self._members.dimension_separator @property - def unclaimed_fields(self) -> dict[Loc, UnclaimedField]: - """Every field nothing in scope claims, its dtype or a codec, located as `reading.fields()` locates it: at its place in the document, where a struct dtype's field types are at their record and position, as their problems are. + def unclaimed_declarations(self) -> dict[Loc, UnclaimedDeclaration]: + """Every declaration nothing in scope claims, its dtype or a codec, located as `reading.declarations()` locates it: at its place in the document, where a struct dtype's field types are at their record and position, as their problems are. This answers whether a reader can open the array: a reader cannot decode data with a dtype or a codec it does not understand, and the scope is what the reader understands. The model only reports these - fields; refusing to open is up to the reader. + declarations; refusing to open is up to the reader. """ - return unclaimed_fields_of(self._reading.fields(), ()) + return unclaimed_declarations_of(self._reading.declarations(), ()) @property def attributes(self) -> Mapping[str, JSONValue]: @@ -683,12 +709,14 @@ def extra_fields(self) -> Mapping[str, JSONValue]: return self._shown[2] @property - def dtype(self) -> AcceptedField[ZarrV2DataTypeDefinition[Any]] | UnclaimedField: + def dtype(self) -> AcceptedDeclaration[ZarrV2DataTypeDefinition[Any]] | UnclaimedDeclaration: """The dtype as the scope read it: read by its family's definition, or unclaimed.""" return held(self._reading.dtype) @property - def compressor(self) -> AcceptedField[ZarrV2CodecDefinition[Any]] | UnclaimedField | None: + def compressor( + self, + ) -> AcceptedDeclaration[ZarrV2CodecDefinition[Any]] | UnclaimedDeclaration | None: """The compressor as the scope read it; None when written as `null`.""" compressor = self._reading.compressor return None if compressor is None else held(compressor) @@ -696,7 +724,7 @@ def compressor(self) -> AcceptedField[ZarrV2CodecDefinition[Any]] | UnclaimedFie @property def filters( self, - ) -> tuple[AcceptedField[ZarrV2CodecDefinition[Any]] | UnclaimedField, ...] | None: + ) -> tuple[AcceptedDeclaration[ZarrV2CodecDefinition[Any]] | UnclaimedDeclaration, ...] | None: """The filters, each as the scope read it; None when written as `null`.""" filters = self._reading.filters if filters is None: @@ -731,8 +759,8 @@ def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetad in `context`. The existing reading is reused when `context` reads every claim identically. """ - context = scoped(context, CORE_V2) - if disagreements(context, claims_of(self._reading.fields())).agrees: + context = scoped(context, CORE_V2_CONTEXT) + if disagreements(context, claims_of(self._reading.declarations())).agrees: return self._of(self._document, context, self._reading, self._members) return type(self)(self._document, context=context) @@ -745,14 +773,16 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadat in the document. Raises `MetadataValidationError` when a definition that `context` claims rejects what was written. """ - context = scoped(context, CORE_V2) - found = disagreements(context, claims_of(self._reading.fields())) + context = scoped(context, CORE_V2_CONTEXT) + found = disagreements(context, claims_of(self._reading.declarations())) if len(found.conflicts) != 0: - raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) + raise ScopeConflictError( + located_conflicts(self._reading.declarations(), found.conflicts) + ) return self.with_context(context) def refines(self, other: ZarrV2ArrayMetadata) -> bool: - """Whether this model holds everything `other` holds. Each field must refine its counterpart. A `null` compressor or `null` filters refine only a `null`. Every other member must be the same, with the fill value compared in the spelling of the more informed dtype. If that dtype rejects the other model's fill value, this is not a refinement.""" + """Whether this model holds everything `other` holds. Each declaration must refine its counterpart. A `null` compressor or `null` filters refine only a `null`. Every other member must be the same, with the fill value compared in the spelling of the more informed dtype. If that dtype rejects the other model's fill value, this is not a refinement.""" if type(other) is not type(self): return False if (self.compressor is None) != (other.compressor is None): @@ -766,7 +796,7 @@ def refines(self, other: ZarrV2ArrayMetadata) -> bool: pairs = [(self.dtype, other.dtype), *zip(mine, theirs, strict=True)] if self.compressor is not None and other.compressor is not None: pairs.append((self.compressor, other.compressor)) - if not all(refines_field(one, another) for one, another in pairs): + if not all(refines_declaration(one, another) for one, another in pairs): return False if len(fill_value_problems(self.dtype, other._members.fill_value)) != 0: return False @@ -807,7 +837,9 @@ def create_default( if lengths is not None: given["chunks"] = lengths if "dtype" in given and "fill_value" not in given: - dtype, _ = resolve(given["dtype"], ZarrV2DataTypeDefinition, scoped(context, CORE_V2)) + dtype, _ = resolve( + given["dtype"], ZarrV2DataTypeDefinition, scoped(context, CORE_V2_CONTEXT) + ) if len(fill_value_problems(dtype, 0)) != 0: given["fill_value"] = None merged = {key: value for key, value in {**document, **given}.items() if value is not UNSET} @@ -836,7 +868,7 @@ def from_key_value( valid, with every problem the read finds. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) files, problems = decoded_node_files_v2( mapping, (ZARR_V2_ARRAY_METADATA_STORE_KEY, ZARR_V2_ATTRIBUTES_STORE_KEY) ) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 53d12254bc..061819fb3d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -30,20 +30,20 @@ from zarr_metadata._typed_json import typeddict_keys from zarr_metadata.definition._context import disagreements, scoped from zarr_metadata.definition._definition import ( - AcceptedField, + AcceptedDeclaration, Definition, - ResolvedField, - UnclaimedField, - field_key, - field_kind, - fields_of, + ResolvedDeclaration, + UnclaimedDeclaration, + declaration_key, + declaration_kind, + declarations_of, fill_value_problems, held, resolve_refined, spelled_canonically, ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of -from zarr_metadata.definition._scope import refines as refines_field +from zarr_metadata.definition._scope import refines as refines_declaration from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, @@ -56,13 +56,13 @@ member_changes, missing_keys, reading_of, - unclaimed_fields_of, + unclaimed_declarations_of, ) -from zarr_metadata.v3._common import ZarrV3ExtensionField, ZarrV3MetadataFieldJSON +from zarr_metadata.v3._common import ZarrV3DeclarationJSON, ZarrV3ExtensionField # A runtime import: `get_type_hints` resolves the public signatures. from zarr_metadata.v3._context import ZarrV3Context # noqa: TC001 -from zarr_metadata.v3._core import CORE_AND_EXTENSIONS +from zarr_metadata.v3._core import CORE_AND_EXTENSIONS_CONTEXT from zarr_metadata.v3._definition import ( ArrayExtraFieldDefinition, Chunk, @@ -119,20 +119,21 @@ def members_past_the_levels( _EXTENSION_POINTS_V3: Final[tuple[tuple[str, type[Definition[Any]]], ...]] = tuple( (key, kind) for key, (annotation, _) in _MEMBERS_V3.items() - if (kind := field_kind(annotation)) is not None + if (kind := declaration_kind(annotation)) is not None ) -"""The single extension points of a v3 array document, and the kind each is read as: each member that the TypedDict annotates with a field alias.""" +"""The single extension points of a v3 array document, and the kind each is read as: each member that the TypedDict annotates with a declaration alias.""" _EXTENSION_LISTS_V3: Final[tuple[tuple[str, type[Definition[Any]]], ...]] = tuple( (key, kind) for key, (annotation, _) in _MEMBERS_V3.items() - if get_origin(annotation) is tuple and (kind := field_kind(get_args(annotation)[0])) is not None + if get_origin(annotation) is tuple + and (kind := declaration_kind(get_args(annotation)[0])) is not None ) -"""The lists of extension points of a v3 array document, and the kind each entry is read as: each member that the TypedDict annotates as a tuple of a field alias.""" +"""The lists of extension points of a v3 array document, and the kind each entry is read as: each member that the TypedDict annotates as a tuple of a declaration alias.""" -def no_extra_fields() -> Mapping[str, ResolvedField[Any]]: +def no_extra_fields() -> Mapping[str, ResolvedDeclaration[Any]]: """The extra fields of a reading of a document that has none.""" return MappingProxyType({}) @@ -141,30 +142,30 @@ def no_extra_fields() -> Mapping[str, ResolvedField[Any]]: class ZarrV3ArrayMetadataReading: """A v3 array document as a scope read it, whatever it holds: each extension point, its codecs as a pipeline, every problem, and the model when there are no problems. - A field the document does not have is `UNSET`. A list of fields that + A declaration the document does not have is `UNSET`. A list of declarations that the document does not have as a list is empty. """ - data_type: ResolvedField[DataTypeDefinition[Any]] | UNSET = UNSET + data_type: ResolvedDeclaration[DataTypeDefinition[Any]] | UNSET = UNSET """The data type, as the scope read it.""" - chunk_grid: ResolvedField[ChunkGridDefinition[Any]] | UNSET = UNSET + chunk_grid: ResolvedDeclaration[ChunkGridDefinition[Any]] | UNSET = UNSET """The chunk grid, as the scope read it.""" - chunk_key_encoding: ResolvedField[ChunkKeyEncodingDefinition[Any]] | UNSET = UNSET + chunk_key_encoding: ResolvedDeclaration[ChunkKeyEncodingDefinition[Any]] | UNSET = UNSET """The chunk key encoding, as the scope read it.""" chunk: Chunk = dataclasses.field(default_factory=Chunk) """The chunks the first codec receives: the length of the grid's chunks along each axis of the shape, and the data type.""" pipeline: tuple[Stage, ...] = () """The codecs, read as a pipeline: each as the scope read it, with the chunk it receives.""" - storage_transformers: tuple[ResolvedField[StorageTransformerDefinition[Any]], ...] = () + storage_transformers: tuple[ResolvedDeclaration[StorageTransformerDefinition[Any]], ...] = () """The storage transformers, each as the scope read it.""" - extra_fields: Mapping[str, ResolvedField[ArrayExtraFieldDefinition[Any]]] = dataclasses.field( - default_factory=no_extra_fields + extra_fields: Mapping[str, ResolvedDeclaration[ArrayExtraFieldDefinition[Any]]] = ( + dataclasses.field(default_factory=no_extra_fields) ) - """Each member the spec does not define, as the scope read it, by key: an `UnclaimedField` when no definition in scope claims the key.""" + """Each member the spec does not define, as the scope read it, by key: an `UnclaimedDeclaration` when no definition in scope claims the key.""" problems: tuple[ValidationProblem, ...] = () """Every reason the document is not valid.""" metadata: ZarrV3ArrayMetadata | None = None - """The document's model, which holds these fields, when there is no problem; None otherwise.""" + """The document's model, which holds these declarations, when there is no problem; None otherwise.""" def __reduce__(self) -> str | tuple[object, ...]: # A reading that holds a model pickles and copies the same way the @@ -177,29 +178,29 @@ def __reduce__(self) -> str | tuple[object, ...]: values = {item.name: getattr(self, item.name) for item in dataclasses.fields(self)} return (_array_reading, ({**values, "extra_fields": dict(self.extra_fields)},)) - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Each field in the document, as the scope read it, with its location in the document. + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Each declaration in the document, as the scope read it, with its location in the document. The order is: the extension points, then each codec and storage transformer at its index, then each extra field at its key. Each is - followed by the fields nested in it, as `fields_of` returns them: a + followed by the declarations nested in it, as `declarations_of` returns them: a shard's codecs, or a struct's field types. `with_problems` returns - each field with its problems. + each declaration with its problems. """ - own: tuple[tuple[str, ResolvedField[Any] | UNSET], ...] = ( + own: tuple[tuple[str, ResolvedDeclaration[Any] | UNSET], ...] = ( ("data_type", self.data_type), ("chunk_grid", self.chunk_grid), ("chunk_key_encoding", self.chunk_key_encoding), ) - for key, field in own: - if field is not UNSET: - yield from fields_of(field, (key,)) + for key, declaration in own: + if declaration is not UNSET: + yield from declarations_of(declaration, (key,)) for index, stage in enumerate(self.pipeline): - yield from fields_of(stage.codec, ("codecs", index)) + yield from declarations_of(stage.codec, ("codecs", index)) for index, transformer in enumerate(self.storage_transformers): - yield from fields_of(transformer, ("storage_transformers", index)) + yield from declarations_of(transformer, ("storage_transformers", index)) for key, extra in self.extra_fields.items(): - yield from fields_of(extra, (key,)) + yield from declarations_of(extra, (key,)) def _array_reading(values: dict[str, Any]) -> ZarrV3ArrayMetadataReading: @@ -210,7 +211,7 @@ def _array_reading(values: dict[str, Any]) -> ZarrV3ArrayMetadataReading: @dataclass(frozen=True, slots=True) class ArrayMembersV3: - """The members other than fields of a v3 array document that a read found no problems with, refined as the model holds them.""" + """The members that hold no declaration in a v3 array document that a read found no problems with, refined as the model holds them.""" shape: tuple[int, ...] fill_value: JSONValue @@ -227,7 +228,7 @@ def read_array_v3( """Read `value`, a v3 array document, in `context`, without building its model. Return the reading and the other members, refined; the members are None when the document has a problem. `read_array_metadata_v3` builds the model from these two results. A - field object in the document (an `AcceptedField` built by hand) is + declaration object in the document (an `AcceptedDeclaration` built by hand) is not JSON, and is refused as such. `at` is the location of this document within the input document; a document in consolidated metadata is three levels below its group's document. Nesting depth @@ -267,7 +268,7 @@ def read_array_v3( "node_type": "array", "shape": shape, } - read: dict[str, ResolvedField[Any]] = {} + read: dict[str, ResolvedDeclaration[Any]] = {} for key, kind in _EXTENSION_POINTS_V3: if key in doc: refined[key], found = refine_json(doc[key], (*at, key)) @@ -292,7 +293,7 @@ def read_array_v3( if "chunk_grid" in read and shape is not None: lengths, found = chunk_grid_lengths(read["chunk_grid"], shape, ("chunk_grid",)) problems.extend(found) - listed: dict[str, list[ResolvedField[Any]]] = {} + listed: dict[str, list[ResolvedDeclaration[Any]]] = {} for key, kind in _EXTENSION_LISTS_V3: if key in doc: entries = doc[key] @@ -327,7 +328,7 @@ def read_array_v3( if "dimension_names" in doc: # Simple typed sequences (dimension_names, shape, chunks) report a single # field-level loc, not per-bad-item locs; per-index locs are reserved for - # the metadata-field lists (codecs, storage_transformers). + # the declaration lists (codecs, storage_transformers). names = doc["dimension_names"] if not is_array(names): problems.append( @@ -381,7 +382,9 @@ def read_extra_fields_v3( kind: type[ExtraFieldDefinition[Any]], context: ZarrV3Context, at: Loc, -) -> tuple[dict[str, JSONValue], dict[str, ResolvedField[Any]], tuple[ValidationProblem, ...]]: +) -> tuple[ + dict[str, JSONValue], dict[str, ResolvedDeclaration[Any]], tuple[ValidationProblem, ...] +]: """Read each member of `doc` (a document at `at`) that is not one of the `spec_members` of `kind` as an extra field of `kind`. Returns three values: each member that is JSON, refined; each member @@ -389,11 +392,11 @@ def read_extra_fields_v3( member that a definition in `context` claims is read by that definition, and must be an object whose numbers are finite, unless the definition reads user data. Any other member is not read: it is an - `UnclaimedField`, and, like attributes, need only be JSON as Python's + `UnclaimedDeclaration`, and, like attributes, need only be JSON as Python's `json` module reads it. """ values: dict[str, JSONValue] = {} - fields: dict[str, ResolvedField[Any]] = {} + declarations: dict[str, ResolvedDeclaration[Any]] = {} problems: list[ValidationProblem] = [] for key, value in doc.items(): if not isinstance(key, str): @@ -409,15 +412,17 @@ def read_extra_fields_v3( if isinstance(refined, dict): values[key] = refined[key] if definition is None: - fields[key] = UnclaimedField(json=refined, name=key, read_as=kind) + declarations[key] = UnclaimedDeclaration(json=refined, name=key, read_as=kind) continue - fields[key], found = resolve_refined(member, refined, found, kind, context, (*at, key)) + declarations[key], found = resolve_refined( + member, refined, found, kind, context, (*at, key) + ) problems.extend(within(found, at)) - return values, fields, tuple(problems) + return values, declarations, tuple(problems) def must_understand_subset( - fields: Mapping[str, AcceptedField[Any] | UnclaimedField], + declarations: Mapping[str, AcceptedDeclaration[Any] | UnclaimedDeclaration], values: Mapping[str, ZarrV3ExtensionField], ) -> dict[str, ZarrV3ExtensionField]: """The extra fields, by key and as written, that no definition in scope reads and that a reader is obligated to understand. @@ -433,8 +438,8 @@ def must_understand_subset( """ return { name: values[name] - for name, field in fields.items() - if isinstance(field, UnclaimedField) and not _waived(values[name]) + for name, declaration in declarations.items() + if isinstance(declaration, UnclaimedDeclaration) and not _waived(values[name]) } @@ -459,15 +464,37 @@ class ZarrV3ArrayMetadataUpdate(TypedDict, total=False, closed=True): """ shape: tuple[int, ...] - data_type: ZarrV3MetadataFieldJSON - chunk_grid: ZarrV3MetadataFieldJSON - chunk_key_encoding: ZarrV3MetadataFieldJSON + """The `shape` member: the length of each dimension.""" + data_type: ZarrV3DeclarationJSON + """The `data_type` member: the data type of the array's elements.""" + chunk_grid: ZarrV3DeclarationJSON + """The `chunk_grid` member: how the array is divided into chunks.""" + chunk_key_encoding: ZarrV3DeclarationJSON + """The `chunk_key_encoding` member: how a chunk's grid coordinates become its store key.""" fill_value: JSONValue - codecs: tuple[ZarrV3MetadataFieldJSON, ...] + """The `fill_value` member: the value of an element in a chunk that was never written.""" + codecs: tuple[ZarrV3DeclarationJSON, ...] + """The `codecs` member: the codecs that encode each chunk, in order.""" attributes: Mapping[str, JSONValue] | UNSET - storage_transformers: tuple[ZarrV3MetadataFieldJSON, ...] | UNSET + """The `attributes` member: user attributes. + + `UNSET` removes it, so the array has no attributes. + """ + storage_transformers: tuple[ZarrV3DeclarationJSON, ...] | UNSET + """The `storage_transformers` member: the storage transformers, in order. + + `UNSET` removes it, which means none. + """ dimension_names: tuple[str | None, ...] | UNSET + """The `dimension_names` member: a name for each dimension, `None` for an unnamed one. + + `UNSET` removes it, which means every dimension is unnamed. + """ extra_fields: Mapping[str, ZarrV3ExtensionField | UNSET] + """The members the spec does not define, by key: each one given is set, or removed when its value is `UNSET`, and any other is kept. + + A key the spec defines, or that the update takes by name, raises `TypeError`. + """ class ZarrV3ArrayMetadata(Keyed): @@ -475,11 +502,11 @@ class ZarrV3ArrayMetadata(Keyed): The model is this pair. `to_json` returns the document as written, refined (arrays as tuples, string keys), and `context` is the scope. - Every typed member is a view of the reading of the document in the + Every typed attribute is a view of the reading of the document in the scope. `data_type`, `chunk_grid`, `chunk_key_encoding`, and each codec - and storage transformer are fields as the scope read them: an - `AcceptedField` (read by the definition that claims its name) or an - `UnclaimedField`, and so is each member in `extra_fields`, which the + and storage transformer are declarations as the scope read them: an + `AcceptedDeclaration` (read by the definition that claims its name) or an + `UnclaimedDeclaration`, and so is each member in `extra_fields`, which the spec does not define. `shape`, `fill_value`, `dimension_names` and `attributes` are the values the read refined. A model is only built by reading: the constructor reads `document` in @@ -496,10 +523,12 @@ class ZarrV3ArrayMetadata(Keyed): __slots__ = ("_context", "_document", "_members", "_reading", "_shown") zarr_format: Final[Literal[3]] = 3 # noqa: PYI064 - a bare Final does not narrow a union of models + """The Zarr format version, `3`. A class constant: comparing it narrows a union of models.""" node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above + """The kind of node, `"array"`. A class constant: comparing it narrows a union of models.""" def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_array_v3(document, context) if members is None: raise MetadataValidationError(reading.problems) @@ -548,7 +577,7 @@ def context(self) -> ZarrV3Context: @property def reading(self) -> ZarrV3ArrayMetadataReading: - """The document as the scope read it: each field, the pipeline, and the chunk each codec receives.""" + """The document as the scope read it: each declaration, the pipeline, and the chunk each codec receives.""" return self._reading def to_json(self) -> ZarrV3ArrayMetadataJSON: @@ -574,9 +603,9 @@ def _key_of(self) -> tuple[object, ...]: return array_key_of(self._reading, self._members) def _plain_key( - self, data_type: AcceptedField[DataTypeDefinition[Any]] | UnclaimedField + self, data_type: AcceptedDeclaration[DataTypeDefinition[Any]] | UnclaimedDeclaration ) -> tuple[object, ...]: - """The members other than fields that `refines` compares. The fill value is in the spelling of `data_type`, the data type of the more informed side.""" + """The members that hold no declaration, which `refines` compares. The fill value is in the spelling of `data_type`, the data type of the more informed side.""" members = self._members return ( members.shape, @@ -615,21 +644,23 @@ def attributes(self) -> Mapping[str, JSONValue]: @property def extra_fields( self, - ) -> Mapping[str, AcceptedField[ArrayExtraFieldDefinition[Any]] | UnclaimedField]: - """Each member the spec does not define, by key, as the scope read it: an `AcceptedField` when a definition in scope claims the key, and an `UnclaimedField` otherwise.""" + ) -> Mapping[str, AcceptedDeclaration[ArrayExtraFieldDefinition[Any]] | UnclaimedDeclaration]: + """Each member the spec does not define, by key, as the scope read it: an `AcceptedDeclaration` when a definition in scope claims the key, and an `UnclaimedDeclaration` otherwise.""" return self._shown[3] @property - def unclaimed_fields(self) -> dict[Loc, UnclaimedField]: - """Every field nothing in scope claims, at any depth, by its location in the document, except an extra member that says `"must_understand": false` and every field inside it, claimed or not, since a reader may ignore such a member whole. + def unclaimed_declarations(self) -> dict[Loc, UnclaimedDeclaration]: + """Every declaration nothing in scope claims, at any depth, by its location in the document, except an extra field that says `"must_understand": false` and every declaration inside it, claimed or not, since a reader may ignore such a member whole. This answers whether a reader can open the array: the spec requires - a reader to refuse a node with a field it does not understand, and + a reader to refuse a node with a declaration it does not understand, and the scope is what the reader understands. A codec inside a shard is located inside the shard's configuration. The model only reports - these fields; refusing to open is up to the reader. + these declarations; refusing to open is up to the reader. """ - return unclaimed_fields_of(self._reading.fields(), waived_members(self._shown[2])) + return unclaimed_declarations_of( + self._reading.declarations(), waived_members(self._shown[2]) + ) @property def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: @@ -644,29 +675,33 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: return must_understand_subset(self.extra_fields, self._shown[2]) @property - def data_type(self) -> AcceptedField[DataTypeDefinition[Any]] | UnclaimedField: + def data_type(self) -> AcceptedDeclaration[DataTypeDefinition[Any]] | UnclaimedDeclaration: """The data type, as the scope read it.""" return held(self._reading.data_type) @property - def chunk_grid(self) -> AcceptedField[ChunkGridDefinition[Any]] | UnclaimedField: + def chunk_grid(self) -> AcceptedDeclaration[ChunkGridDefinition[Any]] | UnclaimedDeclaration: """The chunk grid, as the scope read it.""" return held(self._reading.chunk_grid) @property - def chunk_key_encoding(self) -> AcceptedField[ChunkKeyEncodingDefinition[Any]] | UnclaimedField: + def chunk_key_encoding( + self, + ) -> AcceptedDeclaration[ChunkKeyEncodingDefinition[Any]] | UnclaimedDeclaration: """The chunk key encoding, as the scope read it.""" return held(self._reading.chunk_key_encoding) @property - def codecs(self) -> tuple[AcceptedField[CodecDefinition[Any]] | UnclaimedField, ...]: + def codecs( + self, + ) -> tuple[AcceptedDeclaration[CodecDefinition[Any]] | UnclaimedDeclaration, ...]: """The codecs, each as the scope read it, in pipeline order.""" return tuple(held(stage.codec) for stage in self._reading.pipeline) @property def storage_transformers( self, - ) -> tuple[AcceptedField[StorageTransformerDefinition[Any]] | UnclaimedField, ...]: + ) -> tuple[AcceptedDeclaration[StorageTransformerDefinition[Any]] | UnclaimedDeclaration, ...]: """The storage transformers, each as the scope read it.""" return tuple(held(entry) for entry in self._reading.storage_transformers) @@ -696,8 +731,8 @@ def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetad in `context`. The existing reading is reused when `context` reads every claim identically. """ - context = scoped(context, CORE_AND_EXTENSIONS) - if disagreements(context, claims_of(self._reading.fields())).agrees: + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) + if disagreements(context, claims_of(self._reading.declarations())).agrees: return self._of(self._document, context, self._reading, self._members) return type(self)(self._document, context=context) @@ -713,14 +748,16 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadat gain can expose a problem. `with_context` reads the document in any scope. """ - context = scoped(context, CORE_AND_EXTENSIONS) - found = disagreements(context, claims_of(self._reading.fields())) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) + found = disagreements(context, claims_of(self._reading.declarations())) if len(found.conflicts) != 0: - raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) + raise ScopeConflictError( + located_conflicts(self._reading.declarations(), found.conflicts) + ) return self.with_context(context) def refines(self, other: ZarrV3ArrayMetadata) -> bool: - """Whether this model holds everything `other` holds. Each field must refine its counterpart, in the order `refines` defines for fields (including the fields nested in each field). Every other member must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement.""" + """Whether this model holds everything `other` holds. Each declaration must refine its counterpart, in the order `refines` defines for declarations (including the declarations nested in each declaration). Every other member must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement.""" if type(other) is not type(self): return False if len(self.codecs) != len(other.codecs) or len(self.storage_transformers) != len( @@ -737,7 +774,7 @@ def refines(self, other: ZarrV3ArrayMetadata) -> bool: *zip(self.storage_transformers, other.storage_transformers, strict=True), *((self.extra_fields[key], other.extra_fields[key]) for key in self.extra_fields), ) - if not all(refines_field(mine, theirs) for mine, theirs in pairs): + if not all(refines_declaration(mine, theirs) for mine, theirs in pairs): return False if len(fill_value_problems(self.data_type, other._members.fill_value)) != 0: return False @@ -811,7 +848,7 @@ def from_key_value( are not JSON, or the document is not valid. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) return cls(load_store_json(mapping, ZARR_V3_ARRAY_METADATA_STORE_KEY), context=context) @@ -820,7 +857,7 @@ def array_key_of( ) -> tuple[object, ...]: """A key for what a v3 array document means, from its `reading` and refined `members`. `==` and `hash` compare this key for the model, and two listings of one node are compared by it. - The key holds each field's `field_key`, and the fill value as JSON + The key holds each declaration's `declaration_key`, and the fill value as JSON text: in its canonical spelling when a definition in scope read the data type, and as written otherwise. Every other member is included as it is, with JSON members as text. @@ -828,27 +865,27 @@ def array_key_of( data_type = held(reading.data_type) fill_value = ( spelled_canonically(data_type, members.fill_value) - if isinstance(data_type, AcceptedField) + if isinstance(data_type, AcceptedDeclaration) else members.fill_value ) return ( members.shape, json_text(fill_value), - field_key(data_type), - field_key(held(reading.chunk_grid)), - tuple(field_key(held(stage.codec)) for stage in reading.pipeline), - field_key(held(reading.chunk_key_encoding)), + declaration_key(data_type), + declaration_key(held(reading.chunk_grid)), + tuple(declaration_key(held(stage.codec)) for stage in reading.pipeline), + declaration_key(held(reading.chunk_key_encoding)), members.dimension_names, json_text(members.attributes), - tuple(field_key(held(entry)) for entry in reading.storage_transformers), + tuple(declaration_key(held(entry)) for entry in reading.storage_transformers), extra_fields_key(reading.extra_fields), ) -def extra_fields_key(extra_fields: Mapping[str, ResolvedField[Any]]) -> tuple[object, ...]: - """A key for what a document's extra fields mean: each key with its field's `field_key`, sorted by key.""" +def extra_fields_key(extra_fields: Mapping[str, ResolvedDeclaration[Any]]) -> tuple[object, ...]: + """A key for what a document's extra fields mean: each key with its declaration's `declaration_key`, sorted by key.""" return tuple( - (key, field_key(extra)) + (key, declaration_key(extra)) for key, extra in sorted(extra_fields.items(), key=lambda item: item[0]) ) @@ -859,8 +896,8 @@ def read_array_metadata_v3( """Read `value`, a v3 array document, in `context`, whatever it holds. The result holds everything the read finds: each extension point as - `context` read it (an `AcceptedField` read by the definition that - claims its name, an `UnclaimedField`, or a `RefusedField`), the chunks + `context` read it (an `AcceptedDeclaration` read by the definition that + claims its name, an `UnclaimedDeclaration`, or a `RefusedDeclaration`), the chunks the codecs receive, each codec with the chunk it receives, every problem, and, when there are no problems, the document's model, which holds the same reading. @@ -884,11 +921,11 @@ def read_array_metadata_v3( declare `must_understand: false`; the model reports these as `must_understand_fields`. - To apply a policy over the fields (for example, allow only the core - spec's fields), iterate over the reading's `fields()`. A value that - is not an object has no fields. + To apply a policy over the declarations (for example, allow only the core + spec's declarations), iterate over the reading's `declarations()`. A value that + is not an object has no declarations. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_array_v3(value, context) if members is None: return reading diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py index eb290a6cb7..f5c7e8b405 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py @@ -63,14 +63,14 @@ from zarr_metadata.v2.array import ZARR_V2_ARRAY_METADATA_STORE_KEY from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY from zarr_metadata.v2.consolidated import ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY -from zarr_metadata.v2.definition import CORE_V2 +from zarr_metadata.v2.definition import CORE_V2_CONTEXT from zarr_metadata.v2.group import ZARR_V2_GROUP_METADATA_STORE_KEY, ZarrV2GroupMetadataJSON if TYPE_CHECKING: from collections.abc import Iterator from zarr_metadata._typed_json import Loc - from zarr_metadata.definition._definition import ResolvedField, UnclaimedField + from zarr_metadata.definition._definition import ResolvedDeclaration, UnclaimedDeclaration from zarr_metadata.v2.attributes import ZarrV2AttributesStoreKey from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataStoreKey from zarr_metadata.v2.group import ZarrV2GroupMetadataStoreKey @@ -92,7 +92,7 @@ def _group_problems_v2(value: object) -> tuple[ValidationProblem, ...]: - """Every reason `value` is not a v2 group document in the merged form: the `.zgroup` members, plus an optional `attributes` object from `.zattrs`. A group has no fields for a scope to read, so no scope is needed.""" + """Every reason `value` is not a v2 group document in the merged form: the `.zgroup` members, plus an optional `attributes` object from `.zattrs`. A group has no declarations for a scope to read, so no scope is needed.""" if not is_object(value): return not_an_object(value) doc = value @@ -115,15 +115,15 @@ def _group_problems_v2(value: object) -> tuple[ValidationProblem, ...]: @dataclass(frozen=True, slots=True) class ZarrV2GroupMetadataReading: - """A v2 group document as a scope read it: every problem, and the model when there are no problems. A group has no fields for a scope to read.""" + """A v2 group document as a scope read it: every problem, and the model when there are no problems. A group has no declarations for a scope to read.""" problems: tuple[ValidationProblem, ...] = () """Every reason the document is not valid.""" metadata: ZarrV2GroupMetadata | None = None """The document's model when there is no problem; None otherwise.""" - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Its fields as read: none, since a group has no fields.""" + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Its declarations as read: none, since a group has no declarations.""" return iter(()) def __reduce__(self) -> str | tuple[object, ...]: @@ -136,8 +136,8 @@ def __reduce__(self) -> str | tuple[object, ...]: def read_group_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2GroupMetadataReading: - """Read `value`, a v2 group document in the merged form (`.zgroup` plus `attributes` from `.zattrs`), in `context` (`CORE_V2` when none is given), whatever it holds: every problem, and the model when there are none. A group has no fields for a scope to read; `context` is taken for consistency with every other v2 reader.""" - context = scoped(context, CORE_V2) + """Read `value`, a v2 group document in the merged form (`.zgroup` plus `attributes` from `.zattrs`), in `context` (`CORE_V2_CONTEXT` when none is given), whatever it holds: every problem, and the model when there are none. A group has no declarations for a scope to read; `context` is taken for consistency with every other v2 reader.""" + context = scoped(context, CORE_V2_CONTEXT) problems = _group_problems_v2(value) if len(problems) != 0: return ZarrV2GroupMetadataReading(problems=problems) @@ -158,8 +158,8 @@ def metadata(self) -> None: """Its model: always None, since there is no node to model.""" return None - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Its fields as read: none, since no document is read.""" + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Its declarations as read: none, since no document is read.""" return iter(()) @@ -173,7 +173,7 @@ def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: def read_node_metadata_v2( files: Mapping[str, object], *, context: ZarrV2Context | None = None ) -> ZarrV2NodeMetadataReading: - """Read the v2 node described by `files`, in `context` (`CORE_V2` when none is given), as the node type its files declare. + """Read the v2 node described by `files`, in `context` (`CORE_V2_CONTEXT` when none is given), as the node type its files declare. `files` maps each of the node's files to its decoded JSON: `.zarray` or `.zgroup`, and `.zattrs` when there is one. They can come from a @@ -188,13 +188,13 @@ def read_node_metadata_v2( reading of its node type with no model, and with that problem followed by the problems of the rest of the file. Each problem is located in the file that holds it, such as `(".zarray", "order")` or - `(".zattrs", "x")`, and so is each field of an array's reading, whose + `(".zattrs", "x")`, and so is each declaration of an array's reading, whose `at` is `(".zarray",)`. The model reads its merged document, so its - own `reading` locates fields there, at `("dtype",)`. This is the + own `reading` locates declarations there, at `("dtype",)`. This is the v2 counterpart of `read_node_metadata_v3`, which reads one `zarr.json` and its `node_type`. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) is_array = ZARR_V2_ARRAY_METADATA_STORE_KEY in files is_group = ZARR_V2_GROUP_METADATA_STORE_KEY in files if is_array and is_group: @@ -234,7 +234,7 @@ def node_metadata_from_key_value_v2( mapping: Mapping[StoreKey, bytes], *, context: ZarrV2Context | None = None ) -> ZarrV2NodeMetadata: """Return the model of the v2 node whose files are in `mapping`, a store's keys and bytes: each file is decoded, then read as `read_node_metadata_v2` reads it. Raises `MetadataValidationError` with every problem, including a file whose bytes are not JSON, and `TypeError` for a scope of another format before any bytes are decoded.""" - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) files, problems = decoded_node_files_v2(mapping) if len(problems) != 0: raise MetadataValidationError(problems) @@ -248,6 +248,10 @@ class ZarrV2GroupMetadataUpdate(TypedDict, total=False): """The members `ZarrV2GroupMetadata.update` replaces: `attributes` in the form a `.zattrs` writes them, or `UNSET` for no `.zattrs`.""" attributes: Mapping[str, JSONValue] | UNSET + """The `attributes` member: user attributes, which a store holds in `.zattrs`. + + `UNSET` removes it, so the group has no `.zattrs`. + """ class ZarrV2GroupMetadata(Keyed): @@ -258,7 +262,7 @@ class ZarrV2GroupMetadata(Keyed): them) as written, refined. `attributes` is empty when no `.zattrs` exists, as in a v3 model. The document still tells the two apart: `to_json` and `to_key_value` write `attributes` and `.zattrs` only when - a `.zattrs` exists, even an empty one. A group has no fields + a `.zattrs` exists, even an empty one. A group has no declarations for a scope to read, so the scope is held only for consistency with the other models: `update` reads new attributes in it, and no other scope conflicts with the reading. A model is only built by reading: @@ -275,10 +279,15 @@ class ZarrV2GroupMetadata(Keyed): _shown: Mapping[str, JSONValue] zarr_format: Final[Literal[2]] = 2 # noqa: PYI064 - a bare Final does not narrow a union of models + """The Zarr format version, `2`. A class constant: comparing it narrows a union of models.""" node_type: Final[Literal["group"]] = "group" # noqa: PYI064 - as above + """The kind of node, `"group"`. A class constant: comparing it narrows a union of models. + + A v2 document does not write this member: its `.zgroup` key says the node is a group. + """ def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) problems = _group_problems_v2(document) if len(problems) != 0: raise MetadataValidationError(problems) @@ -313,8 +322,8 @@ def context(self) -> ZarrV2Context: return self._context @property - def unclaimed_fields(self) -> dict[Loc, UnclaimedField]: - """Every field nothing in scope claims. A v2 group holds no fields, so this is always empty; it is here so a reader can ask any node the same question.""" + def unclaimed_declarations(self) -> dict[Loc, UnclaimedDeclaration]: + """Every declaration nothing in scope claims. A v2 group holds no declarations, so this is always empty; it is here so a reader can ask any node the same question.""" return {} @property @@ -368,11 +377,11 @@ def update(self, **members: Unpack[ZarrV2GroupMetadataUpdate]) -> ZarrV2GroupMet def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: """Return this document read in `context`: the same group, holding that scope.""" - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) return self._of(self._document, context) def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: - """Return this document read in `context`. A group has no fields, so no scope conflicts with its reading, and this is the same as `with_context`.""" + """Return this document read in `context`. A group has no declarations, so no scope conflicts with its reading, and this is the same as `with_context`.""" return self.with_context(context) def refines(self, other: ZarrV2GroupMetadata) -> bool: @@ -406,7 +415,7 @@ def from_key_value( valid, with every problem the read finds. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) files, problems = decoded_node_files_v2( mapping, (ZARR_V2_GROUP_METADATA_STORE_KEY, ZARR_V2_ATTRIBUTES_STORE_KEY) ) @@ -433,11 +442,11 @@ class ZarrV2ConsolidatedMetadataReading: metadata: ZarrV2ConsolidatedMetadata | None = None """The document's model when there is no problem; None otherwise.""" - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Each field of each array node, as read, located in the `.zmetadata`: under the node's `.zarray` entry, at the key the document writes for it. Nothing when there is no model, and nothing for a group, which has no fields.""" + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Each declaration of each array node, as read, located in the `.zmetadata`: under the node's `.zarray` entry, at the key the document writes for it. Nothing when there is no model, and nothing for a group, which has no declarations.""" if self.metadata is None: return - yield from self.metadata._located_fields() # pyright: ignore[reportPrivateUsage] + yield from self.metadata._located_declarations() # pyright: ignore[reportPrivateUsage] def __reduce__(self) -> str | tuple[object, ...]: # Same as the other readings: pickle through the model, when there is one. @@ -449,8 +458,8 @@ def __reduce__(self) -> str | tuple[object, ...]: def read_consolidated_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2ConsolidatedMetadataReading: - """Read `value`, a v2 `.zmetadata` document, in `context` (`CORE_V2` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV2ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" - context = scoped(context, CORE_V2) + """Read `value`, a v2 `.zmetadata` document, in `context` (`CORE_V2_CONTEXT` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV2ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" + context = scoped(context, CORE_V2_CONTEXT) document, nodes, files, problems = _read_consolidated_v2(value, context) if len(problems) != 0: return ZarrV2ConsolidatedMetadataReading(problems=problems) @@ -485,9 +494,10 @@ class ZarrV2ConsolidatedMetadata(Keyed): _metadata: dict[str, ZarrV2NodeMetadata] zarr_consolidated_format: Final = 1 + """The version of the consolidated metadata format, `1`.""" def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) read, nodes, files, problems = _read_consolidated_v2(document, context) if len(problems) != 0: raise MetadataValidationError(problems) @@ -563,20 +573,20 @@ def _other_entries_text(self) -> str: consumed = {key for names in self._files.values() for key in names.values()} return json_text({key: value for key, value in entries.items() if key not in consumed}) - def _located_fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Each field of each array node, as read, located in the document: under the node's `.zarray` entry, at the key the document writes for it.""" + def _located_declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Each declaration of each array node, as read, located in the document: under the node's `.zarray` entry, at the key the document writes for it.""" for path, node in self._metadata.items(): if isinstance(node, ZarrV2ArrayMetadata): key = self._files[path][ZARR_V2_ARRAY_METADATA_STORE_KEY] - for loc, field in node.reading.fields(): - yield ("metadata", key, *loc), field + for loc, declaration in node.reading.declarations(): + yield ("metadata", key, *loc), declaration def __reduce__(self) -> tuple[type[ZarrV2ConsolidatedMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: """Return this document with every node read in `context`, whatever that changes. Raises `MetadataValidationError` when a node has a problem in `context`.""" - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) return type(self)(self._document, context=context) def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: @@ -586,19 +596,19 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2Consolidated node's entry. Raises `MetadataValidationError` when a gain exposes a problem. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) conflicts: list[Conflict] = [] for path, node in self._metadata.items(): if not isinstance(node, ZarrV2ArrayMetadata): continue - found = disagreements(context, claims_of(node.reading.fields())) + found = disagreements(context, claims_of(node.reading.declarations())) key = self._files[path][ZARR_V2_ARRAY_METADATA_STORE_KEY] conflicts.extend( dataclasses.replace( conflict, loc=("metadata", key, *(() if conflict.loc is None else conflict.loc)), ) - for conflict in located_conflicts(node.reading.fields(), found.conflicts) + for conflict in located_conflicts(node.reading.declarations(), found.conflicts) ) if len(conflicts) != 0: raise ScopeConflictError(conflicts) @@ -633,7 +643,7 @@ def from_key_value( are not JSON, or the document is not valid. Raises `TypeError` for a scope of another format before the bytes are decoded. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) return cls( load_store_json(mapping, ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY), context=context ) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index 155d8e8019..42f3e61357 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -39,7 +39,7 @@ from zarr_metadata._json import prefixed as _prefix from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._context import disagreements, scoped -from zarr_metadata.definition._definition import UnclaimedField, fields_of, held +from zarr_metadata.definition._definition import UnclaimedDeclaration, declarations_of, held from zarr_metadata.definition._scope import ( Conflict, ScopeConflictError, @@ -47,7 +47,7 @@ definition_said, kind_name, ) -from zarr_metadata.definition._scope import refines as refines_field +from zarr_metadata.definition._scope import refines as refines_declaration from zarr_metadata.model._array_v3 import ( ArrayMembersV3, ZarrV3ArrayMetadata, @@ -73,14 +73,14 @@ member_changes, missing_keys, reading_of, - unclaimed_fields_of, + unclaimed_declarations_of, unexpected_keys, ) from zarr_metadata.v3._common import ZarrV3ExtensionField # A runtime import: `get_type_hints` resolves the public signatures. from zarr_metadata.v3._context import ZarrV3Context # noqa: TC001 -from zarr_metadata.v3._core import CORE_AND_EXTENSIONS +from zarr_metadata.v3._core import CORE_AND_EXTENSIONS_CONTEXT from zarr_metadata.v3._definition import GroupExtraFieldDefinition from zarr_metadata.v3._hierarchy import hierarchy_problems, path_faults from zarr_metadata.v3.consolidated import ( @@ -94,7 +94,7 @@ from zarr_metadata._hierarchy import NodeType from zarr_metadata._typed_json import Loc - from zarr_metadata.definition._definition import AcceptedField, ResolvedField + from zarr_metadata.definition._definition import AcceptedDeclaration, ResolvedDeclaration from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataStoreKey @@ -133,8 +133,11 @@ class ZarrV3ConsolidatedMetadataInput(TypedDict, closed=True): """ kind: Literal["inline"] + """How the entries are carried, `"inline"`: in the group's own document.""" must_understand: Literal[False] + """Always `False`: a reader that does not recognize consolidated metadata may ignore it.""" metadata: Mapping[str, ZarrV3NodeMetadataInput] + """The document of each node below the group, or a model of one, by its path from the group.""" class ZarrV3GroupMetadataUpdate(TypedDict, total=False, closed=True): @@ -153,13 +156,25 @@ class ZarrV3GroupMetadataUpdate(TypedDict, total=False, closed=True): """ attributes: Mapping[str, JSONValue] | UNSET + """The `attributes` member: user attributes. + + `UNSET` removes it, so the group has no attributes. + """ consolidated_metadata: ( ZarrV3ConsolidatedMetadataJSON | Mapping[str, JSONValue] | ZarrV3ConsolidatedMetadata | UNSET ) + """The `consolidated_metadata` member: the documents of the nodes below the group, carried in its own document. + + `UNSET` removes it. + """ extra_fields: Mapping[str, ZarrV3ExtensionField | UNSET] + """The members the spec does not define, by key: each one given is set, or removed when its value is `UNSET`, and any other is kept. + + A key the spec defines, or that the update takes by name, raises `TypeError`. + """ class ZarrV3GroupMetadata(Keyed): @@ -169,7 +184,7 @@ class ZarrV3GroupMetadata(Keyed): the document as written, refined, and `context` is the scope. `attributes` and `consolidated_metadata` are views of the values the read refined. Each member the spec does not define is an extra field, - in `extra_fields`, as the scope read it. `CORE` reads a + in `extra_fields`, as the scope read it. `CORE_CONTEXT` reads a `consolidated_metadata` member as the envelope the core spec fixes, and no definition reads the documents the member lists: they follow a convention of the reference implementation, not the core spec. To @@ -183,10 +198,12 @@ class ZarrV3GroupMetadata(Keyed): __slots__ = ("_context", "_document", "_members", "_reading", "_shown") zarr_format: Final[Literal[3]] = 3 # noqa: PYI064 - a bare Final does not narrow a union of models + """The Zarr format version, `3`. A class constant: comparing it narrows a union of models.""" node_type: Final[Literal["group"]] = "group" # noqa: PYI064 - as above + """The kind of node, `"group"`. A class constant: comparing it narrows a union of models.""" def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_group_v3(document, context) if members is None or len(reading.problems) != 0: raise MetadataValidationError(reading.problems) @@ -280,8 +297,8 @@ def attributes(self) -> Mapping[str, JSONValue]: @property def extra_fields( self, - ) -> Mapping[str, AcceptedField[GroupExtraFieldDefinition[Any]] | UnclaimedField]: - """Each member the spec does not define, by key, as the scope read it: an `AcceptedField` when a definition in scope claims the key (as `CORE` claims `consolidated_metadata`), and an `UnclaimedField` otherwise.""" + ) -> Mapping[str, AcceptedDeclaration[GroupExtraFieldDefinition[Any]] | UnclaimedDeclaration]: + """Each member the spec does not define, by key, as the scope read it: an `AcceptedDeclaration` when a definition in scope claims the key (as `CORE_CONTEXT` claims `consolidated_metadata`), and an `UnclaimedDeclaration` otherwise.""" return self._shown[3] @property @@ -307,18 +324,20 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: return must_understand_subset(self.extra_fields, self._shown[1]) @property - def unclaimed_fields(self) -> dict[Loc, UnclaimedField]: - """Every field nothing in scope claims, at any depth, by its location in the document, except an extra member that says `"must_understand": false` and every field inside it, claimed or not, since a reader may ignore such a member whole. + def unclaimed_declarations(self) -> dict[Loc, UnclaimedDeclaration]: + """Every declaration nothing in scope claims, at any depth, by its location in the document, except an extra field that says `"must_understand": false` and every declaration inside it, claimed or not, since a reader may ignore such a member whole. This answers whether a reader can open the group: the spec requires - a reader to refuse a node with a field it does not understand, and - the scope is what the reader understands. The fields of the + a reader to refuse a node with a declaration it does not understand, and + the scope is what the reader understands. The declarations of the documents that the group's `consolidated_metadata` lists are not included, because the group does not read those documents; read - them with `consolidated()`. The model only reports these fields; + them with `consolidated()`. The model only reports these declarations; refusing to open is up to the reader. """ - return unclaimed_fields_of(self._reading.fields(), waived_members(self._shown[1])) + return unclaimed_declarations_of( + self._reading.declarations(), waived_members(self._shown[1]) + ) def consolidated(self) -> ZarrV3ConsolidatedMetadata | None: """The documents this group's `consolidated_metadata` lists, read in this group's own scope; None when the group writes no such member. @@ -365,8 +384,8 @@ def update(self, **members: Unpack[ZarrV3GroupMetadataUpdate]) -> ZarrV3GroupMet def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadata: """Return this document read in `context`, whatever that changes. Raises `MetadataValidationError` when the document has a problem in `context`. The existing reading is reused when `context` reads every claim identically.""" - context = scoped(context, CORE_AND_EXTENSIONS) - if disagreements(context, claims_of(self._reading.fields())).agrees: + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) + if disagreements(context, claims_of(self._reading.declarations())).agrees: return self._of(self._document, context, self._reading, self._members) return type(self)(self._document, context=context) @@ -382,14 +401,16 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadat in `context`, and `ZarrV3ConsolidatedMetadata.refined_in` checks that they read the same there. """ - context = scoped(context, CORE_AND_EXTENSIONS) - found = disagreements(context, claims_of(self._reading.fields())) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) + found = disagreements(context, claims_of(self._reading.declarations())) if len(found.conflicts) != 0: - raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) + raise ScopeConflictError( + located_conflicts(self._reading.declarations(), found.conflicts) + ) return self.with_context(context) def refines(self, other: ZarrV3GroupMetadata) -> bool: - """Whether this group holds everything `other` holds: the same attributes, and the same extra fields, each refining its counterpart in the order `refines` defines for fields. False when `other` is not a v3 group.""" + """Whether this group holds everything `other` holds: the same attributes, and the same extra fields, each refining its counterpart in the order `refines` defines for declarations. False when `other` is not a v3 group.""" if type(other) is not type(self): return False if json_text(self._members.attributes) != json_text(other._members.attributes): @@ -397,7 +418,7 @@ def refines(self, other: ZarrV3GroupMetadata) -> bool: mine, theirs = self.extra_fields, other.extra_fields if mine.keys() != theirs.keys(): return False - return all(refines_field(mine[key], theirs[key]) for key in mine) + return all(refines_declaration(mine[key], theirs[key]) for key in mine) # --- constructors ----------------------------------------------------- @@ -409,7 +430,7 @@ def create_default( **overrides: Unpack[ZarrV3GroupMetadataUpdate], ) -> ZarrV3GroupMetadata: """Return a group with no attributes, with any document members in `overrides` applied, read in `context`. `overrides` takes what `update` takes, including `UNSET` to drop a member and a `ZarrV3ConsolidatedMetadata` as `consolidated_metadata`. Raises `MetadataValidationError` when the resulting document has a problem.""" - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) changes = member_changes( overrides, ZarrV3GroupMetadataUpdate, GroupExtraFieldDefinition.spec_members ) @@ -426,7 +447,7 @@ def from_json( Raises `MetadataValidationError` with every problem the read finds. A `consolidated_metadata` of `null`, which zarr-python 3.0 and 3.1 wrote, is a value the document wrote that is not an object. In a - scope that claims the member, as `CORE` does, this is a problem, + scope that claims the member, as `CORE_CONTEXT` does, this is a problem, because the spec requires an object. Use `read_repaired_node_metadata_v3` to read such a store. A member the spec does not define is kept in `extra_fields`. @@ -443,7 +464,7 @@ def from_key_value( are not JSON, or the document is not valid. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) return cls(load_store_json(mapping, ZARR_V3_GROUP_METADATA_STORE_KEY), context=context) @@ -492,10 +513,12 @@ class ZarrV3ConsolidatedMetadata(Keyed): __slots__ = ("_context", "_document", "_metadata") kind: Final = "inline" + """How the entries are carried, `"inline"`: in the group's own document.""" must_understand: Final = False + """Always `False`: a reader that does not recognize consolidated metadata may ignore it.""" def __init__(self, member: object, context: ZarrV3Context | None = None) -> None: - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) # The member is always under a group's key, even when read on its # own, so nesting depth is counted from there, as in the group. readings, members, document, problems = _read_consolidated_v3( @@ -561,9 +584,9 @@ def __reduce__(self) -> tuple[type[ZarrV3ConsolidatedMetadata], tuple[object, Za def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ConsolidatedMetadata: """Return this member with every document read in `context`, whatever that changes. Raises `MetadataValidationError` when a document has a problem in `context`. The existing readings are reused when `context` reads every claim of every document identically.""" - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) if all( - disagreements(context, claims_of(node.reading.fields())).agrees + disagreements(context, claims_of(node.reading.declarations())).agrees for node in self._metadata.values() ): moved = {path: node.with_context(context) for path, node in self._metadata.items()} @@ -577,15 +600,15 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3Consolidated document's path. Raises `MetadataValidationError` when a gain exposes a problem. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) conflicts = [ dataclasses.replace( conflict, loc=("metadata", path, *(() if conflict.loc is None else conflict.loc)) ) for path, node in self._metadata.items() for conflict in located_conflicts( - node.reading.fields(), - disagreements(context, claims_of(node.reading.fields())).conflicts, + node.reading.declarations(), + disagreements(context, claims_of(node.reading.declarations())).conflicts, ) ] if len(conflicts) != 0: @@ -625,15 +648,15 @@ class ZarrV3GroupMetadataReading: """Every reason the document is not valid.""" metadata: ZarrV3GroupMetadata | None = None """The document's model when there is no problem; None otherwise.""" - extra_fields: Mapping[str, ResolvedField[GroupExtraFieldDefinition[Any]]] = dataclasses.field( - default_factory=no_extra_fields + extra_fields: Mapping[str, ResolvedDeclaration[GroupExtraFieldDefinition[Any]]] = ( + dataclasses.field(default_factory=no_extra_fields) ) - """Each member the spec does not define, as the scope read it, by key: an `UnclaimedField` when no definition in scope claims the key.""" + """Each member the spec does not define, as the scope read it, by key: an `UnclaimedDeclaration` when no definition in scope claims the key.""" - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Each field in the document, as the scope read it, with its location: the extra fields, each at its key.""" + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Each declaration in the document, as the scope read it, with its location: each extra field, as a declaration at its key.""" for key, extra in self.extra_fields.items(): - yield from fields_of(extra, (key,)) + yield from declarations_of(extra, (key,)) def __reduce__(self) -> str | tuple[object, ...]: # Same as the other readings: pickle through the model, when there @@ -646,7 +669,7 @@ def __reduce__(self) -> str | tuple[object, ...]: def _group_reading( problems: tuple[ValidationProblem, ...], - extra_fields: dict[str, ResolvedField[GroupExtraFieldDefinition[Any]]], + extra_fields: dict[str, ResolvedDeclaration[GroupExtraFieldDefinition[Any]]], ) -> ZarrV3GroupMetadataReading: """Rebuild a group reading without a model from the values `ZarrV3GroupMetadataReading.__reduce__` returns, with its extra fields held read-only.""" return ZarrV3GroupMetadataReading( @@ -663,13 +686,13 @@ class ZarrV3ConsolidatedMetadataReading: metadata: ZarrV3ConsolidatedMetadata | None = None """The member's model when there is no problem; None otherwise.""" - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Each field of each document, as read, located in the member: under `metadata` and the document's path. Nothing when there is no model.""" + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Each declaration of each document, as read, located in the member: under `metadata` and the document's path. Nothing when there is no model.""" if self.metadata is None: return for path, node in self.metadata.metadata.items(): - for loc, field in node.reading.fields(): - yield ("metadata", path, *loc), field + for loc, declaration in node.reading.declarations(): + yield ("metadata", path, *loc), declaration def __reduce__(self) -> str | tuple[object, ...]: # Same as the other readings: pickle through the model, when there is one. @@ -681,8 +704,8 @@ def __reduce__(self) -> str | tuple[object, ...]: def read_consolidated_metadata_v3( value: object, *, context: ZarrV3Context | None = None ) -> ZarrV3ConsolidatedMetadataReading: - """Read `value`, a group's `consolidated_metadata` member, in `context` (`CORE_AND_EXTENSIONS` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV3ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" - context = scoped(context, CORE_AND_EXTENSIONS) + """Read `value`, a group's `consolidated_metadata` member, in `context` (`CORE_AND_EXTENSIONS_CONTEXT` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV3ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) readings, members, document, problems = _read_consolidated_v3( value, context, (ZARR_V3_CONSOLIDATED_METADATA_KEY,) ) @@ -711,8 +734,8 @@ def metadata(self) -> None: """Its model: always None, since without a node type there is no model to choose.""" return None - def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: - """Its fields as read: none, since no field is read.""" + def declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: + """Its declarations as read: none, since no declaration is read.""" return iter(()) @@ -737,7 +760,7 @@ def read_node_metadata_v3( read `node_type` from unvalidated JSON, and a document in another format is reported as not v3. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) node_type, problems = _node_type(value) if node_type == "array": return read_array_metadata_v3(value, context=context) @@ -761,7 +784,7 @@ def node_metadata_from_json_v3( `MetadataValidationError` with every problem `read_node_metadata_v3` finds, including a `node_type` that is neither array nor group. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading = read_node_metadata_v3(data, context=context) if reading.metadata is None: raise MetadataValidationError(reading.problems) @@ -776,7 +799,7 @@ def node_metadata_from_key_value_v3( Raises `MetadataValidationError` when the key is missing, its bytes are not JSON, or the document is not a valid array or group. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) # Array documents and group documents are both stored at `zarr.json`. document = load_store_json(mapping, ZARR_V3_GROUP_METADATA_STORE_KEY) return node_metadata_from_json_v3(document, context=context) @@ -853,13 +876,13 @@ def read_group_metadata_v3( reader must understand each one that does not declare `must_understand: false`; the model reports these as `must_understand_fields`. A `consolidated_metadata` member, if - present and claimed by the scope, as `CORE` claims it, is checked + present and claimed by the scope, as `CORE_CONTEXT` claims it, is checked against the envelope the core spec fixes. The documents it lists are not read; `read_consolidated_metadata_v3` reads them. This is the form of `ZarrV3GroupMetadata(value, context)` that returns problems instead of raising them. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_group_v3(value, context) if members is None or members.document is None: return reading @@ -950,13 +973,13 @@ def _read_node_model( location. """ document = entry.to_json() - found = disagreements(context, claims_of(entry.reading.fields())) + found = disagreements(context, claims_of(entry.reading.declarations())) if len(found.conflicts) != 0: # Read the document again in this scope, so the reading belongs to # it, like every other reading the consolidated metadata holds. - # The conflicts become its problems, each at the field's location. + # The conflicts become its problems, each at the declaration's location. reading, _ = _read_node_v3(document, context, at) - written = {loc: field.name for loc, field in entry.reading.fields()} + written = {loc: declaration.name for loc, declaration in entry.reading.declarations()} problems = tuple( ValidationProblem( conflict.loc if conflict.loc is not None else (), @@ -965,7 +988,7 @@ def _read_node_model( ), "invalid_value", ) - for conflict in located_conflicts(entry.reading.fields(), found.conflicts) + for conflict in located_conflicts(entry.reading.declarations(), found.conflicts) ) # List the conflicts first: they are the cause of what the re-read finds. return _with_problems(reading, (*problems, *reading.problems)), None diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_repair.py b/packages/zarr-metadata/src/zarr_metadata/model/_repair.py index d16663b1fe..679a584fc8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_repair.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_repair.py @@ -45,9 +45,9 @@ ) from zarr_metadata.model._group_v3 import ZarrV3NodeMetadataReading, read_node_metadata_v3 from zarr_metadata.v2.array import ZARR_V2_ARRAY_METADATA_STORE_KEY -from zarr_metadata.v2.definition import CORE_V2 +from zarr_metadata.v2.definition import CORE_V2_CONTEXT from zarr_metadata.v2.group import ZARR_V2_GROUP_METADATA_STORE_KEY -from zarr_metadata.v3._core import CORE_AND_EXTENSIONS +from zarr_metadata.v3._core import CORE_AND_EXTENSIONS_CONTEXT from zarr_metadata.v3.consolidated import ZARR_V3_CONSOLIDATED_METADATA_KEY if TYPE_CHECKING: @@ -278,7 +278,7 @@ def read_repaired_node_metadata_v3( Content that no repair applies to is read as it is, and reported the same way `read_node_metadata_v3` reports it. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) repaired, repairs = repair_node_metadata_v3(value) return ZarrV3RepairedNodeMetadataReading( read_node_metadata_v3(repaired, context=context), repairs @@ -482,7 +482,7 @@ def read_repaired_node_metadata_v2( Content that no repair applies to is read as it is, and reported the same way `read_node_metadata_v2` reports it. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) repaired, repairs = repair_node_metadata_v2(files) return ZarrV2RepairedNodeMetadataReading( read_node_metadata_v2(repaired, context=context), repairs @@ -510,7 +510,7 @@ def read_repaired_consolidated_metadata_v2( read as it is, and reported the same way `read_consolidated_metadata_v2` reports it. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) repaired, repairs = repair_consolidated_metadata_v2(value) return ZarrV2RepairedConsolidatedMetadataReading( read_consolidated_metadata_v2(repaired, context=context), repairs diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py index a1f3d5c5e9..1a5ab0156a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py @@ -19,7 +19,11 @@ within, ) from zarr_metadata._typed_json import typeddict_keys -from zarr_metadata.definition._definition import RefusedField, UnclaimedField, is_field +from zarr_metadata.definition._definition import ( + RefusedDeclaration, + UnclaimedDeclaration, + is_declaration, +) from zarr_metadata.definition._scope import Conflict, claim_key from zarr_metadata.v3._definition import ExtraFieldDefinition @@ -28,7 +32,7 @@ from zarr_metadata._common import JSONValue from zarr_metadata._typed_json import Loc - from zarr_metadata.definition._definition import ResolvedField + from zarr_metadata.definition._definition import ResolvedDeclaration def missing_keys( @@ -109,7 +113,7 @@ def member_changes( `spec_members`) or that `update` takes by name. So a misspelled member is never written as a member the spec does not define, which a reader that must understand every such member would refuse. Also - raises `TypeError` for a field given where a member's JSON goes. + raises `TypeError` for a declaration given where a member's JSON goes. """ declared = typeddict_keys(update).members unknown = [key for key in members if key not in declared] @@ -124,7 +128,7 @@ def member_changes( raise TypeError(msg) changes = {key: value for key, value in members.items() if key != "extra_fields"} for key, value in changes.items(): - _refuse_fields(key, value) + _refuse_declarations(key, value) extra = members.get("extra_fields", {}) if not is_object(extra): msg = f"extra_fields maps members to their values, got {extra!r}" @@ -143,36 +147,37 @@ def member_changes( if key in spec_members: msg = f"{key!r} is a member the spec defines, which extra_fields does not hold" raise TypeError(msg) - _refuse_fields(key, value) + _refuse_declarations(key, value) changes[key] = value return changes -def unclaimed_fields_of( - fields: Iterable[tuple[Loc, ResolvedField[Any]]], waived: Iterable[str] -) -> dict[Loc, UnclaimedField]: - """Each of `fields` (a reading's fields, at every depth) that nothing in scope claims, by its location, except one inside an extra member whose key is in `waived`: a member that says `"must_understand": false`, which a reader may ignore whole, whether or not a definition claims it.""" +def unclaimed_declarations_of( + declarations: Iterable[tuple[Loc, ResolvedDeclaration[Any]]], waived: Iterable[str] +) -> dict[Loc, UnclaimedDeclaration]: + """Each of `declarations` (a reading's declarations, at every depth) that nothing in scope claims, by its location, except one inside an extra field whose key is in `waived`: a member that says `"must_understand": false`, which a reader may ignore whole, whether or not a definition claims it.""" ignored = frozenset(waived) return { - loc: field - for loc, field in fields - if isinstance(field, UnclaimedField) and not (len(loc) != 0 and loc[0] in ignored) + loc: declaration + for loc, declaration in declarations + if isinstance(declaration, UnclaimedDeclaration) + and not (len(loc) != 0 and loc[0] in ignored) } -def _refuse_fields(key: str, value: object) -> None: - """Raise `TypeError` when `value`, given as the member at `key`, is a field or holds fields, rather than the JSON a document writes, with the JSON to give instead.""" +def _refuse_declarations(key: str, value: object) -> None: + """Raise `TypeError` when `value`, given as the member at `key`, is a declaration or holds declarations, rather than the JSON a document writes, with the JSON to give instead.""" for item in value if is_array(value) else (value,): - if not is_field(item): + if not is_declaration(item): continue - if isinstance(item, RefusedField): - advice = "a refused field has no JSON to write; give the member as JSON" + if isinstance(item, RefusedDeclaration): + advice = "a refused declaration has no JSON to write; give the member as JSON" elif issubclass(item.read_as, ExtraFieldDefinition): # An extra field is read as the member alone, `{key: value}`. - advice = f"give the member's value, field.to_json()[{item.name!r}]" + advice = f"give the member's value, declaration.to_json()[{item.name!r}]" else: - advice = "give the field as JSON, field.to_json()" - msg = f"{key}: a member is given as a document writes it, got a field; {advice}" + advice = "give the declaration as JSON, declaration.to_json()" + msg = f"{key}: a member is given as a document writes it, got a declaration; {advice}" raise TypeError(msg) @@ -317,16 +322,21 @@ def dump_store_json(value: object, *, indent: int | str | None = None) -> bytes: def located_conflicts( - fields: Iterable[tuple[Loc, ResolvedField[Any]]], conflicts: Sequence[Conflict] + declarations: Iterable[tuple[Loc, ResolvedDeclaration[Any]]], conflicts: Sequence[Conflict] ) -> tuple[Conflict, ...]: - """Locate each of `conflicts` (found against a reading's claims) in the document, as a problem is located. A conflict appears once for each place in `fields` where its name occurs.""" + """Locate each of `conflicts` (found against a reading's claims) in the document, as a problem is located. A conflict appears once for each place in `declarations` where its name occurs.""" located: list[Conflict] = [] - placed = list(fields) + placed = list(declarations) for conflict in conflicts: - places = [(loc, field) for loc, field in placed if claim_key(field) == conflict.key] + places = [ + (loc, declaration) + for loc, declaration in placed + if claim_key(declaration) == conflict.key + ] if len(places) == 0: located.append(conflict) located.extend( - dataclasses.replace(conflict, loc=loc, written=field.name) for loc, field in places + dataclasses.replace(conflict, loc=loc, written=declaration.name) + for loc, declaration in places ) return tuple(located) diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index f369f2beec..4434660d8f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -8,7 +8,7 @@ freely with non-pydantic code (equality, isinstance, nesting). Validation delegates to the library: a raw document routes through `from_json` (the single source of truth for validation and normalization, so pydantic's -field-level coercion can never bypass it). A field type reads the fields +field-level coercion can never bypass it). A field type reads the declarations of its document in the scope given by pydantic's validation context (pydantic passes this context to every validator). The scope is chosen as follows: @@ -20,7 +20,7 @@ `"zarr_metadata_context_v2"` item. This lets a model with fields of both formats give a scope for each format. - If no scope is given for a format, its field types read in the default - scope for that format, `CORE_AND_EXTENSIONS` or `CORE_V2`. + scope for that format, `CORE_AND_EXTENSIONS_CONTEXT` or `CORE_V2_CONTEXT`. A mapping key that starts like these keys but is not one of them raises `TypeError`, so a misspelled key is not read as no scope. To fix a field's @@ -128,8 +128,8 @@ class ArrayManifest(BaseModel): from zarr_metadata.definition._context import scoped from zarr_metadata.definition._definition import Context from zarr_metadata.definition._scope import ScopeConflictError -from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, ZarrV3Context +from zarr_metadata.v2.definition import CORE_V2_CONTEXT, ZarrV2Context +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, ZarrV3Context if TYPE_CHECKING: from collections.abc import Callable, Mapping @@ -247,7 +247,7 @@ def coerce(value: object) -> _M: def _refined(cls: type[_M], value: _M, refine: Callable[[_M, _C], _M], context: _C) -> _M: - """`value`, an instance of `cls`, read again in `context` with `refine` (its `refined_in`), or a `pydantic_core.ValidationError` with one line error for each problem the scope finds, as for a document, and one `scope_conflict` line error for each conflict or lost meaning, at the field it is in.""" + """`value`, an instance of `cls`, read again in `context` with `refine` (its `refined_in`), or a `pydantic_core.ValidationError` with one line error for each problem the scope finds, as for a document, and one `scope_conflict` line error for each conflict or lost meaning, at the declaration it is in.""" try: return refine(value, context) except MetadataValidationError as error: @@ -321,7 +321,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json, _model.ZarrV3ArrayMetadata.refined_in, - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, CONTEXT_KEY, ), json_schema_input_type=_ZarrV3ArrayMetadataSchema, @@ -337,7 +337,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json, _model.ZarrV2ArrayMetadata.refined_in, - CORE_V2, + CORE_V2_CONTEXT, CONTEXT_KEY_V2, ), json_schema_input_type=_ZarrV2ArrayMetadataSchema, @@ -353,7 +353,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json, _model.ZarrV3GroupMetadata.refined_in, - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, CONTEXT_KEY, ), json_schema_input_type=_ZarrV3GroupMetadataSchema, @@ -369,7 +369,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json, _model.ZarrV2GroupMetadata.refined_in, - CORE_V2, + CORE_V2_CONTEXT, CONTEXT_KEY_V2, ), json_schema_input_type=_ZarrV2GroupMetadataSchema, @@ -385,7 +385,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV3ConsolidatedMetadata, _model.ZarrV3ConsolidatedMetadata.from_json, _model.ZarrV3ConsolidatedMetadata.refined_in, - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, CONTEXT_KEY, ), json_schema_input_type=_ZarrV3ConsolidatedMetadataSchema, @@ -404,7 +404,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV2ConsolidatedMetadata, _model.ZarrV2ConsolidatedMetadata.from_json, _model.ZarrV2ConsolidatedMetadata.refined_in, - CORE_V2, + CORE_V2_CONTEXT, CONTEXT_KEY_V2, ), json_schema_input_type=_ZarrV2ConsolidatedMetadataSchema, @@ -419,15 +419,15 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _FIELD_PARTS: Final[Mapping[type[Any], tuple[ZarrV3Context | ZarrV2Context, object]]] = ( MappingProxyType( { - _model.ZarrV3ArrayMetadata: (CORE_AND_EXTENSIONS, _ZarrV3ArrayMetadataSchema), - _model.ZarrV2ArrayMetadata: (CORE_V2, _ZarrV2ArrayMetadataSchema), - _model.ZarrV3GroupMetadata: (CORE_AND_EXTENSIONS, _ZarrV3GroupMetadataSchema), - _model.ZarrV2GroupMetadata: (CORE_V2, _ZarrV2GroupMetadataSchema), + _model.ZarrV3ArrayMetadata: (CORE_AND_EXTENSIONS_CONTEXT, _ZarrV3ArrayMetadataSchema), + _model.ZarrV2ArrayMetadata: (CORE_V2_CONTEXT, _ZarrV2ArrayMetadataSchema), + _model.ZarrV3GroupMetadata: (CORE_AND_EXTENSIONS_CONTEXT, _ZarrV3GroupMetadataSchema), + _model.ZarrV2GroupMetadata: (CORE_V2_CONTEXT, _ZarrV2GroupMetadataSchema), _model.ZarrV3ConsolidatedMetadata: ( - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, _ZarrV3ConsolidatedMetadataSchema, ), - _model.ZarrV2ConsolidatedMetadata: (CORE_V2, _ZarrV2ConsolidatedMetadataSchema), + _model.ZarrV2ConsolidatedMetadata: (CORE_V2_CONTEXT, _ZarrV2ConsolidatedMetadataSchema), } ) ) @@ -449,6 +449,7 @@ class InScope: """ context: ZarrV3Context | ZarrV2Context + """The scope the field is read in, in place of any scope the validation context holds.""" def __post_init__(self) -> None: if not _is_context(self.context): diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/__init__.py b/packages/zarr-metadata/src/zarr_metadata/testing/__init__.py new file mode 100644 index 0000000000..271a4dfe51 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/__init__.py @@ -0,0 +1,6 @@ +"""Testing utilities for code that reads or writes Zarr metadata. + +`zarr_metadata.testing.strategies` holds Hypothesis strategies that draw +valid metadata. It needs the `testing` extra: +`pip install zarr-metadata[testing]`. +""" diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py new file mode 100644 index 0000000000..b7528a3907 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py @@ -0,0 +1,105 @@ +"""Strategies for each chunk grid this package defines. + +Over an array of a known shape, a grid is drawn to fit it: a chunk +length for each dimension, never zero ("Chunk sizes must be greater +than zero", https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-grids/regular-grid/index.rst#L40), +and rectilinear edges that cover each dimension. Over no shape, what is +drawn is what the grid's rules accept. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from hypothesis import strategies as st + +from zarr_metadata.testing._strategy import MAX_RANK, ChunkGridStrategy +from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID +from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import ChunkGridDefinition + +_MAX_LENGTH: Final = 16 +"""The longest chunk drawn along a dimension whose length is unknown.""" + +_MAX_RUNS: Final = 5 +"""The most runs of equal lengths a rectilinear dimension is drawn with.""" + +_MAX_COUNT: Final = 4 +"""The most chunks a run is drawn with.""" + + +def _length(extent: int | None) -> st.SearchStrategy[int]: + """A chunk length along a dimension of `extent`: at least 1, and up to twice the extent when it is known and not 0.""" + if extent is None or extent == 0: + return st.integers(1, _MAX_LENGTH) + return st.integers(1, 2 * extent) + + +def _regular(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]: + """A chunk length for each dimension.""" + if shape is None: + lengths = st.lists(_length(None), max_size=MAX_RANK).map(tuple) + else: + lengths = st.tuples(*(_length(extent) for extent in shape)) + return lengths.map(lambda chunk_shape: {"chunk_shape": chunk_shape}) + + +def _spelled_run(draw: st.DrawFn, size: int, count: int) -> list[JSONValue]: + """`count` chunks of `size`: one `[size, count]` pair, or the size written once for each.""" + if draw(st.booleans()): + return [(size, count)] + return [size] * count + + +@st.composite +def _edges(draw: st.DrawFn, extent: int | None) -> JSONValue: + """Edge lengths along a dimension, covering `extent`, in runs spelled as pairs or written out. + + Runs are drawn apart, so two of one size can sit side by side and a + pair can count one chunk: spellings the canonical form merges. A + dimension of length zero, or of a length not known, may be covered by + no edges at all. + """ + min_runs = 0 if extent is None or extent == 0 else 1 + runs = draw( + st.lists( + st.tuples(st.integers(1, _MAX_LENGTH), st.integers(1, _MAX_COUNT)), + min_size=min_runs, + max_size=_MAX_RUNS, + ) + ) + covered = sum(size * count for size, count in runs) + if extent is not None and covered < extent: + runs.append((extent - covered, 1)) + spelled: list[JSONValue] = [] + for size, count in runs: + spelled.extend(_spelled_run(draw, size, count)) + return tuple(spelled) + + +def _rectilinear_dimension(extent: int | None) -> st.SearchStrategy[JSONValue]: + """A bare length, which repeats until it covers the dimension, or edges that cover it.""" + return _length(extent) | _edges(extent) + + +def _rectilinear(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]: + """Inline chunk shapes, one entry for each dimension.""" + if shape is None: + dimensions = st.lists(_rectilinear_dimension(None), max_size=MAX_RANK).map(tuple) + else: + dimensions = st.tuples(*(_rectilinear_dimension(extent) for extent in shape)) + return dimensions.map(lambda chunk_shapes: {"kind": "inline", "chunk_shapes": chunk_shapes}) + + +CHUNK_GRID_STRATEGIES: Final[Mapping[ChunkGridDefinition[Any], ChunkGridStrategy]] = { + REGULAR_CHUNK_GRID: ChunkGridStrategy(configurations=_regular), + RECTILINEAR_CHUNK_GRID: ChunkGridStrategy(configurations=_rectilinear), +} +"""A strategy for each chunk grid.""" + +__all__ = ["CHUNK_GRID_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py new file mode 100644 index 0000000000..9c525118c9 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py @@ -0,0 +1,40 @@ +"""Strategies for each chunk key encoding this package defines.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from hypothesis import strategies as st + +from zarr_metadata.testing._strategy import ChunkKeyEncodingStrategy +from zarr_metadata.v3.chunk_key_encoding.default import ( + DEFAULT_CHUNK_KEY_ENCODING, + DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR, +) +from zarr_metadata.v3.chunk_key_encoding.v2 import ( + V2_CHUNK_KEY_ENCODING, + V2_CHUNK_KEY_ENCODING_SEPARATOR, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.v3._definition import ChunkKeyEncodingDefinition + +CHUNK_KEY_ENCODING_STRATEGIES: Final[ + Mapping[ChunkKeyEncodingDefinition[Any], ChunkKeyEncodingStrategy] +] = { + DEFAULT_CHUNK_KEY_ENCODING: ChunkKeyEncodingStrategy( + configurations=st.fixed_dictionaries( + {}, optional={"separator": st.sampled_from(DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR)} + ) + ), + V2_CHUNK_KEY_ENCODING: ChunkKeyEncodingStrategy( + configurations=st.fixed_dictionaries( + {}, optional={"separator": st.sampled_from(V2_CHUNK_KEY_ENCODING_SEPARATOR)} + ) + ), +} +"""A strategy for each chunk key encoding: a `separator`, or none, which means the encoding's default.""" + +__all__ = ["CHUNK_KEY_ENCODING_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py new file mode 100644 index 0000000000..bd6c0927d9 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -0,0 +1,194 @@ +"""Strategies for each codec this package defines that holds no other declaration. + +A codec handed an array draws what fits the chunk: a `bytes` codec an +`endian` for a type whose values take several bytes, a `transpose` an +`order` with an entry for each axis. Handed `Chunk()`, nothing is known +of the chunk, and what is drawn is what the codec's rules accept. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from hypothesis import strategies as st + +from zarr_metadata.definition._definition import AcceptedDeclaration, EmptyConfiguration +from zarr_metadata.testing._data_type import ( + COMPLEX_WIDTHS, + FLOAT_WIDTHS, + INTEGER_RANGES, + FloatWidth, + finite_floats, + float_spellings, +) +from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, MAX_RANK, CodecStrategy +from zarr_metadata.v3._definition import Chunk, storage_of +from zarr_metadata.v3.codec._arithmetic import NOT_NUMBERS +from zarr_metadata.v3.codec.blosc import BLOSC_CNAME, BLOSC_CODEC, BLOSC_NO_SHUFFLE, BLOSC_SHUFFLE +from zarr_metadata.v3.codec.bytes import BYTES_CODEC, ENDIANNESS +from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC +from zarr_metadata.v3.codec.gzip import GZIP_CODEC +from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC +from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC +from zarr_metadata.v3.codec.vlen_bytes import VLEN_BYTES_CODEC +from zarr_metadata.v3.codec.vlen_utf8 import VLEN_UTF8_CODEC +from zarr_metadata.v3.codec.zstd import ZSTD_CODEC, ZSTD_MAX_LEVEL, ZSTD_MIN_LEVEL +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import CodecDefinition + +_BLOSC_MAX_TYPESIZE: Final = 255 +"""The widest element c-blosc takes: `BLOSC_MAX_TYPESIZE`.""" + +_BLOSC_MAX_BLOCKSIZE: Final = 2**31 - 1 +"""The largest block c-blosc takes, a signed 32-bit size; 0 lets it choose.""" + +_ENDIAN: Final = st.sampled_from(ENDIANNESS) + +_NUMPY_TIMES: Final = frozenset({NUMPY_DATETIME64_DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE}) +"""The numpy time types, each stored as a signed 64-bit integer.""" + +_NUMPY_TIME_RANGE: Final = (-(2**63) + 1, 2**63 - 1) +"""The signed 64-bit integers but the least, which numpy reads as `NaT`: the finite numpy times.""" + + +def _bytes(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: + """An `endian`, required for a data type whose values take several bytes; None for one whose values vary in size.""" + storage = None if chunk.data_type is None else storage_of(chunk.data_type) + if storage == "variable_length": + return None + if storage == "multi_byte": + return st.fixed_dictionaries({"endian": _ENDIAN}) + return st.fixed_dictionaries({}, optional={"endian": _ENDIAN}) + + +@st.composite +def _blosc(draw: st.DrawFn) -> Mapping[str, JSONValue]: + """A compressor, a level, a shuffle and a block size; and a `typesize`, which only `noshuffle` may leave out. + + Under `noshuffle` the spec says `typesize` "is ignored", so it is + drawn there too, a spelling that is not the simplest. + """ + shuffle = draw(st.sampled_from(BLOSC_SHUFFLE)) + typesize = st.integers(1, _BLOSC_MAX_TYPESIZE) + required: dict[str, st.SearchStrategy[JSONValue]] = { + "cname": st.sampled_from(BLOSC_CNAME), + "clevel": st.integers(0, 9), + "shuffle": st.just(shuffle), + "blocksize": st.integers(0, _BLOSC_MAX_BLOCKSIZE), + } + if shuffle == BLOSC_NO_SHUFFLE: + return draw(st.fixed_dictionaries(required, optional={"typesize": typesize})) + return draw(st.fixed_dictionaries({**required, "typesize": typesize})) + + +def _order(rank: int) -> st.SearchStrategy[Mapping[str, JSONValue]]: + return st.permutations(range(rank)).map(lambda order: {"order": tuple(order)}) + + +def _transpose(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]]: + """A permutation of the chunk's axes; of some number of axes, when the chunk does not say.""" + if chunk.rank is not None: + return _order(chunk.rank) + return st.integers(0, MAX_RANK).flatmap(_order) + + +def _finite_float_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A finite float of `width` bits, as a number or the hex string of its bits.""" + return finite_floats(width).flatmap(lambda value: float_spellings(value, width)) + + +def _non_zero_float_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A finite, non-zero float of `width` bits, as a number or the hex string of its bits.""" + return ( + finite_floats(width) + .filter(lambda value: value != 0) + .flatmap(lambda value: float_spellings(value, width)) + ) + + +def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: + """An `offset` and a `scale`, each a finite value of the data type the codec is handed, the scale not zero; None for a data type whose values are not numbers. + + Each is a fill value of the chunk's data type + (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L39-L43), + a float's as a number or the hex string of its bits, a complex + number's as a pair of them. Each is drawn finite, and the scale + non-zero, so that decoding never divides by zero; a scale small + enough that decoding overflows is still drawn. A numpy time is drawn + as the signed 64-bit integer it is stored as, never `NaT`, since it + "is compatible with any codec that supports arrays of signed 64-bit + integers" + (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120). + A data type this package does not define, whose values it cannot + draw, takes only the configuration with neither, the identity. + Handed a chunk whose data type is unknown, the values are small + non-negative integers, which every integer and float type holds. + """ + data_type = chunk.data_type + definition = data_type.definition if isinstance(data_type, AcceptedDeclaration) else None + if definition is None: + offset: st.SearchStrategy[JSONValue] = st.integers(0, 127) + scale: st.SearchStrategy[JSONValue] = st.integers(1, 127) + elif definition.name in NOT_NUMBERS: + return None + elif definition in INTEGER_RANGES or definition in _NUMPY_TIMES: + low, high = INTEGER_RANGES.get(definition, _NUMPY_TIME_RANGE) + offset = st.integers(low, high) + scale = st.integers(low, high).filter(lambda value: value != 0) + elif definition in FLOAT_WIDTHS: + offset = _finite_float_values(FLOAT_WIDTHS[definition]) + scale = _non_zero_float_values(FLOAT_WIDTHS[definition]) + elif definition in COMPLEX_WIDTHS: + width = COMPLEX_WIDTHS[definition] + offset = st.tuples(_finite_float_values(width), _finite_float_values(width)) + # Non-zero in either part is a non-zero complex number. + scale = st.one_of( + st.tuples(_non_zero_float_values(width), _finite_float_values(width)), + st.tuples(_finite_float_values(width), _non_zero_float_values(width)), + ) + else: + return EMPTY_CONFIGURATIONS + return st.fixed_dictionaries({}, optional={"offset": offset, "scale": scale}) + + +def _vlen(definition: CodecDefinition[Any]) -> CodecStrategy: + """A codec with nothing to configure that takes only some data types: none for a chunk of another, which its own chunk rules say.""" + return CodecStrategy( + configurations=lambda chunk: ( + None + if any(definition.chunk_rules(EmptyConfiguration(), {}, chunk)) + else EMPTY_CONFIGURATIONS + ) + ) + + +def _unchunked(configurations: st.SearchStrategy[Mapping[str, JSONValue]]) -> CodecStrategy: + """A codec whose configurations do not depend on the chunk it is handed.""" + return CodecStrategy(configurations=lambda _chunk: configurations) + + +CODEC_STRATEGIES: Final[Mapping[CodecDefinition[Any], CodecStrategy]] = { + BLOSC_CODEC: _unchunked(_blosc()), + BYTES_CODEC: CodecStrategy(configurations=_bytes), + CRC32C_CODEC: _unchunked(EMPTY_CONFIGURATIONS), + GZIP_CODEC: _unchunked(st.fixed_dictionaries({"level": st.integers(0, 9)})), + SCALE_OFFSET_CODEC: CodecStrategy(configurations=_scale_offset), + TRANSPOSE_CODEC: CodecStrategy(configurations=_transpose), + VLEN_BYTES_CODEC: _vlen(VLEN_BYTES_CODEC), + VLEN_UTF8_CODEC: _vlen(VLEN_UTF8_CODEC), + ZSTD_CODEC: _unchunked( + st.fixed_dictionaries( + {"level": st.integers(ZSTD_MIN_LEVEL, ZSTD_MAX_LEVEL)}, + optional={"checksum": st.booleans()}, + ) + ), +} +"""A strategy for each codec that holds no other declaration.""" + +__all__ = ["CODEC_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py new file mode 100644 index 0000000000..be68ce7b10 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py @@ -0,0 +1,256 @@ +"""Strategies for each data type this package defines that holds no other declaration. + +A fill value is drawn in every spelling the data type's spec gives it: +a float's as a number, a named non-finite value or a hex string of its +bits, a `bytes` value's as an array of byte values or base64. +""" + +from __future__ import annotations + +import base64 +import struct +from typing import TYPE_CHECKING, Any, Final, Literal, cast, get_args + +from hypothesis import strategies as st + +from zarr_metadata._numpy import NUMPY_TIME_MAX_SCALE_FACTOR +from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, DataTypeStrategy +from zarr_metadata.v3.data_type._float import FloatSpecialFillValue +from zarr_metadata.v3.data_type._numpy_time import NUMPY_TIME_UNIT +from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE +from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE +from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE +from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE +from zarr_metadata.v3.data_type.fixed_length_utf32 import ( + FIXED_LENGTH_UTF32_DATA_TYPE, + FixedLengthUtf32Configuration, +) +from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE +from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE +from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE +from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE +from zarr_metadata.v3.data_type.int16 import INT16_DATA_TYPE +from zarr_metadata.v3.data_type.int32 import INT32_DATA_TYPE +from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE +from zarr_metadata.v3.data_type.null_terminated_bytes import ( + NULL_TERMINATED_BYTES_DATA_TYPE, + NullTerminatedBytesConfiguration, +) +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE +from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE, RawBytesConfiguration +from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE +from zarr_metadata.v3.data_type.uint8 import UINT8_DATA_TYPE +from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE +from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE +from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE +from zarr_metadata.v3.data_type.variable_length_bytes import VARIABLE_LENGTH_BYTES_DATA_TYPE +from zarr_metadata.v3.data_type.zarr_python_raw_bytes import ( + ZARR_PYTHON_RAW_BYTES_DATA_TYPE, + ZarrPythonRawBytesConfiguration, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import DataTypeDefinition + +FloatWidth = Literal[16, 32, 64] + +INTEGER_RANGES: Final[Mapping[DataTypeDefinition[Any], tuple[int, int]]] = { + INT8_DATA_TYPE: (-(2**7), 2**7 - 1), + INT16_DATA_TYPE: (-(2**15), 2**15 - 1), + INT32_DATA_TYPE: (-(2**31), 2**31 - 1), + INT64_DATA_TYPE: (-(2**63), 2**63 - 1), + UINT8_DATA_TYPE: (0, 2**8 - 1), + UINT16_DATA_TYPE: (0, 2**16 - 1), + UINT32_DATA_TYPE: (0, 2**32 - 1), + UINT64_DATA_TYPE: (0, 2**64 - 1), +} +"""The values each integer data type holds.""" + +FLOAT_WIDTHS: Final[Mapping[DataTypeDefinition[Any], FloatWidth]] = { + FLOAT16_DATA_TYPE: 16, + FLOAT32_DATA_TYPE: 32, + FLOAT64_DATA_TYPE: 64, +} +"""The width in bits of each floating-point data type.""" + +COMPLEX_WIDTHS: Final[Mapping[DataTypeDefinition[Any], FloatWidth]] = { + COMPLEX64_DATA_TYPE: 32, + COMPLEX128_DATA_TYPE: 64, +} +"""The width in bits of each part of each complex data type.""" + +_MAX_RAW_BYTES: Final = 32 +"""The widest raw bits drawn, in bytes: `r256`.""" + +_MAX_BYTES: Final = 16 +"""The longest `bytes` fill value drawn.""" + + +def finite_floats(width: FloatWidth) -> st.SearchStrategy[float]: + """Finite numbers a float of `width` bits holds exactly.""" + return st.floats(width=width, allow_nan=False, allow_infinity=False) + + +_PACKED: Final[Mapping[FloatWidth, str]] = {16: ">e", 32: ">f", 64: ">d"} +"""How `struct` packs a float of each width, most significant byte first.""" + + +def _hex_spellings(digits: str) -> st.SearchStrategy[str]: + """`0x` and the digits, the letters in either case.""" + return st.sampled_from((f"0x{digits}", f"0x{digits.upper()}")) + + +def float_spellings(value: float, width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """`value`, a float of `width` bits, as a number or as the hex string of its bits.""" + return st.just(value) | _hex_spellings(struct.pack(_PACKED[width], value).hex()) + + +def _float_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A number, an integer the type holds exactly, a named non-finite value, or the bits in hex. + + A float's fill value is any of these + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79). + Numbers are drawn from those the type holds, so none is rounded: a + `float16` takes `1e300` by the letter of the spec, and a reader is + free to make of it what it will. + """ + # Every integer of magnitude up to 2**11 is a float16, the narrowest type. + numbers = finite_floats(width) | st.integers(-(2**11), 2**11) + named = st.sampled_from(get_args(FloatSpecialFillValue)) + bits = st.integers(0, 2**width - 1).flatmap( + lambda value: _hex_spellings(f"{value:0{width // 4}x}") + ) + return st.one_of(numbers, named, bits) + + +def _fill_values_of(values: st.SearchStrategy[JSONValue]) -> DataTypeStrategy: + """A data type with nothing to configure, whose fill values are `values`.""" + return DataTypeStrategy( + configurations=EMPTY_CONFIGURATIONS, fill_values=lambda _configuration: values + ) + + +def _byte_values(size: int) -> st.SearchStrategy[JSONValue]: + """`size` byte values, each an integer in `[0, 255]`, as a JSON array.""" + return st.lists(st.integers(0, 255), min_size=size, max_size=size).map( + lambda values: cast("JSONValue", tuple(values)) + ) + + +def _raw_bytes_fill_values(configuration: Mapping[str, JSONValue]) -> st.SearchStrategy[JSONValue]: + """One byte value for each 8 bits of the size.""" + return _byte_values(cast("RawBytesConfiguration", configuration)["bits"] // 8) + + +def _base64(value: bytes) -> JSONValue: + return base64.b64encode(value).decode("ascii") + + +def _complex_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A pair of float fill values of `width` bits, real then imaginary.""" + component = _float_fill_values(width) + return st.tuples(component, component).map(lambda pair: cast("JSONValue", pair)) + + +_BYTES_FILL_VALUES: Final = st.integers(0, _MAX_BYTES).flatmap(_byte_values) | st.binary( + max_size=_MAX_BYTES +).map(_base64) +"""A `bytes` fill value: byte values, or the bytes in standard-alphabet base64.""" + +_LENGTH_BYTES: Final = st.integers(1, _MAX_BYTES) +"""The `length_bytes` of zarr-python's fixed-length byte types.""" + + +def _base64_fill_values( + configuration: Mapping[str, JSONValue], +) -> st.SearchStrategy[JSONValue]: + """At most `length_bytes` bytes in standard-alphabet base64, as zarr-python's fixed-length byte types take them.""" + length_bytes = cast("ZarrPythonRawBytesConfiguration", configuration)["length_bytes"] + return st.binary(max_size=length_bytes).map(_base64) + + +_TEXT: Final = st.text(st.characters(exclude_categories=("Cs",)), max_size=16) +"""Text a JSON document can hold: no lone surrogate, which is no Unicode scalar value.""" + +_MAX_CODE_POINTS: Final = 16 +"""The most code points a drawn `fixed_length_utf32` value holds.""" + + +def _fixed_length_utf32_fill_values( + configuration: Mapping[str, JSONValue], +) -> st.SearchStrategy[JSONValue]: + """Text of at most one code point for each 4 bytes of the size, some of them trailing `U+0000` padding.""" + capacity = cast("FixedLengthUtf32Configuration", configuration)["length_bytes"] // 4 + return st.text(st.characters(exclude_categories=("Cs",)) | st.just("\x00"), max_size=capacity) + + +_NUMPY_TIME: Final = DataTypeStrategy( + configurations=st.fixed_dictionaries( + { + "unit": st.sampled_from(NUMPY_TIME_UNIT), + "scale_factor": st.integers(1, NUMPY_TIME_MAX_SCALE_FACTOR), + } + ), + fill_values=lambda _configuration: st.integers(-(2**63), 2**63 - 1) | st.just("NaT"), +) +"""Both numpy time types: a unit, and how many of it one tick is; a signed 64-bit count of ticks, or `"NaT"`.""" + +DATA_TYPE_STRATEGIES: Final[Mapping[DataTypeDefinition[Any], DataTypeStrategy]] = { + BOOL_DATA_TYPE: _fill_values_of(st.booleans()), + **{ + definition: _fill_values_of(st.integers(low, high)) + for definition, (low, high) in INTEGER_RANGES.items() + }, + **{ + definition: _fill_values_of(_float_fill_values(width)) + for definition, width in FLOAT_WIDTHS.items() + }, + **{ + definition: _fill_values_of(_complex_fill_values(width)) + for definition, width in COMPLEX_WIDTHS.items() + }, + RAW_BYTES_DATA_TYPE: DataTypeStrategy( + configurations=st.integers(1, _MAX_RAW_BYTES).map( + lambda size: RawBytesConfiguration(bits=8 * size) + ), + fill_values=_raw_bytes_fill_values, + ), + BYTES_DATA_TYPE: _fill_values_of(_BYTES_FILL_VALUES), + STRING_DATA_TYPE: _fill_values_of(_TEXT), + NULL_TERMINATED_BYTES_DATA_TYPE: DataTypeStrategy( + configurations=_LENGTH_BYTES.map( + lambda length_bytes: NullTerminatedBytesConfiguration(length_bytes=length_bytes) + ), + fill_values=_base64_fill_values, + ), + VARIABLE_LENGTH_BYTES_DATA_TYPE: _fill_values_of(st.binary(max_size=_MAX_BYTES).map(_base64)), + ZARR_PYTHON_RAW_BYTES_DATA_TYPE: DataTypeStrategy( + configurations=_LENGTH_BYTES.map( + lambda length_bytes: ZarrPythonRawBytesConfiguration(length_bytes=length_bytes) + ), + fill_values=_base64_fill_values, + ), + FIXED_LENGTH_UTF32_DATA_TYPE: DataTypeStrategy( + configurations=st.integers(1, _MAX_CODE_POINTS).map( + lambda count: FixedLengthUtf32Configuration(length_bytes=4 * count) + ), + fill_values=_fixed_length_utf32_fill_values, + ), + NUMPY_DATETIME64_DATA_TYPE: _NUMPY_TIME, + NUMPY_TIMEDELTA64_DATA_TYPE: _NUMPY_TIME, +} +"""A strategy for each data type that holds no other declaration.""" + +__all__ = [ + "COMPLEX_WIDTHS", + "DATA_TYPE_STRATEGIES", + "FLOAT_WIDTHS", + "INTEGER_RANGES", + "FloatWidth", + "finite_floats", + "float_spellings", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py new file mode 100644 index 0000000000..6c526a042d --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py @@ -0,0 +1,155 @@ +"""What draws one definition's configurations: a record for each kind of definition. + +Each record holds what its kind's configurations depend on, the way +each kind of definition does: a data type's fill values depend on its +configuration, a chunk grid's on the shape of the array, a codec's on +the chunk it is handed. A chunk key encoding's depend on nothing. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, ClassVar, Final, TypeAlias, cast + +from hypothesis import strategies as st + +from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata.v3._definition import ( + Chunk, + ChunkGridDefinition, + ChunkKeyEncodingDefinition, + CodecDefinition, + DataTypeDefinition, +) + +if TYPE_CHECKING: + from hypothesis.strategies import SearchStrategy + + from zarr_metadata._common import JSONValue + from zarr_metadata.definition._definition import Definition + +Configurations: TypeAlias = "SearchStrategy[Mapping[str, JSONValue]]" +"""Configurations of one definition, each the mapping every configuration is.""" + +NOTHING_KNOWN: Final = Chunk() +"""A chunk nothing is known of, which every codec takes.""" + +MAX_RANK: Final = 5 +"""The most axes drawn where the number of axes is not known.""" + +EMPTY_CONFIGURATIONS: Final = st.builds(EmptyConfiguration) +"""A fresh empty configuration for each draw, so a test that changes one changes no other.""" + + +class DefinitionStrategy(ABC): + """How to draw the configurations of one kind of definition.""" + + __slots__ = () + + kind: ClassVar[type[Definition[Any]]] + """The kind of definition whose configurations it draws.""" + + @abstractmethod + def unconstrained(self) -> Configurations: + """Configurations when nothing is known of what they depend on: what the definition's rules accept.""" + + +@dataclass(frozen=True, kw_only=True, slots=True) +class DataTypeStrategy(DefinitionStrategy): + """How to draw a data type's configurations, and the fill values an array of one takes.""" + + kind: ClassVar[type[Definition[Any]]] = DataTypeDefinition + """The kind of definition it draws configurations of: a data type.""" + + configurations: Configurations + """Configurations its definition accepts.""" + fill_values: Callable[[Mapping[str, JSONValue]], SearchStrategy[JSONValue]] + """Fill values of the data type a configuration describes.""" + + def unconstrained(self) -> Configurations: + """Configurations when nothing is known of what they depend on: what the definition's rules accept.""" + return self.configurations + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ChunkGridStrategy(DefinitionStrategy): + """How to draw a chunk grid's configurations over an array of a shape.""" + + kind: ClassVar[type[Definition[Any]]] = ChunkGridDefinition + """The kind of definition it draws configurations of: a chunk grid.""" + + configurations: Callable[[tuple[int, ...] | None], Configurations] + """Configurations of grids that fit an array of the shape. + + Handed None, no shape is known, and what is drawn is what the grid's + rules accept, whatever shape it is put over. + """ + + def unconstrained(self) -> Configurations: + """Configurations when nothing is known of what they depend on: what the definition's rules accept.""" + return self.configurations(None) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ChunkKeyEncodingStrategy(DefinitionStrategy): + """How to draw a chunk key encoding's configurations.""" + + kind: ClassVar[type[Definition[Any]]] = ChunkKeyEncodingDefinition + """The kind of definition it draws configurations of: a chunk key encoding.""" + + configurations: Configurations + """Configurations its definition accepts.""" + + def unconstrained(self) -> Configurations: + """Configurations when nothing is known of what they depend on: what the definition's rules accept.""" + return self.configurations + + +@dataclass(frozen=True, kw_only=True, slots=True) +class CodecStrategy(DefinitionStrategy): + """How to draw a codec's configurations, handed a chunk. + + Every codec takes `Chunk()`, a chunk nothing is known of, so a + strategy that draws nothing for it is refused when it is built. + """ + + kind: ClassVar[type[Definition[Any]]] = CodecDefinition + """The kind of definition it draws configurations of: a codec.""" + + configurations: Callable[[Chunk], Configurations | None] + """Configurations of the codec handed the chunk; None when no configuration takes it. + + A `bytes` codec handed a `string` chunk is None, since the spec gives + that data type a codec of its own. + """ + + def __post_init__(self) -> None: + if self.configurations(NOTHING_KNOWN) is None: + msg = "a codec strategy draws configurations for Chunk(), a chunk nothing is known of; got None" + raise ValueError(msg) + + def unconstrained(self) -> Configurations: + """Configurations when nothing is known of what they depend on: what the definition's rules accept.""" + return cast("Configurations", self.configurations(NOTHING_KNOWN)) + + +Strategies: TypeAlias = Mapping[Any, DefinitionStrategy] +"""How to draw each definition's configurations, by the definition. + +The key is `Any`, not `Definition[Any]`: a mapping's key type is +invariant, so `{MY_CODEC: CodecStrategy(...)}`, a mapping keyed by a +`CodecDefinition`, would not be a `Mapping[Definition[Any], ...]`. +""" + +__all__ = [ + "NOTHING_KNOWN", + "ChunkGridStrategy", + "ChunkKeyEncodingStrategy", + "CodecStrategy", + "Configurations", + "DataTypeStrategy", + "DefinitionStrategy", + "Strategies", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py new file mode 100644 index 0000000000..683959c4ea --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py @@ -0,0 +1,299 @@ +"""Hypothesis strategies for valid Zarr v3 metadata declarations. + +Every strategy here draws metadata its definition accepts, and that the +spec allows: a reader, writer or converter can test itself against the +whole space of a codec's, data type's, chunk grid's or chunk key +encoding's configurations instead of the handful its authors thought to +write down. + +```python +from hypothesis import given +from zarr_metadata.testing.strategies import declarations +from zarr_metadata.v3.codec.blosc import BLOSC_CODEC + +@given(declarations(BLOSC_CODEC)) +def test_my_reader_reads_every_blosc_codec(declaration): + my_reader.read_codec(declaration) +``` + +What a configuration depends on, each kind's function takes: +`fill_values` a data type's configuration, `chunk_grid_configurations` +the shape of the array, `codec_configurations` the `Chunk` the codec is +handed. Left out, nothing is known of it, and what is drawn is what the +definition's rules accept. `declarations` wraps configurations in the +envelopes a document writes them in. + +How each definition's configurations are drawn is a value, +`Strategies`, keyed by the definition: `DEFAULT_STRATEGIES` holds one +for each definition this package defines that holds no other declaration. An +extension author adds their own with +`DEFAULT_STRATEGIES | {MY_CODEC: CodecStrategy(...)}`. + +Configurations are drawn at the data level the TypedDicts declare, JSON +arrays as tuples; `json.dumps` takes them as they are. + +Requires the `testing` extra (`pip install zarr-metadata[testing]`). +""" + +from __future__ import annotations + +from functools import partial +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast + +from hypothesis import strategies as st + +from zarr_metadata._sentinel import UNSET +from zarr_metadata.definition._definition import Definition, as_kind, canonicalize, kind_of +from zarr_metadata.testing._chunk_grid import CHUNK_GRID_STRATEGIES +from zarr_metadata.testing._chunk_key_encoding import CHUNK_KEY_ENCODING_STRATEGIES +from zarr_metadata.testing._codec import CODEC_STRATEGIES +from zarr_metadata.testing._data_type import DATA_TYPE_STRATEGIES +from zarr_metadata.testing._strategy import ( + NOTHING_KNOWN, + ChunkGridStrategy, + ChunkKeyEncodingStrategy, + CodecStrategy, + Configurations, + DataTypeStrategy, + DefinitionStrategy, + Strategies, +) +from zarr_metadata.v3._context import ZarrV3Context +from zarr_metadata.v3._core import CORE_CONTEXT +from zarr_metadata.v3._definition import Chunk, DataTypeDefinition + +if TYPE_CHECKING: + from collections.abc import Mapping + + from hypothesis.strategies import SearchStrategy + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3 import ZarrV3DeclarationJSON + from zarr_metadata.v3._context import ZarrV3ContextSlots + from zarr_metadata.v3._definition import ( + ChunkGridDefinition, + ChunkKeyEncodingDefinition, + CodecDefinition, + ) + +C = TypeVar("C") + +Spellings = Literal["all", "canonical"] +"""How declarations are written: every spelling the spec allows, or only the simplest. + +`"all"` draws each envelope the spec gives a declaration -- a bare name, an +object without a configuration, an empty configuration, a +`must_understand` of `true` -- and configurations that say what a +simpler one says, as a `blosc` `typesize` under `noshuffle` does. +`"canonical"` draws each in the spelling `canonicalize` gives it, which +is what a writer that writes the fewest words writes. +""" + +DEFAULT_STRATEGIES: Final[MappingProxyType[Definition[Any], DefinitionStrategy]] = MappingProxyType( + { + **DATA_TYPE_STRATEGIES, + **CHUNK_GRID_STRATEGIES, + **CHUNK_KEY_ENCODING_STRATEGIES, + **CODEC_STRATEGIES, + } +) +"""A strategy for each definition this package defines that holds no other declaration.""" + + +def _strategy_for(definition: Definition[Any], strategies: Strategies) -> DefinitionStrategy: + """The strategy `strategies` holds for `definition`, which draws configurations of its kind.""" + found = strategies.get(definition) + if found is None: + msg = f"no strategy for {definition.name!r}; pass one in `strategies`" + raise LookupError(msg) + if not isinstance(definition, found.kind): + msg = ( + f"the strategy for {definition.name!r}, a {type(definition).__name__}, draws " + f"configurations of a {found.kind.__name__}: {found!r}" + ) + raise TypeError(msg) + return found + + +def data_type_configurations( + definition: DataTypeDefinition[C], *, strategies: Strategies = DEFAULT_STRATEGIES +) -> SearchStrategy[C]: + """Configurations of the data type `definition` defines.""" + found = cast("DataTypeStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations) + + +def fill_values( + definition: DataTypeDefinition[C], + configuration: C, + *, + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[JSONValue]: + """Fill values of the data type `definition` and `configuration` describe, in every spelling the spec gives them.""" + found = cast("DataTypeStrategy", _strategy_for(definition, strategies)) + return found.fill_values(cast("Mapping[str, JSONValue]", configuration)) + + +def chunk_grid_configurations( + definition: ChunkGridDefinition[C], + shape: tuple[int, ...] | None = None, + *, + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[C]: + """Configurations of the chunk grid `definition` defines, that fit an array of `shape`. + + With no shape, what is drawn is what the grid's rules accept. + """ + found = cast("ChunkGridStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations(shape)) + + +def chunk_key_encoding_configurations( + definition: ChunkKeyEncodingDefinition[C], *, strategies: Strategies = DEFAULT_STRATEGIES +) -> SearchStrategy[C]: + """Configurations of the chunk key encoding `definition` defines.""" + found = cast("ChunkKeyEncodingStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations) + + +def codec_configurations( + definition: CodecDefinition[C], + chunk: Chunk = NOTHING_KNOWN, + *, + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[C] | None: + """Configurations of the codec `definition` defines, handed `chunk`; None when none takes it. + + With no chunk, nothing is known of it, and what is drawn is what the + codec's rules accept. + """ + found = cast("CodecStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C] | None", found.configurations(chunk)) + + +def _written_bare(definition: Definition[Any]) -> bool: + """Whether the spec has the declaration written as its bare name: a data type it defines itself. + + "If the data type is defined in this specification, then the value + must be the data type identifier provided as a string" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L509-L512). + """ + if not isinstance(definition, DataTypeDefinition): + return False + return CORE_CONTEXT.claimant(DataTypeDefinition, definition.name) is definition + + +@st.composite +def _any_spelling( + draw: st.DrawFn, definition: Definition[Any], configurations: SearchStrategy[Any] +) -> ZarrV3DeclarationJSON: + """A configuration of `definition` in one of the envelopes the spec allows it. + + A configuration with nothing in it is written as the bare name, an + object without one, or an object with an empty one; any object may + say `must_understand` is `true`, which is what absence means. A name + that carries the configuration, as `r16` does, and a data type the + spec defines itself, are written as the bare name. + """ + configuration = cast("Mapping[str, JSONValue]", draw(configurations)) + carried = definition._carrying_name(configuration) # pyright: ignore[reportPrivateUsage] + if carried is not None: + return carried + if _written_bare(definition): + return definition.name + empty = len(configuration) == 0 + if empty and draw(st.booleans()): + return definition.name + declaration: dict[str, JSONValue] = {"name": definition.name} + if not empty or draw(st.booleans()): + declaration["configuration"] = configuration + if draw(st.booleans()): + declaration["must_understand"] = True + return cast("ZarrV3DeclarationJSON", declaration) + + +def _scope_of(definition: Definition[Any]) -> ZarrV3Context: + """The scope that claims `definition` and nothing else; a `TypeError` for a definition no slot of a Zarr v3 scope holds.""" + for slot, kind in ZarrV3Context.kinds.items(): + if isinstance(definition, kind): + return ZarrV3Context(**cast("ZarrV3ContextSlots", {slot: (definition,)})) + msg = ( + f"{definition.name!r} is a {type(definition).__name__}, " + "which no slot of a Zarr v3 scope holds" + ) + raise TypeError(msg) + + +def _checked( + declaration: ZarrV3DeclarationJSON, + *, + definition: Definition[Any], + kind: type[Definition[Any]], + scope: ZarrV3Context, + spellings: Spellings, +) -> ZarrV3DeclarationJSON: + """`declaration`, or its canonical spelling, once it reads without a problem in `scope`; a `ValueError` when it does not.""" + canonical, problems = canonicalize(declaration, kind, scope) + if len(problems) != 0: + msg = f"{declaration!r}, drawn for {definition.name!r}, which is refused: {problems!r}" + raise ValueError(msg) + if spellings == "canonical": + return cast("ZarrV3DeclarationJSON", canonical) + return declaration + + +def declarations( + definition: Definition[C], + configurations: SearchStrategy[C] | UNSET = UNSET, + *, + spellings: Spellings = "all", + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[ZarrV3DeclarationJSON]: + """Metadata declarations of `definition`: its name and a configuration, in an envelope a document writes. + + `configurations` is what to draw the configuration from, by default + the definition's own with nothing known of what they depend on: pass + `codec_configurations(definition, chunk)` for a codec handed a chunk, + once it is known not to be None. `spellings` says whether every + spelling the spec allows is drawn, or only the simplest. + + Each declaration is read in a scope of its definition alone, with no chunk + known, and one its definition's rules refuse, as they refuse one + drawn from `configurations` they do not accept, raises a + `ValueError`. A codec's chunk rules are not checked: what fits the + chunk is what `codec_configurations` draws. + """ + drawn: Configurations | SearchStrategy[C] = ( + _strategy_for(definition, strategies).unconstrained() + if configurations is UNSET + else configurations + ) + checked = partial( + _checked, + definition=definition, + kind=as_kind(kind_of(definition)), + scope=_scope_of(definition), + spellings=spellings, + ) + return _any_spelling(definition, drawn).map(checked) + + +__all__ = [ + "DEFAULT_STRATEGIES", + "NOTHING_KNOWN", + "ChunkGridStrategy", + "ChunkKeyEncodingStrategy", + "CodecStrategy", + "Configurations", + "DataTypeStrategy", + "DefinitionStrategy", + "Spellings", + "Strategies", + "chunk_grid_configurations", + "chunk_key_encoding_configurations", + "codec_configurations", + "data_type_configurations", + "declarations", + "fill_values", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py index 741ff5cfb5..af6dda6c22 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py @@ -6,12 +6,16 @@ and `data_type`, and the v2 kinds and scope are in `definition`. This module also exports every v2 definition (`GZIP_CODEC_V2`, -`FLOAT_DATA_TYPE_V2`, ...), the scope `CORE_V2`, and the scope class +`FLOAT_DATA_TYPE_V2`, ...), the scope `CORE_V2_CONTEXT`, and the scope class `ZarrV2Context`, so that one import builds or updates a scope. Each is the same object as in its own submodule, where a definition's configuration types and constants are. """ +from typing import TYPE_CHECKING + +from zarr_metadata._renamed import renamed_scope +from zarr_metadata.v2._core import RENAMED_SCOPES_V2 from zarr_metadata.v2.codec.checksum import ( ADLER32_CODEC_V2, CRC32_CODEC_V2, @@ -59,7 +63,7 @@ DATETIME64_DATA_TYPE_V2, TIMEDELTA64_DATA_TYPE_V2, ) -from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context +from zarr_metadata.v2.definition import CORE_V2_CONTEXT, ZarrV2Context __all__ = [ "ADLER32_CODEC_V2", @@ -70,7 +74,7 @@ "BYTES_DATA_TYPE_V2", "BZ2_CODEC_V2", "COMPLEX_DATA_TYPE_V2", - "CORE_V2", + "CORE_V2_CONTEXT", "CRC32C_CODEC_V2", "CRC32_CODEC_V2", "DATETIME64_DATA_TYPE_V2", @@ -98,3 +102,11 @@ "ZSTD_CODEC_V2", "ZarrV2Context", ] + + +if not TYPE_CHECKING: + # Hidden from type checkers, which would otherwise read every missing + # name of this module as a scope. + def __getattr__(name: str) -> object: + """`CORE_V2`, the scope's name before it ended in `_CONTEXT`, with a `DeprecationWarning`.""" + return renamed_scope(__name__, name, RENAMED_SCOPES_V2) diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_context.py b/packages/zarr-metadata/src/zarr_metadata/v2/_context.py index b13aa90620..09094c3882 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/_context.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_context.py @@ -1,4 +1,4 @@ -"""The scope a Zarr v2 document is read in: one slot for each kind of v2 field.""" +"""The scope a Zarr v2 document is read in: one slot for each kind of v2 declaration.""" from __future__ import annotations @@ -35,10 +35,10 @@ from zarr_metadata.definition._definition import D, Definition ZarrV2Kind: TypeAlias = ZarrV2DataTypeDefinition[Any] | ZarrV2CodecDefinition[Any] -"""A definition of a kind of v2 field.""" +"""A definition of a kind of v2 declaration.""" _K = TypeVar("_K", bound=ZarrV2Kind) -"""A kind of v2 field, so a type checker refuses a kind of the other format.""" +"""A kind of v2 declaration, so a type checker refuses a kind of the other format.""" _KINDS: Final[frozenset[type[Definition[Any]]]] = frozenset( get_origin(member) for member in get_args(ZarrV2Kind) @@ -46,7 +46,7 @@ def is_v2_kind(kind: type[Definition[Any]]) -> TypeIs[type[ZarrV2Kind]]: - """Whether `kind` is a kind of v2 field: one of the kinds in `ZarrV2Kind`.""" + """Whether `kind` is a kind of v2 declaration: one of the kinds in `ZarrV2Kind`.""" return kind in _KINDS @@ -73,7 +73,7 @@ class ZarrV2ContextSlots(TypedDict, total=False, closed=True): class ZarrV2Context: """A scope that reads Zarr v2 documents. Every v2 reader takes one. - It implements `Context`, and has one slot for each kind of field a v2 + It implements `Context`, and has one slot for each kind of declaration a v2 document holds, as `ZarrV2ContextSlots` declares them: `data_types`, read from `dtype`, and `codecs`, read from `compressor` and `filters`. Each slot is a read-only mapping from name to definition. The diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_core.py b/packages/zarr-metadata/src/zarr_metadata/v2/_core.py new file mode 100644 index 0000000000..0daaa6c344 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_core.py @@ -0,0 +1,17 @@ +"""The scope this package provides for Zarr v2.""" + +from __future__ import annotations + +from typing import Final + +from zarr_metadata.v2._context import ZarrV2Context +from zarr_metadata.v2.codec import CODECS_V2 +from zarr_metadata.v2.data_type import DATA_TYPES_V2 + +CORE_V2_CONTEXT: Final = ZarrV2Context(data_types=DATA_TYPES_V2, codecs=CODECS_V2) +"""The data types that zarr-python 2.x writes and the codecs that numcodecs 0.16 configures.""" + +RENAMED_SCOPES_V2: Final = {"CORE_V2": ("CORE_V2_CONTEXT", CORE_V2_CONTEXT)} +"""The v2 scope's name before it ended in `_CONTEXT`, with the scope's name now and the scope.""" + +__all__ = ["CORE_V2_CONTEXT", "RENAMED_SCOPES_V2"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_field_json.py b/packages/zarr-metadata/src/zarr_metadata/v2/_declaration_json.py similarity index 95% rename from packages/zarr-metadata/src/zarr_metadata/v2/_field_json.py rename to packages/zarr-metadata/src/zarr_metadata/v2/_declaration_json.py index f7a004bd17..e6f9763b22 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/_field_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_declaration_json.py @@ -40,6 +40,7 @@ class ZarrV2CodecMetadata(TypedDict, extra_items=JSONValue): """ id: str + """The numcodecs id that names the codec, such as `"blosc"`.""" __all__ = ["ZarrV2CodecMetadata", "ZarrV2DataTypeMetadata"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py index 789c8bbd86..b3bd11dc12 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py @@ -1,4 +1,4 @@ -"""The kinds of field in a Zarr v2 array document: a dtype, and a codec in `compressor` or `filters`. +"""The kinds of declaration in a Zarr v2 array document: a dtype, and a codec in `compressor` or `filters`. A v2 dtype is written as a NumPy typestr, which is a byte order, a type code and a size, such as ` str | None: if parse_typestr(self.name) is not None: @@ -242,7 +244,7 @@ def _in_configuration(cls, loc: Loc, place: Loc) -> Loc: # `_named_configuration` gives a v2 dtype a configuration only as a # struct's `{_RECORDS: records}`, and every other dtype's # configuration is encoded in its name, which is located at the - # field. So a place is the configuration itself, `()`, or starts + # declaration. So a place is the configuration itself, `()`, or starts # with `_RECORDS`, which the document does not write: it writes the # records as the dtype itself. return (*loc, *place[1:]) @@ -257,11 +259,12 @@ class ZarrV2CodecDefinition(Definition[C], kind=True, format=2): """A v2 codec: a numcodecs id, and the TypedDict that describes its parameters. A document writes `{"id": name, **parameters}`. The configuration of - the definition is the parameters, read at the location of the field + the definition is the parameters, read at the location of the declaration itself. """ label: ClassVar[str] = "v2 codec" + """The name of the kind in error messages, "v2 codec".""" @classmethod def _name_problem(cls, name: str, at: Loc) -> ValidationProblem | None: @@ -325,8 +328,8 @@ def _name_loc(cls, loc: Loc) -> Loc: "STRUCT_NAME", "TYPESTR_PATTERN", "ZarrV2CodecDefinition", + "ZarrV2DataTypeDeclaration", "ZarrV2DataTypeDefinition", - "ZarrV2DataTypeField", "item_bytes", "parse_typestr", "typestr_problem", diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index f4bd7803a7..d0eacd29eb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -7,7 +7,7 @@ from typing_extensions import TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v2._field_json import ZarrV2CodecMetadata, ZarrV2DataTypeMetadata +from zarr_metadata.v2._declaration_json import ZarrV2CodecMetadata, ZarrV2DataTypeMetadata ZarrV2ArrayOrder = Literal["C", "F"] """Literal type of permitted values for the `order` field of v2 array metadata. @@ -45,14 +45,39 @@ class ZarrV2ZArrayJSON(TypedDict): """ zarr_format: Literal[2] + """The Zarr format version, `2`.""" shape: tuple[Annotated[int, Ge(0)], ...] + """The length of each dimension of the array.""" chunks: tuple[Annotated[int, Ge(0)], ...] + """The length of each dimension of a chunk. + + Every chunk of the array has this shape, with one length per dimension of `shape`. + """ dtype: ZarrV2DataTypeMetadata + """The data type of the array's elements. + + A NumPy typestr such as `" ZarrV2ZlibParameters: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py index 4c5ad5c94b..f840ef531f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py @@ -23,22 +23,29 @@ class ZarrV2ShuffleParameters(TypedDict, closed=True): """`numcodecs.Shuffle(elementsize=4)`. An element size of 1 or less leaves the bytes as they are. numcodecs passes a larger one to C as a `Py_ssize_t`, so it is at most 2**63 - 1 on a 64-bit platform, and raises `OverflowError` for any other.""" elementsize: NotRequired[ReadOnly[Annotated[int, Le(2**63 - 1)]]] + """The size in bytes of the elements whose bytes are shuffled. Absent means 4.""" class ZarrV2DeltaParameters(TypedDict, closed=True): """`numcodecs.Delta(dtype, astype=None)`. When `astype` is omitted or null, it equals `dtype`.""" dtype: ReadOnly[str] + """The dtype of the decoded data.""" astype: NotRequired[ReadOnly[str | None]] + """The dtype of the encoded data. Absent or null means `dtype`.""" class ZarrV2FixedScaleOffsetParameters(TypedDict, closed=True): """`numcodecs.FixedScaleOffset(offset, scale, dtype, astype=None)`. When `astype` is omitted or null, it equals `dtype`.""" offset: ReadOnly[float | int] + """The value subtracted from each element before it is scaled.""" scale: ReadOnly[float | int] + """The factor that each element is multiplied by after the offset is subtracted. The product is rounded to the nearest integer.""" dtype: ReadOnly[str] + """The dtype of the decoded data.""" astype: NotRequired[ReadOnly[str | None]] + """The dtype of the encoded data. Absent or null means `dtype`.""" class ZarrV2QuantizeParameters(TypedDict, closed=True): @@ -50,21 +57,31 @@ class ZarrV2QuantizeParameters(TypedDict, closed=True): """ digits: ReadOnly[Annotated[int, Interval(ge=-308, le=307)]] + """The number of decimal digits of precision to keep.""" dtype: ReadOnly[str] + """The float dtype of the decoded data.""" astype: NotRequired[ReadOnly[str | None]] + """The float dtype of the encoded data. Absent or null means `dtype`.""" class ZarrV2BitRoundParameters(TypedDict, closed=True): """`numcodecs.BitRound(keepbits)`.""" keepbits: ReadOnly[Annotated[int, Ge(0)]] + """The number of mantissa bits to keep. + + numcodecs refuses, when encoding, a value above the mantissa width of the + data's float type: 10, 23 or 52 bits. + """ class ZarrV2AsTypeParameters(TypedDict, closed=True): """`numcodecs.AsType(encode_dtype, decode_dtype)`.""" encode_dtype: ReadOnly[str] + """The dtype of the encoded data.""" decode_dtype: ReadOnly[str] + """The dtype of the decoded data.""" def _shuffle_canonical(configuration: ZarrV2ShuffleParameters) -> ZarrV2ShuffleParameters: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py index 39b822f098..4497ab64e5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py @@ -15,6 +15,7 @@ class ZarrV2VLenArrayParameters(TypedDict, closed=True): """`numcodecs.VLenArray(dtype)`: the element type of each array.""" dtype: ReadOnly[str] + """The dtype of the elements of each variable-length array.""" def _vlen_array_canonical(configuration: ZarrV2VLenArrayParameters) -> ZarrV2VLenArrayParameters: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py index a6b8e4a58e..236a8616ab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py @@ -35,7 +35,12 @@ class ZarrV2ConsolidatedMetadataJSON(TypedDict): """ zarr_consolidated_format: Literal[1] + """The version of the consolidated metadata format, `1`.""" metadata: Mapping[str, ZarrV2ZArrayJSON | ZarrV2ZGroupJSON | ZarrV2ZAttrsJSON] + """The content of each metadata file in the hierarchy, by its store key. + + See the class docstring for the shape each key's value takes. + """ ZarrV2ConsolidatedMetadataStoreKey = Literal[".zmetadata"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/__init__.py index 05e7ebfcf4..b6c54b904f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/__init__.py @@ -8,8 +8,8 @@ from typing import Any, Final +from zarr_metadata.v2._declaration_json import ZarrV2DataTypeMetadata from zarr_metadata.v2._definition import ZarrV2DataTypeDefinition -from zarr_metadata.v2._field_json import ZarrV2DataTypeMetadata from zarr_metadata.v2.data_type.fixed_width import ( BYTES_DATA_TYPE_V2, STR_DATA_TYPE_V2, diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py index 01a1fd42cd..9e4dbf4605 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py @@ -16,6 +16,7 @@ class ZarrV2ObjectConfiguration(TypedDict, closed=True): """The configuration that `|O` carries: a byte order, which the type ignores.""" byteorder: ReadOnly[ZarrV2ByteOrder] + """The byte order that the typestr writes. The canonical form writes `|`, as NumPy does.""" def _canonical(configuration: ZarrV2ObjectConfiguration) -> ZarrV2ObjectConfiguration: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py index 6f94fa793b..2f9ec2393b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py @@ -27,7 +27,14 @@ class ZarrV2ScalarConfiguration(TypedDict, closed=True): """The configuration that a scalar typestr carries: its byte order and its size in bytes, such as `{"byteorder": "<", "itemsize": 4}` for ` Iterator[ValidationProblem]: @@ -65,10 +70,10 @@ def _record_size(configuration: ZarrV2StructConfiguration, nested: Nested) -> in """Return the number of bytes that one record of `configuration` takes. Each field takes its item size in bytes times the product of its subarray shape, and a nested struct takes the size of its own record. Return None when the size of a field is not known (as for `|O`) or its dtype was not read.""" total = 0 for index, record in enumerate(configuration["fields"]): - field = nested.get(("fields", index, 1)) - if not isinstance(field, AcceptedField): + declaration = nested.get(("fields", index, 1)) + if not isinstance(declaration, AcceptedDeclaration): return None - size = _item_size(field) + size = _item_size(declaration) if size is None: return None shape = record[2] if len(record) == 3 else () @@ -76,11 +81,13 @@ def _record_size(configuration: ZarrV2StructConfiguration, nested: Nested) -> in return total -def _item_size(field: AcceptedField[Any]) -> int | None: - """Return the number of bytes that one item of the dtype read as `field` takes, or None when that is not known. A typestr gives the size of its type, which for `U` counts code points of four bytes each.""" - if field.name == STRUCT_NAME or "fields" in field.configuration: - return _record_size(cast("ZarrV2StructConfiguration", field.configuration), field.nested) - return item_bytes(field.name) +def _item_size(declaration: AcceptedDeclaration[Any]) -> int | None: + """Return the number of bytes that one item of the dtype read as `declaration` takes, or None when that is not known. A typestr gives the size of its type, which for `U` counts code points of four bytes each.""" + if declaration.name == STRUCT_NAME or "fields" in declaration.configuration: + return _record_size( + cast("ZarrV2StructConfiguration", declaration.configuration), declaration.nested + ) + return item_bytes(declaration.name) STRUCT_DATA_TYPE_V2: Final = ZarrV2DataTypeDefinition( diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py index ffa0379053..ffca4e8662 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py @@ -26,9 +26,17 @@ class ZarrV2TimeConfiguration(TypedDict, closed=True): """The configuration that a time typestr carries. For example, ``.""" itemsize: ReadOnly[Annotated[int, Ge(0)]] + """The size in bytes that the typestr writes, which must be 8.""" unit: NotRequired[ReadOnly[str]] + """The NumPy time unit in brackets, such as `ns`. The canonical form writes `μs` as `us`. + + An array's dtype must have one. Only a codec's dtype parameter may leave + it out, as in ` ZarrV2TimeConfiguration: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/definition.py b/packages/zarr-metadata/src/zarr_metadata/v2/definition.py index 8cf4990992..9c77d4dcd7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/definition.py @@ -1,16 +1,16 @@ -"""Read Zarr v2 fields against their definitions. This is the public module for v2 reading. +"""Read Zarr v2 declarations against their definitions. This is the public module for v2 reading. -A v2 array document has two kinds of field: its `dtype`, and the codecs +A v2 array document has two kinds of declaration: its `dtype`, and the codecs in `compressor` and `filters`. Zarr v2 has no extension registry, but it still has a scope: the data types that zarr-python 2.x writes (one definition per NumPy family), and the codecs that numcodecs 0.16 -configures (one per id that this package models). `CORE_V2` is that -scope. To read one field in it, call `zarr_metadata.definition.resolve` -with the field's kind, as for a v3 field: -`resolve("