From 3a3c41b612977a0b9ba858121ddb41355438c3d8 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 2 Nov 2026 09:52:46 +0100 Subject: [PATCH 1/9] docs(zarr-metadata): split the landing page into guide pages and link each spec claim Shrink index.md to a short pitch, one example and a reader-by-reader list of where to go. Move the rest into guide pages: models and readers, what the readers check, consolidated metadata, repairing known writer bugs, writing an extension, JSON Schema and Pydantic. Each claim backed by a Zarr spec links to the spec section that makes it; the consolidated metadata page says what the specs do and do not settle and links the zarr-specs PRs and issues and the original zarr-python implementation. Trim the README so it no longer mirrors the landing page, and say "context" where the pages said "scope". Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/README.md | 410 +--------------- .../changes/+docs-restructure.doc.md | 1 + packages/zarr-metadata/docs/api/index.md | 9 +- .../docs/consolidated-metadata.md | 95 ++++ packages/zarr-metadata/docs/context.md | 2 +- packages/zarr-metadata/docs/extensions.md | 83 ++++ packages/zarr-metadata/docs/index.md | 449 +++--------------- packages/zarr-metadata/docs/json-schema.md | 33 ++ packages/zarr-metadata/docs/models.md | 146 ++++++ .../zarr-metadata/docs/parsing-validation.md | 24 +- packages/zarr-metadata/docs/pydantic.md | 58 +++ packages/zarr-metadata/docs/repair.md | 40 ++ .../zarr-metadata/docs/validation-rules.md | 86 ++++ packages/zarr-metadata/mkdocs.yml | 12 +- 14 files changed, 656 insertions(+), 792 deletions(-) create mode 100644 packages/zarr-metadata/changes/+docs-restructure.doc.md create mode 100644 packages/zarr-metadata/docs/consolidated-metadata.md create mode 100644 packages/zarr-metadata/docs/extensions.md create mode 100644 packages/zarr-metadata/docs/json-schema.md create mode 100644 packages/zarr-metadata/docs/models.md create mode 100644 packages/zarr-metadata/docs/pydantic.md create mode 100644 packages/zarr-metadata/docs/repair.md create mode 100644 packages/zarr-metadata/docs/validation-rules.md diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index a01ed26842..4b3c1293fe 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -2,410 +2,44 @@ Documentation: -Basic tools for modelling Zarr metadata, with minimal dependencies. +Types, models and validators for Zarr metadata documents, with minimal +dependencies. `zarr-metadata` is developed in the [zarr-python repository](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata) -and released independently of `zarr` itself. Install it with: +and released independently of `zarr`. Install it with: ``` pip install zarr-metadata ``` -## Who needs this - -This library supports software that interacts with Zarr metadata documents. - ## What this is -This is *not* a full Zarr implementation. It's a collection of data structures and routines that -closely model the content of the Zarr specifications, such as: - -- **Typed JSON shapes** ([`zarr_metadata.v2`](https://zarr-metadata.readthedocs.io/en/latest/api/v2/) and - [`zarr_metadata.v3`](https://zarr-metadata.readthedocs.io/en/latest/api/v3/)): `TypedDict` definitions and - `Literal` aliases for the JSON documents specified by the - [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) and - [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) - specifications, plus types for - [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/) and a - few widely-used-but-unspecified entities (e.g. consolidated metadata). -- **Document models** ([`zarr_metadata.model`](https://zarr-metadata.readthedocs.io/en/latest/api/model/)): immutable - models of whole metadata documents, each one a document and the scope - it was read in, with readers that report every problem at its - location, and store-key (de)serialization. A document - produced by `to_json` shares no mutable state with the model that produced - it. -- **Definitions and scopes** ([`zarr_metadata.definition`](https://zarr-metadata.readthedocs.io/en/latest/api/definition/), - with each format's kinds in `zarr_metadata.v3.definition` and - `zarr_metadata.v2.definition`): each extension's configuration as a - `TypedDict` with its rules, and the scope that decides which definitions - read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every - definition and scope of their format, such as `BYTES_CODEC` and - `CORE_AND_EXTENSIONS_CONTEXT`. -- **JSON type checking** ([`zarr_metadata.typed_json`](https://zarr-metadata.readthedocs.io/en/latest/api/typed_json/)): - `check` checks a JSON value against any `TypedDict` and reports every - problem with its location. -- **Problems** ([`zarr_metadata.problems`](https://zarr-metadata.readthedocs.io/en/latest/api/problems/)): the problem - values every reader returns, and the error that carries them. -- **JSON Schema** ([`zarr_metadata.json_schema`](https://zarr-metadata.readthedocs.io/en/latest/api/json_schema/)): the - JSON Schema of a `TypedDict`, a v3 declaration, or a whole `zarr.json`. -- **Optional Pydantic integration** ([`zarr_metadata.pydantic`](https://zarr-metadata.readthedocs.io/en/latest/api/pydantic/), - requires Pydantic 2.13 or newer): each model as a Pydantic field type that - validates raw documents through the same strict parser. - -## What this is for - -- **Static Type Checking** The public `TypedDict` definitions describe the static JSON shape of Zarr - metadata. - -- **Dynamic JSON validation** For strict, loc-aware validation of JSON loaded from disk, use the - model parser: - - ```python test="true" session="readme" - import json - from zarr_metadata.model import ZarrV3ArrayMetadata - - with open("zarr.json", "rb") as f: - raw = json.load(f) - - metadata = ZarrV3ArrayMetadata.from_json(raw) - ``` - - The optional Pydantic integration delegates raw input to the same strict - parser and returns the same normalized model class: - - ```python test="true" session="readme" - from pydantic import TypeAdapter - import zarr_metadata.pydantic as zmp - - adapter: TypeAdapter[zmp.ZarrV3ArrayMetadataField] = TypeAdapter(zmp.ZarrV3ArrayMetadataField) - metadata = adapter.validate_python(raw) - encoded = metadata.to_key_value()["zarr.json"] - ``` - - A field annotated `Annotated[ZarrV3ArrayMetadata, zmp.InScope(CONTEXT)]` - is read in `CONTEXT`, whatever the validation context holds, and a model - instance given to it is read again in `CONTEXT`. - - Note that a bare `TypeAdapter` over a public `TypedDict` is a **coercive** shape - adapter, not a Zarr conformance validator; it may coerce values or discard - members that the strict model parser rejects. This is better suited for handling input provided - from Python objects (e.g., user-authored dicts) than JSON. - -## Validation boundary - -The readers and model constructors enforce the declared document structure and a small set -of context-free consistency rules, e.g.: -- fixed format literals, -- finite JSON numbers outside `attributes` and the members nothing in scope reads -- non-negative dimensions -- one `dimension_names` entry per array dimension. - -In a v3 document, validation also checks each extension point: the data -type, the chunk grid, the chunk key encoding, each codec, and each storage -transformer. Each one is checked by the definition that claims its name in -a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. -`ZARR_PYTHON_CONTEXT` adds the data types that zarr-python writes but no -specification registers. -If the definition rejects a configuration, the reader rejects it too, -and a configuration key the definition does not declare is reported as -`unknown_key`. If no definition in the scope claims a name, the reader -does not check that extension point, and the consumer decides whether to -support it. - -Some checks relate one part of a v3 document to another: - -- The fill value is checked against the data type the document names, by - that data type's definition. -- The chunk grid is checked against the shape, by the grid's definition. -- The codecs are checked as a pipeline. In order, each codec's definition - checks the codec against the chunk it receives. This includes a shard's - inner codecs and index codecs. - -The readers do no arithmetic on values. For example, they do not check -whether a fill value survives a `cast_value` round trip. - -Some choices that the spec text leaves open, or answers in two ways: - -- **`attributes` may hold `NaN`, `Infinity` and `-Infinity`.** The spec - does not interpret attributes, and zarr-python and xarray write these - numbers there (for example, a CF `_FillValue`). The models read them, - and `to_key_value` writes them back as bare tokens, which a strict JSON - parser rejects. So may a member the spec does not define when no - definition in scope claims it: nothing reads it, so it is kept as - written. `check`, from `zarr_metadata.typed_json`, rejects a non-finite - number anywhere, including in attributes. -- **A reader walks at most 256 levels of nesting.** A value nested deeper - is an `invalid_value` problem at the first level past the limit, - wherever it is. Every reader, writer and comparison uses one stack - frame per level, and `copy.deepcopy` (and `pickle` before Python 3.12) - uses two. A document at the limit therefore uses about half of the - interpreter's default recursion limit, and the rest is left for the - caller. -- **`consolidated_metadata: null` is a problem.** A zarr-python 3.0.x bug - wrote it. The spec requires an object, and the package accepts only an - object. A reader of those stores removes the key before reading. -- **An extension name must follow the spec's naming rule**, - `^[a-z][a-z0-9-_.]+$`, or be a URI, which earlier versions of the spec - required. Any other name is rejected before any definition is - consulted, so `""` and `"foo/bar"` are problems, not unknown extensions. -- **`must_understand: false` is rejected at every extension point**, - including codecs and storage transformers, although the core spec names - only the data type, chunk grid and chunk key encoding. A reader that - skips a codec reads wrong bytes, just as a reader that skips a data type - reads wrong values. On an unknown top-level member, `must_understand` - keeps its meaning: a reader can skip that member. -- **A v2 datetime or timedelta dtype must have a unit**, as in ` Iterator[ValidationProblem]: + if "dictionary" in configuration and "dictionary_size" not in configuration: + yield ValidationProblem( + ("dictionary_size",), "a dictionary needs its size", "missing_key" + ) + + +ACME_LZ4 = CodecDefinition( + name="acme.lz4", + configuration=AcmeLz4Configuration, + kind="bytes_bytes", + size="dynamic", + rules=acme_lz4_rules, +) +context = CORE_AND_EXTENSIONS_CONTEXT.update(codecs=[*CORE_AND_EXTENSIONS_CONTEXT.codecs.values(), ACME_LZ4]) +``` + +Pass `context=context` to any reader or model constructor, and a document +that names `acme.lz4` is checked against this definition. + +Some rules for the pieces: + +- The `TypedDict` must be a `typing_extensions.TypedDict`, and must say + how undeclared keys are handled: `closed=True`, `extra_items=`, or + `closed=False`. An open-by-default `TypedDict` is refused, because it + would accept a misspelled key silently. +- Its fields must be JSON shapes: `int`, `float`, `bool`, `str`, `None`, + `JSONValue`, a `Literal`, `tuple[T, ...]`, `tuple[T1, T2]`, a union, a + `TypedDict`, `Mapping[str, V]`, a `NewType` or a type alias. A number + may carry bounds from annotated-types (`Gt`, `Ge`, `Lt`, `Le`, + `Interval`). Any other `Annotated` metadata is refused when the + definition is built. +- The rules run only on a configuration within its bounds, so a rule can + rely on them. A rule reports the location of a problem, and the value + there becomes the problem's `input`. Show values in messages with + `zarr_metadata.problems.shown`, so they read like the package's own. +- Define each function at module top level. A model holds the definitions + that read it, so it can be pickled only if they can. + +A data type also declares its fill value (`fill_value`, +`fill_value_rules`, `fill_value_canonical`) and how its values are stored +(`storage`). A chunk grid declares which shapes it fits (`shape_rules`) and +its chunk lengths (`chunk_lengths`). A codec declares what it checks about +the chunk it receives (`chunk_rules`), what it hands on (`transition`), and +any pipelines it contains (`pipelines`). The +[`zarr_metadata.v3.definition`](api/v3/definition.md) docstring describes +each of these. diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md index b139dfd307..a241a4f62d 100644 --- a/packages/zarr-metadata/docs/index.md +++ b/packages/zarr-metadata/docs/index.md @@ -1,411 +1,90 @@ # zarr-metadata -Basic tools for modelling Zarr metadata, with minimal dependencies. +Types, models and validators for Zarr metadata documents, with minimal +dependencies. `zarr-metadata` is developed in the [zarr-python repository](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata) -and released independently of `zarr` itself. Install it with: +and released independently of `zarr`. Install it with: ``` pip install zarr-metadata ``` -## Who needs this - -This library supports software that interacts with Zarr metadata documents. - ## What this is -This is *not* a full Zarr implementation. It's a collection of data structures and routines that -closely model the content of the Zarr specifications, such as: - -- **Typed JSON shapes** ([`zarr_metadata.v2`](api/v2.md) and - [`zarr_metadata.v3`](api/v3/index.md)): `TypedDict` definitions and - `Literal` aliases for the JSON documents specified by the - [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) and - [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) - specifications, plus types for - [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/) and a - few widely-used-but-unspecified entities (e.g. consolidated metadata). -- **Document models** ([`zarr_metadata.model`](api/model.md)): immutable - models of whole metadata documents, each one a document and the scope - it was read in, with readers that report every problem at its - location, and store-key (de)serialization. A document - produced by `to_json` shares no mutable state with the model that produced - it. -- **Definitions and scopes** ([`zarr_metadata.definition`](api/definition.md), - with each format's kinds in `zarr_metadata.v3.definition` and - `zarr_metadata.v2.definition`): each extension's configuration as a - `TypedDict` with its rules, and the scope that decides which definitions - read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every - definition and scope of their format, such as `BYTES_CODEC` and - `CORE_AND_EXTENSIONS_CONTEXT`. -- **JSON type checking** ([`zarr_metadata.typed_json`](api/typed_json.md)): - `check` checks a JSON value against any `TypedDict` and reports every - problem with its location. -- **Problems** ([`zarr_metadata.problems`](api/problems.md)): the problem - values every reader returns, and the error that carries them. -- **JSON Schema** ([`zarr_metadata.json_schema`](api/json_schema.md)): the - JSON Schema of a `TypedDict`, a v3 declaration, or a whole `zarr.json`. -- **Optional Pydantic integration** ([`zarr_metadata.pydantic`](api/pydantic.md), - requires Pydantic 2.13 or newer): each model as a Pydantic field type that - validates raw documents through the same strict parser. - -## What this is for - -- **Static Type Checking** The public `TypedDict` definitions describe the static JSON shape of Zarr - metadata. - -- **Dynamic JSON validation** For strict, location-aware validation of JSON loaded from disk, use the - model parser: - - ```python test="true" session="index" - import json - from zarr_metadata.model import ZarrV3ArrayMetadata - - with open("zarr.json", "rb") as f: - raw = json.load(f) - - metadata = ZarrV3ArrayMetadata.from_json(raw) - ``` - - The optional Pydantic integration delegates raw input to the same strict - parser and returns the same normalized model class: - - ```python test="true" session="index" - from pydantic import TypeAdapter - import zarr_metadata.pydantic as zmp - - adapter: TypeAdapter[zmp.ZarrV3ArrayMetadataField] = TypeAdapter(zmp.ZarrV3ArrayMetadataField) - metadata = adapter.validate_python(raw) - encoded = metadata.to_key_value()["zarr.json"] - ``` - - A field annotated `Annotated[ZarrV3ArrayMetadata, zmp.InScope(CONTEXT)]` - is read in `CONTEXT`, whatever the validation context holds, and a model - instance given to it is read again in `CONTEXT`. - - Note that a bare `TypeAdapter` over a public `TypedDict` is a **coercive** shape - adapter, not a Zarr conformance validator; it may coerce values or discard - members that the strict model parser rejects. This is better suited for handling input provided - from Python objects (e.g., user-authored dicts) than JSON. - -## Validation boundary - -The readers and model constructors enforce the declared document structure and a small set -of context-free consistency rules, e.g.: -- fixed format literals, -- finite JSON numbers outside `attributes` and the members nothing in scope reads -- non-negative dimensions -- one `dimension_names` entry per array dimension. - -In a v3 document, validation also checks each extension point: the data -type, the chunk grid, the chunk key encoding, each codec, and each storage -transformer. Each one is checked by the definition that claims its name in -a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. -`ZARR_PYTHON_CONTEXT` adds the data types that zarr-python writes but no -specification registers. -If the definition rejects a configuration, the reader rejects it too, -and a configuration key the definition does not declare is reported as -`unknown_key`. If no definition in the scope claims a name, the reader -does not check that extension point, and the consumer decides whether to -support it. - -Making the context and explicit part of metadata parsing allows different libraries, which might support different -sets of Zarr features, to formally represent exactly the codecs, data types, etc they support when -parsing metadata. - -Some checks relate one part of a v3 document to another: - -- The fill value is checked against the data type the document names, by - that data type's definition. -- The chunk grid is checked against the shape, by the grid's definition. -- The codecs are checked as a pipeline. In order, each codec's definition - checks the codec against the chunk it receives. This includes a shard's - inner codecs and index codecs. - -The readers do no arithmetic on values. For example, they do not check -whether a fill value survives a `cast_value` round trip. - -Some choices that the spec text leaves open, or answers in two ways: - -- **`attributes` may hold `NaN`, `Infinity` and `-Infinity`.** The spec - does not interpret attributes, and zarr-python and xarray write these - numbers there (for example, a CF `_FillValue`). The models read them, - and `to_key_value` writes them back as bare tokens, which a strict JSON - parser rejects. So may a member the spec does not define when no - definition in scope claims it: nothing reads it, so it is kept as - written. `check`, from `zarr_metadata.typed_json`, rejects a non-finite - number anywhere, including in attributes. -- **A reader walks at most 256 levels of nesting.** A value nested deeper - is an `invalid_value` problem at the first level past the limit, - wherever it is. Every reader, writer and comparison uses one stack - frame per level, and `copy.deepcopy` (and `pickle` before Python 3.12) - uses two. A document at the limit therefore uses about half of the - interpreter's default recursion limit, and the rest is left for the - caller. -- **`consolidated_metadata: null` is a problem.** A zarr-python 3.0.x bug - wrote it. The spec requires an object, and the package accepts only an - object. A reader of those stores removes the key before reading. -- **An extension name must follow the spec's naming rule**, - `^[a-z][a-z0-9-_.]+$`, or be a URI, which earlier versions of the spec - required. Any other name is rejected before any definition is - consulted, so `""` and `"foo/bar"` are problems, not unknown extensions. -- **`must_understand: false` is rejected at every extension point**, - including codecs and storage transformers, although the core spec names - only the data type, chunk grid and chunk key encoding. A reader that - skips a codec reads wrong bytes, just as a reader that skips a data type - reads wrong values. On an unknown top-level member, `must_understand` - keeps its meaning: a reader can skip that member. -- **A v2 datetime or timedelta dtype must have a unit**, as in ` zarr_metadata.definition': api/definition.md From 2b9ff03196e245a65d499ea7bbfebbbae337010b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 2 Nov 2026 10:29:41 +0100 Subject: [PATCH 2/9] docs(zarr-metadata): number the guide-pages fragment after its fork PR Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../changes/{+docs-restructure.doc.md => 434.doc.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{+docs-restructure.doc.md => 434.doc.md} (100%) diff --git a/packages/zarr-metadata/changes/+docs-restructure.doc.md b/packages/zarr-metadata/changes/434.doc.md similarity index 100% rename from packages/zarr-metadata/changes/+docs-restructure.doc.md rename to packages/zarr-metadata/changes/434.doc.md From 47a4b5d7c53e8ba5813e99512717fe12f36b13d8 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 2 Nov 2026 11:48:45 +0100 Subject: [PATCH 3/9] docs(zarr-metadata): say "member" for a JSON name/value pair in the guide pages The rename branch keeps "member" for a name/value pair of a metadata object, so the new pages follow: "a top-level member that the spec does not define is an extra field". "Extra field" and "extension field" stay, after the names extra_fields and ZarrV3ExtensionField. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../docs/consolidated-metadata.md | 10 ++++----- packages/zarr-metadata/docs/extensions.md | 2 +- packages/zarr-metadata/docs/json-schema.md | 2 +- packages/zarr-metadata/docs/models.md | 22 +++++++++---------- packages/zarr-metadata/docs/repair.md | 2 +- .../zarr-metadata/docs/validation-rules.md | 8 +++---- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/packages/zarr-metadata/docs/consolidated-metadata.md b/packages/zarr-metadata/docs/consolidated-metadata.md index b249b86c53..1428c67be5 100644 --- a/packages/zarr-metadata/docs/consolidated-metadata.md +++ b/packages/zarr-metadata/docs/consolidated-metadata.md @@ -17,10 +17,10 @@ before it describes the API. format, is open. - **Zarr v3.** zarr-specs [PR 309](https://github.com/zarr-developers/zarr-specs/pull/309), open since August 2024 and following [issue 136](https://github.com/zarr-developers/zarr-specs/issues/136), proposes the - `consolidated_metadata` field with the `kind: "inline"` envelope that + `consolidated_metadata` member with the `kind: "inline"` envelope that zarr-python 3 writes. It is not merged. The core spec does not define the - field's entries. Since [PR 373](https://github.com/zarr-developers/zarr-specs/pull/373), merged in July 2026, - it names the field "for historical reasons" and fixes its envelope: an + member's entries. Since [PR 373](https://github.com/zarr-developers/zarr-specs/pull/373), merged in July 2026, + it names the member "for historical reasons" and fixes its envelope: an object with `must_understand: false`, `kind: "inline"` and a `metadata` object ([spec text](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816)). That change closed [issue 371](https://github.com/zarr-developers/zarr-specs/issues/371), which found that the v3.1.0 extension @@ -49,12 +49,12 @@ the matching `node_path` functions, check a string against those rules. A group does not read the documents its `consolidated_metadata` lists. Consolidated metadata is a convention of the reference implementation, not -core metadata, so a group holds the field as written, after it checks the +core metadata, so a group holds the member as written, after it checks the [envelope that the core spec fixes](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816). `CORE_CONTEXT` holds `CONSOLIDATED_METADATA_EXTRA_FIELD`, the definition that reads a group's `consolidated_metadata` as that envelope. -A reader can use the field as a source of child documents in place of the +A reader can use the member as a source of child documents in place of the store: read one document with `read_node_metadata_v3` when that node is opened, or read all of them with the group's `consolidated()`, with `ZarrV3ConsolidatedMetadata`, or with `read_consolidated_metadata_v3`, diff --git a/packages/zarr-metadata/docs/extensions.md b/packages/zarr-metadata/docs/extensions.md index c768850663..f68da53a5a 100644 --- a/packages/zarr-metadata/docs/extensions.md +++ b/packages/zarr-metadata/docs/extensions.md @@ -60,7 +60,7 @@ Some rules for the pieces: how undeclared keys are handled: `closed=True`, `extra_items=`, or `closed=False`. An open-by-default `TypedDict` is refused, because it would accept a misspelled key silently. -- Its fields must be JSON shapes: `int`, `float`, `bool`, `str`, `None`, +- Its members must be JSON shapes: `int`, `float`, `bool`, `str`, `None`, `JSONValue`, a `Literal`, `tuple[T, ...]`, `tuple[T1, T2]`, a union, a `TypedDict`, `Mapping[str, V]`, a `NewType` or a type alias. A number may carry bounds from annotated-types (`Gt`, `Ge`, `Lt`, `Le`, diff --git a/packages/zarr-metadata/docs/json-schema.md b/packages/zarr-metadata/docs/json-schema.md index 9bc5172f1c..783476e514 100644 --- a/packages/zarr-metadata/docs/json-schema.md +++ b/packages/zarr-metadata/docs/json-schema.md @@ -17,7 +17,7 @@ bounds, and a name that nothing in the context claims may have any configuration. The fill value must be a valid fill value for the data type the document names. -A schema describes each field on its own, not the rules that relate fields +A schema describes each member on its own, not the rules that relate members to each other, so a document the schema accepts may still have a problem. Every JSON document the readers accept, the schema also accepts. A schema validator reads JSON as a parser produces it, with arrays as lists. A diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index f914ef8632..df77cd8cf3 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -54,7 +54,7 @@ metadata = reading.metadata # None when reading.problems is not empty `read_group_metadata_v3` reads a group the same way. It checks the envelope of the group's `consolidated_metadata`, and does not read the documents the -field lists (see [Consolidated metadata](consolidated-metadata.md)). +member lists (see [Consolidated metadata](consolidated-metadata.md)). `read_node_metadata_v3` reads a `zarr.json` of either kind as the node type its [`node_type`](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#node-type) names, the same way a discriminated union uses its tag. A document whose `node_type` is neither reads as @@ -77,30 +77,30 @@ writes that same document to store bytes. Every typed attribute is a view of that read: each declaration as the context read it (an `AcceptedDeclaration` or an `UnclaimedDeclaration`), -and `shape`, `attributes` and the fields that hold no declaration as the +and `shape`, `attributes` and the members that hold no declaration as the read refined them. These views are read-only at every level: a list in the document is shown as a tuple, and an object as a read-only mapping. `to_json` returns plain containers. A model holds nothing that can be changed in place. -To change a model, call `update`. It replaces fields of the document with -the given JSON fields and reads the result in the model's own context, so -you do not pass the context again. A field the spec defines is given by its +To change a model, call `update`. It replaces members of the document with +the given JSON members and reads the result in the model's own context, so +you do not pass the context again. A member the spec defines is given by its name, and any other field in `extra_fields`, a mapping from its key to its value, or to `UNSET` to remove it. A name `update` does not take raises -`TypeError`, so a misspelled field is never written. `with_context` reads +`TypeError`, so a misspelled member is never written. `with_context` reads the document in another context. `refined_in` also reads the document in another context, but only one that claims what this context left unclaimed and contradicts nothing; otherwise it raises `ScopeConflictError`. ## Extra fields -A top-level field that the spec does not define is an extra field. The -core spec hands such a field to its [extension rules](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#unknown). A +A top-level member that the spec does not define is an extra field. The +core spec hands such a member to its [extension rules](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#unknown). A context can hold a definition for one, by its key, in the `array_extra_fields` or `group_extra_fields` slot of `ZarrV3Context`. A -field whose key a definition claims is read by that definition: an extra -field is a declaration written as the field alone, `{key: value}`. Any +member whose key a definition claims is read by that definition: an extra +field is a declaration written as the member alone, `{key: value}`. Any other extra field is kept as written, as an `UnclaimedDeclaration`. The context is what a reader recognizes, so the model's @@ -111,7 +111,7 @@ grid, chunk key encoding, codec or storage transformer that nothing in the context claims is an `UnclaimedDeclaration` too, also when it sits inside another declaration, such as a codec inside a shard. A model's `unclaimed_declarations` lists every declaration nothing in the context -claims, at any depth, by its location, and leaves out the fields that waive +claims, at any depth, by its location, and leaves out the members that waive `must_understand`: a reader can open the node when it is empty. ## Equality diff --git a/packages/zarr-metadata/docs/repair.md b/packages/zarr-metadata/docs/repair.md index cebdd239e6..696b08bea7 100644 --- a/packages/zarr-metadata/docs/repair.md +++ b/packages/zarr-metadata/docs/repair.md @@ -12,7 +12,7 @@ the result strictly, and return each change they made as a `Repair`. dimension, which zarr-python 3.0 and 3.1 wrote, is written as 1. - `consolidated_metadata: null` on a v3 group, which zarr-python 3.0 and 3.1 wrote, is removed. The spec [requires an object](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816). -- The `consolidated_metadata` field that zarr-python 3 writes into each +- The `consolidated_metadata` member that zarr-python 3 writes into each `.zgroup` entry of a `.zmetadata` below the root is removed. The [v2 spec](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html#groups) allows no key but `zarr_format` in a `.zgroup`. - A non-finite v2 fill value that zarr-python 3.0.0 to 3.0.6 wrote as a diff --git a/packages/zarr-metadata/docs/validation-rules.md b/packages/zarr-metadata/docs/validation-rules.md index 3293b56aff..b023d1bb33 100644 --- a/packages/zarr-metadata/docs/validation-rules.md +++ b/packages/zarr-metadata/docs/validation-rules.md @@ -5,7 +5,7 @@ and a small set of context-free consistency rules, for example: - fixed format literals ([`zarr_format`](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#zarr-format) and [`node_type`](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#node-type)), -- finite JSON numbers outside `attributes` and the fields nothing in the +- finite JSON numbers outside `attributes` and the members nothing in the context reads, - non-negative [dimensions](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#array-metadata-shape), - one [`dimension_names`](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#dimension-names) entry per array dimension. @@ -40,7 +40,7 @@ Some choices that the spec text leaves open, or answers in two ways: [does not interpret attributes](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#attributes), and zarr-python and xarray write these numbers there (for example, a CF `_FillValue`). The models read them, and `to_key_value` writes them back as bare tokens, which a strict JSON - parser rejects. So may a field the spec does not define when no + parser rejects. So may a member the spec does not define when no definition in the context claims it: nothing reads it, so it is kept as written. `check`, from `zarr_metadata.typed_json`, rejects a non-finite number anywhere, including in attributes. @@ -63,8 +63,8 @@ Some choices that the spec text leaves open, or answers in two ways: [core spec](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#must-understand) names only the data type, chunk grid and chunk key encoding. A reader that skips a codec reads wrong bytes, just as a reader that skips a data type - reads wrong values. On an unknown top-level field, `must_understand` - keeps its meaning: a reader can skip that field. + reads wrong values. On an unknown top-level member, `must_understand` + keeps its meaning: a reader can skip that member. - **A v3 `regular` chunk grid's chunk lengths must be at least 1**, even along a dimension of length 0. The [regular grid spec](https://zarr-specs.readthedocs.io/en/latest/v3/chunk-grids/regular-grid/index.html#description) says "Chunk sizes must be greater than zero", a sentence added after v3.1.0 in From 0a073c96d68f5605d15d60c4c8f8590001147057 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 2 Nov 2026 11:55:54 +0100 Subject: [PATCH 4/9] docs(zarr-metadata): say "member" for a JSON name/value pair on every page Sweep the parsing-validation and context pages, which said "field" for a name/value pair of a metadata object, and state the one exception on the models page: "extra field" follows the spec's "additional fields" and the names extra_fields, ExtraFieldDefinition and must_understand_fields; "field" otherwise means a field of a Python class, or a part of a structured data type. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/docs/context.md | 4 +- packages/zarr-metadata/docs/models.md | 8 ++++ .../zarr-metadata/docs/parsing-validation.md | 42 +++++++++---------- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/packages/zarr-metadata/docs/context.md b/packages/zarr-metadata/docs/context.md index 296e666855..f759416428 100644 --- a/packages/zarr-metadata/docs/context.md +++ b/packages/zarr-metadata/docs/context.md @@ -4,7 +4,7 @@ Resolving metadata documents requires a context, a `Context`, that defines which ## Why do we need a context -The Zarr V3 spec defines the JSON structure of `zarr.json` for arrays and groups exactly, by defining the fields that must be present and the JSON types they must have. But the __interpretation__ of those fields as actual data types, codecs, chunk grids, etc is not strictly defined by the spec. Relating a JSON value like `"bytes"` to the `bytes` codec requires a mapping from names to codec types. +The Zarr V3 spec defines the JSON structure of `zarr.json` for arrays and groups exactly, by defining the members that must be present and the JSON types they must have. But the __interpretation__ of those members as actual data types, codecs, chunk grids, etc is not strictly defined by the spec. Relating a JSON value like `"bytes"` to the `bytes` codec requires a mapping from names to codec types. `zarr-metadata` makes this explicit by validating JSON documents with a __context object__. That context object contains definitions of codecs, data types, etc, which can be matched to the declarations in the JSON document. @@ -47,7 +47,7 @@ document: ZarrV3ArrayMetadataJSON = { } ``` -In the above code snippet we define a `zarr.json` document with three codec declarations (we will just call them codecs for brevity): one named `bytes`, another named `gzip`, and a third named `unknown`. The `bytes` codec here is correctly configured according to the `bytes` codec spec. The `gzip` declaration here violates the `gzip` codec spec because the `level` configuration field is too high. And the `unknown` codec is unknown, it has no specification and cannot be resolved. +In the above code snippet we define a `zarr.json` document with three codec declarations (we will just call them codecs for brevity): one named `bytes`, another named `gzip`, and a third named `unknown`. The `bytes` codec here is correctly configured according to the `bytes` codec spec. The `gzip` declaration here violates the `gzip` codec spec because the `level` configuration member is too high. And the `unknown` codec is unknown, it has no specification and cannot be resolved. These three codecs represent three different scenarios that this library handles, two with distinct failure modes. We demonstrate how this works in the next code snippet. diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index df77cd8cf3..a17d621f5c 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -103,6 +103,14 @@ member whose key a definition claims is read by that definition: an extra field is a declaration written as the member alone, `{key: value}`. Any other extra field is kept as written, as an `UnclaimedDeclaration`. +The spec calls these members [additional fields](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#additional-fields), +and the names `extra_fields`, `ExtraFieldDefinition` and +`must_understand_fields` follow it. Everywhere else this package says +*member* for a name/value pair of a JSON object and *declaration* for the +value at an extension point. It keeps *field* for the fields of a Python +class, such as a dataclass or a pydantic model, and, after the spec and +NumPy, for the parts of a structured data type (`StructField`). + The context is what a reader recognizes, so the model's `must_understand_fields` lists each unclaimed field that does not say `"must_understand": false`: the spec [requires](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#must-understand) a reader to fail to open a diff --git a/packages/zarr-metadata/docs/parsing-validation.md b/packages/zarr-metadata/docs/parsing-validation.md index d509ebc23e..8feee576b6 100644 --- a/packages/zarr-metadata/docs/parsing-validation.md +++ b/packages/zarr-metadata/docs/parsing-validation.md @@ -25,26 +25,26 @@ This procedure yields one of two values: ### JSON document shape -At this stage we check that the document is a JSON object with per-field structure consistent with rules in the Zarr V3 spec. +At this stage we check that the document is a JSON object with per-member structure consistent with rules in the Zarr V3 spec. After confirming that the object named `zarr.json` is indeed valid JSON, we now need to check that the structure of this object is consistent with the rules in the Zarr V3 spec, namely the rules about the [structure of array metadata](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#array-metadata). At this stage we need to ensure that: -- every **required** field is present -- every **required** and **optional** field has the correct shape +- every **required** member is present +- every **required** and **optional** member has the correct shape -The correct shape varies per field. The Zarr V3 spec defines a [**generic metadata element type**](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#extension-definition) that is reused for data types, codecs, etc. Inhabitants of that type are: +The correct shape varies per member. The Zarr V3 spec defines a [**generic metadata element type**](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#extension-definition) that is reused for data types, codecs, etc. Inhabitants of that type are: - strings -- JSON objects with a `"name"` field and optionally `"configuration"` and `"must_understand"` fields +- JSON objects with a `"name"` member and optionally `"configuration"` and `"must_understand"` members -The `data_type` field must comply with this shape. The `codecs` field must be a non-empty JSON array of values that comply with this shape. +The `data_type` member must comply with this shape. The `codecs` member must be a non-empty JSON array of values that comply with this shape. The spec does not define a shape for **extra** fields. Instead, the [relevant section of the spec](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#unknown) says that a reader must refuse to open an array with an extra field, unless that field is a JSON object with [`"must_understand": false`](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#must-understand). Like an unknown name, which we will see later, an extra field is not a problem with the document. It's a field that some readers can't use. So at this stage we don't check extra fields; we just note which of them a reader must understand. !!! note - At this stage we don't need to know **which** data type, or chunk grid, or codecs, are declared in the metadata document. We are only looking at **how** those fields are declared. We will resolve the raw JSON to a specific data type, chunk grid, etc. in a later validation pass. + At this stage we don't need to know **which** data type, or chunk grid, or codecs, are declared in the metadata document. We are only looking at **how** those members are declared. We will resolve the raw JSON to a specific data type, chunk grid, etc. in a later validation pass. This procedure yields one of two values: @@ -55,22 +55,22 @@ For a program that repairs broken `zarr.json` documents, the list of problems is ### Context-free consistency checks -At this stage we check if correlated fields in the document are consistent. +At this stage we check if correlated members in the document are consistent. -Some of the fields in the array metadata document are semantically coupled in a way that can be directly checked at the structural level. As of this writing, the only such fields are the `shape` field (a JSON array of non-negative integers), and the `dimension_names` field (an optional JSON array of strings or `null`s). +Some of the members in the array metadata document are semantically coupled in a way that can be directly checked at the structural level. As of this writing, the only such members are the `shape` member (a JSON array of non-negative integers), and the `dimension_names` member (an optional JSON array of strings or `null`s). -The length of the `shape` field must match the length of the `dimension_names` field, if that field is declared. +The length of the `shape` member must match the length of the `dimension_names` member, if that member is declared. This procedure yields one of two values: -1. A Zarr-metadata-shaped JSON object with consistent `shape` and `dimension_names` fields -2. A problem localized to the `dimension_names` field. Strictly speaking, we can't say which of the two fields is wrong. But we treat `shape` as the reference for the number of axes in the array, so the problem goes on the field that disagrees with it. `shape` is required, and other fields like `chunk_grid` depend on it; `dimension_names` is optional, and only names the axes. This metadata has to be repaired; there's no other way to handle it. +1. A Zarr-metadata-shaped JSON object with consistent `shape` and `dimension_names` members +2. A problem localized to the `dimension_names` member. Strictly speaking, we can't say which of the two members is wrong. But we treat `shape` as the reference for the number of axes in the array, so the problem goes on the member that disagrees with it. `shape` is required, and other members like `chunk_grid` depend on it; `dimension_names` is optional, and only names the axes. This metadata has to be repaired; there's no other way to handle it. ### Resolving and validating each declaration At this stage we check that each metadata object names a known type, given a registry of data types, codec types, etc., and that its configuration is valid for that type. -We take the JSON given for the `data_type`, elements of `codecs`, `chunk_grid`, `chunk_key_encoding`, and `storage_transformers` fields and attempt to **resolve** that JSON into a concrete data type, chunk grid, etc. +We take the JSON given for the `data_type`, elements of `codecs`, `chunk_grid`, `chunk_key_encoding`, and `storage_transformers` members and attempt to **resolve** that JSON into a concrete data type, chunk grid, etc. In the previous stages, we just had two elements to work with: @@ -92,7 +92,7 @@ For each declaration, we do two steps: What if a name isn't in the registry? This is **not** a problem with the document. The document might be perfectly valid, and written by a program with a bigger registry than ours. An unknown name tells us something about our registry, not about the document. So we don't report it as a problem. Instead we mark the declaration as **unknown**, skip the second step, and leave it to the program that reads the document to decide what to do with it. A program that only needs to list the arrays in a hierarchy might be fine with an unknown codec. A program that decodes chunks is not, and it can try again with a more comprehensive registry. -If the name is in the registry, we validate the configuration. Suppose our metadata JSON had the following `chunk_grid` field: `"chunk_grid": {"name": "regular", "configuration": "error"}`. If we resolved the name of the chunk grid (`"regular"`) to the [regular chunk grid](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/chunk-grids/regular-grid/index.html) defined in the core spec, then we must do additional checks on the structure of the `configuration` field before we can interpret this chunk grid. In this case, the spec requires that the `configuration` field be a JSON object with a `chunk_shape` field, which is a JSON array of positive integers. Our `"configuration": "error"` field has the wrong type, so this is a problem localized to `/chunk_grid/configuration`. +If the name is in the registry, we validate the configuration. Suppose our metadata JSON had the following `chunk_grid` member: `"chunk_grid": {"name": "regular", "configuration": "error"}`. If we resolved the name of the chunk grid (`"regular"`) to the [regular chunk grid](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/chunk-grids/regular-grid/index.html) defined in the core spec, then we must do additional checks on the structure of the `configuration` member before we can interpret this chunk grid. In this case, the spec requires that the `configuration` member be a JSON object with a `chunk_shape` member, which is a JSON array of positive integers. Our `"configuration": "error"` member has the wrong type, so this is a problem localized to `/chunk_grid/configuration`. Validating a configuration might require resolving more declarations. Consider this example: @@ -111,18 +111,18 @@ Validating a configuration might require resolving more declarations. Consider t } ``` -The name `"sharding_indexed"` should resolve to the [codec with the same name](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/codecs/sharding-indexed/index.html), and the definition of that codec states that its `codecs` field contains more codec metadata. These JSON array elements go through the same two steps, recursively: `"not_a_codec"` is not in the registry, so it is marked as unknown, not reported as a problem. +The name `"sharding_indexed"` should resolve to the [codec with the same name](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/codecs/sharding-indexed/index.html), and the definition of that codec states that its `codecs` member contains more codec metadata. These JSON array elements go through the same two steps, recursively: `"not_a_codec"` is not in the registry, so it is marked as unknown, not reported as a problem. Repeating these steps, recursively if necessary, for all metadata objects, we yield one of two values: 1. A Zarr metadata object where each declaration is either unknown, or resolved to a type and internally consistent with the definition of that type 2. An enumeration of localized validation problems. -### Consistency checks between fields +### Consistency checks between members At this stage we check that our resolved metadata is consistent across the entire metadata document. -Here we are in the space of interactions between metadata objects. It's a big space. One key invariant that cuts across many fields is dimensionality, i.e. the number of array axes. The `chunk_grid` and `shape` fields must agree on the dimensionality of the array. That is, +Here we are in the space of interactions between metadata objects. It's a big space. One key invariant that cuts across many members is dimensionality, i.e. the number of array axes. The `chunk_grid` and `shape` members must agree on the dimensionality of the array. That is, ```json { @@ -131,11 +131,11 @@ Here we are in the space of interactions between metadata objects. It's a big sp } ``` -is a snippet of an invalid metadata document because the `shape` field applies to an array with three axes while the `chunk_grid` field applies to an array with just one. As with `dimension_names`, `shape` is the reference, so the problem is localized to `/chunk_grid/configuration/chunk_shape`. Dimensionality shows up in codecs too: the `order` of a [`transpose`](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/codecs/transpose/index.html) codec must name each array axis exactly once. +is a snippet of an invalid metadata document because the `shape` member applies to an array with three axes while the `chunk_grid` member applies to an array with just one. As with `dimension_names`, `shape` is the reference, so the problem is localized to `/chunk_grid/configuration/chunk_shape`. Dimensionality shows up in codecs too: the `order` of a [`transpose`](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/codecs/transpose/index.html) codec must name each array axis exactly once. -The `fill_value` field is another example. The spec does not define a single shape for `fill_value`. Instead, each data type defines which JSON values it accepts as a fill value. So `"fill_value": "hello"` might be fine for a string data type, but it's a problem for `"float64"`. We can only check `fill_value` after we have resolved `data_type`. +The `fill_value` member is another example. The spec does not define a single shape for `fill_value`. Instead, each data type defines which JSON values it accepts as a fill value. So `"fill_value": "hello"` might be fine for a string data type, but it's a problem for `"float64"`. We can only check `fill_value` after we have resolved `data_type`. -The `codecs` field has the most interactions of all, because a list of codecs is a **pipeline**. [The spec](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#id22) divides codecs into three kinds: +The `codecs` member has the most interactions of all, because a list of codecs is a **pipeline**. [The spec](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#id22) divides codecs into three kinds: - array → array codecs, like `transpose`, which take an array and return an array - array → bytes codecs, like `bytes`, which take an array and return bytes @@ -145,7 +145,7 @@ A valid pipeline is zero or more array → array codecs, then exactly one array Each codec in the pipeline is also handed a value produced by everything that came before it. The first codec is handed a chunk with the data type of the array and the shape of the chunk grid's chunks. An array → array codec may change that shape or data type before handing it on. Some codecs have rules about what they are handed. For example: -- The [`bytes`](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/codecs/bytes/index.html#configuration-parameters) codec requires an `endian` field in its configuration when the data type it is handed takes more than one byte per element. `{"name": "bytes"}` is valid for a `uint8` array, but a problem for a `float64` array. +- The [`bytes`](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/codecs/bytes/index.html#configuration-parameters) codec requires an `endian` member in its configuration when the data type it is handed takes more than one byte per element. `{"name": "bytes"}` is valid for a `uint8` array, but a problem for a `float64` array. - The [`sharding_indexed`](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/codecs/sharding-indexed/index.html#configuration-parameters) codec requires that its inner `chunk_shape` evenly divides the shape of the chunk it is handed, which is the outer chunk (or shard) shape. To check these rules, we walk the pipeline in order, tracking the chunk that each codec is handed. For codecs that hold pipelines of their own, like `sharding_indexed`, we walk those inner pipelines too. From 0088e6a233a42a27ccc9a5b4cbf4d3a586c5ca56 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 2 Nov 2026 11:58:33 +0100 Subject: [PATCH 5/9] docs(zarr-metadata): say "member" in the two must_understand_fields sentences The models page said "each unclaimed field" and "any such field" where the landing page and every neighboring sentence say "member". Roborev flagged the mismatch. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/docs/models.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index a17d621f5c..d4c82a03f3 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -112,9 +112,9 @@ class, such as a dataclass or a pydantic model, and, after the spec and NumPy, for the parts of a structured data type (`StructField`). The context is what a reader recognizes, so the model's -`must_understand_fields` lists each unclaimed field that does not say +`must_understand_fields` lists each unclaimed member that does not say `"must_understand": false`: the spec [requires](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#must-understand) a reader to fail to open a -node with any such field. It covers extra fields only. A data type, chunk +node with any such member. It covers extra fields only. A data type, chunk grid, chunk key encoding, codec or storage transformer that nothing in the context claims is an `UnclaimedDeclaration` too, also when it sits inside another declaration, such as a codec inside a shard. A model's From 785e5c2e1f7f63445ad378d092a2dc66995c2ee0 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 2 Nov 2026 12:00:41 +0100 Subject: [PATCH 6/9] docs(zarr-metadata): say why the additional-fields link points at the latest spec v3.1.0 has no "Additional fields" heading; PR 373 added it after the release, so the link cannot be pinned. Say so beside the link. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/docs/models.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index d4c82a03f3..2e55df0a6d 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -104,7 +104,9 @@ field is a declaration written as the member alone, `{key: value}`. Any other extra field is kept as written, as an `UnclaimedDeclaration`. The spec calls these members [additional fields](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#additional-fields), -and the names `extra_fields`, `ExtraFieldDefinition` and +the heading [PR 373](https://github.com/zarr-developers/zarr-specs/pull/373) +gave the section that v3.1.0 calls "Unknown" (no tagged release carries it +yet), and the names `extra_fields`, `ExtraFieldDefinition` and `must_understand_fields` follow it. Everywhere else this package says *member* for a name/value pair of a JSON object and *declaration* for the value at an extension point. It keeps *field* for the fields of a Python From 6f081183428a7ded4c5391a5b55427fcd182f23a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 2 Nov 2026 12:09:38 +0100 Subject: [PATCH 7/9] docs(zarr-metadata): split the additional-fields sentence and date its claim Roborev found the three-clause sentence hard to parse and its "no tagged release yet" unanchored in time. Two sentences now, anchored to v3.1.0, with a note to pin the link once a release carries the heading. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/docs/models.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index 2e55df0a6d..d2a5eac37c 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -104,10 +104,12 @@ field is a declaration written as the member alone, `{key: value}`. Any other extra field is kept as written, as an `UnclaimedDeclaration`. The spec calls these members [additional fields](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#additional-fields), -the heading [PR 373](https://github.com/zarr-developers/zarr-specs/pull/373) -gave the section that v3.1.0 calls "Unknown" (no tagged release carries it -yet), and the names `extra_fields`, `ExtraFieldDefinition` and -`must_understand_fields` follow it. Everywhere else this package says +and the names `extra_fields`, `ExtraFieldDefinition` and +`must_understand_fields` follow it. That heading comes from +[PR 373](https://github.com/zarr-developers/zarr-specs/pull/373), which +renamed the section v3.1.0 calls [Unknown](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#unknown). +As of v3.1.0 no tagged release carries the new heading, so the link points +at the latest spec; pin it once a release does. Everywhere else this package says *member* for a name/value pair of a JSON object and *declaration* for the value at an extension point. It keeps *field* for the fields of a Python class, such as a dataclass or a pydantic model, and, after the spec and From 84438babe2b2edeccc77e58413dccf4611dad057 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 2 Nov 2026 12:23:58 +0100 Subject: [PATCH 8/9] docs(zarr-metadata): fix six wording and example issues roborev found on the guide pages The pydantic example now builds the manifest with model_validate, so it type-checks as the next sentence claims. The consolidated page says three other readers take a context, not two. The equality section quotes the v2 spec on a missing .zattrs in two sentences. The two numbered Sphinx anchors are replaced by line ranges in the spec source at the v3.1.0 tag. The landing page says once that the library does no chunk or store I/O. The extensions page example and the zarr_metadata.definition docstring example are pinned to each other by a test. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../docs/consolidated-metadata.md | 2 +- packages/zarr-metadata/docs/extensions.md | 4 +-- packages/zarr-metadata/docs/index.md | 11 ++++---- packages/zarr-metadata/docs/models.md | 8 +++--- .../zarr-metadata/docs/parsing-validation.md | 2 +- packages/zarr-metadata/docs/pydantic.md | 2 +- .../zarr-metadata/docs/validation-rules.md | 4 +-- packages/zarr-metadata/tests/test_docs.py | 25 +++++++++++++++++++ 8 files changed, 42 insertions(+), 16 deletions(-) diff --git a/packages/zarr-metadata/docs/consolidated-metadata.md b/packages/zarr-metadata/docs/consolidated-metadata.md index 1428c67be5..e7e10d3d96 100644 --- a/packages/zarr-metadata/docs/consolidated-metadata.md +++ b/packages/zarr-metadata/docs/consolidated-metadata.md @@ -60,7 +60,7 @@ opened, or read all of them with the group's `consolidated()`, with `ZarrV3ConsolidatedMetadata`, or with `read_consolidated_metadata_v3`, which returns problems instead of raising them. A document with a problem then fails only when it is read. `consolidated()` reads the documents in -the group's own context. The other two take a `context`; pass +the group's own context. The other three take a `context`; pass `context=group.context` to read the documents in the group's own context, and with no `context` they are read in `CORE_AND_EXTENSIONS_CONTEXT`. diff --git a/packages/zarr-metadata/docs/extensions.md b/packages/zarr-metadata/docs/extensions.md index f68da53a5a..357d29d493 100644 --- a/packages/zarr-metadata/docs/extensions.md +++ b/packages/zarr-metadata/docs/extensions.md @@ -48,10 +48,10 @@ ACME_LZ4 = CodecDefinition( size="dynamic", rules=acme_lz4_rules, ) -context = CORE_AND_EXTENSIONS_CONTEXT.update(codecs=[*CORE_AND_EXTENSIONS_CONTEXT.codecs.values(), ACME_LZ4]) +CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.update(codecs=[*CORE_AND_EXTENSIONS_CONTEXT.codecs.values(), ACME_LZ4]) ``` -Pass `context=context` to any reader or model constructor, and a document +Pass `context=CONTEXT` to any reader or model constructor, and a document that names `acme.lz4` is checked against this definition. Some rules for the pieces: diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md index a241a4f62d..50058a3041 100644 --- a/packages/zarr-metadata/docs/index.md +++ b/packages/zarr-metadata/docs/index.md @@ -77,12 +77,11 @@ Zarr v2 and v3 specs, consolidated metadata, and a subset of the metadata defined in zarr-extensions. We are generally open to contributions that add types, models, or validation for Zarr metadata with a published spec. -Runtime array behavior is out of the library's remit: nothing here encodes -or decodes chunks, resolves codec or data type names to implementations, -or performs store I/O. The models begin and end at the metadata documents -themselves. `from_key_value` and `to_key_value` map documents to store keys -and bytes, and everything past that belongs to consumer libraries. This -rule limits how far some codec combinations can be validated. For example, +The models begin and end at the metadata documents themselves: +`from_key_value` and `to_key_value` map documents to store keys and bytes, +and everything past that, including resolving a codec or data type name to +an implementation, belongs to consumer libraries. This boundary limits how +far some codec combinations can be validated. For example, the readers cannot check that a `cast_value` codec maps scalars one to one. ## Reference diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index d2a5eac37c..2deb6c2b36 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -146,9 +146,11 @@ kept whatever the `shuffle`, unlike in v3: v2 codecs are defined by numcodecs, which writes it into each chunk's header, so two v2 blosc codecs that differ only in it encode different bytes, though they decode the same data. The [v3 blosc spec](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/codecs/blosc/index.html#configuration-parameters) says `noshuffle` ignores `typesize`. A v2 node -with no `.zattrs` equals one with an empty `.zattrs`, as the -[v2 spec](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html#attributes) reads a missing `.zattrs`, -as a v3 document with and without an empty `attributes` are equal. +with no `.zattrs` equals one with an empty `.zattrs`: the +[v2 spec](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html#attributes) says the key +"does not have to be present, and if it is absent the attributes should be +treated as empty". In the same way, a v3 document with no `attributes` +equals one whose `attributes` is empty. Values the package does not interpret (attributes, extra fields, and the configuration of a declaration that nothing in the context claims) are diff --git a/packages/zarr-metadata/docs/parsing-validation.md b/packages/zarr-metadata/docs/parsing-validation.md index 8feee576b6..a60e6618c0 100644 --- a/packages/zarr-metadata/docs/parsing-validation.md +++ b/packages/zarr-metadata/docs/parsing-validation.md @@ -135,7 +135,7 @@ is a snippet of an invalid metadata document because the `shape` member applies The `fill_value` member is another example. The spec does not define a single shape for `fill_value`. Instead, each data type defines which JSON values it accepts as a fill value. So `"fill_value": "hello"` might be fine for a string data type, but it's a problem for `"float64"`. We can only check `fill_value` after we have resolved `data_type`. -The `codecs` member has the most interactions of all, because a list of codecs is a **pipeline**. [The spec](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#id22) divides codecs into three kinds: +The `codecs` member has the most interactions of all, because a list of codecs is a **pipeline**. [The spec](https://github.com/zarr-developers/zarr-specs/blob/v3.1.0/docs/v3/core/index.rst#L931-L1024) divides codecs into three kinds: - array → array codecs, like `transpose`, which take an array and return an array - array → bytes codecs, like `bytes`, which take an array and return bytes diff --git a/packages/zarr-metadata/docs/pydantic.md b/packages/zarr-metadata/docs/pydantic.md index a80a9f9207..3d573a4e9c 100644 --- a/packages/zarr-metadata/docs/pydantic.md +++ b/packages/zarr-metadata/docs/pydantic.md @@ -24,7 +24,7 @@ class ArrayManifest(BaseModel): metadata: zmp.ZarrV3ArrayMetadataField -manifest = ArrayManifest(path="a/b", metadata=raw) +manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": raw}) ``` A static type checker sees `manifest.metadata` as a diff --git a/packages/zarr-metadata/docs/validation-rules.md b/packages/zarr-metadata/docs/validation-rules.md index b023d1bb33..a70d4265b7 100644 --- a/packages/zarr-metadata/docs/validation-rules.md +++ b/packages/zarr-metadata/docs/validation-rules.md @@ -25,7 +25,7 @@ Some checks relate one part of a v3 document to another: - The [fill value](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#fill-value) is checked against the data type the document names, by that data type's definition. - The [chunk grid](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#chunk-grids) is checked against the shape, by the grid's definition. -- The codecs are checked as a [pipeline](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#id22). In order, each codec's definition +- The codecs are checked as a [pipeline](https://github.com/zarr-developers/zarr-specs/blob/v3.1.0/docs/v3/core/index.rst#L931-L1024). In order, each codec's definition checks the codec against the chunk it receives. This includes a [shard's](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/codecs/sharding-indexed/index.html#configuration-parameters) inner codecs and index codecs. @@ -54,7 +54,7 @@ Some choices that the spec text leaves open, or answers in two ways: - **`consolidated_metadata: null` is a problem.** A zarr-python 3.0.x bug wrote it. The spec [requires an object](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816), and the package accepts only an object. The [repair readers](repair.md) remove the key before reading. -- **An extension name must follow the spec's [naming rule](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#id31)**, +- **An extension name must follow the spec's [naming rule](https://github.com/zarr-developers/zarr-specs/blob/v3.1.0/docs/v3/core/index.rst#L1571-L1607)**, `^[a-z][a-z0-9-_.]+$`, or be a URI, which earlier versions of the spec required. Any other name is rejected before any definition is consulted, so `""` and `"foo/bar"` are problems, not unknown extensions. diff --git a/packages/zarr-metadata/tests/test_docs.py b/packages/zarr-metadata/tests/test_docs.py index 7726151126..6672347b2c 100644 --- a/packages/zarr-metadata/tests/test_docs.py +++ b/packages/zarr-metadata/tests/test_docs.py @@ -71,3 +71,28 @@ def test_every_python_block_says_whether_it_runs() -> None: ) ] assert unmarked == [] + + +def test_the_extensions_page_example_is_the_definition_module_docstring_example() -> None: + """The acme.lz4 example on the extensions page is, character for character, the example in the `zarr_metadata.definition` module docstring. Only the page's copy runs under these tests, so this is what keeps the docstring's copy from drifting.""" + import textwrap + + import zarr_metadata.definition + + page = next( + example + for example in _examples() + if example.path == PACKAGE_ROOT / "docs" / "extensions.md" + and example.prefix_settings().get("session") == "extensions" + ) + docstring = zarr_metadata.definition.__doc__ + assert docstring is not None + lines = docstring.splitlines() + start = next( + i for i, line in enumerate(lines) if line.strip() == "from collections.abc import Iterator" + ) + end = start + while end < len(lines) and (lines[end].startswith(" ") or lines[end].strip() == ""): + end += 1 + docstring_example = textwrap.dedent("\n".join(lines[start:end])).strip() + assert page.source.strip() == docstring_example From fa63627201a0d69604b7d47a8575a829c92a1918 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 2 Nov 2026 12:29:10 +0100 Subject: [PATCH 9/9] test(zarr-metadata): make the example-drift test say what it could not find Roborev asked for diagnostics: the test now asserts there is exactly one extensions-session block and exactly one marker line in the docstring, each with a message, and compares line lists so a mismatch points at the differing line. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/tests/test_docs.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/zarr-metadata/tests/test_docs.py b/packages/zarr-metadata/tests/test_docs.py index 6672347b2c..9c1d72122b 100644 --- a/packages/zarr-metadata/tests/test_docs.py +++ b/packages/zarr-metadata/tests/test_docs.py @@ -74,25 +74,28 @@ def test_every_python_block_says_whether_it_runs() -> None: def test_the_extensions_page_example_is_the_definition_module_docstring_example() -> None: - """The acme.lz4 example on the extensions page is, character for character, the example in the `zarr_metadata.definition` module docstring. Only the page's copy runs under these tests, so this is what keeps the docstring's copy from drifting.""" + """The acme.lz4 example on the extensions page is, line for line, the example in the `zarr_metadata.definition` module docstring. Only the page's copy runs under these tests, so this is what keeps the docstring's copy from drifting.""" import textwrap import zarr_metadata.definition - page = next( + page_blocks = [ example for example in _examples() if example.path == PACKAGE_ROOT / "docs" / "extensions.md" and example.prefix_settings().get("session") == "extensions" + ] + assert len(page_blocks) == 1, ( + "expected exactly one extensions-session block on docs/extensions.md" ) docstring = zarr_metadata.definition.__doc__ - assert docstring is not None + assert docstring is not None, "zarr_metadata.definition has no module docstring" lines = docstring.splitlines() - start = next( - i for i, line in enumerate(lines) if line.strip() == "from collections.abc import Iterator" - ) - end = start + marker = "from collections.abc import Iterator" + starts = [i for i, line in enumerate(lines) if line.strip() == marker] + assert len(starts) == 1, f"expected one line {marker!r} in the docstring, found {len(starts)}" + end = starts[0] while end < len(lines) and (lines[end].startswith(" ") or lines[end].strip() == ""): end += 1 - docstring_example = textwrap.dedent("\n".join(lines[start:end])).strip() - assert page.source.strip() == docstring_example + docstring_example = textwrap.dedent("\n".join(lines[starts[0] : end])).strip() + assert page_blocks[0].source.strip().splitlines() == docstring_example.splitlines()