From 139d2c1546f61382770a718312cb496839151680 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Fri, 6 Nov 2026 13:32:30 +0100 Subject: [PATCH 1/2] feat(zarr-metadata): add a reader that reports what a reader cannot open `read_openable_node_metadata_v3` reads a v3 `zarr.json` as `read_node_metadata_v3` does and adds a problem of the new kind `unclaimed_declaration` for each declaration a reader must understand that nothing in scope claims: an unknown codec, data type or chunk grid, or an unknown top-level member without `"must_understand": false`. Its reading has no problems exactly when a reader that understands the scope can open the node. The other readers keep accepting unclaimed declarations, so the format stays open; their docstrings and the parsing guide now point to `unclaimed_declarations` and the new reader. Prompted by refkv's report that an unknown v3 member parses without error. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../changes/+openable-reader.feature.md | 1 + .../zarr-metadata/docs/parsing-validation.md | 24 +++++- .../zarr-metadata/src/zarr_metadata/_json.py | 13 +++- .../src/zarr_metadata/model/__init__.py | 9 ++- .../src/zarr_metadata/model/_array_v3.py | 14 +++- .../src/zarr_metadata/model/_group_v3.py | 46 ++++++++++- .../model/test_unclaimed_declarations.py | 78 +++++++++++++++++++ 7 files changed, 174 insertions(+), 11 deletions(-) create mode 100644 packages/zarr-metadata/changes/+openable-reader.feature.md diff --git a/packages/zarr-metadata/changes/+openable-reader.feature.md b/packages/zarr-metadata/changes/+openable-reader.feature.md new file mode 100644 index 0000000000..21ba7def2d --- /dev/null +++ b/packages/zarr-metadata/changes/+openable-reader.feature.md @@ -0,0 +1 @@ +`zarr_metadata.model.read_openable_node_metadata_v3` reads a v3 `zarr.json` as `read_node_metadata_v3` does, and also reports each declaration a reader must understand to open the node that nothing in scope claims, such as an unknown codec or an unknown top-level member without `"must_understand": false`, as a problem of the new kind `unclaimed_declaration`. Its reading has no problems exactly when a reader that understands the scope can open the node, so a validator does not have to check `unclaimed_declarations` itself. The other readers still accept unclaimed declarations, and the `from_json` and reader docstrings now point to `unclaimed_declarations` and the new reader. diff --git a/packages/zarr-metadata/docs/parsing-validation.md b/packages/zarr-metadata/docs/parsing-validation.md index 1be626d5ee..22ad32a6a8 100644 --- a/packages/zarr-metadata/docs/parsing-validation.md +++ b/packages/zarr-metadata/docs/parsing-validation.md @@ -203,11 +203,29 @@ for problem in problems: # ('configuration', 'level') invalid_value ``` -A declaration with an unknown name is called **unclaimed**, because no definition in the scope claims that name. It is not reported as a problem: it is read as an `UnclaimedDeclaration`, and the checks that depend on it are skipped, as described above. So an empty `reading.problems` does not mean that every declaration is known. A program that decodes chunks should also check that no declaration is unclaimed. A model's `unclaimed_declarations` lists each one, at any depth, by its location, and leaves out an extra field that says `"must_understand": false`, with everything in it, since a reader may ignore it. A reading with problems has no model, and a node with problems cannot be opened either: +A declaration with an unknown name is called **unclaimed**, because no definition in the scope claims that name. It is not reported as a problem: it is read as an `UnclaimedDeclaration`, and the checks that depend on it are skipped, as described above. So an empty `reading.problems` does not mean that every declaration is known. A program that decodes chunks should also check that no declaration is unclaimed. A model's `unclaimed_declarations` lists each one, at any depth, by its location, and leaves out an extra field that says `"must_understand": false`, with everything in it, since a reader may ignore it. To report each of these as a problem too, read the document with `read_openable_node_metadata_v3`. Its problems are those of `read_node_metadata_v3`, then one of kind `unclaimed_declaration` for each declaration a reader must understand that nothing in scope claims. So its reading has no problems exactly when a reader that understands the scope can open the node: ```python test="true" session="parsing-validation" -unclaimed = None if reading.metadata is None else reading.metadata.unclaimed_declarations -can_open = unclaimed is not None and len(unclaimed) == 0 +from zarr_metadata.model import read_openable_node_metadata_v3 + +openable = read_openable_node_metadata_v3( + { + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": "uint8", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [2]}}, + "chunk_key_encoding": {"name": "default"}, + "fill_value": 0, + "codecs": [{"name": "bytes"}, {"name": "my_codec"}], + "my_member": {"name": "my_member"}, + "ignorable": {"must_understand": False}, + } +) +for problem in openable.problems: + print(problem.loc, problem.kind) +# ('codecs', 1) unclaimed_declaration +# ('my_member',) unclaimed_declaration ``` To make an unknown data type or codec known, add its definition to the scope's slot and read the document again: diff --git a/packages/zarr-metadata/src/zarr_metadata/_json.py b/packages/zarr-metadata/src/zarr_metadata/_json.py index 69829d32f4..d5de1537a2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_json.py @@ -20,7 +20,14 @@ from zarr_metadata._common import JSONValue from zarr_metadata._sentinel import UNSET -ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json", "unknown_key"] +ProblemKind = Literal[ + "missing_key", + "invalid_type", + "invalid_value", + "invalid_json", + "unknown_key", + "unclaimed_declaration", +] """Machine-readable classification of a `ValidationProblem`. - `missing_key`: a required key (document key or store key) is absent. @@ -35,6 +42,10 @@ open since 2023), and many readers reject such a key. It has its own kind so that a caller who tolerates it can distinguish it from a wrong value, and so that it never hides other problems in the same object. +- `unclaimed_declaration`: a declaration that nothing in scope claims, which + a reader must understand to open the node. Only + `read_openable_node_metadata_v3` reports it: the other readers accept an + unclaimed declaration, since that is what keeps the format open. """ diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 0228861744..2501347273 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -25,7 +25,12 @@ `read_repaired_node_metadata_v3` or `read_repaired_node_metadata_v2`. They undo each bug with `repair_node_metadata_v3` or `repair_node_metadata_v2` before the strict read, and report what they -changed. `zarr_metadata.json_schema` writes a JSON Schema of what the v3 +changed. A reader accepts a declaration that nothing in its scope claims +(an unknown codec, or an unknown top-level member), and a model lists each +one a reader must understand in `unclaimed_declarations`. To also report +each one as a problem, so that a reading has no problems exactly when a +reader can open the node, use `read_openable_node_metadata_v3`. +`zarr_metadata.json_schema` writes a JSON Schema of what the v3 readers read, except for the rules. One declaration on its own is read with `zarr_metadata.definition.resolve`. @@ -91,6 +96,7 @@ read_consolidated_metadata_v3, read_group_metadata_v3, read_node_metadata_v3, + read_openable_node_metadata_v3, ) from zarr_metadata.model._repair import ( Repair, @@ -160,6 +166,7 @@ "read_group_metadata_v3", "read_node_metadata_v2", "read_node_metadata_v3", + "read_openable_node_metadata_v3", "read_repaired_consolidated_metadata_v2", "read_repaired_node_metadata_v2", "read_repaired_node_metadata_v3", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 061819fb3d..fa175ed005 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -443,14 +443,14 @@ def must_understand_subset( } -def waived_members(values: Mapping[str, ZarrV3ExtensionField]) -> list[str]: +def waived_members(values: Mapping[str, object]) -> list[str]: """The keys of the extra fields in `values` (by key, as written) that say `"must_understand": false`, which a reader may ignore.""" return [key for key, value in values.items() if _waived(value)] -def _waived(value: ZarrV3ExtensionField) -> bool: +def _waived(value: object) -> bool: """Whether `value`, an extra field as written, is an object that says `"must_understand": false`.""" - return isinstance(value, Mapping) and value.get("must_understand") is False + return is_object(value) and value.get("must_understand") is False class ZarrV3ArrayMetadataUpdate(TypedDict, total=False, closed=True): @@ -835,6 +835,10 @@ def from_json( Raises `MetadataValidationError` with every problem the read finds. `read_array_metadata_v3` returns the reading this model is built from, and the problems of a document that has any. + A declaration that nothing in `context` claims, such as an unknown + codec or an unknown top-level member, is not a problem; the model + lists each one a reader must understand in `unclaimed_declarations`. + `read_openable_node_metadata_v3` reports each one as a problem. """ return cls(data, context=context) @@ -919,7 +923,9 @@ def read_array_metadata_v3( type. Unknown top-level keys are allowed (they map to `extra_fields`). A reader must understand each one that does not declare `must_understand: false`; the model reports these as - `must_understand_fields`. + `must_understand_fields`, and every declaration a reader must + understand, at any depth, as `unclaimed_declarations`. + `read_openable_node_metadata_v3` reports each of those as a problem. To apply a policy over the declarations (for example, allow only the core spec's declarations), iterate over the reading's `declarations()`. A value that diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index 42f3e61357..82aad927ab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -450,7 +450,10 @@ def from_json( scope that claims the member, as `CORE_CONTEXT` does, this is a problem, because the spec requires an object. Use `read_repaired_node_metadata_v3` to read such a store. A member the - spec does not define is kept in `extra_fields`. + spec does not define is kept in `extra_fields`, and is not a + problem when nothing in `context` claims it; the model lists each + such member a reader must understand in `unclaimed_declarations`. + `read_openable_node_metadata_v3` reports each one as a problem. """ return cls(data, context=context) @@ -769,6 +772,44 @@ def read_node_metadata_v3( return ZarrV3UnknownNodeMetadataReading(problems) +def read_openable_node_metadata_v3( + value: object, *, context: ZarrV3Context | None = None +) -> ZarrV3NodeMetadataReading: + """Read `value`, a v3 `zarr.json`, in `context` as `read_node_metadata_v3` does, and also report each declaration a reader must understand to open the node that nothing in `context` claims. + + The spec requires a reader to refuse a node with a declaration it does + not understand: an unknown codec, data type or chunk grid, or an + unknown top-level member that does not say `"must_understand": false`. + `read_node_metadata_v3` accepts these, as `UnclaimedDeclaration`s, + since that is what keeps the format open. This reader reports each one + as a problem of kind `unclaimed_declaration`, at its location, and + gives no model when there is one. A member that says + `"must_understand": false` is ignored whole, with every declaration in + it, as `unclaimed_declarations` ignores it. So the reading has no + problems exactly when a reader that understands `context` can open the + node. The documents a group's `consolidated_metadata` lists are not + read, as `read_group_metadata_v3` does not read them. + """ + reading = read_node_metadata_v3(value, context=context) + # A node reading is only of an array or a group when `value` is an object. + if isinstance(reading, ZarrV3UnknownNodeMetadataReading) or not is_object(value): + return reading + waived = waived_members({key: value[key] for key in reading.extra_fields}) + unclaimed = [ + ValidationProblem( + loc, + f"nothing in scope claims the {kind_name(declaration.read_as)} {declaration.name!r}, " + "which a reader must understand to open the node", + "unclaimed_declaration", + ) + for loc, declaration in unclaimed_declarations_of(reading.declarations(), waived).items() + ] + if len(unclaimed) == 0: + return reading + problems = (*reading.problems, *with_input(unclaimed, value)) + return dataclasses.replace(reading, problems=problems, metadata=None) + + ZarrV3NodeMetadata = TypeAliasType("ZarrV3NodeMetadata", ZarrV3ArrayMetadata | ZarrV3GroupMetadata) """The model of a v3 `zarr.json`: an array model or a group model, as its `node_type` declares.""" @@ -875,7 +916,8 @@ def read_group_metadata_v3( Unknown top-level keys are allowed (they map to `extra_fields`). A reader must understand each one that does not declare `must_understand: false`; the model reports these as - `must_understand_fields`. A `consolidated_metadata` member, if + `must_understand_fields`, and `read_openable_node_metadata_v3` reports + each one as a problem. A `consolidated_metadata` member, if present and claimed by the scope, as `CORE_CONTEXT` claims it, is checked against the envelope the core spec fixes. The documents it lists are not read; `read_consolidated_metadata_v3` reads them. This is the diff --git a/packages/zarr-metadata/tests/model/test_unclaimed_declarations.py b/packages/zarr-metadata/tests/model/test_unclaimed_declarations.py index 1650899d8e..9148e7ae2d 100644 --- a/packages/zarr-metadata/tests/model/test_unclaimed_declarations.py +++ b/packages/zarr-metadata/tests/model/test_unclaimed_declarations.py @@ -7,6 +7,7 @@ import pytest from typing_extensions import TypedDict +from zarr_metadata import UNSET from zarr_metadata.definition import UnclaimedDeclaration from zarr_metadata.model import ( ZarrV2ArrayMetadata, @@ -15,6 +16,8 @@ ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ZarrV3NodeMetadata, + read_node_metadata_v3, + read_openable_node_metadata_v3, ) from zarr_metadata.v3 import CORE_CONTEXT from zarr_metadata.v3.definition import ( @@ -187,3 +190,78 @@ def test_node_type_tells_a_type_checker_which_model_a_node_is() -> None: for node in nodes: if node.zarr_format == 3 and node.node_type == "group": assert node.consolidated() is None + + +@pytest.mark.parametrize( + ("document", "context", "unclaimed", "problems"), + [ + (ARRAY, CORE_CONTEXT, set(), set()), + ({**ARRAY, "codecs": ("bytes", ZSTD)}, CORE_CONTEXT, {("codecs", 1)}, set()), + ( + {**ARRAY, "codecs": (SHARD,)}, + CORE_CONTEXT, + {("codecs", 0, "configuration", "codecs", 1)}, + set(), + ), + ({**ARRAY, "acme": 1}, CORE_CONTEXT, {("acme",)}, set()), + ({**ARRAY, "acme": {"must_understand": False}}, CORE_CONTEXT, set(), set()), + ({**GROUP, "acme": {"name": "acme"}}, CORE_CONTEXT, {("acme",)}, set()), + ({**GROUP, "acme": {"must_understand": False}}, CORE_CONTEXT, set(), set()), + ( + {**ARRAY, "acme.mirror": {"codec": {"name": "acme.zip"}}}, + MIRRORING, + {("acme.mirror", "codec")}, + set(), + ), + ( + {**ARRAY, "acme.mirror": {"codec": {"name": "acme.zip"}, "must_understand": False}}, + MIRRORING, + set(), + set(), + ), + ( + {**ARRAY, "fill_value": "x", "codecs": ("bytes", ZSTD)}, + CORE_CONTEXT, + {("codecs", 1)}, + {("fill_value",)}, + ), + ({**GROUP, "node_type": "acme"}, CORE_CONTEXT, set(), {("node_type",)}), + ("not a node", CORE_CONTEXT, set(), {()}), + ], + ids=[ + "all-claimed", + "a-codec", + "a-codec-inside-a-shard", + "an-extra-member", + "a-waived-extra-member", + "a-group-member", + "a-waived-group-member", + "a-codec-inside-a-claimed-member", + "a-codec-inside-a-claimed-member-that-waives", + "with-other-problems", + "an-unknown-node-type", + "not-an-object", + ], +) +def test_read_openable_node_metadata_v3_reports_what_a_reader_must_understand( + document: object, + context: Any, # noqa: ANN401 - a v3 scope + unclaimed: set[tuple[str | int, ...]], + problems: set[tuple[str | int, ...]], +) -> None: + """`read_openable_node_metadata_v3` keeps every problem `read_node_metadata_v3` finds and adds one of kind `unclaimed_declaration` at the location of each declaration `unclaimed_declarations` lists, with what the document writes there as its input, whether or not the document has other problems. It gives a model exactly when there are no problems, and the model is the one `read_node_metadata_v3` gives.""" + plain = read_node_metadata_v3(document, context=context) + reading = read_openable_node_metadata_v3(document, context=context) + assert reading.problems[: len(plain.problems)] == plain.problems + assert {problem.loc for problem in plain.problems} == problems + added = reading.problems[len(plain.problems) :] + assert {problem.loc for problem in added} == unclaimed + assert all(problem.kind == "unclaimed_declaration" for problem in added) + assert all(problem.input is not UNSET for problem in added) + if plain.metadata is not None: + assert set(plain.metadata.unclaimed_declarations) == unclaimed + if len(reading.problems) == 0: + assert reading.metadata == plain.metadata + assert reading.metadata is not None + else: + assert reading.metadata is None From ad1ffa72f52f22b0430a221ce52b6f18f2cc4236 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Fri, 6 Nov 2026 13:41:25 +0100 Subject: [PATCH 2/2] docs(zarr-metadata): number the changelog fragment for #440 Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../changes/{+openable-reader.feature.md => 440.feature.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{+openable-reader.feature.md => 440.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/+openable-reader.feature.md b/packages/zarr-metadata/changes/440.feature.md similarity index 100% rename from packages/zarr-metadata/changes/+openable-reader.feature.md rename to packages/zarr-metadata/changes/440.feature.md