feat(zarr-metadata): export base64_fill_value_rules for data type definitions

packages/zarr-metadata/changes/428.feature.md

@@ -0,0 +1 @@
+Export `base64_fill_value_rules` from `zarr_metadata.v3.data_type.bytes`, the rule that rejects a fill value string that is not standard-alphabet base64, so a data type defined outside the package can pass it as `fill_value_rules` for a `Base64Bytes` fill value, or yield from it in a rule of its own. The guide to parsing and validation now shows how to define a data type with `fill_value_rules`.

packages/zarr-metadata/docs/parsing-validation.md

@@ -227,4 +227,50 @@ reading = read_array_metadata_v3(raw, context=context)
 
 `update` replaces each slot it is given and keeps the others. A slot can also be given as a mapping from each definition's name to it, so `CORE_AND_EXTENSIONS_CONTEXT.update(codecs={**CORE_AND_EXTENSIONS_CONTEXT.codecs, "gzip": MY_GZIP})` replaces the gzip definition with your own.
 
+A data type's definition also says what its fill value may be. `fill_value` is the fill value's JSON shape, as a type annotation. `fill_value_rules` is a function of the configuration, the nested declarations and the fill value, that yields each problem the shape cannot express. For example, `Base64Bytes` is a string, so an annotation accepts any string, and the rule `base64_fill_value_rules` rejects one that is not base64. A rule of your own can yield from it, then check what depends on the configuration:
+
+```python test="true" session="parsing-validation"
+import base64
+
+from typing_extensions import TypedDict
+
+from zarr_metadata.definition import fill_value_problems
+from zarr_metadata.problems import ValidationProblem
+from zarr_metadata.v3.data_type.bytes import Base64Bytes, base64_fill_value_rules
+from zarr_metadata.v3.definition import DataTypeDefinition
+
+
+class PackedConfiguration(TypedDict, closed=True):
+    length_bytes: int
+
+
+def packed_fill_value_rules(configuration, nested, value):
+    # The fill value is base64, of at most `length_bytes` bytes.
+    problems = list(base64_fill_value_rules(configuration, nested, value))
+    yield from problems
+    if not problems and len(base64.b64decode(value)) > configuration["length_bytes"]:
+        yield ValidationProblem((), "expected fewer bytes than the type holds", "invalid_value")
+
+
+PACKED = DataTypeDefinition(
+    name="acme.packed",
+    configuration=PackedConfiguration,
+    fill_value=Base64Bytes,
+    fill_value_rules=packed_fill_value_rules,
+)
+context = CORE_AND_EXTENSIONS_CONTEXT.update(
+    data_types=[*CORE_AND_EXTENSIONS_CONTEXT.data_types.values(), PACKED]
+)
+data_type, _ = resolve(
+    {"name": "acme.packed", "configuration": {"length_bytes": 2}}, DataTypeDefinition, context
+)
+for fill_value in ("YWI=", "not base64!", "YWJj"):
+    print(fill_value, [problem.message for problem in fill_value_problems(data_type, fill_value)])
+# YWI= []
+# not base64! ['expected standard-alphabet base64, got "not base64!"']
+# YWJj ['expected fewer bytes than the type holds']
+```
+
+A reader of a document in this scope checks its fill value with these rules, and so does a pydantic field. A JSON Schema written for the scope does not contain the rules, so it accepts any string here.
+
 [^1]: The one exception is the `r*` data type, which is identified by a templated string with no `configuration` object. This mixes the identifier with the configuration in the same JSON value, which is not good design. So this library normalizes `r*` into the same form as all the other metadata objects: the declaration `"r16"` keeps its name, and is read by the `r*` definition with the configuration `{"bits": 16}`.

packages/zarr-metadata/src/zarr_metadata/v3/data_type/__init__.py

@@ -37,6 +37,7 @@
     BytesDataTypeName,
     BytesFillValue,
     base64_bytes,
+    base64_fill_value_rules,
 )
 from zarr_metadata.v3.data_type.complex64 import (
     COMPLEX64_DATA_TYPE,
@@ -338,6 +339,7 @@
     "ZarrPythonRawBytesDataTypeName",
     "ZarrPythonRawBytesFillValue",
     "base64_bytes",
+    "base64_fill_value_rules",
     "hex_float16",
     "hex_float32",
     "hex_float64",

packages/zarr-metadata/src/zarr_metadata/v3/data_type/_byte.py

@@ -28,6 +28,23 @@ def base64_problems(value: str) -> Iterator[ValidationProblem]:
         )
 
 
+def base64_fill_value_rules(
+    configuration: object, nested: object, value: object
+) -> Iterator[ValidationProblem]:
+    """Check that a fill value written as a string is standard-alphabet base64: a data type's `fill_value_rules`.
+
+    `Base64Bytes` is a `NewType` of `str`, so a fill value annotated with
+    it accepts any string. Pass this rule as the data type's
+    `fill_value_rules` to reject a string that is not base64. A value of
+    any other shape is not checked here, so the rule also serves a fill
+    value that may be written another way, such as
+    `tuple[ByteValue, ...] | Base64Bytes`. A rule that checks more, such
+    as the number of bytes, yields from this one first.
+    """
+    if isinstance(value, str):
+        yield from base64_problems(value)
+
+
 def base64_decoded(value: str) -> bytes | None:
     """The bytes that `value` encodes in standard-alphabet base64; None when it is not such base64."""
     try:
@@ -41,4 +58,10 @@ def base64_spelling(data: bytes) -> str:
     return base64.b64encode(data).decode("ascii")
 
 
-__all__ = ["ByteValue", "base64_decoded", "base64_problems", "base64_spelling"]
+__all__ = [
+    "ByteValue",
+    "base64_decoded",
+    "base64_fill_value_rules",
+    "base64_problems",
+    "base64_spelling",
+]

packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py

@@ -5,10 +5,8 @@
 """
 
 import base64
-from collections.abc import Iterator
 from typing import Final, Literal
 
-from zarr_metadata._json import ValidationProblem
 from zarr_metadata._numpy import Base64Bytes, base64_bytes
 from zarr_metadata.definition._definition import (
     EmptyConfiguration,
@@ -18,7 +16,11 @@
     DataTypeDefinition,
     variable_length,
 )
-from zarr_metadata.v3.data_type._byte import ByteValue, base64_problems, base64_spelling
+from zarr_metadata.v3.data_type._byte import (
+    ByteValue,
+    base64_fill_value_rules,
+    base64_spelling,
+)
 
 BYTES_DATA_TYPE_NAME: Final = "bytes"
 """The `data_type` value for the variable-length `bytes` type."""
@@ -34,14 +36,6 @@
 """
 
 
-def _fill_value_rules(
-    configuration: EmptyConfiguration, nested: Nested, value: BytesFillValue
-) -> Iterator[ValidationProblem]:
-    """Check that a string fill value is standard-alphabet base64. A fill value written as byte values is not checked here."""
-    if isinstance(value, str):
-        yield from base64_problems(value)
-
-
 def _fill_value_canonical(
     configuration: EmptyConfiguration, nested: Nested, value: BytesFillValue
 ) -> str:
@@ -60,7 +54,7 @@ def _fill_value_canonical(
     name=BYTES_DATA_TYPE_NAME,
     configuration=EmptyConfiguration,
     fill_value=BytesFillValue,
-    fill_value_rules=_fill_value_rules,
+    fill_value_rules=base64_fill_value_rules,
     fill_value_canonical=_fill_value_canonical,
     storage=variable_length,
 )
@@ -79,4 +73,5 @@ def _fill_value_canonical(
     "BytesDataTypeName",
     "BytesFillValue",
     "base64_bytes",
+    "base64_fill_value_rules",
 ]

packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py

@@ -10,14 +10,12 @@
 """
 
 import base64
-from collections.abc import Iterator
 from typing import Final, Literal
 
-from zarr_metadata._json import ValidationProblem
 from zarr_metadata._numpy import Base64Bytes
 from zarr_metadata.definition._definition import EmptyConfiguration, Nested
 from zarr_metadata.v3._definition import DataTypeDefinition, variable_length
-from zarr_metadata.v3.data_type._byte import base64_problems, base64_spelling
+from zarr_metadata.v3.data_type._byte import base64_fill_value_rules, base64_spelling
 
 VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME: Final = "variable_length_bytes"
 """The `data_type` value for the `variable_length_bytes` type."""
@@ -29,13 +27,6 @@
 """Permitted JSON shape of the `fill_value` field for `variable_length_bytes`: a `Base64Bytes` string, which is what zarr-python writes and reads."""
 
 
-def _fill_value_rules(
-    configuration: EmptyConfiguration, nested: Nested, value: VariableLengthBytesFillValue
-) -> Iterator[ValidationProblem]:
-    """Check that the fill value is standard-alphabet base64."""
-    yield from base64_problems(value)
-
-
 def _fill_value_canonical(
     configuration: EmptyConfiguration, nested: Nested, value: VariableLengthBytesFillValue
 ) -> str:
@@ -47,7 +38,7 @@ def _fill_value_canonical(
     name=VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME,
     configuration=EmptyConfiguration,
     fill_value=VariableLengthBytesFillValue,
-    fill_value_rules=_fill_value_rules,
+    fill_value_rules=base64_fill_value_rules,
     fill_value_canonical=_fill_value_canonical,
     storage=variable_length,
 )

packages/zarr-metadata/tests/v3/data_type/bytes/test_validators.py

@@ -1,16 +1,16 @@
-"""Cover the `base64_bytes` brand validator.
+"""Cover the `base64_bytes` brand validator and the `base64_fill_value_rules` fill value rule.
 
 The pydantic-driven fixture tests don't enforce the base64 alphabet or
 length-multiple-of-4 constraint because `Base64Bytes` is a `NewType`,
 which pydantic treats as plain `str`. Direct coverage of the validator
-function lives here.
+function and the rule built on it lives here.
 """
 
 from __future__ import annotations
 
 import pytest
 
-from zarr_metadata.v3.data_type.bytes import base64_bytes
+from zarr_metadata.v3.data_type.bytes import base64_bytes, base64_fill_value_rules
 
 VALID = [
     "",  # empty is valid base64 (length 0, multiple of 4)
@@ -41,3 +41,16 @@ def test_valid(value: str) -> None:
 def test_invalid(value: str) -> None:
     with pytest.raises(ValueError, match="standard-alphabet base64"):
         base64_bytes(value)
+
+
+# A value written in another shape, such as byte values, is the annotation's to check.
+@pytest.mark.parametrize("value", [*VALID, (1, 2, 3), None])
+def test_base64_fill_value_rules_accepts(value: object) -> None:
+    assert list(base64_fill_value_rules({}, (), value)) == []
+
+
+@pytest.mark.parametrize("value", INVALID)
+def test_error_base64_fill_value_rules_rejects_a_string_that_is_not_base64(value: str) -> None:
+    assert [(p.loc, p.kind) for p in base64_fill_value_rules({}, (), value)] == [
+        ((), "invalid_value")
+    ]