perf(zarr-indexing): skip the sort in _last_occurrences for sorted indices

.github/workflows/check_changelogs.yml

@@ -22,7 +22,7 @@ jobs:
           persist-credentials: false
 
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
 
       - name: Install just
         uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1

.github/workflows/docs.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
-      - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+      - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
       - name: Install just
         uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1
         with:

.github/workflows/downstream.yml

@@ -45,7 +45,7 @@ jobs:
           python-version: '3.13'
 
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
 
       - name: Install xarray and test dependencies
         working-directory: xarray
@@ -106,7 +106,7 @@ jobs:
           python-version: '3.13'
 
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
 
       - name: Install numcodecs with test-zarr-main group
         working-directory: numcodecs

.github/workflows/gpu_test.yml

@@ -62,7 +62,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'
     - name: Install uv
-      uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+      uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
     - name: Install Hatch
       uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60
       with:

.github/workflows/hypothesis.yaml

@@ -60,7 +60,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'
     - name: Install uv
-      uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+      uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
     - name: Install Hatch
       uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60
       with:

.github/workflows/lint.yml

@@ -27,7 +27,7 @@ jobs:
         with:
           python-version: "3.12"
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
         with:
           enable-cache: true
       - name: Install just

.github/workflows/nightly_wheels.yml

@@ -42,7 +42,7 @@ jobs:
         run: hatch build
 
       - name: Upload nightly wheels
-        uses: scientific-python/upload-nightly-action@16fa02eacee1655195143de09f03676e60ef2bf5
+        uses: scientific-python/upload-nightly-action@5a74a2a4ec19d7ab8b5317c1cdd356c150d92cd0
         with:
           artifacts_path: dist
           anaconda_nightly_upload_token: ${{ secrets.ANACONDA_ORG_UPLOAD_TOKEN }}

.github/workflows/test.yml

@@ -66,7 +66,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'
     - name: Install uv
-      uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+      uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
     - name: Install Hatch
       run: python -m pip install hatch==1.16.5
     - name: Install just
@@ -119,7 +119,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'
     - name: Install uv
-      uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+      uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
     - name: Install Hatch
       run: python -m pip install hatch==1.16.5
     - name: Install just
@@ -158,7 +158,7 @@ jobs:
         python-version: '3.13'
         cache: 'pip'
     - name: Install uv
-      uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+      uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
     - name: Install Hatch
       run: python -m pip install hatch==1.16.5
     - name: Install just
@@ -187,7 +187,7 @@ jobs:
         python-version: '3.13'
         cache: 'pip'
     - name: Install uv
-      uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+      uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
     - name: Install Hatch
       run: python -m pip install hatch==1.16.5
     - name: Install just
@@ -215,7 +215,7 @@ jobs:
         python-version: '3.13'
         cache: 'pip'
     - name: Install uv
-      uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+      uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
     - name: Install Hatch
       run: python -m pip install hatch==1.16.5
     - name: Install just

.github/workflows/zarr-http-server-release.yml

@@ -51,7 +51,7 @@ jobs:
           path: dist
 
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
         with:
           enable-cache: false
 

.github/workflows/zarr-http-server.yml

@@ -53,7 +53,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
         with:
           enable-cache: true
       - name: Set up Python ${{ matrix.python-version }}
@@ -83,7 +83,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
       - name: Install just
         uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1
         with:
@@ -104,7 +104,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
         with:
           enable-cache: true
       - name: Set up Python
@@ -131,7 +131,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
         with:
           enable-cache: true
       - name: Install just

.github/workflows/zarr-indexing-release.yml

@@ -51,7 +51,7 @@ jobs:
           path: dist
 
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
         with:
           enable-cache: false
 

.github/workflows/zarr-indexing.yml

@@ -46,7 +46,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
         with:
           enable-cache: true
       - name: Install just
@@ -86,7 +86,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
       - name: Install just
         uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1
         with:
@@ -108,7 +108,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
         with:
           enable-cache: true
       - name: Set up Python
@@ -136,7 +136,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
         with:
           enable-cache: true
       - name: Install just

.github/workflows/zarr-metadata-release.yml

@@ -51,7 +51,7 @@ jobs:
           path: dist
 
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
         with:
           enable-cache: false
 

.github/workflows/zarr-metadata.yml

@@ -49,7 +49,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
         with:
           enable-cache: true
       - name: Install just
@@ -76,7 +76,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
       - name: Install just
         uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1
         with:
@@ -97,7 +97,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
         with:
           enable-cache: true
       - name: Install just
@@ -121,7 +121,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Install uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
         with:
           enable-cache: true
       - name: Install just

.github/workflows/zizmor.yml

@@ -32,4 +32,4 @@ jobs:
           persist-credentials: false
 
       - name: Run zizmor
-        uses: zizmorcore/zizmor-action@85b2b9fb8e3325e7a23d7d0a9d7ba5ae0f999e3e # v0.7.1
+        uses: zizmorcore/zizmor-action@53290badfa369cb0bf7ad16342cb62c2fa47ac7b # v0.8.0

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ default_language_version:
 
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.16.10
+    rev: v0.17.3
     hooks:
       - id: ruff-check
         args: ["--fix", "--show-fixes"]
@@ -34,7 +34,7 @@ repos:
         exclude: mkdocs.yml
       - id: trailing-whitespace
   - repo: https://github.com/DavidAnson/markdownlint-cli2
-    rev: v0.23.3
+    rev: v0.24.0
     hooks:
       # Markdown structure/hygiene. Rule selection and ignores are in
       # .markdownlint-cli2.jsonc; complements ci/lint_docs.py (RST residue,
@@ -89,7 +89,7 @@ repos:
         types: [python]
         files: ^(src|tests)/
   - repo: https://github.com/zizmorcore/zizmor-pre-commit
-    rev: v1.30.1
+    rev: v1.31.1
     hooks:
       - id: zizmor
   - repo: https://github.com/twisted/towncrier

packages/zarr-indexing/changes/4603.misc.md

@@ -0,0 +1 @@
+Writes whose indices land on a chunk in sorted order no longer sort them to drop repeated indices, so deduplication is linear in the number of points; writes without repeats also skip a gather copy.

packages/zarr-indexing/src/zarr_indexing/writer.py

@@ -167,8 +167,22 @@ def _domain_cells(shape: tuple[int, ...]) -> np.ndarray[Any, np.dtype[np.intp]]:
     return np.indices(shape, dtype=np.intp).reshape(len(shape), -1).T
 
 
-def _last_occurrences(flat: np.ndarray[Any, Any]) -> np.ndarray[Any, np.dtype[np.intp]]:
-    """Positions to keep so that repeated flat indices take their last value."""
+def _last_occurrences(
+    flat: np.ndarray[Any, Any],
+) -> slice | np.ndarray[Any, np.dtype[np.intp]]:
+    """Positions to keep so that repeated flat indices take their last value.
+
+    A full slice means every position is kept, so callers index without a copy.
+    """
+    if flat.size < 2:
+        return slice(None)
+    if (flat[1:] >= flat[:-1]).all():
+        # Non-decreasing input, the usual C-order case: repeats are adjacent,
+        # so no sort is needed.
+        steps = flat[1:] != flat[:-1]
+        if steps.all():
+            return slice(None)
+        return np.flatnonzero(np.append(steps, True))
     _, first_in_reversed = np.unique(flat[::-1], return_index=True)
     return np.asarray(len(flat) - 1 - first_in_reversed, dtype=np.intp)
 

packages/zarr-indexing/tests/test_writer.py

@@ -460,3 +460,31 @@ def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None:
     view.write([7, 8, 9])
     np.testing.assert_array_equal(source, [0, 7, 8, 9, 4, 5])
     np.testing.assert_array_equal(view.result(), [1, 2, 3])
+
+
+@pytest.mark.parametrize(
+    "flat",
+    [
+        [],
+        [3],
+        [0, 2, 5, 7],
+        [0, 0, 2, 5, 5, 5, 7],
+        [5, 0, 5, 2, 0, 7],
+        [7, 5, 5, 2, 0],
+    ],
+    ids=["empty", "single", "sorted", "sorted_repeats", "unsorted_repeats", "reversed"],
+)
+def test_last_occurrences_matches_sequential_assignment(flat: list[int]) -> None:
+    from zarr_indexing.writer import _last_occurrences
+
+    indices = np.asarray(flat, dtype=np.intp)
+    values = np.arange(1, indices.size + 1)
+    expected = np.zeros(8, dtype=values.dtype)
+    for index, value in zip(indices, values, strict=True):
+        expected[index] = value
+    keep = _last_occurrences(indices)
+    kept = indices[keep]
+    assert np.unique(kept).size == kept.size
+    actual = np.zeros(8, dtype=values.dtype)
+    actual[kept] = values[keep]
+    np.testing.assert_array_equal(actual, expected)

packages/zarr-metadata/changes/427.removal.md

@@ -0,0 +1 @@
+Replace `ZarrV3Context.update(**slots)` and `ZarrV2Context.update(**slots)` with `copy(*, update=None)`, after pydantic's `model_copy(update=...)`: `context.update(codecs=[...])` becomes `context.copy(update={"codecs": [...]})`, and still replaces each slot it is given whole. To add definitions to a scope and keep the ones it has, use `union`: `CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(data_types=[MY_DATA_TYPE]))`, which the docs now show in place of splicing a slot's definitions back in.

packages/zarr-metadata/changes/4597.bugfix.md

@@ -0,0 +1 @@
+`cast_value` and `scale_offset` reject `fixed_length_utf32`, `null_terminated_bytes`, `raw_bytes` and `variable_length_bytes`, whose values are not numbers. The strategies no longer draw them for either codec. The `scale_offset` strategy now draws `offset` and `scale` in every spelling a fill value of the data type takes, including integers, `NaN`, the infinities and the hex strings of their bits; a scale is still never zero.

packages/zarr-metadata/changes/4597.feature.md

@@ -0,0 +1 @@
+Added strategies to `zarr_metadata.testing.strategies` for the definitions that hold other declarations: `struct`, `cast_value` and `sharding_indexed`. Their nested declarations are drawn from the same `Strategies` mapping, so a struct's field types, a cast's target data type and a shard's codecs come from every definition the mapping holds. A new `pipelines` strategy draws a whole codec pipeline for a `Chunk`, drawing each codec for the chunk the codec before it hands on, as that codec's `transition` gives it. Shards are drawn at most one inside another, and structs likewise. Every function a strategy record holds now also takes the `Strategies` mapping as its last argument, and `declarations` reads what it draws in the scope of every definition the mapping holds.

packages/zarr-metadata/changes/4602.feature.md

@@ -0,0 +1 @@
+Added `array_documents`, `group_documents` and `consolidated_group_documents` to `zarr_metadata.testing.strategies`, which draw whole Zarr v3 documents. An array's members are drawn to fit each other: the chunk grid for the shape, the fill value for the data type, and the codecs for the grid's chunks of that data type. `pipelines` now takes `spellings`, as `declarations` does.

packages/zarr-metadata/docs/context.md

@@ -73,10 +73,10 @@ In this example, our context object only contained one codec definition, for the
 After resolution the `gzip` codec was explicitly flagged as unrecognized. But we can actually check that codec declaration against a real codec definition by expanding our context to include the `GZIP_CODEC` definition:
 
 ```python test="true" session="context"
-new_context = context.update(codecs={**context.codecs, "gzip": GZIP_CODEC})
+new_context = context.union(ZarrV3Context(codecs=[GZIP_CODEC]))
 ```
 
-`update` returns a new context with the `codecs` slot replaced. We pass the codecs the old context already has, plus `gzip`, so the new context claims both `bytes` and `gzip`. The old context is unchanged.
+`union` returns a new context that files every definition of both, so the new context claims both `bytes` and `gzip`. The old context is unchanged. To replace a slot instead of adding to it, use `copy(update=...)`.
 
 Now read the same document in the new context. `read_array_metadata_v3` returns everything the read found, instead of raising, so we can look at each codec:
 

packages/zarr-metadata/docs/parsing-validation.md

@@ -214,17 +214,17 @@ To make an unknown data type or codec known, add its definition to the scope's s
 
 ```python test="true" session="parsing-validation"
 from zarr_metadata.definition import EmptyConfiguration
-from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition
+from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition, ZarrV3Context
 
 # The definition of the codec that was unknown: here, a codec named
 # `acme.codec` that takes no configuration.
 MY_CODEC = CodecDefinition(
     name="acme.codec", configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic"
 )
-context = CORE_AND_EXTENSIONS_CONTEXT.update(codecs=[*CORE_AND_EXTENSIONS_CONTEXT.codecs.values(), MY_CODEC])
+context = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(codecs=[MY_CODEC]))
 reading = read_array_metadata_v3(raw, context=context)
 ```
 
-`update` replaces each slot it is given and keeps the others. A slot can also be given as a mapping from each definition's name to it, so `CORE_AND_EXTENSIONS_CONTEXT.update(codecs={**CORE_AND_EXTENSIONS_CONTEXT.codecs, "gzip": MY_GZIP})` replaces the gzip definition with your own.
+`union` keeps every definition the scope has, and raises `ScopeConflictError` if `MY_CODEC` has a name the scope already files with another definition. To replace a definition on purpose, use `copy`, which, like pydantic's `model_copy(update=...)`, replaces whole each slot it is given and keeps the others. A slot can be given as a mapping from each definition's name to it, so `CORE_AND_EXTENSIONS_CONTEXT.copy(update={"codecs": {**CORE_AND_EXTENSIONS_CONTEXT.codecs, "gzip": MY_GZIP}})` replaces the gzip definition with your own. Passing `copy(update={"codecs": [MY_CODEC]})` instead would leave `MY_CODEC` as the only codec.
 
 [^1]: The one exception is the `r*` data type, which is identified by a templated string with no `configuration` object. This mixes the identifier with the configuration in the same JSON value, which is not good design. So this library normalizes `r*` into the same form as all the other metadata objects: the declaration `"r16"` keeps its name, and is read by the `r*` definition with the configuration `{"bits": 16}`.