feat(music-data): reuse cached chart statistics of unchanged model sources

docs/commands.md

@@ -679,7 +679,7 @@ in the printed summary and in `SITE.failures.json`, models that fail in the summ
 ```
 nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N]
                    [--workers N] [--no-gekisou-aptitude] [--aptitude-max-seeds N] [--aptitude-cross-seeds N]
-                   [--no-bgm] [--jackets DIR] -o FILE
+                   [--stats-cache DIR] [--no-bgm] [--jackets DIR] -o FILE
 ```
 
 Writes one JSON file with every `MasterLiveMusic` song and its charts for one master data version: titles, readings
@@ -697,7 +697,9 @@ master data tables about cards, skills, bonuses, scores and events. Single-skill
 default (not an optimal deck); `--no-gekisou-aptitude` omits it, `--aptitude-max-seeds N` (1024) and
 `--aptitude-cross-seeds N` (64) cap its sampling and cross terms. `--no-deck` skips the deck model (every chart's
 `deck` is null); `--seeds N` (default 8) and `--workers N` (default: every processor) set its seeds on charts with a
-luck range and its threads. The master data is decoded from the files as served: `--master-files DIR` reads
+luck range and its threads. `--stats-cache DIR` keeps every chart's statistics in `DIR` under the SHA-256 of what
+they are a function of (the deck model's sources, these options, the master data tables and the chart) and measures
+only the charts it lacks; `DIR` then holds this file's charts only. The master data is decoded from the files as served: `--master-files DIR` reads
 `DIR/MasterManifest.json` and the `.bin` files it lists (`master download`; the file's region is `[catalog] region`),
 `--apk-master` the same files inside `[paths] apk` (region `embedded`); each file is checked against the manifest's
 SHA-256. `--decoded-master` reads master data decoded elsewhere instead, without the master key: the `<Table>.json`

docs/music-data.md

@@ -12,7 +12,7 @@ as its extension module `nnnotes._deck`. The format is `nnnotes.music-data/1`; i
 ```
 nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N]
                    [--workers N] [--no-gekisou-aptitude] [--aptitude-max-seeds N] [--aptitude-cross-seeds N]
-                   [--no-bgm] [--jackets DIR] -o FILE
+                   [--stats-cache DIR] [--no-bgm] [--jackets DIR] -o FILE
 ```
 
 - `--master-files DIR`: master data files as served, `DIR/MasterManifest.json` and the `.bin` files it lists
@@ -33,6 +33,11 @@ nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--ful
 - `--no-gekisou-aptitude`: keep the existing deck statistics but skip single-skill aptitude measurements.
   `--aptitude-max-seeds N` caps their samples (default 65536), and `--aptitude-cross-seeds N` caps cross-term samples
   (default 64). Lower caps reduce work but may leave the standard-error target unmet.
+- `--stats-cache DIR`: keep every chart's deck statistics in `DIR`, one file per chart named by the SHA-256 of what
+  they are a function of: the deck model's sources (`provenance.deck.sourceSha256`), `--seeds` and the aptitude
+  options, the master data tables of the deck input and the chart's runtime notes. A chart found there is not measured
+  again, so an export after a deck model release that keeps the model's sources, or after a master data update that
+  keeps those tables, measures only the charts that changed. `DIR` then holds this file's charts only.
 - `--no-bgm`: do not read the cue sheets (every `bgm.length` is null).
 - `--jackets DIR`: also write every song's jacket, the Texture2D `Image/Jacket/<jacket>`, as `DIR/<jacket>.webp`
   (WebP quality 88, scaled down with Lanczos to at most 320 pixels on the longer side, without alpha when opaque); a
@@ -439,7 +444,8 @@ release in `rust/Cargo.toml`
 `pip install .` or `pip install -e .` in a checkout builds the module with the Rust toolchain. The same nnnotes
 version always carries the same deck model: the commit moves only through a pull request (`.github/workflows/deck.yml`
 opens one when ournotes-deck publishes a newer release with its WASM packages), and `provenance.deck.commit` names it in
-every file. The replay and recommendation engines of `--replay-engine` and `--recommend-engine` are that release's WASM
+every file. `provenance.deck.sourceSha256` is the SHA-256 of the deck model's sources (`ournotes_sim::SOURCE_SHA256`):
+releases with the same value measure the same statistics. The replay and recommendation engines of `--replay-engine` and `--recommend-engine` are that release's WASM
 packages.
 
 ## Versions

docs/schema/music-data.schema.json

@@ -169,6 +169,11 @@
               "type": "string",
               "pattern": "^[0-9a-f]{40}$"
             },
+            "sourceSha256": {
+              "type": "string",
+              "pattern": "^[0-9a-f]{64}$",
+              "description": "the SHA-256 of the deck model's sources; releases with the same value measure the same statistics"
+            },
             "format": {
               "type": "string",
               "description": "the statistics format, ournotes-deck.chart-stats/2"

rust/Cargo.toml

@@ -16,7 +16,7 @@ crate-type = ["cdylib"]
 pyo3 = { version = "0.29", features = ["abi3-py311"] }
 # the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned to the commit of a release;
 # .github/workflows/deck.yml opens a pull request when ournotes-deck publishes a newer release
-ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "76e96b617842bdcc031cce98e3dfeead27ddbb85" }
+ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "31d74487c4fd9e5dc3c1d4f34be8dee8f1a75b0e" }
 serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] }
 
 [profile.release]

rust/src/lib.rs

@@ -95,14 +95,16 @@ fn chart_stats(
 }
 
 /// info() -> dict: the deck model of this module (name: the ournotes-deck repository; version, source and commit:
-/// its crate ournotes-sim as built) and the formats it reads and writes.
+/// its crate ournotes-sim as built; sourceSha256: the SHA-256 of that crate's sources, equal for builds that run the
+/// same model) and the formats it reads and writes.
 #[pyfunction]
 fn info(py: Python<'_>) -> PyResult<Bound<'_, PyDict>> {
     let d = PyDict::new(py);
     d.set_item("name", "ournotes-deck")?;
     d.set_item("version", env!("DECK_VERSION"))?;
     d.set_item("source", env!("DECK_SOURCE"))?;
     d.set_item("commit", env!("DECK_COMMIT"))?;
+    d.set_item("sourceSha256", ournotes_sim::SOURCE_SHA256)?;
     d.set_item("dataFormat", ournotes_sim::data::FORMAT)?;
     d.set_item("format", chartstats::FORMAT)?;
     Ok(d)

src/nnnotes/cli.py

@@ -37,7 +37,8 @@
                          [--font emoji=<file>]
                          [--region <region> [--region ...] | --all-regions]
     nnnotes music-data --master-files <master download dir> | --apk-master | --decoded-master [--full] [--no-deck]
-                       [--no-gekisou-aptitude] [--no-bgm] [--jackets DIR] -o out/music-data.json[.gz]
+                       [--no-gekisou-aptitude] [--stats-cache DIR] [--no-bgm] [--jackets DIR]
+                       -o out/music-data.json[.gz]
     nnnotes export -o out/assets [--select group:<group> | key:<prefix> | bundle:<glob> ...] [--layout original,cas]
     nnnotes plan [--select ...] [--json] [--check] [--emit-tasks <dir>]
     nnnotes run-stage <task.json> [...]
@@ -678,7 +679,7 @@ def cmd_music_data(args, cfg):
         deck = None if args.no_deck else musicdata.Deck(
             seeds=args.seeds, workers=args.workers, aptitude=not args.no_gekisou_aptitude,
             aptitude_max_seeds=args.aptitude_max_seeds, aptitude_cross_seeds=args.aptitude_cross_seeds,
-            require_convergence=not args.allow_unconverged_aptitude)
+            require_convergence=not args.allow_unconverged_aptitude, cache=args.stats_cache)
         if args.apk_master:
             src, region = deckdata.apk_master(apk), deckdata.EMBEDDED
         elif args.decoded_master:                    # decoded elsewhere: no master key
@@ -998,6 +999,9 @@ def target(m, what):
                    help="diagnostic export only: retain unmet SE flags at the sample cap; final exports reject them")
     c.add_argument("--aptitude-cross-seeds", type=int, metavar="N",
                    help="seeds of a Gekisou aptitude variant's cross terms (default: the deck model's, 64)")
+    c.add_argument("--stats-cache", type=Path, metavar="DIR",
+                   help="keep the charts' deck statistics in DIR: a chart whose model sources, options, master tables "
+                        "and chart are unchanged is not measured again; DIR then holds this export's charts only")
     c.add_argument("--no-bgm", action="store_true",
                    help="do not read the BGM cue sheets (every song's bgm.length is null)")
     c.add_argument("--replay-dir", metavar="DIR",

src/nnnotes/musicdata.py

@@ -306,11 +306,13 @@ class Deck:
     """The deck model (nnnotes._deck): chart statistics of a deck input document, on `workers` threads (None: the
     available parallelism), `seeds` seeds for a chart with a luck range; with `aptitude` every chart's Gekisou aptitude
     too, a variant on at most `aptitude_max_seeds` seeds and its cross terms on at most `aptitude_cross_seeds` (None:
-    the deck model's defaults)."""
+    the deck model's defaults). With a `cache` directory, a chart's statistics are kept there under the SHA-256 of
+    what they are a function of: the model's sources (its `sourceSha256`), the options above, the master tables and
+    the chart; a chart found there is not measured again."""
 
     def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=None, aptitude: bool = True,
                  aptitude_max_seeds: int | None = None, aptitude_cross_seeds: int | None = None,
-                 require_convergence: bool = True):
+                 require_convergence: bool = True, cache: Path | None = None):
         if module is None:
             try:
                 from . import _deck as module
@@ -321,14 +323,15 @@ def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=N
         self.aptitude, self.aptitude_max_seeds, self.aptitude_cross_seeds = \
             aptitude, aptitude_max_seeds, aptitude_cross_seeds
         self.require_convergence = require_convergence
+        self.cache = None if cache is None else Path(cache)
+        self.counts: dict | None = None                 # {cached, measured} charts of the last stats()
 
     def info(self) -> dict:
-        """{name, version, source, commit, format} of the deck model."""
+        """{name, version, source, commit, sourceSha256, format} of the deck model."""
         i = self.module.info()
-        return {k: i[k] for k in ("name", "version", "source", "commit", "format")}
+        return {k: i[k] for k in ("name", "version", "source", "commit", "sourceSha256", "format")}
 
-    def stats(self, deck_input: dict) -> dict:
-        """The chart statistics document of a deck input document (deckdata.build), its numbers as written."""
+    def _measure(self, deck_input: dict) -> dict:
         try:
             text = self.module.chart_stats(deckdata.encode(deck_input).decode("utf-8"), self.seeds, self.workers,
                                            aptitude=self.aptitude, aptitude_max_seeds=self.aptitude_max_seeds,
@@ -338,6 +341,52 @@ def stats(self, deck_input: dict) -> dict:
         doc = json.loads(text, parse_float=deckdata._Num)
         if doc.get("format") != self.info()["format"]:
             raise MusicDataError(f"deck model: wrote {doc.get('format')!r}, expected {self.info()['format']!r}")
+        return doc
+
+    def cache_keys(self, deck_input: dict) -> list[str]:
+        """The cache key of each chart of a deck input document, in chart order."""
+        info = self.info()
+        base = {"sourceSha256": info["sourceSha256"], "format": info["format"], "seeds": self.seeds,
+                "aptitude": self.aptitude, "aptitudeMaxSeeds": self.aptitude_max_seeds,
+                "aptitudeCrossSeeds": self.aptitude_cross_seeds,
+                "master": hashlib.sha256(deckdata.encode(deck_input["master"])).hexdigest()}
+        return [hashlib.sha256(deckdata.encode({**base, "chart": hashlib.sha256(deckdata.encode(c)).hexdigest()}))
+                .hexdigest() for c in deck_input["charts"]]
+
+    def _cached_stats(self, deck_input: dict) -> dict:
+        """stats() through the cache: measure the charts it lacks, add them, then keep only this document's charts."""
+        from .cache import write_atomic
+        keys = self.cache_keys(deck_input)
+        found = {}
+        for k in keys:
+            path = self.cache / f"{k}.json"
+            if path.is_file():
+                found[k] = json.loads(path.read_bytes(), parse_float=deckdata._Num)
+        missing = [c for c, k in zip(deck_input["charts"], keys) if k not in found]
+        doc = self._measure({**deck_input, "charts": missing})
+        if len(doc.get("charts") or []) != len(missing):
+            raise MusicDataError("deck model: the charts measured differ from the charts asked for")
+        self.cache.mkdir(parents=True, exist_ok=True)
+        measured = iter(doc["charts"])
+        for k in keys:
+            if k not in found:
+                found[k] = next(measured)
+                write_atomic(self.cache / f"{k}.json", deckdata.encode(found[k]))
+        for path in self.cache.glob("*.json"):
+            if path.stem not in found:
+                path.unlink()
+        doc["charts"] = [found[k] for k in keys]
+        self.counts = {"cached": len(keys) - len(missing), "measured": len(missing)}
+        return doc
+
+    def stats(self, deck_input: dict) -> dict:
+        """The chart statistics document of a deck input document (deckdata.build), its numbers as written. With a
+        cache, the cache then holds exactly this document's charts."""
+        if self.cache is None:
+            doc = self._measure(deck_input)
+            self.counts = {"cached": 0, "measured": len(deck_input["charts"])}
+        else:
+            doc = self._cached_stats(deck_input)
         if self.require_convergence and self.aptitude:
             unmet = [(c.get("scoreId"), v.get("shape"), v.get("bandMatch"), v.get("seeds"))
                      for c in doc.get("charts", [])
@@ -1140,6 +1189,7 @@ def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes],
     return {"out": str(out), "format": FORMAT, "region": region, "masterSource": src.source,
             "masterVersion": src.version, "songs": len(doc["songs"]), "charts": len(charts),
             "deck": doc["provenance"]["deck"]["commit"] if deck is not None else None,
+            **({"deckStats": deck.counts} if deck is not None else {}),
             "unplayable": sum(1 for c in charts if c["deck"] and c["deck"]["unplayable"]),
             "full": full, "bgm": bgm is not None, "jackets": len(jackets),
             "bytes": len(data), "fileBytes": len(written), "sha256": hashlib.sha256(written).hexdigest(),