diff --git a/docs/commands.md b/docs/commands.md index c8e9b58..1280243 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -679,7 +679,7 @@ in the printed summary and in `SITE.failures.json`, models that fail in the summ ``` nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] [--workers N] [--no-gekisou-aptitude] [--aptitude-max-seeds N] [--aptitude-cross-seeds N] - [--no-bgm] [--jackets DIR] -o FILE + [--stats-cache DIR] [--no-bgm] [--jackets DIR] -o FILE ``` Writes one JSON file with every `MasterLiveMusic` song and its charts for one master data version: titles, readings @@ -697,7 +697,9 @@ master data tables about cards, skills, bonuses, scores and events. Single-skill default (not an optimal deck); `--no-gekisou-aptitude` omits it, `--aptitude-max-seeds N` (1024) and `--aptitude-cross-seeds N` (64) cap its sampling and cross terms. `--no-deck` skips the deck model (every chart's `deck` is null); `--seeds N` (default 8) and `--workers N` (default: every processor) set its seeds on charts with a -luck range and its threads. The master data is decoded from the files as served: `--master-files DIR` reads +luck range and its threads. `--stats-cache DIR` keeps every chart's statistics in `DIR` under the SHA-256 of what +they are a function of (the deck model's sources, these options, the master data tables and the chart) and measures +only the charts it lacks; `DIR` then holds this file's charts only. The master data is decoded from the files as served: `--master-files DIR` reads `DIR/MasterManifest.json` and the `.bin` files it lists (`master download`; the file's region is `[catalog] region`), `--apk-master` the same files inside `[paths] apk` (region `embedded`); each file is checked against the manifest's SHA-256. `--decoded-master` reads master data decoded elsewhere instead, without the master key: the `.json` diff --git a/docs/music-data.md b/docs/music-data.md index 7f30142..8b8290c 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -12,7 +12,7 @@ as its extension module `nnnotes._deck`. The format is `nnnotes.music-data/1`; i ``` nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] [--workers N] [--no-gekisou-aptitude] [--aptitude-max-seeds N] [--aptitude-cross-seeds N] - [--no-bgm] [--jackets DIR] -o FILE + [--stats-cache DIR] [--no-bgm] [--jackets DIR] -o FILE ``` - `--master-files DIR`: master data files as served, `DIR/MasterManifest.json` and the `.bin` files it lists @@ -33,6 +33,11 @@ nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--ful - `--no-gekisou-aptitude`: keep the existing deck statistics but skip single-skill aptitude measurements. `--aptitude-max-seeds N` caps their samples (default 65536), and `--aptitude-cross-seeds N` caps cross-term samples (default 64). Lower caps reduce work but may leave the standard-error target unmet. +- `--stats-cache DIR`: keep every chart's deck statistics in `DIR`, one file per chart named by the SHA-256 of what + they are a function of: the deck model's sources (`provenance.deck.sourceSha256`), `--seeds` and the aptitude + options, the master data tables of the deck input and the chart's runtime notes. A chart found there is not measured + again, so an export after a deck model release that keeps the model's sources, or after a master data update that + keeps those tables, measures only the charts that changed. `DIR` then holds this file's charts only. - `--no-bgm`: do not read the cue sheets (every `bgm.length` is null). - `--jackets DIR`: also write every song's jacket, the Texture2D `Image/Jacket/`, as `DIR/.webp` (WebP quality 88, scaled down with Lanczos to at most 320 pixels on the longer side, without alpha when opaque); a @@ -439,7 +444,8 @@ release in `rust/Cargo.toml` `pip install .` or `pip install -e .` in a checkout builds the module with the Rust toolchain. The same nnnotes version always carries the same deck model: the commit moves only through a pull request (`.github/workflows/deck.yml` opens one when ournotes-deck publishes a newer release with its WASM packages), and `provenance.deck.commit` names it in -every file. The replay and recommendation engines of `--replay-engine` and `--recommend-engine` are that release's WASM +every file. `provenance.deck.sourceSha256` is the SHA-256 of the deck model's sources (`ournotes_sim::SOURCE_SHA256`): +releases with the same value measure the same statistics. The replay and recommendation engines of `--replay-engine` and `--recommend-engine` are that release's WASM packages. ## Versions diff --git a/docs/schema/music-data.schema.json b/docs/schema/music-data.schema.json index b1aff1d..44e3cdb 100644 --- a/docs/schema/music-data.schema.json +++ b/docs/schema/music-data.schema.json @@ -169,6 +169,11 @@ "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "sourceSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "description": "the SHA-256 of the deck model's sources; releases with the same value measure the same statistics" + }, "format": { "type": "string", "description": "the statistics format, ournotes-deck.chart-stats/2" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 9de4642..3146643 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -95,8 +95,8 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "ournotes-sim" -version = "0.0.2" -source = "git+https://github.com/empty-sekai/ournotes-deck?rev=76e96b617842bdcc031cce98e3dfeead27ddbb85#76e96b617842bdcc031cce98e3dfeead27ddbb85" +version = "0.0.3" +source = "git+https://github.com/empty-sekai/ournotes-deck?rev=31d74487c4fd9e5dc3c1d4f34be8dee8f1a75b0e#31d74487c4fd9e5dc3c1d4f34be8dee8f1a75b0e" dependencies = [ "serde", "serde_json", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index bb84b9d..54ef161 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -16,7 +16,7 @@ crate-type = ["cdylib"] pyo3 = { version = "0.29", features = ["abi3-py311"] } # the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned to the commit of a release; # .github/workflows/deck.yml opens a pull request when ournotes-deck publishes a newer release -ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "76e96b617842bdcc031cce98e3dfeead27ddbb85" } +ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "31d74487c4fd9e5dc3c1d4f34be8dee8f1a75b0e" } serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] } [profile.release] diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 4849505..aec368c 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -95,7 +95,8 @@ fn chart_stats( } /// info() -> dict: the deck model of this module (name: the ournotes-deck repository; version, source and commit: -/// its crate ournotes-sim as built) and the formats it reads and writes. +/// its crate ournotes-sim as built; sourceSha256: the SHA-256 of that crate's sources, equal for builds that run the +/// same model) and the formats it reads and writes. #[pyfunction] fn info(py: Python<'_>) -> PyResult> { let d = PyDict::new(py); @@ -103,6 +104,7 @@ fn info(py: Python<'_>) -> PyResult> { d.set_item("version", env!("DECK_VERSION"))?; d.set_item("source", env!("DECK_SOURCE"))?; d.set_item("commit", env!("DECK_COMMIT"))?; + d.set_item("sourceSha256", ournotes_sim::SOURCE_SHA256)?; d.set_item("dataFormat", ournotes_sim::data::FORMAT)?; d.set_item("format", chartstats::FORMAT)?; Ok(d) diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index 14cc82b..cf4f083 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -37,7 +37,8 @@ [--font emoji=] [--region [--region ...] | --all-regions] nnnotes music-data --master-files | --apk-master | --decoded-master [--full] [--no-deck] - [--no-gekisou-aptitude] [--no-bgm] [--jackets DIR] -o out/music-data.json[.gz] + [--no-gekisou-aptitude] [--stats-cache DIR] [--no-bgm] [--jackets DIR] + -o out/music-data.json[.gz] nnnotes export -o out/assets [--select group: | key: | bundle: ...] [--layout original,cas] nnnotes plan [--select ...] [--json] [--check] [--emit-tasks ] nnnotes run-stage [...] @@ -678,7 +679,7 @@ def cmd_music_data(args, cfg): deck = None if args.no_deck else musicdata.Deck( seeds=args.seeds, workers=args.workers, aptitude=not args.no_gekisou_aptitude, aptitude_max_seeds=args.aptitude_max_seeds, aptitude_cross_seeds=args.aptitude_cross_seeds, - require_convergence=not args.allow_unconverged_aptitude) + require_convergence=not args.allow_unconverged_aptitude, cache=args.stats_cache) if args.apk_master: src, region = deckdata.apk_master(apk), deckdata.EMBEDDED elif args.decoded_master: # decoded elsewhere: no master key @@ -998,6 +999,9 @@ def target(m, what): help="diagnostic export only: retain unmet SE flags at the sample cap; final exports reject them") c.add_argument("--aptitude-cross-seeds", type=int, metavar="N", help="seeds of a Gekisou aptitude variant's cross terms (default: the deck model's, 64)") + c.add_argument("--stats-cache", type=Path, metavar="DIR", + help="keep the charts' deck statistics in DIR: a chart whose model sources, options, master tables " + "and chart are unchanged is not measured again; DIR then holds this export's charts only") c.add_argument("--no-bgm", action="store_true", help="do not read the BGM cue sheets (every song's bgm.length is null)") c.add_argument("--replay-dir", metavar="DIR", diff --git a/src/nnnotes/musicdata.py b/src/nnnotes/musicdata.py index b2ef42e..a32d900 100644 --- a/src/nnnotes/musicdata.py +++ b/src/nnnotes/musicdata.py @@ -306,11 +306,13 @@ class Deck: """The deck model (nnnotes._deck): chart statistics of a deck input document, on `workers` threads (None: the available parallelism), `seeds` seeds for a chart with a luck range; with `aptitude` every chart's Gekisou aptitude too, a variant on at most `aptitude_max_seeds` seeds and its cross terms on at most `aptitude_cross_seeds` (None: - the deck model's defaults).""" + the deck model's defaults). With a `cache` directory, a chart's statistics are kept there under the SHA-256 of + what they are a function of: the model's sources (its `sourceSha256`), the options above, the master tables and + the chart; a chart found there is not measured again.""" def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=None, aptitude: bool = True, aptitude_max_seeds: int | None = None, aptitude_cross_seeds: int | None = None, - require_convergence: bool = True): + require_convergence: bool = True, cache: Path | None = None): if module is None: try: from . import _deck as module @@ -321,14 +323,15 @@ def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=N self.aptitude, self.aptitude_max_seeds, self.aptitude_cross_seeds = \ aptitude, aptitude_max_seeds, aptitude_cross_seeds self.require_convergence = require_convergence + self.cache = None if cache is None else Path(cache) + self.counts: dict | None = None # {cached, measured} charts of the last stats() def info(self) -> dict: - """{name, version, source, commit, format} of the deck model.""" + """{name, version, source, commit, sourceSha256, format} of the deck model.""" i = self.module.info() - return {k: i[k] for k in ("name", "version", "source", "commit", "format")} + return {k: i[k] for k in ("name", "version", "source", "commit", "sourceSha256", "format")} - def stats(self, deck_input: dict) -> dict: - """The chart statistics document of a deck input document (deckdata.build), its numbers as written.""" + def _measure(self, deck_input: dict) -> dict: try: text = self.module.chart_stats(deckdata.encode(deck_input).decode("utf-8"), self.seeds, self.workers, aptitude=self.aptitude, aptitude_max_seeds=self.aptitude_max_seeds, @@ -338,6 +341,52 @@ def stats(self, deck_input: dict) -> dict: doc = json.loads(text, parse_float=deckdata._Num) if doc.get("format") != self.info()["format"]: raise MusicDataError(f"deck model: wrote {doc.get('format')!r}, expected {self.info()['format']!r}") + return doc + + def cache_keys(self, deck_input: dict) -> list[str]: + """The cache key of each chart of a deck input document, in chart order.""" + info = self.info() + base = {"sourceSha256": info["sourceSha256"], "format": info["format"], "seeds": self.seeds, + "aptitude": self.aptitude, "aptitudeMaxSeeds": self.aptitude_max_seeds, + "aptitudeCrossSeeds": self.aptitude_cross_seeds, + "master": hashlib.sha256(deckdata.encode(deck_input["master"])).hexdigest()} + return [hashlib.sha256(deckdata.encode({**base, "chart": hashlib.sha256(deckdata.encode(c)).hexdigest()})) + .hexdigest() for c in deck_input["charts"]] + + def _cached_stats(self, deck_input: dict) -> dict: + """stats() through the cache: measure the charts it lacks, add them, then keep only this document's charts.""" + from .cache import write_atomic + keys = self.cache_keys(deck_input) + found = {} + for k in keys: + path = self.cache / f"{k}.json" + if path.is_file(): + found[k] = json.loads(path.read_bytes(), parse_float=deckdata._Num) + missing = [c for c, k in zip(deck_input["charts"], keys) if k not in found] + doc = self._measure({**deck_input, "charts": missing}) + if len(doc.get("charts") or []) != len(missing): + raise MusicDataError("deck model: the charts measured differ from the charts asked for") + self.cache.mkdir(parents=True, exist_ok=True) + measured = iter(doc["charts"]) + for k in keys: + if k not in found: + found[k] = next(measured) + write_atomic(self.cache / f"{k}.json", deckdata.encode(found[k])) + for path in self.cache.glob("*.json"): + if path.stem not in found: + path.unlink() + doc["charts"] = [found[k] for k in keys] + self.counts = {"cached": len(keys) - len(missing), "measured": len(missing)} + return doc + + def stats(self, deck_input: dict) -> dict: + """The chart statistics document of a deck input document (deckdata.build), its numbers as written. With a + cache, the cache then holds exactly this document's charts.""" + if self.cache is None: + doc = self._measure(deck_input) + self.counts = {"cached": 0, "measured": len(deck_input["charts"])} + else: + doc = self._cached_stats(deck_input) if self.require_convergence and self.aptitude: unmet = [(c.get("scoreId"), v.get("shape"), v.get("bandMatch"), v.get("seeds")) for c in doc.get("charts", []) @@ -1140,6 +1189,7 @@ def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes], return {"out": str(out), "format": FORMAT, "region": region, "masterSource": src.source, "masterVersion": src.version, "songs": len(doc["songs"]), "charts": len(charts), "deck": doc["provenance"]["deck"]["commit"] if deck is not None else None, + **({"deckStats": deck.counts} if deck is not None else {}), "unplayable": sum(1 for c in charts if c["deck"] and c["deck"]["unplayable"]), "full": full, "bgm": bgm is not None, "jackets": len(jackets), "bytes": len(data), "fileBytes": len(written), "sha256": hashlib.sha256(written).hexdigest(), diff --git a/tests/test_musicdata.py b/tests/test_musicdata.py index 54b4aed..db077db 100644 --- a/tests/test_musicdata.py +++ b/tests/test_musicdata.py @@ -260,6 +260,7 @@ class FakeDeck: """The interface of nnnotes._deck: statistics made from the deck input as the model reports them, with the Gekisou aptitude (SHAPES; `aptitude`) unless it is left out.""" COMMIT = "7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6" + SOURCE = "5" * 64 FORMAT = "ournotes-deck.chart-stats/2" def __init__(self, change=None, fail=None, header=None): @@ -267,7 +268,8 @@ def __init__(self, change=None, fail=None, header=None): def info(self): return {"name": "ournotes-deck", "version": "0.0.1", "source": "https://github.com/empty-sekai/ournotes-deck", - "commit": self.COMMIT, "dataFormat": deckdata.DECK_FORMAT, "format": self.FORMAT} + "commit": self.COMMIT, "sourceSha256": self.SOURCE, "dataFormat": deckdata.DECK_FORMAT, + "format": self.FORMAT} def chart_stats(self, data, seeds, workers, aptitude=True, aptitude_max_seeds=None, aptitude_cross_seeds=None): if self.fail: @@ -343,6 +345,44 @@ def export(tmp_path, rows=TABLE_ROWS, charts=CHARTS, bgm=bgm, out="music.json", deck=musicdata.Deck(module=deck, workers=3) if deck is not None else None, **kw) +def test_the_stats_cache_measures_only_charts_it_lacks(tmp_path): + cache = tmp_path / "stats" + + def run(name, deck, seeds=musicdata.DECK_SEEDS, stats_cache=cache): + (tmp_path / name).mkdir() + d = master_dir(tmp_path / name) + return musicdata.export(tmp_path / name / "music.json", deckdata.master_files(d), KEY, CHARTS.__getitem__, + bgm, **PROV, deck=musicdata.Deck(module=deck, seeds=seeds, cache=stats_cache)) + plain = run("plain", FakeDeck(), stats_cache=None) + fake = FakeDeck() + first = run("first", fake) + n = plain["charts"] + assert len(fake.inputs[0][0]["charts"]) == n and first["deckStats"] == {"cached": 0, "measured": n} + assert plain["deckStats"] == {"cached": 0, "measured": n} and len(list(cache.glob("*.json"))) == n + again = run("again", fake) + assert fake.inputs[1][0]["charts"] == [] and again["deckStats"] == {"cached": n, "measured": 0} + assert plain["sha256"] == first["sha256"] == again["sha256"] + # the key names the model's sources and the options: changing either measures every chart again, and the + # cache then holds the new statistics only + other = FakeDeck() + other.SOURCE = "6" * 64 + assert run("other", other)["deckStats"] == {"cached": 0, "measured": n} and len(list(cache.glob("*.json"))) == n + assert run("seeds", FakeDeck(), seeds=4)["deckStats"] == {"cached": 0, "measured": n} + + +def test_the_stats_cache_key_reads_the_master_tables_and_the_chart(): + deck = musicdata.Deck(module=FakeDeck()) + doc = {"master": {"MasterLiveSetting": {"columns": ["_id"], "rows": [[1]]}}, + "charts": [{"scoreId": 1, "notes": [1]}, {"scoreId": 2, "notes": [2]}]} + keys = deck.cache_keys(doc) + assert len(set(keys)) == 2 and all(re.fullmatch(r"[0-9a-f]{64}", k) for k in keys) + assert deck.cache_keys({**doc, "charts": doc["charts"][1:]}) == keys[1:] + changed = {**doc, "charts": [doc["charts"][0], {"scoreId": 2, "notes": [3]}]} + assert deck.cache_keys(changed)[0] == keys[0] and deck.cache_keys(changed)[1] != keys[1] + other = {**doc, "master": {"MasterLiveSetting": {"columns": ["_id"], "rows": [[2]]}}} + assert not set(deck.cache_keys(other)) & set(keys) + + def schema_validator(): jsonschema = pytest.importorskip("jsonschema") doc = json.loads((ROOT / "docs" / "schema" / "music-data.schema.json").read_text(encoding="utf-8")) @@ -435,7 +475,7 @@ def test_deck(tmp_path): assert [c["scoreId"] for c in deck_input["charts"]] == [10, 20, 30] # the songs' charts, not chart 40 assert deck_input["provenance"]["master"] == {"source": "api", "version": "v-test"} p = doc["provenance"] - assert p["deck"] == {k: fake.info()[k] for k in ("name", "version", "source", "commit", "format")} + assert p["deck"] == {k: fake.info()[k] for k in ("name", "version", "source", "commit", "sourceSha256", "format")} assert list(p["master"]["tables"]) == list(musicdata.tables_of(True, False)) assert doc["deck"]["model"]["power"] == 300000 assert doc["deck"]["gekisouAptitude"]["shapes"] == SHAPES @@ -630,6 +670,7 @@ def test_deck_module(): info = deck.info() assert info["name"] == "ournotes-deck" and info["format"] == "ournotes-deck.chart-stats/2" assert info["dataFormat"] == deckdata.DECK_FORMAT and re.fullmatch(r"[0-9a-f]{40}", info["commit"]) + assert re.fullmatch(r"[0-9a-f]{64}", info["sourceSha256"]) lock = (ROOT / "rust" / "Cargo.lock").read_text(encoding="utf-8") assert f"#{info['commit']}\"" in lock # the commit Cargo.lock pins with pytest.raises(ValueError, match="not a deck data file"):