diff --git a/README.md b/README.md index bf06f2d..5d35b74 100644 --- a/README.md +++ b/README.md @@ -229,7 +229,7 @@ python scripts/check_docs.py Tests marked `slow` need network access, model weights, or inference, and are not run in CI; run them with `python -m pytest -m slow`. `check_docs.py` checks -that every relative link in `README.md`, `NOTICE.md`, `docs/*.md` and +that every relative link in `README.md`, `NOTICE.md`, `RESULTS_8WAY.md`, `docs/*.md` and `paper/README.md` resolves, and that every `--flag` in a documented `python -m ...` or `python scripts/....py` command is accepted by that program. Use `python scripts/check_docs.py --skip-flags` to check links only. diff --git a/RESULTS_8WAY.md b/RESULTS_8WAY.md index f793c83..441dfd4 100644 --- a/RESULTS_8WAY.md +++ b/RESULTS_8WAY.md @@ -15,8 +15,23 @@ v1.1 purged test partition (n = 2,224; see `v1.1/RESULTS_v1.1.md`). | DeepFense EAT+Nes2Net (FakeMusicCaps) | — | [`DeepFense/FakeMusicCaps_EAT_Nes2Net_NoAug_Seed42`](https://huggingface.co/DeepFense/FakeMusicCaps_EAT_Nes2Net_NoAug_Seed42) (Kheir et al., arXiv:2604.08450) | Adapters: `artifactbench/models/{ast_60s,deepfense,deezer_ismir,fst,spectttra_variants}.py`. -Run with `--model ast_60s --model deepfense --model deezer_ismir --model fst --model spectttra_beta5s` -alongside the existing `--model artifactnet --model spectttra --model clam`. +CLAM and FST import their upstream code, so both need a local clone (with the checkpoints obtained as +each upstream README describes); FST reads its clone from `--fst-repo` or `ARTIFACTBENCH_FST_REPO`. + +```bash +git clone https://github.com/StarkVision-AI/MoM-CLAM ./MoM-CLAM +git clone https://github.com/Mippia/FST-AI-music-detection ./FST-AI-music-detection + +python -m artifactbench.bench \ + --model artifactnet --model spectttra --model clam \ + --model ast_60s --model deepfense --model deezer_ismir --model fst --model spectttra_beta5s \ + --clam-repo ./MoM-CLAM \ + --clam-ckpt ./MoM-CLAM/model_wts/best_model_triplet_loss_margin_0.2.pth \ + --fst-repo ./FST-AI-music-detection \ + --manifest v1.1/artifactbench_v1.1_test_purged.json \ + --split test \ + --output results/eight_way +``` ## Results (n = 2,104: 1,388 AI / 716 real, τ = 0.5) diff --git a/artifactbench/bench.py b/artifactbench/bench.py index a7963b2..10771df 100644 --- a/artifactbench/bench.py +++ b/artifactbench/bench.py @@ -9,8 +9,8 @@ # Four-model comparison (CLAM requires --clam-repo + --clam-ckpt) python -m artifactbench.bench \ --model artifactnet --model spectttra --model clam \ - --clam-repo ~/dev/MoM-CLAM \ - --clam-ckpt ~/dev/MoM-CLAM/model_wts/best_model_triplet_loss_margin_0.2.pth \ + --clam-repo ./MoM-CLAM \ + --clam-ckpt ./MoM-CLAM/model_wts/best_model_triplet_loss_margin_0.2.pth \ --manifest manifest.json --output out/ # Smoke test with tiny sample @@ -39,6 +39,7 @@ BenchModel, CLAMModel, DeezerISMIRModel, + FSTModel, SpecTTTraModel, ) from .report.markdown import comparison_report, single_model_report @@ -64,6 +65,12 @@ def build_model(model_name: str, args) -> "BenchModel": onnx_path=args.deezer_onnx, hf_revision=args.deezer_revision, ) + if model_name == "fst": + return FSTModel( + fst_repo=args.fst_repo, + stage1_ckpt=args.fst_stage1_ckpt, + stage2_ckpt=args.fst_stage2_ckpt, + ) cls = MODEL_REGISTRY[model_name] return cls() @@ -169,7 +176,7 @@ def run_single_model(model, entries, by_source, args): } -def main(): +def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="ArtifactBench v2-compatible runner") parser.add_argument("--model", action="append", required=True, choices=list(MODEL_REGISTRY.keys()), @@ -208,8 +215,18 @@ def main(): help="path to cloned MoM-CLAM repository") parser.add_argument("--clam-ckpt", default=None, help="path to CLAM checkpoint (.pth)") - - args = parser.parse_args() + parser.add_argument("--fst-repo", default=None, + help="path to cloned FST-AI-music-detection repository " + "(default: $ARTIFACTBENCH_FST_REPO)") + parser.add_argument("--fst-stage1-ckpt", default=None, + help="FST stage-1 checkpoint (default: /checkpoints/backbone_stage1.ckpt)") + parser.add_argument("--fst-stage2-ckpt", default=None, + help="FST stage-2 checkpoint (default: /checkpoints/classifier_stage2.ckpt)") + return parser + + +def main(argv=None): + args = build_parser().parse_args(argv) np.random.seed(args.seed) torch.manual_seed(args.seed) diff --git a/artifactbench/models/fst.py b/artifactbench/models/fst.py index f40700a..3579387 100644 --- a/artifactbench/models/fst.py +++ b/artifactbench/models/fst.py @@ -15,6 +15,7 @@ Paper: arXiv:2601.13647 """ import gc +import os import sys from contextlib import contextmanager from pathlib import Path @@ -63,7 +64,9 @@ def pinned_mert_loaders(module, snapshot): setattr(module, n, originals[n]) -FST_REPO_DEFAULT = "/home/unohee/dev/FST-AI-music-detection" +# Location of a clone of https://github.com/Mippia/FST-AI-music-detection. There is no default: the +# adapter imports that repository's code, so it must be told where the clone lives. +FST_REPO_ENV = "ARTIFACTBENCH_FST_REPO" STAGE1_CKPT_DEFAULT = "checkpoints/backbone_stage1.ckpt" STAGE2_CKPT_DEFAULT = "checkpoints/classifier_stage2.ckpt" @@ -77,6 +80,17 @@ def pinned_mert_loaders(module, snapshot): MAX_DURATION_SEC = 90 +def resolve_fst_repo(fst_repo: str | Path | None = None) -> Path: + """The FST clone: the argument, else ``$ARTIFACTBENCH_FST_REPO``; an error names both when neither is set.""" + location = fst_repo if fst_repo else os.environ.get(FST_REPO_ENV) + if not location: + raise ValueError( + "FST needs a clone of https://github.com/Mippia/FST-AI-music-detection: pass --fst-repo " + f"(FSTModel(fst_repo=...)) or set {FST_REPO_ENV}" + ) + return Path(location).expanduser() + + class FSTModel(BenchModel): """FST (Mippia 2025) — two-stage MERT + Fusion Segment Transformer.""" @@ -87,12 +101,12 @@ class FSTModel(BenchModel): paper_ref = "arXiv:2601.13647" def __init__(self, - fst_repo: str = FST_REPO_DEFAULT, + fst_repo: str | None = None, stage1_ckpt: str | None = None, stage2_ckpt: str | None = None, mert_path: str | None = None, beat_checkpoint: str | None = None): - self.fst_repo = Path(fst_repo) + self.fst_repo = resolve_fst_repo(fst_repo) self.stage1_ckpt = stage1_ckpt or str(self.fst_repo / STAGE1_CKPT_DEFAULT) self.stage2_ckpt = stage2_ckpt or str(self.fst_repo / STAGE2_CKPT_DEFAULT) # Pinned local MERT snapshot (v12.model_assets); None keeps the upstream Hub lookup. diff --git a/artifactbench/rerun_current.py b/artifactbench/rerun_current.py index ccb4fde..a9dd6e6 100644 --- a/artifactbench/rerun_current.py +++ b/artifactbench/rerun_current.py @@ -3,6 +3,7 @@ from collections import Counter, defaultdict import hashlib import json +import os from pathlib import Path import random import subprocess @@ -29,7 +30,12 @@ def write_json(path, value): path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n") -def main(): +ARTIFACTNET_REPO_ENV = "ARTIFACTBENCH_ARTIFACTNET_REPO" +FMA_FULL_ROOT_ENV = "ARTIFACTBENCH_FMA_FULL_ROOT" + + +def parse_args(argv=None): + """Parse the CLI; the private ArtifactNet tree and the local fma_full root have no built-in location.""" ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--output", required=True, type=Path) ap.add_argument("--device", default="cuda") @@ -37,10 +43,26 @@ def main(): ap.add_argument("--codec-pairs", type=int, default=50) ap.add_argument("--recover-fma-from", type=Path, help="reuse verified track results and retry HTML-disguised FMA files using local fma_full") - args = ap.parse_args() + ap.add_argument("--artifactnet-repo", type=Path, default=os.environ.get(ARTIFACTNET_REPO_ENV), + help=f"private ArtifactNet source tree (default: ${ARTIFACTNET_REPO_ENV})") + ap.add_argument("--fma-full-root", type=Path, default=os.environ.get(FMA_FULL_ROOT_ENV), + help=f"local fma_full root used by --recover-fma-from (default: ${FMA_FULL_ROOT_ENV})") + args = ap.parse_args(argv) + if args.artifactnet_repo is None: + ap.error(f"the ArtifactNet source tree is required: pass --artifactnet-repo or set {ARTIFACTNET_REPO_ENV}") + if args.recover_fma_from is not None and args.fma_full_root is None: + ap.error(f"--recover-fma-from needs the local fma_full root: pass --fma-full-root or set {FMA_FULL_ROOT_ENV}") + args.artifactnet_repo = Path(args.artifactnet_repo).expanduser() + if args.fma_full_root is not None: + args.fma_full_root = Path(args.fma_full_root).expanduser() + return args + + +def main(argv=None): + args = parse_args(argv) out = args.output out.mkdir(parents=True, exist_ok=True) - rd = Path('/home/unohee/dev/ArtifactNet') + rd = args.artifactnet_repo sys.path.insert(0, str(rd)) import src.pipeline.infer as infer from src.pipeline import model_config as config @@ -103,7 +125,7 @@ def main(): assert p.stem.isdigit() and len(p.stem) == 6 with p.open('rb') as fh: assert fh.read(30).lstrip().lower().startswith(b'/, /Users//, /media//, /Volumes//, /mnt//. +# Bare prefixes such as '/home/' (used by the release guards to reject private paths) do not match. +USER_PATH_RE = re.compile(r"/(?:home|Users|media|Volumes|mnt)/[A-Za-z0-9_.-]+/") + + +def test_no_user_specific_path_in_package_or_scripts(): + offenders = [] + for base in ("artifactbench", "scripts"): + for path in sorted((ROOT / base).rglob("*.py")): + for n, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if USER_PATH_RE.search(line): + offenders.append(f"{path.relative_to(ROOT)}:{n}: {line.strip()}") + assert not offenders, "user-specific absolute paths:\n" + "\n".join(offenders) + + +def test_user_path_pattern_matches_maintainer_paths_but_not_guard_prefixes(): + assert USER_PATH_RE.search("root = Path('/home/someone/dev/ArtifactNet')") + assert USER_PATH_RE.search("'/media/someone/Archive/dataset/fma_full'") + assert not USER_PATH_RE.search("prefix in ('/home/', '/media/', '/Volumes/', '/Users/')") + + +# --- FST ----------------------------------------------------------------------------------------------------------- + +def test_fst_without_repo_raises_naming_flag_and_variable(monkeypatch): + from artifactbench.models.fst import FSTModel + monkeypatch.delenv("ARTIFACTBENCH_FST_REPO", raising=False) + with pytest.raises(ValueError) as err: + FSTModel() + assert "--fst-repo" in str(err.value) and "ARTIFACTBENCH_FST_REPO" in str(err.value) + + +def test_fst_repo_from_environment(monkeypatch, tmp_path): + from artifactbench.models.fst import FSTModel + monkeypatch.setenv("ARTIFACTBENCH_FST_REPO", str(tmp_path)) + model = FSTModel() + assert model.fst_repo == tmp_path + assert model.stage1_ckpt == str(tmp_path / "checkpoints/backbone_stage1.ckpt") + assert model.stage2_ckpt == str(tmp_path / "checkpoints/classifier_stage2.ckpt") + + +def test_fst_repo_argument_wins_over_environment(monkeypatch, tmp_path): + from artifactbench.models.fst import FSTModel + monkeypatch.setenv("ARTIFACTBENCH_FST_REPO", str(tmp_path / "env")) + assert FSTModel(fst_repo=str(tmp_path / "arg")).fst_repo == tmp_path / "arg" + + +def test_bench_cli_wires_fst_flags_into_the_adapter(monkeypatch, tmp_path): + from artifactbench.bench import build_model, build_parser + from artifactbench.models.fst import FSTModel + monkeypatch.delenv("ARTIFACTBENCH_FST_REPO", raising=False) + args = build_parser().parse_args([ + "--model", "fst", "--manifest", "m.json", "--output", str(tmp_path / "out"), + "--fst-repo", str(tmp_path / "fst"), + "--fst-stage1-ckpt", str(tmp_path / "s1.ckpt"), + "--fst-stage2-ckpt", str(tmp_path / "s2.ckpt"), + ]) + model = build_model("fst", args) + assert isinstance(model, FSTModel) + assert model.fst_repo == tmp_path / "fst" + assert model.stage1_ckpt == str(tmp_path / "s1.ckpt") + assert model.stage2_ckpt == str(tmp_path / "s2.ckpt") + + +def test_bench_cli_fst_without_repo_fails_before_any_audio(monkeypatch, tmp_path): + from artifactbench.bench import build_model, build_parser + monkeypatch.delenv("ARTIFACTBENCH_FST_REPO", raising=False) + args = build_parser().parse_args(["--model", "fst", "--manifest", "m.json", "--output", str(tmp_path)]) + with pytest.raises(ValueError, match="--fst-repo"): + build_model("fst", args) + + +# --- pinned v1.2 assets ---------------------------------------------------------------------------------------------- + +def test_asset_location_without_variable_or_default_raises(monkeypatch): + from artifactbench.v12.model_assets import asset_location + monkeypatch.delenv("ARTIFACTBENCH_ARTIFACTNET_REPO", raising=False) + with pytest.raises(ValueError, match="ARTIFACTBENCH_ARTIFACTNET_REPO"): + asset_location("ARTIFACTBENCH_ARTIFACTNET_REPO") + + +def test_asset_location_variable_wins_over_default(monkeypatch, tmp_path): + from artifactbench.v12.model_assets import asset_location + monkeypatch.setenv("ARTIFACTBENCH_CLAM_REPO", str(tmp_path / "env")) + assert asset_location("ARTIFACTBENCH_CLAM_REPO", str(tmp_path / "default")) == tmp_path / "env" + + +@pytest.mark.parametrize("name", ["artifactnet", "clam", "fst"]) +def test_pinned_source_trees_need_a_location(monkeypatch, tmp_path, name): + from artifactbench.v12.model_assets import build_pinned + for variable in ("ARTIFACTBENCH_ARTIFACTNET_REPO", "ARTIFACTBENCH_CLAM_REPO", "ARTIFACTBENCH_FST_REPO"): + monkeypatch.delenv(variable, raising=False) + snapshots = tmp_path / "snapshots.json" + snapshots.write_text("[]") + with pytest.raises(ValueError, match="ARTIFACTBENCH_"): + build_pinned(name, snapshots) + + +def test_freeze_exposure_audit_needs_the_artifactnet_tree(monkeypatch): + from artifactbench.v12.freeze import local_exposure_audit + monkeypatch.delenv("ARTIFACTBENCH_ARTIFACTNET_REPO", raising=False) + with pytest.raises(ValueError, match="ARTIFACTBENCH_ARTIFACTNET_REPO"): + local_exposure_audit([], {"input_hashes": {}}) + + +# --- maintainer re-run tool ------------------------------------------------------------------------------------------ + +def test_rerun_current_requires_the_artifactnet_tree(monkeypatch, tmp_path, capsys): + from artifactbench import rerun_current + monkeypatch.delenv("ARTIFACTBENCH_ARTIFACTNET_REPO", raising=False) + with pytest.raises(SystemExit): + rerun_current.parse_args(["--output", str(tmp_path)]) + message = capsys.readouterr().err + assert "--artifactnet-repo" in message and "ARTIFACTBENCH_ARTIFACTNET_REPO" in message + + +def test_rerun_current_fma_recovery_requires_a_root(monkeypatch, tmp_path, capsys): + from artifactbench import rerun_current + monkeypatch.setenv("ARTIFACTBENCH_ARTIFACTNET_REPO", str(tmp_path / "an")) + monkeypatch.delenv("ARTIFACTBENCH_FMA_FULL_ROOT", raising=False) + with pytest.raises(SystemExit): + rerun_current.parse_args(["--output", str(tmp_path), "--recover-fma-from", str(tmp_path / "prev")]) + message = capsys.readouterr().err + assert "--fma-full-root" in message and "ARTIFACTBENCH_FMA_FULL_ROOT" in message + + +def test_rerun_current_reads_locations_from_flags_and_environment(monkeypatch, tmp_path): + from artifactbench import rerun_current + monkeypatch.setenv("ARTIFACTBENCH_ARTIFACTNET_REPO", str(tmp_path / "an")) + monkeypatch.setenv("ARTIFACTBENCH_FMA_FULL_ROOT", str(tmp_path / "fma")) + args = rerun_current.parse_args(["--output", str(tmp_path), "--recover-fma-from", str(tmp_path / "prev")]) + assert args.artifactnet_repo == tmp_path / "an" and args.fma_full_root == tmp_path / "fma" + args = rerun_current.parse_args(["--output", str(tmp_path), "--artifactnet-repo", str(tmp_path / "flag")]) + assert args.artifactnet_repo == tmp_path / "flag" + assert isinstance(args, argparse.Namespace)