Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ python scripts/check_docs.py

Tests marked `slow` need network access, model weights, or inference, and are
not run in CI; run them with `python -m pytest -m slow`. `check_docs.py` checks
that every relative link in `README.md`, `NOTICE.md`, `docs/*.md` and
that every relative link in `README.md`, `NOTICE.md`, `RESULTS_8WAY.md`, `docs/*.md` and
`paper/README.md` resolves, and that every `--flag` in a documented
`python -m ...` or `python scripts/....py` command is accepted by that program.
Use `python scripts/check_docs.py --skip-flags` to check links only.
Expand Down
19 changes: 17 additions & 2 deletions RESULTS_8WAY.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,23 @@ v1.1 purged test partition (n = 2,224; see `v1.1/RESULTS_v1.1.md`).
| DeepFense EAT+Nes2Net (FakeMusicCaps) | — | [`DeepFense/FakeMusicCaps_EAT_Nes2Net_NoAug_Seed42`](https://huggingface.co/DeepFense/FakeMusicCaps_EAT_Nes2Net_NoAug_Seed42) (Kheir et al., arXiv:2604.08450) |

Adapters: `artifactbench/models/{ast_60s,deepfense,deezer_ismir,fst,spectttra_variants}.py`.
Run with `--model ast_60s --model deepfense --model deezer_ismir --model fst --model spectttra_beta5s`
alongside the existing `--model artifactnet --model spectttra --model clam`.
CLAM and FST import their upstream code, so both need a local clone (with the checkpoints obtained as
each upstream README describes); FST reads its clone from `--fst-repo` or `ARTIFACTBENCH_FST_REPO`.

```bash
git clone https://github.com/StarkVision-AI/MoM-CLAM ./MoM-CLAM
git clone https://github.com/Mippia/FST-AI-music-detection ./FST-AI-music-detection

python -m artifactbench.bench \
--model artifactnet --model spectttra --model clam \
--model ast_60s --model deepfense --model deezer_ismir --model fst --model spectttra_beta5s \
--clam-repo ./MoM-CLAM \
--clam-ckpt ./MoM-CLAM/model_wts/best_model_triplet_loss_margin_0.2.pth \
--fst-repo ./FST-AI-music-detection \
--manifest v1.1/artifactbench_v1.1_test_purged.json \
--split test \
--output results/eight_way
```

## Results (n = 2,104: 1,388 AI / 716 real, τ = 0.5)

Expand Down
27 changes: 22 additions & 5 deletions artifactbench/bench.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
# Four-model comparison (CLAM requires --clam-repo + --clam-ckpt)
python -m artifactbench.bench \
--model artifactnet --model spectttra --model clam \
--clam-repo ~/dev/MoM-CLAM \
--clam-ckpt ~/dev/MoM-CLAM/model_wts/best_model_triplet_loss_margin_0.2.pth \
--clam-repo ./MoM-CLAM \
--clam-ckpt ./MoM-CLAM/model_wts/best_model_triplet_loss_margin_0.2.pth \
--manifest manifest.json --output out/

# Smoke test with tiny sample
Expand Down Expand Up @@ -39,6 +39,7 @@
BenchModel,
CLAMModel,
DeezerISMIRModel,
FSTModel,
SpecTTTraModel,
)
from .report.markdown import comparison_report, single_model_report
Expand All @@ -64,6 +65,12 @@ def build_model(model_name: str, args) -> "BenchModel":
onnx_path=args.deezer_onnx,
hf_revision=args.deezer_revision,
)
if model_name == "fst":
return FSTModel(
fst_repo=args.fst_repo,
stage1_ckpt=args.fst_stage1_ckpt,
stage2_ckpt=args.fst_stage2_ckpt,
)
cls = MODEL_REGISTRY[model_name]
return cls()

Expand Down Expand Up @@ -169,7 +176,7 @@ def run_single_model(model, entries, by_source, args):
}


def main():
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="ArtifactBench v2-compatible runner")
parser.add_argument("--model", action="append", required=True,
choices=list(MODEL_REGISTRY.keys()),
Expand Down Expand Up @@ -208,8 +215,18 @@ def main():
help="path to cloned MoM-CLAM repository")
parser.add_argument("--clam-ckpt", default=None,
help="path to CLAM checkpoint (.pth)")

args = parser.parse_args()
parser.add_argument("--fst-repo", default=None,
help="path to cloned FST-AI-music-detection repository "
"(default: $ARTIFACTBENCH_FST_REPO)")
parser.add_argument("--fst-stage1-ckpt", default=None,
help="FST stage-1 checkpoint (default: <fst-repo>/checkpoints/backbone_stage1.ckpt)")
parser.add_argument("--fst-stage2-ckpt", default=None,
help="FST stage-2 checkpoint (default: <fst-repo>/checkpoints/classifier_stage2.ckpt)")
return parser


def main(argv=None):
args = build_parser().parse_args(argv)

np.random.seed(args.seed)
torch.manual_seed(args.seed)
Expand Down
20 changes: 17 additions & 3 deletions artifactbench/models/fst.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
Paper: arXiv:2601.13647
"""
import gc
import os
import sys
from contextlib import contextmanager
from pathlib import Path
Expand Down Expand Up @@ -63,7 +64,9 @@ def pinned_mert_loaders(module, snapshot):
setattr(module, n, originals[n])


FST_REPO_DEFAULT = "/home/unohee/dev/FST-AI-music-detection"
# Location of a clone of https://github.com/Mippia/FST-AI-music-detection. There is no default: the
# adapter imports that repository's code, so it must be told where the clone lives.
FST_REPO_ENV = "ARTIFACTBENCH_FST_REPO"
STAGE1_CKPT_DEFAULT = "checkpoints/backbone_stage1.ckpt"
STAGE2_CKPT_DEFAULT = "checkpoints/classifier_stage2.ckpt"

Expand All @@ -77,6 +80,17 @@ def pinned_mert_loaders(module, snapshot):
MAX_DURATION_SEC = 90


def resolve_fst_repo(fst_repo: str | Path | None = None) -> Path:
"""The FST clone: the argument, else ``$ARTIFACTBENCH_FST_REPO``; an error names both when neither is set."""
location = fst_repo if fst_repo else os.environ.get(FST_REPO_ENV)
if not location:
raise ValueError(
"FST needs a clone of https://github.com/Mippia/FST-AI-music-detection: pass --fst-repo "
f"(FSTModel(fst_repo=...)) or set {FST_REPO_ENV}"
)
return Path(location).expanduser()


class FSTModel(BenchModel):
"""FST (Mippia 2025) — two-stage MERT + Fusion Segment Transformer."""

Expand All @@ -87,12 +101,12 @@ class FSTModel(BenchModel):
paper_ref = "arXiv:2601.13647"

def __init__(self,
fst_repo: str = FST_REPO_DEFAULT,
fst_repo: str | None = None,
stage1_ckpt: str | None = None,
stage2_ckpt: str | None = None,
mert_path: str | None = None,
beat_checkpoint: str | None = None):
self.fst_repo = Path(fst_repo)
self.fst_repo = resolve_fst_repo(fst_repo)
self.stage1_ckpt = stage1_ckpt or str(self.fst_repo / STAGE1_CKPT_DEFAULT)
self.stage2_ckpt = stage2_ckpt or str(self.fst_repo / STAGE2_CKPT_DEFAULT)
# Pinned local MERT snapshot (v12.model_assets); None keeps the upstream Hub lookup.
Expand Down
30 changes: 26 additions & 4 deletions artifactbench/rerun_current.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from collections import Counter, defaultdict
import hashlib
import json
import os
from pathlib import Path
import random
import subprocess
Expand All @@ -29,18 +30,39 @@ def write_json(path, value):
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n")


def main():
ARTIFACTNET_REPO_ENV = "ARTIFACTBENCH_ARTIFACTNET_REPO"
FMA_FULL_ROOT_ENV = "ARTIFACTBENCH_FMA_FULL_ROOT"


def parse_args(argv=None):
"""Parse the CLI; the private ArtifactNet tree and the local fma_full root have no built-in location."""
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--output", required=True, type=Path)
ap.add_argument("--device", default="cuda")
ap.add_argument("--limit", type=int, default=0, help="0 = full corrected test set")
ap.add_argument("--codec-pairs", type=int, default=50)
ap.add_argument("--recover-fma-from", type=Path,
help="reuse verified track results and retry HTML-disguised FMA files using local fma_full")
args = ap.parse_args()
ap.add_argument("--artifactnet-repo", type=Path, default=os.environ.get(ARTIFACTNET_REPO_ENV),
help=f"private ArtifactNet source tree (default: ${ARTIFACTNET_REPO_ENV})")
ap.add_argument("--fma-full-root", type=Path, default=os.environ.get(FMA_FULL_ROOT_ENV),
help=f"local fma_full root used by --recover-fma-from (default: ${FMA_FULL_ROOT_ENV})")
args = ap.parse_args(argv)
if args.artifactnet_repo is None:
ap.error(f"the ArtifactNet source tree is required: pass --artifactnet-repo or set {ARTIFACTNET_REPO_ENV}")
if args.recover_fma_from is not None and args.fma_full_root is None:
ap.error(f"--recover-fma-from needs the local fma_full root: pass --fma-full-root or set {FMA_FULL_ROOT_ENV}")
args.artifactnet_repo = Path(args.artifactnet_repo).expanduser()
if args.fma_full_root is not None:
args.fma_full_root = Path(args.fma_full_root).expanduser()
return args


def main(argv=None):
args = parse_args(argv)
out = args.output
out.mkdir(parents=True, exist_ok=True)
rd = Path('/home/unohee/dev/ArtifactNet')
rd = args.artifactnet_repo
sys.path.insert(0, str(rd))
import src.pipeline.infer as infer
from src.pipeline import model_config as config
Expand Down Expand Up @@ -103,7 +125,7 @@ def main():
assert p.stem.isdigit() and len(p.stem) == 6
with p.open('rb') as fh:
assert fh.read(30).lstrip().lower().startswith(b'<!doctype html')
original = Path('/media/unohee/Archive/dataset/fma_full') / p.stem[:3] / p.name
original = args.fma_full_root / p.stem[:3] / p.name
assert original.is_file()
recovered[r['path']] = str(original)
parent_files = [parent / 'provenance.json', parent / 'rows.jsonl', snapshot]
Expand Down
12 changes: 9 additions & 3 deletions artifactbench/v12/freeze.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from urllib.parse import urlsplit, urlunsplit

from .common import digest, manifest_rows, rank, write_once
from .model_assets import asset_location
from .duplicates import Groups, entry_key, compare_waveforms
from .inventory import EXPOSURE_INPUTS
from .review_duplicates import full_audio
Expand Down Expand Up @@ -54,8 +55,10 @@ def recording_groups(entries, audit, review):
return labels, representatives


def local_exposure_audit(entries, inventory):
root = Path('/home/unohee/dev/ArtifactNet')
def local_exposure_audit(entries, inventory, artifactnet_repo=None):
# The exposure inputs are training manifests inside the private ArtifactNet tree; its location comes
# from --artifactnet-repo or ARTIFACTBENCH_ARTIFACTNET_REPO (hash checks below still pin the content).
root = asset_location('ARTIFACTBENCH_ARTIFACTNET_REPO', artifactnet_repo)
sys.path.insert(0, str(root))
from src.data.dataset_guard import stem_key, same_seed_different_generator
findings, input_hashes = [], {}
Expand Down Expand Up @@ -113,6 +116,9 @@ def main():
ap = argparse.ArgumentParser(description=__doc__)
for argument in ('selection', 'validation', 'duplicates', 'review', 'inventory', 'variants', 'variant-validation', 'output'):
ap.add_argument('--'+argument, required=True, type=Path)
ap.add_argument('--artifactnet-repo', type=Path, default=None,
help='private ArtifactNet tree holding the exposure manifests '
'(default: $ARTIFACTBENCH_ARTIFACTNET_REPO)')
args = ap.parse_args()
entries = json.loads(args.selection.read_text())['bench']
variations = json.loads(args.variants.read_text())['bench']
Expand All @@ -136,7 +142,7 @@ def main():
if review['pairs_reviewed'] != audit['perceptual_candidates']:
raise ValueError('All fingerprint candidates need explicit review')
inventory = json.loads(args.inventory.read_text())
exposure = local_exposure_audit(entries, inventory)
exposure = local_exposure_audit(entries, inventory, args.artifactnet_repo)
if exposure['candidate_collisions']:
raise ValueError('Unresolved exposure ID candidates require investigation')
groups, representatives = recording_groups(entries, audit, review)
Expand Down
19 changes: 13 additions & 6 deletions artifactbench/v12/model_assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,16 @@
from .common import digest


def asset_location(variable, default):
"""경로만 기계별로 바꾸며, 실제 구현과 가중치의 해시 검사는 유지한다."""
return Path(os.environ.get(variable, default)).expanduser().resolve()
def asset_location(variable, default=None):
"""Machine-specific location of an external asset; hash checks on its code and weights still apply.

The environment variable wins; ``default`` is used only when it is given. Without either the call
raises, naming the variable, rather than looking in a directory that exists on one machine only.
"""
location = os.environ.get(variable) or default
if not location:
raise ValueError(f'Set {variable} to the location of this asset (no default is assumed)')
return Path(location).expanduser().resolve()


def source_identity(path):
Expand Down Expand Up @@ -42,7 +49,7 @@ def local(path):
return str(path)
if name == 'artifactnet':
import sys
root = asset_location('ARTIFACTBENCH_ARTIFACTNET_REPO', '/home/unohee/dev/ArtifactNet')
root = asset_location('ARTIFACTBENCH_ARTIFACTNET_REPO')
sys.path.insert(0, str(root))
from src.pipeline import model_config as cfg
import src.pipeline.infer as infer
Expand All @@ -67,13 +74,13 @@ def local(path):
elif name == 'deezer_ismir':
model = DeezerISMIRModel(onnx_path=str(Path(snapshot('lofcz/ai-music-detector')) / 'ai_music_detector.onnx'))
elif name == 'clam':
root = asset_location('ARTIFACTBENCH_CLAM_REPO', '/home/unohee/dev/MoM-CLAM')
root = asset_location('ARTIFACTBENCH_CLAM_REPO')
evidence['source_repos']['MoM-CLAM'] = source_identity(root)
model = CLAMModel(clam_repo=str(root), clam_ckpt=local(root / 'model_wts/best_model_triplet_loss_margin_0.2.pth'),
mert_repo=snapshot('m-a-p/MERT-v1-95M'),
wav2vec_repo=snapshot('m3hrdadfi/wav2vec2-base-100k-gtzan-music-genres'))
elif name == 'fst':
root = asset_location('ARTIFACTBENCH_FST_REPO', '/home/unohee/dev/FST-AI-music-detection')
root = asset_location('ARTIFACTBENCH_FST_REPO')
evidence['source_repos']['FST'] = source_identity(root)
model = FSTModel(fst_repo=str(root), stage1_ckpt=local(root / 'checkpoints/backbone_stage1.ckpt'),
stage2_ckpt=local(root / 'checkpoints/classifier_stage2.ckpt'),
Expand Down
6 changes: 4 additions & 2 deletions docs/REPRODUCING_v1.2.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,16 @@ absolute paths is not portable by itself; acquire the same revisions into the
new machine's cache and retain its new location manifest.

Three source checkouts and the BeatThis checkpoint are separate assets. Their
locations can be set with these environment variables before model creation:
locations are set with these environment variables before model creation. The
three source checkouts have no built-in location: when a variable is unset, model
creation stops with an error naming it.

| Variable | Required content |
|---|---|
| `ARTIFACTBENCH_ARTIFACTNET_REPO` | Matching ArtifactNet checkout, `src`, and checkpoint files selected by its pinned `model_config` |
| `ARTIFACTBENCH_CLAM_REPO` | MoM-CLAM checkout and `model_wts/best_model_triplet_loss_margin_0.2.pth` |
| `ARTIFACTBENCH_FST_REPO` | FST checkout and `checkpoints/{backbone_stage1,classifier_stage2}.ckpt` |
| `ARTIFACTBENCH_BEAT_CHECKPOINT` | BeatThis `final0.ckpt` |
| `ARTIFACTBENCH_BEAT_CHECKPOINT` | BeatThis `final0.ckpt` (default `.tools/beat-this/final0.ckpt`) |

Overrides change paths only: the factory still hashes all used files and source.
The [external-asset acquisition and verification workflow](EXTERNAL_ASSETS_v1.2.md)
Expand Down
4 changes: 2 additions & 2 deletions scripts/check_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
# Test Status: tests/test_check_docs.py
"""Documentation checker.

Link check: relative ``[text](target)`` links in README.md, NOTICE.md, docs/*.md and
Link check: relative ``[text](target)`` links in README.md, NOTICE.md, RESULTS_8WAY.md, docs/*.md and
paper/README.md must point at an existing file or directory (anchors and URLs skipped).

Flag check: in fenced code blocks, commands of the form ``python -m pkg.mod ...`` or
Expand All @@ -22,7 +22,7 @@
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
DOC_GLOBS = ("README.md", "NOTICE.md", "docs/*.md", "paper/README.md")
DOC_GLOBS = ("README.md", "NOTICE.md", "RESULTS_8WAY.md", "docs/*.md", "paper/README.md")

LINK_RE = re.compile(r"(?<!!)\[[^\]\n]*\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)")
FENCE_RE = re.compile(r"^(\s*)(```|~~~)")
Expand Down
Loading
Loading