From 011962de2cbbcad6a2c043bf81992f8b1cb896c1 Mon Sep 17 00:00:00 2001 From: Prince Ravi Leow Date: Wed, 23 Sep 2026 11:46:59 +0200 Subject: [PATCH 1/4] chore: pre-release cleanup snapshot Snapshot of the working tree before syncing main with upstream PR #5. Mixes two sets of uncommitted edits: - Restore changes lost when forking from PlayClass 26f99ca: SAM3 tracker features (grounding gap fill, frame_loader, ID-match ratio and discontinuity, best-overlap matching, tqdm progress, no timestamped batch dir), 7-way tracker evaluation with per-video mean/std aggregation, classification checkpoint dir, compute_chunk_boundaries --video-path/--video-dir, V-JEPA --resume, and missing docstrings. - Parallel docs/pixi-task cleanup: pixi task names in docs and docstrings, mask crop modes dropped, poster/ablation scripts removed, numpy 2 shims in tracker_eval. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 71 +- docs/2_tracking.md | 16 + docs/4_postprocessing.md | 4 +- docs/5_dataset.md | 18 +- docs/6_embeddings.md | 65 +- docs/7_classification.md | 17 +- docs/8_analysis.md | 6 +- pipeline/build_dataset.py | 5 +- pipeline/clustering_grid.py | 4 +- pipeline/compute_chunk_boundaries.py | 42 +- pipeline/eval_classification.py | 6 +- pipeline/extract_embeddings_dinov3.py | 4 +- pipeline/extract_embeddings_videoprism.py | 10 +- pipeline/extract_embeddings_vjepa2.py | 22 +- pipeline/run_tracker.py | 17 +- pipeline/train.py | 12 +- pipeline/train_ablate_best.sh | 2 +- pipeline/train_sweep_k.sh | 2 +- scripts/setup_gs2.sh | 2 +- src/_config.py | 6 +- src/classification/models.py | 10 + src/classification/stats.py | 2 + src/dataset/README.md | 13 +- src/dataset/embeddings/__init__.py | 3 +- src/dataset/embeddings/vjepa2.py | 7 +- src/dataset/labels.py | 1 + src/dataset/utils.py | 6 + src/tracker_eval/create_ablation_table.py | 124 ---- src/tracker_eval/evaluate.py | 153 +++-- src/tracker_eval/frame_selection.py | 15 +- src/tracker_eval/manifest.py | 44 +- src/tracker_eval/paths.py | 16 +- src/tracker_eval/predictions.py | 26 +- src/tracker_eval/render_poster_figures.py | 787 ---------------------- src/tracking/grounding.py | 108 ++- src/tracking/sam3.py | 165 ++++- src/tracking/scan.py | 11 +- src/tracking/utils.py | 2 + src/tracking/viz.py | 4 + tests/check_crop_modes.py | 2 - 40 files changed, 614 insertions(+), 1216 deletions(-) delete mode 100644 src/tracker_eval/create_ablation_table.py delete mode 100644 src/tracker_eval/render_poster_figures.py diff --git a/README.md b/README.md index 655453f..4565c7d 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,12 @@ All code was developed and tested on Ubuntu 24.04 (linux-64) with CUDA 12.6. ### Dependencies -Run the following commands to setup the main dependencies for this project. +- Install [pixi](https://pixi.sh), a conda-based package manager: +```sh +curl -fsSL https://pixi.sh/install.sh | bash +``` + +- Run the following commands to setup the main dependencies for this project. ```sh git submodule update --init --recursive @@ -21,19 +26,19 @@ pixi install - To install SAM 3, you will need apply for approval at: -- Pixi environments: - - `default` (base) - - `tracker` (SAM3, tracker evaluation) - - `gs2` (Grounded-SAM-2; tracker evaluation) - - `dataset` (build dataset + features) - - `embeddings` (DINOv3/V-JEPA) - - `videoprism` (JAX) - - `classifier` (training, evaluation) - -Pixi environments: `default` (base), `tracker` (SAM3), `dataset` (build + features), `embeddings` (DINOv3/V-JEPA), `classifier` (training), `videoprism` (JAX), `gs2` (Grounded-SAM-2; used for tracker benchmarking), `tracker-evaluation` (CPU-only tracker scoring; motmetrics + pycocotools). Platform is Linux-only (CUDA 12.6). - -All commands use [pixi](https://pixi.sh) task definitions from `pixi.toml`. -Scripts in `script/` and `pipeline/` are the executables; `src/` holds reusable library modules. +- Pixi environments (every environment builds on `default`): + - `default`: PyTorch (CUDA 12.6), `transformers`, Lightning, scikit-learn, + SHAP. Used for building the dataset, DINOv3/V-JEPA 2 embeddings, + classification and analysis. + - `tracker`: adds Ultralytics (YOLO), OmegaConf and motmetrics. Used for + tracking (SAM 3, YOLO) and tracker evaluation. + - `gs2`: Grounded-SAM-2, used for tracker benchmarking. Install it once with + `pixi run -e gs2 setup_gs2`. + - `videoprism`: adds JAX + VideoPrism for VideoPrism embeddings. Solved + separately so JAX doesn't clash with PyTorch. + +All commands use `pixi` task definitions from `pixi.toml`. +Scripts in `scripts/` (one-time setup) and `pipeline/` are the executables; `src/` holds reusable library modules. ``` data/ @@ -41,33 +46,31 @@ data/ tracking/ Symlinks to tracking run output dirs (gitignored) postprocessing/ Version-controlled per-video postprocessing JSONs + parquets (day_28/, day_29/) tracker_eval/ Version-controlled tracker benchmark artefacts (video manifest, keyframes, ablation configs, scored results) -ext-data/ Symlink to large data outputs (results, image sequences, embeddings, etc.) ``` -| Stage | Docs | Environment | -| ---------------------------------------------- | ---------------------------------------------------- | -------------------------- | -| 1. Data (TO-DO: Zenodo deposit in preparation) | [data/README.md](data/README.md) | — | -| 2. Tracking | [docs/2_tracking.md](docs/2_tracking.md) | `tracker`, `gs2` | -| 3. Tracker evaluation _(optional)_ | [docs/3_tracker_eval.md](docs/3_tracker_eval.md) | `tracker`, `gs2` | -| 4. Postprocessing | [docs/4_postprocessing.md](docs/4_postprocessing.md) | `tracker` | -| 5. Build dataset | [docs/5_dataset.md](docs/5_dataset.md) | `tracker` | -| 6. Embeddings | [docs/6_embeddings.md](docs/6_embeddings.md) | `embeddings`, `videoprism` | -| 7. Classification | [docs/7_classification.md](docs/7_classification.md) | `classifier` | +| Stage | Docs | Environment | +| ---------------------------------------------- | ---------------------------------------------------- | ----------------------- | +| 1. Data (TO-DO: Zenodo deposit in preparation) | [data/README.md](data/README.md) | — | +| 2. Tracking | [docs/2_tracking.md](docs/2_tracking.md) | `tracker`, `gs2` | +| 3. Tracker evaluation _(optional)_ | [docs/3_tracker_eval.md](docs/3_tracker_eval.md) | `tracker`, `gs2` | +| 4. Postprocessing | [docs/4_postprocessing.md](docs/4_postprocessing.md) | `default` | +| 5. Build dataset | [docs/5_dataset.md](docs/5_dataset.md) | `default` | +| 6. Embeddings | [docs/6_embeddings.md](docs/6_embeddings.md) | `default`, `videoprism` | +| 7. Classification | [docs/7_classification.md](docs/7_classification.md) | `default` | ### Analysis -| Stage | Docs | Environment | -| --------------------------------------------------------------------- | ---------------------------------------- | ------------ | -| 8. Analysis (clustering, classification figures, feature attribution) | [docs/8_analysis.md](docs/8_analysis.md) | `classifier` | +| Stage | Docs | Environment | +| --------------------------------------------------------------------- | ---------------------------------------- | ----------- | +| 8. Analysis (clustering, classification figures, feature attribution) | [docs/8_analysis.md](docs/8_analysis.md) | `default` | ### Tests ```sh -# 1. Labels, postprocessing, windows (fast, ~seconds) -pixi run -e dataset build_dataset - -pixi run test_features # Feature extraction unit tests (pytest) -pixi run test_postprocessing # Postprocessing logic unit tests (pytest) -pixi run test_labels # Label parsing unit tests (pytest) -pixi run test_post_build # Data integrity checks on a built dataset (pytest; skipped if no dataset) +pixi run test_features # Feature extraction unit tests +pixi run test_labels # Label parsing unit tests +pixi run test_pooling # Temporal pooling module unit tests +pixi run test_dataset_integrity # Data integrity checks on a built dataset (needs `data/dataset/`) +pixi run -e tracker test_tracking_postprocessing # Postprocessing logic unit tests +pixi run -e tracker test_tracker # SAM 3 smoke test on a short clip (GPU) ``` diff --git a/docs/2_tracking.md b/docs/2_tracking.md index ee1f655..716ff82 100644 --- a/docs/2_tracking.md +++ b/docs/2_tracking.md @@ -64,6 +64,22 @@ data/results/tracking/{config_stem}/ | **Manual** | `manual_chunk_frames: [[0,375], ...]` | Explicit `[start, end]` frame pairs. | | **Reuse** | `reuse_chunk_info: true` + `reuse_run_dir` | Load boundaries from a previous run. | +## Grounding (SAM 3) + +At each chunk boundary, a text prompt (`text_prompt: "bird"`) searches +`grounding_frames` frames for the best initialisation frame. IDs are matched +across chunks by mask IoU (`id_match_iou_threshold`). If grounding fails, +falls back to the previous chunk's masks (`fallback_to_prev_chunk`). + +Optional keys (off by default in every config): + +| Key | Effect | +| ------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | +| `text_grounding.fill_grounding_gap: true` | Fill the frames before the chosen grounding frame with the grounding model's own outputs. The day-28 production runs used this. | +| `text_grounding.best_overlap_id_matching: true` | Pick the previous-chunk reference frame by mask overlap instead of the latest frame with enough objects. Legacy; no production run. | +| `text_grounding.id_match_min_ratio` (0.5) | Chunks whose ID match ratio falls below this are flagged with `id_discontinuity` in `chunk_info.json`. | +| `frame_loader` (`torchcodec`) | Frame decoder: `torchcodec`, `cv2_seek` or `cv2_sequential`. The day-28 runs predate torchcodec and need `cv2_seek` to reproduce. | + ## Recomputing chunk boundaries Recompute YOLO scan metrics and chunk boundaries from an existing run without diff --git a/docs/4_postprocessing.md b/docs/4_postprocessing.md index c4daf9e..7bdcaf8 100644 --- a/docs/4_postprocessing.md +++ b/docs/4_postprocessing.md @@ -10,7 +10,7 @@ build. **Run 1 — generate templates:** ```sh -pixi run -e tracker python -m pipeline.build_dataset +pixi run python -m pipeline.build_dataset ``` Reads `tracking_outputs.parquet` from `--tracking-dir` (default: @@ -32,7 +32,7 @@ script stops and tells you which entries need attention. **Run 2 — build the dataset:** ```sh -pixi run -e tracker python -m pipeline.build_dataset +pixi run python -m pipeline.build_dataset ``` --- diff --git a/docs/5_dataset.md b/docs/5_dataset.md index a2dd632..3f41e54 100644 --- a/docs/5_dataset.md +++ b/docs/5_dataset.md @@ -1,9 +1,9 @@ # Dataset Build -| Script | Description | -| ---------------------------- | -------------------------------------------------------------------- | -| `script/build_dataset.py` | Postprocess tracking outputs, match bird IDs, build dataset parquets | -| `script/extract_features.py` | Extract mask features + window summaries from dataset tracks (CPU) | +| Script | Description | +| ------------------------------ | -------------------------------------------------------------------- | +| `pipeline/build_dataset.py` | Postprocess tracking outputs, match bird IDs, build dataset parquets | +| `pipeline/extract_features.py` | Extract mask features + window summaries from dataset tracks (CPU) | --- @@ -13,7 +13,7 @@ The dataset is built in three steps from two sources: the SAM3 tracking outputs (`tracking_outputs.parquet` per video) and the registration protocol Excel files (behaviour labels + bird identity). -**Step 1** (`build_dataset`) is lightweight and produces the canonical +**Step 1** (`pipeline.build_dataset`) is lightweight and produces the canonical `tracks.parquet` and `labels.parquet`. Steps 2 and 3 are slow and independent of each other — run them in parallel if you have the resources. @@ -22,7 +22,7 @@ each other — run them in parallel if you have the resources. ## Step 1 — Labels, postprocessing, windows ```sh -pixi run -e dataset build_dataset +pixi run python -m pipeline.build_dataset ``` **What it does:** @@ -47,7 +47,7 @@ by `(video_id, bird_id, window)`. ## Step 2 — Mask features (CPU) ```sh -pixi run -e dataset extract_features +pixi run python -m pipeline.extract_features ``` **What it does:** Decodes the RLE masks in `tracks.parquet` frame-by-frame and @@ -87,10 +87,10 @@ CV, q10, q90 → `features_windowed.parquet`). ```sh # DINOv3 ViT-L (default) -pixi run -e embeddings extract_embeddings_dinov3 +pixi run extract_dinov3 # V-JEPA 2.1 ViT-L temporal -pixi run -e embeddings python -m script.extract_embeddings_vjepa2 --temporal +pixi run extract_vjepa2 --temporal # VideoPrism Base temporal pixi run -e videoprism extract_videoprism --temporal diff --git a/docs/6_embeddings.md b/docs/6_embeddings.md index 511b65e..40892ea 100644 --- a/docs/6_embeddings.md +++ b/docs/6_embeddings.md @@ -1,10 +1,10 @@ # Embeddings Extraction -| Script | Env | Description | -| ----------------------------------------- | ------------ | -------------------------------------------- | -| `script/extract_embeddings_dinov3.py` | `embeddings` | DINOv3 CLS-token embeddings (image backbone) | -| `script/extract_embeddings_vjepa2.py` | `embeddings` | V-JEPA 2 / 2.1 video embeddings | -| `script/extract_embeddings_videoprism.py` | `videoprism` | VideoPrism video embeddings (JAX) | +| Script | Env | Description | +| ------------------------------------------- | ------------ | -------------------------------------------- | +| `pipeline/extract_embeddings_dinov3.py` | `default` | DINOv3 CLS-token embeddings (image backbone) | +| `pipeline/extract_embeddings_vjepa2.py` | `default` | V-JEPA 2 / 2.1 video embeddings | +| `pipeline/extract_embeddings_videoprism.py` | `videoprism` | VideoPrism video embeddings (JAX) | All three scripts read `tracks.parquet` from the dataset dir, load video frames, and save a `.pt` dict keyed by `(video_id, bird_id, window)`. @@ -18,16 +18,16 @@ run. ```sh # Default: ViT-L, bbox crop -pixi run -e embeddings extract_embeddings_dinov3 \ +pixi run extract_dinov3 \ --video-dir data/videos/day_28 data/videos/day_29 # ViT-B backbone -pixi run -e embeddings extract_embeddings_dinov3 \ +pixi run extract_dinov3 \ --video-dir data/videos/day_28 data/videos/day_29 \ --model-name facebook/dinov3-vitb16-pretrain-lvd1689m # Custom resolution (DINOv3 was trained at 256; supports up to 768) -pixi run -e embeddings extract_embeddings_dinov3 \ +pixi run extract_dinov3 \ --video-dir data/videos/day_28 data/videos/day_29 --resolution 256 ``` @@ -41,7 +41,7 @@ auto-generated from args). ### V-JEPA 2 (HuggingFace) — no setup needed ```sh -pixi run -e embeddings python -m script.extract_embeddings_vjepa2 \ +pixi run extract_vjepa2 \ --video-dir data/videos --device cuda:0 --temporal ``` @@ -52,9 +52,8 @@ loaded via `torch.hub`. The setup script also patches a namespace collision between the hub repo's `src/` directory and this project's own `src/` package. ```sh -# Download checkpoint + patch hub cache (run once) -bash script/setup_vjepa2.1.sh # default: vjepa2_1_vit_large_384 -bash script/setup_vjepa2.1.sh vjepa2_1_vit_base_384 # ViT-B variant +# Download ViT-B + ViT-L checkpoints and patch hub cache (run once) +pixi run setup_vjepa2 ``` Available models: @@ -69,7 +68,7 @@ Available models: Then extract: ```sh -pixi run -e embeddings python -m script.extract_embeddings_vjepa2 \ +pixi run extract_vjepa2 \ --video-dir data/videos/day_28 data/videos/day_29 \ --device cuda:0 --temporal \ --model-name vjepa2_1_vit_large_384 @@ -81,8 +80,8 @@ Output: `embeddings_vjepa21_vitl_temporal.pt` (name auto-generated from args). ## VideoPrism (JAX) -Runs in its own `videoprism` pixi env (JAX + TensorFlow, separate from the -PyTorch `embeddings` env). TF is imported only to be blocked from grabbing the +Runs in its own `videoprism` pixi env (JAX + TensorFlow, solved separately +from the PyTorch envs). TF is imported only to be blocked from grabbing the GPU — the actual inference runs in JAX. No extra setup: model weights download automatically on first run via the @@ -104,20 +103,22 @@ Output: `embeddings_videoprism_temporal.pt` or `embeddings_videoprism_raw.pt`. ## Crop modes -All three scripts support `--crop-mode` to control what region of the frame is -extracted: - -| Mode | Description | -| ----------------------- | --------------------------------------------------------------- | -| `bbox` | Per-frame bounding box (default) | -| `plain256` / `plain384` | Fixed square around the bbox centroid | -| `union512` / `union384` | Fixed square around the centroid of all three birds' union bbox | -| `maskbbox` | Bbox crop with non-bird pixels blacked out | -| `dimbbox` | Same, but dimmed to 40% | -| `silbbox` | Flat silhouette (shape only, no texture) | - -Mask-based modes (`maskbbox`, `dimbbox`, `silbbox`) require the RLE mask columns -in `tracks.parquet` and are matched controls against the default `bbox` crop. - -The crop mode is appended to the output filename: e.g. -`embeddings_vjepa21_vitl_maskbbox_temporal.pt`. +`--crop-mode` controls which part of the frame is fed to the backbone. Not every +script accepts every mode: + +| Mode | Description | DINOv3 | V-JEPA 2 | VideoPrism | +| ----------- | ------------------------------------------------------------------------------------------- | :----: | :------: | :--------: | +| `bbox` | Per-frame bounding box crop (default) | ✓ | ✓ | ✓ | +| `plain256` | Fixed 256×256 square around the bird's bbox centroid, per frame | ✓ | ✓ | ✓ | +| `plain384` | Same, 384×384 | | ✓ | ✓ | +| `union512` | Fixed 512×512 square around the centre of the bird's bboxes, same for the whole window | ✓ | ✓ | ✓ | +| `union384` | Same, 384×384 | | ✓ | ✓ | +| `union` | Union of the bird's bboxes in the window, resized to `--frame-size` | | | ✓ | +| `darken512` | `union512` with everything outside the bird's bbox dimmed to 40% _(untested)_ | ✓ | ✓ | ✓ | +| `roi512` | `union512`; VideoPrism then pools only the patch tokens inside the bird's bbox _(untested)_ | ✓ | ✓ | ✓ | + +`roi512` only differs from `union512` in VideoPrism; DINOv3 and V-JEPA 2 ignore +the ROI patch indices. + +Any mode other than `bbox` is appended to the output filename, e.g. +`embeddings_vjepa21_vitl_union512_temporal.pt`. diff --git a/docs/7_classification.md b/docs/7_classification.md index 62f6742..4a54f2e 100644 --- a/docs/7_classification.md +++ b/docs/7_classification.md @@ -7,26 +7,23 @@ morphokinematic features + V-JEPA 2.1 embeddings). ## Commands -- **Command:** `pixi run -e classifier train [options]` +- **Command:** `pixi run train [options]` - **Requires:** dataset pipeline outputs (steps 4–5) - **Inputs:** `data/dataset/` (tracks, features, embeddings) - **Outputs:** `data/results/eval_classification/{timestamp}_{model}/` ```sh # Features only (MLP baseline) -pixi run -e classifier train --model mlp --input features --exclude social +pixi run train --model mlp --input features --exclude social # Best model: TemporalCNNv2 + V-JEPA 2.1 temporal embeddings -pixi run -e classifier train \ +pixi run train \ --model temporal_cnn2 \ --input features+embeddings_vjepa21_vitl_temporal \ --exclude social --dropout 0.0 --n-segments 24 -# XGBoost baseline -pixi run -e classifier train_xgboost --exclude social - # Dry run (first fold only, 1 batch, no checkpoints) -pixi run -e classifier train --model mlp --input features --dry-run +pixi run train --model mlp --input features --dry-run ``` ## Models @@ -90,11 +87,11 @@ checkpoints, or assemble the segment sweep and ablation tables: ```sh # Evaluate a single run (produces CMs, recall, predictions.csv) -pixi run -e classifier python -m pipeline.eval_classification evaluate +pixi run eval_classification evaluate # Evaluate all runs (skips those with existing outputs) -pixi run -e classifier python -m pipeline.eval_classification evaluate --all +pixi run eval_classification evaluate --all # Assemble Supp Tables 3 and 4 (no GPU needed) -pixi run -e classifier python -m pipeline.eval_classification tables +pixi run eval_classification tables ``` diff --git a/docs/8_analysis.md b/docs/8_analysis.md index cee796b..4d0b201 100644 --- a/docs/8_analysis.md +++ b/docs/8_analysis.md @@ -38,7 +38,7 @@ Runs the k × solver silhouette grid for the dropna variant and writes before the notebook: ```sh -pixi run -e classifier python -m pipeline.clustering_grid +pixi run clustering_grid ``` ## Classification evaluation and tables @@ -47,12 +47,12 @@ Assemble the segment sweep (Supp Table 3) and ablation (Supp Table 4) CSVs from existing `loco_summary.csv` files (no GPU needed): ```sh -pixi run -e classifier python -m pipeline.eval_classification tables +pixi run eval_classification tables ``` To backfill per-fold confusion matrices, recall, and per-sample predictions from checkpoints (requires GPU): ```sh -pixi run -e classifier python -m pipeline.eval_classification evaluate --all +pixi run eval_classification evaluate --all ``` diff --git a/pipeline/build_dataset.py b/pipeline/build_dataset.py index 64d9438..f3fd893 100644 --- a/pipeline/build_dataset.py +++ b/pipeline/build_dataset.py @@ -18,12 +18,12 @@ Typical workflow:: # First run: generates JSONs in data/postprocessing/ - pixi run -e tracker python -m pipeline.build_dataset + pixi run python -m pipeline.build_dataset # Manually fill in "to" values in each tracking_postprocessing.json # Second run: validates remaps, builds dataset - pixi run -e tracker python -m pipeline.build_dataset + pixi run python -m pipeline.build_dataset """ import json @@ -162,6 +162,7 @@ def process_tracking_subdir(tracking_dir, pp_dir, bird_info): def save_data(save_dict, output_dir): + """Save a dict of DataFrames/JSON to an output directory (parquet or JSON by extension).""" output_dir = Path(output_dir) output_dir.mkdir(parents=True, exist_ok=True) for fname, data in save_dict.items(): diff --git a/pipeline/clustering_grid.py b/pipeline/clustering_grid.py index d1eb9ed..af5b3cf 100644 --- a/pipeline/clustering_grid.py +++ b/pipeline/clustering_grid.py @@ -3,7 +3,7 @@ Writes the grid CSV and scaled array to the clustering output directory. Run once per dataset release; the figure notebook reads the outputs. - pixi run -e classifier python -m pipeline.clustering_grid + pixi run clustering_grid """ from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser @@ -62,7 +62,7 @@ def grid_search(X): grid = pd.DataFrame(rows) pivot = grid.pivot(index="k", columns="solver", values="silhouette").round(3) - print(pivot.to_markdown()) + print(pivot.to_string()) mean_sil = grid.groupby("solver")["silhouette"].mean() best_solver = mean_sil.idxmax() diff --git a/pipeline/compute_chunk_boundaries.py b/pipeline/compute_chunk_boundaries.py index 8a09a2a..0409988 100644 --- a/pipeline/compute_chunk_boundaries.py +++ b/pipeline/compute_chunk_boundaries.py @@ -1,7 +1,12 @@ """Recompute chunk boundary metrics from an existing run directory. Given a run dir that contains ``yolo_tracking.parquet``, recomputes per-frame -metrics, adaptive chunk boundaries, and saves updated outputs in-place. +metrics, adaptive chunk boundaries, and saves updated outputs in-place: + +- metrics/yolo_scan_metrics.parquet, metrics/yolo_scan_summary.parquet +- chunk_info.json +- visualizations/yolo_scan_overview.png +- visualizations/chunk_boundaries.png (frame screengrab grid, requires video) Usage:: @@ -104,6 +109,16 @@ def parse_args(): default=None, help="Override YAML config; defaults to the .yaml found in --run-dir", ) + parser.add_argument( + "--video-path", + default=None, + help="Explicit path to source video (for boundary frame grid)", + ) + parser.add_argument( + "--video-dir", + default=None, + help="Directory containing source videos (matched by run dir name)", + ) return parser.parse_args() @@ -265,16 +280,25 @@ def main(): logger.info(f"Saved: {viz_dir / 'yolo_scan_overview.png'}") # ------------------------------------------------------------------------- - # 11. Generate chunk_boundaries_.png (N rows × 2 cols: start + end) + # 11. Generate chunk_boundaries.png (N rows × 2 cols: start + end) # ------------------------------------------------------------------------- - video_path = cfg.get("video_path") - plot_chunk_boundary_frames( - chunk_info=chunk_info, - video_path=video_path, - fps=fps, - yolo_scan_df=yolo_scan_metrics_df, - save_path=viz_dir / f"chunk_boundaries_{run_dir.stem}.png", + video_path = _resolve_video_path( + run_dir, + video_path_override=Path(args.video_path) if args.video_path else None, + video_dir_override=Path(args.video_dir) if args.video_dir else None, ) + if video_path is not None: + boundary_path = viz_dir / "chunk_boundaries.png" + plot_chunk_boundary_frames( + chunk_info=chunk_info, + video_path=video_path, + fps=fps, + yolo_scan_df=yolo_scan_metrics_df, + save_path=boundary_path, + ) + logger.info(f"Saved: {boundary_path}") + else: + logger.warning("Skipping chunk_boundaries.png (no video found)") if __name__ == "__main__": diff --git a/pipeline/eval_classification.py b/pipeline/eval_classification.py index 4fbba8e..f1d65e1 100644 --- a/pipeline/eval_classification.py +++ b/pipeline/eval_classification.py @@ -12,9 +12,9 @@ Usage:: - pixi run -e classifier python -m pipeline.eval_classification evaluate - pixi run -e classifier python -m pipeline.eval_classification evaluate --all - pixi run -e classifier python -m pipeline.eval_classification tables + pixi run eval_classification evaluate + pixi run eval_classification evaluate --all + pixi run eval_classification tables """ import json diff --git a/pipeline/extract_embeddings_dinov3.py b/pipeline/extract_embeddings_dinov3.py index 6b33813..298dfaa 100644 --- a/pipeline/extract_embeddings_dinov3.py +++ b/pipeline/extract_embeddings_dinov3.py @@ -6,8 +6,8 @@ Usage:: - pixi run -e sam3-hf extract-embeddings \ - --video-dir data/video/batch data/video/batch2 + pixi run extract_dinov3 \ + --video-dir data/videos/day_28 data/videos/day_29 """ import sys diff --git a/pipeline/extract_embeddings_videoprism.py b/pipeline/extract_embeddings_videoprism.py index c5fc316..7749fdc 100644 --- a/pipeline/extract_embeddings_videoprism.py +++ b/pipeline/extract_embeddings_videoprism.py @@ -7,11 +7,11 @@ Usage:: pixi run -e videoprism extract_videoprism \ - --video-dir data/video/batch data/video/batch2 --device 0 + --video-dir data/videos/day_28 data/videos/day_29 --device 0 # Save raw patch tokens for trainable pooler pixi run -e videoprism extract_videoprism \ - --video-dir data/video/batch data/video/batch2 --device 0 --raw + --video-dir data/videos/day_28 data/videos/day_29 --device 0 --raw """ import os @@ -40,12 +40,6 @@ from tqdm import tqdm from videoprism import models as vp -# Lazy torch import — only needed for saving .pt files -import torch - -# Block TensorFlow from grabbing GPU -import tensorflow as tf - tf.config.set_visible_devices([], "GPU") tf.config.set_visible_devices([], "TPU") diff --git a/pipeline/extract_embeddings_vjepa2.py b/pipeline/extract_embeddings_vjepa2.py index 15ed00c..dd4f6f1 100644 --- a/pipeline/extract_embeddings_vjepa2.py +++ b/pipeline/extract_embeddings_vjepa2.py @@ -7,10 +7,9 @@ V-JEPA 2.1 setup (one-time):: - bash script/setup_vjepa21.sh # default: vjepa2_1_vit_large_384 - bash script/setup_vjepa21.sh # other variants + pixi run setup_vjepa2 -This downloads the checkpoint and patches the torch.hub cache to rename +This downloads the ViT-B and ViT-L checkpoints and patches the torch.hub cache to rename ``src/`` to ``vjepa2/`` (avoids collision with this project's ``src/``). """ @@ -30,7 +29,7 @@ DEFAULT_TRACKING_DIR, DEFAULT_VIDEO_DIR, ) -from src.dataset.crops import CROP_MODES, needs_mask +from src.dataset.crops import CROP_MODES from src.dataset.embeddings.vjepa2 import ( VJEPA21Wrapper, extract_video_embeddings, @@ -112,6 +111,11 @@ def parse_args(): default=None, help="Output filename (default: auto-generated from model name)", ) + parser.add_argument( + "--resume", + action="store_true", + help="Reuse an existing output file and skip videos already extracted", + ) parser.add_argument( "--cid-checkpoint", type=Path, @@ -229,10 +233,16 @@ def main(): output_path = ( Path(output_name) if "/" in str(output_name) else args.dataset_dir / output_name ) - save_incremental = args.raw - all_embeddings = {} + if args.resume and not args.dry_run and output_path.exists(): + all_embeddings = torch.load(output_path, map_location="cpu", weights_only=False) + logger.info(f"Resuming from {output_path}: {len(all_embeddings)} embeddings") + done_videos = {k[0] for k in all_embeddings} + for video_id in video_ids: + if video_id in done_videos: + logger.info(f"--- Skipping {video_id} (already extracted) ---") + continue logger.info(f"--- Processing video: {video_id} ---") video_path = resolve_video_path(video_id, args.tracking_dir, args.video_dir) if video_path is None: diff --git a/pipeline/run_tracker.py b/pipeline/run_tracker.py index c54fef8..93bccad 100644 --- a/pipeline/run_tracker.py +++ b/pipeline/run_tracker.py @@ -1,4 +1,16 @@ -"""Unified launcher for tracking pipelines (SAM 3, Grounded-SAM-2).""" +""" +Unified launcher for tracking pipelines (SAM 3, Grounded-SAM-2). + +Each video is processed in chunks: chunk 0 uses Sam3VideoModel (text-prompted +segmentation); subsequent chunks use Sam3TrackerVideoModel (point-prompted), +initialised from masks sampled at the cleanest frame in the previous chunk. + +Usage: + # Run with default config (config/sam3_best.yaml) + pixi run -e tracker track_best + # Run with custom config + pixi run -e tracker python -m pipeline.run_tracker --config config/sam3_baseline.yaml +""" import os @@ -108,7 +120,8 @@ def run( _run_batch(cfg, batch_dir, video_path, config_path=config_path) else: videos = sorted( - f for f in video_path.iterdir() + f + for f in video_path.iterdir() if f.is_file() and f.suffix in VIDEO_EXTENSIONS ) for video_file in videos: diff --git a/pipeline/train.py b/pipeline/train.py index b59039c..b8ea2e2 100644 --- a/pipeline/train.py +++ b/pipeline/train.py @@ -6,22 +6,22 @@ Usage:: # Features only (MLP, LOCO default) - pixi run -e classifier train --model mlp --input features + pixi run train --model mlp --input features # LOVO (v0.1.0) - pixi run -e classifier train --model mlp --input features --cv lovo + pixi run train --model mlp --input features --cv lovo # Mean-pooled embeddings (MLP) - pixi run -e classifier train --model mlp --input embeddings + pixi run train --model mlp --input embeddings # Features + embeddings combined - pixi run -e classifier train --model mlp --input features+embeddings + pixi run train --model mlp --input features+embeddings # Temporal model on embeddings - pixi run -e classifier train --model temporal_mlp --input embeddings + pixi run train --model temporal_mlp --input embeddings # Dry run — first fold, 1 batch, no checkpoints - pixi run -e classifier train --model mlp --input features --dry-run + pixi run train --model mlp --input features --dry-run """ import gc diff --git a/pipeline/train_ablate_best.sh b/pipeline/train_ablate_best.sh index 6a0025f..6cf2026 100755 --- a/pipeline/train_ablate_best.sh +++ b/pipeline/train_ablate_best.sh @@ -3,7 +3,7 @@ # Each run changes one hyperparameter from the reference config. # # Usage: -# pixi run ablate-best [-- --device cuda:0 --dry-run] +# pixi run train_ablate_best [--device cuda:0] [--dry-run] # bash pipeline/train_ablate_best.sh [--device cuda:0] [--dry-run] set -euo pipefail diff --git a/pipeline/train_sweep_k.sh b/pipeline/train_sweep_k.sh index 120c231..d8d39e2 100755 --- a/pipeline/train_sweep_k.sh +++ b/pipeline/train_sweep_k.sh @@ -3,7 +3,7 @@ # Embeddings-only, TemporalCNNv2, excluding social. # # Usage: -# pixi run sweep-k [-- --device cuda:0 --dry-run] +# pixi run train_sweep_k [--device cuda:0] [--dry-run] # bash pipeline/train_sweep_k.sh [--device cuda:0] [--dry-run] set -euo pipefail diff --git a/scripts/setup_gs2.sh b/scripts/setup_gs2.sh index 592ea4e..393b355 100755 --- a/scripts/setup_gs2.sh +++ b/scripts/setup_gs2.sh @@ -3,7 +3,7 @@ # checkpoints, PyTorch, editable packages). # # Usage: -# pixi run -e gs2 bash script/setup_gs2.sh +# pixi run -e gs2 setup_gs2 set -euo pipefail diff --git a/src/_config.py b/src/_config.py index 1046ee0..b6ba6ab 100644 --- a/src/_config.py +++ b/src/_config.py @@ -1,12 +1,14 @@ +"""Project-wide constants: default paths, dataset settings, and classification/clustering defaults.""" + from typing import Final DEFAULT_CONFIG_DIR: Final = "config" DEFAULT_DATA_DIR: Final = "data" DEFAULT_DATASET_DIR: Final = f"{DEFAULT_DATA_DIR}/dataset" -DEFAULT_CHECKPOINT_DIR: Final = f"{DEFAULT_DATA_DIR}/eval" DEFAULT_LABEL_DIR: Final = f"{DEFAULT_DATA_DIR}/labels" DEFAULT_POSTPROCESSING_DIR: Final = f"{DEFAULT_DATA_DIR}/postprocessing" DEFAULT_RESULTS_DIR: Final = f"{DEFAULT_DATA_DIR}/results" +DEFAULT_CHECKPOINT_DIR: Final = f"{DEFAULT_RESULTS_DIR}/eval_classification" DEFAULT_CLUSTERING_DIR: Final = f"{DEFAULT_RESULTS_DIR}/clustering" DEFAULT_TRACKING_DIR: Final = f"{DEFAULT_RESULTS_DIR}/tracking" DEFAULT_VIDEO_DIR: Final = f"{DEFAULT_DATA_DIR}/videos" @@ -17,3 +19,5 @@ DEFAULT_FPS: Final = 25.0 DEFAULT_N_BIRDS: Final = 3 LABEL_ORDER: Final = ["none", "worm", "locomotor", "social"] +K_RANGE: Final = range(2, 13) +RANDOM_SEED: Final = 42 diff --git a/src/classification/models.py b/src/classification/models.py index f5e2615..00e3753 100644 --- a/src/classification/models.py +++ b/src/classification/models.py @@ -1,3 +1,9 @@ +"""Neural network backbones for play-behaviour classification. + +Provides non-temporal (SimpleLinear, SimpleMLP) and temporal (TemporalMLP, +TemporalCNNv2) architectures. MODEL_REGISTRY maps CLI names to (class, temporal_flag). +""" + import torch import torch.nn as nn import torch.nn.functional as F @@ -17,6 +23,8 @@ def forward(self, x): class SimpleLinear(nn.Module): + """Single linear layer classifier (linear probe).""" + def __init__(self, d_input, n_classes): super().__init__() self.fc = nn.Linear(d_input, n_classes) @@ -27,6 +35,8 @@ def forward(self, x): class SimpleMLP(nn.Module): + """Two-layer MLP classifier with ReLU and dropout.""" + def __init__(self, d_input, n_classes, d_hidden=64, dropout=0.0): super().__init__() self.net = nn.Sequential( diff --git a/src/classification/stats.py b/src/classification/stats.py index 3f79910..51d7822 100644 --- a/src/classification/stats.py +++ b/src/classification/stats.py @@ -1,3 +1,5 @@ +"""LOCO/LOVO cross-validation aggregation: scalar summary CSV, summed and per-fold confusion matrices, and per-class recall.""" + from pathlib import Path import numpy as np diff --git a/src/dataset/README.md b/src/dataset/README.md index 9bd5791..24e31d7 100644 --- a/src/dataset/README.md +++ b/src/dataset/README.md @@ -193,12 +193,12 @@ cap.release() ```sh # First run: generates tracking_issues.json + tracking_postprocessing.json -pixi run -e sam3-hf build_dataset +pixi run python -m pipeline.build_dataset # User fills in "to" values and reviews trim entries # Second run: applies postprocessing, builds dataset -pixi run -e sam3-hf build_dataset +pixi run python -m pipeline.build_dataset ``` Output: `tracks.parquet` and `labels.parquet` saved to `data/dataset/`. @@ -206,7 +206,7 @@ Output: `tracks.parquet` and `labels.parquet` saved to `data/dataset/`. ### Feature extraction (CPU) ```sh -pixi run -e sam3-hf extract_features +pixi run python -m pipeline.extract_features ``` Output: `features_all.parquet` (per-frame) and `features_windowed.parquet` @@ -215,11 +215,12 @@ Output: `features_all.parquet` (per-frame) and `features_windowed.parquet` ### Embedding extraction (GPU) ```sh -pixi run -e sam3-hf extract_embeddings \ - --video-dir video-data/batch +pixi run extract_dinov3 \ + --video-dir data/videos/day_28 data/videos/day_29 ``` -Output: `embeddings.pt` saved to `data/dataset/`. +Output: `embeddings_dinov3_vitl.pt` saved to `data/dataset/` (see +[docs/6_embeddings.md](../../docs/6_embeddings.md) for other backbones). ## Module overview diff --git a/src/dataset/embeddings/__init__.py b/src/dataset/embeddings/__init__.py index 0994b46..a94c216 100644 --- a/src/dataset/embeddings/__init__.py +++ b/src/dataset/embeddings/__init__.py @@ -1,4 +1,5 @@ -# Re-exports for backwards compatibility +"""Re-exports from dinov3.py for backwards compatibility.""" + from src.dataset.embeddings.dinov3 import ( _split_mask_thirds, extract_bodypart_embeddings, diff --git a/src/dataset/embeddings/vjepa2.py b/src/dataset/embeddings/vjepa2.py index 9bf57d5..cc2dcf2 100644 --- a/src/dataset/embeddings/vjepa2.py +++ b/src/dataset/embeddings/vjepa2.py @@ -9,12 +9,7 @@ from PIL import Image from tqdm import tqdm -from src.dataset.crops import ( - compute_union_origin, - crop_frame, - needs_mask, - union_crop_size, -) +from src.dataset.crops import compute_union_origin, crop_frame from src.utils.io import load_video_frames_torchcodec as load_video_frames _HUB_MODEL_NAMES = { diff --git a/src/dataset/labels.py b/src/dataset/labels.py index 60663dc..c67b1b4 100644 --- a/src/dataset/labels.py +++ b/src/dataset/labels.py @@ -47,6 +47,7 @@ def merge_behaviours(behav_str): + """Map an individual behaviour string to its merged class (locomotor/social/worm/none).""" if behav_str == "none": return "none" merged = set() diff --git a/src/dataset/utils.py b/src/dataset/utils.py index 39d9d75..8cc56ec 100644 --- a/src/dataset/utils.py +++ b/src/dataset/utils.py @@ -1,3 +1,9 @@ +"""Shared helpers for the dataset package. + +Video ID parsing, FPS lookup, time formatting, video path resolution, and +embedding-label alignment checks used across the three dataset pipeline steps. +""" + import os import re from pathlib import Path diff --git a/src/tracker_eval/create_ablation_table.py b/src/tracker_eval/create_ablation_table.py deleted file mode 100644 index d40cb10..0000000 --- a/src/tracker_eval/create_ablation_table.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Render the tracker-eval LaTeX table from results CSVs. - -Mean column = TrackEval COMBINED_SEQ aggregate (from `metrics_aggregate.csv`). -SD column = per-video standard deviation of that variant's HOTA / IDF1 - (from `metrics_per_video.csv`). For ablation rows, the mean - is the delta of aggregates (variant - proposed); the SD is - still the per-video SD of the variant being ablated. - -Usage: - pixi run -e tracker-evaluation python -m src.tracker_eval.create_ablation_table -""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -import pandas as pd - -from .paths import RESULTS_DIR - -PROPOSED = "E_sam3_adaptive" - -# (label, variant_key, role) -# role: "baseline" | "proposed" | "ablation" -ROWS = [ - ("YOLO26x + BoT-SORT", "A_yolo_botsort", "baseline"), - (r"Grounded-SAM-2~\cite{renGroundedSAMAssembling2024}", "B_gs2_strict", "baseline"), - (r"\quad + adaptive grounding", "B_gs2_fixed", "baseline"), - ( - r"\makecell[l]{SAM 3 + adaptive grounding \\ \quad \& adaptive chunking}", - PROPOSED, - "proposed", - ), - (r"\quad $-$ adaptive grounding", "C_sam3_frame_zero", "ablation"), - (r"\quad $-$ adaptive chunking", "D_sam3_fixed", "ablation"), -] - - -def fmt_cell( - mean: float, sd: float, *, bold: bool = False, signed: bool = False -) -> str: - if signed: - sign = "$-$" if mean < 0 else "" - mean_str = f"{sign}{abs(mean):.3f}" - else: - mean_str = f"{mean:.3f}" - if bold: - mean_str = rf"\textbf{{{mean_str}}}" - return rf"{mean_str} \scriptsize{{$\pm${sd:.3f}}}" - - -def build_table(results_dir: Path) -> str: - agg = pd.read_csv(results_dir / "metrics_aggregate.csv").set_index("variant") - perv = pd.read_csv(results_dir / "metrics_per_video.csv") - sd = perv.groupby("variant")[["HOTA", "idf1"]].std() - - e_hota = float(agg.loc[PROPOSED, "HOTA"]) - e_idf1 = float(agg.loc[PROPOSED, "idf1"]) - - lines: list[str] = [] - lines += [ - r"\begin{table}[t]", - r" \centering", - r" \small", - r" \setlength{\tabcolsep}{6pt}", - r" \caption{Tracking evaluation and ablation (mean HOTA and IDF$_1$ $\pm$ SD " - r"aggregated over sparse human-verified keyframes from five videos). " - r"The best score is shown in \textbf{bold}.}", - r" \label{tab:tracker_eval}", - r" \begin{tabular}{lcc}", - r" \toprule", - r" \textbf{Tracking method} & \textbf{HOTA} & \textbf{IDF$_1$} \\", - r" \midrule", - r" \multicolumn{3}{l}{\emph{Baselines:}} \\", - ] - - for label, key, role in ROWS: - hota_mean = float(agg.loc[key, "HOTA"]) - idf1_mean = float(agg.loc[key, "idf1"]) - hota_sd = float(sd.loc[key, "HOTA"]) - idf1_sd = float(sd.loc[key, "idf1"]) - - if role == "ablation": - hota_mean -= e_hota - idf1_mean -= e_idf1 - hota_cell = fmt_cell(hota_mean, hota_sd, signed=True) - idf1_cell = fmt_cell(idf1_mean, idf1_sd, signed=True) - elif role == "proposed": - hota_cell = fmt_cell(hota_mean, hota_sd, bold=True) - idf1_cell = fmt_cell(idf1_mean, idf1_sd, bold=True) - else: - hota_cell = fmt_cell(hota_mean, hota_sd) - idf1_cell = fmt_cell(idf1_mean, idf1_sd) - - if role == "proposed": - lines += [ - r" \midrule", - r" \multicolumn{3}{l}{\emph{Proposed method:}} \\", - f" {label}", - f" & {hota_cell} & {idf1_cell} \\\\", - r" \midrule", - r" \multicolumn{3}{l}{\emph{Ablations on proposed method:}} \\", - ] - else: - lines.append(f" {label} & {hota_cell} & {idf1_cell} \\\\") - - lines += [ - r" \bottomrule", - r" \end{tabular}", - r"\end{table}", - ] - return "\n".join(lines) - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("--results-dir", type=Path, default=RESULTS_DIR) - args = parser.parse_args() - print(build_table(args.results_dir)) - - -if __name__ == "__main__": - main() diff --git a/src/tracker_eval/evaluate.py b/src/tracker_eval/evaluate.py index 123f462..6f10d4d 100644 --- a/src/tracker_eval/evaluate.py +++ b/src/tracker_eval/evaluate.py @@ -1,13 +1,13 @@ -"""Sparse-keyframe MOT evaluation for the 6-way ablation. +"""Sparse-keyframe MOT evaluation for the 7-way ablation. Reads sparse GT (from `cvat_to_mot.py`) and dense predictions (from -`predictions.py`) for all 5 evaluation videos × 6 tracker variants -(A_yolo_botsort, B_gs2_strict, B_gs2_fixed, C_sam3_frame_zero, -D_sam3_fixed, E_sam3_adaptive) and emits: +`predictions.py`) for all 5 evaluation videos × 7 tracker variants +(A_yolo_botsort, A1_yolo_botsort_reid, B_gs2_strict, B_gs2_fixed, +C_sam3_frame_zero, D_sam3_fixed, E_sam3_adaptive) and emits: - data/tracker_eval/results/metrics_per_video.csv - data/tracker_eval/results/metrics_per_cage.csv - data/tracker_eval/results/metrics_aggregate.csv + data/results/eval_tracking/results/metrics_per_video.csv + data/results/eval_tracking/results/metrics_per_cage.csv + data/results/eval_tracking/results/metrics_aggregate.csv Two metric libraries are used: @@ -23,6 +23,9 @@ data dict are built only over GT-present frames so we only score frames that have human-verified ground truth. +Aggregation: per-cage and aggregate metrics are computed as the mean (± std) +of per-video metrics. + Usage: pixi run -e tracker python -m pipeline.eval_tracker_all evaluate """ @@ -32,6 +35,7 @@ from pathlib import Path +import numpy as np import pandas as pd from .paths import ( @@ -43,23 +47,25 @@ ) mm = None -np = None HOTA = None def _ensure_eval_deps() -> None: """Import evaluation-only dependencies lazily. - The `prepare` command runs in the tracker pixi env, which does not include - motmetrics. Keeping these imports lazy lets the shared CLI dispatcher - register pre-CVAT commands without requiring post-CVAT dependencies. + Keeping these imports lazy lets the shared CLI dispatcher register the + pre-CVAT (`prepare`) commands without requiring motmetrics or the + TrackEval submodule. """ - global HOTA, mm, np - if mm is not None and np is not None and HOTA is not None: + global HOTA, mm + if mm is not None and HOTA is not None: return import motmetrics as _mm - import numpy as _np + + # motmetrics 1.4 calls np.asfarray, removed in numpy 2.0. + if not hasattr(np, "asfarray"): + np.asfarray = lambda a, dtype=float: np.asarray(a, dtype=dtype) # TrackEval is vendored as a git submodule at ext/TrackEval, not pip-installed. # Insert it on sys.path so `import trackeval` resolves to the submodule. @@ -75,17 +81,17 @@ def _ensure_eval_deps() -> None: ("bool", bool), ("object", object), ): - if not hasattr(_np, _name): - setattr(_np, _name, _alias) + if not hasattr(np, _name): + setattr(np, _name, _alias) from trackeval.metrics.hota import HOTA as _HOTA mm = _mm - np = _np HOTA = _HOTA VARIANTS = ( "A_yolo_botsort", + "A1_yolo_botsort_reid", "B_gs2_strict", "B_gs2_fixed", "C_sam3_frame_zero", @@ -167,7 +173,7 @@ def iou_xywh(boxes_a: np.ndarray, boxes_b: np.ndarray) -> np.ndarray: def build_motmetrics_accumulator( gt: pd.DataFrame, pred: pd.DataFrame -) -> mm.MOTAccumulator: +) -> "mm.MOTAccumulator": """motmetrics accumulator at IoU ≥ 0.5, over GT-present frames.""" acc = mm.MOTAccumulator(auto_id=False) gt_by_frame = {fr: g for fr, g in gt.groupby("FrameId")} @@ -326,65 +332,82 @@ def run(args: argparse.Namespace) -> None: per_video = per_video[front + [c for c in per_video.columns if c not in front]] per_video.to_csv(args.out_dir / "metrics_per_video.csv", index=False) - # === per-cage === (one video per cage in this study, but generalised) + # Columns to aggregate (everything except identifiers) + id_cols = {"variant", "video_id", "cage"} + numeric_cols = [c for c in per_video.columns if c not in id_cols] + + # === per-cage (mean ± std of per-video metrics within each cage) === rows_per_cage: list[dict] = [] - for variant in variants: - groups: dict[str, list[mm.MOTAccumulator]] = defaultdict(list) - names_in_cage: dict[str, list[str]] = defaultdict(list) - seqs_in_cage: dict[str, list[dict]] = defaultdict(list) - for acc, vid in zip(accs[variant], names[variant]): - c = cage_by_video[vid] - groups[c].append(acc) - names_in_cage[c].append(vid) - seqs_in_cage[c].append(hota_per_seq[variant][vid]) - for cage in sorted(groups): - mm_df = mh.compute_many( - groups[cage], - names=names_in_cage[cage], - metrics=MOTMETRICS_FIELDS, - generate_overall=True, - ) - overall = mm_df.loc["OVERALL"].to_dict() - overall["variant"] = variant - overall["cage"] = cage - cage_hota_res = hota_metric.combine_sequences( - {vid: r for vid, r in zip(names_in_cage[cage], seqs_in_cage[cage])} - ) - overall.update(hota_summary(cage_hota_res)) - rows_per_cage.append(overall) + for (variant, cage), group in per_video.groupby(["variant", "cage"]): + row: dict = {"variant": variant, "cage": cage, "n_videos": len(group)} + for col in numeric_cols: + row[col] = group[col].mean() + row[f"{col}_std"] = group[col].std() + rows_per_cage.append(row) per_cage = pd.DataFrame(rows_per_cage) - front = ["variant", "cage", "HOTA", "DetA", "AssA", "LocA"] - per_cage = per_cage[front + [c for c in per_cage.columns if c not in front]] + front_cage = [ + "variant", + "cage", + "n_videos", + "HOTA", + "HOTA_std", + "DetA", + "DetA_std", + "AssA", + "AssA_std", + "LocA", + "LocA_std", + ] + per_cage = per_cage[ + front_cage + [c for c in per_cage.columns if c not in front_cage] + ] per_cage.to_csv(args.out_dir / "metrics_per_cage.csv", index=False) - # === aggregate === + # === aggregate (mean ± std of per-video metrics across all videos) === rows_agg: list[dict] = [] - for variant in variants: - mm_df = mh.compute_many( - accs[variant], - names=names[variant], - metrics=MOTMETRICS_FIELDS, - generate_overall=True, - ) - overall = mm_df.loc["OVERALL"].to_dict() - overall["variant"] = variant - agg_hota_res = hota_metric.combine_sequences( - {vid: r for vid, r in hota_per_seq[variant].items()} - ) - overall.update(hota_summary(agg_hota_res)) - rows_agg.append(overall) + for variant, group in per_video.groupby("variant"): + row: dict = {"variant": variant, "n_videos": len(group)} + for col in numeric_cols: + row[col] = group[col].mean() + row[f"{col}_std"] = group[col].std() + rows_agg.append(row) aggregate = pd.DataFrame(rows_agg) - front = ["variant", "HOTA", "DetA", "AssA", "LocA"] - aggregate = aggregate[front + [c for c in aggregate.columns if c not in front]] + front_agg = [ + "variant", + "n_videos", + "HOTA", + "HOTA_std", + "DetA", + "DetA_std", + "AssA", + "AssA_std", + "LocA", + "LocA_std", + ] + aggregate = aggregate[ + front_agg + [c for c in aggregate.columns if c not in front_agg] + ] aggregate.to_csv(args.out_dir / "metrics_aggregate.csv", index=False) # Headline print() print("=" * 80) - print("AGGREGATE (5 videos, sparse GT)") + print(f"AGGREGATE ({len(videos)} videos, sparse GT, per-video mean ± std)") print("=" * 80) - headline = ["variant", "HOTA", "DetA", "AssA", "idf1", "mota", "num_switches"] - print(aggregate[headline].to_string(index=False)) + headline = [ + "variant", + "n_videos", + "HOTA", + "HOTA_std", + "idf1", + "idf1_std", + "mota", + "mota_std", + "num_switches", + "num_switches_std", + ] + available = [c for c in headline if c in aggregate.columns] + print(aggregate[available].to_string(index=False)) def _main() -> None: diff --git a/src/tracker_eval/frame_selection.py b/src/tracker_eval/frame_selection.py index ab76d7d..8e204fd 100644 --- a/src/tracker_eval/frame_selection.py +++ b/src/tracker_eval/frame_selection.py @@ -1,7 +1,10 @@ """Select annotation frames for the tracker-eval ground-truth pass. -For each selected video in ``data/tracker_eval/video_manifest.csv`` (selected=True), -this script: +For each selected video in ``data/results/eval_tracking/video_manifest.csv`` +(selected=True), samples frames from three sources (priority order): + chunk_guided — ±5 frames around each adaptive chunk boundary + occlusion_bracketing — start/mid/end ±3 of the top-K longest occlusion periods + uniform — one frame every UNIFORM_INTERVAL_SECONDS Output: ``data/results/eval_tracking/annotation_frames.csv`` (video_id, frame_idx, source). @@ -20,11 +23,9 @@ from .paths import ANNOTATION_FRAMES, DEFAULT_TRACKER_CONFIG, MANIFEST_CSV, ROOT -# `src.metrics` / `src.tracker.*` pull in torch transitively and only the -# `tracker` pixi env has it. The top-level CLI dispatcher imports this -# module unconditionally to register the `select-frames` subparser, so the -# heavy imports are deferred to call time — letting `tracker-evaluation` -# env users invoke the post-CVAT stages without torch. +# `src.tracking.*` pulls in torch transitively. The top-level CLI dispatcher +# imports this module unconditionally to register the `select-frames` +# subparser, so the heavy imports are deferred to call time. DEFAULT_UNIFORM_INTERVAL_SECONDS = 30.0 BOUNDARY_OFFSETS = (-5, 0, 5) diff --git a/src/tracker_eval/manifest.py b/src/tracker_eval/manifest.py index b6f365e..4f24f40 100644 --- a/src/tracker_eval/manifest.py +++ b/src/tracker_eval/manifest.py @@ -1,4 +1,4 @@ -"""Build `data/tracker_eval/video_manifest.csv` from the cached YOLO scan parquets. +"""Build `data/results/eval_tracking/video_manifest.csv` from the cached YOLO scan parquets. Ranks the 30 candidate videos (5 cages × 3 groups × 2 days) by a composite difficulty score computed across 7 proxies derived from @@ -26,9 +26,7 @@ selected videos come from day 28. The joint cross-day ranking has historically produced 2 day-28 picks without forcing. -Scan-dir discovery walks `--scan-runs-root` (default -`ext-data/output/results/sam3-hf`) for `{YYYYMMDD_HHMMSS}_sam3_hf/{stem}/` -directories containing `yolo_tracking.parquet`. When the same video stem +Scan-dir discovery walks `--scan-runs-root` directories containing `yolo_tracking.parquet`. When the same video stem appears in multiple timestamped runs, the lexicographically latest run is used (timestamp prefix orders correctly). @@ -38,12 +36,11 @@ import argparse import re - from pathlib import Path import pandas as pd -from .paths import MANIFEST_CSV, RAW_VIDEO_ROOT, ROOT, SCAN_RUNS_ROOT +from .paths import MANIFEST_CSV, RAW_VIDEO_ROOT, SCAN_RUNS_ROOT STEM_PATTERN = re.compile(r"^(?PC\d)(?PG\d)_Test_\d+_day_(?P\d+)_") @@ -102,10 +99,14 @@ def discover_scan_dirs(scan_runs_root: Path) -> dict[str, Path]: def compute_difficulty_proxies(scan_dir: Path) -> dict[str, float]: pf = pd.read_parquet(scan_dir / "metrics" / "yolo_scan_metrics.parquet") - summary = pd.read_parquet(scan_dir / "metrics" / "yolo_scan_summary.parquet").iloc[0] + summary = pd.read_parquet(scan_dir / "metrics" / "yolo_scan_summary.parquet").iloc[ + 0 + ] duration_min = float(summary["video_duration_seconds"]) / 60.0 - finite_mcd = pf.loc[pf["mean_centroid_distance"] != float("inf"), "mean_centroid_distance"] + finite_mcd = pf.loc[ + pf["mean_centroid_distance"] != float("inf"), "mean_centroid_distance" + ] return { "frac_high_occlusion": float(pf["is_high_occlusion"].mean()), "mean_overlapping_pairs": float(pf["num_overlapping_pairs"].mean()), @@ -164,17 +165,15 @@ def run(args: argparse.Namespace) -> None: continue proxies = compute_difficulty_proxies(scan_dir) raw_video_path = args.raw_video_root / f"day_{parsed['day']}" / f"{stem}.mp4" - rows.append( - { - "video_id": f"{parsed['cage']}{parsed['group']}_day_{parsed['day']}", - "cage": parsed["cage"], - "group": parsed["group"], - "day": parsed["day"], - "path": str(raw_video_path.relative_to(ROOT)), - "scan_dir": str(scan_dir.relative_to(ROOT)), - **proxies, - } - ) + rows.append({ + "video_id": f"{parsed['cage']}{parsed['group']}_day_{parsed['day']}", + "cage": parsed["cage"], + "group": parsed["group"], + "day": parsed["day"], + "path": str(raw_video_path), + "scan_dir": str(scan_dir), + **proxies, + }) df = pd.DataFrame(rows) counts_by_day = df["day"].value_counts().sort_index().to_dict() @@ -194,9 +193,10 @@ def run(args: argparse.Namespace) -> None: df.groupby("cage")["difficulty"].rank(method="min", ascending=False).astype(int) ) df["selected"] = df["rank_in_cage"] == 1 - df["notes"] = df["selected"].map( - {True: "hardest group in cage across both days", False: ""} - ) + df["notes"] = df["selected"].map({ + True: "hardest group in cage across both days", + False: "", + }) n_day_28 = int(((df["selected"]) & (df["day"] == 28)).sum()) if n_day_28 < args.min_day_28: diff --git a/src/tracker_eval/paths.py b/src/tracker_eval/paths.py index a4c42a6..5bdc74d 100644 --- a/src/tracker_eval/paths.py +++ b/src/tracker_eval/paths.py @@ -1,10 +1,10 @@ """Single source of truth for all paths used by the tracker-eval pipeline. -Data-side paths host small, version-controlled artefacts (manifests, -keyframe schedules, per-variant YAML configs under `data/tracker_eval/config/`, -results CSVs). Ext-data-side paths host heavy artefacts (CVAT Backup, -tracker run parquets, source MP4 clips, MOT files) — symlinked at -`ext-data/`; a regular directory on local dev. +Small artefacts (video manifest, keyframe schedule, results CSVs) live under +`data/results/eval_tracking/`; heavy ones (CVAT backup, ground-truth and +prediction MOT files) under its `tracker_benchmark/` subdir. Tracker run +outputs are read from `data/results/tracking/{config_stem}/`, and the +per-variant YAML configs live in `config/`. Each subcommand's CLI accepts `--manifest` / `--out` / `--predictions-root` overrides; these constants supply the defaults. @@ -13,7 +13,7 @@ from pathlib import Path from typing import Final -from src._config import DEFAULT_RESULTS_DIR, DEFAULT_TRACKING_DIR +from src._config import DEFAULT_RESULTS_DIR, DEFAULT_TRACKING_DIR, DEFAULT_VIDEO_DIR ROOT: Final = Path(__file__).resolve().parents[2] @@ -46,3 +46,7 @@ TRACKER_RUNS_YOLO_BOTSORT_REID_ON: Final = ( f"{DEFAULT_TRACKING_DIR}/yolo_botsort_reid_on" ) + +# build-manifest inputs: YOLO scans (`day_{N}/{video_stem}/`) + source videos +SCAN_RUNS_ROOT: Final = TRACKER_RUNS_SAM3_BEST +RAW_VIDEO_ROOT: Final = DEFAULT_VIDEO_DIR diff --git a/src/tracker_eval/predictions.py b/src/tracker_eval/predictions.py index bcc4795..0dc02d7 100644 --- a/src/tracker_eval/predictions.py +++ b/src/tracker_eval/predictions.py @@ -1,7 +1,8 @@ """Convert raw tracker predictions to MOTChallenge 1.1 .txt files. -Six-way ablation (ordered by Family → Recovery): - - Variant A — YOLO + BoT-SORT only. +Seven-way ablation (ordered by Family → Recovery): + - Variant A — YOLO + BoT-SORT (no re-ID). + - Variant A1 — YOLO + BoT-SORT (re-ID enabled). - Variant B-strict — Grounded-SAM-2 strict (no recovery scaffolding). - Variant B-parity — Grounded-SAM-2 with parity recovery. - Variant C-strict — SAM 3 frame-zero (no scan, no fallbacks). @@ -182,7 +183,7 @@ def _add_args(parser: argparse.ArgumentParser) -> None: def add_subparser(subparsers) -> argparse.ArgumentParser: p = subparsers.add_parser( "convert-preds", - help="Convert tracker prediction parquets (6-way ablation) to MOTChallenge .txt files.", + help="Convert tracker prediction parquets (7-way ablation) to MOTChallenge .txt files.", ) _add_args(p) p.set_defaults(func=run) @@ -235,18 +236,29 @@ def run(args: argparse.Namespace) -> None: ) summary.append( - (video_id, len(a_rows), n_b_strict, n_b_parity, n_c_strict, n_d, len(e_rows)) + ( + video_id, + len(a_rows), + len(a1_rows), + n_b_strict, + n_b_parity, + n_c_strict, + n_d, + len(e_rows), + ) ) print() header = ( - f"{'video_id':<14} {'A':>10} {'B-strict':>10} {'B-parity':>10} " + f"{'video_id':<14} {'A':>10} {'A1-reid':>10} {'B-strict':>10} {'B-parity':>10} " f"{'C-strict':>10} {'D':>10} {'E':>10}" ) print(header) print("-" * len(header)) - for vid, na, nbs, nbp, ncs, nd, ne in summary: - print(f"{vid:<14} {na:>10} {nbs:>10} {nbp:>10} {ncs:>10} {nd:>10} {ne:>10}") + for vid, na, na1, nbs, nbp, ncs, nd, ne in summary: + print( + f"{vid:<14} {na:>10} {na1:>10} {nbs:>10} {nbp:>10} {ncs:>10} {nd:>10} {ne:>10}" + ) def _main() -> None: diff --git a/src/tracker_eval/render_poster_figures.py b/src/tracker_eval/render_poster_figures.py deleted file mode 100644 index 645658e..0000000 --- a/src/tracker_eval/render_poster_figures.py +++ /dev/null @@ -1,787 +0,0 @@ -"""Poster-ready visualisations for the tracker evaluation and classifier. - -Outputs (PNG @ 300 dpi + PDF vector) under ``img/``: - - tracker_eval_aggregate.{png,pdf} — grouped-bar of HOTA/DetA/AssA/IDF1 x variants - - tracker_eval_aggregate_strip.{png,pdf} — same with per-video dots overlaid - - tracker_eval_per_cage_hota.{png,pdf} — HOTA per cage x variants - - tracker_eval_hota_decomposition.{png,pdf} — DetA vs AssA scatter (poster scale) - - tracker_eval_robustness.{png,pdf} — per-video HOTA across the 3 SAM 3 variants - - tracker_eval_summary.{png,pdf} — single 2x2 poster panel combining the above - - confusion_matrix_poster.{png,pdf} — row-normalised confusion matrix with marginal metrics - -Usage: - pixi run -e tracker-evaluation python -m src.tracker_eval.render_poster_figures - -ID-switch counts are deliberately omitted: gs2 has very few switches but -worse tracking (HOTA, DetA), because mask-propagation failures register as -misses and persistent false-positives, not as re-assignments. Showing raw -switch counts invites the misleading "fewer is better" reading. - -Variant palette is family-grouped: gray for A (YOLO), two greens for the -gs2 pair (strict / parity), two blues for SAM 3 strict + fixed, and orange -for the full method E. This groups same-family variants visually so the -within-family steps read as recovery-mechanism ablations and the between- -family steps read as model-backbone swaps. -""" - -from __future__ import annotations - -import matplotlib.colors as mcolors -import matplotlib.lines as mlines -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd - -from .paths import RESULTS_DIR, ROOT - -IMG_DIR = ROOT / "img" - -# ---------------------------------------------------------------------------- -# Style -# ---------------------------------------------------------------------------- - -VARIANT_ORDER = [ - "A_yolo_botsort", - "B_gs2_strict", - "B_gs2_fixed", - "C_sam3_frame_zero", - "D_sam3_fixed", - "E_sam3_adaptive", -] -VARIANT_LABELS = { - "A_yolo_botsort": "A: YOLO + BoT-SORT", - "B_gs2_strict": "B-strict: Grounded-SAM-2 (no recovery)", - "B_gs2_fixed": "B-parity: Grounded-SAM-2 (parity recovery)", - "C_sam3_frame_zero": "C-strict: SAM 3 (frame-0, no fallback)", - "D_sam3_fixed": "D: SAM 3 + adaptive grounding", - "E_sam3_adaptive": "E: SAM 3 + adaptive grounding + chunking (proposed)", -} -# Shortened labels used in the poster-scale HOTA decomposition legend so the -# single-column legend stays compact. -VARIANT_LABELS_SHORT = { - "A_yolo_botsort": "A: YOLO + BoT-SORT", - "B_gs2_strict": "B-s: Grounded-SAM-2 (no recovery)", - "B_gs2_fixed": "B-p: Grounded-SAM-2 (parity recovery)", - "C_sam3_frame_zero": "C: SAM 3 (frame-0, no fallback)", - "D_sam3_fixed": "D: SAM 3 + adaptive grounding", - "E_sam3_adaptive": "E: SAM 3 + adapt. grounding + chunking", -} -# Family-grouped palette: gray YOLO, two greens for gs2 family, two blues for -# SAM 3 strict/fixed, orange for the full method. -VARIANT_COLORS = { - "A_yolo_botsort": "#9A9A9A", # gray - "B_gs2_strict": "#B5D69E", # pale green — gs2 no recovery - "B_gs2_fixed": "#5E8B3C", # green — gs2 with parity recovery - "C_sam3_frame_zero": "#A8C5E3", # pale blue — SAM 3 strict - "D_sam3_fixed": "#2E75B6", # blue — SAM 3 adaptive grounding - "E_sam3_adaptive": "#ED7D31", # orange — full method -} -LETTER_MAP = { - "A_yolo_botsort": "A", - "B_gs2_strict": "B-s", - "B_gs2_fixed": "B-p", - "C_sam3_frame_zero": "C", - "D_sam3_fixed": "D", - "E_sam3_adaptive": "E", -} - -BASE_RC = { - "font.family": "DejaVu Sans", - "font.size": 12, - "axes.titlesize": 14, - "axes.labelsize": 12, - "xtick.labelsize": 11, - "ytick.labelsize": 11, - "legend.fontsize": 10, - "axes.spines.top": False, - "axes.spines.right": False, - "axes.grid": True, - "grid.alpha": 0.25, - "grid.linestyle": "-", - "axes.axisbelow": True, -} - -# Poster-scale overrides used only by the HOTA-decomposition scatter. Scoped -# via plt.rc_context so other figures keep BASE_RC. -POSTER_RC = { - **BASE_RC, - "font.size": 20, - "axes.titlesize": 24, - "axes.labelsize": 22, - "xtick.labelsize": 18, - "ytick.labelsize": 18, -} - -plt.rcParams.update(BASE_RC) - - -def _save(fig, stem: str) -> None: - for ext in ("png", "pdf"): - fig.savefig(IMG_DIR / f"{stem}.{ext}", dpi=300, bbox_inches="tight") - - -def _annotate_bars(ax, bars, fmt="{:.3f}", offset=0.005, fontsize=8): - for b in bars: - h = b.get_height() - if h >= 0: - y = h + offset - va = "bottom" - else: - y = h - offset - va = "top" - ax.text( - b.get_x() + b.get_width() / 2, - y, - fmt.format(h), - ha="center", - va=va, - fontsize=fontsize, - color="#333", - ) - - -# ---------------------------------------------------------------------------- -# Load -# ---------------------------------------------------------------------------- - -agg = ( - pd - .read_csv(RESULTS_DIR / "metrics_aggregate.csv") - .set_index("variant") - .loc[VARIANT_ORDER] -) -per_video = pd.read_csv(RESULTS_DIR / "metrics_per_video.csv") -per_cage = pd.read_csv(RESULTS_DIR / "metrics_per_cage.csv") - -N_VARIANTS = len(VARIANT_ORDER) -BAR_WIDTH = 0.13 # 6 variants × 0.13 = 0.78, leaves 0.22 gap between groups - - -def _offset(i: int) -> float: - """Within-group x-offset for the i-th variant out of N_VARIANTS.""" - return (i - (N_VARIANTS - 1) / 2) * BAR_WIDTH - - -# ---------------------------------------------------------------------------- -# Figure 1 — aggregate grouped bar (HOTA, DetA, AssA, IDF1) -# ---------------------------------------------------------------------------- - - -def fig_aggregate(ax=None, show_legend=True): - metrics = ["HOTA", "DetA", "AssA", "idf1"] - display = ["HOTA", "DetA", "AssA", "IDF1"] - x = np.arange(len(metrics)) - - standalone = ax is None - if standalone: - fig, ax = plt.subplots(figsize=(13, 6)) - - for i, v in enumerate(VARIANT_ORDER): - vals = agg.loc[v, metrics].astype(float).values - bars = ax.bar( - x + _offset(i), - vals, - BAR_WIDTH, - label=VARIANT_LABELS[v], - color=VARIANT_COLORS[v], - edgecolor="white", - linewidth=0.5, - ) - _annotate_bars(ax, bars, fontsize=7) - - ax.set_ylim(0, 0.85) - ax.axhline(0, color="#888", lw=0.8, zorder=1) - ax.set_xticks(x) - ax.set_xticklabels(display) - ax.set_ylabel("Score (higher is better)") - ax.set_title( - "Aggregate tracker metrics — 6-way ablation on 5 occlusion-stressed videos (sparse-keyframe GT)" - ) - if show_legend: - ax.legend( - loc="upper center", - bbox_to_anchor=(0.5, -0.13), - frameon=False, - ncol=3, - fontsize=9.5, - ) - - if standalone: - fig.tight_layout() - _save(fig, "tracker_eval_aggregate") - plt.close(fig) - - -# ---------------------------------------------------------------------------- -# Figure 1b — aggregate grouped bar WITH per-video strip overlay -# ---------------------------------------------------------------------------- - - -def _darken(hex_color: str, amount: float = 0.45) -> str: - h = hex_color.lstrip("#") - r, g, b = [int(h[i : i + 2], 16) for i in (0, 2, 4)] - r, g, b = [max(0, int(c * (1 - amount))) for c in (r, g, b)] - return f"#{r:02x}{g:02x}{b:02x}" - - -def fig_aggregate_strip(ax=None): - metrics = ["HOTA", "DetA", "AssA", "idf1"] - display = ["HOTA", "DetA", "AssA", "IDF1"] - x = np.arange(len(metrics)) - - standalone = ax is None - if standalone: - fig, ax = plt.subplots(figsize=(13.5, 6)) - - per_video_std = per_video.groupby("variant")[metrics].std().loc[VARIANT_ORDER] - for i, v in enumerate(VARIANT_ORDER): - vals = agg.loc[v, metrics].astype(float).values - stds = per_video_std.loc[v, metrics].astype(float).values - bar_x = x + _offset(i) - ax.bar( - bar_x, - vals, - BAR_WIDTH, - label=VARIANT_LABELS[v], - color=VARIANT_COLORS[v], - edgecolor="white", - linewidth=0.5, - alpha=0.55, - zorder=2, - ) - ax.errorbar( - bar_x, - vals, - yerr=stds, - fmt="none", - ecolor="#222", - elinewidth=0.9, - capsize=2.5, - capthick=0.9, - zorder=3, - ) - - videos_sorted = sorted(per_video["video_id"].unique()) - jitter_range = BAR_WIDTH * 0.55 - jitter = np.linspace(-jitter_range / 2, jitter_range / 2, len(videos_sorted)) - jitter_map = dict(zip(videos_sorted, jitter)) - - for i, v in enumerate(VARIANT_ORDER): - dot_color = _darken(VARIANT_COLORS[v], 0.40) - sub = per_video[per_video["variant"] == v] - for j, metric in enumerate(metrics): - for _, row in sub.iterrows(): - vid = row["video_id"] - xj = x[j] + _offset(i) + jitter_map[vid] - ax.scatter( - xj, - row[metric], - s=18, - color=dot_color, - edgecolor="white", - linewidth=0.5, - zorder=4, - ) - - ax.set_ylim(0, 0.85) - ax.axhline(0, color="#888", lw=0.8, zorder=1) - ax.set_xticks(x) - ax.set_xticklabels(display) - ax.set_ylabel("Score (higher is better)") - ax.set_title( - "Aggregate tracker metrics with per-video distribution (5 videos × 6 variants)" - ) - - bar_handles = [ - plt.Rectangle((0, 0), 1, 1, color=VARIANT_COLORS[v], alpha=0.55, ec="white") - for v in VARIANT_ORDER - ] - bar_labels = [VARIANT_LABELS[v] for v in VARIANT_ORDER] - ax.legend( - handles=bar_handles, - labels=bar_labels, - loc="upper center", - bbox_to_anchor=(0.5, -0.10), - ncol=3, - frameon=False, - fontsize=9.5, - ) - - if standalone: - fig.tight_layout() - _save(fig, "tracker_eval_aggregate_strip") - plt.close(fig) - - -# ---------------------------------------------------------------------------- -# Figure 2 — per-cage HOTA grouped bar -# ---------------------------------------------------------------------------- - - -def fig_per_cage_hota(ax=None, show_legend=True): - standalone = ax is None - if standalone: - fig, ax = plt.subplots(figsize=(12, 5.5)) - - cages = sorted(per_cage["cage"].unique()) - x = np.arange(len(cages)) - - for i, v in enumerate(VARIANT_ORDER): - sub = per_cage[per_cage["variant"] == v].set_index("cage").loc[cages] - bars = ax.bar( - x + _offset(i), - sub["HOTA"].values, - BAR_WIDTH, - label=VARIANT_LABELS[v], - color=VARIANT_COLORS[v], - edgecolor="white", - linewidth=0.5, - ) - _annotate_bars(ax, bars, fmt="{:.2f}", offset=0.008, fontsize=6.5) - - ax.set_xticks(x) - ax.set_xticklabels(cages) - ax.set_ylim(0, 0.85) - ax.set_xlabel("Cage") - ax.set_ylabel("HOTA") - ax.set_title("HOTA per cage — cross-environment consistency") - if show_legend: - ax.legend( - loc="upper center", - bbox_to_anchor=(0.5, -0.16), - frameon=False, - ncol=3, - fontsize=9.5, - ) - - if standalone: - fig.tight_layout() - _save(fig, "tracker_eval_per_cage_hota") - plt.close(fig) - - -# ---------------------------------------------------------------------------- -# Figure 3 — DetA vs AssA scatter (HOTA decomposition story) -# ---------------------------------------------------------------------------- - - -def _draw_hota_decomposition(ax, *, poster_scale: bool): - """Shared body for the HOTA-decomposition scatter. - - ``poster_scale=True`` is the standalone (poster) variant: large markers, - bolder annotations, single-column bottom legend with shortened labels. - ``poster_scale=False`` keeps it compact for the 2x2 summary panel. - """ - marker_size = 500 if poster_scale else 240 - edge_lw = 2 if poster_scale else 1.5 - letter_fs = 20 if poster_scale else 12 - iso_lw = 1.2 if poster_scale else 0.8 - iso_label_fs = 14 if poster_scale else 8 - label_offset = (10, 10) if poster_scale else (8, 8) - - for v in VARIANT_ORDER: - ax.scatter( - agg.loc[v, "DetA"], - agg.loc[v, "AssA"], - s=marker_size, - color=VARIANT_COLORS[v], - edgecolor="white", - linewidth=edge_lw, - label=VARIANT_LABELS_SHORT[v] if poster_scale else VARIANT_LABELS[v], - zorder=3, - ) - ax.annotate( - LETTER_MAP[v], - xy=(agg.loc[v, "DetA"], agg.loc[v, "AssA"]), - xytext=label_offset, - textcoords="offset points", - fontsize=letter_fs, - fontweight="bold", - color=VARIANT_COLORS[v], - ) - - # Iso-HOTA contours (HOTA ≈ sqrt(DetA · AssA)) - xs = np.linspace(0.01, 0.85, 200) - for hota_iso in (0.1, 0.3, 0.5, 0.7): - ys = (hota_iso**2) / xs - ax.plot(xs, ys, color="#bbb", lw=iso_lw, ls="--", zorder=1) - x_lab = 0.83 - y_lab = (hota_iso**2) / x_lab - if 0.02 < y_lab < 0.83: - ax.text( - x_lab, - y_lab, - f"HOTA={hota_iso}", - fontsize=iso_label_fs, - color="#888", - va="bottom", - ) - - ax.set_xlim(0, 0.85) - ax.set_ylim(0, 0.85) - ax.set_aspect("equal", adjustable="box") - ax.set_xlabel("DetA (detection quality)") - ax.set_ylabel("AssA (association quality)") - - -def fig_deta_vs_assa(ax=None): - """Compact HOTA-decomposition scatter for use inside the 2x2 summary.""" - standalone = ax is None - if standalone: - fig, ax = plt.subplots(figsize=(6.0, 7.0)) - - _draw_hota_decomposition(ax, poster_scale=False) - ax.set_title("Tracking HOTA performance decomposition") - ax.legend( - loc="upper center", - bbox_to_anchor=(0.5, -0.13), - frameon=False, - fontsize=9.5, - ncol=1, - handletextpad=0.6, - borderaxespad=0.0, - ) - - if standalone: - fig.tight_layout() - _save(fig, "tracker_eval_hota_decomposition") - plt.close(fig) - - -def fig_hota_decomposition_poster(): - """Poster-scale HOTA-decomposition scatter (overrides fig_deta_vs_assa output).""" - with plt.rc_context(POSTER_RC): - fig, ax = plt.subplots(figsize=(10, 10)) - _draw_hota_decomposition(ax, poster_scale=True) - ax.set_title( - "Tracking HOTA performance\ndecomposition", fontweight="bold", pad=12 - ) - - handles = [ - mlines.Line2D( - [], - [], - marker="o", - color=VARIANT_COLORS[v], - markersize=14, - linestyle="None", - markeredgecolor="white", - markeredgewidth=1.2, - ) - for v in VARIANT_ORDER - ] - labels = [VARIANT_LABELS_SHORT[v] for v in VARIANT_ORDER] - ax.legend( - handles, - labels, - loc="upper center", - bbox_to_anchor=(0.5, -0.08), - ncol=1, - frameon=False, - fontsize=16, - handletextpad=0.8, - labelspacing=0.5, - ) - - fig.tight_layout() - _save(fig, "tracker_eval_hota_decomposition") - plt.close(fig) - - -# ---------------------------------------------------------------------------- -# Figure 4 — per-video HOTA trajectory across the 3 SAM 3 variants -# ---------------------------------------------------------------------------- - - -def fig_per_video_robustness(ax=None): - standalone = ax is None - if standalone: - fig, ax = plt.subplots(figsize=(10, 6.2)) - - variants_three = ["C_sam3_frame_zero", "D_sam3_fixed", "E_sam3_adaptive"] - x_pos = {v: i for i, v in enumerate(variants_three)} - - pv = per_video.copy() - sub = pv[pv["variant"].isin(variants_three)][["variant", "video_id", "HOTA"]] - wide = sub.pivot(index="video_id", columns="variant", values="HOTA") - wide = wide[variants_three] - - line_color = "#888" - - for vid, row in wide.iterrows(): - xs = [x_pos[v] for v in variants_three] - ys = [row[v] for v in variants_three] - ax.plot(xs, ys, color=line_color, lw=1.2, alpha=0.55, zorder=2) - for v in variants_three: - ax.scatter( - x_pos[v], - row[v], - s=80, - color=VARIANT_COLORS[v], - edgecolor="white", - linewidth=1.3, - zorder=4, - ) - - label_x_anchor = x_pos["E_sam3_adaptive"] + 0.10 - label_x_text = x_pos["E_sam3_adaptive"] + 0.16 - min_gap = 0.028 - - labelled = ( - wide - .assign(vid=wide.index) - .sort_values(by="E_sam3_adaptive", ascending=False) - .reset_index(drop=True) - ) - label_ys = labelled["E_sam3_adaptive"].astype(float).tolist() - for i in range(1, len(label_ys)): - if label_ys[i - 1] - label_ys[i] < min_gap: - label_ys[i] = label_ys[i - 1] - min_gap - - for i, r in labelled.iterrows(): - vid = r["vid"] - y_dot = r["E_sam3_adaptive"] - y_lab = label_ys[i] - if abs(y_dot - y_lab) > 1e-4: - ax.plot( - [x_pos["E_sam3_adaptive"] + 0.025, label_x_anchor], - [y_dot, y_lab], - color="#444", - lw=0.7, - alpha=0.7, - zorder=3, - ) - ax.text( - label_x_text, - y_lab, - vid.replace("_day_", " · day "), - fontsize=10, - va="center", - color="#444", - ) - - means = wide.mean() - for v in variants_three: - ax.hlines( - means[v], - x_pos[v] - 0.20, - x_pos[v] + 0.20, - colors="#222", - linestyles="--", - lw=1.2, - zorder=3, - ) - ax.text( - x_pos[v] - 0.22, - means[v], - f"{means[v]:.3f}", - ha="right", - va="center", - fontsize=9, - color="#222", - ) - - stds = wide.std() - for v in variants_three: - ax.text( - x_pos[v], - 0.04, - f"std = {stds[v]:.3f}", - ha="center", - va="bottom", - fontsize=9.5, - color="#222", - bbox=dict(boxstyle="round,pad=0.32", fc="#f3f3f3", ec="#ccc", lw=0.6), - ) - - ax.set_xlim(-0.55, 2.85) - ax.set_ylim(0, 0.85) - ax.set_xticks([x_pos[v] for v in variants_three]) - ax.set_xticklabels([ - "C-strict\n(no scan, no fallback)", - "D\n(+ adaptive grounding)", - "E\n(+ adaptive chunking)", - ]) - ax.set_ylabel("HOTA") - ax.set_title( - "Per-video HOTA — SAM 3 variants (adaptive grounding drives the big jump)", - fontsize=12.5, - ) - ax.grid(True, axis="y", alpha=0.25) - - if standalone: - fig.tight_layout() - _save(fig, "tracker_eval_robustness") - plt.close(fig) - - -# ---------------------------------------------------------------------------- -# Figure 5 — single 2x2 poster panel -# ---------------------------------------------------------------------------- - - -def fig_summary(): - fig = plt.figure(figsize=(18, 11)) - gs = fig.add_gridspec( - 2, 2, hspace=0.46, wspace=0.22, top=0.83, bottom=0.07, left=0.06, right=0.98 - ) - - ax1 = fig.add_subplot(gs[0, :]) - fig_aggregate(ax=ax1, show_legend=False) - - ax2 = fig.add_subplot(gs[1, 0]) - fig_per_cage_hota(ax=ax2, show_legend=False) - - ax3 = fig.add_subplot(gs[1, 1]) - fig_deta_vs_assa(ax=ax3) - leg = ax3.get_legend() - if leg is not None: - leg.remove() - - handles = [ - plt.Rectangle((0, 0), 1, 1, color=VARIANT_COLORS[v], ec="white") - for v in VARIANT_ORDER - ] - labels = [VARIANT_LABELS[v] for v in VARIANT_ORDER] - fig.legend( - handles, - labels, - loc="upper center", - bbox_to_anchor=(0.5, 0.92), - ncol=3, - frameon=False, - fontsize=11, - ) - - fig.suptitle( - "Tracker evaluation — 6-way ablation on 5 occlusion-stressed videos (sparse-keyframe GT)", - fontsize=17, - fontweight="bold", - y=0.97, - ) - _save(fig, "tracker_eval_summary") - plt.close(fig) - - -# ---------------------------------------------------------------------------- -# Figure 6 — row-normalised confusion matrix with marginal metrics -# ---------------------------------------------------------------------------- - -# Values from Table 4 (3-class, social excluded). -CM_LABELS = ["Other", "Obj.", "Loc."] -CM_PCT = np.array([ - [93.5, 4.8, 1.7], - [31.2, 66.1, 2.7], - [8.5, 6.7, 84.8], -]) -CM_F1 = [94.8, 61.9, 74.4] -CM_PRECISIONS = [96.2, 58.2, 66.2] -CM_SUPPORTS = [12_585, 1_345, 585] - - -def fig_confusion_matrix(): - fig, ax = plt.subplots(figsize=(6, 5.2), dpi=200) - - cmap = plt.cm.Blues - norm = mcolors.Normalize(vmin=0, vmax=100) - im = ax.imshow(CM_PCT, cmap=cmap, norm=norm, aspect="equal") - - n = len(CM_LABELS) - for i in range(n): - for j in range(n): - val = CM_PCT[i, j] - color = "white" if val > 55 else "black" - ax.text( - j, - i, - f"{val:.1f}", - ha="center", - va="center", - fontsize=16, - fontweight="bold", - color=color, - ) - - ax.set_xticks(range(n)) - ax.set_xticklabels(CM_LABELS, fontsize=13) - ax.set_yticks(range(n)) - ax.set_yticklabels(CM_LABELS, fontsize=13) - ax.set_xlabel("Predicted", fontsize=14, labelpad=8) - ax.set_ylabel("True", fontsize=14, labelpad=8) - ax.xaxis.set_ticks_position("top") - ax.xaxis.set_label_position("top") - - margin = 0.15 - for i in range(n): - ax.text( - n - 0.5 + margin, - i, - f"F₁ {CM_F1[i]:.1f}", - ha="left", - va="center", - fontsize=11, - color="#333", - ) - ax.text( - n - 0.5 + margin, - i + 0.30, - f"n={CM_SUPPORTS[i]:,}", - ha="left", - va="center", - fontsize=8.5, - color="#888", - ) - ax.text( - i, - n - 0.5 + margin, - f"P {CM_PRECISIONS[i]:.1f}", - ha="center", - va="top", - fontsize=11, - color="#333", - ) - - ax.set_xlim(-0.5, n - 0.5 + 1.35) - ax.set_ylim(n - 0.5 + 0.55, -0.5) - - ax.set_xticks([x - 0.5 for x in range(1, n)], minor=True) - ax.set_yticks([y - 0.5 for y in range(1, n)], minor=True) - ax.grid(which="minor", color="white", linewidth=2) - ax.tick_params(which="minor", length=0) - - cbar = fig.colorbar(im, ax=ax, fraction=0.035, pad=0.04, shrink=0.82) - cbar.set_label("Row-normalised (%)", fontsize=11) - cbar.ax.tick_params(labelsize=10) - - fig.tight_layout() - _save(fig, "confusion_matrix_poster") - plt.close(fig) - - -# ---------------------------------------------------------------------------- -# Main -# ---------------------------------------------------------------------------- - - -def main() -> None: - IMG_DIR.mkdir(parents=True, exist_ok=True) - fig_aggregate() - fig_aggregate_strip() - fig_per_cage_hota() - fig_per_video_robustness() - fig_summary() - # Poster-scale variant runs last so it overrides the compact HOTA - # decomposition output written by fig_summary(). - fig_hota_decomposition_poster() - fig_confusion_matrix() - print("Wrote:") - for p in sorted(IMG_DIR.glob("tracker_eval_*.png")): - print(f" {p.relative_to(ROOT)}") - for p in sorted(IMG_DIR.glob("tracker_eval_*.pdf")): - print(f" {p.relative_to(ROOT)}") - for p in sorted(IMG_DIR.glob("confusion_matrix_poster.*")): - print(f" {p.relative_to(ROOT)}") - - -if __name__ == "__main__": - main() diff --git a/src/tracking/grounding.py b/src/tracking/grounding.py index 48ffb09..8f3479b 100644 --- a/src/tracking/grounding.py +++ b/src/tracking/grounding.py @@ -8,7 +8,6 @@ from typing import Callable import numpy as np - from loguru import logger from ..utils.memory import free_gpu_memory @@ -95,9 +94,9 @@ def find_best_grounding_frame( masks_list, boxes_list, object_ids_list = get_all_objects_from_results(results) if len(object_ids_list) < min_objects: continue - masks_array = np.stack( - [m.squeeze(0) if m.ndim == 3 and m.shape[0] == 1 else m for m in masks_list] - ).astype(bool) + masks_array = np.stack([ + m.squeeze(0) if m.ndim == 3 and m.shape[0] == 1 else m for m in masks_list + ]).astype(bool) max_iou = compute_max_pairwise_iou(masks_array) scores = results.get("scores") @@ -111,9 +110,14 @@ def find_best_grounding_frame( else: mean_score = 0.0 - candidates.append( - (frame_idx, masks_list, boxes_list, object_ids_list, max_iou, mean_score) - ) + candidates.append(( + frame_idx, + masks_list, + boxes_list, + object_ids_list, + max_iou, + mean_score, + )) if not candidates: return None, [], [], [] @@ -135,7 +139,8 @@ def match_grounding_ids_to_previous( prev_masks: list, prev_ids: list, iou_threshold: float = 0.10, -) -> dict[int, int]: + return_matched_ids: bool = False, +) -> dict[int, int] | tuple[dict[int, int], set[int]]: """ Greedy IoU-based assignment of grounding IDs to previous-chunk IDs. @@ -144,9 +149,16 @@ def match_grounding_ids_to_previous( their original value if it does not collide with an already-mapped ID; otherwise they are assigned a fresh ID to avoid silent key collisions in grounding_prompt_points. + + Args: + return_matched_ids: When True, also return the set of grounding IDs + that were genuinely matched to a previous-chunk ID (as opposed to + passed through unmatched). Callers use this to compute the match + ratio reported in chunk_info. """ if not grounding_masks or not prev_masks: - return {int(pid): int(pid) for pid in grounding_ids} + id_map = {int(pid): int(pid) for pid in grounding_ids} + return (id_map, set()) if return_matched_ids else id_map def _to_bool(m): arr = m.squeeze(0) if m.ndim == 3 and m.shape[0] == 1 else m @@ -193,4 +205,82 @@ def _to_bool(m): claimed.add(next_fresh) next_fresh += 1 + if return_matched_ids: + matched_ids = {int(grounding_ids[i]) for i in assigned_gs} + return id_map, matched_ids return id_map + + +def find_best_overlap_prev_frame( + prev_chunk_outputs: dict, + grounding_masks: list, + min_objects: int = 3, + max_lookback: int = 50, +) -> tuple[int | None, list, list, list]: + """ + Scan backward through previous chunk outputs and find the frame whose masks + have the highest total greedy IoU overlap with the current grounding masks. + + For each candidate frame with >= min_objects, computes a greedy total IoU: + for each grounding mask, find the best IoU with any prev mask, then sum. + Returns the frame with the highest total overlap. + + Drop-in replacement for find_frame_with_enough_objects() — same return + signature: (frame_idx, masks_list, boxes_list, object_ids_list). + """ + if not grounding_masks: + return None, [], [], [] + + def _to_bool(m): + arr = m.squeeze(0) if m.ndim == 3 and m.shape[0] == 1 else m + return arr.astype(bool) + + gr_bool = [_to_bool(m) for m in grounding_masks] + + sorted_frames = sorted(prev_chunk_outputs.keys(), reverse=True) + candidates = [] + + for frame_idx in sorted_frames[:max_lookback]: + masks_list, boxes_list, object_ids_list = get_all_objects_from_results( + prev_chunk_outputs[frame_idx] + ) + if len(object_ids_list) < min_objects: + continue + + prev_bool = [_to_bool(m) for m in masks_list] + + # Greedy total IoU: for each grounding mask, find best IoU with any prev mask + total_iou = 0.0 + for gm in gr_bool: + best_iou = 0.0 + for pm in prev_bool: + inter = float((gm & pm).sum()) + union = float((gm | pm).sum()) + iou = inter / union if union > 0 else 0.0 + if iou > best_iou: + best_iou = iou + total_iou += best_iou + + candidates.append(( + total_iou, + frame_idx, + masks_list, + boxes_list, + object_ids_list, + )) + + if not candidates: + logger.warning( + f"[best-overlap] no frame with >= {min_objects} objects " + f"in last {max_lookback} frames" + ) + return None, [], [], [] + + # Pick highest total overlap + candidates.sort(key=lambda x: -x[0]) + best = candidates[0] + logger.debug( + f"[best-overlap] selected frame {best[1]} with total_iou={best[0]:.3f} " + f"(searched {len(candidates)} candidate frames)" + ) + return best[1], best[2], best[3], best[4] diff --git a/src/tracking/sam3.py b/src/tracking/sam3.py index 2d3ec85..8738f71 100644 --- a/src/tracking/sam3.py +++ b/src/tracking/sam3.py @@ -1,4 +1,4 @@ -"""Tracking pipeline.""" +"""Tracking pipeline""" import json import shutil @@ -14,6 +14,7 @@ from loguru import logger from omegaconf import OmegaConf from simpler_timer import SimplerTimer +from tqdm import tqdm from transformers import ( Sam3TrackerVideoConfig, Sam3TrackerVideoModel, @@ -39,6 +40,7 @@ ) from .grounding import ( find_best_grounding_frame, + find_best_overlap_prev_frame, match_grounding_ids_to_previous, run_grounding, ) @@ -51,13 +53,14 @@ per_run_metrics_to_multiindex_df, summary_metrics_to_df, ) -from .._config import DEFAULT_TRACKER_CONFIG +from .._config import DEFAULT_TRACKER_CONFIG, DEFAULT_TRACKING_DIR from .scan import run_yolo_scan, yolo_scan_to_df from .utils import ( extract_equidistant_points_from_masks, find_frame_with_enough_objects, process_tracking_outputs, reseed_tracker_memory, + to_numpy, ) from .viz import annotate_video_with_sam3_outputs, generate_all_visualizations @@ -70,6 +73,24 @@ # --------------------------------------------------------------------------- +def _remap_output_ids(frame_output: dict, id_map: dict) -> dict: + """Remap object IDs in a single frame's output dict (object_ids, tracker scores, removed/suppressed sets).""" + out = dict(frame_output) + old_ids = to_numpy(out["object_ids"]) + new_ids = np.array([id_map.get(int(oid), int(oid)) for oid in old_ids]) + out["object_ids"] = new_ids + + if "obj_id_to_tracker_score" in out and out["obj_id_to_tracker_score"]: + out["obj_id_to_tracker_score"] = { + id_map.get(int(k), int(k)): v + for k, v in out["obj_id_to_tracker_score"].items() + } + for set_key in ("removed_obj_ids", "suppressed_obj_ids"): + if set_key in out and out[set_key]: + out[set_key] = {id_map.get(int(x), int(x)) for x in out[set_key]} + return out + + def _process_video_chunk(chunk_frames, start_idx, cfg, device): """ Process a chunk with Sam3VideoModel (text-prompted segmentation). @@ -135,9 +156,12 @@ def _process_video_chunk(chunk_frames, start_idx, cfg, device): total_frames = len(chunk_frames) outputs_per_frame = {} with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16): - for model_outputs in model.propagate_in_video_iterator( - inference_session=inference_session, - max_frame_num_to_track=total_frames, + for model_outputs in tqdm( + model.propagate_in_video_iterator( + inference_session=inference_session, + max_frame_num_to_track=total_frames, + ), + total=total_frames, ): processed_outputs = processor.postprocess_outputs( inference_session, model_outputs @@ -152,12 +176,6 @@ def _process_video_chunk(chunk_frames, start_idx, cfg, device): ) global_frame_idx = start_idx + model_outputs.frame_idx outputs_per_frame[global_frame_idx] = processed_outputs - local_frame_idx = model_outputs.frame_idx - if local_frame_idx % 25 == 0 or local_frame_idx == total_frames - 1: - logger.info( - f" [text] frame {local_frame_idx + 1}/{total_frames} " - f"({100 * (local_frame_idx + 1) / total_frames:.0f}%)" - ) # Cleanup if hasattr(inference_session, "reset_inference_session"): @@ -257,11 +275,10 @@ def _process_tracker_chunk(chunk_frames, start_idx, all_prompt_points, cfg, devi ) # Process all frames - total_frames = len(chunk_frames) outputs_per_frame = {} with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16): for tracker_output in model.propagate_in_video_iterator( - inference_session, show_progress_bar=False + inference_session, show_progress_bar=True ): # Post-process masks video_res_masks = processor.post_process_masks( @@ -364,11 +381,6 @@ def _process_tracker_chunk(chunk_frames, start_idx, all_prompt_points, cfg, devi "scores": active_scores, "obj_id_to_tracker_score": obj_id_to_tracker_score, } - if local_frame_idx % 25 == 0 or local_frame_idx == total_frames - 1: - logger.info( - f" [tracker] frame {local_frame_idx + 1}/{total_frames} " - f"({100 * (local_frame_idx + 1) / total_frames:.0f}%)" - ) # Cleanup if hasattr(inference_session, "reset_inference_session"): @@ -388,7 +400,7 @@ def _run_single_video(cfg, run_dir: Path, config_path: Path | None = None): metrics_dir = run_dir / "metrics" metrics_dir.mkdir(parents=True, exist_ok=True) - job_type = cfg.job_type + job_type = cfg.get("job_type", "tracker") log_file = setup_logger(run_dir, job_type=job_type) logger.info("=" * 60) @@ -723,12 +735,29 @@ def _run_single_video(cfg, run_dir: Path, config_path: Path | None = None): results_path = run_dir / "tracking_outputs.parquet" grounding_results_path = run_dir / "grounding_outputs.parquet" + frame_loader = str(cfg.get("frame_loader", "torchcodec")) + if frame_loader not in FRAME_LOADERS: + raise ValueError( + f"Unknown frame_loader {frame_loader!r}; expected one of {FRAME_LOADERS}" + ) + if frame_loader != "torchcodec": + logger.warning( + f"Frame loader: {frame_loader} (not the default torchcodec) — " + f"decoded frames will differ from a default run" + ) + else: + logger.info(f"Frame loader: {frame_loader}") + for chunk_idx, (start_idx, end_idx, chunk_type) in enumerate(chunks): # Load only this chunk's frames from disk — avoids holding the full video in RAM global_chunk_start = start_frame + start_idx global_chunk_end = start_frame + end_idx - chunk_frames = load_video_frames_torchcodec( - video_path, global_chunk_start, global_chunk_end, device=str(device) + chunk_frames = load_video_frames( + video_path, + global_chunk_start, + global_chunk_end, + loader=frame_loader, + device=str(device), ) num_frames = len(chunk_frames) @@ -763,6 +792,7 @@ def _run_single_video(cfg, run_dir: Path, config_path: Path | None = None): use_tracker = False all_prompt_points = {} grounding_frame_offset = 0 # local offset into chunk_frames for tracker init + grounding_prefix_outputs = {} # gap fill, see fill_grounding_gap below grounding_cfg = cfg.get("text_grounding", {}) grounding_enabled = grounding_cfg.get("enabled", False) @@ -816,16 +846,39 @@ def _run_single_video(cfg, run_dir: Path, config_path: Path | None = None): if previous_chunk_outputs is not None and grounding_cfg.get( "id_matching", True ): - _, prev_masks_for_iou, _, prev_ids_for_iou = ( - find_frame_with_enough_objects( - previous_chunk_outputs, - # min_objects=3, - min_objects=cfg.min_objects_for_tracking, - max_lookback=cfg.max_lookback_frames, + # best_overlap_id_matching defaults to FALSE + if grounding_cfg.get("best_overlap_id_matching", False): + # Score each candidate prev frame by greedy IoU against + # the grounding masks instead of taking the first frame + # with enough objects. + dynamic_lookback = max( + cfg.max_lookback_frames, + len(previous_chunk_outputs) // 2, ) + prev_ref_frame, prev_masks_for_iou, _, prev_ids_for_iou = ( + find_best_overlap_prev_frame( + previous_chunk_outputs, + gr_out_masks, + min_objects=cfg.min_objects_for_tracking, + max_lookback=dynamic_lookback, + ) + ) + chunk_info["best_overlap_used"] = True + else: + prev_ref_frame, prev_masks_for_iou, _, prev_ids_for_iou = ( + find_frame_with_enough_objects( + previous_chunk_outputs, + min_objects=cfg.min_objects_for_tracking, + max_lookback=cfg.max_lookback_frames, + ) + ) + chunk_info["best_overlap_used"] = False + chunk_info["prev_reference_frame_idx"] = ( + int(prev_ref_frame) if prev_ref_frame is not None else None ) + if prev_masks_for_iou: - id_map = match_grounding_ids_to_previous( + id_map, matched_grounding_ids = match_grounding_ids_to_previous( gr_out_masks, gr_out_ids, prev_masks_for_iou, @@ -833,7 +886,22 @@ def _run_single_video(cfg, run_dir: Path, config_path: Path | None = None): iou_threshold=grounding_cfg.get( "id_match_iou_threshold", 0.10 ), + return_matched_ids=True, + ) + match_ratio = ( + len(matched_grounding_ids) / len(gr_out_ids) + if gr_out_ids + else 0.0 ) + chunk_info["id_match_ratio"] = round(match_ratio, 3) + min_ratio = grounding_cfg.get("id_match_min_ratio", 0.5) + if match_ratio < min_ratio: + discontinuity = "full" if match_ratio == 0.0 else "partial" + chunk_info["id_discontinuity"] = discontinuity + logger.warning( + f" [grounding] ID discontinuity ({discontinuity}): " + f"match_ratio={match_ratio:.2f} < {min_ratio}" + ) else: id_map = {int(gid): int(gid) for gid in gr_out_ids} else: @@ -943,6 +1011,29 @@ def _run_single_video(cfg, run_dir: Path, config_path: Path | None = None): f"Grounding outputs saved to {grounding_results_path} (chunk {chunk_idx})" ) + # --- Grounding gap fill --- + # The tracker initialises at the best grounding frame, so frames + # [chunk_start, best_grounding_frame) would otherwise have no + # predictions at all. When enabled, reuse the grounding model's own + # outputs for that prefix, remapped to the tracker's IDs. + fill_gap = grounding_cfg.get("fill_grounding_gap", False) + if fill_gap and grounding_frame_offset > 0 and use_tracker: + for fidx in range( + global_chunk_start, global_chunk_start + grounding_frame_offset + ): + if fidx in grounding_outputs: + grounding_prefix_outputs[fidx] = _remap_output_ids( + grounding_outputs[fidx], id_map + ) + if grounding_prefix_outputs: + logger.info( + f" [grounding] gap fill: {len(grounding_prefix_outputs)} " + f"prefix frame(s) extracted from grounding outputs" + ) + chunk_info["grounding_gap_frames_filled"] = len( + grounding_prefix_outputs + ) + del grounding_outputs free_gpu_memory() @@ -1037,6 +1128,11 @@ def _run_single_video(cfg, run_dir: Path, config_path: Path | None = None): chunk_frames, global_start_idx, cfg, device ) + # Merge grounding prefix frames (gap fill) — prefix goes first so the + # tracker's own output wins on any overlapping frame. + if grounding_prefix_outputs: + chunk_outputs = {**grounding_prefix_outputs, **chunk_outputs} + # Stop timer and calculate metrics elapsed_seconds = timer.end() avg_sec_per_frame = elapsed_seconds / max(1, num_frames) @@ -1093,7 +1189,9 @@ def _run_single_video(cfg, run_dir: Path, config_path: Path | None = None): # Save chunk info JSON chunk_info_path = run_dir / "chunk_info.json" with open(chunk_info_path, "w") as f: - json.dump({"chunks": chunk_info_list}, f, indent=2) + json.dump( + {"frame_loader": frame_loader, "chunks": chunk_info_list}, f, indent=2 + ) logger.info(f"Chunk info saved to: {chunk_info_path}") # ----------------------------------------------------------------------- @@ -1250,10 +1348,11 @@ def run(cfg, config_path: str | Path): if not video_path and not video_dir: raise ValueError("Must specify either video_path or video_dir.") - job_type = "yolo_scan" if cfg.get("yolo_scan_only", False) else cfg.job_type - - # Timestamped parent isolates each run from previous ones - batch_dir = create_run_directory(Path(cfg.output_dir), job_type) + # Same layout as pipeline/run_tracker.py: {DEFAULT_TRACKING_DIR}/{config_stem}/ + batch_dir = Path( + cfg.get("output_dir", None) or Path(DEFAULT_TRACKING_DIR) / config_path.stem + ) + batch_dir.mkdir(parents=True, exist_ok=True) if video_dir: _run_batch(cfg, batch_dir, Path(video_dir), config_path=config_path) diff --git a/src/tracking/scan.py b/src/tracking/scan.py index 002e103..e33343c 100644 --- a/src/tracking/scan.py +++ b/src/tracking/scan.py @@ -1,12 +1,9 @@ """ YOLO-based raw inference scan. -Runs YOLO+ByteTrack on a video and returns a per-detection DataFrame -(``yolo_tracking.parquet``). All downstream analysis — per-frame metrics, -occlusion periods, separation windows, and adaptive chunking — lives in -``src.yolo.boundaries``. - -Re-exports from ``src.yolo.boundaries`` are provided so that existing callers continue to work unchanged. +Runs YOLO + BoT-SORT on a video and returns a per-detection DataFrame +(``yolo_tracking.parquet``). Also holds the scan analysis used by adaptive +chunking: high-separation windows, occlusion periods, and scan summaries. """ from pathlib import Path @@ -351,7 +348,7 @@ def run_yolo_scan( except ImportError as exc: raise ImportError( "ultralytics is required for YOLO scan. " - "Install it with: pixi install -e sam3-hf" + "Install it with: pixi install -e tracker" ) from exc video_path = str(video_path) diff --git a/src/tracking/utils.py b/src/tracking/utils.py index aa085e2..afa3b81 100644 --- a/src/tracking/utils.py +++ b/src/tracking/utils.py @@ -14,6 +14,7 @@ def to_numpy(x): + """Convert a torch tensor (or array-like) to a numpy array.""" if hasattr(x, "cpu"): x = x.cpu() if hasattr(x, "numpy"): @@ -278,6 +279,7 @@ def reseed_tracker_memory( def process_tracking_outputs(outputs_per_frame): + """Convert per-frame SAM3 output dicts to a parquet-ready DataFrame with RLE masks.""" index_tuples = [] bboxes = [] counts_list = [] diff --git a/src/tracking/viz.py b/src/tracking/viz.py index bfdf836..523da3c 100644 --- a/src/tracking/viz.py +++ b/src/tracking/viz.py @@ -51,6 +51,7 @@ def overlay_masks(image, masks): def convert_numpy_to_pil(array): + """Convert a numpy array to a PIL RGB Image.""" return Image.fromarray(np.uint8(array)).convert("RGB") @@ -1320,6 +1321,9 @@ def plot_chunk_boundary_frames( save_path : str or Path, optional Path to save the PNG. If None the plot is shown interactively. """ + if video_path is None: + logger.warning("plot_chunk_boundary_frames: no video_path provided, skipping") + return video_path = Path(video_path) if not video_path.exists(): logger.warning( diff --git a/tests/check_crop_modes.py b/tests/check_crop_modes.py index 3d59b1e..94c5310 100644 --- a/tests/check_crop_modes.py +++ b/tests/check_crop_modes.py @@ -19,8 +19,6 @@ CROP_MODES, compute_union_origin, crop_frame, - needs_mask, - union_crop_size, ) from src.dataset.embeddings import _split_mask_thirds from src.dataset.utils import resolve_video_path From 28ebeb9a98f4f07a76af2a18d152646d513d611b Mon Sep 17 00:00:00 2001 From: Prince Ravi Leow Date: Wed, 23 Sep 2026 16:17:22 +0200 Subject: [PATCH 2/4] chore: pre-release follow-ups on top of upstream merge - pixi.toml: add test_imports_full (tracker env) so modules that CI skips for lack of torch get import-checked before a release; document it in the README Tests section. - pixi.toml/pixi.lock: add tabulate to the default env. pandas DataFrame.to_markdown (clustering_grid) and TrackEval need it; it was missing from every env. - src/tracking/sam3.py, pipeline/run_tracker.py: put the chunked-SAM3 description back on sam3.py; run_tracker.py keeps its launcher summary plus usage. - README.md: postprocessing runs in the default env, not tracker. - config/sam3_best.yaml: reuse_run_dir example points at a per-video run dir ({video_stem}), not a timestamped batch name. - pipeline/build_dataset.py: usage uses the build_dataset pixi task. - Drop unused imports (eval_classification.py, extract_embeddings_videoprism.py, tests/test_labels.py, os in features.py); features.py imports torch under TYPE_CHECKING for its annotation. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 6 +++++- config/sam3_best.yaml | 2 +- pipeline/build_dataset.py | 4 ++-- pipeline/eval_classification.py | 6 +----- pipeline/extract_embeddings_videoprism.py | 2 -- pipeline/run_tracker.py | 10 ++-------- pixi.lock | 21 +++++++++++++++++++++ pixi.toml | 5 +++++ src/dataset/features.py | 6 +++++- src/tracking/sam3.py | 9 ++++++++- tests/test_labels.py | 1 - 11 files changed, 50 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 87f1a9e..7e014fb 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ holds reusable library modules. | 1. Data | [data/README.md](data/README.md) | — | | 2. Tracking | [docs/2_tracking.md](docs/2_tracking.md) | `tracker`, `gs2` | | 3. Tracker evaluation _(optional)_ | [docs/3_tracker_eval.md](docs/3_tracker_eval.md) | `tracker`, `gs2` | -| 4. Postprocessing | [docs/4_postprocessing.md](docs/4_postprocessing.md) | `tracker` | +| 4. Postprocessing | [docs/4_postprocessing.md](docs/4_postprocessing.md) | `default` | | 5. Build dataset | [docs/5_dataset.md](docs/5_dataset.md) | `default` | | 6. Embeddings | [docs/6_embeddings.md](docs/6_embeddings.md) | `default`, `videoprism` | | 7. Classification | [docs/7_classification.md](docs/7_classification.md) | `default` | @@ -87,8 +87,12 @@ pixi run test_val_rotation # Cross-validation tests pixi run test_tracking_postprocessing # Tracking postprocessing unit tests pixi run test_tracker # Tracker unit tests (requires CUDA) pixi run test_dataset_integrity # Data integrity checks (requires a built dataset) +pixi run test_imports_full # Import every src/ and pipeline/ module with all deps (run before a release) ``` +CI runs `pixi run -e ci test` in a lightweight environment without torch, so +modules that import torch are skipped there. `test_imports_full` covers them. + ## Citation ```bibtex diff --git a/config/sam3_best.yaml b/config/sam3_best.yaml index 4398ee9..2c94014 100644 --- a/config/sam3_best.yaml +++ b/config/sam3_best.yaml @@ -21,7 +21,7 @@ max_frames_to_track: 0 # set to 0 to track the entire video # --- Mode 1: Reuse a previous run's chunk boundaries --- reuse_chunk_info: false -reuse_run_dir: null # e.g. "data/results/tracking/sam3_best/day_28/20260306_152733_sam3_hf" +reuse_run_dir: null # e.g. "data/results/tracking/sam3_best/day_28/{video_stem}" # --- Mode 3/4: Adaptive or fixed chunking (active when reuse_chunk_info: false) --- yolo_scan_only: false diff --git a/pipeline/build_dataset.py b/pipeline/build_dataset.py index f3fd893..550b670 100644 --- a/pipeline/build_dataset.py +++ b/pipeline/build_dataset.py @@ -18,12 +18,12 @@ Typical workflow:: # First run: generates JSONs in data/postprocessing/ - pixi run python -m pipeline.build_dataset + pixi run build_dataset # Manually fill in "to" values in each tracking_postprocessing.json # Second run: validates remaps, builds dataset - pixi run python -m pipeline.build_dataset + pixi run build_dataset """ import json diff --git a/pipeline/eval_classification.py b/pipeline/eval_classification.py index f1d65e1..3cee6a1 100644 --- a/pipeline/eval_classification.py +++ b/pipeline/eval_classification.py @@ -19,7 +19,6 @@ import json from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser -from glob import glob from pathlib import Path import numpy as np @@ -48,12 +47,9 @@ def _load_cfg(run_dir: Path) -> dict: def _build_dm_and_model_cls(cfg: dict): """Reconstruct datamodule and model class from a run config.""" - import torch - from src.classification.datamodule import BehaviourDataModule from src.classification.model_selection import LOCO from src.classification.models import MODEL_REGISTRY - from src.classification.trainer import BehaviourClassifier from src.classification.utils import parse_input use_features, use_embeddings, embeddings_files = parse_input(cfg["input"]) @@ -113,7 +109,7 @@ def evaluate_run(run_dir: Path, device: str = "cuda:0", predictions: bool = True import torch import torch.nn.functional as F - from src.classification.stats import _save_per_fold_cms, _save_recall_txt + from src.classification.stats import _save_recall_txt run_dir = Path(run_dir) cfg = _load_cfg(run_dir) diff --git a/pipeline/extract_embeddings_videoprism.py b/pipeline/extract_embeddings_videoprism.py index 7749fdc..98f5aef 100644 --- a/pipeline/extract_embeddings_videoprism.py +++ b/pipeline/extract_embeddings_videoprism.py @@ -36,8 +36,6 @@ import torch from loguru import logger -from PIL import Image -from tqdm import tqdm from videoprism import models as vp tf.config.set_visible_devices([], "GPU") diff --git a/pipeline/run_tracker.py b/pipeline/run_tracker.py index 93bccad..f4cb560 100644 --- a/pipeline/run_tracker.py +++ b/pipeline/run_tracker.py @@ -1,9 +1,4 @@ -""" -Unified launcher for tracking pipelines (SAM 3, Grounded-SAM-2). - -Each video is processed in chunks: chunk 0 uses Sam3VideoModel (text-prompted -segmentation); subsequent chunks use Sam3TrackerVideoModel (point-prompted), -initialised from masks sampled at the cleanest frame in the previous chunk. +"""Unified launcher for tracking pipelines (SAM 3, Grounded-SAM-2). Usage: # Run with default config (config/sam3_best.yaml) @@ -120,8 +115,7 @@ def run( _run_batch(cfg, batch_dir, video_path, config_path=config_path) else: videos = sorted( - f - for f in video_path.iterdir() + f for f in video_path.iterdir() if f.is_file() and f.suffix in VIDEO_EXTENSIONS ) for video_file in videos: diff --git a/pixi.lock b/pixi.lock index b9fa8cf..eef8860 100644 --- a/pixi.lock +++ b/pixi.lock @@ -5,6 +5,8 @@ environments: - url: https://conda.anaconda.org/conda-forge/ indexes: - https://pypi.org/simple + options: + pypi-prerelease-mode: if-necessary-or-explicit packages: linux-64: - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda @@ -71,6 +73,8 @@ environments: indexes: - https://pypi.org/simple - https://download.pytorch.org/whl/cu126 + options: + pypi-prerelease-mode: if-necessary-or-explicit packages: linux-64: - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda @@ -226,6 +230,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/34/a138692ced038c135d1a45b73a433c89c57cad2eac9dd55bb38d9de89783/supervision-0.27.0.post2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/43/3f/f88a53f60a472b46f4023f56d204dd7de33d34c5d2acbfa0d70a674e639e/threadpoolctl-3.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://download-r2.pytorch.org/whl/cu126/torch-2.9.1%2Bcu126-cp311-cp311-manylinux_2_28_x86_64.whl @@ -249,6 +254,8 @@ environments: indexes: - https://pypi.org/simple - https://download.pytorch.org/whl/cu126 + options: + pypi-prerelease-mode: if-necessary-or-explicit packages: linux-64: - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda @@ -404,6 +411,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/34/a138692ced038c135d1a45b73a433c89c57cad2eac9dd55bb38d9de89783/supervision-0.27.0.post2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/43/3f/f88a53f60a472b46f4023f56d204dd7de33d34c5d2acbfa0d70a674e639e/threadpoolctl-3.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://download-r2.pytorch.org/whl/cu126/torch-2.9.1%2Bcu126-cp311-cp311-manylinux_2_28_x86_64.whl @@ -427,6 +435,8 @@ environments: indexes: - https://pypi.org/simple - https://download.pytorch.org/whl/cu126 + options: + pypi-prerelease-mode: if-necessary-or-explicit packages: linux-64: - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda @@ -587,6 +597,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/34/a138692ced038c135d1a45b73a433c89c57cad2eac9dd55bb38d9de89783/supervision-0.27.0.post2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/43/3f/f88a53f60a472b46f4023f56d204dd7de33d34c5d2acbfa0d70a674e639e/threadpoolctl-3.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://download-r2.pytorch.org/whl/cu126/torch-2.9.1%2Bcu126-cp311-cp311-manylinux_2_28_x86_64.whl @@ -613,6 +624,8 @@ environments: indexes: - https://pypi.org/simple - https://download.pytorch.org/whl/cu126 + options: + pypi-prerelease-mode: if-necessary-or-explicit packages: linux-64: - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda @@ -801,6 +814,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/34/a138692ced038c135d1a45b73a433c89c57cad2eac9dd55bb38d9de89783/supervision-0.27.0.post2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/59/03/9a99555a762de8cd4a405efbd39a732d926d8e58791441889560b7675b3e/tensorflow_cpu-2.21.0-cp311-cp311-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4f/44/baeeed4d40b45b26b6a9649904977a7d31cb605eb2a889a0ded4b089076c/tensorstore-0.1.85-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/33/d1/8bb87d21e9aeb323cc03034f5eaf2c8f69841e40e4853c2627edf8111ed3/termcolor-3.3.0-py3-none-any.whl @@ -3957,6 +3971,13 @@ packages: - pytest>=7.1.0 ; extra == 'dev' - hypothesis>=6.70.0 ; extra == 'dev' requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + name: tabulate + version: 0.10.0 + sha256: f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3 + requires_dist: + - wcwidth ; extra == 'widechars' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/59/03/9a99555a762de8cd4a405efbd39a732d926d8e58791441889560b7675b3e/tensorflow_cpu-2.21.0-cp311-cp311-manylinux_2_27_x86_64.whl name: tensorflow-cpu version: 2.21.0 diff --git a/pixi.toml b/pixi.toml index 51310d1..10bce27 100644 --- a/pixi.toml +++ b/pixi.toml @@ -88,6 +88,8 @@ torch = "==2.9.1+cu126" torchvision = "==0.24.1+cu126" lightning = ">=2.5" kmedoids = ">=0.5.5,<0.6" +# pandas DataFrame.to_markdown (clustering_grid) and TrackEval output tables +tabulate = ">=0.9" [pypi-options] extra-index-urls = ["https://download.pytorch.org/whl/cu126"] @@ -108,6 +110,9 @@ ultralytics = "==8.4.23" compute_chunk_boundaries = "python -m pipeline.compute_chunk_boundaries" test_tracker = "CUDA_VISIBLE_DEVICES=0 python -m pytest tests/test_tracker.py -v" test_tracking_postprocessing = "python -m pytest tests/test_tracking_postprocessing.py -v" +# Import smoke test with all heavy deps installed. CI skips modules that need +# torch, so run this before a release to cover them. +test_imports_full = "python -m pytest tests/test_imports.py -v" [feature.ci.dependencies] python = ">=3.11,<3.12" diff --git a/src/dataset/features.py b/src/dataset/features.py index ed79fd6..f31e7f3 100644 --- a/src/dataset/features.py +++ b/src/dataset/features.py @@ -1,7 +1,8 @@ """Handcrafted mask feature extraction: spatial, temporal, and pairwise.""" import multiprocessing -import os + +from typing import TYPE_CHECKING import cv2 import numpy as np @@ -11,6 +12,9 @@ from loguru import logger from tqdm.auto import tqdm +if TYPE_CHECKING: + import torch # annotation only; imported lazily in bin_features_per_window + _N_WORKERS = 16 # --------------------------------------------------------------------------- diff --git a/src/tracking/sam3.py b/src/tracking/sam3.py index 8738f71..c65eb66 100644 --- a/src/tracking/sam3.py +++ b/src/tracking/sam3.py @@ -1,4 +1,11 @@ -"""Tracking pipeline""" +"""SAM3 chunked tracking pipeline. + +Each video is processed in chunks: chunk 0 uses Sam3VideoModel (text-prompted +segmentation); subsequent chunks use Sam3TrackerVideoModel (point-prompted), +initialised from masks sampled at the cleanest frame in the previous chunk. + +Launched via ``pipeline/run_tracker.py`` (see ``docs/2_tracking.md``). +""" import json import shutil diff --git a/tests/test_labels.py b/tests/test_labels.py index 2a20013..7200698 100644 --- a/tests/test_labels.py +++ b/tests/test_labels.py @@ -1,7 +1,6 @@ """Tests for label parsing and bird_info correctness.""" from glob import glob -from pathlib import Path import pytest From 653edc008f8b188d8fd77ad24f260de015824ddb Mon Sep 17 00:00:00 2001 From: Prince Ravi Leow Date: Thu, 24 Sep 2026 14:30:06 +0200 Subject: [PATCH 3/4] chore: update code repo zenodo link in main README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7e014fb..5c79916 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # PlayClass -[![Code DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22896784.svg)](https://doi.org/10.5281/zenodo.22896784) +[![Code DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22896785.svg)](https://zenodo.org/records/22896785) [![Data DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22896874.svg)](https://doi.org/10.5281/zenodo.22896874) Repository for _Hybrid morphokinematic and learnable video representations for From 108d4049820504102c5dc9d96ef8b0fbaba8b5ab Mon Sep 17 00:00:00 2001 From: Prince Ravi Leow Date: Thu, 24 Sep 2026 14:39:03 +0200 Subject: [PATCH 4/4] chore: update code and data DOI's in main README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c79916..7e014fb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # PlayClass -[![Code DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22896785.svg)](https://zenodo.org/records/22896785) +[![Code DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22896784.svg)](https://doi.org/10.5281/zenodo.22896784) [![Data DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22896874.svg)](https://doi.org/10.5281/zenodo.22896874) Repository for _Hybrid morphokinematic and learnable video representations for