From 121bb7cc1e16326e1028eb1a626a3fe6ab11c735 Mon Sep 17 00:00:00 2001 From: Ricardo-M-L Date: Tue, 13 Oct 2026 22:07:52 +0800 Subject: [PATCH] fix(video-agent-kit): retain videos without scene cuts --- marketplace.json | 2 +- .../.claude-plugin/plugin.json | 2 +- .../video-agent-kit/.zcode-plugin/plugin.json | 2 +- plugins/video-agent-kit/README.md | 9 ++ .../video-agent-kit/mcp/ve_tools/schemas.py | 2 +- plugins/video-agent-kit/mcp/ve_tools/shots.py | 27 ++-- .../tests/test_shots_coverage.py | 119 ++++++++++++++++++ 7 files changed, 145 insertions(+), 18 deletions(-) create mode 100644 plugins/video-agent-kit/tests/test_shots_coverage.py diff --git a/marketplace.json b/marketplace.json index a82aec6e..92ca692f 100644 --- a/marketplace.json +++ b/marketplace.json @@ -94,7 +94,7 @@ "en": "Automated video editing toolkit: standard cloud speech transcription and synthesis, full video frame extraction and understanding, local re-inspection, timeline, preview rendering, QC, and TTS.", "zh-CN": "自动化视频剪辑工具包: 标准云端语音转录与合成、完整视频抽帧理解、局部复看、时间线、预览渲染、QC 和 TTS。" }, - "version": "0.4.3", + "version": "0.4.4", "author": { "name": "Z.ai", "url": "https://z.ai" diff --git a/plugins/video-agent-kit/.claude-plugin/plugin.json b/plugins/video-agent-kit/.claude-plugin/plugin.json index c0c93dfe..b90561ca 100644 --- a/plugins/video-agent-kit/.claude-plugin/plugin.json +++ b/plugins/video-agent-kit/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "video-agent-kit", - "version": "0.4.3", + "version": "0.4.4", "description": "自动化视频剪辑工具包: 标准云端语音转录与合成、完整视频抽帧理解、局部复看、时间线、预览渲染、QC 和 TTS。", "author": { "name": "Z.ai", diff --git a/plugins/video-agent-kit/.zcode-plugin/plugin.json b/plugins/video-agent-kit/.zcode-plugin/plugin.json index a6179d52..7521928a 100644 --- a/plugins/video-agent-kit/.zcode-plugin/plugin.json +++ b/plugins/video-agent-kit/.zcode-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "video-agent-kit", - "version": "0.4.3", + "version": "0.4.4", "description": "自动化视频剪辑工具包: 标准云端语音转录与合成、完整视频抽帧理解、局部复看、时间线、预览渲染、QC 和 TTS。", "author": { "name": "Z.ai", diff --git a/plugins/video-agent-kit/README.md b/plugins/video-agent-kit/README.md index 4da1bd85..aaef09ff 100644 --- a/plugins/video-agent-kit/README.md +++ b/plugins/video-agent-kit/README.md @@ -26,6 +26,15 @@ `video-recap-workflows` 覆盖完整电影、足球、LoL 和篮球解说剪辑。电影解说使用 `detect_shots` / `arrange_footage` / `synthesize_narration` / `bind_narration` / `render_narrated`；足球使用 `soccer_ingest` / `soccer_arrange` / `soccer_tts` / `soccer_render`；LoL 与篮球复用 `lol_ingest` / `lol_arrange` / `lol_tts` / `lol_render`。这些流程通常产出 `out/final.mp4` 和 workflow-specific QC，而不是通用 `out/preview.mp4`。 +`detect_shots` 在没有切点的连续镜头视频中也会保留覆盖整片的一条镜头，普通检测与 +`frame_skip` 隔帧检测行为一致；即使整片短于 `min_len` 也不会丢弃。镜头缓存版本 +升级为 `cover-all-v3`，旧缓存会自动重新检测。 + +For videos with no detected cuts, `detect_shots` retains one shot spanning the +whole video in both normal and frame-skipping modes, including videos shorter +than `min_len`. The `cover-all-v3` cache version automatically rebuilds older +shot caches. + `out/media.json` 保留给源素材探测。`qc_preview` 和 `tts_generate` 内部做媒体探测时会写 `out/preview_media.json`、`out/tts_*_media.json` 这类 sidecar，不覆盖源素材 metadata。`validate_timeline` 和 `qc_preview` 会在报告中写入文件 hash；Stop hook 会拒绝已经过期的 validation / QC report。 `schemas/video_observation.schema.json` 对应工具真实落盘的 observation package，即 `video_ingest` / `video_watch_segment` 生成的帧清单、sheet 清单、媒体元数据、抽帧参数和匹配转录。模型读图后的主观摘要、候选片段和风险判断应写入 `out/report.md` 或后续独立的 review 文件，不写回这个工具产物 schema。 diff --git a/plugins/video-agent-kit/mcp/ve_tools/schemas.py b/plugins/video-agent-kit/mcp/ve_tools/schemas.py index 82dc7f74..dcf7816b 100644 --- a/plugins/video-agent-kit/mcp/ve_tools/schemas.py +++ b/plugins/video-agent-kit/mcp/ve_tools/schemas.py @@ -523,7 +523,7 @@ RECAP_TOOL_SCHEMAS: dict[str, dict] = {'detect_shots': {'description': 'Detect shot boundaries across the whole video with PySceneDetect ContentDetector and ' - 'cache them as a shots.json timeline (segment_version cover-all-v2). Shots cover the ' + 'cache them as a shots.json timeline (segment_version cover-all-v3). Shots cover the ' 'full video contiguously with no gaps: sub-min_len shots are merged into the previous ' 'shot (or accumulated into the first real shot when they open the video), so adjacent ' 'shots always satisfy end == next start. Output JSON fields ' diff --git a/plugins/video-agent-kit/mcp/ve_tools/shots.py b/plugins/video-agent-kit/mcp/ve_tools/shots.py index 49c6f3da..cd077990 100644 --- a/plugins/video-agent-kit/mcp/ve_tools/shots.py +++ b/plugins/video-agent-kit/mcp/ve_tools/shots.py @@ -17,8 +17,8 @@ from .result import ToolResult from .run_context import RunContext -# 与 movie_cut 保持同一版本串: 两边产物可互相复用缓存, 改合并算法时必须换版本号 -SEGMENT_VERSION = "cover-all-v2" +# 镜头覆盖语义变化时升级版本，避免复用旧实现留下的空镜头缓存。 +SEGMENT_VERSION = "cover-all-v3" PROBE_TIMEOUT_SECONDS = 60 @@ -92,18 +92,17 @@ def detect_shots(args: dict, ctx: RunContext) -> ToolResult: started = time.time() det = ContentDetector(threshold=threshold) try: - if frame_skip > 0: - # 隔帧检测: 解码量减少, 提速全片切分; 代价是切点精度下降(可能漏快切), - # 默认关闭。SceneManager 默认 auto_downscale=True, 检测本身已在缩小帧上做。 - from scenedetect import SceneManager, open_video - v = open_video(str(input_path)) - sm = SceneManager() - sm.add_detector(det) - sm.detect_scenes(v, frame_skip=frame_skip, show_progress=False) - scenes = sm.get_scene_list() - else: - from scenedetect import detect - scenes = detect(str(input_path), det, show_progress=False) + # 隔帧检测: 解码量减少, 提速全片切分; 代价是切点精度下降(可能漏快切), + # 默认关闭。SceneManager 默认 auto_downscale=True, 检测本身已在缩小帧上做。 + # Use SceneManager for both modes: its start_in_scene option is available + # throughout SceneDetect 0.6, unlike the detect() convenience wrapper's. + # Without it, a video with no cuts returns no shots at all. + from scenedetect import SceneManager, open_video + v = open_video(str(input_path)) + sm = SceneManager() + sm.add_detector(det) + sm.detect_scenes(v, frame_skip=frame_skip, show_progress=False) + scenes = sm.get_scene_list(start_in_scene=True) except Exception as exc: return ToolResult( text=f"[ERROR] scene detection failed: {exc}", diff --git a/plugins/video-agent-kit/tests/test_shots_coverage.py b/plugins/video-agent-kit/tests/test_shots_coverage.py new file mode 100644 index 00000000..695db54a --- /dev/null +++ b/plugins/video-agent-kit/tests/test_shots_coverage.py @@ -0,0 +1,119 @@ +"""Real SceneDetect/FFmpeg regressions for whole-video shot coverage. + +Run with Python >= 3.10, scenedetect and opencv-python-headless installed: + python -m unittest discover -s plugins/video-agent-kit/tests -p 'test_shots_coverage.py' +""" +from __future__ import annotations + +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + + +MCP_ROOT = Path(__file__).resolve().parents[1] / "mcp" +sys.path.insert(0, str(MCP_ROOT)) + +from ve_tools import shots # noqa: E402 +from ve_tools.run_context import RunContext # noqa: E402 + + +HAS_RUNTIME = ( + shutil.which("ffmpeg") is not None + and shutil.which("ffprobe") is not None + and importlib.util.find_spec("scenedetect") is not None + and importlib.util.find_spec("cv2") is not None +) + + +@unittest.skipUnless(HAS_RUNTIME, "requires FFmpeg, SceneDetect and OpenCV") +class ShotCoverageTest(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory(prefix="shots-coverage-") + self.addCleanup(self.temp.cleanup) + self.project = Path(self.temp.name).resolve() + self.env = patch.dict(os.environ, {"CLAUDE_PROJECT_DIR": str(self.project)}) + self.env.start() + self.addCleanup(self.env.stop) + self.ctx = RunContext() + + def make_video(self, *, cut: bool = False) -> Path: + video = self.project / "source.mp4" + command = ["ffmpeg", "-nostdin", "-hide_banner", "-loglevel", "error", "-y"] + command += ["-f", "lavfi", "-i", "color=c=blue:s=160x90:r=24:d=2"] + if cut: + command += ["-f", "lavfi", "-i", "color=c=red:s=160x90:r=24:d=1"] + command += [ + "-filter_complex", "[0:v]trim=duration=1,setpts=PTS-STARTPTS[a];" + "[a][1:v]concat=n=2:v=1:a=0[v]", "-map", "[v]", + ] + command += ["-c:v", "mpeg4", "-q:v", "2", "-pix_fmt", "yuv420p", str(video)] + subprocess.run(command, check=True, capture_output=True, timeout=30) + return video + + def detect(self, video: Path, **options) -> tuple[dict, dict]: + result = shots.detect_shots({"input_path": str(video), **options}, self.ctx) + self.assertFalse(result.text.startswith("[ERROR]"), result.text) + timeline = json.loads(Path(result.data["output_json"]).read_text(encoding="utf-8")) + return result.data, timeline + + def assert_covers_video(self, timeline: dict, expected_shots: int) -> None: + self.assertEqual(timeline["n_shots"], expected_shots) + self.assertEqual(len(timeline["shots"]), expected_shots) + self.assertEqual(timeline["shots"][0]["start"], 0.0) + # Early SceneDetect 0.6 releases report the final frame's timestamp; + # newer releases report the exclusive end. Allow one frame of rounding. + self.assertAlmostEqual( + timeline["shots"][-1]["end"], 2.0, delta=1 / timeline["fps"] + 0.001 + ) + for previous, following in zip(timeline["shots"], timeline["shots"][1:]): + self.assertEqual(previous["end"], following["start"]) + + def test_no_cuts_cover_the_whole_video_in_both_detection_modes(self) -> None: + video = self.make_video() + for frame_skip in (0, 2): + with self.subTest(frame_skip=frame_skip): + _, timeline = self.detect(video, frame_skip=frame_skip, force=True) + self.assert_covers_video(timeline, 1) + + def test_no_cuts_shorter_than_min_len_are_still_retained(self) -> None: + video = self.make_video() + for frame_skip in (0, 2): + with self.subTest(frame_skip=frame_skip): + _, timeline = self.detect(video, frame_skip=frame_skip, min_len=3.0, force=True) + self.assert_covers_video(timeline, 1) + + def test_real_cut_boundaries_remain_contiguous(self) -> None: + video = self.make_video(cut=True) + for frame_skip in (0, 2): + with self.subTest(frame_skip=frame_skip): + _, timeline = self.detect(video, frame_skip=frame_skip, force=True) + self.assert_covers_video(timeline, 2) + self.assertAlmostEqual(timeline["shots"][0]["end"], 1.0, delta=0.15) + + def test_old_empty_cache_is_rebuilt_and_new_cache_can_be_reused(self) -> None: + video = self.make_video() + output = self.project / "out" / "shots.json" + old_cache = { + "video": str(video), "video_signature": shots.file_signature(video), + "segment_version": "cover-all-v2", "threshold": 27.0, + "min_len": 0.8, "frame_skip": 0, "duration": 2.0, + "fps": 24.0, "n_shots": 0, "shots": [], + } + output.write_text(json.dumps(old_cache), encoding="utf-8") + data, timeline = self.detect(video) + self.assertFalse(data["reused"]) + self.assert_covers_video(timeline, 1) + reused, cached = self.detect(video) + self.assertTrue(reused["reused"]) + self.assertEqual(cached, timeline) + + +if __name__ == "__main__": + unittest.main()