Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion marketplace.json
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@
"en": "Automated video editing toolkit: standard cloud speech transcription and synthesis, full video frame extraction and understanding, local re-inspection, timeline, preview rendering, QC, and TTS.",
"zh-CN": "自动化视频剪辑工具包: 标准云端语音转录与合成、完整视频抽帧理解、局部复看、时间线、预览渲染、QC 和 TTS。"
},
"version": "0.4.3",
"version": "0.4.4",
"author": {
"name": "Z.ai",
"url": "https://z.ai"
Expand Down
2 changes: 1 addition & 1 deletion plugins/video-agent-kit/.claude-plugin/plugin.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "video-agent-kit",
"version": "0.4.3",
"version": "0.4.4",
"description": "自动化视频剪辑工具包: 标准云端语音转录与合成、完整视频抽帧理解、局部复看、时间线、预览渲染、QC 和 TTS。",
"author": {
"name": "Z.ai",
Expand Down
2 changes: 1 addition & 1 deletion plugins/video-agent-kit/.zcode-plugin/plugin.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "video-agent-kit",
"version": "0.4.3",
"version": "0.4.4",
"description": "自动化视频剪辑工具包: 标准云端语音转录与合成、完整视频抽帧理解、局部复看、时间线、预览渲染、QC 和 TTS。",
"author": {
"name": "Z.ai",
Expand Down
9 changes: 9 additions & 0 deletions plugins/video-agent-kit/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,15 @@

`video-recap-workflows` 覆盖完整电影、足球、LoL 和篮球解说剪辑。电影解说使用 `detect_shots` / `arrange_footage` / `synthesize_narration` / `bind_narration` / `render_narrated`;足球使用 `soccer_ingest` / `soccer_arrange` / `soccer_tts` / `soccer_render`;LoL 与篮球复用 `lol_ingest` / `lol_arrange` / `lol_tts` / `lol_render`。这些流程通常产出 `out/final.mp4` 和 workflow-specific QC,而不是通用 `out/preview.mp4`。

`detect_shots` 在没有切点的连续镜头视频中也会保留覆盖整片的一条镜头,普通检测与
`frame_skip` 隔帧检测行为一致;即使整片短于 `min_len` 也不会丢弃。镜头缓存版本
升级为 `cover-all-v3`,旧缓存会自动重新检测。

For videos with no detected cuts, `detect_shots` retains one shot spanning the
whole video in both normal and frame-skipping modes, including videos shorter
than `min_len`. The `cover-all-v3` cache version automatically rebuilds older
shot caches.

`out/media.json` 保留给源素材探测。`qc_preview` 和 `tts_generate` 内部做媒体探测时会写 `out/preview_media.json`、`out/tts_*_media.json` 这类 sidecar,不覆盖源素材 metadata。`validate_timeline` 和 `qc_preview` 会在报告中写入文件 hash;Stop hook 会拒绝已经过期的 validation / QC report。

`schemas/video_observation.schema.json` 对应工具真实落盘的 observation package,即 `video_ingest` / `video_watch_segment` 生成的帧清单、sheet 清单、媒体元数据、抽帧参数和匹配转录。模型读图后的主观摘要、候选片段和风险判断应写入 `out/report.md` 或后续独立的 review 文件,不写回这个工具产物 schema。
Expand Down
2 changes: 1 addition & 1 deletion plugins/video-agent-kit/mcp/ve_tools/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -523,7 +523,7 @@


RECAP_TOOL_SCHEMAS: dict[str, dict] = {'detect_shots': {'description': 'Detect shot boundaries across the whole video with PySceneDetect ContentDetector and '
'cache them as a shots.json timeline (segment_version cover-all-v2). Shots cover the '
'cache them as a shots.json timeline (segment_version cover-all-v3). Shots cover the '
'full video contiguously with no gaps: sub-min_len shots are merged into the previous '
'shot (or accumulated into the first real shot when they open the video), so adjacent '
'shots always satisfy end == next start. Output JSON fields '
Expand Down
27 changes: 13 additions & 14 deletions plugins/video-agent-kit/mcp/ve_tools/shots.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
from .result import ToolResult
from .run_context import RunContext

# 与 movie_cut 保持同一版本串: 两边产物可互相复用缓存, 改合并算法时必须换版本号
SEGMENT_VERSION = "cover-all-v2"
# 镜头覆盖语义变化时升级版本,避免复用旧实现留下的空镜头缓存。
SEGMENT_VERSION = "cover-all-v3"

PROBE_TIMEOUT_SECONDS = 60

Expand Down Expand Up @@ -92,18 +92,17 @@ def detect_shots(args: dict, ctx: RunContext) -> ToolResult:
started = time.time()
det = ContentDetector(threshold=threshold)
try:
if frame_skip > 0:
# 隔帧检测: 解码量减少, 提速全片切分; 代价是切点精度下降(可能漏快切),
# 默认关闭。SceneManager 默认 auto_downscale=True, 检测本身已在缩小帧上做。
from scenedetect import SceneManager, open_video
v = open_video(str(input_path))
sm = SceneManager()
sm.add_detector(det)
sm.detect_scenes(v, frame_skip=frame_skip, show_progress=False)
scenes = sm.get_scene_list()
else:
from scenedetect import detect
scenes = detect(str(input_path), det, show_progress=False)
# 隔帧检测: 解码量减少, 提速全片切分; 代价是切点精度下降(可能漏快切),
# 默认关闭。SceneManager 默认 auto_downscale=True, 检测本身已在缩小帧上做。
# Use SceneManager for both modes: its start_in_scene option is available
# throughout SceneDetect 0.6, unlike the detect() convenience wrapper's.
# Without it, a video with no cuts returns no shots at all.
from scenedetect import SceneManager, open_video
v = open_video(str(input_path))
sm = SceneManager()
sm.add_detector(det)
sm.detect_scenes(v, frame_skip=frame_skip, show_progress=False)
scenes = sm.get_scene_list(start_in_scene=True)
except Exception as exc:
return ToolResult(
text=f"[ERROR] scene detection failed: {exc}",
Expand Down
119 changes: 119 additions & 0 deletions plugins/video-agent-kit/tests/test_shots_coverage.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
"""Real SceneDetect/FFmpeg regressions for whole-video shot coverage.

Run with Python >= 3.10, scenedetect and opencv-python-headless installed:
python -m unittest discover -s plugins/video-agent-kit/tests -p 'test_shots_coverage.py'
"""
from __future__ import annotations

import importlib.util
import json
import os
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch


MCP_ROOT = Path(__file__).resolve().parents[1] / "mcp"
sys.path.insert(0, str(MCP_ROOT))

from ve_tools import shots # noqa: E402
from ve_tools.run_context import RunContext # noqa: E402


HAS_RUNTIME = (
shutil.which("ffmpeg") is not None
and shutil.which("ffprobe") is not None
and importlib.util.find_spec("scenedetect") is not None
and importlib.util.find_spec("cv2") is not None
)


@unittest.skipUnless(HAS_RUNTIME, "requires FFmpeg, SceneDetect and OpenCV")
class ShotCoverageTest(unittest.TestCase):
def setUp(self) -> None:
self.temp = tempfile.TemporaryDirectory(prefix="shots-coverage-")
self.addCleanup(self.temp.cleanup)
self.project = Path(self.temp.name).resolve()
self.env = patch.dict(os.environ, {"CLAUDE_PROJECT_DIR": str(self.project)})
self.env.start()
self.addCleanup(self.env.stop)
self.ctx = RunContext()

def make_video(self, *, cut: bool = False) -> Path:
video = self.project / "source.mp4"
command = ["ffmpeg", "-nostdin", "-hide_banner", "-loglevel", "error", "-y"]
command += ["-f", "lavfi", "-i", "color=c=blue:s=160x90:r=24:d=2"]
if cut:
command += ["-f", "lavfi", "-i", "color=c=red:s=160x90:r=24:d=1"]
command += [
"-filter_complex", "[0:v]trim=duration=1,setpts=PTS-STARTPTS[a];"
"[a][1:v]concat=n=2:v=1:a=0[v]", "-map", "[v]",
]
command += ["-c:v", "mpeg4", "-q:v", "2", "-pix_fmt", "yuv420p", str(video)]
subprocess.run(command, check=True, capture_output=True, timeout=30)
return video

def detect(self, video: Path, **options) -> tuple[dict, dict]:
result = shots.detect_shots({"input_path": str(video), **options}, self.ctx)
self.assertFalse(result.text.startswith("[ERROR]"), result.text)
timeline = json.loads(Path(result.data["output_json"]).read_text(encoding="utf-8"))
return result.data, timeline

def assert_covers_video(self, timeline: dict, expected_shots: int) -> None:
self.assertEqual(timeline["n_shots"], expected_shots)
self.assertEqual(len(timeline["shots"]), expected_shots)
self.assertEqual(timeline["shots"][0]["start"], 0.0)
# Early SceneDetect 0.6 releases report the final frame's timestamp;
# newer releases report the exclusive end. Allow one frame of rounding.
self.assertAlmostEqual(
timeline["shots"][-1]["end"], 2.0, delta=1 / timeline["fps"] + 0.001
)
for previous, following in zip(timeline["shots"], timeline["shots"][1:]):
self.assertEqual(previous["end"], following["start"])

def test_no_cuts_cover_the_whole_video_in_both_detection_modes(self) -> None:
video = self.make_video()
for frame_skip in (0, 2):
with self.subTest(frame_skip=frame_skip):
_, timeline = self.detect(video, frame_skip=frame_skip, force=True)
self.assert_covers_video(timeline, 1)

def test_no_cuts_shorter_than_min_len_are_still_retained(self) -> None:
video = self.make_video()
for frame_skip in (0, 2):
with self.subTest(frame_skip=frame_skip):
_, timeline = self.detect(video, frame_skip=frame_skip, min_len=3.0, force=True)
self.assert_covers_video(timeline, 1)

def test_real_cut_boundaries_remain_contiguous(self) -> None:
video = self.make_video(cut=True)
for frame_skip in (0, 2):
with self.subTest(frame_skip=frame_skip):
_, timeline = self.detect(video, frame_skip=frame_skip, force=True)
self.assert_covers_video(timeline, 2)
self.assertAlmostEqual(timeline["shots"][0]["end"], 1.0, delta=0.15)

def test_old_empty_cache_is_rebuilt_and_new_cache_can_be_reused(self) -> None:
video = self.make_video()
output = self.project / "out" / "shots.json"
old_cache = {
"video": str(video), "video_signature": shots.file_signature(video),
"segment_version": "cover-all-v2", "threshold": 27.0,
"min_len": 0.8, "frame_skip": 0, "duration": 2.0,
"fps": 24.0, "n_shots": 0, "shots": [],
}
output.write_text(json.dumps(old_cache), encoding="utf-8")
data, timeline = self.detect(video)
self.assertFalse(data["reused"])
self.assert_covers_video(timeline, 1)
reused, cached = self.detect(video)
self.assertTrue(reused["reused"])
self.assertEqual(cached, timeline)


if __name__ == "__main__":
unittest.main()