From 54558e3cd2512abd3cd0c8ee95b6576bdea18464 Mon Sep 17 00:00:00 2001 From: Ricardo-M-L Date: Sun, 11 Oct 2026 23:39:27 +0800 Subject: [PATCH] fix(video2code): publish dependency cache portably --- marketplace.json | 2 +- plugins/video2code/.claude-plugin/plugin.json | 2 +- plugins/video2code/.zcode-plugin/plugin.json | 2 +- plugins/video2code/README.md | 2 +- plugins/video2code/README_CN.md | 2 +- .../scripts/relink-node-modules.sh | 90 +++++++- .../tests/test_node_modules_cache.py | 196 ++++++++++++++++++ 7 files changed, 287 insertions(+), 9 deletions(-) create mode 100644 plugins/video2code/tests/test_node_modules_cache.py diff --git a/marketplace.json b/marketplace.json index a82aec6..8b6b661 100644 --- a/marketplace.json +++ b/marketplace.json @@ -117,7 +117,7 @@ "en": "Records WebM through ZCode's built-in Browser Use WebView and converts it to MP4 with ffmpeg for video/URL replication; no Playwright or external Chromium required.", "zh-CN": "基于 ZCode 内置 Browser Use WebView 录制 WebM 并用 ffmpeg 转成 MP4/URL 复刻；无需 Playwright 或外部 Chromium" }, - "version": "0.6.0", + "version": "0.6.1", "author": { "name": "Z.ai", "url": "https://z.ai" diff --git a/plugins/video2code/.claude-plugin/plugin.json b/plugins/video2code/.claude-plugin/plugin.json index cfc189f..80f1ea7 100644 --- a/plugins/video2code/.claude-plugin/plugin.json +++ b/plugins/video2code/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "video2code", - "version": "0.6.0", + "version": "0.6.1", "description": "基于 ZCode 内置 Browser Use WebView 录制 WebM 并用 ffmpeg 转成 MP4/URL 复刻；无需 Playwright 或外部 Chromium", "author": { "name": "Z.ai", diff --git a/plugins/video2code/.zcode-plugin/plugin.json b/plugins/video2code/.zcode-plugin/plugin.json index cfc189f..80f1ea7 100644 --- a/plugins/video2code/.zcode-plugin/plugin.json +++ b/plugins/video2code/.zcode-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "video2code", - "version": "0.6.0", + "version": "0.6.1", "description": "基于 ZCode 内置 Browser Use WebView 录制 WebM 并用 ffmpeg 转成 MP4/URL 复刻；无需 Playwright 或外部 Chromium", "author": { "name": "Z.ai", diff --git a/plugins/video2code/README.md b/plugins/video2code/README.md index 99b8c77..fd0dee8 100644 --- a/plugins/video2code/README.md +++ b/plugins/video2code/README.md @@ -99,7 +99,7 @@ python3 skills/env-setup/scripts/env_doctor.py --fix Enabling this plugin grants code-execution trust. Concretely, it will: - **Execute commands** — `ffmpeg`/`ffprobe` for transcoding and frame extraction, and `skills/web-replicate/scripts/init-webapp.sh`, which runs `npm install` to populate `node_modules`. -- **Write files** — `recordings/` (WebM and MP4), `app/` (the scaffolded project), `out/` (`plan.md`, `verify.jsonl`, `report.md`, comparison images under `out/cmp/`), `.v2c/` (plugin root pointer and hook state, including `.v2c/hook_state/interceptions.jsonl`), and a `node_modules` cache under `/tmp/webapp-node-modules` (override with `NM_LOCAL_ROOT`). +- **Write files** — `recordings/` (WebM and MP4), `app/` (the scaffolded project), `out/` (`plan.md`, `verify.jsonl`, `report.md`, comparison images under `out/cmp/`), `.v2c/` (plugin root pointer and hook state, including `.v2c/hook_state/interceptions.jsonl`), and a `node_modules` cache under `/tmp/webapp-node-modules` (override with `NM_LOCAL_ROOT`). Concurrent initializations share only complete cache entries; cache publication works with the standard macOS/BSD and GNU command-line tools. - **Bind a local port** — `deploy_website` starts a `http.server` on port 8765 or the next free port, bound locally, and stops it when the MCP server process exits. - **Access the network** — the npm registry when installing template dependencies, and whatever site you ask it to record. - **Send no telemetry** and require no API key, token, or account. There are no credentials in this plugin and nothing is uploaded anywhere. diff --git a/plugins/video2code/README_CN.md b/plugins/video2code/README_CN.md index 895863f..175d912 100644 --- a/plugins/video2code/README_CN.md +++ b/plugins/video2code/README_CN.md @@ -99,7 +99,7 @@ python3 skills/env-setup/scripts/env_doctor.py --fix 启用本插件等于授予代码执行信任。具体地，它会： - **执行命令** —— `ffmpeg`/`ffprobe` 做转码与抽帧；`skills/web-replicate/scripts/init-webapp.sh` 会执行 `npm install` 拉起 `node_modules`。 -- **写文件** —— `recordings/`（WebM 与 MP4）、`app/`（脚手架项目）、`out/`（`plan.md`、`verify.jsonl`、`report.md`，以及 `out/cmp/` 下的对照图）、`.v2c/`（插件根路径指针与 hook 状态，含 `.v2c/hook_state/interceptions.jsonl`），以及 `/tmp/webapp-node-modules` 下的 `node_modules` 缓存（可用 `NM_LOCAL_ROOT` 改）。 +- **写文件** —— `recordings/`（WebM 与 MP4）、`app/`（脚手架项目）、`out/`（`plan.md`、`verify.jsonl`、`report.md`，以及 `out/cmp/` 下的对照图）、`.v2c/`（插件根路径指针与 hook 状态，含 `.v2c/hook_state/interceptions.jsonl`），以及 `/tmp/webapp-node-modules` 下的 `node_modules` 缓存（可用 `NM_LOCAL_ROOT` 改）。并发初始化只会共享已完整落位的缓存；缓存发布同时兼容 macOS/BSD 和 GNU 标准命令行工具。 - **监听本地端口** —— `deploy_website` 在 8765 或之后第一个空闲端口起 `http.server`，只监听本地，MCP 服务进程退出时统一清理。 - **访问网络** —— 安装模板依赖时访问 npm registry；以及你让它录制的目标网站。 - **不上报任何遥测**，不需要 API key、token 或账号。插件里没有任何凭据，也不会把内容上传到任何地方。 diff --git a/plugins/video2code/skills/web-replicate/scripts/relink-node-modules.sh b/plugins/video2code/skills/web-replicate/scripts/relink-node-modules.sh index d96b6f3..4755f00 100755 --- a/plugins/video2code/skills/web-replicate/scripts/relink-node-modules.sh +++ b/plugins/video2code/skills/web-replicate/scripts/relink-node-modules.sh @@ -22,12 +22,87 @@ TEMPLATE_NAME="${TEMPLATE_NAME:-default}" NM_LOCAL_ROOT=${NM_LOCAL_ROOT:-/tmp/webapp-node-modules} mkdir -p "$NM_LOCAL_ROOT" +package_hash() { + local output + local digest + + # GNU coreutils exposes md5sum, while older macOS releases only ship BSD + # md5. The digest is only a cache key, but both commands produce the same + # MD5 value for a given package.json. + if command -v md5sum >/dev/null 2>&1; then + output=$(md5sum -- "$1") || return 1 + elif command -v md5 >/dev/null 2>&1; then + output=$(md5 -q "$1") || return 1 + else + echo "ERROR: neither md5sum nor md5 is available; cannot key node_modules cache." >&2 + return 1 + fi + + if [[ "$output" =~ ([[:xdigit:]]{32}) ]]; then + digest="${BASH_REMATCH[1]}" + printf '%.12s\n' "$digest" + return 0 + fi + + echo "ERROR: could not parse package.json checksum." >&2 + return 1 +} + +master_ready() { + [[ -d "$1" && -e "$1/.bin/vite" ]] +} + +publish_master() { + local source=$1 + local destination=$2 + local lock="${destination}.publish-lock" + local attempts=0 + + # mkdir is an atomic, portable lock on both BSD/macOS and GNU/Linux. Build + # work stays parallel; only the final same-filesystem rename is serialized. + # This preserves the old first-writer-wins contract without GNU `mv -T`. + while ! mkdir "$lock" 2>/dev/null; do + if master_ready "$destination"; then + return 0 + fi + attempts=$((attempts + 1)) + if [[ "$attempts" -ge 300 ]]; then + echo "ERROR: timed out waiting for cache publisher lock: $lock" >&2 + return 1 + fi + sleep 0.1 + done + + if ! ( + trap 'rmdir "$lock" 2>/dev/null || true' EXIT + trap 'exit 1' HUP INT TERM + if ! master_ready "$destination"; then + if ! mv "$source" "$destination"; then + echo "ERROR: failed to publish node_modules cache: $destination" >&2 + exit 1 + fi + fi + ); then + return 1 + fi + + if ! master_ready "$destination"; then + echo "ERROR: published node_modules cache is incomplete: $destination" >&2 + return 1 + fi +} + [[ -f "$PROJECT_PATH/package.json" ]] || { echo "ERROR: $PROJECT_PATH/package.json 不存在" >&2; exit 1; } -PKG_HASH=$(md5sum "$PROJECT_PATH/package.json" | cut -c1-12) +PKG_HASH=$(package_hash "$PROJECT_PATH/package.json") NM_MASTER="$NM_LOCAL_ROOT/master-$TEMPLATE_NAME-$PKG_HASH" # ── master 构建: 全部安装动作发生在本地盘构建目录, 永不触碰项目目录 ──────────── -if [[ ! -d "$NM_MASTER" ]]; then +if ! master_ready "$NM_MASTER"; then + if [[ -e "$NM_MASTER" ]]; then + echo "ERROR: node_modules cache exists but is incomplete: $NM_MASTER" >&2 + echo "Remove that cache entry and retry." >&2 + exit 1 + fi NM_TMP=$(mktemp -d "$NM_LOCAL_ROOT/.master.XXXXXX") cp "$PROJECT_PATH/package.json" "$NM_TMP/" [[ -f "$PROJECT_PATH/package-lock.json" ]] && cp "$PROJECT_PATH/package-lock.json" "$NM_TMP/" @@ -40,7 +115,7 @@ if [[ ! -d "$NM_MASTER" ]]; then else echo "无可用底座, 本地盘冷启动全量 npm install (首次较慢)..." fi - if ! (cd "$NM_TMP" && npm install --no-audit --no-fund --prefer-offline 2>&1 | tail -5); then + if ! (set -o pipefail; cd "$NM_TMP" && npm install --no-audit --no-fund --prefer-offline 2>&1 | tail -5); then rm -rf "$NM_TMP" echo "ERROR: npm install failed (check registry/proxy connectivity above)." >&2 exit 1 @@ -51,11 +126,18 @@ if [[ ! -d "$NM_MASTER" ]]; then exit 1 fi # 原子落位: 并发同时建 master 时先到先得, 输家直接丢弃 - mv -T "$NM_TMP/node_modules" "$NM_MASTER" 2>/dev/null || true + if ! publish_master "$NM_TMP/node_modules" "$NM_MASTER"; then + rm -rf "$NM_TMP" + exit 1 + fi rm -rf "$NM_TMP" fi if [[ "$PREPARE_ONLY" == "1" ]]; then + if ! master_ready "$NM_MASTER"; then + echo "ERROR: node_modules cache is not ready: $NM_MASTER" >&2 + exit 1 + fi echo "✓ master ready: $NM_MASTER" exit 0 fi diff --git a/plugins/video2code/tests/test_node_modules_cache.py b/plugins/video2code/tests/test_node_modules_cache.py new file mode 100644 index 0000000..ead00b6 --- /dev/null +++ b/plugins/video2code/tests/test_node_modules_cache.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +SCRIPT = ( + Path(__file__).resolve().parents[1] + / "skills" + / "web-replicate" + / "scripts" + / "relink-node-modules.sh" +) +FAKE_DIGEST = "0123456789abcdef0123456789abcdef" + + +class NodeModulesCacheTest(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory(prefix="video2code cache test ") + self.root = Path(self.tmp.name) + self.cache = self.root / "cache with spaces" + self.bin = self.root / "portable bin" + self.cache.mkdir() + self.bin.mkdir() + self._install_required_commands() + self._write_executable( + "md5", + "#!/bin/sh\n" + "[ \"$#\" -eq 2 ] && [ \"$1\" = '-q' ] || exit 64\n" + f"printf '%s\\n' '{FAKE_DIGEST}'\n", + ) + real_mv = shutil.which("mv") + assert real_mv is not None + self._write_executable( + "mv", + "#!/bin/sh\n" + "if [ \"${1:-}\" = '-T' ]; then\n" + " echo 'mv: illegal option -- T' >&2\n" + " exit 64\n" + "fi\n" + f"exec '{real_mv}' \"$@\"\n", + ) + self.bash = shutil.which("bash") + assert self.bash is not None + + def tearDown(self) -> None: + self.tmp.cleanup() + + def _install_required_commands(self) -> None: + # PATH is intentionally limited to this directory: md5sum must be + # absent so the BSD `md5` fallback is exercised on Linux CI as well. + for name in ( + "cp", + "head", + "ln", + "ls", + "mkdir", + "mktemp", + "nohup", + "rm", + "rmdir", + "sleep", + "tail", + ): + source = shutil.which(name) + assert source is not None, name + (self.bin / name).symlink_to(source) + + def _write_executable(self, name: str, content: str) -> None: + path = self.bin / name + path.write_text(content, encoding="utf-8") + path.chmod(0o755) + + def _project(self, name: str) -> Path: + project = self.root / name + project.mkdir() + (project / "package.json").write_text( + '{"name":"portable-cache-test","private":true}\n', + encoding="utf-8", + ) + return project + + def _env(self, **extra: str) -> dict[str, str]: + return { + **os.environ, + "PATH": str(self.bin), + "NM_LOCAL_ROOT": str(self.cache), + "TEMPLATE_NAME": "default", + **extra, + } + + def _run( + self, + project: Path, + *args: str, + env_extra: dict[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [self.bash, str(SCRIPT), str(project), *args], + capture_output=True, + text=True, + env=self._env(**(env_extra or {})), + timeout=10, + ) + + def test_bsd_tools_prepare_and_relink_paths_with_spaces(self) -> None: + project = self._project("project with spaces") + self._write_executable( + "npm", + "#!/bin/sh\n" + "mkdir -p node_modules/.bin\n" + ": > node_modules/.bin/vite\n", + ) + + prepared = self._run(project, "--prepare-only") + self.assertEqual(prepared.returncode, 0, prepared.stderr or prepared.stdout) + master = self.cache / f"master-default-{FAKE_DIGEST[:12]}" + self.assertTrue((master / ".bin" / "vite").is_file()) + self.assertIn(f"✓ master ready: {master}", prepared.stdout) + + linked = self._run(project) + self.assertEqual(linked.returncode, 0, linked.stderr or linked.stdout) + self.assertTrue((project / "node_modules").is_symlink()) + self.assertTrue((project / "node_modules" / ".bin" / "vite").is_file()) + + def test_concurrent_prepare_publishes_one_complete_master(self) -> None: + project_one = self._project("project one") + project_two = self._project("project two") + barrier = self.root / "both builders reached npm" + self._write_executable( + "npm", + "#!/bin/sh\n" + ": > \"$TEST_BARRIER.$BUILDER_ID\"\n" + "attempts=0\n" + "while [ ! -e \"$TEST_BARRIER.one\" ] || [ ! -e \"$TEST_BARRIER.two\" ]; do\n" + " attempts=$((attempts + 1))\n" + " [ \"$attempts\" -lt 500 ] || exit 70\n" + " sleep 0.01\n" + "done\n" + "mkdir -p node_modules/.bin\n" + "printf '%s\\n' \"$BUILDER_ID\" > node_modules/builder\n" + ": > node_modules/.bin/vite\n", + ) + + common = {"TEST_BARRIER": str(barrier)} + first = subprocess.Popen( + [self.bash, str(SCRIPT), str(project_one), "--prepare-only"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=self._env(**common, BUILDER_ID="one"), + ) + second = subprocess.Popen( + [self.bash, str(SCRIPT), str(project_two), "--prepare-only"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=self._env(**common, BUILDER_ID="two"), + ) + first_out, first_err = first.communicate(timeout=10) + second_out, second_err = second.communicate(timeout=10) + + self.assertEqual(first.returncode, 0, first_err or first_out) + self.assertEqual(second.returncode, 0, second_err or second_out) + master = self.cache / f"master-default-{FAKE_DIGEST[:12]}" + self.assertTrue((master / ".bin" / "vite").is_file()) + self.assertIn((master / "builder").read_text(encoding="utf-8").strip(), {"one", "two"}) + self.assertFalse((master / "node_modules").exists()) + self.assertEqual(list(self.cache.glob("*.publish-lock")), []) + + def test_failed_delta_install_does_not_publish_cloned_base(self) -> None: + project = self._project("project with failed install") + stale_base = self.cache / "master-default-previous" + (stale_base / ".bin").mkdir(parents=True) + (stale_base / ".bin" / "vite").write_text("old vite\n", encoding="utf-8") + self._write_executable( + "npm", + "#!/bin/sh\n" + "echo 'simulated npm failure' >&2\n" + "exit 42\n", + ) + + prepared = self._run(project, "--prepare-only") + + self.assertNotEqual(prepared.returncode, 0) + self.assertIn("npm install failed", prepared.stderr) + master = self.cache / f"master-default-{FAKE_DIGEST[:12]}" + self.assertFalse(master.exists()) + + +if __name__ == "__main__": + unittest.main()