diff --git a/hooks/hook-hosts.mjs b/hooks/hook-hosts.mjs index 1c3c4a9..74ee29d 100644 --- a/hooks/hook-hosts.mjs +++ b/hooks/hook-hosts.mjs @@ -21,6 +21,7 @@ export const HOSTS = { manifest: ".claude-plugin/hooks.json", format: "nested", rootVariable: "CLAUDE_PLUGIN_ROOT", + telemetry: "claude-code", contextOutput: (eventName, text) => ({ hookSpecificOutput: { hookEventName: eventName, additionalContext: text }, }), diff --git a/hooks/telemetry-adapters/claude-code.mjs b/hooks/telemetry-adapters/claude-code.mjs new file mode 100644 index 0000000..e9eabd6 --- /dev/null +++ b/hooks/telemetry-adapters/claude-code.mjs @@ -0,0 +1,113 @@ +// @ts-check + +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { BASH_CLIS } from "../telemetry-contract.mjs"; +import { + arcadeToolProperties, + builtinToolProperties as sharedBuiltinToolProperties, + otherServerToolProperties, +} from "../telemetry-events.mjs"; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const plugin = JSON.parse(readFileSync(path.join(ROOT, "plugin.json"), "utf8")); +const [mcpServerKey] = Object.keys(JSON.parse(readFileSync(path.join(ROOT, "mcp.json"), "utf8")).mcpServers); + +export const ARCADE_TOOL_PREFIX = `mcp__plugin_${plugin.name}_${mcpServerKey}__`; +export const CLAUDE_AI_ARCADE_TOOL_PREFIX = "mcp__claude_ai_arcade__"; + +const preToolMatcher = `^(?:${ARCADE_TOOL_PREFIX}|${CLAUDE_AI_ARCADE_TOOL_PREFIX})`; + +/** + * @param {unknown} toolName + * @param {Record | undefined} toolInput + */ +const claudeToolProperties = (toolName, toolInput) => { + if (typeof toolName !== "string" || !toolName.startsWith("mcp__")) { + return null; + } + const splitAt = toolName.lastIndexOf("__"); + const tool = toolName.slice(splitAt + 2); + if (toolName.startsWith(ARCADE_TOOL_PREFIX)) { + return arcadeToolProperties("arcade", tool, toolInput); + } + if (toolName.startsWith(CLAUDE_AI_ARCADE_TOOL_PREFIX)) { + return arcadeToolProperties("other_arcade", tool, toolInput); + } + return otherServerToolProperties(toolName.slice("mcp__".length, splitAt), tool, toolInput); +}; + +/** + * @param {unknown} toolName + * @param {Record | undefined} toolInput + */ +const claudeAttemptProperties = (toolName, toolInput) => { + if (typeof toolName !== "string") return null; + if (toolName.startsWith(ARCADE_TOOL_PREFIX)) { + return arcadeToolProperties("arcade", toolName.slice(ARCADE_TOOL_PREFIX.length), toolInput); + } + if (toolName.startsWith(CLAUDE_AI_ARCADE_TOOL_PREFIX)) { + return arcadeToolProperties("other_arcade", toolName.slice(CLAUDE_AI_ARCADE_TOOL_PREFIX.length), toolInput); + } + return null; +}; + +/** @param {Record} raw */ +const normalize = (raw) => ({ + hook_event_name: raw.hook_event_name, + session_id: raw.session_id, + prompt_id: raw.prompt_id, + source: raw.source, + prompt: raw.prompt, + tool_name: raw.tool_name, + tool_input: raw.tool_input, + tool_response: raw.tool_response, + error: raw.error, + is_interrupt: raw.is_interrupt, + agent_type: raw.agent_type, + agent_id: raw.agent_id, + last_assistant_message: raw.last_assistant_message, +}); + +// `if` is a Claude Code permission rule so the hook does not start for other commands; +// `extraArgs` pass the CLI name through to builtinToolProperties. +/** @param {"PostToolUse" | "PostToolUseFailure"} event */ +const bashHookRows = (event) => + BASH_CLIS.map((cli) => ({ + event, + matcher: "Bash", + if: `Bash(${cli} *)`, + extraArgs: ["--cli", cli], + })); + +/** @type {import("../telemetry-adapter.mjs").TelemetryAdapter} */ +const claudeCodeAdapter = { + host: "claude-code", + dataVariable: "CLAUDE_PLUGIN_DATA", + // Claude Code reads these as set for any non-empty value, even "0" or "false". + optOutSwitches: [ + { name: "DISABLE_TELEMETRY", anyValue: true }, + { name: "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", anyValue: true }, + ], + requiresTurn: true, + promptReminder: true, + subagentSession: false, + hookRows: [ + { event: "UserPromptSubmit" }, + { event: "PreToolUse", matcher: preToolMatcher }, + { event: "PostToolUse", matcher: "mcp__.*" }, + { event: "PostToolUse", matcher: "WebFetch|WebSearch" }, + ...bashHookRows("PostToolUse"), + { event: "PostToolUseFailure", matcher: "mcp__.*" }, + { event: "PostToolUseFailure", matcher: "WebFetch|WebSearch" }, + ...bashHookRows("PostToolUseFailure"), + { event: "SubagentStop" }, + ], + normalize, + toolProperties: claudeToolProperties, + attemptProperties: claudeAttemptProperties, + builtinToolProperties: sharedBuiltinToolProperties, +}; + +export default claudeCodeAdapter; diff --git a/hooks/telemetry-events.mjs b/hooks/telemetry-events.mjs index 411ff14..1890d81 100644 --- a/hooks/telemetry-events.mjs +++ b/hooks/telemetry-events.mjs @@ -9,6 +9,7 @@ import { isTaskNotification, shouldRemind } from "./prompt-filters.mjs"; import { isOperatorAgentType } from "./routing-guidance.mjs"; import { classifyPrompt, + serviceForToolkit, serviceForToolName, } from "./telemetry-classify.mjs"; import { commandUsesCli } from "./telemetry-commands.mjs"; @@ -75,6 +76,24 @@ export const arcadeToolProperties = (server, tool, toolInput) => { */ export const otherServerProperties = (service) => withService({ server: "other" }, service); +/** + * Properties for a tool on an MCP server other than this plugin's Arcade + * server. A gateway tool name means another Arcade server. Some connectors + * name the service in the server, not the tool (e.g. claude_ai_Gmail); only + * the category is sent. + * @param {string} server + * @param {string} tool + * @param {Record | undefined} toolInput + */ +export const otherServerToolProperties = (server, tool, toolInput) => { + if (/** @type {readonly string[]} */ (GATEWAY_TOOLS).includes(tool)) { + return arcadeToolProperties("other_arcade", tool, toolInput); + } + const service = + serviceForToolName(tool) ?? server.split(/[^A-Za-z0-9]+/).map(serviceForToolkit).find(Boolean); + return otherServerProperties(service); +}; + /** * Properties for a WebFetch, WebSearch, or listed-CLI Bash call, or null to * send nothing. Only the tool name and the CLI name are read into the event; diff --git a/test/fixtures/telemetry/claude-code/auth-status-response.mjs b/test/fixtures/telemetry/claude-code/auth-status-response.mjs new file mode 100644 index 0000000..cac1f49 --- /dev/null +++ b/test/fixtures/telemetry/claude-code/auth-status-response.mjs @@ -0,0 +1,10 @@ +/** Text of a System_ManageAuthorization status answer, as tool_response. */ +export const authStatusResponse = (statuses) => [ + { + type: "text", + text: JSON.stringify({ + message: statuses.includes("authorization_required") ? "Not yet authorized: dropbox." : "All authorized.", + providers: statuses.map((status, index) => ({ provider: `provider${index}`, status })), + }), + }, +]; diff --git a/test/fixtures/telemetry/claude-code/event-cases.mjs b/test/fixtures/telemetry/claude-code/event-cases.mjs new file mode 100644 index 0000000..1e1ae57 --- /dev/null +++ b/test/fixtures/telemetry/claude-code/event-cases.mjs @@ -0,0 +1,154 @@ +import { + ARCADE_TOOL_PREFIX, + CLAUDE_AI_ARCADE_TOOL_PREFIX, +} from "../../../../hooks/telemetry-adapters/claude-code.mjs"; +import { authStatusResponse } from "./auth-status-response.mjs"; + +const SIGN_IN = `${ARCADE_TOOL_PREFIX}System_ManageAuthorization`; +const OPERATOR = "arcade:arcade-operator"; + +const tool = (name, toolInput) => ({ hook_event_name: "PostToolUse", tool_name: name, tool_input: toolInput }); +const failed = (name, error, fields = {}) => ({ + hook_event_name: "PostToolUseFailure", + tool_name: name, + error, + ...fields, +}); +const signInCheck = (name, statuses) => ({ + hook_event_name: "PostToolUse", + tool_name: name, + tool_response: authStatusResponse(statuses), +}); +const stop = (message) => ({ + hook_event_name: "SubagentStop", + agent_type: OPERATOR, + last_assistant_message: message, +}); +const operator = (status) => ({ agent: "arcade-operator", status }); + +/** + * @typedef {object} EventCase + * @property {string} id + * @property {Record} input + * @property {null | { event: string, extra: Record }} expected + * @property {string[]} [argv] + * @property {boolean} [seedAppWork] + */ + +/** @returns {EventCase[]} */ +export const eventCases = () => { + const ATTEMPTED = "Plugin tool attempted"; + const CALLED = "Plugin tool called"; + const FAILED = "Plugin tool failed"; + const STOPPED = "Plugin subagent stopped"; + /** @type {EventCase[]} */ + const cases = [ + { id: "session-start-startup", input: { hook_event_name: "SessionStart", source: "startup" }, expected: null }, + { id: "prompt-app-work", input: { hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }, + expected: { event: "Plugin prompt submitted", extra: { could_use_arcade: true, service_hints: ["calendar"], reminder_sent: true } } }, + { id: "prompt-confirmation-style", input: { hook_event_name: "UserPromptSubmit", prompt: "ok", prompt_id: "confirm-turn" }, + expected: { event: "Plugin prompt submitted", extra: { could_use_arcade: false, service_hints: [], reminder_sent: false } }, + seedAppWork: true }, + { id: "prompt-unrelated", input: { hook_event_name: "UserPromptSubmit", prompt: "Fix the parser" }, expected: null }, + { id: "prompt-task-notification", input: { + hook_event_name: "UserPromptSubmit", + prompt: "\ncompleted calendar", + }, expected: null }, + { id: "pre-arcade-select", input: { hook_event_name: "PreToolUse", tool_name: `${ARCADE_TOOL_PREFIX}Arcade_SelectTools` }, + expected: { event: ATTEMPTED, extra: { server: "arcade", tool: "Arcade_SelectTools" } } }, + { id: "pre-claude-ai-select", input: { hook_event_name: "PreToolUse", tool_name: `${CLAUDE_AI_ARCADE_TOOL_PREFIX}Arcade_SelectTools` }, + expected: { event: ATTEMPTED, extra: { server: "other_arcade", tool: "Arcade_SelectTools" } } }, + { id: "pre-claude-ai-calendar", input: { hook_event_name: "PreToolUse", tool_name: `${CLAUDE_AI_ARCADE_TOOL_PREFIX}GoogleCalendar_ListEvents` }, + expected: { event: ATTEMPTED, extra: { server: "other_arcade", tool: "app_tool", service: "calendar" } } }, + { id: "pre-use-tool", input: { + hook_event_name: "PreToolUse", + tool_name: `${ARCADE_TOOL_PREFIX}Arcade_UseTool`, + tool_input: { tool_name: "GoogleCalendar.ListEvents" }, + }, expected: { event: ATTEMPTED, extra: { server: "arcade", tool: "Arcade_UseTool", service: "calendar" } } }, + { id: "pre-other-arcade-tool", input: { hook_event_name: "PreToolUse", tool_name: `${CLAUDE_AI_ARCADE_TOOL_PREFIX}AcmeHR_RunPayroll` }, + expected: { event: ATTEMPTED, extra: { server: "other_arcade", tool: "other" } } }, + { id: "pre-gmail-connector", input: { hook_event_name: "PreToolUse", tool_name: "mcp__claude_ai_Gmail__search_threads" }, expected: null }, + { id: "pre-wrong-arcade-connector", input: { hook_event_name: "PreToolUse", tool_name: "mcp__claude_ai_Arcade__Arcade_SelectTools" }, expected: null }, + { id: "post-gmail", input: tool(`${ARCADE_TOOL_PREFIX}Gmail_ListEmails`), + expected: { event: CALLED, extra: { server: "arcade", tool: "app_tool", service: "email" } } }, + { id: "post-claude-ai-calendar", input: tool(`${CLAUDE_AI_ARCADE_TOOL_PREFIX}GoogleCalendar_ListEvents`), + expected: { event: CALLED, extra: { server: "other_arcade", tool: "app_tool", service: "calendar" } } }, + { id: "post-sign-in-no-auth", input: tool(SIGN_IN), + expected: { event: CALLED, extra: { server: "arcade", tool: "System_ManageAuthorization", auth_needed: false } } }, + { id: "post-sign-in-auth-needed", input: signInCheck(SIGN_IN, ["authorized", "authorization_required"]), + expected: { event: CALLED, extra: { server: "arcade", tool: "System_ManageAuthorization", auth_needed: true } } }, + { id: "post-sign-in-authorized", input: signInCheck(SIGN_IN, ["authorized"]), + expected: { event: CALLED, extra: { server: "arcade", tool: "System_ManageAuthorization", auth_needed: false } } }, + { id: "post-claude-ai-sign-in", input: signInCheck("mcp__claude_ai_Arcade__System_ManageAuthorization", ["authorization_required"]), + expected: { event: CALLED, extra: { server: "other_arcade", tool: "System_ManageAuthorization", auth_needed: true } } }, + { id: "post-use-tool-gmail", input: tool(`${ARCADE_TOOL_PREFIX}Arcade_UseTool`, { tool_name: "Gmail.ListEmails" }), + expected: { event: CALLED, extra: { server: "arcade", tool: "Arcade_UseTool", service: "email" } } }, + { id: "post-use-tool-calendar", input: tool(`${ARCADE_TOOL_PREFIX}Arcade_UseTool`, { tool_name: "GoogleCalendar.ListEvents" }), + expected: { event: CALLED, extra: { server: "arcade", tool: "Arcade_UseTool", service: "calendar" } } }, + { id: "post-use-tool-unknown", input: tool(`${ARCADE_TOOL_PREFIX}Arcade_UseTool`, { tool_name: "AcmeHR.RunPayroll" }), + expected: { event: CALLED, extra: { server: "arcade", tool: "Arcade_UseTool" } } }, + { id: "post-other-arcade-app", input: tool(`${ARCADE_TOOL_PREFIX}AcmeHR_RunPayroll`), + expected: { event: CALLED, extra: { server: "arcade", tool: "other" } } }, + { id: "post-claude-ai-use-tool", input: tool("mcp__claude_ai_Arcade_Production__Arcade_UseTool", { tool_name: "Slack_SendMessage" }), + expected: { event: CALLED, extra: { server: "other_arcade", tool: "Arcade_UseTool", service: "chat" } } }, + { id: "post-other-server-granola", input: tool("mcp__granola__Granola_ListMeetings"), + expected: { event: CALLED, extra: { server: "other", service: "meetings" } }, seedAppWork: true }, + { id: "post-gmail-connector", input: tool("mcp__claude_ai_Gmail__search_threads"), + expected: { event: CALLED, extra: { server: "other", service: "email" } }, seedAppWork: true }, + { id: "post-other-server-unknown", input: tool("mcp__secret-server__DoThing"), + expected: { event: CALLED, extra: { server: "other" } }, seedAppWork: true }, + { id: "post-read-builtin", input: tool("Read"), expected: null }, + { id: "failure-slack", input: { hook_event_name: "PostToolUseFailure", tool_name: `${ARCADE_TOOL_PREFIX}Slack_SendMessage` }, + expected: { event: FAILED, extra: { server: "arcade", tool: "app_tool", service: "chat", failure_kind: "tool_error" } } }, + { id: "failure-auth", input: failed(`${ARCADE_TOOL_PREFIX}Gmail_ListEmails`, + '{"message":"The tool was not executed because it requires authorization.","authorization_url":"https://example.com/auth"}'), + expected: { event: FAILED, extra: { server: "arcade", tool: "app_tool", service: "email", failure_kind: "auth_required" } } }, + { id: "failure-session-expired", input: failed(`${ARCADE_TOOL_PREFIX}Arcade_UseTool`, 'MCP server "plugin:arcade:arcade" session expired', + { tool_input: { tool_name: "Gmail.ListEmails" } }), + expected: { event: FAILED, extra: { server: "arcade", tool: "Arcade_UseTool", service: "email", failure_kind: "session_expired" } } }, + { id: "failure-unreachable", input: failed(`${ARCADE_TOOL_PREFIX}Gmail_ListEmails`, "Connection closed"), + expected: { event: FAILED, extra: { server: "arcade", tool: "app_tool", service: "email", failure_kind: "unreachable" } } }, + { id: "failure-interrupted", input: failed(`${ARCADE_TOOL_PREFIX}Gmail_ListEmails`, "Connection closed", { is_interrupt: true }), + expected: { event: FAILED, extra: { server: "arcade", tool: "app_tool", service: "email", failure_kind: "interrupted" } } }, + { id: "failure-other-http", input: failed("mcp__secret-server__DoThing", "Error POSTing to endpoint: internal error"), + expected: { event: FAILED, extra: { server: "other", failure_kind: "http_error" } }, seedAppWork: true }, + { id: "failure-read", input: failed("Read", "File does not exist."), expected: null }, + { id: "subagent-operator-needs-auth", input: stop("Done.\n\nstatus: needs_auth\nsummary: sign in"), + expected: { event: STOPPED, extra: operator("needs_auth") } }, + { id: "subagent-operator-needs-confirmation", input: stop("**status:** needs_confirmation"), + expected: { event: STOPPED, extra: operator("needs_confirmation") } }, + { id: "subagent-operator-needs-clarification", input: stop("- status: `needs_clarification`"), + expected: { event: STOPPED, extra: operator("needs_clarification") } }, + { id: "subagent-operator-failed", input: stop("STATUS: FAILED"), + expected: { event: STOPPED, extra: operator("failed") } }, + { id: "subagent-operator-unknown", input: stop("status: exploded"), + expected: { event: STOPPED, extra: operator("unknown") } }, + { id: "subagent-operator-no-message", input: stop(undefined), + expected: { event: STOPPED, extra: operator("unknown") } }, + { id: "subagent-non-operator", input: { + hook_event_name: "SubagentStop", + agent_type: "general-purpose", + last_assistant_message: "status: completed", + }, expected: null }, + { id: "subagent-operator-completed", input: { + hook_event_name: "SubagentStop", + agent_type: OPERATOR, + agent_id: "agent-1", + last_assistant_message: "status: completed", + }, expected: { event: STOPPED, extra: operator("completed") } }, + { id: "builtin-webfetch", input: { hook_event_name: "PostToolUse", tool_name: "WebFetch", tool_input: { url: "https://example.com" } }, + expected: { event: "Plugin built-in tool called", extra: { tool: "WebFetch" } }, seedAppWork: true }, + { id: "builtin-websearch", input: { hook_event_name: "PostToolUse", tool_name: "WebSearch", tool_input: { query: "weather" } }, + expected: { event: "Plugin built-in tool called", extra: { tool: "WebSearch" } }, seedAppWork: true }, + { id: "builtin-bash-gh", input: { hook_event_name: "PostToolUse", tool_name: "Bash", tool_input: { command: "gh pr list" } }, + expected: { event: "Plugin built-in tool called", extra: { tool: "Bash", cli: "gh", service: "code_hosting" } }, + argv: ["--cli", "gh"], seedAppWork: true }, + { id: "builtin-bash-curl", input: { hook_event_name: "PostToolUseFailure", tool_name: "Bash", tool_input: { command: "curl x" } }, + expected: { event: "Plugin built-in tool failed", extra: { tool: "Bash", cli: "curl" } }, + argv: ["--cli", "curl"], seedAppWork: true }, + { id: "builtin-bash-non-listed", input: { hook_event_name: "PostToolUse", tool_name: "Bash", tool_input: { command: "echo hi" } }, + expected: null, argv: ["--cli", "gh"], seedAppWork: true }, + { id: "session-start-compact", input: { hook_event_name: "SessionStart", source: "compact" }, expected: null }, + ]; + return cases; +}; diff --git a/test/fixtures/telemetry/claude-code/hooks.enabled.json b/test/fixtures/telemetry/claude-code/hooks.enabled.json new file mode 100644 index 0000000..71cd3f7 --- /dev/null +++ b/test/fixtures/telemetry/claude-code/hooks.enabled.json @@ -0,0 +1,191 @@ +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/session-start.mjs\" --host claude-code", + "timeout": 5 + } + ] + } + ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/user-prompt-submit.mjs\" --host claude-code", + "timeout": 5 + }, + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + } + ], + "SubagentStart": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-start.mjs\" --host claude-code", + "timeout": 5 + } + ] + } + ], + "PreToolUse": [ + { + "matcher": "^(?:mcp__plugin_arcade_arcade__|mcp__claude_ai_arcade__)", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + } + ], + "PostToolUse": [ + { + "matcher": "mcp__.*", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + }, + { + "matcher": "WebFetch|WebSearch", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + }, + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "if": "Bash(gh *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli gh", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(glab *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli glab", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(curl *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli curl", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(wget *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli wget", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(http *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli http", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(osascript *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli osascript", + "timeout": 5 + } + ] + } + ], + "PostToolUseFailure": [ + { + "matcher": "mcp__.*", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + }, + { + "matcher": "WebFetch|WebSearch", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + }, + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "if": "Bash(gh *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli gh", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(glab *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli glab", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(curl *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli curl", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(wget *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli wget", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(http *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli http", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(osascript *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli osascript", + "timeout": 5 + } + ] + } + ], + "SubagentStop": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + } + ] + } +} diff --git a/test/telemetry-claude.test.mjs b/test/telemetry-claude.test.mjs new file mode 100644 index 0000000..36e3ec5 --- /dev/null +++ b/test/telemetry-claude.test.mjs @@ -0,0 +1,378 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { readFileSync, readdirSync, statSync } from "node:fs"; +import path from "node:path"; +import { test } from "node:test"; +import { BASH_CLIS } from "../hooks/telemetry-contract.mjs"; +import { HOOKS, HOSTS, telemetryHookRows } from "../hooks/hook-hosts.mjs"; +import { SCOPE_TTL_MS } from "../hooks/hook-scope.mjs"; +import { buildEvent, isArcadeCall } from "../hooks/telemetry-events.mjs"; +import adapter, { + ARCADE_TOOL_PREFIX, + CLAUDE_AI_ARCADE_TOOL_PREFIX, +} from "../hooks/telemetry-adapters/claude-code.mjs"; +import { buildHookManifest } from "../scripts/generate-manifests.mjs"; +import { readRepoFile, ROOT } from "./helpers.mjs"; +import { eventCases } from "./fixtures/telemetry/claude-code/event-cases.mjs"; +import { + assertMatchesContract, + assertNoLeak, + assertSilentThenServerReachable, + captureTelemetry, + expectedEvent, + hookEnv, + hookInput, + runTelemetryScript, + startServer, + telemetryEnv, + tempDataDir, +} from "./telemetry-helpers.mjs"; + +const OPERATOR = "arcade:arcade-operator"; +const SESSION_ID = "raw-session-id-123"; +const PROMPT_ID = "raw-prompt-id-456"; +const LEAK_RE = /SECRET|private-repo|raw-session|raw-prompt|authorization_url|example\.com/i; + +const seedAppWork = async (dataDir, promptId = PROMPT_ID, now) => { + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ + hook_event_name: "UserPromptSubmit", + prompt: "Check my calendar", + prompt_id: promptId, + }), + }); +}; + +test("Arcade tool prefixes match plugin.json and mcp.json", () => { + const plugin = JSON.parse(readRepoFile("plugin.json")); + const [server] = Object.keys(JSON.parse(readRepoFile("mcp.json")).mcpServers); + assert.equal(ARCADE_TOOL_PREFIX, `mcp__plugin_${plugin.name}_${server}__`); + assert.equal(CLAUDE_AI_ARCADE_TOOL_PREFIX, "mcp__claude_ai_arcade__"); +}); + +test("every fixture maps through captureTelemetry to the expected contract event", async () => { + for (const { id, input, expected, argv = [], seedAppWork: seed } of eventCases()) { + const dataDir = tempDataDir(); + if (seed) await seedAppWork(dataDir, PROMPT_ID); + const { sent } = await captureTelemetry({ adapter, dataDir, input: hookInput(input), argv }); + if (expected === null) { + assert.equal(sent.length, 0, id); + continue; + } + assert.equal(sent.length, 1, id); + const promptId = typeof input.prompt_id === "string" ? input.prompt_id : PROMPT_ID; + const want = expectedEvent(expected.event, expected.extra, adapter, SESSION_ID, promptId); + want.properties.os = process.platform; + assert.deepEqual(sent[0], want, id); + assertMatchesContract(sent[0], id); + assertNoLeak(JSON.stringify(sent[0]), LEAK_RE); + } +}); + +test("scope transitions across a sequence sharing one data dir", async () => { + const dataDir = tempDataDir(); + const now = 1_000_000; + const calendar = "calendar-turn"; + const unrelated = "unrelated-turn"; + const confirm = "confirm-turn"; + + let r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Fix the parser", prompt_id: unrelated }), + }); + assert.equal(r.sent.length, 0); + r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: unrelated }), + }); + assert.equal(r.sent.length, 0); + + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar", prompt_id: calendar }), + }); + r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: calendar }), + }); + assert.equal(r.sent.length, 1); + + r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: "other-turn" }), + }); + assert.equal(r.sent.length, 0); + + await captureTelemetry({ + adapter, + dataDir, + now: now + 1000, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "yes, send it", prompt_id: confirm }), + }); + r = await captureTelemetry({ + adapter, + dataDir, + now: now + 1000, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: confirm }), + }); + assert.equal(r.sent.length, 1); + + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar", prompt_id: calendar }), + }); + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "SessionStart", source: "compact", session_id: SESSION_ID }), + }); + r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: calendar }), + }); + assert.equal(r.sent.length, 1, "compact SessionStart keeps scope"); + + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar", prompt_id: "after-compact" }), + }); + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "SessionStart", source: "startup", session_id: SESSION_ID }), + }); + r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: calendar }), + }); + assert.equal(r.sent.length, 0, "normal SessionStart clears scope"); + + const expTurn = "expiry-turn"; + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar", prompt_id: expTurn }), + }); + r = await captureTelemetry({ + adapter, + dataDir, + now: now + SCOPE_TTL_MS + 1, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: expTurn }), + }); + assert.equal(r.sent.length, 0, "scope expires after 30 minutes"); +}); + +test("direct Arcade calls are observed without prompt scope; other and built-in tools need scope", async () => { + const fresh = tempDataDir(); + const arcade = await captureTelemetry({ + adapter, + dataDir: fresh, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: `${ARCADE_TOOL_PREFIX}Gmail_ListEmails` }), + }); + assert.equal(arcade.sent.length, 1); + + const scoped = tempDataDir(); + const other = await captureTelemetry({ + adapter, + dataDir: scoped, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "mcp__granola__Granola_ListMeetings" }), + }); + assert.equal(other.sent.length, 0); + await seedAppWork(scoped); + const otherInScope = await captureTelemetry({ + adapter, + dataDir: scoped, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "mcp__granola__Granola_ListMeetings" }), + }); + assert.equal(otherInScope.sent.length, 1); + + const builtinDir = tempDataDir(); + const builtin = await captureTelemetry({ + adapter, + dataDir: builtinDir, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch" }), + }); + assert.equal(builtin.sent.length, 0); +}); + +test("arcade-used is set only by a successful Arcade gateway call", async () => { + const dataDir = tempDataDir(); + const other = await captureTelemetry({ + adapter, + dataDir, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "mcp__granola__Granola_ListMeetings" }), + }); + assert.deepEqual(readdirSync(dataDir), []); + const attempt = await captureTelemetry({ + adapter, + dataDir, + input: hookInput({ + hook_event_name: "PreToolUse", + tool_name: `${CLAUDE_AI_ARCADE_TOOL_PREFIX}GoogleCalendar_ListEvents`, + tool_input: { query: "SECRET private meeting" }, + }), + }); + assert.deepEqual(readdirSync(dataDir), []); + const arcade = await captureTelemetry({ + adapter, + dataDir, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: `${ARCADE_TOOL_PREFIX}Gmail_ListEmails` }), + }); + assert.equal(readFileSync(path.join(dataDir, "arcade-used"), "utf8"), "true"); + if (process.platform !== "win32") { + assert.equal(statSync(path.join(dataDir, "arcade-used")).mode & 0o777, 0o600); + } + assert.equal(other.sent.length, 0); + assert.equal(attempt.sent.length, 1); + assert.equal(attempt.sent[0].event, "Plugin tool attempted"); + assertNoLeak(JSON.stringify(attempt.sent[0]), /SECRET|private meeting/); + assert.equal(arcade.sent.length, 1); + assert.deepEqual([attempt.sent[0], arcade.sent[0]].map((e) => e.properties.arcade_used_before), [false, false]); +}); + +test("Claude opt-out switches send nothing and write no data dir files", async () => { + const base = hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar" }); + const sessionStart = hookInput({ hook_event_name: "SessionStart", source: "startup" }); + const cases = [ + ["DISABLE_TELEMETRY=1", { DISABLE_TELEMETRY: "1" }], + ["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1", { CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1" }], + ["DISABLE_TELEMETRY=0", { DISABLE_TELEMETRY: "0" }], + ["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=false", { CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "false" }], + ["ARCADE_PLUGIN_TELEMETRY=0", { ARCADE_PLUGIN_TELEMETRY: "0" }], + ["DO_NOT_TRACK=1", { DO_NOT_TRACK: "1" }], + ["missing CLAUDE_PLUGIN_DATA", { CLAUDE_PLUGIN_DATA: "" }], + ["relative CLAUDE_PLUGIN_DATA", { CLAUDE_PLUGIN_DATA: "relative/data" }], + ]; + for (const [label, env] of cases) { + const dataDir = tempDataDir(); + for (const input of [sessionStart, base]) { + const { sent } = await captureTelemetry({ adapter, input, dataDir, env }); + assert.deepEqual(sent, [], label); + } + assert.deepEqual(readdirSync(dataDir), [], label); + } + const control = await captureTelemetry({ adapter, input: base }); + assert.equal(control.sent.length, 1, "the same prompt sends without an opt-out"); +}); + +test("enabled Claude manifest wiring matches hooks.enabled.json", async () => { + const rows = [ + ...HOOKS.filter((hook) => hook.script !== "telemetry.mjs"), + ...(await telemetryHookRows()).filter((row) => row.hosts.includes("claude-code")), + ]; + const built = buildHookManifest("claude-code", rows); + const expected = JSON.parse(readFileSync( + path.join(ROOT, "test/fixtures/telemetry/claude-code/hooks.enabled.json"), + "utf8", + )); + assert.deepEqual(built, expected); +}); + +test("checked-in Claude manifest has no telemetry rows", () => { + assert.equal(readRepoFile(HOSTS["claude-code"].manifest).includes("telemetry.mjs"), false); +}); + +test("PreToolUse and PostToolUse nest MCP, web, and per-CLI Bash telemetry groups", () => { + const enabled = JSON.parse(readFileSync( + path.join(ROOT, "test/fixtures/telemetry/claude-code/hooks.enabled.json"), + "utf8", + )).hooks; + const [pre] = enabled.PreToolUse; + assert.equal(pre.matcher, `^(?:${ARCADE_TOOL_PREFIX}|${CLAUDE_AI_ARCADE_TOOL_PREFIX})`); + for (const event of ["PostToolUse", "PostToolUseFailure"]) { + const groups = enabled[event]; + assert.equal(groups.length, 3); + assert.equal(groups[0].matcher, "mcp__.*"); + assert.equal(groups[1].matcher, "WebFetch|WebSearch"); + assert.equal(groups[2].matcher, "Bash"); + assert.equal(groups[2].hooks.length, BASH_CLIS.length); + for (const hook of groups[2].hooks) { + assert.ok(hook.if); + assert.ok(hook.command.endsWith(`--cli ${hook.if.match(/^Bash\((\w+) \*\)$/)[1]}`)); + } + } +}); + +test("every enabled telemetry command exits quietly with telemetry off", async () => { + const enabled = JSON.parse(readFileSync( + path.join(ROOT, "test/fixtures/telemetry/claude-code/hooks.enabled.json"), + "utf8", + )); + const commands = Object.values(enabled.hooks) + .flatMap((groups) => groups.flatMap((group) => group.hooks)) + .map((hook) => hook.command) + .filter((command) => command.includes("/hooks/telemetry.mjs")); + const rows = (await telemetryHookRows()).filter((row) => row.hosts.includes("claude-code")); + assert.equal(commands.length, rows.length); + const server = await startServer(); + try { + const dataDir = tempDataDir(); + // A sendable prompt; runTelemetryScript drops inherited opt-outs, so an open gate would post. + const input = hookInput({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }); + const env = telemetryEnv(adapter, dataDir, server.url); + for (const row of rows) { + const result = runTelemetryScript(JSON.stringify(input), env, "claude-code", row.extraArgs ?? []); + assert.equal(result.status, 0, `${row.event} ${row.matcher ?? ""}: ${result.stderr}`); + assert.equal(result.stdout, "", row.event); + assert.equal(result.stderr, "", row.event); + } + await assertSilentThenServerReachable({ server, adapter, input, env: hookEnv(env) }); + assert.deepEqual(readdirSync(dataDir), []); + } finally { + await server.close(); + } +}); + +test("invalid telemetry inputs are ignored without affecting hooks", async () => { + const dataDir = tempDataDir(); + const env = telemetryEnv(adapter, dataDir, "http://127.0.0.1:9"); + const cases = [ + "not-json", + JSON.stringify({ hook_event_name: "UserPromptSubmit" }), + JSON.stringify(hookInput({ hook_event_name: "UnknownHook" })), + JSON.stringify(hookInput({ hook_event_name: "PreToolUse", tool_name: "not-mcp" })), + ]; + for (const stdin of cases) { + const result = runTelemetryScript(stdin, env); + assert.equal(result.status, 0, stdin); + assert.equal(result.stdout, ""); + } + const routing = spawnSync(process.execPath, [path.join(ROOT, "hooks", "session-start.mjs"), "--host", "claude-code"], { + input: "not-json", + encoding: "utf8", + }); + assert.equal(routing.status, 0); +}); + +test("buildEvent uses adapter tool properties for malformed tool names", () => { + const options = { adapter, os: "darwin", arcadeUsedBefore: false, appWork: true }; + assert.equal(buildEvent(hookInput({ hook_event_name: "PreToolUse", tool_name: 42 }), options), null); + assert.equal(isArcadeCall(buildEvent(hookInput({ + hook_event_name: "PostToolUse", + tool_name: `${ARCADE_TOOL_PREFIX}Gmail_ListEmails`, + }), options)), true); +}); diff --git a/test/telemetry-helpers.mjs b/test/telemetry-helpers.mjs index fca1824..eab341c 100644 --- a/test/telemetry-helpers.mjs +++ b/test/telemetry-helpers.mjs @@ -4,7 +4,7 @@ */ import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdtempSync, rmSync } from "node:fs"; import http from "node:http"; @@ -13,33 +13,46 @@ import path from "node:path"; import { after } from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; import { eventSchema } from "../hooks/telemetry-contract.mjs"; -import { PLUGIN_VERSION } from "../hooks/telemetry-config.mjs"; +import { EVENT_ENV, PLUGIN_VERSION } from "../hooks/telemetry-config.mjs"; +import { buildEvent } from "../hooks/telemetry-events.mjs"; import { runTelemetry } from "../hooks/telemetry-run.mjs"; import { ROOT } from "./helpers.mjs"; +// An inherited opt-out, destination, plugin root or data dir would make a +// silence test pass for the wrong reason or send somewhere other than its server. const TELEMETRY_ENV_BLOCKLIST = [ "ARCADE_PLUGIN_TELEMETRY", + "ARCADE_PLUGIN_TELEMETRY_HOST", "DO_NOT_TRACK", "DISABLE_TELEMETRY", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "COPILOT_OFFLINE", "CLAUDE_PLUGIN_DATA", "COPILOT_PLUGIN_DATA", + "PLUGIN_ROOT", ]; +/** + * process.env without inherited opt-outs, telemetry destination, plugin root + * or data dirs, plus `overrides`. + * @param {NodeJS.ProcessEnv} [overrides] + */ +export const hookEnv = (overrides = {}) => { + const env = { ...process.env }; + for (const key of TELEMETRY_ENV_BLOCKLIST) delete env[key]; + return { ...env, ...overrides }; +}; + /** @type {string[]} */ const tempDataDirs = []; -let tempDataCleanupRegistered = false; +// Registered at load so it runs after the whole file; `after` inside a test runs after that test only. +after(() => { + for (const dir of tempDataDirs) rmSync(dir, { recursive: true, force: true }); + tempDataDirs.length = 0; +}); /** Absolute path of a fresh temp directory; removed after the test run finishes. */ export const tempDataDir = () => { - if (!tempDataCleanupRegistered) { - after(() => { - for (const dir of tempDataDirs) rmSync(dir, { recursive: true, force: true }); - tempDataDirs.length = 0; - }); - tempDataCleanupRegistered = true; - } const dir = mkdtempSync(path.join(os.tmpdir(), "arcade-telemetry-")); tempDataDirs.push(dir); return dir; @@ -50,12 +63,8 @@ export const tempDataDir = () => { * @param {string | null} dataDir * @param {NodeJS.ProcessEnv} overrides */ -const captureEnv = (adapter, dataDir, overrides) => { - const env = { ...process.env }; - for (const key of TELEMETRY_ENV_BLOCKLIST) delete env[key]; - if (dataDir !== null) env[adapter.dataVariable] = dataDir; - return { ...env, ...overrides }; -}; +const captureEnv = (adapter, dataDir, overrides) => + hookEnv({ ...(dataDir === null ? {} : { [adapter.dataVariable]: dataDir }), ...overrides }); /** * @param {object} options @@ -182,5 +191,43 @@ export const runTelemetryScript = (stdin, env, hostName = "claude-code", extraAr spawnSync(process.execPath, [path.join(ROOT, "hooks", "telemetry.mjs"), "--host", hostName, ...extraArgs], { input: stdin, encoding: "utf8", - env: { ...process.env, ...env }, + env: hookEnv(env), }); + +export const SEND_SCRIPT = path.join(ROOT, "hooks", "telemetry-send.mjs"); + +/** + * Async spawn keeps this process's event loop free so an in-process server can answer. + * @param {string} command + * @param {string[]} args + * @param {import("node:child_process").SpawnOptions} options + * @returns {Promise} + */ +export const spawnExitCode = (command, args, options) => + new Promise((resolve) => spawn(command, args, { ...options, stdio: "ignore" }).on("exit", resolve)); + +/** @param {NodeJS.ProcessEnv} env */ +export const sendWithNode = (env) => spawnExitCode(process.execPath, [SEND_SCRIPT], { env }); + +/** + * Asserts nothing reached `server`, then sends the event `input` would build + * through `send` with `env` and asserts it arrives. That proves the hooks' + * environment resolves to the server, so the empty request list could not + * come from sending somewhere else. + * @param {object} options + * @param {{ requests: { body: string }[] }} options.server + * @param {import("../hooks/telemetry-adapter.mjs").TelemetryAdapter} options.adapter + * @param {Record} options.input A hook input classified as app work. + * @param {NodeJS.ProcessEnv} options.env + * @param {(env: NodeJS.ProcessEnv) => Promise} [options.send] + */ +export const assertSilentThenServerReachable = async ({ server, adapter, input, env, send = sendWithNode }) => { + await sleep(300); + assert.deepEqual(server.requests, []); + const event = buildEvent(input, { adapter, os: "linux", arcadeUsedBefore: false, appWork: true }); + assert.ok(event); + assert.equal(await send({ ...env, [EVENT_ENV]: JSON.stringify(event) }), 0); + await waitForRequests(server.requests, 1); + assert.equal(server.requests.length, 1); + assert.equal(JSON.parse(server.requests[0].body).event, event.event); +}; diff --git a/test/telemetry-helpers.test.mjs b/test/telemetry-helpers.test.mjs new file mode 100644 index 0000000..c9b06b8 --- /dev/null +++ b/test/telemetry-helpers.test.mjs @@ -0,0 +1,39 @@ +// @ts-check + +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { test } from "node:test"; +import { pathToFileURL } from "node:url"; +import { ROOT } from "./helpers.mjs"; +import { tempDataDir } from "./telemetry-helpers.mjs"; + +test("tempDataDir keeps directories from every test until the file ends, then removes them all", () => { + const helpers = pathToFileURL(path.join(ROOT, "test", "telemetry-helpers.mjs")).href; + const file = path.join(tempDataDir(), "uses-temp-dirs.test.mjs"); + writeFileSync(file, ` +import { existsSync } from "node:fs"; +import { test } from "node:test"; +import { tempDataDir } from ${JSON.stringify(helpers)}; +const made = []; +for (const name of ["first", "second", "third"]) { + test(name, () => { + const dir = tempDataDir(); + made.push(dir); + console.log("TEMP_DIR " + dir); + }); +} +test("directories from earlier tests still exist", () => { + for (const dir of made) if (!existsSync(dir)) throw new Error("removed early: " + dir); +}); +`); + // Without NODE_TEST_CONTEXT the child prints readable TAP instead of the runner's binary protocol. + const env = { ...process.env }; + delete env.NODE_TEST_CONTEXT; + const result = spawnSync(process.execPath, [file], { encoding: "utf8", env }); + assert.equal(result.status, 0, result.stdout + result.stderr); + const dirs = [...result.stdout.matchAll(/^TEMP_DIR (.+)$/gm)].map((match) => match[1]); + assert.equal(dirs.length, 3, result.stdout); + for (const dir of dirs) assert.equal(existsSync(dir), false, `left behind: ${dir}`); +});