From bda3a8525c52d90dcf4147d6828a79de7dcac0e4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 12 Oct 2026 19:08:02 +0000 Subject: [PATCH 1/6] feat(telemetry): add Claude Code telemetry adapter (2/4, GRO-305) Replays the net #27 slice (old foundation 26710300..1c51f07f) onto main 6f4cdc76, which now contains the squash-merged foundation (#26). Telemetry stays hard-OFF (TELEMETRY_ENABLED = false). Co-authored-by: Teal Larson --- hooks/hook-hosts.mjs | 1 + hooks/telemetry-adapters/claude-code.mjs | 122 ++++++ .../claude-code/auth-status-response.mjs | 10 + .../telemetry/claude-code/event-cases.mjs | 154 +++++++ .../telemetry/claude-code/hooks.enabled.json | 191 +++++++++ test/telemetry-claude.test.mjs | 393 ++++++++++++++++++ 6 files changed, 871 insertions(+) create mode 100644 hooks/telemetry-adapters/claude-code.mjs create mode 100644 test/fixtures/telemetry/claude-code/auth-status-response.mjs create mode 100644 test/fixtures/telemetry/claude-code/event-cases.mjs create mode 100644 test/fixtures/telemetry/claude-code/hooks.enabled.json create mode 100644 test/telemetry-claude.test.mjs diff --git a/hooks/hook-hosts.mjs b/hooks/hook-hosts.mjs index 1c3c4a9..74ee29d 100644 --- a/hooks/hook-hosts.mjs +++ b/hooks/hook-hosts.mjs @@ -21,6 +21,7 @@ export const HOSTS = { manifest: ".claude-plugin/hooks.json", format: "nested", rootVariable: "CLAUDE_PLUGIN_ROOT", + telemetry: "claude-code", contextOutput: (eventName, text) => ({ hookSpecificOutput: { hookEventName: eventName, additionalContext: text }, }), diff --git a/hooks/telemetry-adapters/claude-code.mjs b/hooks/telemetry-adapters/claude-code.mjs new file mode 100644 index 0000000..abff60c --- /dev/null +++ b/hooks/telemetry-adapters/claude-code.mjs @@ -0,0 +1,122 @@ +// @ts-check + +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { serviceForToolkit, serviceForToolName } from "../telemetry-classify.mjs"; +import { BASH_CLIS, GATEWAY_TOOLS } from "../telemetry-contract.mjs"; +import { + arcadeToolProperties, + builtinToolProperties as sharedBuiltinToolProperties, + otherServerProperties, +} from "../telemetry-events.mjs"; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const plugin = JSON.parse(readFileSync(path.join(ROOT, "plugin.json"), "utf8")); +const [mcpServerKey] = Object.keys(JSON.parse(readFileSync(path.join(ROOT, "mcp.json"), "utf8")).mcpServers); + +export const ARCADE_TOOL_PREFIX = `mcp__plugin_${plugin.name}_${mcpServerKey}__`; +export const CLAUDE_AI_ARCADE_TOOL_PREFIX = "mcp__claude_ai_arcade__"; + +const preToolMatcher = `^(?:${ARCADE_TOOL_PREFIX}|${CLAUDE_AI_ARCADE_TOOL_PREFIX})`; + +/** + * @param {unknown} toolName + * @param {Record | undefined} toolInput + */ +const claudeToolProperties = (toolName, toolInput) => { + if (typeof toolName !== "string" || !toolName.startsWith("mcp__")) { + return null; + } + const splitAt = toolName.lastIndexOf("__"); + const tool = toolName.slice(splitAt + 2); + if (toolName.startsWith(ARCADE_TOOL_PREFIX)) { + return arcadeToolProperties("arcade", tool, toolInput); + } + if (toolName.startsWith(CLAUDE_AI_ARCADE_TOOL_PREFIX)) { + return arcadeToolProperties("other_arcade", tool, toolInput); + } + if (/** @type {readonly string[]} */ (GATEWAY_TOOLS).includes(tool)) { + return arcadeToolProperties("other_arcade", tool, toolInput); + } + // Some connectors name the service in the server, not the tool, e.g. + // mcp__claude_ai_Gmail__search_threads. Only the category is sent. + const serverParts = toolName.slice("mcp__".length, splitAt).split(/[^A-Za-z0-9]+/); + const service = + serviceForToolName(tool) ?? serverParts.map(serviceForToolkit).find(Boolean); + return otherServerProperties(service ?? undefined); +}; + +/** + * @param {unknown} toolName + * @param {Record | undefined} toolInput + */ +const claudeAttemptProperties = (toolName, toolInput) => { + if (typeof toolName !== "string") return null; + if (toolName.startsWith(ARCADE_TOOL_PREFIX)) { + return arcadeToolProperties("arcade", toolName.slice(ARCADE_TOOL_PREFIX.length), toolInput); + } + if (toolName.startsWith(CLAUDE_AI_ARCADE_TOOL_PREFIX)) { + return arcadeToolProperties("other_arcade", toolName.slice(CLAUDE_AI_ARCADE_TOOL_PREFIX.length), toolInput); + } + return null; +}; + +/** @param {Record} raw */ +const normalize = (raw) => ({ + hook_event_name: raw.hook_event_name, + session_id: raw.session_id, + prompt_id: raw.prompt_id, + source: raw.source, + prompt: raw.prompt, + tool_name: raw.tool_name, + tool_input: raw.tool_input, + tool_response: raw.tool_response, + error: raw.error, + is_interrupt: raw.is_interrupt, + agent_type: raw.agent_type, + agent_id: raw.agent_id, + last_assistant_message: raw.last_assistant_message, +}); + +// `if` is a Claude Code permission rule so the hook does not start for other commands; +// `extraArgs` pass the CLI name through to builtinToolProperties. +/** @param {"PostToolUse" | "PostToolUseFailure"} event */ +const bashHookRows = (event) => + BASH_CLIS.map((cli) => ({ + event, + matcher: "Bash", + if: `Bash(${cli} *)`, + extraArgs: ["--cli", cli], + })); + +/** @type {import("../telemetry-adapter.mjs").TelemetryAdapter} */ +const claudeCodeAdapter = { + host: "claude-code", + dataVariable: "CLAUDE_PLUGIN_DATA", + // Claude Code reads these as set for any non-empty value, even "0" or "false". + optOutSwitches: [ + { name: "DISABLE_TELEMETRY", anyValue: true }, + { name: "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", anyValue: true }, + ], + requiresTurn: true, + promptReminder: true, + subagentSession: false, + hookRows: [ + { event: "UserPromptSubmit" }, + { event: "PreToolUse", matcher: preToolMatcher }, + { event: "PostToolUse", matcher: "mcp__.*" }, + { event: "PostToolUse", matcher: "WebFetch|WebSearch" }, + ...bashHookRows("PostToolUse"), + { event: "PostToolUseFailure", matcher: "mcp__.*" }, + { event: "PostToolUseFailure", matcher: "WebFetch|WebSearch" }, + ...bashHookRows("PostToolUseFailure"), + { event: "SubagentStop" }, + ], + normalize, + toolProperties: claudeToolProperties, + attemptProperties: claudeAttemptProperties, + builtinToolProperties: sharedBuiltinToolProperties, +}; + +export default claudeCodeAdapter; diff --git a/test/fixtures/telemetry/claude-code/auth-status-response.mjs b/test/fixtures/telemetry/claude-code/auth-status-response.mjs new file mode 100644 index 0000000..cac1f49 --- /dev/null +++ b/test/fixtures/telemetry/claude-code/auth-status-response.mjs @@ -0,0 +1,10 @@ +/** Text of a System_ManageAuthorization status answer, as tool_response. */ +export const authStatusResponse = (statuses) => [ + { + type: "text", + text: JSON.stringify({ + message: statuses.includes("authorization_required") ? "Not yet authorized: dropbox." : "All authorized.", + providers: statuses.map((status, index) => ({ provider: `provider${index}`, status })), + }), + }, +]; diff --git a/test/fixtures/telemetry/claude-code/event-cases.mjs b/test/fixtures/telemetry/claude-code/event-cases.mjs new file mode 100644 index 0000000..1e1ae57 --- /dev/null +++ b/test/fixtures/telemetry/claude-code/event-cases.mjs @@ -0,0 +1,154 @@ +import { + ARCADE_TOOL_PREFIX, + CLAUDE_AI_ARCADE_TOOL_PREFIX, +} from "../../../../hooks/telemetry-adapters/claude-code.mjs"; +import { authStatusResponse } from "./auth-status-response.mjs"; + +const SIGN_IN = `${ARCADE_TOOL_PREFIX}System_ManageAuthorization`; +const OPERATOR = "arcade:arcade-operator"; + +const tool = (name, toolInput) => ({ hook_event_name: "PostToolUse", tool_name: name, tool_input: toolInput }); +const failed = (name, error, fields = {}) => ({ + hook_event_name: "PostToolUseFailure", + tool_name: name, + error, + ...fields, +}); +const signInCheck = (name, statuses) => ({ + hook_event_name: "PostToolUse", + tool_name: name, + tool_response: authStatusResponse(statuses), +}); +const stop = (message) => ({ + hook_event_name: "SubagentStop", + agent_type: OPERATOR, + last_assistant_message: message, +}); +const operator = (status) => ({ agent: "arcade-operator", status }); + +/** + * @typedef {object} EventCase + * @property {string} id + * @property {Record} input + * @property {null | { event: string, extra: Record }} expected + * @property {string[]} [argv] + * @property {boolean} [seedAppWork] + */ + +/** @returns {EventCase[]} */ +export const eventCases = () => { + const ATTEMPTED = "Plugin tool attempted"; + const CALLED = "Plugin tool called"; + const FAILED = "Plugin tool failed"; + const STOPPED = "Plugin subagent stopped"; + /** @type {EventCase[]} */ + const cases = [ + { id: "session-start-startup", input: { hook_event_name: "SessionStart", source: "startup" }, expected: null }, + { id: "prompt-app-work", input: { hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }, + expected: { event: "Plugin prompt submitted", extra: { could_use_arcade: true, service_hints: ["calendar"], reminder_sent: true } } }, + { id: "prompt-confirmation-style", input: { hook_event_name: "UserPromptSubmit", prompt: "ok", prompt_id: "confirm-turn" }, + expected: { event: "Plugin prompt submitted", extra: { could_use_arcade: false, service_hints: [], reminder_sent: false } }, + seedAppWork: true }, + { id: "prompt-unrelated", input: { hook_event_name: "UserPromptSubmit", prompt: "Fix the parser" }, expected: null }, + { id: "prompt-task-notification", input: { + hook_event_name: "UserPromptSubmit", + prompt: "\ncompleted calendar", + }, expected: null }, + { id: "pre-arcade-select", input: { hook_event_name: "PreToolUse", tool_name: `${ARCADE_TOOL_PREFIX}Arcade_SelectTools` }, + expected: { event: ATTEMPTED, extra: { server: "arcade", tool: "Arcade_SelectTools" } } }, + { id: "pre-claude-ai-select", input: { hook_event_name: "PreToolUse", tool_name: `${CLAUDE_AI_ARCADE_TOOL_PREFIX}Arcade_SelectTools` }, + expected: { event: ATTEMPTED, extra: { server: "other_arcade", tool: "Arcade_SelectTools" } } }, + { id: "pre-claude-ai-calendar", input: { hook_event_name: "PreToolUse", tool_name: `${CLAUDE_AI_ARCADE_TOOL_PREFIX}GoogleCalendar_ListEvents` }, + expected: { event: ATTEMPTED, extra: { server: "other_arcade", tool: "app_tool", service: "calendar" } } }, + { id: "pre-use-tool", input: { + hook_event_name: "PreToolUse", + tool_name: `${ARCADE_TOOL_PREFIX}Arcade_UseTool`, + tool_input: { tool_name: "GoogleCalendar.ListEvents" }, + }, expected: { event: ATTEMPTED, extra: { server: "arcade", tool: "Arcade_UseTool", service: "calendar" } } }, + { id: "pre-other-arcade-tool", input: { hook_event_name: "PreToolUse", tool_name: `${CLAUDE_AI_ARCADE_TOOL_PREFIX}AcmeHR_RunPayroll` }, + expected: { event: ATTEMPTED, extra: { server: "other_arcade", tool: "other" } } }, + { id: "pre-gmail-connector", input: { hook_event_name: "PreToolUse", tool_name: "mcp__claude_ai_Gmail__search_threads" }, expected: null }, + { id: "pre-wrong-arcade-connector", input: { hook_event_name: "PreToolUse", tool_name: "mcp__claude_ai_Arcade__Arcade_SelectTools" }, expected: null }, + { id: "post-gmail", input: tool(`${ARCADE_TOOL_PREFIX}Gmail_ListEmails`), + expected: { event: CALLED, extra: { server: "arcade", tool: "app_tool", service: "email" } } }, + { id: "post-claude-ai-calendar", input: tool(`${CLAUDE_AI_ARCADE_TOOL_PREFIX}GoogleCalendar_ListEvents`), + expected: { event: CALLED, extra: { server: "other_arcade", tool: "app_tool", service: "calendar" } } }, + { id: "post-sign-in-no-auth", input: tool(SIGN_IN), + expected: { event: CALLED, extra: { server: "arcade", tool: "System_ManageAuthorization", auth_needed: false } } }, + { id: "post-sign-in-auth-needed", input: signInCheck(SIGN_IN, ["authorized", "authorization_required"]), + expected: { event: CALLED, extra: { server: "arcade", tool: "System_ManageAuthorization", auth_needed: true } } }, + { id: "post-sign-in-authorized", input: signInCheck(SIGN_IN, ["authorized"]), + expected: { event: CALLED, extra: { server: "arcade", tool: "System_ManageAuthorization", auth_needed: false } } }, + { id: "post-claude-ai-sign-in", input: signInCheck("mcp__claude_ai_Arcade__System_ManageAuthorization", ["authorization_required"]), + expected: { event: CALLED, extra: { server: "other_arcade", tool: "System_ManageAuthorization", auth_needed: true } } }, + { id: "post-use-tool-gmail", input: tool(`${ARCADE_TOOL_PREFIX}Arcade_UseTool`, { tool_name: "Gmail.ListEmails" }), + expected: { event: CALLED, extra: { server: "arcade", tool: "Arcade_UseTool", service: "email" } } }, + { id: "post-use-tool-calendar", input: tool(`${ARCADE_TOOL_PREFIX}Arcade_UseTool`, { tool_name: "GoogleCalendar.ListEvents" }), + expected: { event: CALLED, extra: { server: "arcade", tool: "Arcade_UseTool", service: "calendar" } } }, + { id: "post-use-tool-unknown", input: tool(`${ARCADE_TOOL_PREFIX}Arcade_UseTool`, { tool_name: "AcmeHR.RunPayroll" }), + expected: { event: CALLED, extra: { server: "arcade", tool: "Arcade_UseTool" } } }, + { id: "post-other-arcade-app", input: tool(`${ARCADE_TOOL_PREFIX}AcmeHR_RunPayroll`), + expected: { event: CALLED, extra: { server: "arcade", tool: "other" } } }, + { id: "post-claude-ai-use-tool", input: tool("mcp__claude_ai_Arcade_Production__Arcade_UseTool", { tool_name: "Slack_SendMessage" }), + expected: { event: CALLED, extra: { server: "other_arcade", tool: "Arcade_UseTool", service: "chat" } } }, + { id: "post-other-server-granola", input: tool("mcp__granola__Granola_ListMeetings"), + expected: { event: CALLED, extra: { server: "other", service: "meetings" } }, seedAppWork: true }, + { id: "post-gmail-connector", input: tool("mcp__claude_ai_Gmail__search_threads"), + expected: { event: CALLED, extra: { server: "other", service: "email" } }, seedAppWork: true }, + { id: "post-other-server-unknown", input: tool("mcp__secret-server__DoThing"), + expected: { event: CALLED, extra: { server: "other" } }, seedAppWork: true }, + { id: "post-read-builtin", input: tool("Read"), expected: null }, + { id: "failure-slack", input: { hook_event_name: "PostToolUseFailure", tool_name: `${ARCADE_TOOL_PREFIX}Slack_SendMessage` }, + expected: { event: FAILED, extra: { server: "arcade", tool: "app_tool", service: "chat", failure_kind: "tool_error" } } }, + { id: "failure-auth", input: failed(`${ARCADE_TOOL_PREFIX}Gmail_ListEmails`, + '{"message":"The tool was not executed because it requires authorization.","authorization_url":"https://example.com/auth"}'), + expected: { event: FAILED, extra: { server: "arcade", tool: "app_tool", service: "email", failure_kind: "auth_required" } } }, + { id: "failure-session-expired", input: failed(`${ARCADE_TOOL_PREFIX}Arcade_UseTool`, 'MCP server "plugin:arcade:arcade" session expired', + { tool_input: { tool_name: "Gmail.ListEmails" } }), + expected: { event: FAILED, extra: { server: "arcade", tool: "Arcade_UseTool", service: "email", failure_kind: "session_expired" } } }, + { id: "failure-unreachable", input: failed(`${ARCADE_TOOL_PREFIX}Gmail_ListEmails`, "Connection closed"), + expected: { event: FAILED, extra: { server: "arcade", tool: "app_tool", service: "email", failure_kind: "unreachable" } } }, + { id: "failure-interrupted", input: failed(`${ARCADE_TOOL_PREFIX}Gmail_ListEmails`, "Connection closed", { is_interrupt: true }), + expected: { event: FAILED, extra: { server: "arcade", tool: "app_tool", service: "email", failure_kind: "interrupted" } } }, + { id: "failure-other-http", input: failed("mcp__secret-server__DoThing", "Error POSTing to endpoint: internal error"), + expected: { event: FAILED, extra: { server: "other", failure_kind: "http_error" } }, seedAppWork: true }, + { id: "failure-read", input: failed("Read", "File does not exist."), expected: null }, + { id: "subagent-operator-needs-auth", input: stop("Done.\n\nstatus: needs_auth\nsummary: sign in"), + expected: { event: STOPPED, extra: operator("needs_auth") } }, + { id: "subagent-operator-needs-confirmation", input: stop("**status:** needs_confirmation"), + expected: { event: STOPPED, extra: operator("needs_confirmation") } }, + { id: "subagent-operator-needs-clarification", input: stop("- status: `needs_clarification`"), + expected: { event: STOPPED, extra: operator("needs_clarification") } }, + { id: "subagent-operator-failed", input: stop("STATUS: FAILED"), + expected: { event: STOPPED, extra: operator("failed") } }, + { id: "subagent-operator-unknown", input: stop("status: exploded"), + expected: { event: STOPPED, extra: operator("unknown") } }, + { id: "subagent-operator-no-message", input: stop(undefined), + expected: { event: STOPPED, extra: operator("unknown") } }, + { id: "subagent-non-operator", input: { + hook_event_name: "SubagentStop", + agent_type: "general-purpose", + last_assistant_message: "status: completed", + }, expected: null }, + { id: "subagent-operator-completed", input: { + hook_event_name: "SubagentStop", + agent_type: OPERATOR, + agent_id: "agent-1", + last_assistant_message: "status: completed", + }, expected: { event: STOPPED, extra: operator("completed") } }, + { id: "builtin-webfetch", input: { hook_event_name: "PostToolUse", tool_name: "WebFetch", tool_input: { url: "https://example.com" } }, + expected: { event: "Plugin built-in tool called", extra: { tool: "WebFetch" } }, seedAppWork: true }, + { id: "builtin-websearch", input: { hook_event_name: "PostToolUse", tool_name: "WebSearch", tool_input: { query: "weather" } }, + expected: { event: "Plugin built-in tool called", extra: { tool: "WebSearch" } }, seedAppWork: true }, + { id: "builtin-bash-gh", input: { hook_event_name: "PostToolUse", tool_name: "Bash", tool_input: { command: "gh pr list" } }, + expected: { event: "Plugin built-in tool called", extra: { tool: "Bash", cli: "gh", service: "code_hosting" } }, + argv: ["--cli", "gh"], seedAppWork: true }, + { id: "builtin-bash-curl", input: { hook_event_name: "PostToolUseFailure", tool_name: "Bash", tool_input: { command: "curl x" } }, + expected: { event: "Plugin built-in tool failed", extra: { tool: "Bash", cli: "curl" } }, + argv: ["--cli", "curl"], seedAppWork: true }, + { id: "builtin-bash-non-listed", input: { hook_event_name: "PostToolUse", tool_name: "Bash", tool_input: { command: "echo hi" } }, + expected: null, argv: ["--cli", "gh"], seedAppWork: true }, + { id: "session-start-compact", input: { hook_event_name: "SessionStart", source: "compact" }, expected: null }, + ]; + return cases; +}; diff --git a/test/fixtures/telemetry/claude-code/hooks.enabled.json b/test/fixtures/telemetry/claude-code/hooks.enabled.json new file mode 100644 index 0000000..71cd3f7 --- /dev/null +++ b/test/fixtures/telemetry/claude-code/hooks.enabled.json @@ -0,0 +1,191 @@ +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/session-start.mjs\" --host claude-code", + "timeout": 5 + } + ] + } + ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/user-prompt-submit.mjs\" --host claude-code", + "timeout": 5 + }, + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + } + ], + "SubagentStart": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-start.mjs\" --host claude-code", + "timeout": 5 + } + ] + } + ], + "PreToolUse": [ + { + "matcher": "^(?:mcp__plugin_arcade_arcade__|mcp__claude_ai_arcade__)", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + } + ], + "PostToolUse": [ + { + "matcher": "mcp__.*", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + }, + { + "matcher": "WebFetch|WebSearch", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + }, + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "if": "Bash(gh *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli gh", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(glab *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli glab", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(curl *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli curl", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(wget *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli wget", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(http *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli http", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(osascript *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli osascript", + "timeout": 5 + } + ] + } + ], + "PostToolUseFailure": [ + { + "matcher": "mcp__.*", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + }, + { + "matcher": "WebFetch|WebSearch", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + }, + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "if": "Bash(gh *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli gh", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(glab *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli glab", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(curl *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli curl", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(wget *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli wget", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(http *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli http", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(osascript *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli osascript", + "timeout": 5 + } + ] + } + ], + "SubagentStop": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + } + ] + } +} diff --git a/test/telemetry-claude.test.mjs b/test/telemetry-claude.test.mjs new file mode 100644 index 0000000..3247549 --- /dev/null +++ b/test/telemetry-claude.test.mjs @@ -0,0 +1,393 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { readFileSync, readdirSync, statSync } from "node:fs"; +import path from "node:path"; +import { test } from "node:test"; +import { BASH_CLIS } from "../hooks/telemetry-contract.mjs"; +import { HOOKS, HOSTS, telemetryHookRows } from "../hooks/hook-hosts.mjs"; +import { SCOPE_TTL_MS } from "../hooks/hook-scope.mjs"; +import { buildEvent, isArcadeCall } from "../hooks/telemetry-events.mjs"; +import { runTelemetry } from "../hooks/telemetry-run.mjs"; +import adapter, { + ARCADE_TOOL_PREFIX, + CLAUDE_AI_ARCADE_TOOL_PREFIX, +} from "../hooks/telemetry-adapters/claude-code.mjs"; +import { buildHookManifest } from "../scripts/generate-manifests.mjs"; +import { readRepoFile, ROOT } from "./helpers.mjs"; +import { eventCases } from "./fixtures/telemetry/claude-code/event-cases.mjs"; +import { + assertMatchesContract, + assertNoLeak, + captureTelemetry, + expectedEvent, + hookInput, + runTelemetryScript, + sleep, + startServer, + telemetryEnv, + tempDataDir, + waitForRequests, +} from "./telemetry-helpers.mjs"; + +const OPERATOR = "arcade:arcade-operator"; +const SESSION_ID = "raw-session-id-123"; +const PROMPT_ID = "raw-prompt-id-456"; +const LEAK_RE = /SECRET|private-repo|raw-session|raw-prompt|authorization_url|example\.com/i; + +const seedAppWork = async (dataDir, promptId = PROMPT_ID, now) => { + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ + hook_event_name: "UserPromptSubmit", + prompt: "Check my calendar", + prompt_id: promptId, + }), + }); +}; + +test("Arcade tool prefixes match plugin.json and mcp.json", () => { + const plugin = JSON.parse(readRepoFile("plugin.json")); + const [server] = Object.keys(JSON.parse(readRepoFile("mcp.json")).mcpServers); + assert.equal(ARCADE_TOOL_PREFIX, `mcp__plugin_${plugin.name}_${server}__`); + assert.equal(CLAUDE_AI_ARCADE_TOOL_PREFIX, "mcp__claude_ai_arcade__"); +}); + +test("every fixture maps through captureTelemetry to the expected contract event", async () => { + for (const { id, input, expected, argv = [], seedAppWork: seed } of eventCases()) { + const dataDir = tempDataDir(); + if (seed) await seedAppWork(dataDir, PROMPT_ID); + const { sent } = await captureTelemetry({ adapter, dataDir, input: hookInput(input), argv }); + if (expected === null) { + assert.equal(sent.length, 0, id); + continue; + } + assert.equal(sent.length, 1, id); + const promptId = typeof input.prompt_id === "string" ? input.prompt_id : PROMPT_ID; + const want = expectedEvent(expected.event, expected.extra, adapter, SESSION_ID, promptId); + want.properties.os = process.platform; + assert.deepEqual(sent[0], want, id); + assertMatchesContract(sent[0], id); + assertNoLeak(JSON.stringify(sent[0]), LEAK_RE); + } +}); + +test("scope transitions across a sequence sharing one data dir", async () => { + const dataDir = tempDataDir(); + const now = 1_000_000; + const calendar = "calendar-turn"; + const unrelated = "unrelated-turn"; + const confirm = "confirm-turn"; + + let r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Fix the parser", prompt_id: unrelated }), + }); + assert.equal(r.sent.length, 0); + r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: unrelated }), + }); + assert.equal(r.sent.length, 0); + + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar", prompt_id: calendar }), + }); + r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: calendar }), + }); + assert.equal(r.sent.length, 1); + + r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: "other-turn" }), + }); + assert.equal(r.sent.length, 0); + + await captureTelemetry({ + adapter, + dataDir, + now: now + 1000, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "yes, send it", prompt_id: confirm }), + }); + r = await captureTelemetry({ + adapter, + dataDir, + now: now + 1000, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: confirm }), + }); + assert.equal(r.sent.length, 1); + + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar", prompt_id: calendar }), + }); + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "SessionStart", source: "compact", session_id: SESSION_ID }), + }); + r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: calendar }), + }); + assert.equal(r.sent.length, 1, "compact SessionStart keeps scope"); + + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar", prompt_id: "after-compact" }), + }); + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "SessionStart", source: "startup", session_id: SESSION_ID }), + }); + r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: calendar }), + }); + assert.equal(r.sent.length, 0, "normal SessionStart clears scope"); + + const expTurn = "expiry-turn"; + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar", prompt_id: expTurn }), + }); + r = await captureTelemetry({ + adapter, + dataDir, + now: now + SCOPE_TTL_MS + 1, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: expTurn }), + }); + assert.equal(r.sent.length, 0, "scope expires after 30 minutes"); +}); + +test("direct Arcade calls are observed without prompt scope; other and built-in tools need scope", async () => { + const fresh = tempDataDir(); + const arcade = await captureTelemetry({ + adapter, + dataDir: fresh, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: `${ARCADE_TOOL_PREFIX}Gmail_ListEmails` }), + }); + assert.equal(arcade.sent.length, 1); + + const scoped = tempDataDir(); + const other = await captureTelemetry({ + adapter, + dataDir: scoped, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "mcp__granola__Granola_ListMeetings" }), + }); + assert.equal(other.sent.length, 0); + await seedAppWork(scoped); + const otherInScope = await captureTelemetry({ + adapter, + dataDir: scoped, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "mcp__granola__Granola_ListMeetings" }), + }); + assert.equal(otherInScope.sent.length, 1); + + const builtinDir = tempDataDir(); + const builtin = await captureTelemetry({ + adapter, + dataDir: builtinDir, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch" }), + }); + assert.equal(builtin.sent.length, 0); +}); + +test("arcade-used is set only by a successful Arcade gateway call", async () => { + const dataDir = tempDataDir(); + const env = telemetryEnv(adapter, dataDir, "http://127.0.0.1:9"); + const send = []; + await runTelemetry({ + adapter, + env, + enabled: true, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "mcp__granola__Granola_ListMeetings" }), + send: (e) => send.push(e), + }); + assert.deepEqual(readdirSync(dataDir), []); + await runTelemetry({ + adapter, + env, + enabled: true, + input: hookInput({ + hook_event_name: "PreToolUse", + tool_name: `${CLAUDE_AI_ARCADE_TOOL_PREFIX}GoogleCalendar_ListEvents`, + tool_input: { query: "SECRET private meeting" }, + }), + send: (e) => send.push(e), + }); + assert.deepEqual(readdirSync(dataDir), []); + await runTelemetry({ + adapter, + env, + enabled: true, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: `${ARCADE_TOOL_PREFIX}Gmail_ListEmails` }), + send: (e) => send.push(e), + }); + assert.equal(readFileSync(path.join(dataDir, "arcade-used"), "utf8"), "true"); + if (process.platform !== "win32") { + assert.equal(statSync(path.join(dataDir, "arcade-used")).mode & 0o777, 0o600); + } + const attempt = send.find((e) => e.event === "Plugin tool attempted"); + assert.ok(attempt); + assertNoLeak(JSON.stringify(attempt), /SECRET|private meeting/); + const flags = send.map((e) => e.properties.arcade_used_before).sort(); + assert.deepEqual(flags, [false, false]); +}); + +test("Claude opt-out switches send nothing and write no data dir files", async () => { + const server = await startServer(); + try { + const sent = []; + const base = hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar" }); + const sessionStart = hookInput({ hook_event_name: "SessionStart", source: "startup" }); + const cases = [ + ["DISABLE_TELEMETRY=1", { DISABLE_TELEMETRY: "1" }], + ["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1", { CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1" }], + ["DISABLE_TELEMETRY=0", { DISABLE_TELEMETRY: "0" }], + ["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=false", { CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "false" }], + ["ARCADE_PLUGIN_TELEMETRY=0", { ARCADE_PLUGIN_TELEMETRY: "0" }], + ["DO_NOT_TRACK=1", { DO_NOT_TRACK: "1" }], + ["missing CLAUDE_PLUGIN_DATA", { CLAUDE_PLUGIN_DATA: "" }], + ["relative CLAUDE_PLUGIN_DATA", { CLAUDE_PLUGIN_DATA: "relative/data" }], + ]; + for (const [label, extra] of cases) { + const dataDir = tempDataDir(); + const env = telemetryEnv(adapter, dataDir, server.url, extra); + for (const input of [sessionStart, base]) { + await runTelemetry({ adapter, input, env, enabled: true, send: (e) => sent.push(e) }); + } + assert.deepEqual(readdirSync(dataDir), [], label); + } + await sleep(200); + assert.deepEqual(server.requests, []); + assert.deepEqual(sent, []); + } finally { + await server.close(); + } +}); + +test("enabled Claude manifest wiring matches hooks.enabled.json", async () => { + const rows = [ + ...HOOKS.filter((hook) => hook.script !== "telemetry.mjs"), + ...(await telemetryHookRows()).filter((row) => row.hosts.includes("claude-code")), + ]; + const built = buildHookManifest("claude-code", rows); + const expected = JSON.parse(readFileSync( + path.join(ROOT, "test/fixtures/telemetry/claude-code/hooks.enabled.json"), + "utf8", + )); + assert.deepEqual(built, expected); +}); + +test("checked-in Claude manifest has no telemetry rows", () => { + assert.equal(readRepoFile(HOSTS["claude-code"].manifest).includes("telemetry.mjs"), false); +}); + +test("PreToolUse and PostToolUse nest MCP, web, and per-CLI Bash telemetry groups", () => { + const enabled = JSON.parse(readFileSync( + path.join(ROOT, "test/fixtures/telemetry/claude-code/hooks.enabled.json"), + "utf8", + )).hooks; + const [pre] = enabled.PreToolUse; + assert.equal(pre.matcher, `^(?:${ARCADE_TOOL_PREFIX}|${CLAUDE_AI_ARCADE_TOOL_PREFIX})`); + for (const event of ["PostToolUse", "PostToolUseFailure"]) { + const groups = enabled[event]; + assert.equal(groups.length, 3); + assert.equal(groups[0].matcher, "mcp__.*"); + assert.equal(groups[1].matcher, "WebFetch|WebSearch"); + assert.equal(groups[2].matcher, "Bash"); + assert.equal(groups[2].hooks.length, BASH_CLIS.length); + for (const hook of groups[2].hooks) { + assert.ok(hook.if); + assert.ok(hook.command.endsWith(`--cli ${hook.if.match(/^Bash\((\w+) \*\)$/)[1]}`)); + } + } +}); + +test("every enabled telemetry command exits quietly with telemetry off", async () => { + const enabled = JSON.parse(readFileSync( + path.join(ROOT, "test/fixtures/telemetry/claude-code/hooks.enabled.json"), + "utf8", + )); + const commands = Object.values(enabled.hooks) + .flatMap((groups) => groups.flatMap((group) => group.hooks)) + .map((hook) => hook.command) + .filter((command) => command.includes("/hooks/telemetry.mjs")); + const rows = (await telemetryHookRows()).filter((row) => row.hosts.includes("claude-code")); + assert.equal(commands.length, rows.length); + const server = await startServer(); + try { + const dataDir = tempDataDir(); + const stdin = JSON.stringify(hookInput({ hook_event_name: "SessionStart" })); + const env = telemetryEnv(adapter, dataDir, server.url); + for (const row of rows) { + const result = runTelemetryScript(stdin, env, "claude-code", row.extraArgs ?? []); + assert.equal(result.status, 0, `${row.event} ${row.matcher ?? ""}: ${result.stderr}`); + assert.equal(result.stdout, "", row.event); + assert.equal(result.stderr, "", row.event); + } + await sleep(300); + assert.deepEqual(server.requests, []); + assert.deepEqual(readdirSync(dataDir), []); + } finally { + await server.close(); + } +}); + +test("invalid telemetry inputs are ignored without affecting hooks", async () => { + const dataDir = tempDataDir(); + const env = telemetryEnv(adapter, dataDir, "http://127.0.0.1:9"); + const cases = [ + "not-json", + JSON.stringify({ hook_event_name: "UserPromptSubmit" }), + JSON.stringify(hookInput({ hook_event_name: "UnknownHook" })), + JSON.stringify(hookInput({ hook_event_name: "PreToolUse", tool_name: "not-mcp" })), + ]; + for (const stdin of cases) { + const result = runTelemetryScript(stdin, env); + assert.equal(result.status, 0, stdin); + assert.equal(result.stdout, ""); + } + const routing = spawnSync(process.execPath, [path.join(ROOT, "hooks", "session-start.mjs"), "--host", "claude-code"], { + input: "not-json", + encoding: "utf8", + }); + assert.equal(routing.status, 0); +}); + +test("buildEvent uses adapter tool properties for malformed tool names", () => { + const options = { adapter, os: "darwin", arcadeUsedBefore: false, appWork: true }; + assert.equal(buildEvent(hookInput({ hook_event_name: "PreToolUse", tool_name: 42 }), options), null); + assert.equal(isArcadeCall(buildEvent(hookInput({ + hook_event_name: "PostToolUse", + tool_name: `${ARCADE_TOOL_PREFIX}Gmail_ListEmails`, + }), options)), true); +}); From 45edb2c6366f9f6fa4abd3e9cba2fedb8a7886d6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 12 Oct 2026 19:11:03 +0000 Subject: [PATCH 2/6] test(telemetry-claude): make the hard-OFF boundary test detect an open gate The 19 enabled-fixture commands were fed a SessionStart input, which maps to no event, so the test passed even with TELEMETRY_ENABLED flipped true. Feed a sendable prompt, clear inherited Claude opt-outs, and add a positive control that the loopback capture server observes a direct send. Co-authored-by: Teal Larson --- test/telemetry-claude.test.mjs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/test/telemetry-claude.test.mjs b/test/telemetry-claude.test.mjs index 3247549..fb428ba 100644 --- a/test/telemetry-claude.test.mjs +++ b/test/telemetry-claude.test.mjs @@ -3,6 +3,7 @@ import { spawnSync } from "node:child_process"; import { readFileSync, readdirSync, statSync } from "node:fs"; import path from "node:path"; import { test } from "node:test"; +import { EVENT_ENV } from "../hooks/telemetry-config.mjs"; import { BASH_CLIS } from "../hooks/telemetry-contract.mjs"; import { HOOKS, HOSTS, telemetryHookRows } from "../hooks/hook-hosts.mjs"; import { SCOPE_TTL_MS } from "../hooks/hook-scope.mjs"; @@ -346,10 +347,12 @@ test("every enabled telemetry command exits quietly with telemetry off", async ( const server = await startServer(); try { const dataDir = tempDataDir(); - const stdin = JSON.stringify(hookInput({ hook_event_name: "SessionStart" })); - const env = telemetryEnv(adapter, dataDir, server.url); + // A sendable prompt with no inherited opt-out, so an open gate would post. + const input = hookInput({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }); + const clearedOptOuts = Object.fromEntries(adapter.optOutSwitches.map(({ name }) => [name, ""])); + const env = telemetryEnv(adapter, dataDir, server.url, clearedOptOuts); for (const row of rows) { - const result = runTelemetryScript(stdin, env, "claude-code", row.extraArgs ?? []); + const result = runTelemetryScript(JSON.stringify(input), env, "claude-code", row.extraArgs ?? []); assert.equal(result.status, 0, `${row.event} ${row.matcher ?? ""}: ${result.stderr}`); assert.equal(result.stdout, "", row.event); assert.equal(result.stderr, "", row.event); @@ -357,6 +360,15 @@ test("every enabled telemetry command exits quietly with telemetry off", async ( await sleep(300); assert.deepEqual(server.requests, []); assert.deepEqual(readdirSync(dataDir), []); + + const event = buildEvent(input, { adapter, os: "linux", arcadeUsedBefore: false, appWork: true }); + assert.ok(event); + const sent = spawnSync(process.execPath, [path.join(ROOT, "hooks", "telemetry-send.mjs")], { + env: { ...process.env, ...env, [EVENT_ENV]: JSON.stringify(event) }, + }); + assert.equal(sent.status, 0); + await waitForRequests(server.requests, 1); + assert.equal(server.requests.length, 1); } finally { await server.close(); } From c188452b403621720617ecd0edb384c5a372522d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 12 Oct 2026 22:24:01 +0000 Subject: [PATCH 3/6] Share other-server tool classification and telemetry test scaffolding Move the gateway-on-another-server and service inference fallback into otherServerToolProperties so adapters keep only their own name parsing. Export hookEnv, spawnExitCode and assertSilentThenServerReachable from the telemetry test helpers, and use captureTelemetry instead of hand-rolled runTelemetry calls in the Claude tests. Co-authored-by: Teal Larson --- hooks/telemetry-adapters/claude-code.mjs | 15 +--- hooks/telemetry-events.mjs | 19 +++++ test/telemetry-claude.test.mjs | 101 +++++++++-------------- test/telemetry-helpers.mjs | 68 +++++++++++++-- 4 files changed, 118 insertions(+), 85 deletions(-) diff --git a/hooks/telemetry-adapters/claude-code.mjs b/hooks/telemetry-adapters/claude-code.mjs index abff60c..e9eabd6 100644 --- a/hooks/telemetry-adapters/claude-code.mjs +++ b/hooks/telemetry-adapters/claude-code.mjs @@ -3,12 +3,11 @@ import { readFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { serviceForToolkit, serviceForToolName } from "../telemetry-classify.mjs"; -import { BASH_CLIS, GATEWAY_TOOLS } from "../telemetry-contract.mjs"; +import { BASH_CLIS } from "../telemetry-contract.mjs"; import { arcadeToolProperties, builtinToolProperties as sharedBuiltinToolProperties, - otherServerProperties, + otherServerToolProperties, } from "../telemetry-events.mjs"; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); @@ -36,15 +35,7 @@ const claudeToolProperties = (toolName, toolInput) => { if (toolName.startsWith(CLAUDE_AI_ARCADE_TOOL_PREFIX)) { return arcadeToolProperties("other_arcade", tool, toolInput); } - if (/** @type {readonly string[]} */ (GATEWAY_TOOLS).includes(tool)) { - return arcadeToolProperties("other_arcade", tool, toolInput); - } - // Some connectors name the service in the server, not the tool, e.g. - // mcp__claude_ai_Gmail__search_threads. Only the category is sent. - const serverParts = toolName.slice("mcp__".length, splitAt).split(/[^A-Za-z0-9]+/); - const service = - serviceForToolName(tool) ?? serverParts.map(serviceForToolkit).find(Boolean); - return otherServerProperties(service ?? undefined); + return otherServerToolProperties(toolName.slice("mcp__".length, splitAt), tool, toolInput); }; /** diff --git a/hooks/telemetry-events.mjs b/hooks/telemetry-events.mjs index 411ff14..1890d81 100644 --- a/hooks/telemetry-events.mjs +++ b/hooks/telemetry-events.mjs @@ -9,6 +9,7 @@ import { isTaskNotification, shouldRemind } from "./prompt-filters.mjs"; import { isOperatorAgentType } from "./routing-guidance.mjs"; import { classifyPrompt, + serviceForToolkit, serviceForToolName, } from "./telemetry-classify.mjs"; import { commandUsesCli } from "./telemetry-commands.mjs"; @@ -75,6 +76,24 @@ export const arcadeToolProperties = (server, tool, toolInput) => { */ export const otherServerProperties = (service) => withService({ server: "other" }, service); +/** + * Properties for a tool on an MCP server other than this plugin's Arcade + * server. A gateway tool name means another Arcade server. Some connectors + * name the service in the server, not the tool (e.g. claude_ai_Gmail); only + * the category is sent. + * @param {string} server + * @param {string} tool + * @param {Record | undefined} toolInput + */ +export const otherServerToolProperties = (server, tool, toolInput) => { + if (/** @type {readonly string[]} */ (GATEWAY_TOOLS).includes(tool)) { + return arcadeToolProperties("other_arcade", tool, toolInput); + } + const service = + serviceForToolName(tool) ?? server.split(/[^A-Za-z0-9]+/).map(serviceForToolkit).find(Boolean); + return otherServerProperties(service); +}; + /** * Properties for a WebFetch, WebSearch, or listed-CLI Bash call, or null to * send nothing. Only the tool name and the CLI name are read into the event; diff --git a/test/telemetry-claude.test.mjs b/test/telemetry-claude.test.mjs index fb428ba..36e3ec5 100644 --- a/test/telemetry-claude.test.mjs +++ b/test/telemetry-claude.test.mjs @@ -3,12 +3,10 @@ import { spawnSync } from "node:child_process"; import { readFileSync, readdirSync, statSync } from "node:fs"; import path from "node:path"; import { test } from "node:test"; -import { EVENT_ENV } from "../hooks/telemetry-config.mjs"; import { BASH_CLIS } from "../hooks/telemetry-contract.mjs"; import { HOOKS, HOSTS, telemetryHookRows } from "../hooks/hook-hosts.mjs"; import { SCOPE_TTL_MS } from "../hooks/hook-scope.mjs"; import { buildEvent, isArcadeCall } from "../hooks/telemetry-events.mjs"; -import { runTelemetry } from "../hooks/telemetry-run.mjs"; import adapter, { ARCADE_TOOL_PREFIX, CLAUDE_AI_ARCADE_TOOL_PREFIX, @@ -19,15 +17,15 @@ import { eventCases } from "./fixtures/telemetry/claude-code/event-cases.mjs"; import { assertMatchesContract, assertNoLeak, + assertSilentThenServerReachable, captureTelemetry, expectedEvent, + hookEnv, hookInput, runTelemetryScript, - sleep, startServer, telemetryEnv, tempDataDir, - waitForRequests, } from "./telemetry-helpers.mjs"; const OPERATOR = "arcade:arcade-operator"; @@ -223,76 +221,62 @@ test("direct Arcade calls are observed without prompt scope; other and built-in test("arcade-used is set only by a successful Arcade gateway call", async () => { const dataDir = tempDataDir(); - const env = telemetryEnv(adapter, dataDir, "http://127.0.0.1:9"); - const send = []; - await runTelemetry({ + const other = await captureTelemetry({ adapter, - env, - enabled: true, + dataDir, input: hookInput({ hook_event_name: "PostToolUse", tool_name: "mcp__granola__Granola_ListMeetings" }), - send: (e) => send.push(e), }); assert.deepEqual(readdirSync(dataDir), []); - await runTelemetry({ + const attempt = await captureTelemetry({ adapter, - env, - enabled: true, + dataDir, input: hookInput({ hook_event_name: "PreToolUse", tool_name: `${CLAUDE_AI_ARCADE_TOOL_PREFIX}GoogleCalendar_ListEvents`, tool_input: { query: "SECRET private meeting" }, }), - send: (e) => send.push(e), }); assert.deepEqual(readdirSync(dataDir), []); - await runTelemetry({ + const arcade = await captureTelemetry({ adapter, - env, - enabled: true, + dataDir, input: hookInput({ hook_event_name: "PostToolUse", tool_name: `${ARCADE_TOOL_PREFIX}Gmail_ListEmails` }), - send: (e) => send.push(e), }); assert.equal(readFileSync(path.join(dataDir, "arcade-used"), "utf8"), "true"); if (process.platform !== "win32") { assert.equal(statSync(path.join(dataDir, "arcade-used")).mode & 0o777, 0o600); } - const attempt = send.find((e) => e.event === "Plugin tool attempted"); - assert.ok(attempt); - assertNoLeak(JSON.stringify(attempt), /SECRET|private meeting/); - const flags = send.map((e) => e.properties.arcade_used_before).sort(); - assert.deepEqual(flags, [false, false]); + assert.equal(other.sent.length, 0); + assert.equal(attempt.sent.length, 1); + assert.equal(attempt.sent[0].event, "Plugin tool attempted"); + assertNoLeak(JSON.stringify(attempt.sent[0]), /SECRET|private meeting/); + assert.equal(arcade.sent.length, 1); + assert.deepEqual([attempt.sent[0], arcade.sent[0]].map((e) => e.properties.arcade_used_before), [false, false]); }); test("Claude opt-out switches send nothing and write no data dir files", async () => { - const server = await startServer(); - try { - const sent = []; - const base = hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar" }); - const sessionStart = hookInput({ hook_event_name: "SessionStart", source: "startup" }); - const cases = [ - ["DISABLE_TELEMETRY=1", { DISABLE_TELEMETRY: "1" }], - ["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1", { CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1" }], - ["DISABLE_TELEMETRY=0", { DISABLE_TELEMETRY: "0" }], - ["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=false", { CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "false" }], - ["ARCADE_PLUGIN_TELEMETRY=0", { ARCADE_PLUGIN_TELEMETRY: "0" }], - ["DO_NOT_TRACK=1", { DO_NOT_TRACK: "1" }], - ["missing CLAUDE_PLUGIN_DATA", { CLAUDE_PLUGIN_DATA: "" }], - ["relative CLAUDE_PLUGIN_DATA", { CLAUDE_PLUGIN_DATA: "relative/data" }], - ]; - for (const [label, extra] of cases) { - const dataDir = tempDataDir(); - const env = telemetryEnv(adapter, dataDir, server.url, extra); - for (const input of [sessionStart, base]) { - await runTelemetry({ adapter, input, env, enabled: true, send: (e) => sent.push(e) }); - } - assert.deepEqual(readdirSync(dataDir), [], label); + const base = hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar" }); + const sessionStart = hookInput({ hook_event_name: "SessionStart", source: "startup" }); + const cases = [ + ["DISABLE_TELEMETRY=1", { DISABLE_TELEMETRY: "1" }], + ["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1", { CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1" }], + ["DISABLE_TELEMETRY=0", { DISABLE_TELEMETRY: "0" }], + ["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=false", { CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "false" }], + ["ARCADE_PLUGIN_TELEMETRY=0", { ARCADE_PLUGIN_TELEMETRY: "0" }], + ["DO_NOT_TRACK=1", { DO_NOT_TRACK: "1" }], + ["missing CLAUDE_PLUGIN_DATA", { CLAUDE_PLUGIN_DATA: "" }], + ["relative CLAUDE_PLUGIN_DATA", { CLAUDE_PLUGIN_DATA: "relative/data" }], + ]; + for (const [label, env] of cases) { + const dataDir = tempDataDir(); + for (const input of [sessionStart, base]) { + const { sent } = await captureTelemetry({ adapter, input, dataDir, env }); + assert.deepEqual(sent, [], label); } - await sleep(200); - assert.deepEqual(server.requests, []); - assert.deepEqual(sent, []); - } finally { - await server.close(); + assert.deepEqual(readdirSync(dataDir), [], label); } + const control = await captureTelemetry({ adapter, input: base }); + assert.equal(control.sent.length, 1, "the same prompt sends without an opt-out"); }); test("enabled Claude manifest wiring matches hooks.enabled.json", async () => { @@ -347,28 +331,17 @@ test("every enabled telemetry command exits quietly with telemetry off", async ( const server = await startServer(); try { const dataDir = tempDataDir(); - // A sendable prompt with no inherited opt-out, so an open gate would post. + // A sendable prompt; runTelemetryScript drops inherited opt-outs, so an open gate would post. const input = hookInput({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }); - const clearedOptOuts = Object.fromEntries(adapter.optOutSwitches.map(({ name }) => [name, ""])); - const env = telemetryEnv(adapter, dataDir, server.url, clearedOptOuts); + const env = telemetryEnv(adapter, dataDir, server.url); for (const row of rows) { const result = runTelemetryScript(JSON.stringify(input), env, "claude-code", row.extraArgs ?? []); assert.equal(result.status, 0, `${row.event} ${row.matcher ?? ""}: ${result.stderr}`); assert.equal(result.stdout, "", row.event); assert.equal(result.stderr, "", row.event); } - await sleep(300); - assert.deepEqual(server.requests, []); + await assertSilentThenServerReachable({ server, adapter, input, env: hookEnv(env) }); assert.deepEqual(readdirSync(dataDir), []); - - const event = buildEvent(input, { adapter, os: "linux", arcadeUsedBefore: false, appWork: true }); - assert.ok(event); - const sent = spawnSync(process.execPath, [path.join(ROOT, "hooks", "telemetry-send.mjs")], { - env: { ...process.env, ...env, [EVENT_ENV]: JSON.stringify(event) }, - }); - assert.equal(sent.status, 0); - await waitForRequests(server.requests, 1); - assert.equal(server.requests.length, 1); } finally { await server.close(); } diff --git a/test/telemetry-helpers.mjs b/test/telemetry-helpers.mjs index fca1824..9c70a33 100644 --- a/test/telemetry-helpers.mjs +++ b/test/telemetry-helpers.mjs @@ -4,7 +4,7 @@ */ import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdtempSync, rmSync } from "node:fs"; import http from "node:http"; @@ -13,20 +13,36 @@ import path from "node:path"; import { after } from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; import { eventSchema } from "../hooks/telemetry-contract.mjs"; -import { PLUGIN_VERSION } from "../hooks/telemetry-config.mjs"; +import { EVENT_ENV, PLUGIN_VERSION } from "../hooks/telemetry-config.mjs"; +import { buildEvent } from "../hooks/telemetry-events.mjs"; import { runTelemetry } from "../hooks/telemetry-run.mjs"; import { ROOT } from "./helpers.mjs"; +// An inherited opt-out would make a hook skip even with the hard-OFF gate open, +// so tests that expect silence would pass for the wrong reason. const TELEMETRY_ENV_BLOCKLIST = [ "ARCADE_PLUGIN_TELEMETRY", + "ARCADE_PLUGIN_TELEMETRY_HOST", "DO_NOT_TRACK", "DISABLE_TELEMETRY", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "COPILOT_OFFLINE", "CLAUDE_PLUGIN_DATA", "COPILOT_PLUGIN_DATA", + "PLUGIN_ROOT", ]; +/** + * process.env without inherited opt-outs, telemetry destination, plugin root + * or data dirs, plus `overrides`. + * @param {NodeJS.ProcessEnv} [overrides] + */ +export const hookEnv = (overrides = {}) => { + const env = { ...process.env }; + for (const key of TELEMETRY_ENV_BLOCKLIST) delete env[key]; + return { ...env, ...overrides }; +}; + /** @type {string[]} */ const tempDataDirs = []; let tempDataCleanupRegistered = false; @@ -50,12 +66,8 @@ export const tempDataDir = () => { * @param {string | null} dataDir * @param {NodeJS.ProcessEnv} overrides */ -const captureEnv = (adapter, dataDir, overrides) => { - const env = { ...process.env }; - for (const key of TELEMETRY_ENV_BLOCKLIST) delete env[key]; - if (dataDir !== null) env[adapter.dataVariable] = dataDir; - return { ...env, ...overrides }; -}; +const captureEnv = (adapter, dataDir, overrides) => + hookEnv({ ...(dataDir === null ? {} : { [adapter.dataVariable]: dataDir }), ...overrides }); /** * @param {object} options @@ -182,5 +194,43 @@ export const runTelemetryScript = (stdin, env, hostName = "claude-code", extraAr spawnSync(process.execPath, [path.join(ROOT, "hooks", "telemetry.mjs"), "--host", hostName, ...extraArgs], { input: stdin, encoding: "utf8", - env: { ...process.env, ...env }, + env: hookEnv(env), }); + +export const SEND_SCRIPT = path.join(ROOT, "hooks", "telemetry-send.mjs"); + +/** + * Async spawn keeps this process's event loop free so an in-process server can answer. + * @param {string} command + * @param {string[]} args + * @param {import("node:child_process").SpawnOptions} options + * @returns {Promise} + */ +export const spawnExitCode = (command, args, options) => + new Promise((resolve) => spawn(command, args, { ...options, stdio: "ignore" }).on("exit", resolve)); + +/** @param {NodeJS.ProcessEnv} env */ +export const sendWithNode = (env) => spawnExitCode(process.execPath, [SEND_SCRIPT], { env }); + +/** + * Asserts nothing reached `server`, then sends the event `input` would build + * through `send` with `env` and asserts it arrives. That proves the hooks' + * environment resolves to the server, so the empty request list could not + * come from sending somewhere else. + * @param {object} options + * @param {{ requests: { body: string }[] }} options.server + * @param {import("../hooks/telemetry-adapter.mjs").TelemetryAdapter} options.adapter + * @param {Record} options.input A hook input classified as app work. + * @param {NodeJS.ProcessEnv} options.env + * @param {(env: NodeJS.ProcessEnv) => Promise} [options.send] + */ +export const assertSilentThenServerReachable = async ({ server, adapter, input, env, send = sendWithNode }) => { + await sleep(200); + assert.deepEqual(server.requests, []); + const event = buildEvent(input, { adapter, os: "linux", arcadeUsedBefore: false, appWork: true }); + assert.ok(event); + assert.equal(await send({ ...env, [EVENT_ENV]: JSON.stringify(event) }), 0); + await waitForRequests(server.requests, 1); + assert.equal(server.requests.length, 1); + assert.equal(JSON.parse(server.requests[0].body).event, event.event); +}; From 705fa8ab262bfcce8e9998a4bca95d17c1967ab1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 12 Oct 2026 22:35:07 +0000 Subject: [PATCH 4/6] Clean up every telemetry test temp dir and wait 300ms before silence checks Register tempDataDir cleanup at module load; an after() hook created inside the first test only ran after that test. Co-authored-by: Teal Larson --- test/telemetry-helpers.mjs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/test/telemetry-helpers.mjs b/test/telemetry-helpers.mjs index 9c70a33..eab341c 100644 --- a/test/telemetry-helpers.mjs +++ b/test/telemetry-helpers.mjs @@ -18,8 +18,8 @@ import { buildEvent } from "../hooks/telemetry-events.mjs"; import { runTelemetry } from "../hooks/telemetry-run.mjs"; import { ROOT } from "./helpers.mjs"; -// An inherited opt-out would make a hook skip even with the hard-OFF gate open, -// so tests that expect silence would pass for the wrong reason. +// An inherited opt-out, destination, plugin root or data dir would make a +// silence test pass for the wrong reason or send somewhere other than its server. const TELEMETRY_ENV_BLOCKLIST = [ "ARCADE_PLUGIN_TELEMETRY", "ARCADE_PLUGIN_TELEMETRY_HOST", @@ -45,17 +45,14 @@ export const hookEnv = (overrides = {}) => { /** @type {string[]} */ const tempDataDirs = []; -let tempDataCleanupRegistered = false; +// Registered at load so it runs after the whole file; `after` inside a test runs after that test only. +after(() => { + for (const dir of tempDataDirs) rmSync(dir, { recursive: true, force: true }); + tempDataDirs.length = 0; +}); /** Absolute path of a fresh temp directory; removed after the test run finishes. */ export const tempDataDir = () => { - if (!tempDataCleanupRegistered) { - after(() => { - for (const dir of tempDataDirs) rmSync(dir, { recursive: true, force: true }); - tempDataDirs.length = 0; - }); - tempDataCleanupRegistered = true; - } const dir = mkdtempSync(path.join(os.tmpdir(), "arcade-telemetry-")); tempDataDirs.push(dir); return dir; @@ -225,7 +222,7 @@ export const sendWithNode = (env) => spawnExitCode(process.execPath, [SEND_SCRIP * @param {(env: NodeJS.ProcessEnv) => Promise} [options.send] */ export const assertSilentThenServerReachable = async ({ server, adapter, input, env, send = sendWithNode }) => { - await sleep(200); + await sleep(300); assert.deepEqual(server.requests, []); const event = buildEvent(input, { adapter, os: "linux", arcadeUsedBefore: false, appWork: true }); assert.ok(event); From 0faab06c25df6b5ee722b436d206fe7119da338a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 12 Oct 2026 22:44:59 +0000 Subject: [PATCH 5/6] Test that tempDataDir cleans up directories from every test Co-authored-by: Teal Larson --- test/telemetry-helpers.test.mjs | 36 +++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 test/telemetry-helpers.test.mjs diff --git a/test/telemetry-helpers.test.mjs b/test/telemetry-helpers.test.mjs new file mode 100644 index 0000000..8a729fb --- /dev/null +++ b/test/telemetry-helpers.test.mjs @@ -0,0 +1,36 @@ +// @ts-check + +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { test } from "node:test"; +import { pathToFileURL } from "node:url"; +import { ROOT } from "./helpers.mjs"; +import { tempDataDir } from "./telemetry-helpers.mjs"; + +test("tempDataDir keeps directories from every test until the file ends, then removes them all", () => { + const helpers = pathToFileURL(path.join(ROOT, "test", "telemetry-helpers.mjs")).href; + const file = path.join(tempDataDir(), "uses-temp-dirs.test.mjs"); + writeFileSync(file, ` +import { existsSync } from "node:fs"; +import { test } from "node:test"; +import { tempDataDir } from ${JSON.stringify(helpers)}; +const made = []; +for (const name of ["first", "second", "third"]) { + test(name, () => { + const dir = tempDataDir(); + made.push(dir); + console.log("TEMP_DIR " + dir); + }); +} +test("directories from earlier tests still exist", () => { + for (const dir of made) if (!existsSync(dir)) throw new Error("removed early: " + dir); +}); +`); + const result = spawnSync(process.execPath, [file], { encoding: "utf8" }); + assert.equal(result.status, 0, result.stdout + result.stderr); + const dirs = [...result.stdout.matchAll(/^TEMP_DIR (.+)$/gm)].map((match) => match[1]); + assert.equal(dirs.length, 3, result.stdout); + for (const dir of dirs) assert.equal(existsSync(dir), false, `left behind: ${dir}`); +}); From a3fd91fd3f154031a8c91922bd2455bd830ea68a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 12 Oct 2026 22:49:02 +0000 Subject: [PATCH 6/6] Print readable child output when the tempDataDir cleanup test fails Co-authored-by: Teal Larson --- test/telemetry-helpers.test.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/telemetry-helpers.test.mjs b/test/telemetry-helpers.test.mjs index 8a729fb..c9b06b8 100644 --- a/test/telemetry-helpers.test.mjs +++ b/test/telemetry-helpers.test.mjs @@ -28,7 +28,10 @@ test("directories from earlier tests still exist", () => { for (const dir of made) if (!existsSync(dir)) throw new Error("removed early: " + dir); }); `); - const result = spawnSync(process.execPath, [file], { encoding: "utf8" }); + // Without NODE_TEST_CONTEXT the child prints readable TAP instead of the runner's binary protocol. + const env = { ...process.env }; + delete env.NODE_TEST_CONTEXT; + const result = spawnSync(process.execPath, [file], { encoding: "utf8", env }); assert.equal(result.status, 0, result.stdout + result.stderr); const dirs = [...result.stdout.matchAll(/^TEMP_DIR (.+)$/gm)].map((match) => match[1]); assert.equal(dirs.length, 3, result.stdout);