From 1da44229e9bd68352b22c9418eef9cc4ec7b229a Mon Sep 17 00:00:00 2001 From: Facundo Santiago <32112894+santiagxf@users.noreply.github.com> Date: Mon, 16 Nov 2026 07:22:31 -0500 Subject: [PATCH] Stop the Gemini adapters drifting from the commands they mirror The feedback command's description is written twice, once in commands/feedback.md for the Markdown hosts and once in commands/aigateway/feedback.toml for Gemini, with nothing tying the copies together. So they drifted. The skill moved to returning an anonymized summary, the Markdown followed, and the Gemini adapter went on telling users it packages a local ZIP for GitHub issues -- a flow the current skill explicitly refuses. The stale description was the visible half. The adapter's prompt still drove the whole v1 flow: consent for a session export, a ZIP, GitHub publication. The skill it points at now rejects full transcripts and says sharing decisions belong to the user, so the adapter was instructing Gemini to do what the skill forbids. Rewrite it to mirror the Markdown command. Check the pair from now on: each .toml description must equal the description of the command it mirrors. The other five already matched, and every skill path the adapters reference resolves. Also move the feedback skill's tests out of the skill directory. Whatever sits in skills/ is downloaded by everyone who installs the plugin, and evals.json and test_feedback.py are development files. They move to tests/feedback/, and the check fails if anything like that reappears. Both checks were run against the previous tree and report exactly these three problems. Co-Authored-By: Claude Opus 5 --- .github/scripts/check-plugin.py | 67 ++++++++++++++++++- commands/aigateway/feedback.toml | 18 +++-- .../tests => tests/feedback}/evals.json | 0 .../tests => tests/feedback}/test_feedback.py | 3 +- 4 files changed, 75 insertions(+), 13 deletions(-) rename {skills/feedback-ai-gateway/tests => tests/feedback}/evals.json (100%) rename {skills/feedback-ai-gateway/tests => tests/feedback}/test_feedback.py (98%) diff --git a/.github/scripts/check-plugin.py b/.github/scripts/check-plugin.py index bf88f03f..83072947 100644 --- a/.github/scripts/check-plugin.py +++ b/.github/scripts/check-plugin.py @@ -9,12 +9,18 @@ wants a string. Claude Code accepted all of it, which is why the break reached a release unnoticed. -So this script checks the two halves of what a host needs: +So this script checks what a host needs: * every command and skill the manifests promise parses, and parses to the types a loader expects; * every manifest promises the same thing, because the plugin is published through - five of them and a version bump that misses one ships a lie. + five of them and a version bump that misses one ships a lie; +* each Gemini ``.toml`` adapter describes its command the same way the canonical + Markdown does. The same text is written twice, once per host, so it drifts: the + feedback adapter went on advertising a ZIP long after the skill stopped + producing one; +* no development test or fixture sits inside a skill directory, since whatever is + in there is downloaded by everyone who installs the plugin. The guard against parser differences is quoting, not parseability. A quoted scalar is a string under every YAML implementation regardless of what it @@ -43,6 +49,11 @@ except ImportError: # optional, see module docstring yaml = None +try: + import tomllib # Python 3.11+; the adapter check is skipped without it +except ImportError: + tomllib = None + REPO = Path(__file__).resolve().parents[2] # Every manifest that publishes this plugin. Each entry is the file and the @@ -164,6 +175,55 @@ def check_markdown(rel: str, kind: str) -> None: fail(rel, f"{key} parses as {got}, not str: {data[key]!r}") +def check_adapters(commands: list[str]) -> None: + """Each Gemini .toml must describe its command exactly as the Markdown does. + + The description is authored twice, once per host, with nothing tying the + copies together. That is how the feedback adapter kept advertising a ZIP + after the skill had moved to returning an anonymized summary. + """ + if tomllib is None: # Python < 3.11, see module docstring + return + for entry in commands: + rel = entry.lstrip("./") + name = Path(rel).stem + adapter = REPO / "commands" / "aigateway" / f"{name}.toml" + canonical = REPO / rel + if not adapter.is_file() or not canonical.is_file(): + continue + try: + want = tomllib.loads(adapter.read_text(encoding="utf-8")).get("description") + except tomllib.TOMLDecodeError as exc: + fail(f"commands/aigateway/{name}.toml", f"invalid TOML: {exc}") + continue + + lines = split_frontmatter(canonical.read_text(encoding="utf-8")) or [] + got = None + for line in lines: + match = re.match(r"^description:\s*(.*)$", line) + if match: + got = match.group(1).strip().strip("\"'") + if got is not None and want != got: + fail( + f"commands/aigateway/{name}.toml", + "description does not match " + f"{rel}\n toml: {want}\n md: {got}", + ) + + +def check_no_bundled_tests() -> None: + """Skill directories ship to users; development files do not belong in them.""" + stray = sorted( + str(path.relative_to(REPO)) + for pattern in ("*/tests/*", "*/evals/*", "*/test_*.py") + for path in (REPO / "skills").glob(pattern) + # Local build artifacts are gitignored; do not fail a developer's run. + if "__pycache__" not in path.parts + ) + for path in stray: + fail(path, "development file inside a shipped skill; move it under tests/") + + def main() -> int: manifests: dict[str, dict] = {} for rel in MANIFESTS: @@ -210,6 +270,9 @@ def main() -> int: for entry in base.get("skills", []): check_markdown(f"{entry.strip('./')}/SKILL.md", "skill") + check_adapters(base.get("commands", [])) + check_no_bundled_tests() + strict = "with" if yaml else "without" checked = len(base.get("commands", [])) + len(base.get("skills", [])) if failures: diff --git a/commands/aigateway/feedback.toml b/commands/aigateway/feedback.toml index bb05dc19..2eac0fc8 100644 --- a/commands/aigateway/feedback.toml +++ b/commands/aigateway/feedback.toml @@ -1,16 +1,14 @@ -description = "Package a selected session into a local feedback ZIP, with mandatory anonymization and review for GitHub issues." +description = "Collect session feedback on friction points and completed work for Azure AI Gateway insights. Process and anonymize a session summary, returning a clean file ready for sharing." prompt = """ Follow `skills/feedback-ai-gateway/SKILL.md` and its bundled local Python helper. -Obtain consent for one selected session export and any optional diagnostic files. -Use the host's supported export mechanism for Copilot, Claude, Gemini, Cursor, -or another agent; report missing telemetry and never read unrelated sessions. -Return a local ZIP without uploading. For private handoff, let the user choose -whether to anonymize. For GitHub issue sharing, always semantically anonymize all -selected content, run helper scrubbing, review the final ZIP and issue title/body, -and obtain explicit publication approval. Never post raw/private bundles. -The user can attach the reviewed ZIP through GitHub's browser upload; -do not invent a ZIP attachment option for gh issue create. +Resolve the arguments into the selected summary file and output directory. +Obtain consent before reading the summary file. Verify that the file is a +summary (friction, tasks, outcomes) and not a full transcript. Never accept +full session histories or conversation exports. +Return an anonymized summary file to the user; never upload automatically. +Do not suggest or facilitate sharing to GitHub, email, or any other service. +All sharing decisions belong to the user. Feedback request: {{args}} """ diff --git a/skills/feedback-ai-gateway/tests/evals.json b/tests/feedback/evals.json similarity index 100% rename from skills/feedback-ai-gateway/tests/evals.json rename to tests/feedback/evals.json diff --git a/skills/feedback-ai-gateway/tests/test_feedback.py b/tests/feedback/test_feedback.py similarity index 98% rename from skills/feedback-ai-gateway/tests/test_feedback.py rename to tests/feedback/test_feedback.py index 69a8ed80..57c648d5 100644 --- a/skills/feedback-ai-gateway/tests/test_feedback.py +++ b/tests/feedback/test_feedback.py @@ -9,7 +9,8 @@ SPEC = importlib.util.spec_from_file_location( - "feedback", Path(__file__).parents[1] / "scripts" / "feedback.py" + "feedback", + Path(__file__).parents[2] / "skills" / "feedback-ai-gateway" / "scripts" / "feedback.py", ) feedback = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(feedback)