From b6562965471f460650aef923a7cc513a721b60f5 Mon Sep 17 00:00:00 2001 From: justrach <54503978+justrach@users.noreply.github.com> Date: Sat, 21 Nov 2026 02:24:49 +0800 Subject: [PATCH] Allow explicit no-edit instructions in run_task The workspace guard matched edit words without checking direct negation. Ignore only the negated occurrence so a separate edit request still triggers the guard. Server permissions are unchanged. Co-Authored-By: Codegraff --- CHANGELOG.md | 4 ++ evals/harness_behavior.jsonl | 1 + scripts/eval/tier1-manifest.json | 2 +- src/cli_changelog_source.zig | 3 ++ src/mcp_run_task_guard.zig | 76 +++++++++++++++++++++++++------- 5 files changed, 69 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90d2c7ee..b2b95c6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ The release workflow uses a tag's section here as its release notes (a hand-written `docs/releases/.md` wins if present), so keeping this file current is part of cutting a release. +## v0.302.74.78 + +- Allow read-only `run_task` prompts that say `No edits.` without weakening the guard for actual edit requests (#1914). + ## v0.302.74.77 - Prefer `rlm` for independent file reads, JSON row counts, and field extraction. Direct tools, shell fallback, and `--no-rlm` remain available (#1915). diff --git a/evals/harness_behavior.jsonl b/evals/harness_behavior.jsonl index e767aa78..bc77a97d 100644 --- a/evals/harness_behavior.jsonl +++ b/evals/harness_behavior.jsonl @@ -120,6 +120,7 @@ {"id": "rlm-indexed-print-refuses-and-recovers", "why": "Unsupported Python-style result indexing must report an error without pretending to execute the nested host or executing later side effects; the supported text binding form recovers.", "args": ["--lean"], "prompt": "Run the scripted checks and report only observed results.", "script": [{"tool": "rlm", "arguments": {"code": "print(bash(\"printf ran > nested-marker\")[\"stdout\"])\nx = write_file(path=\"tail-marker\", content=\"ran\")"}}, {"tool": "rlm", "arguments": {"code": "r = bash(\"printf VERIFIED\")\nprint(r)"}}, {"text": "Verified using the supported text result."}], "verify_command": ["python3", "-c", "from pathlib import Path; assert not Path(\"nested-marker\").exists(); assert not Path(\"tail-marker\").exists()"], "assert": [{"event": {"type": "tool_call_finished", "name": "rlm", "is_error": true}}, {"events_at_most": {"match": {"type": "tool_call_finished", "is_error": true}, "count": 1}}, {"request_contains": {"index": 1, "text": "host results are text"}}, {"event": {"type": "tool_result", "name": "rlm", "is_error": false, "text": "VERIFIED"}}, {"final_text_contains": "Verified using the supported text result."}]} {"id":"rlm-routing-preference","why":"The first request advertises a narrow RLM preference without removing the shell fallback.","args":[],"prompt":"Read the synthetic JSON fixture and return its row count.","workspace_seed":{"rows.json":"[{\"id\":1},{\"id\":2}]"},"script":[{"tool":"rlm","arguments":{"code":"rows = read_file(\"rows.json\")\nprint(len(rows))"}},{"text":"2"}],"assert":[{"request_contains":{"index":0,"text":"Prefer one RLM call for independent reads"}},{"request_contains":{"index":0,"text":"Use shell for builds, tests, git, and unsupported transforms"}},{"event":{"type":"tool_result","name":"rlm","is_error":false,"text":"2"}}]} {"id":"rlm-routing-structured-opt-out","why":"Structured-only mode does not advertise the RLM preference.","args":["--old"],"prompt":"Read the synthetic fixture.","workspace_seed":{"rows.json":"[]"},"script":[{"tool":"read_file","arguments":{"path":"rows.json"}},{"text":"The array is empty."}],"assert":[{"request_lacks":{"index":0,"text":"Prefer one RLM call for independent reads"}},{"event":{"type":"tool_result","name":"read_file","is_error":false,"text":"[]"}}]} +{"id":"run-task-no-edits-dispatches","why":"#1914: an explicit no-edit instruction must reach a safe run_task server, but a separate edit request must still be refused.","env":{"GRAFF_MCP_SCHEMA_BUDGET":"0","GRAFF_MCP_PROBE":"0","GRAFF_LEARN_AUTO":"off"},"workspace_seed":{".mcp.json":"{\"mcpServers\":{\"codegraff\":{\"command\":\"python3\",\"args\":[\"mock_mcp.py\"]}}}","mock_mcp.py":"import json, sys\nfor line in sys.stdin:\n r = json.loads(line)\n if 'id' not in r: continue\n method = r.get('method')\n if method == 'initialize':\n result = {'protocolVersion':'2025-06-18','capabilities':{'tools':{}},'serverInfo':{'name':'fixture','version':'1'}}\n elif method == 'tools/list':\n result = {'tools':[{'name':'run_task','description':'Run a task','inputSchema':{'type':'object','properties':{'prompt':{'type':'string'}},'required':['prompt']}}]}\n elif method == 'tools/call':\n result = {'content':[{'type':'text','text':'fixture received: ' + r['params']['arguments']['prompt']}] }\n else: result = {}\n print(json.dumps({'jsonrpc':'2.0','id':r['id'],'result':result}),flush=True)\n"},"prompt":"Delegate a read-only summary and check that the server refuses local edits.","script":[{"tool":"load_tool_schemas","arguments":{"server":"codegraff"}},{"tool":"mcp__codegraff__run_task","arguments":{"prompt":"Summarize the documentation. No edits."}},{"tool":"mcp__codegraff__run_task","arguments":{"prompt":"No edits elsewhere. Update notes.md."}},{"text":"Read-only delegation checked."}],"assert":[{"event":{"type":"tool_result","name":"mcp__codegraff__run_task","is_error":false}},{"event":{"type":"tool_result","name":"mcp__codegraff__run_task","is_error":true}},{"request_contains":{"index":-1,"text":"fixture received: Summarize the documentation. No edits."}},{"request_contains":{"index":-1,"text":"refused detour to external MCP tool mcp__codegraff__run_task for a local workspace edit"}},{"request_lacks":{"index":-1,"text":"fixture received: No edits elsewhere. Update notes.md."}}]} {"id": "gateway-failure-usage-unknown", "why": "A typed gateway failure with no usage must preserve the provider diagnostic and label totals as incomplete instead of implying a free zero-token call.", "args": ["-p", "summarize the fixture"], "prompt": "summarize the fixture", "script": [{"text": "partial", "stream_error": {"type": "gateway_error", "code": "request_failed", "message": "fixture request failed"}}], "expected_exit": 1, "assert": [{"stderr_contains": "gateway_error request_failed"}, {"stderr_contains": "known subtotal: 0 api call(s)"}, {"stderr_contains": "1 failed request attempt(s) without usage (tokens and cost unknown)"}]} {"id": "gateway-retry-usage-remains-incomplete", "why": "A recovered retry retains known successful usage while the failed attempt remains explicitly unknown.", "prompt": "summarize the fixture", "script": [{"text": "partial", "stream_error": {"type": "api_error", "message": "Internal error during token parsing"}}, {"text": "recovered fixture"}], "assert": [{"event": {"type": "turn", "usage_complete": false, "unreported_failed_attempts": 1, "api_calls": 1}}, {"final_text_contains": "recovered fixture"}]} {"id":"terminal-model-unavailable-does-not-retry","why":"#1191: a structured model_unavailable error is terminal when the gateway cannot serve the chosen model; a retry would repeat the same failure. Keep the request id so support can find the failed call.","prompt":"Say hello","script":[{"stream_error":{"type":"service_unavailable","code":"model_unavailable","message":"This model is not available right now. Choose another model or contact Codegraff support with the request ID.","request_id":"cg_req_fixture"}}],"assert":[{"events_at_most":{"match":{"type":"model_call_started"},"count":1}},{"event":{"type":"error","message":"Model lmstudio is currently unavailable. Switch models with /model, or try again later. Technical details — api error: (service_unavailable model_unavailable) This model is not available right now. Choose another model or contact Codegraff support with the request ID. [cg_req_fixture]"}},{"no_event":{"type":"model_call_finished","ok":true}}]} diff --git a/scripts/eval/tier1-manifest.json b/scripts/eval/tier1-manifest.json index ba802fe0..add1ada3 100644 --- a/scripts/eval/tier1-manifest.json +++ b/scripts/eval/tier1-manifest.json @@ -14,7 +14,7 @@ "src/main.zig", "src/repl.zig" ], - "test_count_baseline": 4069, + "test_count_baseline": 4070, "test_count_slack": 25, "required_invariants": [ { diff --git a/src/cli_changelog_source.zig b/src/cli_changelog_source.zig index 301fa99a..fe3b186a 100644 --- a/src/cli_changelog_source.zig +++ b/src/cli_changelog_source.zig @@ -5,6 +5,9 @@ pub const changelog_text = \\What's new \\────────── + \\0.302.74.78 + \\ • Let read-only `run_task` prompts say `No edits.` while still guarding actual edits (#1914). + \\ \\0.302.74.77 \\ • Prefer `rlm` for independent reads, JSON row counts, and field extraction (#1915). \\ diff --git a/src/mcp_run_task_guard.zig b/src/mcp_run_task_guard.zig index 226b2f83..1a0fc881 100644 --- a/src/mcp_run_task_guard.zig +++ b/src/mcp_run_task_guard.zig @@ -42,26 +42,45 @@ pub fn isRunTaskTool(qualified_name: []const u8, original_name: []const u8) bool std.mem.endsWith(u8, qualified_name, "__run_task"); } -fn hasWord(haystack: []const u8, word: []const u8) bool { - if (word.len == 0 or word.len > haystack.len) return false; - var i: usize = 0; - while (i + word.len <= haystack.len) : (i += 1) { - if (!std.ascii.eqlIgnoreCase(haystack[i .. i + word.len], word)) continue; - const left_ok = i == 0 or !std.ascii.isAlphanumeric(haystack[i - 1]); - const right_ok = i + word.len == haystack.len or !std.ascii.isAlphanumeric(haystack[i + word.len]); - if (left_ok and right_ok) return true; +// Match a whole word followed by horizontal whitespace immediately before the +// edit word. Do not carry negation across punctuation or a line break. +fn beforeWord(prefix: []const u8, word: []const u8) ?[]const u8 { + const trimmed = std.mem.trimEnd(u8, prefix, " \t"); + if (trimmed.len == prefix.len or trimmed.len < word.len) return null; + const start = trimmed.len - word.len; + if (start > 0 and std.ascii.isAlphanumeric(trimmed[start - 1])) return null; + if (!std.ascii.eqlIgnoreCase(trimmed[start..], word)) return null; + return trimmed[0..start]; +} + +fn explicitlyNegated(prefix: []const u8) bool { + for ([_][]const u8{ "no", "without", "don't" }) |word| { + if (beforeWord(prefix, word) != null) return true; } - return false; + const before_not = beforeWord(prefix, "not") orelse return false; + return beforeWord(before_not, "do") != null; } -/// True when the delegated task text asks to modify/write/create/delete files. +/// Heuristic only: ignore directly negated edit words, but keep scanning for +/// positive requests. Server-side permissions are independent of this check. pub fn requestsLocalEdit(prompt: []const u8) bool { - for ([_][]const u8{ - "edit", "edits", "modify", "patch", "update", "replace", - "write", "create", "delete", "remove", "rename", "insert", - "append", "apply", "fix", "add", "change", "rewrite", - }) |kw| { - if (hasWord(prompt, kw)) return true; + var i: usize = 0; + while (i < prompt.len) { + if (!std.ascii.isAlphanumeric(prompt[i])) { + i += 1; + continue; + } + const start = i; + while (i < prompt.len and std.ascii.isAlphanumeric(prompt[i])) : (i += 1) {} + for ([_][]const u8{ + "edit", "edits", "modify", "patch", "update", "replace", + "write", "create", "delete", "remove", "rename", "insert", + "append", "apply", "fix", "add", "change", "rewrite", + }) |kw| { + if (!std.ascii.eqlIgnoreCase(prompt[start..i], kw)) continue; + if (!explicitlyNegated(prompt[0..start])) return true; + break; + } } return false; } @@ -196,6 +215,31 @@ pub fn annotateCompleted(gpa: Allocator, qualified_name: []const u8, has_meta: b return joined; } +test "#1914: run_task ignores explicit no-edit instructions without allowing actual edits" { + const boundaries = [_]ServerBoundary{ + .{ .server_name = "codegraff", .mode = .server_owned_external }, + .{ .server_name = "codegraff", .mode = .safe, .server_cwd = "/other" }, + .{ .server_name = "codegraff", .mode = .yolo, .server_cwd = "/other" }, + }; + for (boundaries) |boundary| { + for ([_][]const u8{ + "Look up the public documentation. No edits.", + "Summarize the results. Do not edit any files.", + "Review only; DON'T\tMODIFY the workspace.", + "Summarize without edits.", + }) |prompt| try std.testing.expect(!checkBoundary(boundary, "/workspace", prompt, true).any()); + for ([_][]const u8{ + "Update notes.md.", + "No edits elsewhere. Update notes.md.", + "Do not edit notes.md, but fix config.json.", + "Fix config.json. No edits elsewhere.", + "Do not stop. Edit notes.md.", + "Do not edit notes.md.\nUpdate config.json.", + "Why not fix config.json?", + }) |prompt| try std.testing.expect(checkBoundary(boundary, "/workspace", prompt, true).any()); + } +} + test "#1909: safe-mode external run_task rooted elsewhere is blocked for local edits while native edit verifies bytes" { const gpa = std.testing.allocator; const io = std.testing.io;