Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ The release workflow uses a tag's section here as its release notes (a
hand-written `docs/releases/<tag>.md` wins if present), so keeping this file
current is part of cutting a release.

## v0.302.74.78

- Allow read-only `run_task` prompts that say `No edits.` without weakening the guard for actual edit requests (#1914).

## v0.302.74.77

- Prefer `rlm` for independent file reads, JSON row counts, and field extraction. Direct tools, shell fallback, and `--no-rlm` remain available (#1915).
Expand Down
1 change: 1 addition & 0 deletions evals/harness_behavior.jsonl
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@
{"id": "rlm-indexed-print-refuses-and-recovers", "why": "Unsupported Python-style result indexing must report an error without pretending to execute the nested host or executing later side effects; the supported text binding form recovers.", "args": ["--lean"], "prompt": "Run the scripted checks and report only observed results.", "script": [{"tool": "rlm", "arguments": {"code": "print(bash(\"printf ran > nested-marker\")[\"stdout\"])\nx = write_file(path=\"tail-marker\", content=\"ran\")"}}, {"tool": "rlm", "arguments": {"code": "r = bash(\"printf VERIFIED\")\nprint(r)"}}, {"text": "Verified using the supported text result."}], "verify_command": ["python3", "-c", "from pathlib import Path; assert not Path(\"nested-marker\").exists(); assert not Path(\"tail-marker\").exists()"], "assert": [{"event": {"type": "tool_call_finished", "name": "rlm", "is_error": true}}, {"events_at_most": {"match": {"type": "tool_call_finished", "is_error": true}, "count": 1}}, {"request_contains": {"index": 1, "text": "host results are text"}}, {"event": {"type": "tool_result", "name": "rlm", "is_error": false, "text": "VERIFIED"}}, {"final_text_contains": "Verified using the supported text result."}]}
{"id":"rlm-routing-preference","why":"The first request advertises a narrow RLM preference without removing the shell fallback.","args":[],"prompt":"Read the synthetic JSON fixture and return its row count.","workspace_seed":{"rows.json":"[{\"id\":1},{\"id\":2}]"},"script":[{"tool":"rlm","arguments":{"code":"rows = read_file(\"rows.json\")\nprint(len(rows))"}},{"text":"2"}],"assert":[{"request_contains":{"index":0,"text":"Prefer one RLM call for independent reads"}},{"request_contains":{"index":0,"text":"Use shell for builds, tests, git, and unsupported transforms"}},{"event":{"type":"tool_result","name":"rlm","is_error":false,"text":"2"}}]}
{"id":"rlm-routing-structured-opt-out","why":"Structured-only mode does not advertise the RLM preference.","args":["--old"],"prompt":"Read the synthetic fixture.","workspace_seed":{"rows.json":"[]"},"script":[{"tool":"read_file","arguments":{"path":"rows.json"}},{"text":"The array is empty."}],"assert":[{"request_lacks":{"index":0,"text":"Prefer one RLM call for independent reads"}},{"event":{"type":"tool_result","name":"read_file","is_error":false,"text":"[]"}}]}
{"id":"run-task-no-edits-dispatches","why":"#1914: an explicit no-edit instruction must reach a safe run_task server, but a separate edit request must still be refused.","env":{"GRAFF_MCP_SCHEMA_BUDGET":"0","GRAFF_MCP_PROBE":"0","GRAFF_LEARN_AUTO":"off"},"workspace_seed":{".mcp.json":"{\"mcpServers\":{\"codegraff\":{\"command\":\"python3\",\"args\":[\"mock_mcp.py\"]}}}","mock_mcp.py":"import json, sys\nfor line in sys.stdin:\n r = json.loads(line)\n if 'id' not in r: continue\n method = r.get('method')\n if method == 'initialize':\n result = {'protocolVersion':'2025-06-18','capabilities':{'tools':{}},'serverInfo':{'name':'fixture','version':'1'}}\n elif method == 'tools/list':\n result = {'tools':[{'name':'run_task','description':'Run a task','inputSchema':{'type':'object','properties':{'prompt':{'type':'string'}},'required':['prompt']}}]}\n elif method == 'tools/call':\n result = {'content':[{'type':'text','text':'fixture received: ' + r['params']['arguments']['prompt']}] }\n else: result = {}\n print(json.dumps({'jsonrpc':'2.0','id':r['id'],'result':result}),flush=True)\n"},"prompt":"Delegate a read-only summary and check that the server refuses local edits.","script":[{"tool":"load_tool_schemas","arguments":{"server":"codegraff"}},{"tool":"mcp__codegraff__run_task","arguments":{"prompt":"Summarize the documentation. No edits."}},{"tool":"mcp__codegraff__run_task","arguments":{"prompt":"No edits elsewhere. Update notes.md."}},{"text":"Read-only delegation checked."}],"assert":[{"event":{"type":"tool_result","name":"mcp__codegraff__run_task","is_error":false}},{"event":{"type":"tool_result","name":"mcp__codegraff__run_task","is_error":true}},{"request_contains":{"index":-1,"text":"fixture received: Summarize the documentation. No edits."}},{"request_contains":{"index":-1,"text":"refused detour to external MCP tool mcp__codegraff__run_task for a local workspace edit"}},{"request_lacks":{"index":-1,"text":"fixture received: No edits elsewhere. Update notes.md."}}]}
{"id": "gateway-failure-usage-unknown", "why": "A typed gateway failure with no usage must preserve the provider diagnostic and label totals as incomplete instead of implying a free zero-token call.", "args": ["-p", "summarize the fixture"], "prompt": "summarize the fixture", "script": [{"text": "partial", "stream_error": {"type": "gateway_error", "code": "request_failed", "message": "fixture request failed"}}], "expected_exit": 1, "assert": [{"stderr_contains": "gateway_error request_failed"}, {"stderr_contains": "known subtotal: 0 api call(s)"}, {"stderr_contains": "1 failed request attempt(s) without usage (tokens and cost unknown)"}]}
{"id": "gateway-retry-usage-remains-incomplete", "why": "A recovered retry retains known successful usage while the failed attempt remains explicitly unknown.", "prompt": "summarize the fixture", "script": [{"text": "partial", "stream_error": {"type": "api_error", "message": "Internal error during token parsing"}}, {"text": "recovered fixture"}], "assert": [{"event": {"type": "turn", "usage_complete": false, "unreported_failed_attempts": 1, "api_calls": 1}}, {"final_text_contains": "recovered fixture"}]}
{"id":"terminal-model-unavailable-does-not-retry","why":"#1191: a structured model_unavailable error is terminal when the gateway cannot serve the chosen model; a retry would repeat the same failure. Keep the request id so support can find the failed call.","prompt":"Say hello","script":[{"stream_error":{"type":"service_unavailable","code":"model_unavailable","message":"This model is not available right now. Choose another model or contact Codegraff support with the request ID.","request_id":"cg_req_fixture"}}],"assert":[{"events_at_most":{"match":{"type":"model_call_started"},"count":1}},{"event":{"type":"error","message":"Model lmstudio is currently unavailable. Switch models with /model, or try again later. Technical details — api error: (service_unavailable model_unavailable) This model is not available right now. Choose another model or contact Codegraff support with the request ID. [cg_req_fixture]"}},{"no_event":{"type":"model_call_finished","ok":true}}]}
Expand Down
2 changes: 1 addition & 1 deletion scripts/eval/tier1-manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"src/main.zig",
"src/repl.zig"
],
"test_count_baseline": 4069,
"test_count_baseline": 4070,
"test_count_slack": 25,
"required_invariants": [
{
Expand Down
3 changes: 3 additions & 0 deletions src/cli_changelog_source.zig
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
pub const changelog_text =
\\What's new
\\──────────
\\0.302.74.78
\\ • Let read-only `run_task` prompts say `No edits.` while still guarding actual edits (#1914).
\\
\\0.302.74.77
\\ • Prefer `rlm` for independent reads, JSON row counts, and field extraction (#1915).
\\
Expand Down
76 changes: 60 additions & 16 deletions src/mcp_run_task_guard.zig
Original file line number Diff line number Diff line change
Expand Up @@ -42,26 +42,45 @@ pub fn isRunTaskTool(qualified_name: []const u8, original_name: []const u8) bool
std.mem.endsWith(u8, qualified_name, "__run_task");
}

fn hasWord(haystack: []const u8, word: []const u8) bool {
if (word.len == 0 or word.len > haystack.len) return false;
var i: usize = 0;
while (i + word.len <= haystack.len) : (i += 1) {
if (!std.ascii.eqlIgnoreCase(haystack[i .. i + word.len], word)) continue;
const left_ok = i == 0 or !std.ascii.isAlphanumeric(haystack[i - 1]);
const right_ok = i + word.len == haystack.len or !std.ascii.isAlphanumeric(haystack[i + word.len]);
if (left_ok and right_ok) return true;
// Match a whole word followed by horizontal whitespace immediately before the
// edit word. Do not carry negation across punctuation or a line break.
fn beforeWord(prefix: []const u8, word: []const u8) ?[]const u8 {
const trimmed = std.mem.trimEnd(u8, prefix, " \t");
if (trimmed.len == prefix.len or trimmed.len < word.len) return null;
const start = trimmed.len - word.len;
if (start > 0 and std.ascii.isAlphanumeric(trimmed[start - 1])) return null;
if (!std.ascii.eqlIgnoreCase(trimmed[start..], word)) return null;
return trimmed[0..start];
}

fn explicitlyNegated(prefix: []const u8) bool {
for ([_][]const u8{ "no", "without", "don't" }) |word| {
if (beforeWord(prefix, word) != null) return true;
}
return false;
const before_not = beforeWord(prefix, "not") orelse return false;
return beforeWord(before_not, "do") != null;
}

/// True when the delegated task text asks to modify/write/create/delete files.
/// Heuristic only: ignore directly negated edit words, but keep scanning for
/// positive requests. Server-side permissions are independent of this check.
pub fn requestsLocalEdit(prompt: []const u8) bool {
for ([_][]const u8{
"edit", "edits", "modify", "patch", "update", "replace",
"write", "create", "delete", "remove", "rename", "insert",
"append", "apply", "fix", "add", "change", "rewrite",
}) |kw| {
if (hasWord(prompt, kw)) return true;
var i: usize = 0;
while (i < prompt.len) {
if (!std.ascii.isAlphanumeric(prompt[i])) {
i += 1;
continue;
}
const start = i;
while (i < prompt.len and std.ascii.isAlphanumeric(prompt[i])) : (i += 1) {}
for ([_][]const u8{
"edit", "edits", "modify", "patch", "update", "replace",
"write", "create", "delete", "remove", "rename", "insert",
"append", "apply", "fix", "add", "change", "rewrite",
}) |kw| {
if (!std.ascii.eqlIgnoreCase(prompt[start..i], kw)) continue;
if (!explicitlyNegated(prompt[0..start])) return true;
break;
}
}
return false;
}
Expand Down Expand Up @@ -196,6 +215,31 @@ pub fn annotateCompleted(gpa: Allocator, qualified_name: []const u8, has_meta: b
return joined;
}

test "#1914: run_task ignores explicit no-edit instructions without allowing actual edits" {
const boundaries = [_]ServerBoundary{
.{ .server_name = "codegraff", .mode = .server_owned_external },
.{ .server_name = "codegraff", .mode = .safe, .server_cwd = "/other" },
.{ .server_name = "codegraff", .mode = .yolo, .server_cwd = "/other" },
};
for (boundaries) |boundary| {
for ([_][]const u8{
"Look up the public documentation. No edits.",
"Summarize the results. Do not edit any files.",
"Review only; DON'T\tMODIFY the workspace.",
"Summarize without edits.",
}) |prompt| try std.testing.expect(!checkBoundary(boundary, "/workspace", prompt, true).any());
for ([_][]const u8{
"Update notes.md.",
"No edits elsewhere. Update notes.md.",
"Do not edit notes.md, but fix config.json.",
"Fix config.json. No edits elsewhere.",
"Do not stop. Edit notes.md.",
"Do not edit notes.md.\nUpdate config.json.",
"Why not fix config.json?",
}) |prompt| try std.testing.expect(checkBoundary(boundary, "/workspace", prompt, true).any());
}
}

test "#1909: safe-mode external run_task rooted elsewhere is blocked for local edits while native edit verifies bytes" {
const gpa = std.testing.allocator;
const io = std.testing.io;
Expand Down
Loading