Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions evals/harness_behavior.jsonl
Original file line number Diff line number Diff line change
Expand Up @@ -187,3 +187,4 @@
{"id": "strict-optional-empty-args-edit-read-load", "why": "Strict tool-calling models send empty optional arrays/strings (edit_file edits:[], load_tool_schemas server:''/query:'') and combine read_file contains with start_line/end_line; all three must succeed cleanly.", "args": [], "env": {"GRAFF_NO_CODEDB_GUARD": "1"}, "workspace_seed": {"doc.txt": "line1 alpha\nline2 target\nline3 target\n"}, "prompt": "Search lines 2-2 of doc.txt for target, replace line2 target with line2 updated, load todo_write, and confirm.", "script": [{"tool": "read_file", "arguments": {"path": "doc.txt", "contains": "target", "start_line": 2, "end_line": 2}}, {"tool": "edit_file", "arguments": {"path": "doc.txt", "old_string": "line2 target", "new_string": "line2 updated", "edits": []}}, {"tool": "load_tool_schemas", "arguments": {"tools": ["todo_write"], "server": "", "query": ""}}, {"text": "Verified read_file window contains, edit_file single span with empty edits, and load_tool_schemas with empty server/query."}], "assert": [{"event": {"type": "tool_call_finished", "name": "read_file", "is_error": false}}, {"event": {"type": "tool_call_finished", "name": "edit_file", "is_error": false}}, {"event": {"type": "tool_call_finished", "name": "load_tool_schemas", "is_error": false}}, {"request_contains": {"index": 1, "text": "2: line2 target"}}, {"request_lacks": {"index": 1, "text": "3: line3 target"}}, {"file_equals": {"path": "doc.txt", "content": "line1 alpha\nline2 updated\nline3 target\n"}}, {"final_text_contains": "Verified read_file window contains"}]}
{"id": "python-repl-persistent-session-state", "scripted_only": true, "why": "ADR 0348: python_repl unfolds on demand via load_tool_schemas and preserves variables and trailing _ across calls.", "prompt": "Load python_repl and compute total into answer.txt across two calls.", "script": [{"tool": "load_tool_schemas", "arguments": {"tools": ["python_repl"]}}, {"tool": "python_repl", "arguments": {"code": "nums = [10, 20, 30]\nsum(nums)"}}, {"tool": "python_repl", "arguments": {"code": "open('answer.txt', 'w').write(f'total={sum(nums)},prev={_}')\n'saved'"}}, {"text": "Saved persistent computation to answer.txt."}], "assert": [{"event": {"type": "tool_call", "name": "load_tool_schemas"}}, {"event": {"type": "tool_result", "name": "python_repl", "is_error": false}}, {"request_lacks": {"index": 0, "text": "\"name\": \"python_repl\""}}, {"request_contains": {"index": 1, "text": "\"name\": \"python_repl\""}}, {"request_contains": {"index": -1, "text": "'saved'"}}, {"file_equals": {"path": "answer.txt", "content": "total=60,prev=60"}}, {"final_text_contains": "answer.txt"}]}
{"id": "unharnessed-mode-single-tool-and-json-control", "scripted_only": true, "why": "ADR 0348: set_unharnessed persists to .harness/settings.json, switches to single-tool python catalog without file/shell tools, and preserves persistent REPL state.", "controls_before_prompt": [{"type": "set_unharnessed", "on": true}], "prompt": "Compute 7 * 9 across two persistent python calls and write out.txt.", "script": [{"tool": "python", "arguments": {"code": "val = 7 * 9\nval"}}, {"tool": "python", "arguments": {"code": "open('out.txt', 'w').write(f'{val}:{_}')\n'done'"}}, {"text": "Completed in single-tool python mode."}], "assert": [{"event": {"type": "unharnessed", "ok": true, "on": true}}, {"request_contains": {"index": 0, "text": "\"name\": \"python\""}}, {"request_lacks": {"index": 0, "text": "\"name\": \"read_file\""}}, {"request_lacks": {"index": 0, "text": "\"name\": \"edit_file\""}}, {"request_lacks": {"index": 0, "text": "\"name\": \"shell\""}}, {"file_equals": {"path": ".harness/settings.json", "content": "{\"unharnessed\":true}\n"}}, {"file_equals": {"path": "out.txt", "content": "63:63"}}, {"final_text_contains": "Completed in single-tool python mode."}]}
{"id":"plan-mcp-readonly-diagnostic","why":"#1919: direct and RLM MCP calls share the planning gate. A declared read works; an unannotated lookup or explicit write stays blocked with an accurate reason.","args":["--orchestrator=off"],"env":{"GRAFF_MCP_SCHEMA_BUDGET":"0","GRAFF_MCP_PROBE":"0","GRAFF_LEARN_AUTO":"off"},"workspace_seed":{".mcp.json":"{\"mcpServers\": {\"fixture\": {\"command\": \"python3\", \"args\": [\"mock_mcp.py\"]}}}","mock_mcp.py":"import json, sys\nfor line in sys.stdin:\n call = json.loads(line)\n if 'id' not in call: continue\n method = call.get('method')\n if method == 'initialize':\n result = {'protocolVersion':'2025-06-18','capabilities':{'tools':{}},'serverInfo':{'name':'fixture','version':'1'}}\n elif method == 'tools/list':\n schema = {'type':'object','properties':{'label':{'type':'string'}}}\n result = {'tools':[\n {'name':'list_verified','inputSchema':schema,'annotations':{'readOnlyHint':True}},\n {'name':'list_unannotated','inputSchema':schema},\n {'name':'update_issue','inputSchema':schema,'annotations':{'readOnlyHint':False}},\n ]}\n elif method == 'tools/call':\n params = call['params']\n text = 'dispatched ' + params['name'] + ': ' + params['arguments'].get('label', '')\n result = {'content':[{'type':'text','text':text}]}\n else: result = {}\n print(json.dumps({'jsonrpc':'2.0','id':call['id'],'result':result}),flush=True)\n"},"controls_before_prompt":[{"type":"set_mode","mode":"plan"}],"prompt":"Check the issue lookup tools in read-only mode.","script":[{"tool":"load_tool_schemas","arguments":{"server":"fixture"}},{"tool":"mcp__fixture__list_verified","arguments":{"label":"direct"}},{"tool":"rlm","arguments":{"code":"x = mcp__fixture__list_verified(label=\"via RLM\")\nprint(x)"}},{"tool":"mcp__fixture__list_unannotated","arguments":{}},{"tool":"rlm","arguments":{"code":"x = mcp__fixture__list_unannotated()\nprint(x)"}},{"tool":"mcp__fixture__update_issue","arguments":{}},{"text":"Planning tool checks complete."}],"assert":[{"event":{"type":"tool_result","name":"mcp__fixture__list_verified","is_error":false,"text":"dispatched list_verified: direct"}},{"event":{"type":"tool_result","name":"rlm","is_error":false}},{"event":{"type":"tool_result","name":"mcp__fixture__list_unannotated","is_error":true}},{"event":{"type":"tool_result","name":"rlm","is_error":true}},{"event":{"type":"tool_result","name":"mcp__fixture__update_issue","is_error":true}},{"request_contains":{"index":-1,"text":"dispatched list_verified: via RLM"}},{"request_contains":{"index":-1,"text":"MCP tool mcp__fixture__list_unannotated is not declared read-only"}},{"request_contains":{"index":-1,"text":"MCP tool mcp__fixture__update_issue is not declared read-only"}},{"request_lacks":{"index":-1,"text":"dispatched list_unannotated"}},{"request_lacks":{"index":-1,"text":"dispatched update_issue"}},{"request_lacks":{"index":-1,"text":"Fold this change into the plan"}}]}
13 changes: 11 additions & 2 deletions src/agent_tool_gate.zig
Original file line number Diff line number Diff line change
Expand Up @@ -226,12 +226,21 @@ pub fn gateTool(self: *Agent, call: ToolCall) !?ExecResult {
const approvals = self.approvals orelse return null;

if (main_mod.plan_mode) {
if (mcp.Registry.isMcp(call.name) and !mcpReadOnly(self.registry, call)) {
const msg = if (@import("orchestrator_mode.zig").g_phase == .planning)
"orchestrator Phase 1 planning is read-only"
else
"plan mode is on — read-only";
return .{
.text = try std.fmt.allocPrint(self.arena, "{s}: MCP tool {s} is not declared read-only. Use a verified read-only alternative, or defer this call until execution.", .{ msg, call.name }),
.is_error = true,
};
}
if (std.mem.eql(u8, call.name, "learn_candidate") or
std.mem.eql(u8, call.name, "write_file") or std.mem.eql(u8, call.name, "edit_file") or
@import("python_repl.zig").isTool(call.name) or
std.mem.eql(u8, call.name, "imagegen") or // #352: writes a file and spawns a workspace-write child
std.mem.eql(u8, call.name, "peer_message") or // #469: appends to a peer's inbox — a side effect
(mcp.Registry.isMcp(call.name) and !mcpReadOnly(self.registry, call)))
std.mem.eql(u8, call.name, "peer_message")) // #469: appends to a peer's inbox — a side effect
{
const msg = if (@import("orchestrator_mode.zig").g_phase == .planning)
"orchestrator Phase 1 planning is read-only — include this edit in exit_plan_mode(plan) so the executor runs it in Phase 2."
Expand Down
Loading