diff --git a/evals/harness_behavior.jsonl b/evals/harness_behavior.jsonl index bc77a97d..270eecbd 100644 --- a/evals/harness_behavior.jsonl +++ b/evals/harness_behavior.jsonl @@ -187,3 +187,4 @@ {"id": "strict-optional-empty-args-edit-read-load", "why": "Strict tool-calling models send empty optional arrays/strings (edit_file edits:[], load_tool_schemas server:''/query:'') and combine read_file contains with start_line/end_line; all three must succeed cleanly.", "args": [], "env": {"GRAFF_NO_CODEDB_GUARD": "1"}, "workspace_seed": {"doc.txt": "line1 alpha\nline2 target\nline3 target\n"}, "prompt": "Search lines 2-2 of doc.txt for target, replace line2 target with line2 updated, load todo_write, and confirm.", "script": [{"tool": "read_file", "arguments": {"path": "doc.txt", "contains": "target", "start_line": 2, "end_line": 2}}, {"tool": "edit_file", "arguments": {"path": "doc.txt", "old_string": "line2 target", "new_string": "line2 updated", "edits": []}}, {"tool": "load_tool_schemas", "arguments": {"tools": ["todo_write"], "server": "", "query": ""}}, {"text": "Verified read_file window contains, edit_file single span with empty edits, and load_tool_schemas with empty server/query."}], "assert": [{"event": {"type": "tool_call_finished", "name": "read_file", "is_error": false}}, {"event": {"type": "tool_call_finished", "name": "edit_file", "is_error": false}}, {"event": {"type": "tool_call_finished", "name": "load_tool_schemas", "is_error": false}}, {"request_contains": {"index": 1, "text": "2: line2 target"}}, {"request_lacks": {"index": 1, "text": "3: line3 target"}}, {"file_equals": {"path": "doc.txt", "content": "line1 alpha\nline2 updated\nline3 target\n"}}, {"final_text_contains": "Verified read_file window contains"}]} {"id": "python-repl-persistent-session-state", "scripted_only": true, "why": "ADR 0348: python_repl unfolds on demand via load_tool_schemas and preserves variables and trailing _ across calls.", "prompt": "Load python_repl and compute total into answer.txt across two calls.", "script": [{"tool": "load_tool_schemas", "arguments": {"tools": ["python_repl"]}}, {"tool": "python_repl", "arguments": {"code": "nums = [10, 20, 30]\nsum(nums)"}}, {"tool": "python_repl", "arguments": {"code": "open('answer.txt', 'w').write(f'total={sum(nums)},prev={_}')\n'saved'"}}, {"text": "Saved persistent computation to answer.txt."}], "assert": [{"event": {"type": "tool_call", "name": "load_tool_schemas"}}, {"event": {"type": "tool_result", "name": "python_repl", "is_error": false}}, {"request_lacks": {"index": 0, "text": "\"name\": \"python_repl\""}}, {"request_contains": {"index": 1, "text": "\"name\": \"python_repl\""}}, {"request_contains": {"index": -1, "text": "'saved'"}}, {"file_equals": {"path": "answer.txt", "content": "total=60,prev=60"}}, {"final_text_contains": "answer.txt"}]} {"id": "unharnessed-mode-single-tool-and-json-control", "scripted_only": true, "why": "ADR 0348: set_unharnessed persists to .harness/settings.json, switches to single-tool python catalog without file/shell tools, and preserves persistent REPL state.", "controls_before_prompt": [{"type": "set_unharnessed", "on": true}], "prompt": "Compute 7 * 9 across two persistent python calls and write out.txt.", "script": [{"tool": "python", "arguments": {"code": "val = 7 * 9\nval"}}, {"tool": "python", "arguments": {"code": "open('out.txt', 'w').write(f'{val}:{_}')\n'done'"}}, {"text": "Completed in single-tool python mode."}], "assert": [{"event": {"type": "unharnessed", "ok": true, "on": true}}, {"request_contains": {"index": 0, "text": "\"name\": \"python\""}}, {"request_lacks": {"index": 0, "text": "\"name\": \"read_file\""}}, {"request_lacks": {"index": 0, "text": "\"name\": \"edit_file\""}}, {"request_lacks": {"index": 0, "text": "\"name\": \"shell\""}}, {"file_equals": {"path": ".harness/settings.json", "content": "{\"unharnessed\":true}\n"}}, {"file_equals": {"path": "out.txt", "content": "63:63"}}, {"final_text_contains": "Completed in single-tool python mode."}]} +{"id":"plan-mcp-readonly-diagnostic","why":"#1919: direct and RLM MCP calls share the planning gate. A declared read works; an unannotated lookup or explicit write stays blocked with an accurate reason.","args":["--orchestrator=off"],"env":{"GRAFF_MCP_SCHEMA_BUDGET":"0","GRAFF_MCP_PROBE":"0","GRAFF_LEARN_AUTO":"off"},"workspace_seed":{".mcp.json":"{\"mcpServers\": {\"fixture\": {\"command\": \"python3\", \"args\": [\"mock_mcp.py\"]}}}","mock_mcp.py":"import json, sys\nfor line in sys.stdin:\n call = json.loads(line)\n if 'id' not in call: continue\n method = call.get('method')\n if method == 'initialize':\n result = {'protocolVersion':'2025-06-18','capabilities':{'tools':{}},'serverInfo':{'name':'fixture','version':'1'}}\n elif method == 'tools/list':\n schema = {'type':'object','properties':{'label':{'type':'string'}}}\n result = {'tools':[\n {'name':'list_verified','inputSchema':schema,'annotations':{'readOnlyHint':True}},\n {'name':'list_unannotated','inputSchema':schema},\n {'name':'update_issue','inputSchema':schema,'annotations':{'readOnlyHint':False}},\n ]}\n elif method == 'tools/call':\n params = call['params']\n text = 'dispatched ' + params['name'] + ': ' + params['arguments'].get('label', '')\n result = {'content':[{'type':'text','text':text}]}\n else: result = {}\n print(json.dumps({'jsonrpc':'2.0','id':call['id'],'result':result}),flush=True)\n"},"controls_before_prompt":[{"type":"set_mode","mode":"plan"}],"prompt":"Check the issue lookup tools in read-only mode.","script":[{"tool":"load_tool_schemas","arguments":{"server":"fixture"}},{"tool":"mcp__fixture__list_verified","arguments":{"label":"direct"}},{"tool":"rlm","arguments":{"code":"x = mcp__fixture__list_verified(label=\"via RLM\")\nprint(x)"}},{"tool":"mcp__fixture__list_unannotated","arguments":{}},{"tool":"rlm","arguments":{"code":"x = mcp__fixture__list_unannotated()\nprint(x)"}},{"tool":"mcp__fixture__update_issue","arguments":{}},{"text":"Planning tool checks complete."}],"assert":[{"event":{"type":"tool_result","name":"mcp__fixture__list_verified","is_error":false,"text":"dispatched list_verified: direct"}},{"event":{"type":"tool_result","name":"rlm","is_error":false}},{"event":{"type":"tool_result","name":"mcp__fixture__list_unannotated","is_error":true}},{"event":{"type":"tool_result","name":"rlm","is_error":true}},{"event":{"type":"tool_result","name":"mcp__fixture__update_issue","is_error":true}},{"request_contains":{"index":-1,"text":"dispatched list_verified: via RLM"}},{"request_contains":{"index":-1,"text":"MCP tool mcp__fixture__list_unannotated is not declared read-only"}},{"request_contains":{"index":-1,"text":"MCP tool mcp__fixture__update_issue is not declared read-only"}},{"request_lacks":{"index":-1,"text":"dispatched list_unannotated"}},{"request_lacks":{"index":-1,"text":"dispatched update_issue"}},{"request_lacks":{"index":-1,"text":"Fold this change into the plan"}}]} diff --git a/src/agent_tool_gate.zig b/src/agent_tool_gate.zig index f98382a1..57821ef3 100644 --- a/src/agent_tool_gate.zig +++ b/src/agent_tool_gate.zig @@ -226,12 +226,21 @@ pub fn gateTool(self: *Agent, call: ToolCall) !?ExecResult { const approvals = self.approvals orelse return null; if (main_mod.plan_mode) { + if (mcp.Registry.isMcp(call.name) and !mcpReadOnly(self.registry, call)) { + const msg = if (@import("orchestrator_mode.zig").g_phase == .planning) + "orchestrator Phase 1 planning is read-only" + else + "plan mode is on — read-only"; + return .{ + .text = try std.fmt.allocPrint(self.arena, "{s}: MCP tool {s} is not declared read-only. Use a verified read-only alternative, or defer this call until execution.", .{ msg, call.name }), + .is_error = true, + }; + } if (std.mem.eql(u8, call.name, "learn_candidate") or std.mem.eql(u8, call.name, "write_file") or std.mem.eql(u8, call.name, "edit_file") or @import("python_repl.zig").isTool(call.name) or std.mem.eql(u8, call.name, "imagegen") or // #352: writes a file and spawns a workspace-write child - std.mem.eql(u8, call.name, "peer_message") or // #469: appends to a peer's inbox — a side effect - (mcp.Registry.isMcp(call.name) and !mcpReadOnly(self.registry, call))) + std.mem.eql(u8, call.name, "peer_message")) // #469: appends to a peer's inbox — a side effect { const msg = if (@import("orchestrator_mode.zig").g_phase == .planning) "orchestrator Phase 1 planning is read-only — include this edit in exit_plan_mode(plan) so the executor runs it in Phase 2."