From f56701b8b6708f5e1192dfa8645073c6f3d0f6c2 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 13:52:31 +0100 Subject: [PATCH 01/13] docs(agent): design for the global agent side panel --- .../2026-11-12-agent-side-panel-design.md | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 docs/superpowers/specs/2026-11-12-agent-side-panel-design.md diff --git a/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md b/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md new file mode 100644 index 000000000..69bfa8db5 --- /dev/null +++ b/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md @@ -0,0 +1,78 @@ +# Agent side panel: design + +## Intent + +The agent today lives in the "chat" mode of the Generate page's left panel. It loses its conversation whenever that panel or the page changes. The user wants the agent available **all the time**, in a panel on the right that **pushes the whole app**, so that the app can be driven entirely through the agent. + +## Goals + +- **Global panel.** A right-side panel lives in `MainLayout`, next to `
`, and pushes the content rather than overlaying it. + - A TopBar button toggles it, and so does `Ctrl+L` (my call). + - Its width can be resized by dragging (min 320, max 640, default 380; my call). + - Its open state and width are remembered across restarts. +- **Chat moves out of Generate.** + - `WorkflowPanel` loses its basic/chat toggle and keeps basic mode only. + - `ChatPanel` moves to `src/shared/components/agent/` as the panel body, keeping its current behaviour (streaming, thinking, actions card, mesh undo, workflow progress, image attachments). +- **The agent acts on the whole app without navigating.** It works on the page's state through stores, IPC and the API, never by changing `currentPage`, and confirms what it did in the panel. +- **Destructive actions ask first.** The panel shows a Confirm / Cancel card and the agent waits for the answer. This applies to uninstalling an extension, deleting model weights, deleting or overwriting a workflow, and changing a storage folder. +- **Conversations are persisted.** + - Each conversation is a JSON file in `/agent/conversations/{id}.json`. + - The panel lists past conversations (title, date) and offers New, Open and Delete. + - Images attached to messages are stored inline as data URLs (my call; simple, and good enough at this scale). +- **Context everywhere.** Every request sends the current page, the mesh (path, triangles), workflows, installed extensions, downloaded models and storage paths. + +## New agent tools + +**Server-side tools** run in `execute_tool` and call the local API: + +| Tool | Does | +|---|---| +| `generate_from_image(model_id?, params?)` | Starts a generation from the image attached to the message. The frontend runs it, as with `run_workflow`. | +| `list_extensions` | Lists installed extensions (from context). | +| `get_settings` | Returns the storage paths and a few UI options (from context). | + +**Client-side tools** need Electron IPC or renderer stores, so the frontend runs them: + +| Tool | Confirm? | +|---|---| +| `install_extension(github_url)` | no | +| `uninstall_extension(extension_id)` | **yes** | +| `reload_extensions` | no | +| `download_model_weights(extension_id, variant?)` | no | +| `delete_model_weights(extension_id, variant?)` | **yes** | +| `delete_workflow(workflow_id)` | **yes** | +| `open_workflow(workflow_id)` | no (opens it in the workflows store, does not navigate) | +| `set_setting(key, value)`, with key in `uiScale` / `useAtkinsonFont` / `showRamIndicator` | no | +| `set_storage_folder(kind, path)` | **yes** | +| `undo_mesh` / `redo_mesh` | no | +| `navigate(page)` | no; only when the user explicitly asks to go to a page | + +## Client tool bridge + +This is the core mechanism. + +1. The LLM calls a client tool. `_chat_events` does not run it. It yields a `client_tool` event `{id, tool, arguments, confirm}`, registers an `asyncio.Future` under the call id, and awaits it. +2. The panel runs the tool through a frontend registry (`clientTools.ts`, mapping each tool name to an async handler that returns a result string). When `confirm` is set, it first shows the Confirm / Cancel card. Cancel resolves with "The user declined." +3. The panel sends `POST /agent/tool-result {call_id, result, error?}`. The future resolves, the result goes into `messages` as a real `role: tool` message, and the loop continues in the same turn. +4. Timeouts work differently depending on whether the user is involved: + - Execution is bounded by `tool_timeout`. + - Waiting for the user to confirm is not bounded by it, but is capped at 10 minutes. + - If the stream disconnects, the futures are cancelled. +5. Which tools run on the client, and which of those need confirmation, is declared once on the backend in a `CLIENT_TOOLS` dict. The frontend just obeys `confirm`. + +This replaces nothing that exists today. `mesh_update`, `create_workflow` and `run_workflow` payloads keep working. + +## Non-goals + +- No page navigation as a side effect of an action. +- No change to the providers or to Settings > Agent. +- No change to the MCP server. + +## Anti-goals + +- The agent performing a destructive action without an explicit Confirm click. +- Losing the conversation when the page changes. + +## Left to the builder + +Visual details of the panel and the conversation list, as long as they match the existing zinc/violet style. From 70fe74e13f35726310af48e151416251281c057a Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:00:44 +0100 Subject: [PATCH 02/13] feat(agent): client tool bridge and app-wide tools for the side panel Tools the app must run itself (extensions, model weights, workflows, settings, mesh undo/redo, navigation) are declared in CLIENT_TOOLS. The tool loop hands them to the app as a client_tool event and waits for the answer on POST /agent/tool-result; destructive ones ask the user first and wait up to 10 minutes. Adds list_extensions, get_settings and generate_from_image, and renders the page, models, workflows, storage folders and UI options into the system prompt. --- api/routers/agent.py | 286 +++++++++++++++++++-- api/tests/test_agent_client_tools.py | 365 +++++++++++++++++++++++++++ 2 files changed, 626 insertions(+), 25 deletions(-) create mode 100644 api/tests/test_agent_client_tools.py diff --git a/api/routers/agent.py b/api/routers/agent.py index 2bdd04c13..797e40461 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -93,10 +93,21 @@ def registry_tool_lines(registry: MeshOpsRegistry) -> str: SYSTEM_PROMPT = f"""\ You are Modly's built-in AI assistant, specialized in 3D modeling and workflow automation. You help users generate 3D models from images, optimize meshes, and manage workflows directly inside the Modly application. +You can act on the whole app (extensions, model weights, workflows, settings, generation) from any page, without changing page. ## Available tools - **list_models** — List all downloaded 3D generation models ready to use. +- **generate_from_image(model_id?)** — Generate a 3D model from the image attached to the user's message. Omit model_id to use the model selected in the app. +- **list_extensions** — List installed extensions. +- **get_settings** — Show the storage folders and UI options. +- **install_extension(github_url)** / **uninstall_extension(extension_id)** / **reload_extensions** — Manage extensions. +- **download_model_weights(extension_id, variant?)** / **delete_model_weights(extension_id, variant?)** — Manage model weights. +- **open_workflow(workflow_id)** / **delete_workflow(workflow_id)** — Open a workflow in the editor, or delete it. +- **set_setting(key, value)** — Change `uiScale` (number), `useAtkinsonFont` or `showRamIndicator` (booleans). +- **set_storage_folder(kind, path)** — Move the `models`, `workspace` or `workflows` folder. +- **undo_mesh** / **redo_mesh** — Undo or redo the last change to the current mesh. +- **navigate(page)** — Switch the app to another page. - **unload_models** — Unload all 3D generation models from GPU VRAM to free memory. - **get_mesh_info** — Get info about the current mesh in the 3D viewer (path, triangle count). {registry_tool_lines(mesh_ops_registry)} @@ -117,6 +128,8 @@ def registry_tool_lines(registry: MeshOpsRegistry) -> str: - If you need to run a workflow but don't know the ID, call list_workflows first. - To create a workflow, ONLY use extension ids listed under "Available extensions" in the context. Never invent an id. Chain steps so each step's input type matches the previous step's output type. - For a workflow's input, `input_type` MUST be exactly one of: `image`, `text`, or `mesh`. These map to the Image, Text, and Load 3D Mesh nodes. Never invent another input. Pick the one matching the first step's expected input. +- uninstall_extension, delete_model_weights, delete_workflow and set_storage_folder ask the user to confirm. If the result says the user declined, acknowledge it and do not retry. +- Never call navigate unless the user explicitly asks to go to a page. Act from where the user is. - After each tool call, give a short one-sentence summary of what was done. - Always reply in the same language the user is writing in. - Be concise. No unnecessary explanations.\ @@ -281,6 +294,124 @@ def registry_tool_lines(registry: MeshOpsRegistry) -> str: TOOLS += registry_tools(mesh_ops_registry) +def _function(name: str, description: str, properties: dict | None = None, required: list[str] | None = None) -> dict: + parameters: dict = {"type": "object", "properties": properties or {}} + if required: + parameters["required"] = required + return {"type": "function", "function": {"name": name, "description": description, "parameters": parameters}} + + +_EXTENSION_ID = {"type": "string", "description": "Exact extension id, as listed by list_extensions."} +_WORKFLOW_ID = {"type": "string", "description": "Exact workflow id, as listed by list_workflows."} +_VARIANT = {"type": "string", "description": "Optional weights variant, for extensions that offer several. Omit for the default."} + +# Server-side tools that read the request context or hand a run to the app. +TOOLS += [ + _function("list_extensions", "List the installed extensions, with their ids and input→output types."), + _function("get_settings", "Show Modly's storage folders (models, workspace, workflows, extensions) and UI options."), + _function( + "generate_from_image", + "Generate a 3D model from the image the user attached to their message. The app runs the " + "generation and shows its progress. Needs an attached image.", + {"model_id": { + "type": "string", + "description": "Id of a downloaded generation model (see list_models). Omit to use the model selected in the app.", + }}, + ), +] + +# Tools the app runs itself (Electron IPC, renderer stores), mapped to whether +# the user must confirm them first. The frontend obeys `confirm`. +CLIENT_TOOLS: dict[str, bool] = { + "install_extension": False, + "uninstall_extension": True, + "reload_extensions": False, + "download_model_weights": False, + "delete_model_weights": True, + "delete_workflow": True, + "open_workflow": False, + "set_setting": False, + "set_storage_folder": True, + "undo_mesh": False, + "redo_mesh": False, + "navigate": False, +} + +TOOLS += [ + _function( + "install_extension", + "Install an extension from its GitHub repository URL, then load it.", + {"github_url": {"type": "string", "description": "Repository URL, e.g. https://github.com/owner/repo."}}, + ["github_url"], + ), + _function( + "uninstall_extension", + "Uninstall an extension and remove its files. The user is asked to confirm.", + {"extension_id": _EXTENSION_ID}, + ["extension_id"], + ), + _function("reload_extensions", "Reload every installed extension, e.g. after its files changed on disk."), + _function( + "download_model_weights", + "Download the model weights an extension needs. The download continues in the background.", + {"extension_id": _EXTENSION_ID, "variant": _VARIANT}, + ["extension_id"], + ), + _function( + "delete_model_weights", + "Delete an extension's downloaded model weights from disk. The user is asked to confirm.", + {"extension_id": _EXTENSION_ID, "variant": _VARIANT}, + ["extension_id"], + ), + _function( + "delete_workflow", + "Delete a saved workflow. The user is asked to confirm.", + {"workflow_id": _WORKFLOW_ID}, + ["workflow_id"], + ), + _function( + "open_workflow", + "Open a workflow in the workflow editor, without leaving the current page.", + {"workflow_id": _WORKFLOW_ID}, + ["workflow_id"], + ), + _function( + "set_setting", + "Change a UI setting. uiScale is a number (1 = 100%); useAtkinsonFont and showRamIndicator are booleans.", + { + "key": {"type": "string", "enum": ["uiScale", "useAtkinsonFont", "showRamIndicator"]}, + "value": {"type": ["number", "boolean"], "description": "A number for uiScale, a boolean otherwise."}, + }, + ["key", "value"], + ), + _function( + "set_storage_folder", + "Change where Modly stores models, the workspace or workflows. The user is asked to confirm.", + { + "kind": {"type": "string", "enum": ["models", "workspace", "workflows"]}, + "path": {"type": "string", "description": "Absolute path of the new folder."}, + }, + ["kind", "path"], + ), + _function("undo_mesh", "Undo the last change to the current mesh in the viewer."), + _function("redo_mesh", "Redo the last undone change to the current mesh in the viewer."), + _function( + "navigate", + "Switch the app to another page. Use it ONLY when the user explicitly asks to go to a page; " + "every other tool works from any page.", + {"page": {"type": "string", "enum": ["generate", "workflows", "models", "settings"]}}, + ["page"], + ), +] + +# Seconds the app has to run a client tool that needs the user's click. +CONFIRM_TIMEOUT = 600.0 +_NO_ANSWER = "Error: The app did not answer in time." + +# call_id → the turn waiting for the app's answer (see /agent/tool-result). +_pending_client_tools: dict[str, asyncio.Future] = {} + + # Input kinds the agent may pick, mapped to the real Modly source-node types. # Keep this in sync with the node palette in WorkflowsPage.tsx. INPUT_NODES = { @@ -374,11 +505,40 @@ async def _run_agent_op(client: httpx.AsyncClient, op: MeshOp, arguments: dict) return text, payload +def _extension_lines(extensions: list[dict]) -> list[str]: + return [ + f"- {e['id']} ({e.get('input', '?')}→{e.get('output', '?')}): {e.get('name', e['id'])}" + for e in extensions if e.get("id") + ] + + +def _id_name_lines(items: list[dict]) -> list[str]: + return [f"- {i['id']}: {i.get('name') or i['id']}" for i in items if i.get("id")] + + +_STORAGE_LABELS = { + "modelsDir": "Models folder", + "workspaceDir": "Workspace folder", + "workflowsDir": "Workflows folder", + "extensionsDir": "Extensions folder", +} + + +def _settings_lines(context: dict) -> list[str]: + """The storage folders and UI options the app sent, one per line.""" + storage = context.get("storage") or {} + ui = context.get("ui") or {} + lines = [f"{label}: {storage[key]}" for key, label in _STORAGE_LABELS.items() if storage.get(key)] + lines += [f"{key}: {_show(value)}" for key, value in ui.items() if value is not None] + return lines + + async def execute_tool( - name: str, arguments: dict, context: dict, *, timeout: float = 60.0, + name: str, arguments: dict, context: dict, *, timeout: float = 60.0, has_image: bool = False, ) -> tuple[str, dict | None]: """Execute a tool and return (result_text, action_payload). action_payload carries data the frontend needs to react (e.g. new mesh URL). + has_image: the user's last message carries an image (generate_from_image). """ async with httpx.AsyncClient(timeout=timeout) as client: try: @@ -459,6 +619,34 @@ async def execute_tool( payload = {"type": "run_workflow", "workflow_id": workflow_id, "workflow_name": match["name"]} return f"Executing workflow '{match['name']}'…", payload + elif name == "list_extensions": + lines = _extension_lines(context.get("extensions") or []) + if not lines: + return "No extensions installed.", None + return "Installed extensions:\n" + "\n".join(lines), None + + elif name == "get_settings": + lines = _settings_lines(context) + if not lines: + return "No settings were sent by the app.", None + return "\n".join(lines), None + + elif name == "generate_from_image": + if not has_image: + return "No image is attached. Ask the user to attach the image to generate from.", None + model_id = arguments.get("model_id") or None + models = context.get("models") or [] + label = "the selected model" + if model_id and models: + match = next((m for m in models if m.get("id") == model_id), None) + if not match: + ids = ", ".join(m["id"] for m in models if m.get("id")) + return f"Model '{model_id}' is not downloaded. Use one of: {ids}.", None + label = match.get("name") or model_id + elif model_id: + label = model_id + return f"Generating a 3D model with {label}…", {"type": "generate_from_image", "model_id": model_id} + elif name == "create_workflow": steps = arguments.get("steps") or [] if not steps: @@ -861,6 +1049,8 @@ async def _chat_events(request: AgentChatRequest): token / thinking — answer and reasoning text, in pieces tool_start / tool_done — around each tool call + client_tool — between them, for a tool the app runs: the turn waits for + its answer on POST /agent/tool-result with the same call_id done — always last: the final answer, every action and the full reasoning """ messages: list[dict] = [{"role": "system", "content": SYSTEM_PROMPT}] @@ -868,6 +1058,8 @@ async def _chat_events(request: AgentChatRequest): # Inject scene context so the LLM knows current state if request.context: ctx_lines = [] + if request.context.get("page"): + ctx_lines.append(f"Current page: {request.context['page']}") if request.context.get("currentMeshPath"): ctx_lines.append(f"Current mesh path: {request.context['currentMeshPath']}") if request.context.get("meshTriangles"): @@ -878,19 +1070,17 @@ async def _chat_events(request: AgentChatRequest): "content": "Scene context:\n" + "\n".join(ctx_lines), }) - extensions = request.context.get("extensions") or [] - if extensions: - ext_lines = [ - f"- {e['id']} ({e.get('input', '?')}→{e.get('output', '?')}): {e.get('name', e['id'])}" - for e in extensions - ] - messages.append({ - "role": "system", - "content": ( - "Available extensions (use the exact id when creating workflows):\n" - + "\n".join(ext_lines) - ), - }) + sections = [ + ("Available extensions (use the exact id when creating workflows)", + _extension_lines(request.context.get("extensions") or [])), + ("Downloaded generation models (use the exact id with generate_from_image)", + _id_name_lines(request.context.get("models") or [])), + ("Workflows", _id_name_lines(request.context.get("workflows") or [])), + ("Settings", _settings_lines(request.context)), + ] + for title, lines in sections: + if lines: + messages.append({"role": "system", "content": f"{title}:\n" + "\n".join(lines)}) # ONE system message, first. Qwen3.5's chat template raises "System message # must be at the beginning", which llama-server returns as a flat HTTP 400; @@ -924,6 +1114,10 @@ async def _chat_events(request: AgentChatRequest): for m in request.messages: messages.append(_user_entry(m, send_images)) + # The image generate_from_image would use: the frontend takes it from the + # user's last message, as run_workflow does. + last_user = next((m for m in reversed(request.messages) if m.role == "user"), None) + has_image = bool(last_user and last_user.images) actions_done: list[ActionDone] = [] all_thinking: list[str] = [] @@ -988,17 +1182,43 @@ async def post() -> httpx.Response: break arguments = _tool_arguments(fn.get("arguments")) yield {"type": "tool_start", "id": tc["id"], "tool": fn["name"], "arguments": arguments} - try: - result_text, payload = await asyncio.wait_for( - execute_tool(fn["name"], arguments, request.context, timeout=request.tool_timeout), - request.tool_timeout, - ) - except asyncio.TimeoutError: - # Cancelling the call drops its HTTP request, but a mesh op - # the API already started runs to its end in a worker - # thread; its result is never shown. - stopped = _tool_stop("tool_timeout", fn["name"], request.tool_timeout) - result_text, payload = stopped["message"], None + if fn["name"] in CLIENT_TOOLS: + # The app runs it and answers through /agent/tool-result. + confirm = CLIENT_TOOLS[fn["name"]] + call_id = uuid.uuid4().hex + answer = asyncio.get_running_loop().create_future() + _pending_client_tools[call_id] = answer + try: + yield { + "type": "client_tool", "id": tc["id"], "call_id": call_id, + "tool": fn["name"], "arguments": arguments, "confirm": confirm, + } + # A confirmation waits on the user's click, not on the tool. + limit = CONFIRM_TIMEOUT if confirm else request.tool_timeout + try: + result_text = await asyncio.wait_for(answer, limit) + except asyncio.TimeoutError: + result_text = _NO_ANSWER + finally: + # Also runs when the client leaves: nobody will answer. + _pending_client_tools.pop(call_id, None) + answer.cancel() + payload = None + else: + try: + result_text, payload = await asyncio.wait_for( + execute_tool( + fn["name"], arguments, request.context, + timeout=request.tool_timeout, has_image=has_image, + ), + request.tool_timeout, + ) + except asyncio.TimeoutError: + # Cancelling the call drops its HTTP request, but a mesh op + # the API already started runs to its end in a worker + # thread; its result is never shown. + stopped = _tool_stop("tool_timeout", fn["name"], request.tool_timeout) + result_text, payload = stopped["message"], None actions_done.append(ActionDone(tool=fn["name"], result=result_text, payload=payload, error=stopped)) yield { "type": "tool_done", "id": tc["id"], "tool": fn["name"], @@ -1026,6 +1246,22 @@ async def post() -> httpx.Response: yield _done(final, actions_done, all_thinking) +class ToolResultRequest(BaseModel): + call_id: str + result: str + error: bool = False + + +@router.post("/tool-result") +async def agent_tool_result(req: ToolResultRequest): + """The app's answer to a client_tool event; the waiting turn resumes with it.""" + answer = _pending_client_tools.get(req.call_id) + if answer is None or answer.done(): + raise HTTPException(status_code=404, detail="No tool call is waiting for this call_id.") + answer.set_result(f"Error: {req.result}" if req.error else req.result) + return {"ok": True} + + @router.post("/chat", response_model=AgentChatResponse) async def agent_chat(request: AgentChatRequest): """The whole turn at once, for callers that don't read a stream.""" diff --git a/api/tests/test_agent_client_tools.py b/api/tests/test_agent_client_tools.py new file mode 100644 index 000000000..999a9ce01 --- /dev/null +++ b/api/tests/test_agent_client_tools.py @@ -0,0 +1,365 @@ +import asyncio +import json +import unittest +from unittest import mock + +import httpx +from fastapi import HTTPException + +import routers.agent as agent +from tests.test_agent_streaming import _call, _external, _Scripted, _sse + + +def _client_round(name: str, arguments: dict | None = None, call_id: str = "c1") -> httpx.Response: + return _sse(_call(0, id=call_id, function={"name": name, "arguments": json.dumps(arguments or {})})) + + +async def _answer(call_id: str, result: str, error: bool = False) -> dict: + return await agent.agent_tool_result(agent.ToolResultRequest(call_id=call_id, result=result, error=error)) + + +def _run_turn(scripted: _Scripted, request: agent.AgentChatRequest, on_client_tool) -> list[dict]: + """Drive a turn, letting `on_client_tool(event)` answer each client tool + the way the panel would (it may be a coroutine function).""" + + async def collect(): + events = [] + async for event in agent._chat_events(request): + events.append(event) + if event["type"] == "client_tool": + answered = on_client_tool(event) + if asyncio.iscoroutine(answered): + await answered + return events + + with mock.patch.object(agent.httpx, "AsyncClient", scripted): + return asyncio.run(collect()) + + +class ClientToolBridgeTests(unittest.TestCase): + def tearDown(self) -> None: + agent._pending_client_tools.clear() + + def test_client_tool_is_handed_to_the_app_and_its_answer_reaches_the_model(self) -> None: + scripted = _Scripted( + _client_round("install_extension", {"github_url": "https://github.com/a/b"}), + _sse({"content": "Installed."}), + ) + execute = mock.AsyncMock(return_value=("never", None)) + answers = [] + + async def answer(event): + answers.append(await _answer(event["call_id"], "Installed b v1.0.")) + + with mock.patch.object(agent, "execute_tool", execute): + events = _run_turn(scripted, _external(), answer) + + execute.assert_not_awaited() + self.assertEqual([e["type"] for e in events], ["tool_start", "client_tool", "tool_done", "token", "done"]) + handoff = events[1] + self.assertEqual( + {k: v for k, v in handoff.items() if k != "call_id"}, + { + "type": "client_tool", "id": "c1", "tool": "install_extension", + "arguments": {"github_url": "https://github.com/a/b"}, "confirm": False, + }, + ) + self.assertRegex(handoff["call_id"], r"^[0-9a-f]{32}$") + self.assertEqual(answers, [{"ok": True}]) + self.assertEqual(events[2]["result"], "Installed b v1.0.") + self.assertEqual(events[-1]["actions"][0]["tool"], "install_extension") + self.assertEqual( + scripted.bodies[1]["messages"][-1], + {"role": "tool", "tool_call_id": "c1", "content": "Installed b v1.0."}, + ) + self.assertEqual(agent._pending_client_tools, {}) + + def test_answer_while_the_turn_is_waiting(self) -> None: + scripted = _Scripted(_client_round("reload_extensions"), _sse({"content": "ok"})) + + def answer_later(event): + async def later(): + await asyncio.sleep(0.01) + await _answer(event["call_id"], "Reloaded.") + asyncio.ensure_future(later()) + + events = _run_turn(scripted, _external(), answer_later) + + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Reloaded.") + self.assertFalse(events[-1]["error"]) + + def test_destructive_tool_asks_for_confirmation_and_a_decline_is_relayed(self) -> None: + scripted = _Scripted( + _client_round("uninstall_extension", {"extension_id": "x"}), + _sse({"content": "Kept it."}), + ) + events = _run_turn(scripted, _external(), lambda e: _answer(e["call_id"], "The user declined.")) + + handoff = next(e for e in events if e["type"] == "client_tool") + self.assertTrue(handoff["confirm"]) + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "The user declined.") + + def test_error_answer_is_prefixed(self) -> None: + scripted = _Scripted(_client_round("reload_extensions"), _sse({"content": "ok"})) + events = _run_turn(scripted, _external(), lambda e: _answer(e["call_id"], "disk full", error=True)) + + self.assertEqual(events[2]["result"], "Error: disk full") + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Error: disk full") + + def test_unknown_or_already_answered_call_is_404(self) -> None: + with self.assertRaises(HTTPException) as caught: + asyncio.run(_answer("nope", "x")) + self.assertEqual(caught.exception.status_code, 404) + + seen = [] + + async def answer_twice(event): + await _answer(event["call_id"], "first") + try: + await _answer(event["call_id"], "second") + except HTTPException as e: + seen.append(e.status_code) + + _run_turn(_Scripted(_client_round("reload_extensions"), _sse({"content": "ok"})), _external(), answer_twice) + self.assertEqual(seen, [404]) + + def test_unanswered_tool_times_out_and_the_loop_continues(self) -> None: + scripted = _Scripted(_client_round("reload_extensions"), _sse({"content": "It did not answer."})) + events = _run_turn(scripted, _external().model_copy(update={"tool_timeout": 0.05}), lambda e: None) + + finish = next(e for e in events if e["type"] == "tool_done") + self.assertEqual(finish["result"], "Error: The app did not answer in time.") + self.assertIsNone(finish["error"]) + done = events[-1] + self.assertFalse(done["error"]) + self.assertIsNone(done["stopped"]) + self.assertEqual(done["message"], "It did not answer.") + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Error: The app did not answer in time.") + self.assertEqual(agent._pending_client_tools, {}) + + def test_confirm_tools_wait_for_the_confirm_cap_not_the_tool_timeout(self) -> None: + scripted = _Scripted(_client_round("delete_workflow", {"workflow_id": "w"}), _sse({"content": "ok"})) + + def answer_after_tool_timeout(event): + async def later(): + await asyncio.sleep(0.15) + await _answer(event["call_id"], "Deleted.") + asyncio.ensure_future(later()) + + with mock.patch.object(agent, "CONFIRM_TIMEOUT", 5.0): + _run_turn(scripted, _external().model_copy(update={"tool_timeout": 0.05}), answer_after_tool_timeout) + + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Deleted.") + + def test_confirm_wait_is_capped(self) -> None: + self.assertEqual(agent.CONFIRM_TIMEOUT, 600) + scripted = _Scripted(_client_round("delete_workflow", {"workflow_id": "w"}), _sse({"content": "ok"})) + with mock.patch.object(agent, "CONFIRM_TIMEOUT", 0.05): + events = _run_turn(scripted, _external(), lambda e: None) + + self.assertEqual(events[2]["result"], "Error: The app did not answer in time.") + + def test_client_tools_count_against_the_budget(self) -> None: + both = _sse( + _call(0, id="a", function={"name": "reload_extensions", "arguments": "{}"}), + _call(1, id="b", function={"name": "undo_mesh", "arguments": "{}"}), + ) + events = _run_turn( + _Scripted(both), _external().model_copy(update={"tool_budget": 1}), + lambda e: _answer(e["call_id"], "ok"), + ) + + self.assertEqual(sum(e["type"] == "client_tool" for e in events), 1) + self.assertEqual(events[-1]["stopped"]["code"], "tool_budget_exhausted") + self.assertEqual(events[-1]["stopped"]["tool"], "undo_mesh") + + def test_closing_the_stream_while_a_tool_waits_cancels_it(self) -> None: + async def leave_at_handoff(): + stream = agent._chat_events(_external()) + async for event in stream: + if event["type"] == "client_tool": + future = agent._pending_client_tools[event["call_id"]] + await stream.aclose() + return event, future + + with mock.patch.object(agent.httpx, "AsyncClient", _Scripted(_client_round("reload_extensions"))): + event, future = asyncio.run(leave_at_handoff()) + + self.assertTrue(future.cancelled()) + self.assertNotIn(event["call_id"], agent._pending_client_tools) + + def test_cancelling_the_turn_while_it_awaits_the_app_cancels_the_future(self) -> None: + async def cancel_while_waiting(): + stream = agent._chat_events(_external()) + while (event := await stream.__anext__())["type"] != "client_tool": + pass + future = agent._pending_client_tools[event["call_id"]] + waiting = asyncio.ensure_future(stream.__anext__()) + await asyncio.sleep(0.01) + waiting.cancel() + with self.assertRaises(asyncio.CancelledError): + await waiting + return event, future + + with mock.patch.object(agent.httpx, "AsyncClient", _Scripted(_client_round("reload_extensions"))): + event, future = asyncio.run(cancel_while_waiting()) + + self.assertTrue(future.cancelled()) + self.assertNotIn(event["call_id"], agent._pending_client_tools) + + +_CLIENT_TOOL_ARGS = { + "install_extension": ["github_url"], + "uninstall_extension": ["extension_id"], + "reload_extensions": [], + "download_model_weights": ["extension_id", "variant"], + "delete_model_weights": ["extension_id", "variant"], + "delete_workflow": ["workflow_id"], + "open_workflow": ["workflow_id"], + "set_setting": ["key", "value"], + "set_storage_folder": ["kind", "path"], + "undo_mesh": [], + "redo_mesh": [], + "navigate": ["page"], +} + + +class ClientToolDeclarationTests(unittest.TestCase): + def _tool(self, name: str) -> dict: + return next(t["function"] for t in agent.TOOLS if t["function"]["name"] == name) + + def test_every_client_tool_is_declared_with_its_confirmation(self) -> None: + self.assertEqual(set(agent.CLIENT_TOOLS), set(_CLIENT_TOOL_ARGS)) + confirmed = {name for name, confirm in agent.CLIENT_TOOLS.items() if confirm} + self.assertEqual( + confirmed, + {"uninstall_extension", "delete_model_weights", "delete_workflow", "set_storage_folder"}, + ) + + def test_every_client_tool_has_a_schema(self) -> None: + for name, args in _CLIENT_TOOL_ARGS.items(): + with self.subTest(name=name): + fn = self._tool(name) + self.assertTrue(fn["description"]) + self.assertEqual(sorted(fn["parameters"]["properties"]), sorted(args)) + + def test_enums(self) -> None: + props = self._tool("set_setting")["parameters"]["properties"] + self.assertEqual(props["key"]["enum"], ["uiScale", "useAtkinsonFont", "showRamIndicator"]) + props = self._tool("set_storage_folder")["parameters"]["properties"] + self.assertEqual(props["kind"]["enum"], ["models", "workspace", "workflows"]) + nav = self._tool("navigate") + self.assertEqual(nav["parameters"]["properties"]["page"]["enum"], ["generate", "workflows", "models", "settings"]) + self.assertIn("ONLY", nav["description"]) + + def test_tool_names_are_unique(self) -> None: + names = [t["function"]["name"] for t in agent.TOOLS] + self.assertEqual(len(names), len(set(names))) + + def test_system_prompt_lists_the_new_tools(self) -> None: + for name in [*_CLIENT_TOOL_ARGS, "list_extensions", "get_settings", "generate_from_image"]: + with self.subTest(name=name): + self.assertIn(name, agent.SYSTEM_PROMPT) + self.assertIn("declined", agent.SYSTEM_PROMPT) + + +_CONTEXT = { + "extensions": [{"id": "trellis/gen", "name": "Trellis", "input": "image", "output": "mesh"}], + "models": [{"id": "trellis", "name": "Trellis 2"}, {"id": "sf3d", "name": "SF3D"}], + "storage": { + "modelsDir": "D:/models", "workspaceDir": "D:/ws", + "workflowsDir": "D:/wf", "extensionsDir": "D:/ext", + }, + "ui": {"uiScale": 1.1, "useAtkinsonFont": False, "showRamIndicator": True}, +} + + +def _run(name: str, arguments: dict | None = None, context: dict | None = None, **kwargs): + return asyncio.run(agent.execute_tool(name, arguments or {}, _CONTEXT if context is None else context, **kwargs)) + + +class ServerToolTests(unittest.TestCase): + def test_list_extensions(self) -> None: + text, payload = _run("list_extensions") + self.assertIn("- trellis/gen (image→mesh): Trellis", text) + self.assertIsNone(payload) + self.assertIn("No extensions", _run("list_extensions", context={})[0]) + + def test_get_settings(self) -> None: + text, payload = _run("get_settings") + for expected in ("D:/models", "D:/ws", "D:/wf", "D:/ext", "uiScale: 1.1", "useAtkinsonFont: false", "showRamIndicator: true"): + self.assertIn(expected, text) + self.assertIsNone(payload) + self.assertIn("No settings", _run("get_settings", context={})[0]) + + def test_generate_from_image_needs_an_attached_image(self) -> None: + text, payload = _run("generate_from_image") + self.assertIsNone(payload) + self.assertIn("attach", text.lower()) + + def test_generate_from_image_hands_the_run_to_the_app(self) -> None: + text, payload = _run("generate_from_image", {"model_id": "sf3d"}, has_image=True) + self.assertEqual(payload, {"type": "generate_from_image", "model_id": "sf3d"}) + self.assertIn("SF3D", text) + + text, payload = _run("generate_from_image", has_image=True) + self.assertEqual(payload, {"type": "generate_from_image", "model_id": None}) + + def test_generate_from_image_rejects_an_unknown_model(self) -> None: + text, payload = _run("generate_from_image", {"model_id": "ghost"}, has_image=True) + self.assertIsNone(payload) + self.assertIn("ghost", text) + self.assertIn("trellis", text) + + def test_the_turn_tells_the_tool_whether_the_last_user_message_has_an_image(self) -> None: + def turn(messages): + request = _external().model_copy(update={"messages": messages}) + tool = mock.AsyncMock(return_value=("ok", None)) + with mock.patch.object(agent, "execute_tool", tool): + _run_turn(_Scripted(_client_round("generate_from_image"), _sse({"content": "ok"})), request, None) + return tool.await_args.kwargs["has_image"] + + img = "data:image/png;base64,AAAA" + self.assertTrue(turn([agent.ChatMessage(role="user", content="go", images=[img])])) + self.assertFalse(turn([ + agent.ChatMessage(role="user", content="a", images=[img]), + agent.ChatMessage(role="assistant", content="b"), + agent.ChatMessage(role="user", content="now generate"), + ])) + + +class ContextRenderingTests(unittest.TestCase): + def test_context_is_rendered_into_the_single_system_message(self) -> None: + context = { + **_CONTEXT, + "page": "models", + "workflows": [{"id": "wf1", "name": "My Workflow"}], + "currentMeshPath": "Default/mesh.glb", + } + scripted = _Scripted(_sse({"content": "hi"})) + _run_turn(scripted, _external(context), None) + + messages = scripted.bodies[0]["messages"] + self.assertEqual(sum(m["role"] == "system" for m in messages), 1) + system = messages[0]["content"] + for expected in ( + "Current page: models", + "- trellis: Trellis 2", + "- sf3d: SF3D", + "- wf1: My Workflow", + "D:/models", "D:/ws", "D:/wf", "D:/ext", + "uiScale: 1.1", "useAtkinsonFont: false", "showRamIndicator: true", + "Current mesh path: Default/mesh.glb", + "- trellis/gen (image→mesh): Trellis", + ): + with self.subTest(expected=expected): + self.assertIn(expected, system) + + def test_empty_context_adds_nothing(self) -> None: + scripted = _Scripted(_sse({"content": "hi"})) + _run_turn(scripted, _external(), None) + self.assertEqual(scripted.bodies[0]["messages"][0]["content"], agent.SYSTEM_PROMPT) + + +if __name__ == "__main__": + unittest.main() From be528925fde4e07b55675eb559534312966f6599 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:03:44 +0100 Subject: [PATCH 03/13] feat(agent): persist agent conversations as JSON files over IPC Adds window.electron.agent.conversations (list/read/save/remove), one file per conversation in /agent/conversations. Ids are checked against /^[a-z0-9-]{1,64}$/ and writes go through a temp file plus rename. Pure title/sort/settle helpers live in src/shared/utils/agentConversations. --- electron/main/agent-conversations.test.ts | 109 ++++++++++++++++++++ electron/main/agent-conversations.ts | 93 +++++++++++++++++ electron/main/ipc-handlers.ts | 9 ++ electron/preload/electron-api.ts | 15 +++ package.json | 2 +- src/shared/types/electron.d.ts | 11 ++ src/shared/utils/agentConversations.test.ts | 77 ++++++++++++++ src/shared/utils/agentConversations.ts | 81 +++++++++++++++ 8 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 electron/main/agent-conversations.test.ts create mode 100644 electron/main/agent-conversations.ts create mode 100644 src/shared/utils/agentConversations.test.ts create mode 100644 src/shared/utils/agentConversations.ts diff --git a/electron/main/agent-conversations.test.ts b/electron/main/agent-conversations.test.ts new file mode 100644 index 000000000..7d8387778 --- /dev/null +++ b/electron/main/agent-conversations.test.ts @@ -0,0 +1,109 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { mkdtemp, readdir, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { + createConversationStorage, + isConversation, + isValidConversationId, + sortSummaries, +} from './agent-conversations' + +async function withDir(fn: (dir: string) => Promise): Promise { + const root = await mkdtemp(join(tmpdir(), 'modly-conv-')) + try { + await fn(join(root, 'agent', 'conversations')) + } finally { + await rm(root, { recursive: true, force: true }) + } +} + +const conv = (id: string, updatedAt: number, title = id) => + ({ id, title, createdAt: 1, updatedAt, messages: [{ id: 'u', role: 'user', content: title }] }) + +test('conversation ids: uuids pass, path tricks and odd characters do not', () => { + assert.equal(isValidConversationId('3f2a9c1e-8b7d-4e6f-a5b4-c3d2e1f0a9b8'), true) + assert.equal(isValidConversationId('a'), true) + assert.equal(isValidConversationId('a'.repeat(64)), true) + assert.equal(isValidConversationId('a'.repeat(65)), false) + assert.equal(isValidConversationId(''), false) + assert.equal(isValidConversationId('../secrets'), false) + assert.equal(isValidConversationId('..\\secrets'), false) + assert.equal(isValidConversationId('a/b'), false) + assert.equal(isValidConversationId('ABC'), false) + assert.equal(isValidConversationId('abc.json'), false) + assert.equal(isValidConversationId('abc\n'), false) + assert.equal(isValidConversationId(42), false) +}) + +test('isConversation checks the stored shape', () => { + assert.equal(isConversation(conv('abc-1', 2)), true) + assert.equal(isConversation({ ...conv('abc-1', 2), id: '../x' }), false) + assert.equal(isConversation({ ...conv('abc-1', 2), messages: 'nope' }), false) + assert.equal(isConversation({ ...conv('abc-1', 2), updatedAt: '2' }), false) + assert.equal(isConversation(null), false) +}) + +test('sortSummaries puts the newest first', () => { + const sorted = sortSummaries([conv('a', 10), conv('b', 30), conv('c', 20)]) + assert.deepEqual(sorted.map((s) => s.id), ['b', 'c', 'a']) +}) + +test('list is empty before anything is saved', async () => { + await withDir(async (dir) => { + assert.deepEqual(await createConversationStorage(dir).list(), []) + }) +}) + +test('save, read, list newest first, remove', async () => { + await withDir(async (dir) => { + const storage = createConversationStorage(dir) + await storage.save(conv('old', 10, 'Old')) + await storage.save(conv('new', 20, 'New')) + assert.deepEqual(await storage.read('old'), conv('old', 10, 'Old')) + assert.deepEqual(await storage.list(), [ + { id: 'new', title: 'New', updatedAt: 20 }, + { id: 'old', title: 'Old', updatedAt: 10 }, + ]) + // No temp file is left behind. + assert.deepEqual((await readdir(dir)).sort(), ['new.json', 'old.json']) + + await storage.remove('old') + assert.equal(await storage.read('old'), null) + assert.deepEqual((await storage.list()).map((s) => s.id), ['new']) + }) +}) + +test('saving again overwrites the file', async () => { + await withDir(async (dir) => { + const storage = createConversationStorage(dir) + await storage.save(conv('a', 1, 'First')) + await storage.save(conv('a', 2, 'Second')) + assert.equal((await storage.read('a'))?.title, 'Second') + }) +}) + +test('invalid ids and shapes are refused', async () => { + await withDir(async (dir) => { + const storage = createConversationStorage(dir) + await assert.rejects(storage.save({ ...conv('a', 1), id: '../escape' })) + await assert.rejects(storage.save({ id: 'a' })) + await assert.rejects(storage.remove('../escape')) + assert.equal(await storage.read('../escape'), null) + }) +}) + +test('corrupted or mismatched files are skipped', async () => { + await withDir(async (dir) => { + const storage = createConversationStorage(dir) + await storage.save(conv('good', 5)) + await writeFile(join(dir, 'broken.json'), '{not json', 'utf-8') + await writeFile(join(dir, 'other.json'), JSON.stringify(conv('different', 9)), 'utf-8') + await writeFile(join(dir, 'notes.txt'), 'hi', 'utf-8') + assert.deepEqual((await storage.list()).map((s) => s.id), ['good']) + assert.equal(await storage.read('broken'), null) + assert.equal(await storage.read('other'), null) + }) +}) diff --git a/electron/main/agent-conversations.ts b/electron/main/agent-conversations.ts new file mode 100644 index 000000000..24c130fd2 --- /dev/null +++ b/electron/main/agent-conversations.ts @@ -0,0 +1,93 @@ +// Agent conversations on disk: one JSON file per conversation in +// /agent/conversations/{id}.json. + +import { join } from 'path' +import { mkdir, readFile, readdir, rename, rm, writeFile } from 'fs/promises' + +export interface ConversationSummary { + id: string + title: string + updatedAt: number +} + +export interface Conversation extends ConversationSummary { + createdAt: number + messages: unknown[] +} + +export const CONVERSATION_ID_RE = /^[a-z0-9-]{1,64}$/ + +/** Ids name files on disk: anything else could escape the conversations folder. */ +export function isValidConversationId(id: unknown): id is string { + return typeof id === 'string' && CONVERSATION_ID_RE.test(id) +} + +/** Shape check for a conversation read from disk or received over IPC. */ +export function isConversation(value: unknown): value is Conversation { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false + const c = value as Record + return isValidConversationId(c.id) + && typeof c.title === 'string' + && typeof c.createdAt === 'number' + && typeof c.updatedAt === 'number' + && Array.isArray(c.messages) +} + +/** Newest first; ties keep a stable order by id. */ +export function sortSummaries(list: readonly T[]): T[] { + return [...list].sort((a, b) => (b.updatedAt - a.updatedAt) || a.id.localeCompare(b.id)) +} + +export function createConversationStorage(dir: string) { + const fileFor = (id: string): string => join(dir, `${id}.json`) + + async function readConversation(id: string): Promise { + if (!isValidConversationId(id)) return null + try { + const parsed: unknown = JSON.parse(await readFile(fileFor(id), 'utf-8')) + return isConversation(parsed) && parsed.id === id ? parsed : null + } catch { + return null + } + } + + return { + async list(): Promise { + let files: string[] + try { + files = await readdir(dir) + } catch { + return [] + } + const summaries: ConversationSummary[] = [] + for (const file of files) { + if (!file.endsWith('.json')) continue + const c = await readConversation(file.slice(0, -'.json'.length)) + if (c) summaries.push({ id: c.id, title: c.title, updatedAt: c.updatedAt }) + } + return sortSummaries(summaries) + }, + + read: readConversation, + + async save(conversation: unknown): Promise { + if (!isConversation(conversation)) throw new Error('Invalid conversation') + await mkdir(dir, { recursive: true }) + const target = fileFor(conversation.id) + // Write then rename: a crash mid-write never leaves a truncated file. + const tmp = `${target}.${process.pid}.${Date.now()}.tmp` + await writeFile(tmp, JSON.stringify(conversation), 'utf-8') + try { + await rename(tmp, target) + } catch (err) { + await rm(tmp, { force: true }) + throw err + } + }, + + async remove(id: string): Promise { + if (!isValidConversationId(id)) throw new Error('Invalid conversation id') + await rm(fileFor(id), { force: true }) + }, + } +} diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 33a9b2409..039705780 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -50,6 +50,7 @@ import { checkSetupNeeded, markSetupDone, runFullSetup, getVenvPythonExe, ensure import { logger } from './logger' import { getProcessRunner, getPythonProcessRunner, getExtPythonExe, terminateProcessRunner, terminateAllProcessRunners } from './process-runner' import { getBuiltinExtensionsDir } from './builtin-sync' +import { createConversationStorage } from './agent-conversations' import { spawn, execFile } from 'child_process' import { EXT_INCOMPLETE_MARKER, @@ -1078,6 +1079,14 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe return result.canceled ? null : result.filePaths[0] }) + // Agent conversations: one JSON file each under /agent/conversations. + // Ids are validated in the storage layer, so no path escapes that folder. + const conversations = createConversationStorage(join(app.getPath('userData'), 'agent', 'conversations')) + ipcMain.handle('agent:conversations:list', () => conversations.list()) + ipcMain.handle('agent:conversations:read', (_, id: string) => conversations.read(id)) + ipcMain.handle('agent:conversations:save', (_, conversation: unknown) => conversations.save(conversation)) + ipcMain.handle('agent:conversations:remove', (_, id: string) => conversations.remove(id)) + // Add a local GGUF to the agent's models folder. Picking and copying both // happen here, so the renderer never hands main an arbitrary path to copy. ipcMain.handle('agent:addModel', async (): Promise<{ success: boolean; cancelled?: boolean; fileName?: string; error?: string }> => { diff --git a/electron/preload/electron-api.ts b/electron/preload/electron-api.ts index 04b50ca06..d11863beb 100644 --- a/electron/preload/electron-api.ts +++ b/electron/preload/electron-api.ts @@ -5,6 +5,10 @@ import type { AssetLibraryReadRequest, AssetLibraryReadResult, } from '../../src/shared/types/assetLibrary' +import type { + Conversation as AgentConversation, + ConversationSummary as AgentConversationSummary, +} from '../main/agent-conversations' export interface IpcRendererLike { invoke(channel: string, ...args: unknown[]): Promise @@ -120,6 +124,17 @@ export function createElectronApi(ipcRenderer: IpcRendererLike, webFrame: WebFra // Opens a file picker and copies the chosen .gguf into the agent's models folder. addModel: (): Promise<{ success: boolean; cancelled?: boolean; fileName?: string; error?: string }> => ipcRenderer.invoke('agent:addModel') as Promise<{ success: boolean; cancelled?: boolean; fileName?: string; error?: string }>, + // Persisted chat conversations (/agent/conversations/{id}.json). + conversations: { + list: (): Promise => + ipcRenderer.invoke('agent:conversations:list') as Promise, + read: (id: string): Promise => + ipcRenderer.invoke('agent:conversations:read', id) as Promise, + save: (conversation: AgentConversation): Promise => + ipcRenderer.invoke('agent:conversations:save', conversation) as Promise, + remove: (id: string): Promise => + ipcRenderer.invoke('agent:conversations:remove', id) as Promise, + }, }, // Settings diff --git a/package.json b/package.json index 22e207ac9..a8289b931 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/shared/types/electron.d.ts b/src/shared/types/electron.d.ts index 2be5a5dc1..675827f62 100644 --- a/src/shared/types/electron.d.ts +++ b/src/shared/types/electron.d.ts @@ -8,6 +8,7 @@ import type { AssetLibraryReadRequest, AssetLibraryReadResult, } from './assetLibrary' +import type { Conversation, ConversationSummary } from '../utils/agentConversations' // ─── Extension types ────────────────────────────────────────────────────────── @@ -228,6 +229,16 @@ declare global { agent: { /** Opens a file picker and copies the chosen .gguf into the agent's models folder. */ addModel: () => Promise<{ success: boolean; cancelled?: boolean; fileName?: string; error?: string }> + /** Persisted chat conversations, one JSON file each in /agent/conversations. */ + conversations: { + /** Newest first. */ + list: () => Promise + /** null when the id is invalid, missing or the file is corrupted. */ + read: (id: string) => Promise + /** Rejects on an invalid id or shape. */ + save: (conversation: Conversation) => Promise + remove: (id: string) => Promise + } } cache: { clear: () => Promise<{ success: boolean; error?: string }> diff --git a/src/shared/utils/agentConversations.test.ts b/src/shared/utils/agentConversations.test.ts new file mode 100644 index 000000000..8ed04cc6a --- /dev/null +++ b/src/shared/utils/agentConversations.test.ts @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { + TITLE_MAX_LENGTH, + UNTITLED, + deriveTitle, + relativeDate, + settleMessages, + sortSummaries, + upsertSummary, +} from './agentConversations' + +test('title is the first user message, collapsed to one line', () => { + const title = deriveTitle([ + { role: 'assistant', content: 'Hello' }, + { role: 'user', content: ' make the\n\nmesh smoother ' }, + { role: 'user', content: 'second' }, + ]) + assert.equal(title, 'make the mesh smoother') +}) + +test('a long first message is cut to the title limit', () => { + const title = deriveTitle([{ role: 'user', content: 'x'.repeat(200) }]) + assert.equal(title.length, TITLE_MAX_LENGTH) + assert.ok(title.endsWith('…')) + assert.equal(deriveTitle([{ role: 'user', content: 'y'.repeat(TITLE_MAX_LENGTH) }]), 'y'.repeat(TITLE_MAX_LENGTH)) +}) + +test('no usable user message gives the placeholder title', () => { + assert.equal(deriveTitle([]), UNTITLED) + assert.equal(deriveTitle([{ role: 'assistant', content: 'hi' }, { role: 'user', content: ' ' }, null]), UNTITLED) +}) + +test('summaries sort newest first without mutating the input', () => { + const list = [ + { id: 'a', title: 'A', updatedAt: 10 }, + { id: 'b', title: 'B', updatedAt: 30 }, + { id: 'c', title: 'C', updatedAt: 20 }, + ] + assert.deepEqual(sortSummaries(list).map((s) => s.id), ['b', 'c', 'a']) + assert.deepEqual(list.map((s) => s.id), ['a', 'b', 'c']) +}) + +test('upsertSummary replaces an entry and moves it to the top', () => { + const list = [ + { id: 'b', title: 'B', updatedAt: 30 }, + { id: 'a', title: 'A', updatedAt: 10 }, + ] + assert.deepEqual(upsertSummary(list, { id: 'a', title: 'A2', updatedAt: 40 }), [ + { id: 'a', title: 'A2', updatedAt: 40 }, + { id: 'b', title: 'B', updatedAt: 30 }, + ]) + assert.equal(upsertSummary(list, { id: 'c', title: 'C', updatedAt: 1 }).length, 3) +}) + +test('settleMessages clears streaming and running flags', () => { + const settled = settleMessages([ + { id: 'u', role: 'user', content: 'hi' }, + { id: 'a', role: 'assistant', content: 'part', streaming: true, actions: [{ id: 't', tool: 'x', running: true }] }, + ]) + assert.deepEqual(settled, [ + { id: 'u', role: 'user', content: 'hi' }, + { id: 'a', role: 'assistant', content: 'part', streaming: false, actions: [{ id: 't', tool: 'x', running: false }] }, + ]) +}) + +test('relativeDate buckets by age', () => { + const now = Date.UTC(2026, 10, 12, 12, 0, 0) + assert.equal(relativeDate(now - 5_000, now), 'just now') + assert.equal(relativeDate(now - 5 * 60_000, now), '5m ago') + assert.equal(relativeDate(now - 3 * 3_600_000, now), '3h ago') + assert.equal(relativeDate(now - 2 * 86_400_000, now), '2d ago') + assert.equal(relativeDate(now + 60_000, now), 'just now') + assert.match(relativeDate(now - 30 * 86_400_000, now), /^Oct \d+$/) + assert.match(relativeDate(Date.UTC(2024, 0, 15), now), /2024/) +}) diff --git a/src/shared/utils/agentConversations.ts b/src/shared/utils/agentConversations.ts new file mode 100644 index 000000000..06c8e3dd0 --- /dev/null +++ b/src/shared/utils/agentConversations.ts @@ -0,0 +1,81 @@ +// Pure renderer-side logic for persisted agent conversations. File storage +// and id validation live in electron/main/agent-conversations.ts. + +export interface ConversationSummary { + id: string + title: string + /** Epoch milliseconds. */ + updatedAt: number +} + +export interface Conversation { + id: string + title: string + /** Epoch milliseconds. */ + createdAt: number + /** Epoch milliseconds. */ + updatedAt: number + /** AgentChat messages; attached images stay inline as data URLs. */ + messages: unknown[] +} + +export const TITLE_MAX_LENGTH = 60 +export const UNTITLED = 'New conversation' + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +/** The first user message, on one line, cut to TITLE_MAX_LENGTH characters. */ +export function deriveTitle(messages: readonly unknown[]): string { + for (const m of messages) { + if (!isRecord(m) || m.role !== 'user' || typeof m.content !== 'string') continue + const text = m.content.replace(/\s+/g, ' ').trim() + if (!text) continue + return text.length > TITLE_MAX_LENGTH ? `${text.slice(0, TITLE_MAX_LENGTH - 1).trimEnd()}…` : text + } + return UNTITLED +} + +/** Newest first; ties keep a stable order by id. */ +export function sortSummaries(list: readonly T[]): T[] { + return [...list].sort((a, b) => (b.updatedAt - a.updatedAt) || a.id.localeCompare(b.id)) +} + +/** Replace (or add) one summary and keep the list sorted. */ +export function upsertSummary(list: readonly ConversationSummary[], summary: ConversationSummary): ConversationSummary[] { + return sortSummaries([...list.filter((s) => s.id !== summary.id), summary]) +} + +/** + * A conversation saved mid-turn holds a streaming message and running tools. + * Nothing is streaming once it is loaded again, so settle them. + */ +export function settleMessages(messages: readonly unknown[]): unknown[] { + return messages.map((m) => { + if (!isRecord(m)) return m + const settled: Record = { ...m } + if (settled.streaming) settled.streaming = false + if (Array.isArray(settled.actions)) { + settled.actions = settled.actions.map((a) => (isRecord(a) && a.running ? { ...a, running: false } : a)) + } + return settled + }) +} + +const MINUTE = 60_000 +const HOUR = 60 * MINUTE +const DAY = 24 * HOUR + +/** "just now", "5m ago", "3h ago", "2d ago", then a short date. */ +export function relativeDate(ts: number, now: number = Date.now()): string { + const diff = Math.max(0, now - ts) + if (diff < MINUTE) return 'just now' + if (diff < HOUR) return `${Math.floor(diff / MINUTE)}m ago` + if (diff < DAY) return `${Math.floor(diff / HOUR)}h ago` + if (diff < 7 * DAY) return `${Math.floor(diff / DAY)}d ago` + const d = new Date(ts) + const opts: Intl.DateTimeFormatOptions = { month: 'short', day: 'numeric' } + if (d.getFullYear() !== new Date(now).getFullYear()) opts.year = 'numeric' + return d.toLocaleDateString('en-US', opts) +} From 4e791fd4ba54b1d387138d9604adebcfc2323089 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:03:44 +0100 Subject: [PATCH 04/13] feat(agent): global right-side agent panel The chat leaves the Generate page's basic/chat toggle and becomes the body of an AgentPanel next to
in MainLayout, so it pushes the app. The panel toggles from a TopBar button or Ctrl/Cmd+L, resizes from its left edge (320-640px) and remembers its open state and width. Messages move to agentConversationStore: they survive page changes and panel close, save to disk after 500ms of quiet, and the latest conversation reopens at startup. The header lists past conversations with open and delete. --- .../generate/components/WorkflowPanel.tsx | 112 +++---- .../components/agent/AgentChat.tsx} | 42 +-- src/shared/components/agent/AgentPanel.tsx | 273 ++++++++++++++++++ src/shared/components/layout/MainLayout.tsx | 5 +- src/shared/components/layout/TopBar.tsx | 24 ++ src/shared/stores/agentConversationStore.ts | 170 +++++++++++ src/shared/stores/agentPanelStore.ts | 43 +++ 7 files changed, 571 insertions(+), 98 deletions(-) rename src/{areas/generate/components/ChatPanel.tsx => shared/components/agent/AgentChat.tsx} (96%) create mode 100644 src/shared/components/agent/AgentPanel.tsx create mode 100644 src/shared/stores/agentConversationStore.ts create mode 100644 src/shared/stores/agentPanelStore.ts diff --git a/src/areas/generate/components/WorkflowPanel.tsx b/src/areas/generate/components/WorkflowPanel.tsx index 14b257815..d1be5e9f7 100644 --- a/src/areas/generate/components/WorkflowPanel.tsx +++ b/src/areas/generate/components/WorkflowPanel.tsx @@ -19,9 +19,6 @@ import type { Workflow, WFNode, WFEdge, ParamSchema } from '@shared/types/electr import { PICKER_LABELS, openParamPicker, resolvePickerIntent } from '@shared/utils/paramPicker' import { FloatInput, IntInput, PickerIcon } from '@shared/components/ui' import { isMissingWeightVariant, withWeightVariantAvailability } from '@shared/utils/weightVariants' -import ChatPanel from './ChatPanel' - -type PanelMode = 'basic' | 'chat' // ─── Constants ──────────────────────────────────────────────────────────────── @@ -610,30 +607,6 @@ function EmbeddedCanvas({ workflow, allExtensions }: { ) } -// ─── Mode toggle ────────────────────────────────────────────────────────────── - -function ModeToggle({ mode, onChange }: { mode: PanelMode; onChange: (m: PanelMode) => void }): JSX.Element { - return ( -
-
- {(['basic', 'chat'] as PanelMode[]).map((m) => ( - - ))} -
-
- ) -} - // ─── Main panel ─────────────────────────────────────────────────────────────── export default function WorkflowPanel() { @@ -642,7 +615,6 @@ export default function WorkflowPanel() { const loadExtensions = useExtensionsStore((s) => s.loadExtensions) const { navigate } = useNavStore() const [selectedId, setSelectedId] = useState(activeId) - const [mode, setMode] = useState('basic') const allExtensions = useMemo( () => buildAllWorkflowExtensions(modelExtensions, processExtensions), @@ -667,56 +639,44 @@ export default function WorkflowPanel() { return (
- {/* Mode toggle */} - - - {mode === 'chat' ? ( - <> -
- - - ) : ( - <> - {/* Header */} -
-

Workflow

-
-
- -
- {selectedId && ( - - )} -
+ {/* Header */} +
+

Workflow

+
+
+
- - {/* Canvas or empty state */} - {workflow ? ( - - - - ) : ( -
-

- No workflows yet.
Create one in the Workflows tab. -

-
+ {selectedId && ( + )} - +
+
+ + {/* Canvas or empty state */} + {workflow ? ( + + + + ) : ( +
+

+ No workflows yet.
Create one in the Workflows tab. +

+
)}
) diff --git a/src/areas/generate/components/ChatPanel.tsx b/src/shared/components/agent/AgentChat.tsx similarity index 96% rename from src/areas/generate/components/ChatPanel.tsx rename to src/shared/components/agent/AgentChat.tsx index 24d76f238..d155c80cb 100644 --- a/src/areas/generate/components/ChatPanel.tsx +++ b/src/shared/components/agent/AgentChat.tsx @@ -8,30 +8,16 @@ import { useWorkflowRunStore } from '@areas/workflows/workflowRunStore' import { buildAllWorkflowExtensions } from '@areas/workflows/mockExtensions' import { readAgentEvents, toolStopTitle } from '@shared/utils/agentStream' import { adviceText, ollamaAdvice } from '@shared/utils/ollamaHealth' -import OllamaBanner, { useOllamaHealth } from './OllamaBanner' -import { commitSculpt } from '../sculpt/sculptStore' +import OllamaBanner, { useOllamaHealth } from '@areas/generate/components/OllamaBanner' +import { commitSculpt } from '@areas/generate/sculpt/sculptStore' +import { useAgentConversationStore } from '@shared/stores/agentConversationStore' // ─── Types ──────────────────────────────────────────────────────────────────── import type { ThinkingMode } from '@shared/stores/agentStore' import type { Workflow } from '@shared/types/electron.d' -import type { AgentAction, AgentActionPayload, ToolStop } from '@shared/utils/agentStream' - -interface Message { - id: string - role: 'user' | 'assistant' - content: string - thinking?: string - imageDataUrls?: string[] - actions?: ActionDone[] - stopped?: ToolStop | null - streaming?: boolean -} - -interface ActionDone extends AgentAction { - id?: string - running?: boolean -} +import type { AgentActionPayload, ToolStop } from '@shared/utils/agentStream' +import type { ActionDone, AgentMessage as Message } from '@shared/stores/agentConversationStore' // ─── Constants ──────────────────────────────────────────────────────────────── @@ -270,13 +256,18 @@ function WorkflowProgressCard({ name }: { name: string }): JSX.Element { // ─── Main component ────────────────────────────────────────────────────────── -export default function ChatPanel(): JSX.Element { +export default function AgentChat(): JSX.Element { const { provider, localModel, external, defaultThinking, toolBudget, toolTimeout } = useAgentStore() const { models: llmModels } = useLlmModels() const isLocal = provider === 'local' const externalConfig = external[provider] - const [messages, setMessages] = useState([]) + // Messages live in the conversation store: they survive page changes and + // panel close, and are saved to disk. + const messages = useAgentConversationStore((s) => s.messages) + const setMessages = useAgentConversationStore((s) => s.setMessages) + const currentId = useAgentConversationStore((s) => s.currentId) + const setBusy = useAgentConversationStore((s) => s.setBusy) const [input, setInput] = useState('') const [isLoading, setIsLoading] = useState(false) const [error, setError] = useState(null) @@ -358,6 +349,15 @@ export default function ChatPanel(): JSX.Element { // eslint-disable-next-line react-hooks/exhaustive-deps -- fire on run-status transition; error/outputUrl read atomically }, [runState.status, pendingWorkflow]) + // The panel locks conversation switching while a turn or its workflow runs. + useEffect(() => { setBusy(isLoading || pendingWorkflow !== null) }, [isLoading, pendingWorkflow, setBusy]) + + // Another conversation was opened: drop the view state of the previous one. + useEffect(() => { + setShowAll(false) + setError(null) + }, [currentId]) + useEffect(() => { // Instant while streaming: a smooth scroll per token never catches up. endRef.current?.scrollIntoView({ behavior: isLoading ? 'auto' : 'smooth' }) diff --git a/src/shared/components/agent/AgentPanel.tsx b/src/shared/components/agent/AgentPanel.tsx new file mode 100644 index 000000000..c3ba61c90 --- /dev/null +++ b/src/shared/components/agent/AgentPanel.tsx @@ -0,0 +1,273 @@ +import { useEffect, useRef, useState } from 'react' +import { + AGENT_PANEL_DEFAULT_WIDTH, + AGENT_PANEL_MAX_WIDTH, + AGENT_PANEL_MIN_WIDTH, + clampPanelWidth, + useAgentPanelStore, +} from '@shared/stores/agentPanelStore' +import { useAgentConversationStore } from '@shared/stores/agentConversationStore' +import { UNTITLED, deriveTitle, relativeDate } from '@shared/utils/agentConversations' +import AgentChat from './AgentChat' + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +function isEditable(el: EventTarget | null): boolean { + if (!(el instanceof HTMLElement)) return false + return el.isContentEditable || el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || el.tagName === 'SELECT' +} + +function IconButton({ label, onClick, pressed, disabled, children }: { + label: string + onClick: () => void + pressed?: boolean + disabled?: boolean + children: React.ReactNode +}): JSX.Element { + return ( + + ) +} + +// ─── Conversation list ──────────────────────────────────────────────────────── + +function ConversationList({ onPicked }: { onPicked: () => void }): JSX.Element { + const list = useAgentConversationStore((s) => s.list) + const currentId = useAgentConversationStore((s) => s.currentId) + const busy = useAgentConversationStore((s) => s.busy) + const open = useAgentConversationStore((s) => s.open) + const remove = useAgentConversationStore((s) => s.remove) + // Delete takes two clicks: the first arms it. + const [armedId, setArmedId] = useState(null) + const now = Date.now() + + if (list.length === 0) { + return ( +
+

No saved conversations yet.

+
+ ) + } + + return ( +
    setArmedId(null)}> + {list.map((c) => { + const active = c.id === currentId + const locked = busy && !active + return ( +
  • + + +
    • + ) + })} +
+ ) +} + +// ─── Panel ──────────────────────────────────────────────────────────────────── + +export default function AgentPanel(): JSX.Element { + const open = useAgentPanelStore((s) => s.open) + const width = useAgentPanelStore((s) => s.width) + const setOpen = useAgentPanelStore((s) => s.setOpen) + const toggle = useAgentPanelStore((s) => s.toggle) + const setWidth = useAgentPanelStore((s) => s.setWidth) + + const messages = useAgentConversationStore((s) => s.messages) + const busy = useAgentConversationStore((s) => s.busy) + const newConversation = useAgentConversationStore((s) => s.newConversation) + + const [showList, setShowList] = useState(false) + const [dragWidth, setDragWidth] = useState(null) + const panelRef = useRef(null) + const dragRef = useRef<{ startX: number; startWidth: number } | null>(null) + + const shownWidth = dragWidth ?? width + const title = messages.length > 0 ? deriveTitle(messages) : UNTITLED + + // Load the saved conversations and reopen the latest; save before quitting. + useEffect(() => { + const { init, flush } = useAgentConversationStore.getState() + void init() + const onUnload = () => { void flush() } + window.addEventListener('beforeunload', onUnload) + return () => window.removeEventListener('beforeunload', onUnload) + }, []) + + // Ctrl+L / Cmd+L toggles the panel, except while typing elsewhere. + useEffect(() => { + const handler = (e: KeyboardEvent) => { + if (!(e.ctrlKey || e.metaKey) || e.shiftKey || e.altKey || e.key.toLowerCase() !== 'l') return + const target = e.target as Node | null + if (isEditable(e.target) && !panelRef.current?.contains(target)) return + e.preventDefault() + toggle() + } + window.addEventListener('keydown', handler) + return () => window.removeEventListener('keydown', handler) + }, [toggle]) + + // Opening the panel puts the cursor in the chat input. + useEffect(() => { + if (!open || showList) return + panelRef.current?.querySelector('textarea')?.focus() + }, [open, showList]) + + // ── Resize ── + function onHandlePointerDown(e: React.PointerEvent) { + if (e.button !== 0) return + e.preventDefault() + e.currentTarget.setPointerCapture(e.pointerId) + dragRef.current = { startX: e.clientX, startWidth: width } + setDragWidth(width) + } + + function onHandlePointerMove(e: React.PointerEvent) { + const drag = dragRef.current + if (!drag) return + // The panel sits on the right: dragging left widens it. + setDragWidth(clampPanelWidth(drag.startWidth + drag.startX - e.clientX)) + } + + function onHandlePointerUp(e: React.PointerEvent) { + if (!dragRef.current) return + e.currentTarget.releasePointerCapture(e.pointerId) + dragRef.current = null + if (dragWidth !== null) setWidth(dragWidth) + setDragWidth(null) + } + + function onHandleKeyDown(e: React.KeyboardEvent) { + const step = e.shiftKey ? 64 : 16 + if (e.key === 'ArrowLeft') { e.preventDefault(); setWidth(width + step) } + if (e.key === 'ArrowRight') { e.preventDefault(); setWidth(width - step) } + } + + return ( + + ) +} diff --git a/src/shared/components/layout/MainLayout.tsx b/src/shared/components/layout/MainLayout.tsx index da5bcd9c5..71a0cccac 100644 --- a/src/shared/components/layout/MainLayout.tsx +++ b/src/shared/components/layout/MainLayout.tsx @@ -1,6 +1,7 @@ import TopBar from './TopBar' import Sidebar from './Sidebar' import Router from '@shared/router/Router' +import AgentPanel from '@shared/components/agent/AgentPanel' export default function MainLayout(): JSX.Element { @@ -11,9 +12,11 @@ export default function MainLayout(): JSX.Element {
-
+
+ +
) diff --git a/src/shared/components/layout/TopBar.tsx b/src/shared/components/layout/TopBar.tsx index c73fd02a0..056ddf8af 100644 --- a/src/shared/components/layout/TopBar.tsx +++ b/src/shared/components/layout/TopBar.tsx @@ -1,10 +1,13 @@ import { useEffect, useState } from 'react' import { useAppStore } from '@shared/stores/appStore' +import { useAgentPanelStore } from '@shared/stores/agentPanelStore' import MemoryIndicator from './MemoryIndicator' export default function TopBar(): JSX.Element { const { patchUpdateReady, platform, showRamIndicator } = useAppStore() const isMac = platform === 'darwin' + const agentOpen = useAgentPanelStore((s) => s.open) + const toggleAgent = useAgentPanelStore((s) => s.toggle) const [isMaximized, setIsMaximized] = useState(false) @@ -45,6 +48,27 @@ export default function TopBar(): JSX.Element { {/* Memory indicator */} {showRamIndicator && } + {/* Agent panel toggle */} + + {/* Patch update badge */} {patchUpdateReady && (
diff --git a/src/shared/stores/agentConversationStore.ts b/src/shared/stores/agentConversationStore.ts new file mode 100644 index 000000000..d52261c1c --- /dev/null +++ b/src/shared/stores/agentConversationStore.ts @@ -0,0 +1,170 @@ +import { create } from 'zustand' +import type { AgentAction, ToolStop } from '@shared/utils/agentStream' +import { + deriveTitle, + settleMessages, + upsertSummary, + type Conversation, + type ConversationSummary, +} from '@shared/utils/agentConversations' + +// ─── Message types (AgentChat) ──────────────────────────────────────────────── + +export interface ActionDone extends AgentAction { + id?: string + running?: boolean +} + +export interface AgentMessage { + id: string + role: 'user' | 'assistant' + content: string + thinking?: string + /** Attached images, inline as data URLs (persisted as-is). */ + imageDataUrls?: string[] + actions?: ActionDone[] + stopped?: ToolStop | null + streaming?: boolean +} + +// ─── Store ──────────────────────────────────────────────────────────────────── + +const SAVE_DEBOUNCE_MS = 500 + +interface AgentConversationState { + /** Id of the conversation on screen; not on disk until it has a message. */ + currentId: string + createdAt: number + messages: AgentMessage[] + /** Saved conversations, newest first. */ + list: ConversationSummary[] + /** A turn is streaming or a workflow it started is running: switching would misroute its output. */ + busy: boolean + + setMessages: (update: AgentMessage[] | ((prev: AgentMessage[]) => AgentMessage[])) => void + setBusy: (busy: boolean) => void + newConversation: () => void + open: (id: string) => Promise + remove: (id: string) => Promise + refreshList: () => Promise + /** Loads the list and opens the most recent conversation. Runs once. */ + init: () => Promise + /** Writes a pending debounced save now. */ + flush: () => Promise +} + +function newId(): string { + return crypto.randomUUID() +} + +let saveTimer: ReturnType | null = null +// Unsaved changes: opening or switching away must not bump an untouched conversation. +let dirty = false +let initPromise: Promise | null = null + +export const useAgentConversationStore = create()((set, get) => { + async function save(): Promise { + if (saveTimer) { clearTimeout(saveTimer); saveTimer = null } + const { currentId, createdAt, messages } = get() + // An empty conversation never reaches the disk. + if (!dirty || messages.length === 0) return + dirty = false + const conversation: Conversation = { + id: currentId, + title: deriveTitle(messages), + createdAt, + updatedAt: Date.now(), + messages, + } + try { + await window.electron.agent.conversations.save(conversation) + set((s) => ({ list: upsertSummary(s.list, { id: conversation.id, title: conversation.title, updatedAt: conversation.updatedAt }) })) + } catch (err) { + console.error('[agent] Failed to save conversation:', err) + } + } + + function scheduleSave(): void { + if (saveTimer) clearTimeout(saveTimer) + saveTimer = setTimeout(() => { void save() }, SAVE_DEBOUNCE_MS) + } + + return { + currentId: newId(), + createdAt: Date.now(), + messages: [], + list: [], + busy: false, + + setMessages: (update) => { + set((s) => ({ messages: typeof update === 'function' ? update(s.messages) : update })) + dirty = true + scheduleSave() + }, + + setBusy: (busy) => set({ busy }), + + newConversation: () => { + if (get().busy) return + void save() + set({ currentId: newId(), createdAt: Date.now(), messages: [] }) + dirty = false + }, + + open: async (id) => { + if (get().busy || id === get().currentId) return + await save() + const conversation = await window.electron.agent.conversations.read(id) + if (!conversation) { + // Gone or corrupted on disk: drop it from the list. + set((s) => ({ list: s.list.filter((c) => c.id !== id) })) + return + } + set({ + currentId: conversation.id, + createdAt: conversation.createdAt, + messages: settleMessages(conversation.messages) as AgentMessage[], + }) + dirty = false + }, + + remove: async (id) => { + const isCurrent = id === get().currentId + if (isCurrent && get().busy) return + if (isCurrent) { + if (saveTimer) { clearTimeout(saveTimer); saveTimer = null } + dirty = false + } + try { + await window.electron.agent.conversations.remove(id) + } catch (err) { + console.error('[agent] Failed to delete conversation:', err) + return + } + set((s) => ({ + list: s.list.filter((c) => c.id !== id), + ...(isCurrent ? { currentId: newId(), createdAt: Date.now(), messages: [] } : {}), + })) + }, + + refreshList: async () => { + try { + set({ list: await window.electron.agent.conversations.list() }) + } catch (err) { + console.error('[agent] Failed to list conversations:', err) + } + }, + + init: () => { + initPromise ??= (async () => { + await get().refreshList() + const latest = get().list[0] + // The user may already have started typing a new conversation. + if (latest && get().messages.length === 0 && !get().busy) await get().open(latest.id) + })() + return initPromise + }, + + flush: save, + } +}) diff --git a/src/shared/stores/agentPanelStore.ts b/src/shared/stores/agentPanelStore.ts new file mode 100644 index 000000000..f82997087 --- /dev/null +++ b/src/shared/stores/agentPanelStore.ts @@ -0,0 +1,43 @@ +import { create } from 'zustand' +import { persist } from 'zustand/middleware' + +export const AGENT_PANEL_MIN_WIDTH = 320 +export const AGENT_PANEL_MAX_WIDTH = 640 +export const AGENT_PANEL_DEFAULT_WIDTH = 380 + +export function clampPanelWidth(width: number): number { + if (!Number.isFinite(width)) return AGENT_PANEL_DEFAULT_WIDTH + return Math.round(Math.min(AGENT_PANEL_MAX_WIDTH, Math.max(AGENT_PANEL_MIN_WIDTH, width))) +} + +interface AgentPanelState { + open: boolean + width: number + setOpen: (open: boolean) => void + toggle: () => void + setWidth: (width: number) => void +} + +export const useAgentPanelStore = create()( + persist( + (set) => ({ + open: false, + width: AGENT_PANEL_DEFAULT_WIDTH, + setOpen: (open) => set({ open }), + toggle: () => set((s) => ({ open: !s.open })), + setWidth: (width) => set({ width: clampPanelWidth(width) }), + }), + { + name: 'modly-agent-panel', + partialize: (s) => ({ open: s.open, width: s.width }), + merge: (persisted, current) => { + const p = (persisted ?? {}) as Partial + return { + ...current, + open: typeof p.open === 'boolean' ? p.open : current.open, + width: typeof p.width === 'number' ? clampPanelWidth(p.width) : current.width, + } + }, + }, + ), +) From df1d2c838125f78510f844c61e0bf2d1787539d1 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:04:58 +0100 Subject: [PATCH 05/13] feat(agent): parse client_tool events and the generate_from_image payload --- src/shared/utils/agentStream.test.ts | 29 ++++++++++++++++++++++++++++ src/shared/utils/agentStream.ts | 19 ++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/shared/utils/agentStream.test.ts b/src/shared/utils/agentStream.test.ts index b2fa95159..0d6876679 100644 --- a/src/shared/utils/agentStream.test.ts +++ b/src/shared/utils/agentStream.test.ts @@ -38,6 +38,35 @@ test('malformed and non-data lines are skipped', () => { ]) }) +test('a client_tool event passes through with its call id, arguments and confirm flag', () => { + const parser = createSseParser() + const event = { + type: 'client_tool', + id: 'call_1', + call_id: 'c-42', + tool: 'uninstall_extension', + arguments: { extension_id: 'triposr' }, + confirm: true, + } + const done = { type: 'tool_done', id: 'call_1', tool: 'uninstall_extension', result: 'ok', payload: null } + + const events = parser.push(line(event) + line(done)) + + assert.deepEqual(events, [event, done]) + const [parsed] = events + assert.ok(parsed.type === 'client_tool' && parsed.call_id === 'c-42' && parsed.confirm) +}) + +test('a generate_from_image payload keeps its model id', () => { + const parser = createSseParser() + const payload = { type: 'generate_from_image', model_id: 'sf3d/main' } + + const [event] = parser.push(line({ type: 'tool_done', id: 't', tool: 'generate_from_image', result: 'queued', payload })) + + assert.ok(event.type === 'tool_done') + assert.deepEqual(event.payload, payload) +}) + test('a response body is read into events, multi-byte characters intact', async () => { const bytes = new TextEncoder().encode(line({ type: 'token', text: 'é→' }) + line({ type: 'done', message: 'é→', actions: [], thinking: null, error: false })) // Split inside the multi-byte "é" so the decoder has to carry it over. diff --git a/src/shared/utils/agentStream.ts b/src/shared/utils/agentStream.ts index 14d155430..82fbf38ee 100644 --- a/src/shared/utils/agentStream.ts +++ b/src/shared/utils/agentStream.ts @@ -9,6 +9,10 @@ export interface AgentActionPayload { workflow_id?: string workflow_name?: string workflow?: Omit + /** `generate_from_image`: model to generate with; the Generate page's selection when absent. */ + model_id?: string + /** `generate_from_image`: model parameters overriding the defaults. */ + params?: Record } export interface AgentAction { @@ -17,11 +21,26 @@ export interface AgentAction { payload?: AgentActionPayload | null } +/** + * A tool the frontend must run (it needs IPC or renderer stores). The server + * waits for `POST /agent/tool-result {call_id, result, error?}`, then sends the + * usual `tool_done` for the same `id`. `confirm` asks the user first. + */ +export interface ClientToolEvent { + type: 'client_tool' + id: string + call_id: string + tool: string + arguments: Record + confirm: boolean +} + export type AgentEvent = | { type: 'token'; text: string } | { type: 'thinking'; text: string } | { type: 'tool_start'; id: string; tool: string; arguments: Record } | { type: 'tool_done'; id: string; tool: string; result: string; payload: AgentActionPayload | null } + | ClientToolEvent | { type: 'done'; message: string; actions: AgentAction[]; thinking: string | null; error: boolean } function parseBlock(block: string): AgentEvent | null { From 28119bb541d214eae958739af5646012701c1c65 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:04:59 +0100 Subject: [PATCH 06/13] refactor(generate): extract the image generation path out of useGeneration runGeneration and createGenerationApi hold the from-image submit and status polling without React, so the agent can start a generation outside a hook. useGeneration and useApi now delegate to them with unchanged behaviour. --- src/shared/hooks/useApi.ts | 45 +------ src/shared/hooks/useGeneration.ts | 88 ++------------ src/shared/utils/generationRunner.ts | 171 +++++++++++++++++++++++++++ 3 files changed, 183 insertions(+), 121 deletions(-) create mode 100644 src/shared/utils/generationRunner.ts diff --git a/src/shared/hooks/useApi.ts b/src/shared/hooks/useApi.ts index f013f1c11..93ef6f5bd 100644 --- a/src/shared/hooks/useApi.ts +++ b/src/shared/hooks/useApi.ts @@ -1,48 +1,13 @@ import axios from 'axios' -import { useAppStore, GenerationOptions } from '@shared/stores/appStore' +import { useAppStore } from '@shared/stores/appStore' +import { createGenerationApi } from '../utils/generationRunner' export function useApi() { const apiUrl = useAppStore((s) => s.apiUrl) const client = axios.create({ baseURL: apiUrl }) - async function generateFromImage( - imagePath: string, - options: GenerationOptions, - imageData?: string, - signal?: AbortSignal, - ): Promise<{ jobId: string }> { - // Use provided base64 (drag & drop) or read from disk via IPC - const base64 = imageData ?? await window.electron.fs.readFileBase64(imagePath) - const byteArray = Uint8Array.from(atob(base64), (c) => c.charCodeAt(0)) - const blob = new Blob([byteArray], { type: 'image/png' }) - const filename = imagePath.split(/[\\/]/).pop() ?? 'image.png' - - const formData = new FormData() - formData.append('image', blob, filename) - formData.append('model_id', options.modelId) - formData.append('remesh', options.remesh) - formData.append('enable_texture', String(options.enableTexture)) - formData.append('texture_resolution', String(options.textureResolution)) - formData.append('params', JSON.stringify(options.modelParams)) - const { data } = await client.post<{ job_id: string }>('/generate/from-image', formData, { - headers: { 'Content-Type': 'multipart/form-data' }, - signal, - }) - - return { jobId: data.job_id } - } - - async function pollJobStatus(jobId: string): Promise<{ - status: 'pending' | 'running' | 'done' | 'error' | 'cancelled' - progress: number - step?: string - outputUrl?: string - error?: string - }> { - const { data } = await client.get(`/generate/status/${jobId}`) - return { ...data, outputUrl: data.output_url } - } + const { generateFromImage, pollJobStatus, cancelJob } = createGenerationApi(apiUrl) async function getModelStatus(): Promise<{ downloaded: boolean @@ -96,10 +61,6 @@ export function useApi() { return { url: data.url, faceCount: data.face_count } } - async function cancelJob(jobId: string): Promise { - await client.post(`/generate/cancel/${jobId}`).catch(() => {}) - } - async function smoothMesh( path: string, iterations: number, diff --git a/src/shared/hooks/useGeneration.ts b/src/shared/hooks/useGeneration.ts index 6f3d3a375..a0900d989 100644 --- a/src/shared/hooks/useGeneration.ts +++ b/src/shared/hooks/useGeneration.ts @@ -2,6 +2,7 @@ import { useCallback, useRef } from 'react' import { useAppStore } from '@shared/stores/appStore' import { useApi } from './useApi' import { showCompletionNotification, showErrorNotification } from '@shared/utils/notification' +import { runGeneration } from '@shared/utils/generationRunner' export function useGeneration() { const { currentJob, setCurrentJob, updateCurrentJob, generationOptions, selectedImageData, pushMeshUrl, clearMeshHistory } = useAppStore() @@ -13,90 +14,19 @@ export function useGeneration() { async (imagePath: string) => { cancelledRef.current = false abortControllerRef.current = new AbortController() - clearMeshHistory() - const job = { - id: crypto.randomUUID(), - imageFile: imagePath, - status: 'uploading' as const, - progress: 0, - createdAt: Date.now(), - modelId: generationOptions.modelId, - generationOptions, - } - setCurrentJob(job) - - try { - const { jobId } = await generateFromImage(imagePath, generationOptions, selectedImageData ?? undefined, abortControllerRef.current.signal) - - if (cancelledRef.current) { - await cancelJob(jobId) - setCurrentJob(null) - return - } - - updateCurrentJob({ status: 'generating', progress: 0 }) - - await pollUntilDone(jobId) - } catch (err) { - if (cancelledRef.current) { - setCurrentJob(null) - return - } - let errorMessage: string - if (err && typeof err === 'object' && 'response' in err) { - const axiosErr = err as { response?: { data?: { detail?: string } }; message: string } - errorMessage = axiosErr.response?.data?.detail ?? axiosErr.message - } else { - errorMessage = err instanceof Error ? err.message : String(err) - } - updateCurrentJob({ - status: 'error', - error: errorMessage - }) - void showErrorNotification(errorMessage, 'Generation failed') - } + await runGeneration(imagePath, generationOptions, selectedImageData ?? undefined, { + api: { generateFromImage, pollJobStatus, cancelJob }, + store: { setCurrentJob, updateCurrentJob, pushMeshUrl, clearMeshHistory }, + notifyComplete: (body) => showCompletionNotification(body), + notifyError: (body, title) => showErrorNotification(body, title), + isCancelled: () => cancelledRef.current, + signal: abortControllerRef.current.signal, + }) }, // eslint-disable-next-line react-hooks/exhaustive-deps -- useApi re-creates its fns each render, so this re-memoizes anyway (values stay fresh) [generateFromImage, pollJobStatus, cancelJob, setCurrentJob, updateCurrentJob] ) - const pollUntilDone = async (jobId: string) => { - while (true) { - await new Promise((r) => setTimeout(r, 1000)) - - if (cancelledRef.current) { - await cancelJob(jobId) - setCurrentJob(null) - break - } - - const result = await pollJobStatus(jobId) - - if (result.status === 'cancelled') { - setCurrentJob(null) - break - } - - if (result.status === 'done') { - updateCurrentJob({ status: 'done', progress: 100, outputUrl: result.outputUrl, originalOutputUrl: result.outputUrl }) - if (result.outputUrl) pushMeshUrl(result.outputUrl) - void showCompletionNotification('Generation complete') - break - } - - if (result.status === 'error') { - updateCurrentJob({ status: 'error', error: result.error }) - void showErrorNotification(result.error ?? 'Unknown error', 'Generation failed') - break - } - - updateCurrentJob({ - progress: result.progress, - step: result.step, - }) - } - } - const cancelGeneration = useCallback(() => { cancelledRef.current = true abortControllerRef.current?.abort() diff --git a/src/shared/utils/generationRunner.ts b/src/shared/utils/generationRunner.ts new file mode 100644 index 000000000..8b25e2091 --- /dev/null +++ b/src/shared/utils/generationRunner.ts @@ -0,0 +1,171 @@ +import axios from 'axios' +import type { GenerationJob, GenerationOptions } from '../stores/appStore' + +// The single-image generation path (POST /generate/from-image, then poll +// /generate/status), free of React so the agent can start one outside a hook. +// useGeneration and useApi both go through it. + +export interface JobStatus { + status: 'pending' | 'running' | 'done' | 'error' | 'cancelled' + progress: number + step?: string + outputUrl?: string + error?: string +} + +export interface GenerationApi { + generateFromImage: (imagePath: string, options: GenerationOptions, imageData?: string, signal?: AbortSignal) => Promise<{ jobId: string }> + pollJobStatus: (jobId: string) => Promise + cancelJob: (jobId: string) => Promise +} + +export function createGenerationApi(apiUrl: string): GenerationApi { + const client = axios.create({ baseURL: apiUrl }) + + async function generateFromImage( + imagePath: string, + options: GenerationOptions, + imageData?: string, + signal?: AbortSignal, + ): Promise<{ jobId: string }> { + // Use provided base64 (drag & drop) or read from disk via IPC + const base64 = imageData ?? await window.electron.fs.readFileBase64(imagePath) + const byteArray = Uint8Array.from(atob(base64), (c) => c.charCodeAt(0)) + const blob = new Blob([byteArray], { type: 'image/png' }) + const filename = imagePath.split(/[\\/]/).pop() ?? 'image.png' + + const formData = new FormData() + formData.append('image', blob, filename) + formData.append('model_id', options.modelId) + formData.append('remesh', options.remesh) + formData.append('enable_texture', String(options.enableTexture)) + formData.append('texture_resolution', String(options.textureResolution)) + formData.append('params', JSON.stringify(options.modelParams)) + const { data } = await client.post<{ job_id: string }>('/generate/from-image', formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + signal, + }) + + return { jobId: data.job_id } + } + + async function pollJobStatus(jobId: string): Promise { + const { data } = await client.get(`/generate/status/${jobId}`) + return { ...data, outputUrl: data.output_url } + } + + async function cancelJob(jobId: string): Promise { + await client.post(`/generate/cancel/${jobId}`).catch(() => {}) + } + + return { generateFromImage, pollJobStatus, cancelJob } +} + +export interface GenerationStoreActions { + setCurrentJob: (job: GenerationJob | null) => void + updateCurrentJob: (patch: Partial) => void + pushMeshUrl: (url: string) => void + clearMeshHistory: () => void +} + +export interface GenerationRunDeps { + api: GenerationApi + store: GenerationStoreActions + notifyComplete: (body: string) => void | Promise + notifyError: (body: string, title: string) => void | Promise + isCancelled: () => boolean + signal?: AbortSignal + /** Delay between two status polls; tests pass a fast one. */ + wait?: () => Promise +} + +export type GenerationOutcome = + | { status: 'done'; outputUrl?: string } + | { status: 'error'; error: string } + | { status: 'cancelled' } + +/** Runs one generation into `currentJob`, so the Generate viewer shows it. */ +export async function runGeneration( + imagePath: string, + options: GenerationOptions, + imageData: string | undefined, + deps: GenerationRunDeps, +): Promise { + const { api, store } = deps + const wait = deps.wait ?? (() => new Promise((r) => setTimeout(r, 1000))) + store.clearMeshHistory() + store.setCurrentJob({ + id: crypto.randomUUID(), + imageFile: imagePath, + status: 'uploading', + progress: 0, + createdAt: Date.now(), + modelId: options.modelId, + generationOptions: options, + }) + + try { + const { jobId } = await api.generateFromImage(imagePath, options, imageData, deps.signal) + + if (deps.isCancelled()) { + await api.cancelJob(jobId) + store.setCurrentJob(null) + return { status: 'cancelled' } + } + + store.updateCurrentJob({ status: 'generating', progress: 0 }) + + for (;;) { + await wait() + + if (deps.isCancelled()) { + await api.cancelJob(jobId) + store.setCurrentJob(null) + return { status: 'cancelled' } + } + + const result = await api.pollJobStatus(jobId) + + if (result.status === 'cancelled') { + store.setCurrentJob(null) + return { status: 'cancelled' } + } + + if (result.status === 'done') { + store.updateCurrentJob({ status: 'done', progress: 100, outputUrl: result.outputUrl, originalOutputUrl: result.outputUrl }) + if (result.outputUrl) store.pushMeshUrl(result.outputUrl) + void deps.notifyComplete('Generation complete') + return { status: 'done', outputUrl: result.outputUrl } + } + + if (result.status === 'error') { + store.updateCurrentJob({ status: 'error', error: result.error }) + void deps.notifyError(result.error ?? 'Unknown error', 'Generation failed') + return { status: 'error', error: result.error ?? 'Unknown error' } + } + + store.updateCurrentJob({ + progress: result.progress, + step: result.step, + }) + } + } catch (err) { + if (deps.isCancelled()) { + store.setCurrentJob(null) + return { status: 'cancelled' } + } + let errorMessage: string + if (err && typeof err === 'object' && 'response' in err) { + const axiosErr = err as { response?: { data?: { detail?: string } }; message: string } + errorMessage = axiosErr.response?.data?.detail ?? axiosErr.message + } else { + errorMessage = err instanceof Error ? err.message : String(err) + } + store.updateCurrentJob({ + status: 'error', + error: errorMessage, + }) + void deps.notifyError(errorMessage, 'Generation failed') + return { status: 'error', error: errorMessage } + } +} From 918a3a0951318affe35f76e6b54906fc6782db9f Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:05:00 +0100 Subject: [PATCH 07/13] feat(agent): add client tools, confirm card and agent context clientTools.ts runs the client_tool calls of the agent (extensions, weights, workflows, settings, storage folders, mesh undo/redo, navigation) through injected dependencies, asks for confirmation when required and posts the result to /agent/tool-result. ConfirmCard.tsx renders the Confirm/Cancel card and its promise queue. agentContext.ts builds the request context from the stores. --- package.json | 2 +- .../components/agent/ConfirmCard.test.mjs | 66 ++ src/shared/components/agent/ConfirmCard.tsx | 173 ++++ .../components/agent/agentContext.test.ts | 81 ++ src/shared/components/agent/agentContext.ts | 84 ++ .../components/agent/clientTools.test.ts | 434 ++++++++++ src/shared/components/agent/clientTools.ts | 761 ++++++++++++++++++ 7 files changed, 1600 insertions(+), 1 deletion(-) create mode 100644 src/shared/components/agent/ConfirmCard.test.mjs create mode 100644 src/shared/components/agent/ConfirmCard.tsx create mode 100644 src/shared/components/agent/agentContext.test.ts create mode 100644 src/shared/components/agent/agentContext.ts create mode 100644 src/shared/components/agent/clientTools.test.ts create mode 100644 src/shared/components/agent/clientTools.ts diff --git a/package.json b/package.json index d74c54d74..5a4b69dfe 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/shared/components/agent/ConfirmCard.test.mjs b/src/shared/components/agent/ConfirmCard.test.mjs new file mode 100644 index 000000000..da117e50a --- /dev/null +++ b/src/shared/components/agent/ConfirmCard.test.mjs @@ -0,0 +1,66 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { buildSync } from 'esbuild' +import { createRequire } from 'node:module' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +// Bundle ConfirmCard.tsx with its real react/zustand dependencies; its only +// other import is type-only. Rendering is left to the app; this covers the +// confirm queue that turns confirm(description) into a Promise. +function load() { + const outfile = join(mkdtempSync(join(tmpdir(), 'modly-confirm-test-')), 'ConfirmCard.cjs') + const result = buildSync({ + entryPoints: [resolve('src/shared/components/agent/ConfirmCard.tsx')], + bundle: true, + platform: 'node', + format: 'cjs', + jsx: 'automatic', + write: false, + }) + writeFileSync(outfile, result.outputFiles[0].text, 'utf8') + return createRequire(import.meta.url)(outfile) +} + +const { useConfirmQueue, ConfirmCard } = load() +const queue = () => useConfirmQueue.getState() + +test('confirm shows a pending request and resolves true on Confirm', async () => { + const answer = queue().confirm('Uninstall the extension “TripoSR”', { tool: 'uninstall_extension', id: 'e1', callId: 'c1', destructive: true }) + const [request] = queue().requests + assert.equal(request.status, 'pending') + assert.equal(request.callId, 'c1') + assert.equal(request.destructive, true) + + queue().answer(request.key, true) + + assert.equal(await answer, true) + assert.equal(queue().requests[0].status, 'confirmed') + queue().clearAnswered() + assert.deepEqual(queue().requests, []) +}) + +test('Cancel resolves false, and a second answer is ignored', async () => { + const answer = queue().confirm('Delete the workflow “Turntable”') + const { key } = queue().requests[0] + queue().answer(key, false) + queue().answer(key, true) + + assert.equal(await answer, false) + assert.equal(queue().requests[0].status, 'cancelled') + queue().clearAnswered() +}) + +test('cancelAll declines every pending request', async () => { + const a = queue().confirm('one') + const b = queue().confirm('two') + queue().cancelAll() + assert.deepEqual(await Promise.all([a, b]), [false, false]) + assert.deepEqual(queue().requests.map((r) => r.status), ['cancelled', 'cancelled']) + queue().clearAnswered() +}) + +test('ConfirmCard is a component', () => { + assert.equal(typeof ConfirmCard, 'function') +}) diff --git a/src/shared/components/agent/ConfirmCard.tsx b/src/shared/components/agent/ConfirmCard.tsx new file mode 100644 index 000000000..55aeb6a7d --- /dev/null +++ b/src/shared/components/agent/ConfirmCard.tsx @@ -0,0 +1,173 @@ +import { create } from 'zustand' +import type { ConfirmInfo } from './clientTools' + +// ─── Confirm queue ──────────────────────────────────────────────────────────── + +export type ConfirmStatus = 'pending' | 'confirmed' | 'cancelled' + +export interface ConfirmRequest { + /** Unique per request. */ + key: string + description: string + destructive: boolean + status: ConfirmStatus + /** The client_tool event this request belongs to, when it came from one. */ + tool?: string + eventId?: string + callId?: string +} + +interface ConfirmQueueState { + requests: ConfirmRequest[] + /** Shows a card and resolves with the user's answer. Matches runClientTool's `confirm`. */ + confirm: (description: string, info?: ConfirmInfo) => Promise + answer: (key: string, confirmed: boolean) => void + /** Cancels every pending request (stream aborted, panel closed, new conversation). */ + cancelAll: () => void + /** Forgets answered requests. */ + clearAnswered: () => void +} + +// Resolvers stay out of the state: they are not data to render. +const resolvers = new Map void>() +let nextKey = 0 + +export const useConfirmQueue = create((set, get) => ({ + requests: [], + + confirm(description, info) { + const key = `confirm-${++nextKey}` + const request: ConfirmRequest = { + key, + description, + destructive: info?.destructive ?? true, + status: 'pending', + tool: info?.tool, + eventId: info?.id, + callId: info?.callId, + } + return new Promise((resolve) => { + resolvers.set(key, resolve) + set((s) => ({ requests: [...s.requests, request] })) + }) + }, + + answer(key, confirmed) { + const resolve = resolvers.get(key) + if (!resolve) return + resolvers.delete(key) + set((s) => ({ + requests: s.requests.map((r) => (r.key === key ? { ...r, status: confirmed ? 'confirmed' : 'cancelled' } : r)), + })) + resolve(confirmed) + }, + + cancelAll() { + for (const r of get().requests) if (r.status === 'pending') get().answer(r.key, false) + }, + + clearAnswered() { + set((s) => ({ requests: s.requests.filter((r) => r.status === 'pending') })) + }, +})) + +// ─── Card ───────────────────────────────────────────────────────────────────── + +export interface ConfirmCardProps { + description: string + destructive?: boolean + status: ConfirmStatus + onConfirm: () => void + onCancel: () => void +} + +/** The Confirm / Cancel card shown in the chat before a sensitive action. */ +export function ConfirmCard({ description, destructive = true, status, onConfirm, onCancel }: ConfirmCardProps): JSX.Element { + const pending = status === 'pending' + + function handleKeyDown(e: React.KeyboardEvent) { + if (!pending) return + if (e.key === 'Escape') { + e.preventDefault() + onCancel() + } else if (e.key === 'Enter' && !(e.target as HTMLElement).dataset.confirmCancel) { + // Enter on the Cancel button keeps its native meaning. + e.preventDefault() + onConfirm() + } + } + + return ( +
+
+ + + + +

{description}

+
+ +
+ {pending ? ( + <> + + + + ) : ( + + {status === 'confirmed' ? ( + + + + ) : ( + + + + )} + {status === 'confirmed' ? 'Confirmed' : 'Cancelled'} + + )} +
+
+ ) +} + +/** A ConfirmCard bound to one request of the queue. */ +export function QueuedConfirmCard({ request }: { request: ConfirmRequest }): JSX.Element { + const answer = useConfirmQueue((s) => s.answer) + return ( + answer(request.key, true)} + onCancel={() => answer(request.key, false)} + /> + ) +} diff --git a/src/shared/components/agent/agentContext.test.ts b/src/shared/components/agent/agentContext.test.ts new file mode 100644 index 000000000..93ff023e2 --- /dev/null +++ b/src/shared/components/agent/agentContext.test.ts @@ -0,0 +1,81 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { assembleAgentContext, buildAgentContext } from './agentContext' +import type { AgentContextInputs } from './agentContext' +import type { ModelExtension, ProcessExtension } from '../../types/electron.d' +import { useAppStore } from '../../stores/appStore' +import { useNavStore } from '../../stores/navStore' +import { useWorkflowsStore } from '../../stores/workflowsStore' +import { useExtensionsStore } from '../../stores/extensionsStore' + +const node = (id: string, input: string, output: string) => ({ id, name: id.toUpperCase(), input, output, paramsSchema: [] }) + +const modelExt = { + type: 'model', id: 'triposr', name: 'TripoSR', trusted: true, builtin: false, nodes: [node('main', 'image', 'mesh')], +} as unknown as ModelExtension +const processExt = { + type: 'process', id: 'mesh-ops', name: 'Mesh Ops', trusted: true, builtin: true, entry: 'x', nodes: [node('decimate', 'mesh', 'mesh')], +} as unknown as ProcessExtension +const STORAGE = { modelsDir: 'M', workspaceDir: 'W', workflowsDir: 'F', extensionsDir: 'E' } + +const inputs = (over: Partial = {}): AgentContextInputs => ({ + page: 'models', + currentMeshUrl: '/workspace/Default/mesh.glb', + meshTriangles: 1200, + workflows: [{ id: 'wf1', name: 'Turntable' }], + modelExtensions: [modelExt], + processExtensions: [processExt], + models: [{ id: 'triposr/main', name: 'TripoSR' }], + storage: STORAGE, + ui: { uiScale: 'large', useAtkinsonFont: true, showRamIndicator: false }, + ...over, +}) + +test('the context carries the page, mesh, workflows, extensions, models, storage and UI options', () => { + assert.deepEqual(assembleAgentContext(inputs()), { + page: 'models', + currentMeshPath: 'Default/mesh.glb', + meshTriangles: 1200, + workflows: [{ id: 'wf1', name: 'Turntable' }], + extensions: [ + { id: 'mesh-ops/decimate', name: 'DECIMATE', input: 'mesh', output: 'mesh' }, + { id: 'triposr/main', name: 'MAIN', input: 'image', output: 'mesh' }, + ], + models: [{ id: 'triposr/main', name: 'TripoSR' }], + storage: STORAGE, + ui: { uiScale: 'large', useAtkinsonFont: true, showRamIndicator: false }, + }) +}) + +test('empty or unreadable parts are left out, as buildContext did', () => { + const ctx = assembleAgentContext(inputs({ + currentMeshUrl: undefined, meshTriangles: null, workflows: [], modelExtensions: [], processExtensions: [], models: null, storage: null, + })) + assert.deepEqual(Object.keys(ctx), ['page', 'ui']) +}) + +test('storage keeps only the four folders, never the HF token', () => { + const ctx = assembleAgentContext(inputs({ storage: { ...STORAGE, hfToken: 'secret' } as typeof STORAGE })) + assert.deepEqual(ctx.storage, STORAGE) +}) + +test('buildAgentContext reads the stores and the async sources', async () => { + useNavStore.setState({ currentPage: 'workflows' }) + useAppStore.setState({ currentJob: null, meshStats: null, uiScale: 'small', useAtkinsonFont: false, showRamIndicator: true }) + useWorkflowsStore.setState({ workflows: [{ id: 'w', name: 'W', description: '', nodes: [], edges: [], createdAt: '', updatedAt: '' }] }) + useExtensionsStore.setState({ modelExtensions: [modelExt], processExtensions: [] }) + + const ctx = await buildAgentContext({ + fetchModels: async () => [{ id: 'triposr/main', name: 'TripoSR' }], + getStorage: async () => { throw new Error('IPC down') }, + }) + + assert.deepEqual(ctx, { + page: 'workflows', + workflows: [{ id: 'w', name: 'W' }], + extensions: [{ id: 'triposr/main', name: 'MAIN', input: 'image', output: 'mesh' }], + models: [{ id: 'triposr/main', name: 'TripoSR' }], + ui: { uiScale: 'small', useAtkinsonFont: false, showRamIndicator: true }, + }) +}) diff --git a/src/shared/components/agent/agentContext.ts b/src/shared/components/agent/agentContext.ts new file mode 100644 index 000000000..a13a06de8 --- /dev/null +++ b/src/shared/components/agent/agentContext.ts @@ -0,0 +1,84 @@ +import type { ModelExtension, ProcessExtension } from '../../types/electron.d' +import type { Workflow } from '../../types/electron.d' +import type { UiScale } from '../../stores/appStore' +import type { Page } from '../../stores/navStore' +import { useAppStore } from '../../stores/appStore' +import { useExtensionsStore } from '../../stores/extensionsStore' +import { useWorkflowsStore } from '../../stores/workflowsStore' +import { useNavStore } from '../../stores/navStore' +import { buildAllWorkflowExtensions } from '../../../areas/workflows/mockExtensions' + +// The `context` object sent with every agent request: what the app shows and +// holds right now, so the agent can act without asking. + +export interface AgentContextInputs { + page: Page + currentMeshUrl?: string | null + meshTriangles?: number | null + workflows: Pick[] + modelExtensions: ModelExtension[] + processExtensions: ProcessExtension[] + /** Downloaded generation models; null when they could not be read. */ + models: { id: string; name: string }[] | null + /** Null when the settings could not be read. */ + storage: { modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string } | null + ui: { uiScale: UiScale; useAtkinsonFont: boolean; showRamIndicator: boolean } +} + +/** Pure assembly of the context from already-read state. */ +export function assembleAgentContext(inputs: AgentContextInputs): Record { + const ctx: Record = { page: inputs.page } + if (inputs.currentMeshUrl) ctx.currentMeshPath = inputs.currentMeshUrl.replace('/workspace/', '') + if (inputs.meshTriangles) ctx.meshTriangles = inputs.meshTriangles + if (inputs.workflows.length > 0) ctx.workflows = inputs.workflows.map((w) => ({ id: w.id, name: w.name })) + const extensions = buildAllWorkflowExtensions(inputs.modelExtensions, inputs.processExtensions) + if (extensions.length > 0) ctx.extensions = extensions.map((e) => ({ + id: e.id, name: e.name, input: e.input, output: e.output, + })) + if (inputs.models) ctx.models = inputs.models.map((m) => ({ id: m.id, name: m.name })) + if (inputs.storage) { + const { modelsDir, workspaceDir, workflowsDir, extensionsDir } = inputs.storage + ctx.storage = { modelsDir, workspaceDir, workflowsDir, extensionsDir } + } + ctx.ui = { uiScale: inputs.ui.uiScale, useAtkinsonFont: inputs.ui.useAtkinsonFont, showRamIndicator: inputs.ui.showRamIndicator } + return ctx +} + +export interface AgentContextSources { + /** GET /model/all, filtered to the downloaded ones, as the Generate options list does. */ + fetchModels: () => Promise<{ id: string; name: string }[]> + getStorage: () => Promise<{ modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string }> +} + +function defaultSources(): AgentContextSources { + return { + async fetchModels() { + const res = await fetch(`${useAppStore.getState().apiUrl}/model/all`) + if (!res.ok) throw new Error(`API error ${res.status}`) + const list = await res.json() as { id: string; name?: string; downloaded: boolean }[] + return list.filter((m) => m.downloaded).map((m) => ({ id: m.id, name: m.name ?? m.id })) + }, + getStorage: () => window.electron.settings.get(), + } +} + +/** Reads the stores, the downloaded models and the storage folders, then assembles the context. */ +export async function buildAgentContext(sources: AgentContextSources = defaultSources()): Promise> { + const app = useAppStore.getState() + const ext = useExtensionsStore.getState() + const [models, storage] = await Promise.all([ + sources.fetchModels().catch(() => null), + sources.getStorage().catch(() => null), + ]) + return assembleAgentContext({ + page: useNavStore.getState().currentPage, + currentMeshUrl: app.currentJob?.outputUrl, + meshTriangles: app.meshStats?.triangles, + workflows: useWorkflowsStore.getState().workflows, + modelExtensions: ext.modelExtensions, + processExtensions: ext.processExtensions, + models, + storage, + ui: { uiScale: app.uiScale, useAtkinsonFont: app.useAtkinsonFont, showRamIndicator: app.showRamIndicator }, + }) +} diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts new file mode 100644 index 000000000..cda280082 --- /dev/null +++ b/src/shared/components/agent/clientTools.test.ts @@ -0,0 +1,434 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { + clientTools, + describeClientTool, + isErrorResult, + latestUserImage, + runClientTool, + startGenerationFromImage, +} from './clientTools' +import type { ClientToolDeps, GenerationDeps, StorageKind, ToolResultBody } from './clientTools' +import type { AnyExtension, Workflow } from '../../types/electron.d' +import type { GenerationJob, GenerationOptions, UiScale } from '../../stores/appStore' + +// ─── Fakes ──────────────────────────────────────────────────────────────────── + +const node = (id: string, extra: Record = {}) => ({ + id, name: id.toUpperCase(), input: 'image', output: 'mesh', paramsSchema: [], ...extra, +}) + +function extensions(): AnyExtension[] { + return [ + { + type: 'model', id: 'triposr', name: 'TripoSR', trusted: true, builtin: false, + nodes: [node('main', { hfRepo: 'stabilityai/TripoSR' })], + }, + { + type: 'model', id: 'hunyuan', name: 'Hunyuan3D', trusted: true, builtin: false, + nodes: [ + node('shape', { hfRepo: 'tencent/h', weightVariants: { param: 'v', default: 'full', options: [{ id: 'full', label: 'Full' }, { id: 'mini', label: 'Mini' }] } }), + node('paint', { hfRepo: 'tencent/p' }), + ], + }, + { + type: 'model', id: 'shared', name: 'Shared', trusted: true, builtin: false, + weightGroups: [{ id: 'g', dependentNodeIds: ['a'] }], + nodes: [node('a', { weightGroups: ['g'] })], + }, + { + type: 'process', id: 'mesh-ops', name: 'Mesh Ops', trusted: true, builtin: true, entry: 'x', + nodes: [node('decimate')], + }, + ] as AnyExtension[] +} + +const workflow = (id: string, name: string): Workflow => ({ + id, name, description: '', nodes: [], edges: [], createdAt: '', updatedAt: '', +}) + +interface Fake { + deps: ClientToolDeps + calls: unknown[][] + posted: ToolResultBody[] + ui: { uiScale: UiScale; useAtkinsonFont: boolean; showRamIndicator: boolean } + dirs: { modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string } + history: { index: number; length: number } + dirItems: Record +} + +function fake(overrides: Partial = {}): Fake { + const calls: unknown[][] = [] + const posted: ToolResultBody[] = [] + const rec = (...call: unknown[]) => { calls.push(call) } + const ok = { success: true } + const state = { + ui: { uiScale: 'medium' as UiScale, useAtkinsonFont: false, showRamIndicator: true }, + dirs: { modelsDir: 'C:\\Modly\\models', workspaceDir: 'C:\\Modly\\workspace', workflowsDir: 'C:\\Modly\\workflows', extensionsDir: 'C:\\Modly\\extensions' }, + history: { index: 1, length: 3 }, + dirItems: {} as Record, + } + const list = extensions() + const wfs = [workflow('wf1', 'Turntable'), workflow('wf2', 'Cleanup')] + const deps: ClientToolDeps = { + extensions: { + list: () => list, + installFromGitHub: async (url) => { + rec('install', url) + return { success: true, extensionId: 'sf3d', extension: { type: 'model', id: 'sf3d', name: 'SF3D', trusted: false, builtin: false, nodes: [node('main', { hfRepo: 'x/y' })] } as AnyExtension } + }, + uninstall: async (id) => { rec('uninstall', id); return ok }, + reload: async () => { rec('reload') }, + loadErrors: () => ({ broken: 'bad manifest' }), + }, + weights: { + download: async (id, variant) => { rec('download', id, variant); return ok }, + delete: async (id) => { rec('delete', id); return ok }, + deleteVariant: async (id, variant) => { rec('deleteVariant', id, variant); return ok }, + deleteExtensionWeights: async (id) => { rec('deleteExtensionWeights', id); return ok }, + refresh: async () => { rec('refresh') }, + }, + workflows: { + list: () => wfs, + remove: async (id) => { rec('removeWorkflow', id); return ok }, + open: (id) => { rec('openWorkflow', id) }, + reload: async () => { rec('reloadWorkflows') }, + }, + ui: { + get: () => state.ui, + setUiScale: (v) => { state.ui.uiScale = v }, + setUseAtkinsonFont: (v) => { state.ui.useAtkinsonFont = v }, + setShowRamIndicator: (v) => { state.ui.showRamIndicator = v }, + }, + storage: { + get: async () => state.dirs, + listDir: async (dir) => state.dirItems[dir] ?? [], + moveDirectory: async (args) => { rec('move', args.src, args.dest); return ok }, + apply: async (kind: StorageKind, path: string) => { rec('applyStorage', kind, path) }, + }, + mesh: { + history: () => state.history, + undo: () => { rec('undo') }, + redo: () => { rec('redo') }, + }, + navigate: (page) => { rec('navigate', page) }, + postResult: async (body) => { posted.push(body) }, + downloadSettleMs: 20, + ...overrides, + } + return { deps, calls, posted, ...state } +} + +const run = (tool: string, args: Record, f = fake()) => clientTools[tool].run(args, f.deps) + +// ─── Extensions ─────────────────────────────────────────────────────────────── + +test('install_extension installs from GitHub and names the extension', async () => { + const f = fake() + const result = await run('install_extension', { github_url: 'https://github.com/x/sf3d' }, f) + + assert.deepEqual(f.calls, [['install', 'https://github.com/x/sf3d']]) + assert.match(result, /Installed the extension “SF3D” \(id sf3d\)/) + assert.match(result, /not downloaded yet/) +}) + +test('install_extension refuses a missing or non-GitHub URL without installing', async () => { + const f = fake() + assert.match(await run('install_extension', {}, f), /^Error: github_url is required/) + assert.match(await run('install_extension', { github_url: 'https://evil.example/x' }, f), /^Error: .*github\.com/) + assert.match(await run('install_extension', { github_url: 'not a url' }, f), /^Error: .*not a valid URL/) + assert.deepEqual(f.calls, []) +}) + +test('install_extension reports the installer error', async () => { + const f = fake() + f.deps.extensions.installFromGitHub = async () => ({ success: false, error: 'No manifest.json' }) + assert.equal(await run('install_extension', { github_url: 'https://github.com/x/y' }, f), 'Error: No manifest.json') +}) + +test('uninstall_extension uninstalls by id, describes by name and refuses built-ins and unknown ids', async () => { + const f = fake() + assert.equal(describeClientTool('uninstall_extension', { extension_id: 'triposr' }, f.deps), 'Uninstall the extension “TripoSR”') + assert.match(await run('uninstall_extension', { extension_id: 'triposr' }, f), /^Uninstalled the extension “TripoSR”/) + assert.deepEqual(f.calls, [['uninstall', 'triposr']]) + + assert.match(await run('uninstall_extension', { extension_id: 'mesh-ops' }, f), /^Error: “Mesh Ops” is built into Modly/) + assert.match(await run('uninstall_extension', { extension_id: 'nope' }, f), /^Error: No installed extension has the id “nope”. Installed extensions: triposr, hunyuan, shared, mesh-ops\./) + assert.match(await run('uninstall_extension', {}, f), /^Error: extension_id is required/) + assert.equal(f.calls.length, 1) +}) + +test('reload_extensions reloads and reports the count and load errors', async () => { + const f = fake() + const result = await run('reload_extensions', {}, f) + assert.deepEqual(f.calls, [['reload']]) + assert.equal(result, 'Reloaded the extensions: 4 installed. 1 failed to load: broken (bad manifest).') +}) + +// ─── Weights ────────────────────────────────────────────────────────────────── + +test('download_model_weights downloads every model of the extension in turn', async () => { + const f = fake() + const result = await run('download_model_weights', { extension_id: 'hunyuan' }, f) + assert.deepEqual(f.calls, [['download', 'hunyuan/shape', undefined], ['download', 'hunyuan/paint', undefined], ['refresh']]) + assert.equal(result, 'Downloaded the model weights of “Hunyuan3D”.') +}) + +test('download_model_weights with a variant only targets the models offering it', async () => { + const f = fake() + const result = await run('download_model_weights', { extension_id: 'hunyuan', variant: 'mini' }, f) + assert.deepEqual(f.calls, [['download', 'hunyuan/shape', 'mini'], ['refresh']]) + assert.equal(result, 'Downloaded the model weights of “Hunyuan3D” (variant “mini”).') +}) + +test('download_model_weights validates the extension, model and variant', async () => { + const f = fake() + assert.match(await run('download_model_weights', { extension_id: 'hunyuan', variant: 'huge' }, f), /^Error: “Hunyuan3D” has no weight variant “huge”. Available variants: full, mini\./) + assert.match(await run('download_model_weights', { extension_id: 'triposr', variant: 'x' }, f), /^Error: “TripoSR” does not offer weight variants/) + assert.match(await run('download_model_weights', { extension_id: 'hunyuan/nope' }, f), /^Error: “Hunyuan3D” has no model “nope”/) + assert.match(await run('download_model_weights', { extension_id: 'mesh-ops' }, f), /^Error: “Mesh Ops” is a process extension/) + assert.match(await run('download_model_weights', { extension_id: 'ghost' }, f), /^Error: No installed extension/) + assert.match(await run('download_model_weights', { extension_id: 'triposr', variant: 3 }, f), /^Error: variant must be a string/) + assert.deepEqual(f.calls, []) +}) + +test('download_model_weights reports a long download as started, and an early failure as an error', async () => { + const slow = fake() + slow.deps.weights.download = () => new Promise(() => {}) + assert.match(await run('download_model_weights', { extension_id: 'triposr' }, slow), /^Started downloading the model weights of “TripoSR” \(1 model\)\. The download continues in the background/) + + const broken = fake() + broken.deps.weights.download = async () => ({ success: false, error: 'HF 401' }) + assert.equal(await run('download_model_weights', { extension_id: 'triposr/main' }, broken), 'Error: Downloading “TripoSR / MAIN” failed: triposr/main: HF 401.') +}) + +test('delete_model_weights deletes per model, per variant, or the shared extension weights', async () => { + const f = fake() + assert.equal(await run('delete_model_weights', { extension_id: 'hunyuan' }, f), 'Deleted the downloaded model weights of “Hunyuan3D”.') + assert.equal(await run('delete_model_weights', { extension_id: 'hunyuan', variant: 'full' }, f), 'Deleted the downloaded model weights of “Hunyuan3D” (variant “full”).') + await run('delete_model_weights', { extension_id: 'shared' }, f) + assert.deepEqual(f.calls, [ + ['delete', 'hunyuan/shape'], ['delete', 'hunyuan/paint'], ['refresh'], + ['deleteVariant', 'hunyuan/shape', 'full'], ['refresh'], + ['deleteExtensionWeights', 'shared'], ['refresh'], + ]) + assert.equal(describeClientTool('delete_model_weights', { extension_id: 'hunyuan/shape', variant: 'full' }, f.deps), 'Delete the downloaded model weights of “Hunyuan3D / SHAPE” (variant “full”)') +}) + +test('delete_model_weights reports a failed delete', async () => { + const f = fake() + f.deps.weights.delete = async () => ({ success: false, error: 'file locked' }) + assert.equal(await run('delete_model_weights', { extension_id: 'triposr' }, f), 'Error: Deleting the weights of “TripoSR” failed: file locked.') +}) + +// ─── Workflows ──────────────────────────────────────────────────────────────── + +test('delete_workflow deletes by id and resolves its name', async () => { + const f = fake() + assert.equal(describeClientTool('delete_workflow', { workflow_id: 'wf2' }, f.deps), 'Delete the workflow “Cleanup”') + assert.equal(await run('delete_workflow', { workflow_id: 'wf2' }, f), 'Deleted the workflow “Cleanup”.') + assert.deepEqual(f.calls, [['removeWorkflow', 'wf2']]) + assert.match(await run('delete_workflow', { workflow_id: 'wf9' }, f), /^Error: No workflow has the id “wf9”. Workflows: Turntable \(wf1\), Cleanup \(wf2\)\./) +}) + +test('open_workflow opens the tab without navigating', async () => { + const f = fake() + const result = await run('open_workflow', { workflow_id: 'wf1' }, f) + assert.deepEqual(f.calls, [['openWorkflow', 'wf1']]) + assert.match(result, /“Turntable”.*did not change/) + assert.match(await run('open_workflow', {}, f), /^Error: workflow_id is required/) +}) + +// ─── Settings ───────────────────────────────────────────────────────────────── + +test('set_setting sets each UI option and validates key and value', async () => { + const f = fake() + assert.equal(await run('set_setting', { key: 'uiScale', value: 'large' }, f), 'Set the interface scale to "large".') + assert.equal(await run('set_setting', { key: 'useAtkinsonFont', value: true }, f), 'Set the Atkinson Hyperlegible font to true.') + assert.equal(await run('set_setting', { key: 'showRamIndicator', value: 'false' }, f), 'Set the RAM indicator to false.') + assert.deepEqual(f.ui, { uiScale: 'large', useAtkinsonFont: true, showRamIndicator: false }) + + assert.match(await run('set_setting', { key: 'uiScale', value: 'huge' }, f), /^Error: uiScale must be one of: small, medium, large, very-large/) + assert.match(await run('set_setting', { key: 'showRamIndicator', value: 1 }, f), /^Error: showRamIndicator must be true or false/) + assert.match(await run('set_setting', { key: 'hfToken', value: 'x' }, f), /^Error: key must be one of/) +}) + +test('set_storage_folder moves existing files, then saves the folder', async () => { + const f = fake() + f.dirItems['C:\\Modly\\models'] = ['triposr', 'hunyuan'] + const result = await run('set_storage_folder', { kind: 'models', path: 'D:\\AI\\models' }, f) + assert.deepEqual(f.calls, [['move', 'C:\\Modly\\models', 'D:\\AI\\models'], ['applyStorage', 'models', 'D:\\AI\\models']]) + assert.equal(result, 'Moved 2 items from “C:\\Modly\\models” and set the models folder to “D:\\AI\\models”.') +}) + +test('set_storage_folder with an empty folder only saves it', async () => { + const f = fake() + assert.equal(await run('set_storage_folder', { kind: 'workflows', path: 'D:/flows' }, f), 'Set the workflows folder to “D:/flows”.') + assert.deepEqual(f.calls, [['applyStorage', 'workflows', 'D:/flows']]) +}) + +test('set_storage_folder refuses invalid locations and keeps everything on a failed move', async () => { + const f = fake() + assert.match(await run('set_storage_folder', { kind: 'cache', path: 'D:/x' }, f), /^Error: kind must be one of/) + assert.match(await run('set_storage_folder', { kind: 'models', path: 'models' }, f), /^Error: “models” is not an absolute path/) + assert.match(await run('set_storage_folder', { kind: 'models', path: 'c:/modly/models/sub' }, f), /^Error: .*overlaps the current models folder/) + assert.match(await run('set_storage_folder', { kind: 'models', path: 'C:\\Modly' }, f), /^Error: .*overlaps the current models folder/) + assert.match(await run('set_storage_folder', { kind: 'models', path: 'C:\\Modly\\workspace\\m' }, f), /^Error: .*overlaps another Modly folder/) + assert.match(await run('set_storage_folder', { kind: 'models', path: 'C:/Modly/models/' }, f), /already/) + assert.deepEqual(f.calls, []) + + f.dirItems['C:\\Modly\\workspace'] = ['a.glb'] + f.deps.storage.moveDirectory = async () => ({ success: false, error: 'EACCES' }) + assert.equal(await run('set_storage_folder', { kind: 'workspace', path: 'E:/ws' }, f), 'Error: Could not move the workspace files to “E:/ws”: EACCES. Nothing changed.') + assert.deepEqual(f.calls, []) +}) + +// ─── Mesh, navigation ───────────────────────────────────────────────────────── + +test('undo_mesh and redo_mesh step through the history, and say when there is nothing to do', async () => { + const f = fake() + assert.match(await run('undo_mesh', {}, f), /^Undid/) + assert.match(await run('redo_mesh', {}, f), /^Redid/) + assert.deepEqual(f.calls, [['undo'], ['redo']]) + + f.history.index = 0 + f.history.length = 1 + assert.equal(await run('undo_mesh', {}, f), 'There is no mesh change to undo.') + assert.equal(await run('redo_mesh', {}, f), 'There is no mesh change to redo.') + assert.equal(f.calls.length, 2) +}) + +test('navigate changes page and validates it', async () => { + const f = fake() + assert.equal(await run('navigate', { page: 'models' }, f), 'Opened the models page.') + assert.deepEqual(f.calls, [['navigate', 'models']]) + assert.match(await run('navigate', { page: 'home' }, f), /^Error: page must be one of: generate, workflows, models, settings/) +}) + +test('a throwing dependency comes back as an error text', async () => { + const f = fake() + f.deps.extensions.reload = async () => { throw new Error('IPC gone') } + assert.equal(await run('reload_extensions', {}, f), 'Error: IPC gone') +}) + +// ─── runClientTool ──────────────────────────────────────────────────────────── + +const event = (tool: string, args: Record, confirm = false) => ({ + type: 'client_tool' as const, id: 'evt-1', call_id: 'call-7', tool, arguments: args, confirm, +}) + +test('runClientTool runs the tool and posts the result with its call_id', async () => { + const f = fake() + const outcome = await runClientTool(event('navigate', { page: 'settings' }), f.deps, async () => { throw new Error('not asked') }) + assert.deepEqual(f.posted, [{ call_id: 'call-7', result: 'Opened the settings page.' }]) + assert.deepEqual(outcome, { description: 'Go to the settings page', result: 'Opened the settings page.', error: false, declined: false, posted: true }) +}) + +test('runClientTool asks first when confirm is set, and runs on Confirm', async () => { + const f = fake() + const asked: unknown[] = [] + await runClientTool(event('uninstall_extension', { extension_id: 'triposr' }, true), f.deps, async (description, info) => { + asked.push(description, info) + return true + }) + assert.deepEqual(asked, ['Uninstall the extension “TripoSR”', { tool: 'uninstall_extension', id: 'evt-1', callId: 'call-7', destructive: true }]) + assert.deepEqual(f.calls, [['uninstall', 'triposr']]) + assert.equal(f.posted[0].call_id, 'call-7') +}) + +test('runClientTool reports a declined confirmation without running the tool', async () => { + const f = fake() + const outcome = await runClientTool(event('delete_workflow', { workflow_id: 'wf1' }, true), f.deps, async () => false) + assert.deepEqual(f.calls, []) + assert.deepEqual(f.posted, [{ call_id: 'call-7', result: 'The user declined: Delete the workflow “Turntable”.' }]) + assert.equal(outcome.declined, true) +}) + +test('runClientTool does not ask about a call that cannot run, and flags errors', async () => { + const f = fake() + let asked = false + const outcome = await runClientTool(event('delete_workflow', { workflow_id: 'nope' }, true), f.deps, async () => { asked = true; return true }) + assert.equal(asked, false) + assert.equal(outcome.error, true) + assert.equal(f.posted[0].error, true) + assert.match(f.posted[0].result, /^Error: No workflow has the id “nope”/) +}) + +test('runClientTool answers an unknown tool with an error', async () => { + const f = fake() + const outcome = await runClientTool(event('format_disk', {}), f.deps, async () => true) + assert.match(outcome.result, /^Error: Unknown client tool “format_disk”. Client tools: install_extension,/) + assert.deepEqual(f.posted, [{ call_id: 'call-7', result: outcome.result, error: true }]) +}) + +test('runClientTool survives a failed post', async () => { + const f = fake({ postResult: async () => { throw new Error('offline') } }) + const outcome = await runClientTool(event('reload_extensions', {}), f.deps, async () => true) + assert.equal(outcome.posted, false) + assert.equal(isErrorResult(outcome.result), false) +}) + +// ─── generate_from_image ────────────────────────────────────────────────────── + +const OPTIONS: GenerationOptions = { modelId: 'triposr/main', remesh: 'quad', enableTexture: false, textureResolution: 512, modelParams: { steps: 30 } } + +function generation(statuses: { status: string; progress?: number; outputUrl?: string; error?: string }[]) { + const calls: unknown[][] = [] + let job: GenerationJob | null = null + const deps: GenerationDeps = { + api: { + generateFromImage: async (path, options, data) => { calls.push(['submit', path, options, data]); return { jobId: 'j1' } }, + pollJobStatus: async () => ({ progress: 0, ...statuses.shift()! }) as never, + cancelJob: async () => { calls.push(['cancel']) }, + }, + store: { + setCurrentJob: (j) => { job = j }, + updateCurrentJob: (patch) => { if (job) job = { ...job, ...patch } }, + pushMeshUrl: (url) => { calls.push(['push', url]) }, + clearMeshHistory: () => { calls.push(['clear']) }, + }, + options: () => OPTIONS, + notifyComplete: () => {}, + notifyError: () => {}, + wait: async () => {}, + } + return { deps, calls, job: () => job } +} + +test('startGenerationFromImage generates from the attached image into the current job', async () => { + const g = generation([{ status: 'running', progress: 50 }, { status: 'done', outputUrl: '/workspace/out.glb' }]) + const result = await startGenerationFromImage({ type: 'generate_from_image', model_id: 'sf3d/main' }, 'data:image/jpeg;base64,QUJD', g.deps) + + assert.equal(result, 'Generated a mesh with sf3d/main: /workspace/out.glb.') + const submit = g.calls.find((c) => c[0] === 'submit')! + assert.equal(submit[3], 'QUJD') + assert.deepEqual(submit[2], { ...OPTIONS, modelId: 'sf3d/main', modelParams: {} }) + assert.deepEqual(g.calls.at(-1), ['push', '/workspace/out.glb']) + assert.equal(g.job()?.status, 'done') + assert.equal(g.job()?.outputUrl, '/workspace/out.glb') +}) + +test('startGenerationFromImage keeps the selected model and its params when none is given', async () => { + const g = generation([{ status: 'done', outputUrl: '/workspace/o.glb' }]) + await startGenerationFromImage({ type: 'generate_from_image' }, 'QUJD', g.deps) + assert.deepEqual(g.calls.find((c) => c[0] === 'submit')![2], OPTIONS) +}) + +test('startGenerationFromImage reports a missing image, a wrong payload and a failed job', async () => { + const g = generation([{ status: 'error', error: 'CUDA OOM' }]) + assert.match(await startGenerationFromImage({ type: 'generate_from_image' }, undefined, g.deps), /^Error: No image/) + assert.match(await startGenerationFromImage({ type: 'mesh_update' }, 'QUJD', g.deps), /^Error: Not a generate_from_image payload/) + assert.equal(await startGenerationFromImage({ type: 'generate_from_image' }, 'QUJD', g.deps), 'Error: The generation failed: CUDA OOM') + assert.equal(g.job()?.status, 'error') +}) + +test('latestUserImage takes the most recent user message with an image', () => { + assert.equal(latestUserImage([ + { role: 'user', imageDataUrls: ['data:old'] }, + { role: 'assistant', imageDataUrls: ['data:assistant'] }, + { role: 'user', imageDataUrls: ['data:new', 'data:second'] }, + { role: 'user' }, + ]), 'data:new') + assert.equal(latestUserImage([{ role: 'user' }]), undefined) +}) diff --git a/src/shared/components/agent/clientTools.ts b/src/shared/components/agent/clientTools.ts new file mode 100644 index 000000000..c7ce651f2 --- /dev/null +++ b/src/shared/components/agent/clientTools.ts @@ -0,0 +1,761 @@ +import type { AnyExtension, ExtensionNode, Workflow } from '../../types/electron.d' +import type { AgentActionPayload, ClientToolEvent } from '../../utils/agentStream' +import type { GenerationOptions, UiScale } from '../../stores/appStore' +import type { Page } from '../../stores/navStore' +import type { GenerationApi, GenerationStoreActions } from '../../utils/generationRunner' +import { createGenerationApi, runGeneration } from '../../utils/generationRunner' +import { useAppStore } from '../../stores/appStore' +import { useExtensionsStore } from '../../stores/extensionsStore' +import { useWorkflowsStore } from '../../stores/workflowsStore' +import { useNavStore } from '../../stores/navStore' +import { showCompletionNotification, showErrorNotification } from '../../utils/notification' + +// Tools the agent asks the frontend to run, because they need IPC or the +// renderer stores. Every handler returns the text sent back to the LLM and +// never throws: failures come back as text starting with "Error: ". + +// ─── Dependencies ───────────────────────────────────────────────────────────── + +type OpResult = { success: boolean; error?: string } + +export type StorageKind = 'models' | 'workspace' | 'workflows' + +export interface StorageDirs { + modelsDir: string + workspaceDir: string + workflowsDir: string + extensionsDir: string +} + +export interface UiSettings { + uiScale: UiScale + useAtkinsonFont: boolean + showRamIndicator: boolean +} + +export interface ToolResultBody { + call_id: string + result: string + error?: boolean +} + +export interface ClientToolDeps { + extensions: { + list: () => AnyExtension[] + installFromGitHub: (url: string) => Promise + uninstall: (extensionId: string) => Promise + reload: () => Promise + loadErrors: () => Record + } + weights: { + download: (modelId: string, variantId?: string) => Promise + delete: (modelId: string) => Promise + deleteVariant: (modelId: string, variantId: string) => Promise + deleteExtensionWeights: (extensionId: string) => Promise + /** Re-reads what is installed, so the Models page stays accurate. */ + refresh: () => Promise + } + workflows: { + list: () => Workflow[] + remove: (id: string) => Promise + /** Opens the workflow as the active editor tab, without changing page. */ + open: (id: string) => void + reload: () => Promise + } + ui: { + get: () => UiSettings + setUiScale: (value: UiScale) => void + setUseAtkinsonFont: (value: boolean) => void + setShowRamIndicator: (value: boolean) => void + } + storage: { + get: () => Promise + listDir: (dirPath: string) => Promise + moveDirectory: (args: { src: string; dest: string }) => Promise + /** Saves the new folder, the way Settings > Storage does once files are moved. */ + apply: (kind: StorageKind, path: string) => Promise + } + mesh: { + history: () => { index: number; length: number } + undo: () => void + redo: () => void + } + navigate: (page: Page) => void + postResult: (body: ToolResultBody) => Promise + /** How long a download is watched for an early failure before it is reported as started. */ + downloadSettleMs?: number +} + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +type Args = Record + +const ERROR_PREFIX = 'Error: ' + +const fail = (message: string): string => ERROR_PREFIX + message + +/** True for a result text that reports a failure. */ +export function isErrorResult(result: string): boolean { + return result.startsWith(ERROR_PREFIX) +} + +const quote = (s: string): string => `“${s}”` + +function errorText(err: unknown): string { + return err instanceof Error ? err.message : String(err) +} + +function str(args: Args, key: string): string | null { + const value = args[key] + return typeof value === 'string' && value.trim() ? value.trim() : null +} + +function nodeHasManagedWeights(node: ExtensionNode): boolean { + return Boolean(node.hfRepo || node.hasModelSources || node.weightGroups?.length) +} + +function listOf(items: string[]): string { + return items.length ? items.join(', ') : 'none' +} + +function findExtension(id: string, deps: ClientToolDeps | undefined): AnyExtension | undefined { + return deps?.extensions.list().find((e) => e.id === id.split('/')[0]) +} + +function extensionLabel(id: string, deps: ClientToolDeps | undefined): string { + const ext = findExtension(id, deps) + if (!ext) return quote(id) + const nodeId = id.includes('/') ? id.slice(id.indexOf('/') + 1) : null + const node = nodeId ? ext.nodes.find((n) => n.id === nodeId) : undefined + return quote(node ? `${ext.name} / ${node.name}` : ext.name) +} + +function workflowLabel(id: string, deps: ClientToolDeps | undefined): string { + return quote(deps?.workflows.list().find((w) => w.id === id)?.name ?? id) +} + +function lookupExtension(args: Args, deps: ClientToolDeps): { ext: AnyExtension; id: string } | string { + const id = str(args, 'extension_id') + if (!id) return 'extension_id is required.' + const ext = findExtension(id, deps) + if (!ext) { + const installed = deps.extensions.list().map((e) => e.id) + return `No installed extension has the id ${quote(id)}. Installed extensions: ${listOf(installed)}.` + } + return { ext, id } +} + +function lookupWorkflow(args: Args, deps: ClientToolDeps): Workflow | string { + const id = str(args, 'workflow_id') + if (!id) return 'workflow_id is required.' + const workflows = deps.workflows.list() + const wf = workflows.find((w) => w.id === id) + if (!wf) { + return `No workflow has the id ${quote(id)}. Workflows: ${listOf(workflows.map((w) => `${w.name} (${w.id})`))}.` + } + return wf +} + +interface WeightTarget { + ext: AnyExtension + /** Set when the id named one model ("ext/node") rather than the whole extension. */ + nodeId: string | null + variant: string | null + nodes: { fullId: string; node: ExtensionNode }[] +} + +function lookupWeights(args: Args, deps: ClientToolDeps): WeightTarget | string { + const found = lookupExtension(args, deps) + if (typeof found === 'string') return found + const { ext, id } = found + if (args.variant !== undefined && args.variant !== null && typeof args.variant !== 'string') { + return 'variant must be a string.' + } + const variant = str(args, 'variant') + if (ext.type !== 'model') return `${quote(ext.name)} is a process extension; it has no model weights.` + + const nodeId = id.includes('/') ? id.slice(id.indexOf('/') + 1) : null + let nodes = ext.nodes.filter(nodeHasManagedWeights) + if (nodeId) { + nodes = nodes.filter((n) => n.id === nodeId) + if (!nodes.length) { + const models = ext.nodes.filter(nodeHasManagedWeights).map((n) => `${ext.id}/${n.id}`) + return `${quote(ext.name)} has no model ${quote(nodeId)} with weights. Models with weights: ${listOf(models)}.` + } + } + if (!nodes.length) return `${quote(ext.name)} has no downloadable model weights.` + if (variant) { + const offered = [...new Set(nodes.flatMap((n) => n.weightVariants?.options.map((o) => o.id) ?? []))] + nodes = nodes.filter((n) => n.weightVariants?.options.some((o) => o.id === variant)) + if (!nodes.length) { + return offered.length + ? `${quote(ext.name)} has no weight variant ${quote(variant)}. Available variants: ${listOf(offered)}.` + : `${quote(ext.name)} does not offer weight variants.` + } + } + return { ext, nodeId, variant, nodes: nodes.map((node) => ({ fullId: `${ext.id}/${node.id}`, node })) } +} + +function weightsLabel(args: Args, deps: ClientToolDeps | undefined): string { + const id = typeof args.extension_id === 'string' ? args.extension_id : '?' + const variant = typeof args.variant === 'string' && args.variant ? ` (variant ${quote(args.variant)})` : '' + return `${extensionLabel(id, deps)}${variant}` +} + +const sleep = (ms: number): Promise => new Promise((r) => setTimeout(() => r(null), ms)) + +// ─── Paths ──────────────────────────────────────────────────────────────────── + +const STORAGE_KINDS: StorageKind[] = ['models', 'workspace', 'workflows'] + +const STORAGE_KEY: Record = { + models: 'modelsDir', + workspace: 'workspaceDir', + workflows: 'workflowsDir', +} + +function isAbsolutePath(p: string): boolean { + return /^[A-Za-z]:[\\/]/.test(p) || p.startsWith('\\\\') || p.startsWith('/') +} + +/** Comparable form: forward slashes, no trailing slash, Windows paths lower-cased. */ +function normPath(p: string): string { + const s = p.replace(/\\/g, '/').replace(/\/+$/, '') + return /^[A-Za-z]:/.test(s) || s.startsWith('//') ? s.toLowerCase() : s +} + +function isInside(child: string, parent: string): boolean { + return child === parent || child.startsWith(`${parent}/`) +} + +// ─── Registry ───────────────────────────────────────────────────────────────── + +export interface ClientTool { + /** Destructive: the confirm card shows a red Confirm button. */ + destructive: boolean + /** The human sentence for the confirm card and the action list. */ + describe: (args: Args, deps?: ClientToolDeps) => string + /** Why the arguments cannot be run, or null when they can. */ + check: (args: Args, deps: ClientToolDeps) => string | null + /** Runs the tool; returns the result text for the LLM and never throws. */ + run: (args: Args, deps: ClientToolDeps) => Promise +} + +interface ToolSpec { + destructive?: boolean + describe: (args: Args, deps?: ClientToolDeps) => string + check: (args: Args, deps: ClientToolDeps) => string | null + execute: (args: Args, deps: ClientToolDeps) => Promise +} + +function defineTool(spec: ToolSpec): ClientTool { + return { + destructive: spec.destructive ?? false, + describe: spec.describe, + check: (args, deps) => { + try { + return spec.check(args, deps) + } catch (err) { + return errorText(err) + } + }, + run: async (args, deps) => { + try { + const problem = spec.check(args, deps) + if (problem) return fail(problem) + return await spec.execute(args, deps) + } catch (err) { + return fail(errorText(err)) + } + }, + } +} + +const checked = (value: T | string): string | null => (typeof value === 'string' ? value : null) + +export const clientTools: Record = { + install_extension: defineTool({ + describe: (args) => `Install the extension from ${quote(String(args.github_url ?? '?'))}`, + check: (args) => { + const url = str(args, 'github_url') + if (!url) return 'github_url is required.' + let host: string + try { + host = new URL(url).hostname.toLowerCase() + } catch { + return `${quote(url)} is not a valid URL.` + } + return host === 'github.com' || host === 'www.github.com' ? null : 'github_url must be a github.com repository URL.' + }, + async execute(args, deps) { + const result = await deps.extensions.installFromGitHub(str(args, 'github_url')!) + if (!result.success) return fail(result.error ?? 'Installation failed.') + const ext = result.extension + const id = ext?.id ?? result.extensionId ?? '?' + const needsWeights = ext?.type === 'model' && ext.nodes.some(nodeHasManagedWeights) + return `Installed the extension ${quote(ext?.name ?? id)} (id ${id}).` + + (needsWeights ? ' Its model weights are not downloaded yet; download_model_weights fetches them.' : '') + }, + }), + + uninstall_extension: defineTool({ + destructive: true, + describe: (args, deps) => `Uninstall the extension ${extensionLabel(String(args.extension_id ?? '?'), deps)}`, + check: (args, deps) => { + const found = lookupExtension(args, deps) + if (typeof found === 'string') return found + return found.ext.builtin ? `${quote(found.ext.name)} is built into Modly and cannot be uninstalled.` : null + }, + async execute(args, deps) { + const { ext } = lookupExtension(args, deps) as { ext: AnyExtension } + const result = await deps.extensions.uninstall(ext.id) + if (!result.success) return fail(result.error ?? `Could not uninstall ${quote(ext.name)}.`) + return `Uninstalled the extension ${quote(ext.name)}. Its downloaded model weights, if any, were left on disk.` + }, + }), + + reload_extensions: defineTool({ + describe: () => 'Reload the installed extensions', + check: () => null, + async execute(_args, deps) { + await deps.extensions.reload() + const count = deps.extensions.list().length + const errors = Object.entries(deps.extensions.loadErrors()) + const tail = errors.length + ? ` ${errors.length} failed to load: ${errors.map(([id, err]) => `${id} (${err})`).join('; ')}.` + : '' + return `Reloaded the extensions: ${count} installed.${tail}` + }, + }), + + download_model_weights: defineTool({ + describe: (args, deps) => `Download the model weights of ${weightsLabel(args, deps)}`, + check: (args, deps) => checked(lookupWeights(args, deps)), + async execute(args, deps) { + const target = lookupWeights(args, deps) as WeightTarget + const variant = target.variant ?? undefined + // Downloads run one after the other, as "Install all" does on the Models page. + const queue = (async () => { + const outcomes: { fullId: string; result: OpResult & { paused?: boolean; cancelled?: boolean } }[] = [] + for (const { fullId } of target.nodes) { + let result: OpResult & { paused?: boolean; cancelled?: boolean } + try { + result = await deps.weights.download(fullId, variant) + } catch (err) { + result = { success: false, error: errorText(err) } + } + outcomes.push({ fullId, result }) + } + await deps.weights.refresh().catch(() => {}) + return outcomes + })() + const outcomes = await Promise.race([queue, sleep(deps.downloadSettleMs ?? 1500)]) + const label = weightsLabel(args, deps) + if (!outcomes) { + return `Started downloading the model weights of ${label} (${target.nodes.length} model${target.nodes.length > 1 ? 's' : ''}). ` + + 'The download continues in the background; its progress shows on the Models page.' + } + const failed = outcomes.filter((o) => !o.result.success && !o.result.paused && !o.result.cancelled) + if (failed.length) { + return fail(`Downloading ${label} failed: ${failed.map((o) => `${o.fullId}: ${o.result.error ?? 'unknown error'}`).join('; ')}.`) + } + if (outcomes.some((o) => o.result.paused || o.result.cancelled)) { + return `The download of ${label} was paused or cancelled before it finished.` + } + return `Downloaded the model weights of ${label}.` + }, + }), + + delete_model_weights: defineTool({ + destructive: true, + describe: (args, deps) => `Delete the downloaded model weights of ${weightsLabel(args, deps)}`, + check: (args, deps) => checked(lookupWeights(args, deps)), + async execute(args, deps) { + const target = lookupWeights(args, deps) as WeightTarget + const label = weightsLabel(args, deps) + const results: OpResult[] = [] + if (target.variant) { + for (const { fullId } of target.nodes) results.push(await deps.weights.deleteVariant(fullId, target.variant)) + } else if (!target.nodeId && target.ext.type === 'model' && target.ext.weightGroups?.length) { + // Shared weight groups: the whole-extension delete, as the uninstall dialog does. + results.push(await deps.weights.deleteExtensionWeights(target.ext.id)) + } else { + for (const { fullId } of target.nodes) results.push(await deps.weights.delete(fullId)) + } + await deps.weights.refresh().catch(() => {}) + const errors = results.filter((r) => !r.success).map((r) => r.error ?? 'unknown error') + if (errors.length) return fail(`Deleting the weights of ${label} failed: ${errors.join('; ')}.`) + return `Deleted the downloaded model weights of ${label}.` + }, + }), + + delete_workflow: defineTool({ + destructive: true, + describe: (args, deps) => `Delete the workflow ${workflowLabel(String(args.workflow_id ?? '?'), deps)}`, + check: (args, deps) => checked(lookupWorkflow(args, deps)), + async execute(args, deps) { + const wf = lookupWorkflow(args, deps) as Workflow + const result = await deps.workflows.remove(wf.id) + if (!result.success) return fail(result.error ?? `Could not delete the workflow ${quote(wf.name)}.`) + return `Deleted the workflow ${quote(wf.name)}.` + }, + }), + + open_workflow: defineTool({ + describe: (args, deps) => `Open the workflow ${workflowLabel(String(args.workflow_id ?? '?'), deps)}`, + check: (args, deps) => checked(lookupWorkflow(args, deps)), + async execute(args, deps) { + const wf = lookupWorkflow(args, deps) as Workflow + deps.workflows.open(wf.id) + return `Opened the workflow ${quote(wf.name)} as the active tab of the Workflows editor. The current page did not change.` + }, + }), + + set_setting: defineTool({ + describe: (args) => `Set ${SETTING_LABELS[String(args.key)] ?? String(args.key ?? '?')} to ${JSON.stringify(args.value)}`, + check: (args) => { + const key = args.key + if (typeof key !== 'string' || !(key in SETTING_LABELS)) { + return `key must be one of: ${Object.keys(SETTING_LABELS).join(', ')}.` + } + if (key === 'uiScale') { + return UI_SCALES.includes(args.value as UiScale) ? null : `uiScale must be one of: ${UI_SCALES.join(', ')}.` + } + return toBool(args.value) === null ? `${key} must be true or false.` : null + }, + async execute(args, deps) { + const key = args.key as keyof UiSettings + if (key === 'uiScale') deps.ui.setUiScale(args.value as UiScale) + else if (key === 'useAtkinsonFont') deps.ui.setUseAtkinsonFont(toBool(args.value)!) + else deps.ui.setShowRamIndicator(toBool(args.value)!) + return `Set ${SETTING_LABELS[key]} to ${JSON.stringify(deps.ui.get()[key])}.` + }, + }), + + set_storage_folder: defineTool({ + destructive: true, + describe: (args) => `Change the ${String(args.kind ?? '?')} folder to ${quote(String(args.path ?? '?'))}, moving its current files there`, + check: (args) => { + if (!STORAGE_KINDS.includes(args.kind as StorageKind)) return `kind must be one of: ${STORAGE_KINDS.join(', ')}.` + const path = str(args, 'path') + if (!path) return 'path is required.' + return isAbsolutePath(path) ? null : `${quote(path)} is not an absolute path.` + }, + async execute(args, deps) { + const kind = args.kind as StorageKind + const path = str(args, 'path')! + const dirs = await deps.storage.get() + const current = dirs[STORAGE_KEY[kind]] + const next = normPath(path) + if (current && normPath(current) === next) return `The ${kind} folder is already ${quote(current)}; nothing changed.` + if (current && (isInside(next, normPath(current)) || isInside(normPath(current), next))) { + return fail(`${quote(path)} overlaps the current ${kind} folder ${quote(current)}; choose a folder outside it. Nothing changed.`) + } + for (const [key, dir] of Object.entries(dirs) as [keyof StorageDirs, string][]) { + if (key === STORAGE_KEY[kind] || !dir) continue + if (isInside(next, normPath(dir)) || isInside(normPath(dir), next)) { + return fail(`${quote(path)} overlaps another Modly folder (${quote(dir)}); choose a separate folder. Nothing changed.`) + } + } + + const items = current ? await deps.storage.listDir(current).catch(() => [] as string[]) : [] + if (items.length > 0) { + const moved = await deps.storage.moveDirectory({ src: current, dest: path }) + if (!moved.success) { + return fail(`Could not move the ${kind} files to ${quote(path)}: ${moved.error ?? 'unknown error'}. Nothing changed.`) + } + } + try { + await deps.storage.apply(kind, path) + } catch (err) { + return fail(items.length + ? `Moved the ${kind} files to ${quote(path)}, but saving the new folder failed: ${errorText(err)}. Set it in Settings > Storage.` + : `Saving the new ${kind} folder failed: ${errorText(err)}. Nothing changed.`) + } + return items.length + ? `Moved ${items.length} item${items.length > 1 ? 's' : ''} from ${quote(current)} and set the ${kind} folder to ${quote(path)}.` + : `Set the ${kind} folder to ${quote(path)}.` + }, + }), + + undo_mesh: defineTool({ + describe: () => 'Undo the last mesh change', + check: () => null, + async execute(_args, deps) { + const { index } = deps.mesh.history() + if (index <= 0) return 'There is no mesh change to undo.' + deps.mesh.undo() + return 'Undid the last mesh change; the viewer shows the previous mesh.' + }, + }), + + redo_mesh: defineTool({ + describe: () => 'Redo the mesh change', + check: () => null, + async execute(_args, deps) { + const { index, length } = deps.mesh.history() + if (index >= length - 1) return 'There is no mesh change to redo.' + deps.mesh.redo() + return 'Redid the mesh change; the viewer shows the next mesh.' + }, + }), + + navigate: defineTool({ + describe: (args) => `Go to the ${String(args.page ?? '?')} page`, + check: (args) => (PAGES.includes(args.page as Page) ? null : `page must be one of: ${PAGES.join(', ')}.`), + async execute(args, deps) { + deps.navigate(args.page as Page) + return `Opened the ${String(args.page)} page.` + }, + }), +} + +const SETTING_LABELS: Record = { + uiScale: 'the interface scale', + useAtkinsonFont: 'the Atkinson Hyperlegible font', + showRamIndicator: 'the RAM indicator', +} + +const UI_SCALES: UiScale[] = ['small', 'medium', 'large', 'very-large'] + +const PAGES: Page[] = ['generate', 'workflows', 'models', 'settings'] + +function toBool(value: unknown): boolean | null { + if (typeof value === 'boolean') return value + if (value === 'true') return true + if (value === 'false') return false + return null +} + +/** The sentence for a tool call; falls back to the tool name for an unknown tool. */ +export function describeClientTool(tool: string, args: Args, deps?: ClientToolDeps): string { + const entry = clientTools[tool] + if (!entry) return tool.replace(/_/g, ' ') + try { + return entry.describe(args ?? {}, deps) + } catch { + return tool.replace(/_/g, ' ') + } +} + +// ─── Running a client_tool event ────────────────────────────────────────────── + +export interface ConfirmInfo { + tool: string + /** The `client_tool` event id, which the later `tool_done` carries too. */ + id: string + callId: string + destructive: boolean +} + +export type ConfirmFn = (description: string, info?: ConfirmInfo) => Promise + +export interface ClientToolOutcome { + description: string + result: string + error: boolean + declined: boolean + /** False when the result could not be sent back to the server. */ + posted: boolean +} + +/** + * Runs one `client_tool` event: asks for confirmation when the server says so, + * runs the tool, then posts the result to `/agent/tool-result`. Never throws. + */ +export async function runClientTool( + event: Pick, + deps: ClientToolDeps, + confirm: ConfirmFn, +): Promise { + const tool = clientTools[event.tool] + const args: Args = event.arguments && typeof event.arguments === 'object' && !Array.isArray(event.arguments) + ? event.arguments + : {} + const description = describeClientTool(event.tool, args, deps) + let result: string + let declined = false + + if (!tool) { + result = fail(`Unknown client tool ${quote(event.tool)}. Client tools: ${Object.keys(clientTools).join(', ')}.`) + } else { + const problem = tool.check(args, deps) + if (problem) { + // Nothing to confirm: the call cannot run as given. + result = fail(problem) + } else { + let ok = true + if (event.confirm) { + try { + ok = await confirm(description, { tool: event.tool, id: event.id, callId: event.call_id, destructive: tool.destructive }) + } catch { + ok = false + } + } + if (ok) { + result = await tool.run(args, deps) + } else { + declined = true + result = `The user declined: ${description}.` + } + } + } + + const error = isErrorResult(result) + let posted = true + try { + await deps.postResult({ call_id: event.call_id, result, ...(error ? { error: true } : {}) }) + } catch { + posted = false + } + return { description, result, error, declined, posted } +} + +// ─── generate_from_image ────────────────────────────────────────────────────── + +export interface GenerationDeps { + api: GenerationApi + store: GenerationStoreActions + /** The Generate page's current options (model, remesh, texture, params). */ + options: () => GenerationOptions + notifyComplete: (body: string) => void | Promise + notifyError: (body: string, title: string) => void | Promise + isCancelled?: () => boolean + wait?: () => Promise +} + +/** The first image of the most recent user message that has one. */ +export function latestUserImage(messages: { role: string; imageDataUrls?: string[] }[]): string | undefined { + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i] + if (m.role === 'user' && m.imageDataUrls?.length) return m.imageDataUrls[0] + } + return undefined +} + +/** + * Runs a `generate_from_image` payload through the Generate page's own path, + * so the result appears in its viewer. `image` is a data URL or bare base64. + * Resolves once the generation ends, with a sentence describing the outcome. + */ +export async function startGenerationFromImage( + payload: AgentActionPayload | null | undefined, + image: string | undefined, + deps: GenerationDeps, +): Promise { + if (payload?.type !== 'generate_from_image') return fail('Not a generate_from_image payload.') + const base64 = image?.startsWith('data:') ? image.slice(image.indexOf(',') + 1) : image + if (!base64) return fail('No image is attached to the conversation; attach one and ask again.') + + const current = deps.options() + const modelId = payload.model_id || current.modelId + if (!modelId) return fail('No generation model is selected or downloaded.') + const options: GenerationOptions = { + ...current, + modelId, + // Saved params belong to the selected model; another model starts from its defaults. + modelParams: { ...(modelId === current.modelId ? current.modelParams : {}), ...(payload.params ?? {}) }, + } + + try { + const outcome = await runGeneration('agent-image.png', options, base64, { + api: deps.api, + store: deps.store, + notifyComplete: deps.notifyComplete, + notifyError: deps.notifyError, + isCancelled: deps.isCancelled ?? (() => false), + wait: deps.wait, + }) + if (outcome.status === 'done') return `Generated a mesh with ${modelId}${outcome.outputUrl ? `: ${outcome.outputUrl}` : ''}.` + if (outcome.status === 'cancelled') return 'The generation was cancelled.' + return fail(`The generation failed: ${outcome.error}`) + } catch (err) { + return fail(`The generation failed: ${errorText(err)}`) + } +} + +// ─── App wiring ─────────────────────────────────────────────────────────────── + +/** The real dependencies: stores, IPC and the local API. */ +export function createClientToolDeps(): ClientToolDeps { + const ext = () => useExtensionsStore.getState() + const app = () => useAppStore.getState() + const wfs = () => useWorkflowsStore.getState() + return { + extensions: { + list: () => [...ext().modelExtensions, ...ext().processExtensions], + installFromGitHub: (url) => ext().installFromGitHub(url), + uninstall: (id) => ext().uninstall(id), + reload: () => ext().reload(), + loadErrors: () => ext().loadErrors, + }, + weights: { + download: (modelId, variantId) => window.electron.model.download(modelId, variantId), + delete: (modelId) => window.electron.model.delete(modelId), + deleteVariant: (modelId, variantId) => window.electron.model.deleteWeightVariant(modelId, variantId), + deleteExtensionWeights: (extensionId) => window.electron.model.deleteExtensionWeights(extensionId), + refresh: () => ext().refreshInstalledWeightVariants(), + }, + workflows: { + list: () => wfs().workflows, + remove: (id) => wfs().remove(id), + open: (id) => wfs().openWorkflow(id), + reload: () => wfs().load(), + }, + ui: { + get: () => ({ uiScale: app().uiScale, useAtkinsonFont: app().useAtkinsonFont, showRamIndicator: app().showRamIndicator }), + setUiScale: (v) => app().setUiScale(v), + setUseAtkinsonFont: (v) => app().setUseAtkinsonFont(v), + setShowRamIndicator: (v) => app().setShowRamIndicator(v), + }, + storage: { + get: () => window.electron.settings.get(), + listDir: (dirPath) => window.electron.fs.listDir(dirPath), + moveDirectory: (args) => window.electron.fs.moveDirectory(args), + async apply(kind, path) { + // Same calls as Settings > Storage. + if (kind === 'models') { + await window.electron.settings.set({ modelsDir: path }) + await window.electron.api.updatePaths({ modelsDir: path }) + } else if (kind === 'workspace') { + await window.electron.settings.set({ workspaceDir: path }) + await window.electron.api.updatePaths({ workspaceDir: path }) + } else { + await window.electron.settings.set({ workflowsDir: path }) + await wfs().load() + } + }, + }, + mesh: { + history: () => ({ index: app().historyIndex, length: app().meshHistory.length }), + undo: () => app().undoMesh(), + redo: () => app().redoMesh(), + }, + navigate: (page) => useNavStore.getState().navigate(page), + async postResult(body) { + const res = await fetch(`${app().apiUrl}/agent/tool-result`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + if (!res.ok) throw new Error(`API error ${res.status}`) + }, + } +} + +/** The real dependencies of startGenerationFromImage. */ +export function createGenerationDeps(): GenerationDeps { + const app = () => useAppStore.getState() + return { + api: createGenerationApi(app().apiUrl), + store: { + setCurrentJob: (job) => app().setCurrentJob(job), + updateCurrentJob: (patch) => app().updateCurrentJob(patch), + pushMeshUrl: (url) => app().pushMeshUrl(url), + clearMeshHistory: () => app().clearMeshHistory(), + }, + options: () => app().generationOptions, + notifyComplete: (body) => showCompletionNotification(body), + notifyError: (body, title) => showErrorNotification(body, title), + } +} From e3d3ca530c294c244cdc3ffa6f4deaaf94fb460b Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:16:25 +0100 Subject: [PATCH 08/13] feat(agent): wire client tools, confirm cards and app-wide context into the chat The chat runs client_tool events through the client tool registry (after a Confirm click for destructive ones), starts generate_from_image after the turn, and sends the app-wide context. Undo/redo checks read the mesh history depths. --- src/shared/components/agent/AgentChat.tsx | 42 ++++++++++++------- .../components/agent/clientTools.test.ts | 8 ++-- src/shared/components/agent/clientTools.ts | 12 +++--- 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/src/shared/components/agent/AgentChat.tsx b/src/shared/components/agent/AgentChat.tsx index d155c80cb..93b8e9f43 100644 --- a/src/shared/components/agent/AgentChat.tsx +++ b/src/shared/components/agent/AgentChat.tsx @@ -11,6 +11,9 @@ import { adviceText, ollamaAdvice } from '@shared/utils/ollamaHealth' import OllamaBanner, { useOllamaHealth } from '@areas/generate/components/OllamaBanner' import { commitSculpt } from '@areas/generate/sculpt/sculptStore' import { useAgentConversationStore } from '@shared/stores/agentConversationStore' +import { buildAgentContext } from './agentContext' +import { createClientToolDeps, createGenerationDeps, latestUserImage, runClientTool, startGenerationFromImage } from './clientTools' +import { QueuedConfirmCard, useConfirmQueue } from './ConfirmCard' // ─── Types ──────────────────────────────────────────────────────────────────── @@ -295,6 +298,10 @@ export default function AgentChat(): JSX.Element { const applyMeshEdit = useAppStore((s) => s.applyMeshEdit) const undoMesh = useAppStore((s) => s.undoMesh) + const clientDeps = useMemo(() => createClientToolDeps(), []) + const confirm = useConfirmQueue((s) => s.confirm) + const confirmRequests = useConfirmQueue((s) => s.requests) + const isOllama = provider === 'ollama' const ollama = useOllamaHealth(isOllama, apiUrl, providerBaseUrl(provider, externalConfig), model) @@ -363,19 +370,6 @@ export default function AgentChat(): JSX.Element { endRef.current?.scrollIntoView({ behavior: isLoading ? 'auto' : 'smooth' }) }, [messages, isLoading, pendingWorkflow]) - function buildContext(): Record { - const ctx: Record = {} - // Read from the store, not the render: a sculpt commit may just have swapped the mesh. - const { currentJob, meshStats } = useAppStore.getState() - if (currentJob?.outputUrl) ctx.currentMeshPath = currentJob.outputUrl.replace('/workspace/', '') - if (meshStats?.triangles) ctx.meshTriangles = meshStats.triangles - if (workflows.length > 0) ctx.workflows = workflows.map((w) => ({ id: w.id, name: w.name })) - if (allExtensions.length > 0) ctx.extensions = allExtensions.map((e) => ({ - id: e.id, name: e.name, input: e.input, output: e.output, - })) - return ctx - } - async function callAgent(msgs: Message[], extraContext: Record = {}) { setIsLoading(true) setError(null) @@ -383,7 +377,9 @@ export default function AgentChat(): JSX.Element { let finished = false let failure: string | null = null try { - const context = { ...buildContext(), ...extraContext } + // Read at send time: the page, the mesh (a sculpt commit may just have + // swapped it), workflows, extensions, models, folders and UI settings. + const context: Record = { ...(await buildAgentContext()), ...extraContext } // Inject workflow completion as a system hint if present const apiMessages = msgs.map((m) => { @@ -437,6 +433,12 @@ export default function AgentChat(): JSX.Element { case 'tool_start': patch((m) => ({ ...m, actions: [...(m.actions ?? []), { id: event.id, tool: event.tool, result: '', running: true }] })) break + case 'client_tool': + // Runs in the app (stores / IPC), after a Confirm click when asked; + // the server waits for its result, then sends the usual tool_done. + // Not awaited: the stream must keep reading meanwhile. + void runClientTool(event, clientDeps, confirm) + break case 'tool_done': patch((m) => ({ ...m, @@ -463,6 +465,7 @@ export default function AgentChat(): JSX.Element { // Started only now: the local LLM frees its VRAM at the end of the // turn, and the workflow needs the whole GPU. for (const action of event.actions) startWorkflow(action.payload, msgs) + for (const action of event.actions) startGeneration(action.payload, msgs) break } } @@ -479,6 +482,8 @@ export default function AgentChat(): JSX.Element { ? { ...m, streaming: false, actions: m.actions?.map((a) => ({ ...a, running: false })) } : m))) } + // A card still waiting when the turn ends has no one listening any more. + useConfirmQueue.getState().cancelAll() setIsLoading(false) } } @@ -507,6 +512,14 @@ export default function AgentChat(): JSX.Element { } } + // Started after `done`, like workflows: the local LLM frees the GPU first. + function startGeneration(payload: AgentActionPayload | null | undefined, msgs: Message[]) { + if (payload?.type !== 'generate_from_image') return + void startGenerationFromImage(payload, latestUserImage(msgs), createGenerationDeps()).then((result) => { + setMessages((prev) => [...prev, { id: `sys-${Date.now()}`, role: 'assistant', content: result }]) + }) + } + function startWorkflow(payload: AgentActionPayload | null | undefined, msgs: Message[]) { if (payload?.type !== 'run_workflow' || !payload.workflow_id) return const wf = workflows.find((w) => w.id === payload.workflow_id) @@ -685,6 +698,7 @@ export default function AgentChat(): JSX.Element {
)} + {confirmRequests.map((request) => )}
diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts index cda280082..30bc265db 100644 --- a/src/shared/components/agent/clientTools.test.ts +++ b/src/shared/components/agent/clientTools.test.ts @@ -54,7 +54,7 @@ interface Fake { posted: ToolResultBody[] ui: { uiScale: UiScale; useAtkinsonFont: boolean; showRamIndicator: boolean } dirs: { modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string } - history: { index: number; length: number } + history: { undo: number; redo: number } dirItems: Record } @@ -66,7 +66,7 @@ function fake(overrides: Partial = {}): Fake { const state = { ui: { uiScale: 'medium' as UiScale, useAtkinsonFont: false, showRamIndicator: true }, dirs: { modelsDir: 'C:\\Modly\\models', workspaceDir: 'C:\\Modly\\workspace', workflowsDir: 'C:\\Modly\\workflows', extensionsDir: 'C:\\Modly\\extensions' }, - history: { index: 1, length: 3 }, + history: { undo: 1, redo: 1 }, dirItems: {} as Record, } const list = extensions() @@ -292,8 +292,8 @@ test('undo_mesh and redo_mesh step through the history, and say when there is no assert.match(await run('redo_mesh', {}, f), /^Redid/) assert.deepEqual(f.calls, [['undo'], ['redo']]) - f.history.index = 0 - f.history.length = 1 + f.history.undo = 0 + f.history.redo = 0 assert.equal(await run('undo_mesh', {}, f), 'There is no mesh change to undo.') assert.equal(await run('redo_mesh', {}, f), 'There is no mesh change to redo.') assert.equal(f.calls.length, 2) diff --git a/src/shared/components/agent/clientTools.ts b/src/shared/components/agent/clientTools.ts index fc4221333..e181126e9 100644 --- a/src/shared/components/agent/clientTools.ts +++ b/src/shared/components/agent/clientTools.ts @@ -8,6 +8,7 @@ import { useAppStore } from '../../stores/appStore' import { useExtensionsStore } from '../../stores/extensionsStore' import { useWorkflowsStore } from '../../stores/workflowsStore' import { useNavStore } from '../../stores/navStore' +import { redoDepth, undoDepth } from '../../stores/meshHistory' import { showCompletionNotification, showRunErrorNotification } from '../../utils/notification' // Tools the agent asks the frontend to run, because they need IPC or the @@ -76,7 +77,8 @@ export interface ClientToolDeps { apply: (kind: StorageKind, path: string) => Promise } mesh: { - history: () => { index: number; length: number } + /** Steps available each way on the mesh undo stack. */ + history: () => { undo: number; redo: number } undo: () => void redo: () => void } @@ -482,8 +484,7 @@ export const clientTools: Record = { describe: () => 'Undo the last mesh change', check: () => null, async execute(_args, deps) { - const { index } = deps.mesh.history() - if (index <= 0) return 'There is no mesh change to undo.' + if (deps.mesh.history().undo <= 0) return 'There is no mesh change to undo.' deps.mesh.undo() return 'Undid the last mesh change; the viewer shows the previous mesh.' }, @@ -493,8 +494,7 @@ export const clientTools: Record = { describe: () => 'Redo the mesh change', check: () => null, async execute(_args, deps) { - const { index, length } = deps.mesh.history() - if (index >= length - 1) return 'There is no mesh change to redo.' + if (deps.mesh.history().redo <= 0) return 'There is no mesh change to redo.' deps.mesh.redo() return 'Redid the mesh change; the viewer shows the next mesh.' }, @@ -727,7 +727,7 @@ export function createClientToolDeps(): ClientToolDeps { }, }, mesh: { - history: () => ({ index: app().historyIndex, length: app().meshHistory.length }), + history: () => ({ undo: undoDepth(app().meshHistory), redo: redoDepth(app().meshHistory) }), undo: () => app().undoMesh(), redo: () => app().redoMesh(), }, From 02d77c64dbda46bb292a81c54719fba8e29b6ae9 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:18:36 +0100 Subject: [PATCH 09/13] fix(agent): confirm extension installs, and enforce confirmation in the app install_extension runs code downloaded from GitHub, so it now needs a Confirm click showing the URL. The app also asks for destructive tools and installs whatever the server's confirm flag says, so a backend slip cannot bypass the card. --- api/routers/agent.py | 3 ++- api/tests/test_agent_client_tools.py | 5 +++-- src/shared/components/agent/clientTools.test.ts | 14 ++++++++++++++ src/shared/components/agent/clientTools.ts | 11 ++++++++++- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/api/routers/agent.py b/api/routers/agent.py index 797e40461..884d730d3 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -323,7 +323,8 @@ def _function(name: str, description: str, properties: dict | None = None, requi # Tools the app runs itself (Electron IPC, renderer stores), mapped to whether # the user must confirm them first. The frontend obeys `confirm`. CLIENT_TOOLS: dict[str, bool] = { - "install_extension": False, + # Installing runs code downloaded from GitHub: the user must see the URL and click. + "install_extension": True, "uninstall_extension": True, "reload_extensions": False, "download_model_weights": False, diff --git a/api/tests/test_agent_client_tools.py b/api/tests/test_agent_client_tools.py index 999a9ce01..e27d61773 100644 --- a/api/tests/test_agent_client_tools.py +++ b/api/tests/test_agent_client_tools.py @@ -61,7 +61,7 @@ async def answer(event): {k: v for k, v in handoff.items() if k != "call_id"}, { "type": "client_tool", "id": "c1", "tool": "install_extension", - "arguments": {"github_url": "https://github.com/a/b"}, "confirm": False, + "arguments": {"github_url": "https://github.com/a/b"}, "confirm": True, }, ) self.assertRegex(handoff["call_id"], r"^[0-9a-f]{32}$") @@ -233,7 +233,8 @@ def test_every_client_tool_is_declared_with_its_confirmation(self) -> None: confirmed = {name for name, confirm in agent.CLIENT_TOOLS.items() if confirm} self.assertEqual( confirmed, - {"uninstall_extension", "delete_model_weights", "delete_workflow", "set_storage_folder"}, + # install_extension runs code downloaded from GitHub: never without a click. + {"install_extension", "uninstall_extension", "delete_model_weights", "delete_workflow", "set_storage_folder"}, ) def test_every_client_tool_has_a_schema(self) -> None: diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts index 30bc265db..4a6f49a4f 100644 --- a/src/shared/components/agent/clientTools.test.ts +++ b/src/shared/components/agent/clientTools.test.ts @@ -345,6 +345,20 @@ test('runClientTool reports a declined confirmation without running the tool', a assert.equal(outcome.declined, true) }) +test('runClientTool asks for code installs and destructive tools even when the event says no confirm', async () => { + for (const [tool, args] of [ + ['install_extension', { github_url: 'https://github.com/a/b' }], + ['delete_workflow', { workflow_id: 'wf1' }], + ] as const) { + const f = fake() + let asked = false + const outcome = await runClientTool(event(tool, { ...args }, false), f.deps, async () => { asked = true; return false }) + assert.equal(asked, true, tool) + assert.equal(outcome.declined, true, tool) + assert.deepEqual(f.calls, [], tool) + } +}) + test('runClientTool does not ask about a call that cannot run, and flags errors', async () => { const f = fake() let asked = false diff --git a/src/shared/components/agent/clientTools.ts b/src/shared/components/agent/clientTools.ts index e181126e9..05f162225 100644 --- a/src/shared/components/agent/clientTools.ts +++ b/src/shared/components/agent/clientTools.ts @@ -235,6 +235,11 @@ function isInside(child: string, parent: string): boolean { export interface ClientTool { /** Destructive: the confirm card shows a red Confirm button. */ destructive: boolean + /** + * Always asks before running, whatever the server's `confirm` says: true for + * destructive tools and for installs (they run code downloaded from GitHub). + */ + requiresConfirm: boolean /** The human sentence for the confirm card and the action list. */ describe: (args: Args, deps?: ClientToolDeps) => string /** Why the arguments cannot be run, or null when they can. */ @@ -245,6 +250,8 @@ export interface ClientTool { interface ToolSpec { destructive?: boolean + /** Ask even when not destructive (see ClientTool.requiresConfirm). */ + requiresConfirm?: boolean describe: (args: Args, deps?: ClientToolDeps) => string check: (args: Args, deps: ClientToolDeps) => string | null execute: (args: Args, deps: ClientToolDeps) => Promise @@ -253,6 +260,7 @@ interface ToolSpec { function defineTool(spec: ToolSpec): ClientTool { return { destructive: spec.destructive ?? false, + requiresConfirm: (spec.destructive ?? false) || (spec.requiresConfirm ?? false), describe: spec.describe, check: (args, deps) => { try { @@ -277,6 +285,7 @@ const checked = (value: T | string): string | null => (typeof value === 'stri export const clientTools: Record = { install_extension: defineTool({ + requiresConfirm: true, describe: (args) => `Install the extension from ${quote(String(args.github_url ?? '?'))}`, check: (args) => { const url = str(args, 'github_url') @@ -585,7 +594,7 @@ export async function runClientTool( result = fail(problem) } else { let ok = true - if (event.confirm) { + if (event.confirm || tool.requiresConfirm) { try { ok = await confirm(description, { tool: event.tool, id: event.id, callId: event.call_id, destructive: tool.destructive }) } catch { From 119ce1ccb456431cb558da57165df76c6ecb29d8 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:23:13 +0100 Subject: [PATCH 10/13] fix(agent): review fixes for the side panel - set_storage_folder moves what Settings would (storageEntries, so a workflows folder of JSON files moves too), refuses what Settings refuses, and reports items left behind. - set_setting declares uiScale by its scale names, matching the app. - A queued confirm card opens the panel; a card the server stopped waiting for is cancelled; answered cards are cleared per turn and per conversation. - An agent-started generation keeps the conversation busy, so its result cannot land in another one. - A failed conversation save stays dirty and blocks switching away; temp files get unique names. --- api/routers/agent.py | 6 ++-- api/tests/test_agent_client_tools.py | 9 ++++-- electron/main/agent-conversations.ts | 3 +- src/shared/components/agent/AgentChat.tsx | 17 ++++++++--- src/shared/components/agent/AgentPanel.tsx | 12 ++++++++ .../components/agent/ConfirmCard.test.mjs | 22 +++++++++++++++ src/shared/components/agent/ConfirmCard.tsx | 13 +++++++++ .../components/agent/clientTools.test.ts | 28 +++++++++++++++++-- src/shared/components/agent/clientTools.ts | 27 +++++++++++++----- src/shared/stores/agentConversationStore.ts | 20 ++++++++----- 10 files changed, 129 insertions(+), 28 deletions(-) diff --git a/api/routers/agent.py b/api/routers/agent.py index 884d730d3..1e067b4a3 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -104,7 +104,7 @@ def registry_tool_lines(registry: MeshOpsRegistry) -> str: - **install_extension(github_url)** / **uninstall_extension(extension_id)** / **reload_extensions** — Manage extensions. - **download_model_weights(extension_id, variant?)** / **delete_model_weights(extension_id, variant?)** — Manage model weights. - **open_workflow(workflow_id)** / **delete_workflow(workflow_id)** — Open a workflow in the editor, or delete it. -- **set_setting(key, value)** — Change `uiScale` (number), `useAtkinsonFont` or `showRamIndicator` (booleans). +- **set_setting(key, value)** — Change `uiScale` (one of `small`, `medium`, `large`, `very-large`), `useAtkinsonFont` or `showRamIndicator` (booleans). - **set_storage_folder(kind, path)** — Move the `models`, `workspace` or `workflows` folder. - **undo_mesh** / **redo_mesh** — Undo or redo the last change to the current mesh. - **navigate(page)** — Switch the app to another page. @@ -378,10 +378,10 @@ def _function(name: str, description: str, properties: dict | None = None, requi ), _function( "set_setting", - "Change a UI setting. uiScale is a number (1 = 100%); useAtkinsonFont and showRamIndicator are booleans.", + "Change a UI setting. uiScale is one of small, medium, large, very-large; useAtkinsonFont and showRamIndicator are booleans.", { "key": {"type": "string", "enum": ["uiScale", "useAtkinsonFont", "showRamIndicator"]}, - "value": {"type": ["number", "boolean"], "description": "A number for uiScale, a boolean otherwise."}, + "value": {"type": ["string", "boolean"], "description": "For uiScale one of: small, medium, large, very-large. A boolean otherwise."}, }, ["key", "value"], ), diff --git a/api/tests/test_agent_client_tools.py b/api/tests/test_agent_client_tools.py index e27d61773..d6fabfca1 100644 --- a/api/tests/test_agent_client_tools.py +++ b/api/tests/test_agent_client_tools.py @@ -247,6 +247,9 @@ def test_every_client_tool_has_a_schema(self) -> None: def test_enums(self) -> None: props = self._tool("set_setting")["parameters"]["properties"] self.assertEqual(props["key"]["enum"], ["uiScale", "useAtkinsonFont", "showRamIndicator"]) + # The app's scale names, not a number: the frontend accepts only these. + self.assertEqual(props["value"]["type"], ["string", "boolean"]) + self.assertIn("small, medium, large, very-large", props["value"]["description"]) props = self._tool("set_storage_folder")["parameters"]["properties"] self.assertEqual(props["kind"]["enum"], ["models", "workspace", "workflows"]) nav = self._tool("navigate") @@ -271,7 +274,7 @@ def test_system_prompt_lists_the_new_tools(self) -> None: "modelsDir": "D:/models", "workspaceDir": "D:/ws", "workflowsDir": "D:/wf", "extensionsDir": "D:/ext", }, - "ui": {"uiScale": 1.1, "useAtkinsonFont": False, "showRamIndicator": True}, + "ui": {"uiScale": "large", "useAtkinsonFont": False, "showRamIndicator": True}, } @@ -288,7 +291,7 @@ def test_list_extensions(self) -> None: def test_get_settings(self) -> None: text, payload = _run("get_settings") - for expected in ("D:/models", "D:/ws", "D:/wf", "D:/ext", "uiScale: 1.1", "useAtkinsonFont: false", "showRamIndicator: true"): + for expected in ("D:/models", "D:/ws", "D:/wf", "D:/ext", "uiScale: large", "useAtkinsonFont: false", "showRamIndicator: true"): self.assertIn(expected, text) self.assertIsNone(payload) self.assertIn("No settings", _run("get_settings", context={})[0]) @@ -349,7 +352,7 @@ def test_context_is_rendered_into_the_single_system_message(self) -> None: "- sf3d: SF3D", "- wf1: My Workflow", "D:/models", "D:/ws", "D:/wf", "D:/ext", - "uiScale: 1.1", "useAtkinsonFont: false", "showRamIndicator: true", + "uiScale: large", "useAtkinsonFont: false", "showRamIndicator: true", "Current mesh path: Default/mesh.glb", "- trellis/gen (image→mesh): Trellis", ): diff --git a/electron/main/agent-conversations.ts b/electron/main/agent-conversations.ts index 24c130fd2..8a2db16e4 100644 --- a/electron/main/agent-conversations.ts +++ b/electron/main/agent-conversations.ts @@ -2,6 +2,7 @@ // /agent/conversations/{id}.json. import { join } from 'path' +import { randomUUID } from 'crypto' import { mkdir, readFile, readdir, rename, rm, writeFile } from 'fs/promises' export interface ConversationSummary { @@ -75,7 +76,7 @@ export function createConversationStorage(dir: string) { await mkdir(dir, { recursive: true }) const target = fileFor(conversation.id) // Write then rename: a crash mid-write never leaves a truncated file. - const tmp = `${target}.${process.pid}.${Date.now()}.tmp` + const tmp = `${target}.${randomUUID()}.tmp` await writeFile(tmp, JSON.stringify(conversation), 'utf-8') try { await rename(tmp, target) diff --git a/src/shared/components/agent/AgentChat.tsx b/src/shared/components/agent/AgentChat.tsx index 93b8e9f43..6f1f1c03f 100644 --- a/src/shared/components/agent/AgentChat.tsx +++ b/src/shared/components/agent/AgentChat.tsx @@ -282,6 +282,8 @@ export default function AgentChat(): JSX.Element { const localModels = llmModels.filter((m) => m.downloaded && !(m.tags ?? []).some((t) => t === 'code' || t === 'cad')) const [showModelPicker, setShowModelPicker] = useState(false) const [pendingWorkflow, setPendingWorkflow] = useState<{ id: string; name: string } | null>(null) + // Generations the agent started; the conversation stays put until they end. + const [generations, setGenerations] = useState(0) const [attachments, setAttachments] = useState([]) // data URLs const [isDragging, setIsDragging] = useState(false) const [thinkingMode, setThinkingMode] = useState(defaultThinking) @@ -357,12 +359,14 @@ export default function AgentChat(): JSX.Element { }, [runState.status, pendingWorkflow]) // The panel locks conversation switching while a turn or its workflow runs. - useEffect(() => { setBusy(isLoading || pendingWorkflow !== null) }, [isLoading, pendingWorkflow, setBusy]) + useEffect(() => { setBusy(isLoading || pendingWorkflow !== null || generations > 0) }, [isLoading, pendingWorkflow, generations, setBusy]) // Another conversation was opened: drop the view state of the previous one. useEffect(() => { setShowAll(false) setError(null) + useConfirmQueue.getState().cancelAll() + useConfirmQueue.getState().clearAnswered() }, [currentId]) useEffect(() => { @@ -371,6 +375,7 @@ export default function AgentChat(): JSX.Element { }, [messages, isLoading, pendingWorkflow]) async function callAgent(msgs: Message[], extraContext: Record = {}) { + useConfirmQueue.getState().clearAnswered() setIsLoading(true) setError(null) const id = `a-${Date.now()}` @@ -440,6 +445,7 @@ export default function AgentChat(): JSX.Element { void runClientTool(event, clientDeps, confirm) break case 'tool_done': + useConfirmQueue.getState().cancelEvent(event.id) patch((m) => ({ ...m, actions: (m.actions ?? []).map((a) => (a.id === event.id @@ -515,9 +521,12 @@ export default function AgentChat(): JSX.Element { // Started after `done`, like workflows: the local LLM frees the GPU first. function startGeneration(payload: AgentActionPayload | null | undefined, msgs: Message[]) { if (payload?.type !== 'generate_from_image') return - void startGenerationFromImage(payload, latestUserImage(msgs), createGenerationDeps()).then((result) => { - setMessages((prev) => [...prev, { id: `sys-${Date.now()}`, role: 'assistant', content: result }]) - }) + setGenerations((n) => n + 1) + void startGenerationFromImage(payload, latestUserImage(msgs), createGenerationDeps()) + .then((result) => { + setMessages((prev) => [...prev, { id: `sys-${Date.now()}`, role: 'assistant', content: result }]) + }) + .finally(() => setGenerations((n) => n - 1)) } function startWorkflow(payload: AgentActionPayload | null | undefined, msgs: Message[]) { diff --git a/src/shared/components/agent/AgentPanel.tsx b/src/shared/components/agent/AgentPanel.tsx index c3ba61c90..f2d68b158 100644 --- a/src/shared/components/agent/AgentPanel.tsx +++ b/src/shared/components/agent/AgentPanel.tsx @@ -9,6 +9,7 @@ import { import { useAgentConversationStore } from '@shared/stores/agentConversationStore' import { UNTITLED, deriveTitle, relativeDate } from '@shared/utils/agentConversations' import AgentChat from './AgentChat' +import { useConfirmQueue } from './ConfirmCard' // ─── Helpers ────────────────────────────────────────────────────────────────── @@ -137,6 +138,17 @@ export default function AgentPanel(): JSX.Element { return () => window.removeEventListener('beforeunload', onUnload) }, []) + // A confirm card is waiting for a click: show it, even if the panel was + // closed mid-turn or the conversation list covered the chat. + useEffect(() => { + const { setOnQueued } = useConfirmQueue.getState() + setOnQueued(() => { + setOpen(true) + setShowList(false) + }) + return () => setOnQueued(null) + }, [setOpen]) + // Ctrl+L / Cmd+L toggles the panel, except while typing elsewhere. useEffect(() => { const handler = (e: KeyboardEvent) => { diff --git a/src/shared/components/agent/ConfirmCard.test.mjs b/src/shared/components/agent/ConfirmCard.test.mjs index da117e50a..a9553ed95 100644 --- a/src/shared/components/agent/ConfirmCard.test.mjs +++ b/src/shared/components/agent/ConfirmCard.test.mjs @@ -64,3 +64,25 @@ test('cancelAll declines every pending request', async () => { test('ConfirmCard is a component', () => { assert.equal(typeof ConfirmCard, 'function') }) + +test('cancelEvent cancels the pending card of a tool call the server gave up on', async () => { + const answer = queue().confirm('Delete the workflow “Turntable”', { tool: 'delete_workflow', id: 'e9', callId: 'c9', destructive: true }) + const other = queue().confirm('Uninstall the extension “X”', { tool: 'uninstall_extension', id: 'e10', callId: 'c10', destructive: true }) + queue().cancelEvent('e9') + assert.equal(await answer, false) + assert.deepEqual(queue().requests.map((r) => r.status), ['cancelled', 'pending']) + queue().cancelAll() + assert.equal(await other, false) + queue().clearAnswered() +}) + +test('a queued confirm calls onQueued, so the panel can open itself', async () => { + let opened = 0 + queue().setOnQueued(() => { opened++ }) + const answer = queue().confirm('Install the extension from “https://github.com/a/b”', { tool: 'install_extension', id: 'e11', callId: 'c11', destructive: false }) + assert.equal(opened, 1) + queue().cancelAll() + assert.equal(await answer, false) + queue().setOnQueued(null) + queue().clearAnswered() +}) diff --git a/src/shared/components/agent/ConfirmCard.tsx b/src/shared/components/agent/ConfirmCard.tsx index 55aeb6a7d..e406b28f2 100644 --- a/src/shared/components/agent/ConfirmCard.tsx +++ b/src/shared/components/agent/ConfirmCard.tsx @@ -26,6 +26,11 @@ interface ConfirmQueueState { cancelAll: () => void /** Forgets answered requests. */ clearAnswered: () => void + /** Cancels the pending card of one client_tool event (the server stopped waiting for it). */ + cancelEvent: (eventId: string) => void + /** Called whenever a card is queued; the panel uses it to open itself so the card is seen. */ + onQueued: (() => void) | null + setOnQueued: (callback: (() => void) | null) => void } // Resolvers stay out of the state: they are not data to render. @@ -49,6 +54,7 @@ export const useConfirmQueue = create((set, get) => ({ return new Promise((resolve) => { resolvers.set(key, resolve) set((s) => ({ requests: [...s.requests, request] })) + get().onQueued?.() }) }, @@ -69,6 +75,13 @@ export const useConfirmQueue = create((set, get) => ({ clearAnswered() { set((s) => ({ requests: s.requests.filter((r) => r.status === 'pending') })) }, + + cancelEvent(eventId) { + for (const r of get().requests) if (r.status === 'pending' && r.eventId === eventId) get().answer(r.key, false) + }, + + onQueued: null, + setOnQueued: (callback) => set({ onQueued: callback }), })) // ─── Card ───────────────────────────────────────────────────────────────────── diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts index 4a6f49a4f..62082de08 100644 --- a/src/shared/components/agent/clientTools.test.ts +++ b/src/shared/components/agent/clientTools.test.ts @@ -103,7 +103,8 @@ function fake(overrides: Partial = {}): Fake { }, storage: { get: async () => state.dirs, - listDir: async (dir) => state.dirItems[dir] ?? [], + entries: async (kind: StorageKind) => ({ modly: state.dirItems[kind] ?? [], other: [] }), + checkLocation: async (path: string) => (path.startsWith('C:\\Windows') ? { ok: false, error: 'This is a system folder.' } : { ok: true }), moveDirectory: async (args) => { rec('move', args.src, args.dest); return ok }, apply: async (kind: StorageKind, path: string) => { rec('applyStorage', kind, path) }, }, @@ -256,12 +257,33 @@ test('set_setting sets each UI option and validates key and value', async () => test('set_storage_folder moves existing files, then saves the folder', async () => { const f = fake() - f.dirItems['C:\\Modly\\models'] = ['triposr', 'hunyuan'] + f.dirItems.models = ['triposr', 'hunyuan'] const result = await run('set_storage_folder', { kind: 'models', path: 'D:\\AI\\models' }, f) assert.deepEqual(f.calls, [['move', 'C:\\Modly\\models', 'D:\\AI\\models'], ['applyStorage', 'models', 'D:\\AI\\models']]) assert.equal(result, 'Moved 2 items from “C:\\Modly\\models” and set the models folder to “D:\\AI\\models”.') }) +test('set_storage_folder moves a workflows folder that holds only files', async () => { + const f = fake() + f.dirItems.workflows = ['turntable.json'] + const result = await run('set_storage_folder', { kind: 'workflows', path: 'D:/flows' }, f) + assert.deepEqual(f.calls, [['move', 'C:\\Modly\\workflows', 'D:/flows'], ['applyStorage', 'workflows', 'D:/flows']]) + assert.match(result, /^Moved 1 item from/) +}) + +test('set_storage_folder refuses a location Settings would refuse', async () => { + const f = fake() + assert.equal(await run('set_storage_folder', { kind: 'models', path: 'C:\\Windows\\m' }, f), 'Error: This is a system folder. Nothing changed.') + assert.deepEqual(f.calls, []) +}) + +test('set_storage_folder says which items stayed behind', async () => { + const f = fake() + f.dirItems.models = ['a', 'b'] + f.deps.storage.moveDirectory = async (args) => { f.calls.push(['move', args.src, args.dest]); return { success: true, kept: ['b'] } } + assert.match(await run('set_storage_folder', { kind: 'models', path: 'D:/m' }, f), /^Moved 1 item from .*; 1 item could not be moved and stayed in “C:\\Modly\\models”\.$/) +}) + test('set_storage_folder with an empty folder only saves it', async () => { const f = fake() assert.equal(await run('set_storage_folder', { kind: 'workflows', path: 'D:/flows' }, f), 'Set the workflows folder to “D:/flows”.') @@ -278,7 +300,7 @@ test('set_storage_folder refuses invalid locations and keeps everything on a fai assert.match(await run('set_storage_folder', { kind: 'models', path: 'C:/Modly/models/' }, f), /already/) assert.deepEqual(f.calls, []) - f.dirItems['C:\\Modly\\workspace'] = ['a.glb'] + f.dirItems.workspace = ['a.glb'] f.deps.storage.moveDirectory = async () => ({ success: false, error: 'EACCES' }) assert.equal(await run('set_storage_folder', { kind: 'workspace', path: 'E:/ws' }, f), 'Error: Could not move the workspace files to “E:/ws”: EACCES. Nothing changed.') assert.deepEqual(f.calls, []) diff --git a/src/shared/components/agent/clientTools.ts b/src/shared/components/agent/clientTools.ts index 05f162225..a9bdeeb78 100644 --- a/src/shared/components/agent/clientTools.ts +++ b/src/shared/components/agent/clientTools.ts @@ -71,8 +71,11 @@ export interface ClientToolDeps { } storage: { get: () => Promise - listDir: (dirPath: string) => Promise - moveDirectory: (args: { src: string; dest: string }) => Promise + /** What Modly created in the current folder (moved) and what it did not (left alone), as Settings sees it. */ + entries: (kind: StorageKind) => Promise<{ modly: string[]; other: string[] }> + /** Settings' guard: refuses a drive root, a system folder or an ancestor of home/app data. */ + checkLocation: (path: string) => Promise<{ ok: boolean; error?: string }> + moveDirectory: (args: { src: string; dest: string }) => Promise /** Saves the new folder, the way Settings > Storage does once files are moved. */ apply: (kind: StorageKind, path: string) => Promise } @@ -469,12 +472,18 @@ export const clientTools: Record = { } } - const items = current ? await deps.storage.listDir(current).catch(() => [] as string[]) : [] + const location = await deps.storage.checkLocation(path).catch((err) => ({ ok: false, error: errorText(err) })) + if (!location.ok) return fail(`${location.error ?? `${quote(path)} cannot hold Modly data.`} Nothing changed.`) + + // Files count too: a workflows folder holds only JSON files. + const items = current ? (await deps.storage.entries(kind).catch(() => ({ modly: [] as string[] }))).modly : [] + let kept: string[] = [] if (items.length > 0) { const moved = await deps.storage.moveDirectory({ src: current, dest: path }) if (!moved.success) { return fail(`Could not move the ${kind} files to ${quote(path)}: ${moved.error ?? 'unknown error'}. Nothing changed.`) } + kept = moved.kept ?? [] } try { await deps.storage.apply(kind, path) @@ -483,9 +492,12 @@ export const clientTools: Record = { ? `Moved the ${kind} files to ${quote(path)}, but saving the new folder failed: ${errorText(err)}. Set it in Settings > Storage.` : `Saving the new ${kind} folder failed: ${errorText(err)}. Nothing changed.`) } - return items.length - ? `Moved ${items.length} item${items.length > 1 ? 's' : ''} from ${quote(current)} and set the ${kind} folder to ${quote(path)}.` - : `Set the ${kind} folder to ${quote(path)}.` + if (!items.length) return `Set the ${kind} folder to ${quote(path)}.` + const count = (n: number) => `${n} item${n > 1 ? 's' : ''}` + const movedCount = items.length - kept.length + return kept.length + ? `Moved ${count(movedCount)} from ${quote(current)} and set the ${kind} folder to ${quote(path)}; ${count(kept.length)} could not be moved and stayed in ${quote(current)}.` + : `Moved ${count(movedCount)} from ${quote(current)} and set the ${kind} folder to ${quote(path)}.` }, }), @@ -719,7 +731,8 @@ export function createClientToolDeps(): ClientToolDeps { }, storage: { get: () => window.electron.settings.get(), - listDir: (dirPath) => window.electron.fs.listDir(dirPath), + entries: (kind) => window.electron.fs.storageEntries(kind), + checkLocation: (path) => window.electron.fs.checkStorageLocation(path), moveDirectory: (args) => window.electron.fs.moveDirectory(args), async apply(kind, path) { // Same calls as Settings > Storage. diff --git a/src/shared/stores/agentConversationStore.ts b/src/shared/stores/agentConversationStore.ts index d52261c1c..7ff4e114e 100644 --- a/src/shared/stores/agentConversationStore.ts +++ b/src/shared/stores/agentConversationStore.ts @@ -43,7 +43,7 @@ interface AgentConversationState { setMessages: (update: AgentMessage[] | ((prev: AgentMessage[]) => AgentMessage[])) => void setBusy: (busy: boolean) => void - newConversation: () => void + newConversation: () => Promise open: (id: string) => Promise remove: (id: string) => Promise refreshList: () => Promise @@ -63,11 +63,12 @@ let dirty = false let initPromise: Promise | null = null export const useAgentConversationStore = create()((set, get) => { - async function save(): Promise { + /** False when the write failed: the changes stay unsaved and switching away must wait. */ + async function save(): Promise { if (saveTimer) { clearTimeout(saveTimer); saveTimer = null } const { currentId, createdAt, messages } = get() // An empty conversation never reaches the disk. - if (!dirty || messages.length === 0) return + if (!dirty || messages.length === 0) return true dirty = false const conversation: Conversation = { id: currentId, @@ -79,8 +80,12 @@ export const useAgentConversationStore = create()((set, try { await window.electron.agent.conversations.save(conversation) set((s) => ({ list: upsertSummary(s.list, { id: conversation.id, title: conversation.title, updatedAt: conversation.updatedAt }) })) + return true } catch (err) { console.error('[agent] Failed to save conversation:', err) + // Still unsaved: the next change or switch tries again. + if (get().currentId === conversation.id) dirty = true + return false } } @@ -104,16 +109,17 @@ export const useAgentConversationStore = create()((set, setBusy: (busy) => set({ busy }), - newConversation: () => { + newConversation: async () => { if (get().busy) return - void save() + // Leaving a conversation whose save failed would lose it. + if (!(await save())) return set({ currentId: newId(), createdAt: Date.now(), messages: [] }) dirty = false }, open: async (id) => { if (get().busy || id === get().currentId) return - await save() + if (!(await save())) return const conversation = await window.electron.agent.conversations.read(id) if (!conversation) { // Gone or corrupted on disk: drop it from the list. @@ -165,6 +171,6 @@ export const useAgentConversationStore = create()((set, return initPromise }, - flush: save, + flush: async () => { await save() }, } }) From e580e27d1b25e6a25588e6cab3e910c0bf024163 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:32:06 +0100 Subject: [PATCH 11/13] feat(agent): slide the side panel in and out The panel's slot grows to its width while the panel slides and fades in from the right (220 ms), and the reverse on close. No transition while resizing, none with reduced motion. Closed, it stays mounted but invisible. --- src/shared/components/agent/AgentPanel.tsx | 25 +++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/shared/components/agent/AgentPanel.tsx b/src/shared/components/agent/AgentPanel.tsx index f2d68b158..cca241cf8 100644 --- a/src/shared/components/agent/AgentPanel.tsx +++ b/src/shared/components/agent/AgentPanel.tsx @@ -198,14 +198,28 @@ export default function AgentPanel(): JSX.Element { if (e.key === 'ArrowRight') { e.preventDefault(); setWidth(width - step) } } + const dragging = dragWidth !== null + return ( + // The slot grows from 0 to the panel width, pushing the app aside, while + // the panel slides in from the right. Closed, it stays mounted (a streaming + // turn or a running workflow keeps going) but invisible, so it takes no + // focus once the slide-out has ended. +
+
) } From 362d93940da1ca6e59b706772ee1421302ff8b4c Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:34:28 +0100 Subject: [PATCH 12/13] style(agent): Claude-like composer in the side panel A lighter rounded box with the text on top, a plus to attach and the send button inside; the model picker and the thinking mode sit as quiet text controls under it. The Shift+Enter hint moves to the send button's tooltip. --- src/shared/components/agent/AgentChat.tsx | 131 ++++++++++++---------- 1 file changed, 69 insertions(+), 62 deletions(-) diff --git a/src/shared/components/agent/AgentChat.tsx b/src/shared/components/agent/AgentChat.tsx index 6f1f1c03f..6441924d7 100644 --- a/src/shared/components/agent/AgentChat.tsx +++ b/src/shared/components/agent/AgentChat.tsx @@ -715,8 +715,9 @@ export default function AgentChat(): JSX.Element { {/* Ollama down / model not pulled */} { void ollama.check() }} /> - {/* Input bar */} -
+ {/* Input bar: a composer box (text, attach, send) with the model and + thinking controls on a quiet line under it, as in Claude. */} +
{ if (e.target.files) handleFiles(Array.from(e.target.files)); e.target.value = '' }} /> -
+
{/* Attachment previews */} {attachments.length > 0 && (
@@ -734,6 +735,7 @@ export default function AgentChat(): JSX.Element { - {/* Thinking toggle */} + + {/* Send */} - {/* Model selector */} -
- - - {showModelPicker && ( -
- {!isLocal ? ( -

{PROVIDERS[provider].label} — change the model in Settings → Agent.

- ) : localModels.length === 0 ? ( -

No local model downloaded — open Settings → Agent.

- ) : ( - localModels.map((m) => ( - - )) - )} -
- )} -
-
+
+
+ {/* Model and thinking, under the box */} +
+
+ + {showModelPicker && ( +
+ {!isLocal ? ( +

{PROVIDERS[provider].label} — change the model in Settings → Agent.

+ ) : localModels.length === 0 ? ( +

No local model downloaded — open Settings → Agent.

+ ) : ( + localModels.map((m) => ( + + )) + )} +
+ )}
+ + {/* Thinking: auto → on → off */} +
-

Shift+Enter for new line

From a8c44e6ea3d9645b41185022ac229657f1276dd7 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 18:10:43 +0100 Subject: [PATCH 13/13] fix(agent): review fixes for the side panel and conversation saving Agent tools: - generate_from_image refuses while a generation is already running - the tool-result endpoint no longer doubles the "Error: " prefix - a confirmed tool waits for the click (10 min), then gets tool_timeout for its run, via the new POST /agent/tool-confirmed - the non-streaming /agent/chat no longer offers client tools - uninstall_extension rejects "ext/node" ids so the card can't mislead Conversations: - every disk operation goes through one queue: a save in flight is always awaited, and a delete can't race the write of its file - a failed delete keeps the pending changes - init retries after a failure and skips unreadable conversations - rename retries on transient Windows locks; orphan .tmp files are cleaned --- api/routers/agent.py | 95 ++++++++-- api/tests/test_agent_client_tools.py | 96 ++++++++++ .../2026-11-12-agent-side-panel-design.md | 2 +- electron/main/agent-conversations.test.ts | 64 ++++++- electron/main/agent-conversations.ts | 52 +++++- src/shared/components/agent/AgentChat.tsx | 9 +- .../components/agent/clientTools.test.ts | 55 ++++++ src/shared/components/agent/clientTools.ts | 40 +++- src/shared/stores/agentConversationStore.ts | 126 +++++++++---- src/shared/utils/agentConversations.test.ts | 175 ++++++++++++++++++ src/shared/utils/agentConversations.ts | 13 ++ 11 files changed, 665 insertions(+), 62 deletions(-) diff --git a/api/routers/agent.py b/api/routers/agent.py index 1e067b4a3..39e8cb8cb 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -405,12 +405,55 @@ def _function(name: str, description: str, properties: dict | None = None, requi ), ] -# Seconds the app has to run a client tool that needs the user's click. +# Seconds the user has to click Confirm or Cancel on a confirmed client tool. +# Once they confirm (POST /agent/tool-confirmed), the tool gets tool_timeout. CONFIRM_TIMEOUT = 600.0 _NO_ANSWER = "Error: The app did not answer in time." +# Client tools are run by the Modly app; /agent/chat has nobody to run them. +_NEEDS_APP = "Error: This tool needs the Modly app." # call_id → the turn waiting for the app's answer (see /agent/tool-result). _pending_client_tools: dict[str, asyncio.Future] = {} +# call_id → set when the user clicked Confirm (see /agent/tool-confirmed). +_confirmed_client_tools: dict[str, asyncio.Event] = {} + +# What the non-streaming /agent/chat offers: no client tool, nobody would answer. +SERVER_TOOLS = [t for t in TOOLS if t["function"]["name"] not in CLIENT_TOOLS] + + +def _not_finished(tool: str, limit: float) -> str: + return ( + f"Error: {tool} was confirmed but did not finish within {limit:g} s. " + "It may still be running in the app; check before retrying." + ) + + +async def _await_client_answer( + tool: str, answer: asyncio.Future, confirmed: asyncio.Event, confirm: bool, tool_timeout: float, +) -> str: + """The app's answer to one client tool, or why there is none. + + A confirmed tool waits up to CONFIRM_TIMEOUT for the click (or a direct + answer, e.g. a Cancel), then tool_timeout for the run itself. + """ + if confirm: + clicked = asyncio.ensure_future(confirmed.wait()) + try: + await asyncio.wait({answer, clicked}, timeout=CONFIRM_TIMEOUT, return_when=asyncio.FIRST_COMPLETED) + finally: + clicked.cancel() + if answer.done(): + return answer.result() + if not confirmed.is_set(): + return _NO_ANSWER + try: + return await asyncio.wait_for(answer, tool_timeout) + except asyncio.TimeoutError: + return _not_finished(tool, tool_timeout) + try: + return await asyncio.wait_for(answer, tool_timeout) + except asyncio.TimeoutError: + return _NO_ANSWER # Input kinds the agent may pick, mapped to the real Modly source-node types. @@ -1045,9 +1088,12 @@ async def _claim_slot(model: str, spec: dict): raise -async def _chat_events(request: AgentChatRequest): +async def _chat_events(request: AgentChatRequest, client_tools: bool = True): """Run the tool loop, yielding its events as they happen. + client_tools — False when nobody reads the stream to run them (/agent/chat): + they are not offered, and a call to one fails at once. + token / thinking — answer and reasoning text, in pieces tool_start / tool_done — around each tool call client_tool — between them, for a tool the app runs: the turn waits for @@ -1136,7 +1182,10 @@ async def post() -> httpx.Response: "POST", f"{base_url}/chat/completions", headers=headers, - json={"model": request.model, "messages": messages, "tools": TOOLS, "stream": True, **extra}, + json={ + "model": request.model, "messages": messages, + "tools": TOOLS if client_tools else SERVER_TOOLS, "stream": True, **extra, + }, ), stream=True, ) @@ -1183,26 +1232,29 @@ async def post() -> httpx.Response: break arguments = _tool_arguments(fn.get("arguments")) yield {"type": "tool_start", "id": tc["id"], "tool": fn["name"], "arguments": arguments} - if fn["name"] in CLIENT_TOOLS: + if fn["name"] in CLIENT_TOOLS and not client_tools: + # Not offered here, but a model may still name one. + result_text, payload = _NEEDS_APP, None + elif fn["name"] in CLIENT_TOOLS: # The app runs it and answers through /agent/tool-result. confirm = CLIENT_TOOLS[fn["name"]] call_id = uuid.uuid4().hex answer = asyncio.get_running_loop().create_future() + confirmed = asyncio.Event() _pending_client_tools[call_id] = answer + _confirmed_client_tools[call_id] = confirmed try: yield { "type": "client_tool", "id": tc["id"], "call_id": call_id, "tool": fn["name"], "arguments": arguments, "confirm": confirm, } - # A confirmation waits on the user's click, not on the tool. - limit = CONFIRM_TIMEOUT if confirm else request.tool_timeout - try: - result_text = await asyncio.wait_for(answer, limit) - except asyncio.TimeoutError: - result_text = _NO_ANSWER + result_text = await _await_client_answer( + fn["name"], answer, confirmed, confirm, request.tool_timeout, + ) finally: # Also runs when the client leaves: nobody will answer. _pending_client_tools.pop(call_id, None) + _confirmed_client_tools.pop(call_id, None) answer.cancel() payload = None else: @@ -1259,7 +1311,26 @@ async def agent_tool_result(req: ToolResultRequest): answer = _pending_client_tools.get(req.call_id) if answer is None or answer.done(): raise HTTPException(status_code=404, detail="No tool call is waiting for this call_id.") - answer.set_result(f"Error: {req.result}" if req.error else req.result) + # The app's results already start with "Error: "; don't say it twice. + result = req.result + if req.error and not result.startswith("Error:"): + result = f"Error: {result}" + answer.set_result(result) + return {"ok": True} + + +class ToolConfirmedRequest(BaseModel): + call_id: str + + +@router.post("/tool-confirmed") +async def agent_tool_confirmed(req: ToolConfirmedRequest): + """The user clicked Confirm: the turn now waits tool_timeout for the result.""" + confirmed = _confirmed_client_tools.get(req.call_id) + answer = _pending_client_tools.get(req.call_id) + if confirmed is None or answer is None or answer.done(): + raise HTTPException(status_code=404, detail="No tool call is waiting for this call_id.") + confirmed.set() return {"ok": True} @@ -1267,7 +1338,7 @@ async def agent_tool_result(req: ToolResultRequest): async def agent_chat(request: AgentChatRequest): """The whole turn at once, for callers that don't read a stream.""" done: dict = {} - async for event in _chat_events(request): + async for event in _chat_events(request, client_tools=False): if event["type"] == "done": done = event return AgentChatResponse( diff --git a/api/tests/test_agent_client_tools.py b/api/tests/test_agent_client_tools.py index d6fabfca1..cd43d3b54 100644 --- a/api/tests/test_agent_client_tools.py +++ b/api/tests/test_agent_client_tools.py @@ -18,6 +18,10 @@ async def _answer(call_id: str, result: str, error: bool = False) -> dict: return await agent.agent_tool_result(agent.ToolResultRequest(call_id=call_id, result=result, error=error)) +async def _confirmed(call_id: str) -> dict: + return await agent.agent_tool_confirmed(agent.ToolConfirmedRequest(call_id=call_id)) + + def _run_turn(scripted: _Scripted, request: agent.AgentChatRequest, on_client_tool) -> list[dict]: """Drive a turn, letting `on_client_tool(event)` answer each client tool the way the panel would (it may be a coroutine function).""" @@ -39,6 +43,7 @@ async def collect(): class ClientToolBridgeTests(unittest.TestCase): def tearDown(self) -> None: agent._pending_client_tools.clear() + agent._confirmed_client_tools.clear() def test_client_tool_is_handed_to_the_app_and_its_answer_reaches_the_model(self) -> None: scripted = _Scripted( @@ -106,6 +111,13 @@ def test_error_answer_is_prefixed(self) -> None: self.assertEqual(events[2]["result"], "Error: disk full") self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Error: disk full") + def test_an_error_already_prefixed_by_the_app_is_not_prefixed_again(self) -> None: + scripted = _Scripted(_client_round("reload_extensions"), _sse({"content": "ok"})) + events = _run_turn(scripted, _external(), lambda e: _answer(e["call_id"], "Error: disk full", error=True)) + + self.assertEqual(events[2]["result"], "Error: disk full") + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Error: disk full") + def test_unknown_or_already_answered_call_is_404(self) -> None: with self.assertRaises(HTTPException) as caught: asyncio.run(_answer("nope", "x")) @@ -159,6 +171,63 @@ def test_confirm_wait_is_capped(self) -> None: self.assertEqual(events[2]["result"], "Error: The app did not answer in time.") + def test_after_confirm_the_tool_gets_the_tool_timeout(self) -> None: + scripted = _Scripted(_client_round("set_storage_folder", {"kind": "models", "path": "E:/m"}), _sse({"content": "ok"})) + seen = [] + + async def confirm_then_hang(event): + await asyncio.sleep(0.02) + seen.append(await _confirmed(event["call_id"])) + + def on_tool(event): + asyncio.ensure_future(confirm_then_hang(event)) + + with mock.patch.object(agent, "CONFIRM_TIMEOUT", 5.0): + events = _run_turn(scripted, _external().model_copy(update={"tool_timeout": 0.05}), on_tool) + + self.assertEqual(seen, [{"ok": True}]) + result = events[2]["result"] + self.assertTrue(result.startswith("Error: set_storage_folder was confirmed but did not finish within 0.05 s")) + self.assertIn("may still be running", result) + self.assertEqual(agent._pending_client_tools, {}) + self.assertEqual(agent._confirmed_client_tools, {}) + + def test_a_confirmed_tool_answering_within_the_tool_timeout_is_relayed(self) -> None: + scripted = _Scripted(_client_round("delete_workflow", {"workflow_id": "w"}), _sse({"content": "ok"})) + + def on_tool(event): + async def click_then_run(): + await asyncio.sleep(0.1) # longer than tool_timeout: the click is not timed by it + await _confirmed(event["call_id"]) + await asyncio.sleep(0.02) + await _answer(event["call_id"], "Deleted.") + asyncio.ensure_future(click_then_run()) + + with mock.patch.object(agent, "CONFIRM_TIMEOUT", 5.0): + _run_turn(scripted, _external().model_copy(update={"tool_timeout": 0.05}), on_tool) + + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Deleted.") + + def test_confirmed_for_an_unknown_or_answered_call_is_404(self) -> None: + with self.assertRaises(HTTPException) as caught: + asyncio.run(_confirmed("nope")) + self.assertEqual(caught.exception.status_code, 404) + + seen = [] + + async def answer_then_confirm(event): + await _answer(event["call_id"], "The user declined.") + try: + await _confirmed(event["call_id"]) + except HTTPException as e: + seen.append(e.status_code) + + _run_turn( + _Scripted(_client_round("delete_workflow", {"workflow_id": "w"}), _sse({"content": "ok"})), + _external(), answer_then_confirm, + ) + self.assertEqual(seen, [404]) + def test_client_tools_count_against_the_budget(self) -> None: both = _sse( _call(0, id="a", function={"name": "reload_extensions", "arguments": "{}"}), @@ -208,6 +277,33 @@ async def cancel_while_waiting(): self.assertNotIn(event["call_id"], agent._pending_client_tools) +class NonStreamingChatTests(unittest.TestCase): + def test_chat_does_not_offer_client_tools(self) -> None: + scripted = _Scripted(_sse({"content": "hi"})) + with mock.patch.object(agent.httpx, "AsyncClient", scripted): + response = asyncio.run(agent.agent_chat(_external())) + + self.assertEqual(response.message, "hi") + offered = {t["function"]["name"] for t in scripted.bodies[0]["tools"]} + self.assertFalse(offered & set(agent.CLIENT_TOOLS)) + self.assertIn("list_extensions", offered) + + def test_chat_answers_a_client_tool_call_at_once(self) -> None: + scripted = _Scripted(_client_round("reload_extensions"), _sse({"content": "Could not."})) + with mock.patch.object(agent.httpx, "AsyncClient", scripted): + response = asyncio.run(asyncio.wait_for(agent.agent_chat(_external()), 5)) + + self.assertEqual(response.actions[0].result, "Error: This tool needs the Modly app.") + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Error: This tool needs the Modly app.") + self.assertEqual(agent._pending_client_tools, {}) + + def test_the_stream_still_offers_client_tools(self) -> None: + scripted = _Scripted(_sse({"content": "hi"})) + _run_turn(scripted, _external(), None) + offered = {t["function"]["name"] for t in scripted.bodies[0]["tools"]} + self.assertTrue(set(agent.CLIENT_TOOLS) <= offered) + + _CLIENT_TOOL_ARGS = { "install_extension": ["github_url"], "uninstall_extension": ["extension_id"], diff --git a/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md b/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md index 69bfa8db5..43a784bc0 100644 --- a/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md +++ b/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md @@ -27,7 +27,7 @@ The agent today lives in the "chat" mode of the Generate page's left panel. It l | Tool | Does | |---|---| -| `generate_from_image(model_id?, params?)` | Starts a generation from the image attached to the message. The frontend runs it, as with `run_workflow`. | +| `generate_from_image(model_id?)` | Starts a generation from the image attached to the message. The frontend runs it, as with `run_workflow`. | | `list_extensions` | Lists installed extensions (from context). | | `get_settings` | Returns the storage paths and a few UI options (from context). | diff --git a/electron/main/agent-conversations.test.ts b/electron/main/agent-conversations.test.ts index 7d8387778..b680039d7 100644 --- a/electron/main/agent-conversations.test.ts +++ b/electron/main/agent-conversations.test.ts @@ -1,10 +1,11 @@ import assert from 'node:assert/strict' import test from 'node:test' -import { mkdtemp, readdir, rm, writeFile } from 'node:fs/promises' +import { mkdtemp, readdir, rename, rm, utimes, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { + ORPHAN_TMP_AGE_MS, createConversationStorage, isConversation, isValidConversationId, @@ -107,3 +108,64 @@ test('corrupted or mismatched files are skipped', async () => { assert.equal(await storage.read('other'), null) }) }) + +const errno = (code: string): NodeJS.ErrnoException => Object.assign(new Error(code), { code }) + +test('a rename locked for a moment (antivirus, indexer) is retried', async () => { + await withDir(async (dir) => { + let calls = 0 + const storage = createConversationStorage(dir, { + retryDelayMs: 1, + rename: async (from, to) => { + calls++ + if (calls < 3) throw errno(calls === 1 ? 'EPERM' : 'EBUSY') + await rename(from, to) + }, + }) + await storage.save(conv('a', 1, 'Locked')) + assert.equal(calls, 3) + assert.equal((await storage.read('a'))?.title, 'Locked') + assert.deepEqual(await readdir(dir), ['a.json']) + }) +}) + +test('a rename still locked after the retries fails and leaves no temp file', async () => { + await withDir(async (dir) => { + let calls = 0 + const storage = createConversationStorage(dir, { + retryDelayMs: 1, + rename: async () => { calls++; throw errno('EACCES') }, + }) + await assert.rejects(storage.save(conv('a', 1)), { code: 'EACCES' }) + assert.equal(calls, 3) + assert.deepEqual(await readdir(dir), []) + }) +}) + +test('other rename errors are not retried', async () => { + await withDir(async (dir) => { + let calls = 0 + const storage = createConversationStorage(dir, { + retryDelayMs: 1, + rename: async () => { calls++; throw errno('ENOSPC') }, + }) + await assert.rejects(storage.save(conv('a', 1)), { code: 'ENOSPC' }) + assert.equal(calls, 1) + }) +}) + +test('list removes orphan temp files, but not ones a write may still own', async () => { + await withDir(async (dir) => { + const storage = createConversationStorage(dir) + await storage.save(conv('a', 1)) + const old = join(dir, 'a.json.1111.tmp') + const fresh = join(dir, 'a.json.2222.tmp') + await writeFile(old, '{', 'utf-8') + await writeFile(fresh, '{', 'utf-8') + const past = (Date.now() - ORPHAN_TMP_AGE_MS - 10_000) / 1000 + await utimes(old, past, past) + + assert.deepEqual((await storage.list()).map((s) => s.id), ['a']) + assert.deepEqual((await readdir(dir)).sort(), ['a.json', 'a.json.2222.tmp']) + }) +}) diff --git a/electron/main/agent-conversations.ts b/electron/main/agent-conversations.ts index 8a2db16e4..65d9011f3 100644 --- a/electron/main/agent-conversations.ts +++ b/electron/main/agent-conversations.ts @@ -3,7 +3,7 @@ import { join } from 'path' import { randomUUID } from 'crypto' -import { mkdir, readFile, readdir, rename, rm, writeFile } from 'fs/promises' +import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'fs/promises' export interface ConversationSummary { id: string @@ -39,7 +39,26 @@ export function sortSummaries(list: readonly T[]) return [...list].sort((a, b) => (b.updatedAt - a.updatedAt) || a.id.localeCompare(b.id)) } -export function createConversationStorage(dir: string) { +/** Windows antivirus or indexers briefly lock a freshly written file. */ +const TRANSIENT_RENAME_CODES = new Set(['EPERM', 'EBUSY', 'EACCES']) +const RENAME_ATTEMPTS = 3 +const RENAME_RETRY_MS = 50 +/** A temp file this old belongs to no write in flight: a crash left it behind. */ +export const ORPHAN_TMP_AGE_MS = 60_000 + +export interface ConversationStorageOptions { + /** Injectable for tests. */ + rename?: (from: string, to: string) => Promise + retryDelayMs?: number + now?: () => number +} + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) + +export function createConversationStorage(dir: string, options: ConversationStorageOptions = {}) { + const renameFile = options.rename ?? rename + const retryDelayMs = options.retryDelayMs ?? RENAME_RETRY_MS + const now = options.now ?? Date.now const fileFor = (id: string): string => join(dir, `${id}.json`) async function readConversation(id: string): Promise { @@ -52,6 +71,32 @@ export function createConversationStorage(dir: string) { } } + async function renameWithRetry(from: string, to: string): Promise { + for (let attempt = 1; ; attempt++) { + try { + await renameFile(from, to) + return + } catch (err) { + const code = (err as NodeJS.ErrnoException | null)?.code + if (attempt >= RENAME_ATTEMPTS || !code || !TRANSIENT_RENAME_CODES.has(code)) throw err + await sleep(retryDelayMs * attempt) + } + } + } + + /** Best effort: temp files left by a crash between write and rename. */ + async function removeOrphanTmpFiles(files: readonly string[]): Promise { + for (const file of files) { + if (!file.endsWith('.tmp')) continue + const path = join(dir, file) + try { + if (now() - (await stat(path)).mtimeMs >= ORPHAN_TMP_AGE_MS) await rm(path, { force: true }) + } catch { + // Gone already, or locked: the next listing tries again. + } + } + } + return { async list(): Promise { let files: string[] @@ -60,6 +105,7 @@ export function createConversationStorage(dir: string) { } catch { return [] } + await removeOrphanTmpFiles(files) const summaries: ConversationSummary[] = [] for (const file of files) { if (!file.endsWith('.json')) continue @@ -79,7 +125,7 @@ export function createConversationStorage(dir: string) { const tmp = `${target}.${randomUUID()}.tmp` await writeFile(tmp, JSON.stringify(conversation), 'utf-8') try { - await rename(tmp, target) + await renameWithRetry(tmp, target) } catch (err) { await rm(tmp, { force: true }) throw err diff --git a/src/shared/components/agent/AgentChat.tsx b/src/shared/components/agent/AgentChat.tsx index 6441924d7..a7c1bcab5 100644 --- a/src/shared/components/agent/AgentChat.tsx +++ b/src/shared/components/agent/AgentChat.tsx @@ -521,12 +521,15 @@ export default function AgentChat(): JSX.Element { // Started after `done`, like workflows: the local LLM frees the GPU first. function startGeneration(payload: AgentActionPayload | null | undefined, msgs: Message[]) { if (payload?.type !== 'generate_from_image') return - setGenerations((n) => n + 1) - void startGenerationFromImage(payload, latestUserImage(msgs), createGenerationDeps()) + const deps = createGenerationDeps() + // A refused start (one is already running) is not ours to count. + const counted = !deps.isGenerating() + if (counted) setGenerations((n) => n + 1) + void startGenerationFromImage(payload, latestUserImage(msgs), deps) .then((result) => { setMessages((prev) => [...prev, { id: `sys-${Date.now()}`, role: 'assistant', content: result }]) }) - .finally(() => setGenerations((n) => n - 1)) + .finally(() => { if (counted) setGenerations((n) => n - 1) }) } function startWorkflow(payload: AgentActionPayload | null | undefined, msgs: Message[]) { diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts index 62082de08..e6cb9ecde 100644 --- a/src/shared/components/agent/clientTools.test.ts +++ b/src/shared/components/agent/clientTools.test.ts @@ -5,6 +5,7 @@ import { clientTools, describeClientTool, isErrorResult, + isGenerationRunning, latestUserImage, runClientTool, startGenerationFromImage, @@ -160,6 +161,17 @@ test('uninstall_extension uninstalls by id, describes by name and refuses built- assert.equal(f.calls.length, 1) }) +test('uninstall_extension refuses a model id, which would remove the whole extension', async () => { + const f = fake() + const problem = clientTools.uninstall_extension.check({ extension_id: 'hunyuan/shape' }, f.deps) + assert.match(problem ?? '', /“hunyuan\/shape” names a model; pass the extension id \(“hunyuan”\)/) + let asked = false + const outcome = await runClientTool(event('uninstall_extension', { extension_id: 'hunyuan/shape' }, true), f.deps, async () => { asked = true; return true }) + assert.equal(asked, false) + assert.equal(outcome.error, true) + assert.deepEqual(f.calls, []) +}) + test('reload_extensions reloads and reports the count and load errors', async () => { const f = fake() const result = await run('reload_extensions', {}, f) @@ -359,6 +371,32 @@ test('runClientTool asks first when confirm is set, and runs on Confirm', async assert.equal(f.posted[0].call_id, 'call-7') }) +test('runClientTool tells the server about the Confirm click before running, and not on a decline', async () => { + const order: string[] = [] + const f = fake({ + postConfirmed: async (callId) => { order.push(`confirmed ${callId}`) }, + postResult: async (body) => { order.push(`result ${body.call_id}`) }, + }) + f.deps.extensions.uninstall = async () => { order.push('uninstall'); return { success: true } } + await runClientTool(event('uninstall_extension', { extension_id: 'triposr' }, true), f.deps, async () => true) + assert.deepEqual(order, ['confirmed call-7', 'uninstall', 'result call-7']) + + order.length = 0 + await runClientTool(event('uninstall_extension', { extension_id: 'triposr' }, true), f.deps, async () => false) + assert.deepEqual(order, ['result call-7']) + + order.length = 0 + await runClientTool(event('navigate', { page: 'models' }), f.deps, async () => true) + assert.deepEqual(order, ['result call-7']) +}) + +test('runClientTool still runs the tool when the confirmed notice fails', async () => { + const f = fake({ postConfirmed: async () => { throw new Error('offline') } }) + const outcome = await runClientTool(event('delete_workflow', { workflow_id: 'wf1' }, true), f.deps, async () => true) + assert.deepEqual(f.calls, [['removeWorkflow', 'wf1']]) + assert.equal(outcome.error, false) +}) + test('runClientTool reports a declined confirmation without running the tool', async () => { const f = fake() const outcome = await runClientTool(event('delete_workflow', { workflow_id: 'wf1' }, true), f.deps, async () => false) @@ -413,6 +451,7 @@ function generation(statuses: { status: string; progress?: number; outputUrl?: s const calls: unknown[][] = [] let job: GenerationJob | null = null const deps: GenerationDeps = { + isGenerating: () => isGenerationRunning(job), api: { generateFromImage: async (path, options, data) => { calls.push(['submit', path, options, data]); return { jobId: 'j1' } }, pollJobStatus: async () => ({ progress: 0, ...statuses.shift()! }) as never, @@ -459,6 +498,22 @@ test('startGenerationFromImage reports a missing image, a wrong payload and a fa assert.equal(g.job()?.status, 'error') }) +test('startGenerationFromImage refuses while a generation is running, without touching the current job', async () => { + for (const status of ['uploading', 'generating'] as const) { + const g = generation([{ status: 'done', outputUrl: '/workspace/o.glb' }]) + const running = { id: 'busy', imageFile: 'x.png', status, progress: 40, createdAt: 1 } as GenerationJob + g.deps.store.setCurrentJob(running) + const result = await startGenerationFromImage({ type: 'generate_from_image' }, 'QUJD', g.deps) + assert.equal(result, 'Error: A generation is already running. Wait for it to finish.') + assert.deepEqual(g.calls, [], status) + assert.equal(g.job(), running) + } + for (const status of ['done', 'error', 'idle'] as const) { + assert.equal(isGenerationRunning({ status }), false, status) + } + assert.equal(isGenerationRunning(null), false) +}) + test('latestUserImage takes the most recent user message with an image', () => { assert.equal(latestUserImage([ { role: 'user', imageDataUrls: ['data:old'] }, diff --git a/src/shared/components/agent/clientTools.ts b/src/shared/components/agent/clientTools.ts index a9bdeeb78..526873e64 100644 --- a/src/shared/components/agent/clientTools.ts +++ b/src/shared/components/agent/clientTools.ts @@ -87,6 +87,12 @@ export interface ClientToolDeps { } navigate: (page: Page) => void postResult: (body: ToolResultBody) => Promise + /** + * Tells the server the user clicked Confirm, so it stops waiting on the click + * and gives the tool its own timeout. Optional: without it the server keeps + * waiting under the confirmation cap. + */ + postConfirmed?: (callId: string) => Promise /** How long a download is watched for an early failure before it is reported as started. */ downloadSettleMs?: number } @@ -316,6 +322,12 @@ export const clientTools: Record = { destructive: true, describe: (args, deps) => `Uninstall the extension ${extensionLabel(String(args.extension_id ?? '?'), deps)}`, check: (args, deps) => { + // "ext/node" names one model, but uninstalling removes the whole extension: + // the card must not read as if only that model goes. + const id = str(args, 'extension_id') + if (id?.includes('/')) { + return `${quote(id)} names a model; pass the extension id (${quote(id.split('/')[0])}) to uninstall the whole extension.` + } const found = lookupExtension(args, deps) if (typeof found === 'string') return found return found.ext.builtin ? `${quote(found.ext.name)} is built into Modly and cannot be uninstalled.` : null @@ -606,7 +618,8 @@ export async function runClientTool( result = fail(problem) } else { let ok = true - if (event.confirm || tool.requiresConfirm) { + const asked = event.confirm || tool.requiresConfirm + if (asked) { try { ok = await confirm(description, { tool: event.tool, id: event.id, callId: event.call_id, destructive: tool.destructive }) } catch { @@ -614,6 +627,10 @@ export async function runClientTool( } } if (ok) { + if (asked && deps.postConfirmed) { + // From here the server waits on the tool, not on the click. + await deps.postConfirmed(event.call_id).catch(() => {}) + } result = await tool.run(args, deps) } else { declined = true @@ -635,6 +652,8 @@ export async function runClientTool( // ─── generate_from_image ────────────────────────────────────────────────────── export interface GenerationDeps { + /** True while a generation is uploading or running (the Generate page then disables its upload too). */ + isGenerating: () => boolean api: GenerationApi store: GenerationStoreActions /** The Generate page's current options (model, remesh, texture, params). */ @@ -645,6 +664,13 @@ export interface GenerationDeps { wait?: () => Promise } +export const GENERATION_BUSY = 'A generation is already running. Wait for it to finish.' + +/** Same condition as the Generate page's upload and options lock. */ +export function isGenerationRunning(job: { status: string } | null | undefined): boolean { + return job?.status === 'uploading' || job?.status === 'generating' +} + /** The first image of the most recent user message that has one. */ export function latestUserImage(messages: { role: string; imageDataUrls?: string[] }[]): string | undefined { for (let i = messages.length - 1; i >= 0; i--) { @@ -667,6 +693,9 @@ export async function startGenerationFromImage( if (payload?.type !== 'generate_from_image') return fail('Not a generate_from_image payload.') const base64 = image?.startsWith('data:') ? image.slice(image.indexOf(',') + 1) : image if (!base64) return fail('No image is attached to the conversation; attach one and ask again.') + // Checked synchronously right before runGeneration claims currentJob, so two + // calls cannot both pass it. + if (deps.isGenerating()) return fail(GENERATION_BUSY) const current = deps.options() const modelId = payload.model_id || current.modelId @@ -762,6 +791,14 @@ export function createClientToolDeps(): ClientToolDeps { }) if (!res.ok) throw new Error(`API error ${res.status}`) }, + async postConfirmed(callId) { + const res = await fetch(`${app().apiUrl}/agent/tool-confirmed`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ call_id: callId }), + }) + if (!res.ok) throw new Error(`API error ${res.status}`) + }, } } @@ -769,6 +806,7 @@ export function createClientToolDeps(): ClientToolDeps { export function createGenerationDeps(): GenerationDeps { const app = () => useAppStore.getState() return { + isGenerating: () => isGenerationRunning(app().currentJob), api: createGenerationApi(app().apiUrl), store: { setCurrentJob: (job) => app().setCurrentJob(job), diff --git a/src/shared/stores/agentConversationStore.ts b/src/shared/stores/agentConversationStore.ts index 7ff4e114e..621f5e1a4 100644 --- a/src/shared/stores/agentConversationStore.ts +++ b/src/shared/stores/agentConversationStore.ts @@ -1,12 +1,14 @@ import { create } from 'zustand' import type { AgentAction, ToolStop } from '@shared/utils/agentStream' +// Relative so the store loads under `node --test` (no path aliases there). import { + createSerialQueue, deriveTitle, settleMessages, upsertSummary, type Conversation, type ConversationSummary, -} from '@shared/utils/agentConversations' +} from '../utils/agentConversations' // ─── Message types (AgentChat) ──────────────────────────────────────────────── @@ -30,6 +32,8 @@ export interface AgentMessage { // ─── Store ──────────────────────────────────────────────────────────────────── const SAVE_DEBOUNCE_MS = 500 +/** Unreadable conversations init skips before giving up on reopening one. */ +const MAX_INIT_ATTEMPTS = 5 interface AgentConversationState { /** Id of the conversation on screen; not on disk until it has a message. */ @@ -47,12 +51,14 @@ interface AgentConversationState { open: (id: string) => Promise remove: (id: string) => Promise refreshList: () => Promise - /** Loads the list and opens the most recent conversation. Runs once. */ + /** Loads the list and opens the most recent readable conversation. Runs once, retried if it failed. */ init: () => Promise /** Writes a pending debounced save now. */ flush: () => Promise } +type OpenResult = 'opened' | 'unreadable' | 'skipped' + function newId(): string { return crypto.randomUUID() } @@ -60,16 +66,26 @@ function newId(): string { let saveTimer: ReturnType | null = null // Unsaved changes: opening or switching away must not bump an untouched conversation. let dirty = false +// Bumped on every edit: a save clears `dirty` only if nothing changed while it was writing. +let edits = 0 +// Deleted conversations: a save finishing late must not bring one back into the list. +const removedIds = new Set() let initPromise: Promise | null = null +// Every disk operation runs through this queue, one at a time: a save in +// flight is always awaited, and a delete never races the write of its file. +const enqueue = createSerialQueue() export const useAgentConversationStore = create()((set, get) => { - /** False when the write failed: the changes stay unsaved and switching away must wait. */ - async function save(): Promise { + /** + * Writes the current conversation if it has unsaved changes. Queued tasks only. + * False when the write failed: the changes stay unsaved and switching away must wait. + */ + async function saveNow(): Promise { if (saveTimer) { clearTimeout(saveTimer); saveTimer = null } const { currentId, createdAt, messages } = get() // An empty conversation never reaches the disk. if (!dirty || messages.length === 0) return true - dirty = false + const editsAtStart = edits const conversation: Conversation = { id: currentId, title: deriveTitle(messages), @@ -79,14 +95,20 @@ export const useAgentConversationStore = create()((set, } try { await window.electron.agent.conversations.save(conversation) - set((s) => ({ list: upsertSummary(s.list, { id: conversation.id, title: conversation.title, updatedAt: conversation.updatedAt }) })) - return true } catch (err) { + // Still dirty: the next change or switch tries again. console.error('[agent] Failed to save conversation:', err) - // Still unsaved: the next change or switch tries again. - if (get().currentId === conversation.id) dirty = true return false } + if (get().currentId === conversation.id && edits === editsAtStart) dirty = false + if (!removedIds.has(conversation.id)) { + set((s) => ({ list: upsertSummary(s.list, { id: conversation.id, title: conversation.title, updatedAt: conversation.updatedAt }) })) + } + return true + } + + function save(): Promise { + return enqueue(saveNow) } function scheduleSave(): void { @@ -94,6 +116,32 @@ export const useAgentConversationStore = create()((set, saveTimer = setTimeout(() => { void save() }, SAVE_DEBOUNCE_MS) } + /** Queued tasks only. */ + async function openNow(id: string): Promise { + if (get().busy || id === get().currentId) return 'skipped' + // Leaving a conversation whose save failed would lose it. + if (!(await saveNow())) return 'skipped' + let conversation: Conversation | null + try { + conversation = await window.electron.agent.conversations.read(id) + } catch (err) { + console.error('[agent] Failed to read conversation:', err) + return 'skipped' + } + if (!conversation) { + // Gone or corrupted on disk: drop it from the list. + set((s) => ({ list: s.list.filter((c) => c.id !== id) })) + return 'unreadable' + } + set({ + currentId: conversation.id, + createdAt: conversation.createdAt, + messages: settleMessages(conversation.messages) as AgentMessage[], + }) + dirty = false + return 'opened' + } + return { currentId: newId(), createdAt: Date.now(), @@ -104,70 +152,66 @@ export const useAgentConversationStore = create()((set, setMessages: (update) => { set((s) => ({ messages: typeof update === 'function' ? update(s.messages) : update })) dirty = true + edits++ scheduleSave() }, setBusy: (busy) => set({ busy }), - newConversation: async () => { + newConversation: () => enqueue(async () => { if (get().busy) return // Leaving a conversation whose save failed would lose it. - if (!(await save())) return + if (!(await saveNow())) return set({ currentId: newId(), createdAt: Date.now(), messages: [] }) dirty = false - }, + }), - open: async (id) => { - if (get().busy || id === get().currentId) return - if (!(await save())) return - const conversation = await window.electron.agent.conversations.read(id) - if (!conversation) { - // Gone or corrupted on disk: drop it from the list. - set((s) => ({ list: s.list.filter((c) => c.id !== id) })) - return - } - set({ - currentId: conversation.id, - createdAt: conversation.createdAt, - messages: settleMessages(conversation.messages) as AgentMessage[], - }) - dirty = false - }, + open: async (id) => { await enqueue(() => openNow(id)) }, - remove: async (id) => { + remove: (id) => enqueue(async () => { const isCurrent = id === get().currentId if (isCurrent && get().busy) return - if (isCurrent) { - if (saveTimer) { clearTimeout(saveTimer); saveTimer = null } - dirty = false - } try { await window.electron.agent.conversations.remove(id) } catch (err) { + // Still on disk, and its pending changes still get saved. console.error('[agent] Failed to delete conversation:', err) return } + removedIds.add(id) + if (isCurrent) { + if (saveTimer) { clearTimeout(saveTimer); saveTimer = null } + dirty = false + } set((s) => ({ list: s.list.filter((c) => c.id !== id), ...(isCurrent ? { currentId: newId(), createdAt: Date.now(), messages: [] } : {}), })) - }, + }), - refreshList: async () => { + refreshList: () => enqueue(async () => { try { set({ list: await window.electron.agent.conversations.list() }) } catch (err) { console.error('[agent] Failed to list conversations:', err) } - }, + }), init: () => { initPromise ??= (async () => { - await get().refreshList() - const latest = get().list[0] - // The user may already have started typing a new conversation. - if (latest && get().messages.length === 0 && !get().busy) await get().open(latest.id) - })() + // Throws on failure, so a failed init is forgotten and the next call retries. + const list = await enqueue(() => window.electron.agent.conversations.list()) + set({ list }) + // Open the most recent readable one; unreadable files drop out of the list. + for (const { id } of list.slice(0, MAX_INIT_ATTEMPTS)) { + // The user may already have started typing a new conversation. + if (get().messages.length > 0 || get().busy) return + if ((await enqueue(() => openNow(id))) !== 'unreadable') return + } + })().catch((err: unknown) => { + console.error('[agent] Failed to load conversations:', err) + initPromise = null + }) return initPromise }, diff --git a/src/shared/utils/agentConversations.test.ts b/src/shared/utils/agentConversations.test.ts index 8ed04cc6a..1362f54c9 100644 --- a/src/shared/utils/agentConversations.test.ts +++ b/src/shared/utils/agentConversations.test.ts @@ -4,12 +4,14 @@ import test from 'node:test' import { TITLE_MAX_LENGTH, UNTITLED, + createSerialQueue, deriveTitle, relativeDate, settleMessages, sortSummaries, upsertSummary, } from './agentConversations' +import { useAgentConversationStore } from '../stores/agentConversationStore' test('title is the first user message, collapsed to one line', () => { const title = deriveTitle([ @@ -75,3 +77,176 @@ test('relativeDate buckets by age', () => { assert.match(relativeDate(now - 30 * 86_400_000, now), /^Oct \d+$/) assert.match(relativeDate(Date.UTC(2024, 0, 15), now), /2024/) }) + +test('the serial queue runs tasks one at a time, and a failure does not stop it', async () => { + const enqueue = createSerialQueue() + const log: string[] = [] + const first = deferred() + const a = enqueue(async () => { log.push('a start'); const v = await first.promise; log.push('a end'); return v }) + const b = enqueue(async () => { log.push('b'); throw new Error('b failed') }) + const c = enqueue(async () => { log.push('c'); return 'c' }) + await tick() + assert.deepEqual(log, ['a start']) + first.resolve('a') + assert.equal(await a, 'a') + await assert.rejects(b, /b failed/) + assert.equal(await c, 'c') + assert.deepEqual(log, ['a start', 'a end', 'b', 'c']) +}) + +// ─── Store: disk operations against a fake IPC ─────────────────────────────── + +interface Deferred { promise: Promise; resolve: (v: T) => void; reject: (e: unknown) => void } + +function deferred(): Deferred { + let resolve!: (v: T) => void + let reject!: (e: unknown) => void + const promise = new Promise((res, rej) => { resolve = res; reject = rej }) + return { promise, resolve, reject } +} + +const tick = () => new Promise((resolve) => setTimeout(resolve, 0)) + +type Ipc = { + list: () => Promise + read: (id: string) => Promise + save: (c: { id: string; messages: unknown[] }) => Promise + remove: (id: string) => Promise +} +const ipc: Ipc = { + list: async () => [], + read: async () => null, + save: async () => {}, + remove: async () => {}, +} +const calls: string[] = [] +;(globalThis as Record).window = { + electron: { + agent: { + conversations: { + list: () => { calls.push('list'); return ipc.list() }, + read: (id: string) => { calls.push(`read ${id}`); return ipc.read(id) }, + save: (c: { id: string; messages: unknown[] }) => { calls.push(`save ${c.id}`); return ipc.save(c) }, + remove: (id: string) => { calls.push(`remove ${id}`); return ipc.remove(id) }, + }, + }, + }, +} + +const store = useAgentConversationStore +const msg = (content: string) => ({ id: content, role: 'user' as const, content }) +const stored = (id: string, updatedAt: number) => ({ id, title: id, createdAt: 1, updatedAt, messages: [msg(id)] }) + +function resetIpc(): void { + ipc.list = async () => [] + ipc.read = async () => null + ipc.save = async () => {} + ipc.remove = async () => {} + calls.length = 0 +} + +// Runs first: init is memoized once it succeeds. +test('store: a failed init is retried, and an unreadable latest conversation falls back to the next', async () => { + resetIpc() + const quietError = console.error + console.error = () => {} + try { + ipc.list = async () => { throw new Error('IPC down') } + await store.getState().init() + assert.deepEqual(store.getState().list, []) + + ipc.list = async () => [ + { id: 'broken', title: 'broken', updatedAt: 30 }, + { id: 'good', title: 'good', updatedAt: 20 }, + { id: 'older', title: 'older', updatedAt: 10 }, + ] + ipc.read = async (id) => (id === 'good' ? stored('good', 20) : null) + await store.getState().init() + assert.equal(store.getState().currentId, 'good') + assert.deepEqual(store.getState().list.map((s) => s.id), ['good', 'older']) + assert.deepEqual(calls, ['list', 'list', 'read broken', 'read good']) + + // Memoized now. + await store.getState().init() + assert.equal(calls.filter((c) => c === 'list').length, 2) + } finally { + console.error = quietError + } +}) + +test('store: switching away waits for a save in flight, and stays put when it fails', async () => { + resetIpc() + await store.getState().newConversation() + const id = store.getState().currentId + const quietError = console.error + console.error = () => {} + try { + const first = deferred() + ipc.save = () => first.promise + store.getState().setMessages([msg('hello')]) + const flushing = store.getState().flush() + await tick() + const switching = store.getState().newConversation() + await tick() + // Only the first write has started; the switch is waiting for it. + assert.deepEqual(calls, [`save ${id}`]) + + ipc.save = async () => { throw new Error('disk full') } + first.reject(new Error('disk full')) + await flushing + await switching + // The switch retried the write, which failed too: nothing is lost. + assert.deepEqual(calls, [`save ${id}`, `save ${id}`]) + assert.equal(store.getState().currentId, id) + assert.deepEqual(store.getState().messages, [msg('hello')]) + + ipc.save = async () => {} + await store.getState().newConversation() + assert.notEqual(store.getState().currentId, id) + assert.equal(store.getState().list[0]?.id, id) + } finally { + console.error = quietError + } +}) + +test('store: deleting the current conversation waits for its save and does not come back', async () => { + resetIpc() + await store.getState().newConversation() + const id = store.getState().currentId + const write = deferred() + ipc.save = () => write.promise + store.getState().setMessages([msg('to delete')]) + const flushing = store.getState().flush() + await tick() + const removing = store.getState().remove(id) + await tick() + assert.deepEqual(calls, [`save ${id}`]) + + write.resolve() + await flushing + await removing + assert.deepEqual(calls, [`save ${id}`, `remove ${id}`]) + assert.equal(store.getState().list.some((s) => s.id === id), false) + assert.notEqual(store.getState().currentId, id) + assert.deepEqual(store.getState().messages, []) +}) + +test('store: a failed delete keeps the pending changes', async () => { + resetIpc() + await store.getState().newConversation() + const id = store.getState().currentId + const quietError = console.error + console.error = () => {} + try { + ipc.remove = async () => { throw new Error('locked') } + store.getState().setMessages([msg('keep me')]) + await store.getState().remove(id) + assert.equal(store.getState().currentId, id) + + await store.getState().flush() + assert.deepEqual(calls, [`remove ${id}`, `save ${id}`]) + assert.equal(store.getState().list[0]?.id, id) + } finally { + console.error = quietError + } +}) diff --git a/src/shared/utils/agentConversations.ts b/src/shared/utils/agentConversations.ts index 06c8e3dd0..6f97943bd 100644 --- a/src/shared/utils/agentConversations.ts +++ b/src/shared/utils/agentConversations.ts @@ -47,6 +47,19 @@ export function upsertSummary(list: readonly ConversationSummary[], summary: Con return sortSummaries([...list.filter((s) => s.id !== summary.id), summary]) } +/** + * Runs async tasks one at a time, in call order. A failed task does not stop + * the ones queued after it; its caller still gets the rejection. + */ +export function createSerialQueue(): (task: () => Promise) => Promise { + let tail: Promise = Promise.resolve() + return (task: () => Promise): Promise => { + const run = tail.then(() => task()) + tail = run.catch(() => undefined) + return run + } +} + /** * A conversation saved mid-turn holds a streaming message and running tools. * Nothing is streaming once it is loaded again, so settle them.