diff --git a/api/routers/agent.py b/api/routers/agent.py index 2bdd04c13..39e8cb8cb 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -93,10 +93,21 @@ def registry_tool_lines(registry: MeshOpsRegistry) -> str: SYSTEM_PROMPT = f"""\ You are Modly's built-in AI assistant, specialized in 3D modeling and workflow automation. You help users generate 3D models from images, optimize meshes, and manage workflows directly inside the Modly application. +You can act on the whole app (extensions, model weights, workflows, settings, generation) from any page, without changing page. ## Available tools - **list_models** — List all downloaded 3D generation models ready to use. +- **generate_from_image(model_id?)** — Generate a 3D model from the image attached to the user's message. Omit model_id to use the model selected in the app. +- **list_extensions** — List installed extensions. +- **get_settings** — Show the storage folders and UI options. +- **install_extension(github_url)** / **uninstall_extension(extension_id)** / **reload_extensions** — Manage extensions. +- **download_model_weights(extension_id, variant?)** / **delete_model_weights(extension_id, variant?)** — Manage model weights. +- **open_workflow(workflow_id)** / **delete_workflow(workflow_id)** — Open a workflow in the editor, or delete it. +- **set_setting(key, value)** — Change `uiScale` (one of `small`, `medium`, `large`, `very-large`), `useAtkinsonFont` or `showRamIndicator` (booleans). +- **set_storage_folder(kind, path)** — Move the `models`, `workspace` or `workflows` folder. +- **undo_mesh** / **redo_mesh** — Undo or redo the last change to the current mesh. +- **navigate(page)** — Switch the app to another page. - **unload_models** — Unload all 3D generation models from GPU VRAM to free memory. - **get_mesh_info** — Get info about the current mesh in the 3D viewer (path, triangle count). {registry_tool_lines(mesh_ops_registry)} @@ -117,6 +128,8 @@ def registry_tool_lines(registry: MeshOpsRegistry) -> str: - If you need to run a workflow but don't know the ID, call list_workflows first. - To create a workflow, ONLY use extension ids listed under "Available extensions" in the context. Never invent an id. Chain steps so each step's input type matches the previous step's output type. - For a workflow's input, `input_type` MUST be exactly one of: `image`, `text`, or `mesh`. These map to the Image, Text, and Load 3D Mesh nodes. Never invent another input. Pick the one matching the first step's expected input. +- uninstall_extension, delete_model_weights, delete_workflow and set_storage_folder ask the user to confirm. If the result says the user declined, acknowledge it and do not retry. +- Never call navigate unless the user explicitly asks to go to a page. Act from where the user is. - After each tool call, give a short one-sentence summary of what was done. - Always reply in the same language the user is writing in. - Be concise. No unnecessary explanations.\ @@ -281,6 +294,168 @@ def registry_tool_lines(registry: MeshOpsRegistry) -> str: TOOLS += registry_tools(mesh_ops_registry) +def _function(name: str, description: str, properties: dict | None = None, required: list[str] | None = None) -> dict: + parameters: dict = {"type": "object", "properties": properties or {}} + if required: + parameters["required"] = required + return {"type": "function", "function": {"name": name, "description": description, "parameters": parameters}} + + +_EXTENSION_ID = {"type": "string", "description": "Exact extension id, as listed by list_extensions."} +_WORKFLOW_ID = {"type": "string", "description": "Exact workflow id, as listed by list_workflows."} +_VARIANT = {"type": "string", "description": "Optional weights variant, for extensions that offer several. Omit for the default."} + +# Server-side tools that read the request context or hand a run to the app. +TOOLS += [ + _function("list_extensions", "List the installed extensions, with their ids and input→output types."), + _function("get_settings", "Show Modly's storage folders (models, workspace, workflows, extensions) and UI options."), + _function( + "generate_from_image", + "Generate a 3D model from the image the user attached to their message. The app runs the " + "generation and shows its progress. Needs an attached image.", + {"model_id": { + "type": "string", + "description": "Id of a downloaded generation model (see list_models). Omit to use the model selected in the app.", + }}, + ), +] + +# Tools the app runs itself (Electron IPC, renderer stores), mapped to whether +# the user must confirm them first. The frontend obeys `confirm`. +CLIENT_TOOLS: dict[str, bool] = { + # Installing runs code downloaded from GitHub: the user must see the URL and click. + "install_extension": True, + "uninstall_extension": True, + "reload_extensions": False, + "download_model_weights": False, + "delete_model_weights": True, + "delete_workflow": True, + "open_workflow": False, + "set_setting": False, + "set_storage_folder": True, + "undo_mesh": False, + "redo_mesh": False, + "navigate": False, +} + +TOOLS += [ + _function( + "install_extension", + "Install an extension from its GitHub repository URL, then load it.", + {"github_url": {"type": "string", "description": "Repository URL, e.g. https://github.com/owner/repo."}}, + ["github_url"], + ), + _function( + "uninstall_extension", + "Uninstall an extension and remove its files. The user is asked to confirm.", + {"extension_id": _EXTENSION_ID}, + ["extension_id"], + ), + _function("reload_extensions", "Reload every installed extension, e.g. after its files changed on disk."), + _function( + "download_model_weights", + "Download the model weights an extension needs. The download continues in the background.", + {"extension_id": _EXTENSION_ID, "variant": _VARIANT}, + ["extension_id"], + ), + _function( + "delete_model_weights", + "Delete an extension's downloaded model weights from disk. The user is asked to confirm.", + {"extension_id": _EXTENSION_ID, "variant": _VARIANT}, + ["extension_id"], + ), + _function( + "delete_workflow", + "Delete a saved workflow. The user is asked to confirm.", + {"workflow_id": _WORKFLOW_ID}, + ["workflow_id"], + ), + _function( + "open_workflow", + "Open a workflow in the workflow editor, without leaving the current page.", + {"workflow_id": _WORKFLOW_ID}, + ["workflow_id"], + ), + _function( + "set_setting", + "Change a UI setting. uiScale is one of small, medium, large, very-large; useAtkinsonFont and showRamIndicator are booleans.", + { + "key": {"type": "string", "enum": ["uiScale", "useAtkinsonFont", "showRamIndicator"]}, + "value": {"type": ["string", "boolean"], "description": "For uiScale one of: small, medium, large, very-large. A boolean otherwise."}, + }, + ["key", "value"], + ), + _function( + "set_storage_folder", + "Change where Modly stores models, the workspace or workflows. The user is asked to confirm.", + { + "kind": {"type": "string", "enum": ["models", "workspace", "workflows"]}, + "path": {"type": "string", "description": "Absolute path of the new folder."}, + }, + ["kind", "path"], + ), + _function("undo_mesh", "Undo the last change to the current mesh in the viewer."), + _function("redo_mesh", "Redo the last undone change to the current mesh in the viewer."), + _function( + "navigate", + "Switch the app to another page. Use it ONLY when the user explicitly asks to go to a page; " + "every other tool works from any page.", + {"page": {"type": "string", "enum": ["generate", "workflows", "models", "settings"]}}, + ["page"], + ), +] + +# Seconds the user has to click Confirm or Cancel on a confirmed client tool. +# Once they confirm (POST /agent/tool-confirmed), the tool gets tool_timeout. +CONFIRM_TIMEOUT = 600.0 +_NO_ANSWER = "Error: The app did not answer in time." +# Client tools are run by the Modly app; /agent/chat has nobody to run them. +_NEEDS_APP = "Error: This tool needs the Modly app." + +# call_id → the turn waiting for the app's answer (see /agent/tool-result). +_pending_client_tools: dict[str, asyncio.Future] = {} +# call_id → set when the user clicked Confirm (see /agent/tool-confirmed). +_confirmed_client_tools: dict[str, asyncio.Event] = {} + +# What the non-streaming /agent/chat offers: no client tool, nobody would answer. +SERVER_TOOLS = [t for t in TOOLS if t["function"]["name"] not in CLIENT_TOOLS] + + +def _not_finished(tool: str, limit: float) -> str: + return ( + f"Error: {tool} was confirmed but did not finish within {limit:g} s. " + "It may still be running in the app; check before retrying." + ) + + +async def _await_client_answer( + tool: str, answer: asyncio.Future, confirmed: asyncio.Event, confirm: bool, tool_timeout: float, +) -> str: + """The app's answer to one client tool, or why there is none. + + A confirmed tool waits up to CONFIRM_TIMEOUT for the click (or a direct + answer, e.g. a Cancel), then tool_timeout for the run itself. + """ + if confirm: + clicked = asyncio.ensure_future(confirmed.wait()) + try: + await asyncio.wait({answer, clicked}, timeout=CONFIRM_TIMEOUT, return_when=asyncio.FIRST_COMPLETED) + finally: + clicked.cancel() + if answer.done(): + return answer.result() + if not confirmed.is_set(): + return _NO_ANSWER + try: + return await asyncio.wait_for(answer, tool_timeout) + except asyncio.TimeoutError: + return _not_finished(tool, tool_timeout) + try: + return await asyncio.wait_for(answer, tool_timeout) + except asyncio.TimeoutError: + return _NO_ANSWER + + # Input kinds the agent may pick, mapped to the real Modly source-node types. # Keep this in sync with the node palette in WorkflowsPage.tsx. INPUT_NODES = { @@ -374,11 +549,40 @@ async def _run_agent_op(client: httpx.AsyncClient, op: MeshOp, arguments: dict) return text, payload +def _extension_lines(extensions: list[dict]) -> list[str]: + return [ + f"- {e['id']} ({e.get('input', '?')}→{e.get('output', '?')}): {e.get('name', e['id'])}" + for e in extensions if e.get("id") + ] + + +def _id_name_lines(items: list[dict]) -> list[str]: + return [f"- {i['id']}: {i.get('name') or i['id']}" for i in items if i.get("id")] + + +_STORAGE_LABELS = { + "modelsDir": "Models folder", + "workspaceDir": "Workspace folder", + "workflowsDir": "Workflows folder", + "extensionsDir": "Extensions folder", +} + + +def _settings_lines(context: dict) -> list[str]: + """The storage folders and UI options the app sent, one per line.""" + storage = context.get("storage") or {} + ui = context.get("ui") or {} + lines = [f"{label}: {storage[key]}" for key, label in _STORAGE_LABELS.items() if storage.get(key)] + lines += [f"{key}: {_show(value)}" for key, value in ui.items() if value is not None] + return lines + + async def execute_tool( - name: str, arguments: dict, context: dict, *, timeout: float = 60.0, + name: str, arguments: dict, context: dict, *, timeout: float = 60.0, has_image: bool = False, ) -> tuple[str, dict | None]: """Execute a tool and return (result_text, action_payload). action_payload carries data the frontend needs to react (e.g. new mesh URL). + has_image: the user's last message carries an image (generate_from_image). """ async with httpx.AsyncClient(timeout=timeout) as client: try: @@ -459,6 +663,34 @@ async def execute_tool( payload = {"type": "run_workflow", "workflow_id": workflow_id, "workflow_name": match["name"]} return f"Executing workflow '{match['name']}'…", payload + elif name == "list_extensions": + lines = _extension_lines(context.get("extensions") or []) + if not lines: + return "No extensions installed.", None + return "Installed extensions:\n" + "\n".join(lines), None + + elif name == "get_settings": + lines = _settings_lines(context) + if not lines: + return "No settings were sent by the app.", None + return "\n".join(lines), None + + elif name == "generate_from_image": + if not has_image: + return "No image is attached. Ask the user to attach the image to generate from.", None + model_id = arguments.get("model_id") or None + models = context.get("models") or [] + label = "the selected model" + if model_id and models: + match = next((m for m in models if m.get("id") == model_id), None) + if not match: + ids = ", ".join(m["id"] for m in models if m.get("id")) + return f"Model '{model_id}' is not downloaded. Use one of: {ids}.", None + label = match.get("name") or model_id + elif model_id: + label = model_id + return f"Generating a 3D model with {label}…", {"type": "generate_from_image", "model_id": model_id} + elif name == "create_workflow": steps = arguments.get("steps") or [] if not steps: @@ -856,11 +1088,16 @@ async def _claim_slot(model: str, spec: dict): raise -async def _chat_events(request: AgentChatRequest): +async def _chat_events(request: AgentChatRequest, client_tools: bool = True): """Run the tool loop, yielding its events as they happen. + client_tools — False when nobody reads the stream to run them (/agent/chat): + they are not offered, and a call to one fails at once. + token / thinking — answer and reasoning text, in pieces tool_start / tool_done — around each tool call + client_tool — between them, for a tool the app runs: the turn waits for + its answer on POST /agent/tool-result with the same call_id done — always last: the final answer, every action and the full reasoning """ messages: list[dict] = [{"role": "system", "content": SYSTEM_PROMPT}] @@ -868,6 +1105,8 @@ async def _chat_events(request: AgentChatRequest): # Inject scene context so the LLM knows current state if request.context: ctx_lines = [] + if request.context.get("page"): + ctx_lines.append(f"Current page: {request.context['page']}") if request.context.get("currentMeshPath"): ctx_lines.append(f"Current mesh path: {request.context['currentMeshPath']}") if request.context.get("meshTriangles"): @@ -878,19 +1117,17 @@ async def _chat_events(request: AgentChatRequest): "content": "Scene context:\n" + "\n".join(ctx_lines), }) - extensions = request.context.get("extensions") or [] - if extensions: - ext_lines = [ - f"- {e['id']} ({e.get('input', '?')}→{e.get('output', '?')}): {e.get('name', e['id'])}" - for e in extensions - ] - messages.append({ - "role": "system", - "content": ( - "Available extensions (use the exact id when creating workflows):\n" - + "\n".join(ext_lines) - ), - }) + sections = [ + ("Available extensions (use the exact id when creating workflows)", + _extension_lines(request.context.get("extensions") or [])), + ("Downloaded generation models (use the exact id with generate_from_image)", + _id_name_lines(request.context.get("models") or [])), + ("Workflows", _id_name_lines(request.context.get("workflows") or [])), + ("Settings", _settings_lines(request.context)), + ] + for title, lines in sections: + if lines: + messages.append({"role": "system", "content": f"{title}:\n" + "\n".join(lines)}) # ONE system message, first. Qwen3.5's chat template raises "System message # must be at the beginning", which llama-server returns as a flat HTTP 400; @@ -924,6 +1161,10 @@ async def _chat_events(request: AgentChatRequest): for m in request.messages: messages.append(_user_entry(m, send_images)) + # The image generate_from_image would use: the frontend takes it from the + # user's last message, as run_workflow does. + last_user = next((m for m in reversed(request.messages) if m.role == "user"), None) + has_image = bool(last_user and last_user.images) actions_done: list[ActionDone] = [] all_thinking: list[str] = [] @@ -941,7 +1182,10 @@ async def post() -> httpx.Response: "POST", f"{base_url}/chat/completions", headers=headers, - json={"model": request.model, "messages": messages, "tools": TOOLS, "stream": True, **extra}, + json={ + "model": request.model, "messages": messages, + "tools": TOOLS if client_tools else SERVER_TOOLS, "stream": True, **extra, + }, ), stream=True, ) @@ -988,17 +1232,46 @@ async def post() -> httpx.Response: break arguments = _tool_arguments(fn.get("arguments")) yield {"type": "tool_start", "id": tc["id"], "tool": fn["name"], "arguments": arguments} - try: - result_text, payload = await asyncio.wait_for( - execute_tool(fn["name"], arguments, request.context, timeout=request.tool_timeout), - request.tool_timeout, - ) - except asyncio.TimeoutError: - # Cancelling the call drops its HTTP request, but a mesh op - # the API already started runs to its end in a worker - # thread; its result is never shown. - stopped = _tool_stop("tool_timeout", fn["name"], request.tool_timeout) - result_text, payload = stopped["message"], None + if fn["name"] in CLIENT_TOOLS and not client_tools: + # Not offered here, but a model may still name one. + result_text, payload = _NEEDS_APP, None + elif fn["name"] in CLIENT_TOOLS: + # The app runs it and answers through /agent/tool-result. + confirm = CLIENT_TOOLS[fn["name"]] + call_id = uuid.uuid4().hex + answer = asyncio.get_running_loop().create_future() + confirmed = asyncio.Event() + _pending_client_tools[call_id] = answer + _confirmed_client_tools[call_id] = confirmed + try: + yield { + "type": "client_tool", "id": tc["id"], "call_id": call_id, + "tool": fn["name"], "arguments": arguments, "confirm": confirm, + } + result_text = await _await_client_answer( + fn["name"], answer, confirmed, confirm, request.tool_timeout, + ) + finally: + # Also runs when the client leaves: nobody will answer. + _pending_client_tools.pop(call_id, None) + _confirmed_client_tools.pop(call_id, None) + answer.cancel() + payload = None + else: + try: + result_text, payload = await asyncio.wait_for( + execute_tool( + fn["name"], arguments, request.context, + timeout=request.tool_timeout, has_image=has_image, + ), + request.tool_timeout, + ) + except asyncio.TimeoutError: + # Cancelling the call drops its HTTP request, but a mesh op + # the API already started runs to its end in a worker + # thread; its result is never shown. + stopped = _tool_stop("tool_timeout", fn["name"], request.tool_timeout) + result_text, payload = stopped["message"], None actions_done.append(ActionDone(tool=fn["name"], result=result_text, payload=payload, error=stopped)) yield { "type": "tool_done", "id": tc["id"], "tool": fn["name"], @@ -1026,11 +1299,46 @@ async def post() -> httpx.Response: yield _done(final, actions_done, all_thinking) +class ToolResultRequest(BaseModel): + call_id: str + result: str + error: bool = False + + +@router.post("/tool-result") +async def agent_tool_result(req: ToolResultRequest): + """The app's answer to a client_tool event; the waiting turn resumes with it.""" + answer = _pending_client_tools.get(req.call_id) + if answer is None or answer.done(): + raise HTTPException(status_code=404, detail="No tool call is waiting for this call_id.") + # The app's results already start with "Error: "; don't say it twice. + result = req.result + if req.error and not result.startswith("Error:"): + result = f"Error: {result}" + answer.set_result(result) + return {"ok": True} + + +class ToolConfirmedRequest(BaseModel): + call_id: str + + +@router.post("/tool-confirmed") +async def agent_tool_confirmed(req: ToolConfirmedRequest): + """The user clicked Confirm: the turn now waits tool_timeout for the result.""" + confirmed = _confirmed_client_tools.get(req.call_id) + answer = _pending_client_tools.get(req.call_id) + if confirmed is None or answer is None or answer.done(): + raise HTTPException(status_code=404, detail="No tool call is waiting for this call_id.") + confirmed.set() + return {"ok": True} + + @router.post("/chat", response_model=AgentChatResponse) async def agent_chat(request: AgentChatRequest): """The whole turn at once, for callers that don't read a stream.""" done: dict = {} - async for event in _chat_events(request): + async for event in _chat_events(request, client_tools=False): if event["type"] == "done": done = event return AgentChatResponse( diff --git a/api/tests/test_agent_client_tools.py b/api/tests/test_agent_client_tools.py new file mode 100644 index 000000000..cd43d3b54 --- /dev/null +++ b/api/tests/test_agent_client_tools.py @@ -0,0 +1,465 @@ +import asyncio +import json +import unittest +from unittest import mock + +import httpx +from fastapi import HTTPException + +import routers.agent as agent +from tests.test_agent_streaming import _call, _external, _Scripted, _sse + + +def _client_round(name: str, arguments: dict | None = None, call_id: str = "c1") -> httpx.Response: + return _sse(_call(0, id=call_id, function={"name": name, "arguments": json.dumps(arguments or {})})) + + +async def _answer(call_id: str, result: str, error: bool = False) -> dict: + return await agent.agent_tool_result(agent.ToolResultRequest(call_id=call_id, result=result, error=error)) + + +async def _confirmed(call_id: str) -> dict: + return await agent.agent_tool_confirmed(agent.ToolConfirmedRequest(call_id=call_id)) + + +def _run_turn(scripted: _Scripted, request: agent.AgentChatRequest, on_client_tool) -> list[dict]: + """Drive a turn, letting `on_client_tool(event)` answer each client tool + the way the panel would (it may be a coroutine function).""" + + async def collect(): + events = [] + async for event in agent._chat_events(request): + events.append(event) + if event["type"] == "client_tool": + answered = on_client_tool(event) + if asyncio.iscoroutine(answered): + await answered + return events + + with mock.patch.object(agent.httpx, "AsyncClient", scripted): + return asyncio.run(collect()) + + +class ClientToolBridgeTests(unittest.TestCase): + def tearDown(self) -> None: + agent._pending_client_tools.clear() + agent._confirmed_client_tools.clear() + + def test_client_tool_is_handed_to_the_app_and_its_answer_reaches_the_model(self) -> None: + scripted = _Scripted( + _client_round("install_extension", {"github_url": "https://github.com/a/b"}), + _sse({"content": "Installed."}), + ) + execute = mock.AsyncMock(return_value=("never", None)) + answers = [] + + async def answer(event): + answers.append(await _answer(event["call_id"], "Installed b v1.0.")) + + with mock.patch.object(agent, "execute_tool", execute): + events = _run_turn(scripted, _external(), answer) + + execute.assert_not_awaited() + self.assertEqual([e["type"] for e in events], ["tool_start", "client_tool", "tool_done", "token", "done"]) + handoff = events[1] + self.assertEqual( + {k: v for k, v in handoff.items() if k != "call_id"}, + { + "type": "client_tool", "id": "c1", "tool": "install_extension", + "arguments": {"github_url": "https://github.com/a/b"}, "confirm": True, + }, + ) + self.assertRegex(handoff["call_id"], r"^[0-9a-f]{32}$") + self.assertEqual(answers, [{"ok": True}]) + self.assertEqual(events[2]["result"], "Installed b v1.0.") + self.assertEqual(events[-1]["actions"][0]["tool"], "install_extension") + self.assertEqual( + scripted.bodies[1]["messages"][-1], + {"role": "tool", "tool_call_id": "c1", "content": "Installed b v1.0."}, + ) + self.assertEqual(agent._pending_client_tools, {}) + + def test_answer_while_the_turn_is_waiting(self) -> None: + scripted = _Scripted(_client_round("reload_extensions"), _sse({"content": "ok"})) + + def answer_later(event): + async def later(): + await asyncio.sleep(0.01) + await _answer(event["call_id"], "Reloaded.") + asyncio.ensure_future(later()) + + events = _run_turn(scripted, _external(), answer_later) + + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Reloaded.") + self.assertFalse(events[-1]["error"]) + + def test_destructive_tool_asks_for_confirmation_and_a_decline_is_relayed(self) -> None: + scripted = _Scripted( + _client_round("uninstall_extension", {"extension_id": "x"}), + _sse({"content": "Kept it."}), + ) + events = _run_turn(scripted, _external(), lambda e: _answer(e["call_id"], "The user declined.")) + + handoff = next(e for e in events if e["type"] == "client_tool") + self.assertTrue(handoff["confirm"]) + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "The user declined.") + + def test_error_answer_is_prefixed(self) -> None: + scripted = _Scripted(_client_round("reload_extensions"), _sse({"content": "ok"})) + events = _run_turn(scripted, _external(), lambda e: _answer(e["call_id"], "disk full", error=True)) + + self.assertEqual(events[2]["result"], "Error: disk full") + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Error: disk full") + + def test_an_error_already_prefixed_by_the_app_is_not_prefixed_again(self) -> None: + scripted = _Scripted(_client_round("reload_extensions"), _sse({"content": "ok"})) + events = _run_turn(scripted, _external(), lambda e: _answer(e["call_id"], "Error: disk full", error=True)) + + self.assertEqual(events[2]["result"], "Error: disk full") + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Error: disk full") + + def test_unknown_or_already_answered_call_is_404(self) -> None: + with self.assertRaises(HTTPException) as caught: + asyncio.run(_answer("nope", "x")) + self.assertEqual(caught.exception.status_code, 404) + + seen = [] + + async def answer_twice(event): + await _answer(event["call_id"], "first") + try: + await _answer(event["call_id"], "second") + except HTTPException as e: + seen.append(e.status_code) + + _run_turn(_Scripted(_client_round("reload_extensions"), _sse({"content": "ok"})), _external(), answer_twice) + self.assertEqual(seen, [404]) + + def test_unanswered_tool_times_out_and_the_loop_continues(self) -> None: + scripted = _Scripted(_client_round("reload_extensions"), _sse({"content": "It did not answer."})) + events = _run_turn(scripted, _external().model_copy(update={"tool_timeout": 0.05}), lambda e: None) + + finish = next(e for e in events if e["type"] == "tool_done") + self.assertEqual(finish["result"], "Error: The app did not answer in time.") + self.assertIsNone(finish["error"]) + done = events[-1] + self.assertFalse(done["error"]) + self.assertIsNone(done["stopped"]) + self.assertEqual(done["message"], "It did not answer.") + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Error: The app did not answer in time.") + self.assertEqual(agent._pending_client_tools, {}) + + def test_confirm_tools_wait_for_the_confirm_cap_not_the_tool_timeout(self) -> None: + scripted = _Scripted(_client_round("delete_workflow", {"workflow_id": "w"}), _sse({"content": "ok"})) + + def answer_after_tool_timeout(event): + async def later(): + await asyncio.sleep(0.15) + await _answer(event["call_id"], "Deleted.") + asyncio.ensure_future(later()) + + with mock.patch.object(agent, "CONFIRM_TIMEOUT", 5.0): + _run_turn(scripted, _external().model_copy(update={"tool_timeout": 0.05}), answer_after_tool_timeout) + + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Deleted.") + + def test_confirm_wait_is_capped(self) -> None: + self.assertEqual(agent.CONFIRM_TIMEOUT, 600) + scripted = _Scripted(_client_round("delete_workflow", {"workflow_id": "w"}), _sse({"content": "ok"})) + with mock.patch.object(agent, "CONFIRM_TIMEOUT", 0.05): + events = _run_turn(scripted, _external(), lambda e: None) + + self.assertEqual(events[2]["result"], "Error: The app did not answer in time.") + + def test_after_confirm_the_tool_gets_the_tool_timeout(self) -> None: + scripted = _Scripted(_client_round("set_storage_folder", {"kind": "models", "path": "E:/m"}), _sse({"content": "ok"})) + seen = [] + + async def confirm_then_hang(event): + await asyncio.sleep(0.02) + seen.append(await _confirmed(event["call_id"])) + + def on_tool(event): + asyncio.ensure_future(confirm_then_hang(event)) + + with mock.patch.object(agent, "CONFIRM_TIMEOUT", 5.0): + events = _run_turn(scripted, _external().model_copy(update={"tool_timeout": 0.05}), on_tool) + + self.assertEqual(seen, [{"ok": True}]) + result = events[2]["result"] + self.assertTrue(result.startswith("Error: set_storage_folder was confirmed but did not finish within 0.05 s")) + self.assertIn("may still be running", result) + self.assertEqual(agent._pending_client_tools, {}) + self.assertEqual(agent._confirmed_client_tools, {}) + + def test_a_confirmed_tool_answering_within_the_tool_timeout_is_relayed(self) -> None: + scripted = _Scripted(_client_round("delete_workflow", {"workflow_id": "w"}), _sse({"content": "ok"})) + + def on_tool(event): + async def click_then_run(): + await asyncio.sleep(0.1) # longer than tool_timeout: the click is not timed by it + await _confirmed(event["call_id"]) + await asyncio.sleep(0.02) + await _answer(event["call_id"], "Deleted.") + asyncio.ensure_future(click_then_run()) + + with mock.patch.object(agent, "CONFIRM_TIMEOUT", 5.0): + _run_turn(scripted, _external().model_copy(update={"tool_timeout": 0.05}), on_tool) + + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Deleted.") + + def test_confirmed_for_an_unknown_or_answered_call_is_404(self) -> None: + with self.assertRaises(HTTPException) as caught: + asyncio.run(_confirmed("nope")) + self.assertEqual(caught.exception.status_code, 404) + + seen = [] + + async def answer_then_confirm(event): + await _answer(event["call_id"], "The user declined.") + try: + await _confirmed(event["call_id"]) + except HTTPException as e: + seen.append(e.status_code) + + _run_turn( + _Scripted(_client_round("delete_workflow", {"workflow_id": "w"}), _sse({"content": "ok"})), + _external(), answer_then_confirm, + ) + self.assertEqual(seen, [404]) + + def test_client_tools_count_against_the_budget(self) -> None: + both = _sse( + _call(0, id="a", function={"name": "reload_extensions", "arguments": "{}"}), + _call(1, id="b", function={"name": "undo_mesh", "arguments": "{}"}), + ) + events = _run_turn( + _Scripted(both), _external().model_copy(update={"tool_budget": 1}), + lambda e: _answer(e["call_id"], "ok"), + ) + + self.assertEqual(sum(e["type"] == "client_tool" for e in events), 1) + self.assertEqual(events[-1]["stopped"]["code"], "tool_budget_exhausted") + self.assertEqual(events[-1]["stopped"]["tool"], "undo_mesh") + + def test_closing_the_stream_while_a_tool_waits_cancels_it(self) -> None: + async def leave_at_handoff(): + stream = agent._chat_events(_external()) + async for event in stream: + if event["type"] == "client_tool": + future = agent._pending_client_tools[event["call_id"]] + await stream.aclose() + return event, future + + with mock.patch.object(agent.httpx, "AsyncClient", _Scripted(_client_round("reload_extensions"))): + event, future = asyncio.run(leave_at_handoff()) + + self.assertTrue(future.cancelled()) + self.assertNotIn(event["call_id"], agent._pending_client_tools) + + def test_cancelling_the_turn_while_it_awaits_the_app_cancels_the_future(self) -> None: + async def cancel_while_waiting(): + stream = agent._chat_events(_external()) + while (event := await stream.__anext__())["type"] != "client_tool": + pass + future = agent._pending_client_tools[event["call_id"]] + waiting = asyncio.ensure_future(stream.__anext__()) + await asyncio.sleep(0.01) + waiting.cancel() + with self.assertRaises(asyncio.CancelledError): + await waiting + return event, future + + with mock.patch.object(agent.httpx, "AsyncClient", _Scripted(_client_round("reload_extensions"))): + event, future = asyncio.run(cancel_while_waiting()) + + self.assertTrue(future.cancelled()) + self.assertNotIn(event["call_id"], agent._pending_client_tools) + + +class NonStreamingChatTests(unittest.TestCase): + def test_chat_does_not_offer_client_tools(self) -> None: + scripted = _Scripted(_sse({"content": "hi"})) + with mock.patch.object(agent.httpx, "AsyncClient", scripted): + response = asyncio.run(agent.agent_chat(_external())) + + self.assertEqual(response.message, "hi") + offered = {t["function"]["name"] for t in scripted.bodies[0]["tools"]} + self.assertFalse(offered & set(agent.CLIENT_TOOLS)) + self.assertIn("list_extensions", offered) + + def test_chat_answers_a_client_tool_call_at_once(self) -> None: + scripted = _Scripted(_client_round("reload_extensions"), _sse({"content": "Could not."})) + with mock.patch.object(agent.httpx, "AsyncClient", scripted): + response = asyncio.run(asyncio.wait_for(agent.agent_chat(_external()), 5)) + + self.assertEqual(response.actions[0].result, "Error: This tool needs the Modly app.") + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Error: This tool needs the Modly app.") + self.assertEqual(agent._pending_client_tools, {}) + + def test_the_stream_still_offers_client_tools(self) -> None: + scripted = _Scripted(_sse({"content": "hi"})) + _run_turn(scripted, _external(), None) + offered = {t["function"]["name"] for t in scripted.bodies[0]["tools"]} + self.assertTrue(set(agent.CLIENT_TOOLS) <= offered) + + +_CLIENT_TOOL_ARGS = { + "install_extension": ["github_url"], + "uninstall_extension": ["extension_id"], + "reload_extensions": [], + "download_model_weights": ["extension_id", "variant"], + "delete_model_weights": ["extension_id", "variant"], + "delete_workflow": ["workflow_id"], + "open_workflow": ["workflow_id"], + "set_setting": ["key", "value"], + "set_storage_folder": ["kind", "path"], + "undo_mesh": [], + "redo_mesh": [], + "navigate": ["page"], +} + + +class ClientToolDeclarationTests(unittest.TestCase): + def _tool(self, name: str) -> dict: + return next(t["function"] for t in agent.TOOLS if t["function"]["name"] == name) + + def test_every_client_tool_is_declared_with_its_confirmation(self) -> None: + self.assertEqual(set(agent.CLIENT_TOOLS), set(_CLIENT_TOOL_ARGS)) + confirmed = {name for name, confirm in agent.CLIENT_TOOLS.items() if confirm} + self.assertEqual( + confirmed, + # install_extension runs code downloaded from GitHub: never without a click. + {"install_extension", "uninstall_extension", "delete_model_weights", "delete_workflow", "set_storage_folder"}, + ) + + def test_every_client_tool_has_a_schema(self) -> None: + for name, args in _CLIENT_TOOL_ARGS.items(): + with self.subTest(name=name): + fn = self._tool(name) + self.assertTrue(fn["description"]) + self.assertEqual(sorted(fn["parameters"]["properties"]), sorted(args)) + + def test_enums(self) -> None: + props = self._tool("set_setting")["parameters"]["properties"] + self.assertEqual(props["key"]["enum"], ["uiScale", "useAtkinsonFont", "showRamIndicator"]) + # The app's scale names, not a number: the frontend accepts only these. + self.assertEqual(props["value"]["type"], ["string", "boolean"]) + self.assertIn("small, medium, large, very-large", props["value"]["description"]) + props = self._tool("set_storage_folder")["parameters"]["properties"] + self.assertEqual(props["kind"]["enum"], ["models", "workspace", "workflows"]) + nav = self._tool("navigate") + self.assertEqual(nav["parameters"]["properties"]["page"]["enum"], ["generate", "workflows", "models", "settings"]) + self.assertIn("ONLY", nav["description"]) + + def test_tool_names_are_unique(self) -> None: + names = [t["function"]["name"] for t in agent.TOOLS] + self.assertEqual(len(names), len(set(names))) + + def test_system_prompt_lists_the_new_tools(self) -> None: + for name in [*_CLIENT_TOOL_ARGS, "list_extensions", "get_settings", "generate_from_image"]: + with self.subTest(name=name): + self.assertIn(name, agent.SYSTEM_PROMPT) + self.assertIn("declined", agent.SYSTEM_PROMPT) + + +_CONTEXT = { + "extensions": [{"id": "trellis/gen", "name": "Trellis", "input": "image", "output": "mesh"}], + "models": [{"id": "trellis", "name": "Trellis 2"}, {"id": "sf3d", "name": "SF3D"}], + "storage": { + "modelsDir": "D:/models", "workspaceDir": "D:/ws", + "workflowsDir": "D:/wf", "extensionsDir": "D:/ext", + }, + "ui": {"uiScale": "large", "useAtkinsonFont": False, "showRamIndicator": True}, +} + + +def _run(name: str, arguments: dict | None = None, context: dict | None = None, **kwargs): + return asyncio.run(agent.execute_tool(name, arguments or {}, _CONTEXT if context is None else context, **kwargs)) + + +class ServerToolTests(unittest.TestCase): + def test_list_extensions(self) -> None: + text, payload = _run("list_extensions") + self.assertIn("- trellis/gen (image→mesh): Trellis", text) + self.assertIsNone(payload) + self.assertIn("No extensions", _run("list_extensions", context={})[0]) + + def test_get_settings(self) -> None: + text, payload = _run("get_settings") + for expected in ("D:/models", "D:/ws", "D:/wf", "D:/ext", "uiScale: large", "useAtkinsonFont: false", "showRamIndicator: true"): + self.assertIn(expected, text) + self.assertIsNone(payload) + self.assertIn("No settings", _run("get_settings", context={})[0]) + + def test_generate_from_image_needs_an_attached_image(self) -> None: + text, payload = _run("generate_from_image") + self.assertIsNone(payload) + self.assertIn("attach", text.lower()) + + def test_generate_from_image_hands_the_run_to_the_app(self) -> None: + text, payload = _run("generate_from_image", {"model_id": "sf3d"}, has_image=True) + self.assertEqual(payload, {"type": "generate_from_image", "model_id": "sf3d"}) + self.assertIn("SF3D", text) + + text, payload = _run("generate_from_image", has_image=True) + self.assertEqual(payload, {"type": "generate_from_image", "model_id": None}) + + def test_generate_from_image_rejects_an_unknown_model(self) -> None: + text, payload = _run("generate_from_image", {"model_id": "ghost"}, has_image=True) + self.assertIsNone(payload) + self.assertIn("ghost", text) + self.assertIn("trellis", text) + + def test_the_turn_tells_the_tool_whether_the_last_user_message_has_an_image(self) -> None: + def turn(messages): + request = _external().model_copy(update={"messages": messages}) + tool = mock.AsyncMock(return_value=("ok", None)) + with mock.patch.object(agent, "execute_tool", tool): + _run_turn(_Scripted(_client_round("generate_from_image"), _sse({"content": "ok"})), request, None) + return tool.await_args.kwargs["has_image"] + + img = "data:image/png;base64,AAAA" + self.assertTrue(turn([agent.ChatMessage(role="user", content="go", images=[img])])) + self.assertFalse(turn([ + agent.ChatMessage(role="user", content="a", images=[img]), + agent.ChatMessage(role="assistant", content="b"), + agent.ChatMessage(role="user", content="now generate"), + ])) + + +class ContextRenderingTests(unittest.TestCase): + def test_context_is_rendered_into_the_single_system_message(self) -> None: + context = { + **_CONTEXT, + "page": "models", + "workflows": [{"id": "wf1", "name": "My Workflow"}], + "currentMeshPath": "Default/mesh.glb", + } + scripted = _Scripted(_sse({"content": "hi"})) + _run_turn(scripted, _external(context), None) + + messages = scripted.bodies[0]["messages"] + self.assertEqual(sum(m["role"] == "system" for m in messages), 1) + system = messages[0]["content"] + for expected in ( + "Current page: models", + "- trellis: Trellis 2", + "- sf3d: SF3D", + "- wf1: My Workflow", + "D:/models", "D:/ws", "D:/wf", "D:/ext", + "uiScale: large", "useAtkinsonFont: false", "showRamIndicator: true", + "Current mesh path: Default/mesh.glb", + "- trellis/gen (image→mesh): Trellis", + ): + with self.subTest(expected=expected): + self.assertIn(expected, system) + + def test_empty_context_adds_nothing(self) -> None: + scripted = _Scripted(_sse({"content": "hi"})) + _run_turn(scripted, _external(), None) + self.assertEqual(scripted.bodies[0]["messages"][0]["content"], agent.SYSTEM_PROMPT) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md b/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md new file mode 100644 index 000000000..43a784bc0 --- /dev/null +++ b/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md @@ -0,0 +1,78 @@ +# Agent side panel: design + +## Intent + +The agent today lives in the "chat" mode of the Generate page's left panel. It loses its conversation whenever that panel or the page changes. The user wants the agent available **all the time**, in a panel on the right that **pushes the whole app**, so that the app can be driven entirely through the agent. + +## Goals + +- **Global panel.** A right-side panel lives in `MainLayout`, next to `
`, and pushes the content rather than overlaying it. + - A TopBar button toggles it, and so does `Ctrl+L` (my call). + - Its width can be resized by dragging (min 320, max 640, default 380; my call). + - Its open state and width are remembered across restarts. +- **Chat moves out of Generate.** + - `WorkflowPanel` loses its basic/chat toggle and keeps basic mode only. + - `ChatPanel` moves to `src/shared/components/agent/` as the panel body, keeping its current behaviour (streaming, thinking, actions card, mesh undo, workflow progress, image attachments). +- **The agent acts on the whole app without navigating.** It works on the page's state through stores, IPC and the API, never by changing `currentPage`, and confirms what it did in the panel. +- **Destructive actions ask first.** The panel shows a Confirm / Cancel card and the agent waits for the answer. This applies to uninstalling an extension, deleting model weights, deleting or overwriting a workflow, and changing a storage folder. +- **Conversations are persisted.** + - Each conversation is a JSON file in `/agent/conversations/{id}.json`. + - The panel lists past conversations (title, date) and offers New, Open and Delete. + - Images attached to messages are stored inline as data URLs (my call; simple, and good enough at this scale). +- **Context everywhere.** Every request sends the current page, the mesh (path, triangles), workflows, installed extensions, downloaded models and storage paths. + +## New agent tools + +**Server-side tools** run in `execute_tool` and call the local API: + +| Tool | Does | +|---|---| +| `generate_from_image(model_id?)` | Starts a generation from the image attached to the message. The frontend runs it, as with `run_workflow`. | +| `list_extensions` | Lists installed extensions (from context). | +| `get_settings` | Returns the storage paths and a few UI options (from context). | + +**Client-side tools** need Electron IPC or renderer stores, so the frontend runs them: + +| Tool | Confirm? | +|---|---| +| `install_extension(github_url)` | no | +| `uninstall_extension(extension_id)` | **yes** | +| `reload_extensions` | no | +| `download_model_weights(extension_id, variant?)` | no | +| `delete_model_weights(extension_id, variant?)` | **yes** | +| `delete_workflow(workflow_id)` | **yes** | +| `open_workflow(workflow_id)` | no (opens it in the workflows store, does not navigate) | +| `set_setting(key, value)`, with key in `uiScale` / `useAtkinsonFont` / `showRamIndicator` | no | +| `set_storage_folder(kind, path)` | **yes** | +| `undo_mesh` / `redo_mesh` | no | +| `navigate(page)` | no; only when the user explicitly asks to go to a page | + +## Client tool bridge + +This is the core mechanism. + +1. The LLM calls a client tool. `_chat_events` does not run it. It yields a `client_tool` event `{id, tool, arguments, confirm}`, registers an `asyncio.Future` under the call id, and awaits it. +2. The panel runs the tool through a frontend registry (`clientTools.ts`, mapping each tool name to an async handler that returns a result string). When `confirm` is set, it first shows the Confirm / Cancel card. Cancel resolves with "The user declined." +3. The panel sends `POST /agent/tool-result {call_id, result, error?}`. The future resolves, the result goes into `messages` as a real `role: tool` message, and the loop continues in the same turn. +4. Timeouts work differently depending on whether the user is involved: + - Execution is bounded by `tool_timeout`. + - Waiting for the user to confirm is not bounded by it, but is capped at 10 minutes. + - If the stream disconnects, the futures are cancelled. +5. Which tools run on the client, and which of those need confirmation, is declared once on the backend in a `CLIENT_TOOLS` dict. The frontend just obeys `confirm`. + +This replaces nothing that exists today. `mesh_update`, `create_workflow` and `run_workflow` payloads keep working. + +## Non-goals + +- No page navigation as a side effect of an action. +- No change to the providers or to Settings > Agent. +- No change to the MCP server. + +## Anti-goals + +- The agent performing a destructive action without an explicit Confirm click. +- Losing the conversation when the page changes. + +## Left to the builder + +Visual details of the panel and the conversation list, as long as they match the existing zinc/violet style. diff --git a/electron/main/agent-conversations.test.ts b/electron/main/agent-conversations.test.ts new file mode 100644 index 000000000..b680039d7 --- /dev/null +++ b/electron/main/agent-conversations.test.ts @@ -0,0 +1,171 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { mkdtemp, readdir, rename, rm, utimes, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { + ORPHAN_TMP_AGE_MS, + createConversationStorage, + isConversation, + isValidConversationId, + sortSummaries, +} from './agent-conversations' + +async function withDir(fn: (dir: string) => Promise): Promise { + const root = await mkdtemp(join(tmpdir(), 'modly-conv-')) + try { + await fn(join(root, 'agent', 'conversations')) + } finally { + await rm(root, { recursive: true, force: true }) + } +} + +const conv = (id: string, updatedAt: number, title = id) => + ({ id, title, createdAt: 1, updatedAt, messages: [{ id: 'u', role: 'user', content: title }] }) + +test('conversation ids: uuids pass, path tricks and odd characters do not', () => { + assert.equal(isValidConversationId('3f2a9c1e-8b7d-4e6f-a5b4-c3d2e1f0a9b8'), true) + assert.equal(isValidConversationId('a'), true) + assert.equal(isValidConversationId('a'.repeat(64)), true) + assert.equal(isValidConversationId('a'.repeat(65)), false) + assert.equal(isValidConversationId(''), false) + assert.equal(isValidConversationId('../secrets'), false) + assert.equal(isValidConversationId('..\\secrets'), false) + assert.equal(isValidConversationId('a/b'), false) + assert.equal(isValidConversationId('ABC'), false) + assert.equal(isValidConversationId('abc.json'), false) + assert.equal(isValidConversationId('abc\n'), false) + assert.equal(isValidConversationId(42), false) +}) + +test('isConversation checks the stored shape', () => { + assert.equal(isConversation(conv('abc-1', 2)), true) + assert.equal(isConversation({ ...conv('abc-1', 2), id: '../x' }), false) + assert.equal(isConversation({ ...conv('abc-1', 2), messages: 'nope' }), false) + assert.equal(isConversation({ ...conv('abc-1', 2), updatedAt: '2' }), false) + assert.equal(isConversation(null), false) +}) + +test('sortSummaries puts the newest first', () => { + const sorted = sortSummaries([conv('a', 10), conv('b', 30), conv('c', 20)]) + assert.deepEqual(sorted.map((s) => s.id), ['b', 'c', 'a']) +}) + +test('list is empty before anything is saved', async () => { + await withDir(async (dir) => { + assert.deepEqual(await createConversationStorage(dir).list(), []) + }) +}) + +test('save, read, list newest first, remove', async () => { + await withDir(async (dir) => { + const storage = createConversationStorage(dir) + await storage.save(conv('old', 10, 'Old')) + await storage.save(conv('new', 20, 'New')) + assert.deepEqual(await storage.read('old'), conv('old', 10, 'Old')) + assert.deepEqual(await storage.list(), [ + { id: 'new', title: 'New', updatedAt: 20 }, + { id: 'old', title: 'Old', updatedAt: 10 }, + ]) + // No temp file is left behind. + assert.deepEqual((await readdir(dir)).sort(), ['new.json', 'old.json']) + + await storage.remove('old') + assert.equal(await storage.read('old'), null) + assert.deepEqual((await storage.list()).map((s) => s.id), ['new']) + }) +}) + +test('saving again overwrites the file', async () => { + await withDir(async (dir) => { + const storage = createConversationStorage(dir) + await storage.save(conv('a', 1, 'First')) + await storage.save(conv('a', 2, 'Second')) + assert.equal((await storage.read('a'))?.title, 'Second') + }) +}) + +test('invalid ids and shapes are refused', async () => { + await withDir(async (dir) => { + const storage = createConversationStorage(dir) + await assert.rejects(storage.save({ ...conv('a', 1), id: '../escape' })) + await assert.rejects(storage.save({ id: 'a' })) + await assert.rejects(storage.remove('../escape')) + assert.equal(await storage.read('../escape'), null) + }) +}) + +test('corrupted or mismatched files are skipped', async () => { + await withDir(async (dir) => { + const storage = createConversationStorage(dir) + await storage.save(conv('good', 5)) + await writeFile(join(dir, 'broken.json'), '{not json', 'utf-8') + await writeFile(join(dir, 'other.json'), JSON.stringify(conv('different', 9)), 'utf-8') + await writeFile(join(dir, 'notes.txt'), 'hi', 'utf-8') + assert.deepEqual((await storage.list()).map((s) => s.id), ['good']) + assert.equal(await storage.read('broken'), null) + assert.equal(await storage.read('other'), null) + }) +}) + +const errno = (code: string): NodeJS.ErrnoException => Object.assign(new Error(code), { code }) + +test('a rename locked for a moment (antivirus, indexer) is retried', async () => { + await withDir(async (dir) => { + let calls = 0 + const storage = createConversationStorage(dir, { + retryDelayMs: 1, + rename: async (from, to) => { + calls++ + if (calls < 3) throw errno(calls === 1 ? 'EPERM' : 'EBUSY') + await rename(from, to) + }, + }) + await storage.save(conv('a', 1, 'Locked')) + assert.equal(calls, 3) + assert.equal((await storage.read('a'))?.title, 'Locked') + assert.deepEqual(await readdir(dir), ['a.json']) + }) +}) + +test('a rename still locked after the retries fails and leaves no temp file', async () => { + await withDir(async (dir) => { + let calls = 0 + const storage = createConversationStorage(dir, { + retryDelayMs: 1, + rename: async () => { calls++; throw errno('EACCES') }, + }) + await assert.rejects(storage.save(conv('a', 1)), { code: 'EACCES' }) + assert.equal(calls, 3) + assert.deepEqual(await readdir(dir), []) + }) +}) + +test('other rename errors are not retried', async () => { + await withDir(async (dir) => { + let calls = 0 + const storage = createConversationStorage(dir, { + retryDelayMs: 1, + rename: async () => { calls++; throw errno('ENOSPC') }, + }) + await assert.rejects(storage.save(conv('a', 1)), { code: 'ENOSPC' }) + assert.equal(calls, 1) + }) +}) + +test('list removes orphan temp files, but not ones a write may still own', async () => { + await withDir(async (dir) => { + const storage = createConversationStorage(dir) + await storage.save(conv('a', 1)) + const old = join(dir, 'a.json.1111.tmp') + const fresh = join(dir, 'a.json.2222.tmp') + await writeFile(old, '{', 'utf-8') + await writeFile(fresh, '{', 'utf-8') + const past = (Date.now() - ORPHAN_TMP_AGE_MS - 10_000) / 1000 + await utimes(old, past, past) + + assert.deepEqual((await storage.list()).map((s) => s.id), ['a']) + assert.deepEqual((await readdir(dir)).sort(), ['a.json', 'a.json.2222.tmp']) + }) +}) diff --git a/electron/main/agent-conversations.ts b/electron/main/agent-conversations.ts new file mode 100644 index 000000000..65d9011f3 --- /dev/null +++ b/electron/main/agent-conversations.ts @@ -0,0 +1,140 @@ +// Agent conversations on disk: one JSON file per conversation in +// /agent/conversations/{id}.json. + +import { join } from 'path' +import { randomUUID } from 'crypto' +import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'fs/promises' + +export interface ConversationSummary { + id: string + title: string + updatedAt: number +} + +export interface Conversation extends ConversationSummary { + createdAt: number + messages: unknown[] +} + +export const CONVERSATION_ID_RE = /^[a-z0-9-]{1,64}$/ + +/** Ids name files on disk: anything else could escape the conversations folder. */ +export function isValidConversationId(id: unknown): id is string { + return typeof id === 'string' && CONVERSATION_ID_RE.test(id) +} + +/** Shape check for a conversation read from disk or received over IPC. */ +export function isConversation(value: unknown): value is Conversation { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false + const c = value as Record + return isValidConversationId(c.id) + && typeof c.title === 'string' + && typeof c.createdAt === 'number' + && typeof c.updatedAt === 'number' + && Array.isArray(c.messages) +} + +/** Newest first; ties keep a stable order by id. */ +export function sortSummaries(list: readonly T[]): T[] { + return [...list].sort((a, b) => (b.updatedAt - a.updatedAt) || a.id.localeCompare(b.id)) +} + +/** Windows antivirus or indexers briefly lock a freshly written file. */ +const TRANSIENT_RENAME_CODES = new Set(['EPERM', 'EBUSY', 'EACCES']) +const RENAME_ATTEMPTS = 3 +const RENAME_RETRY_MS = 50 +/** A temp file this old belongs to no write in flight: a crash left it behind. */ +export const ORPHAN_TMP_AGE_MS = 60_000 + +export interface ConversationStorageOptions { + /** Injectable for tests. */ + rename?: (from: string, to: string) => Promise + retryDelayMs?: number + now?: () => number +} + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) + +export function createConversationStorage(dir: string, options: ConversationStorageOptions = {}) { + const renameFile = options.rename ?? rename + const retryDelayMs = options.retryDelayMs ?? RENAME_RETRY_MS + const now = options.now ?? Date.now + const fileFor = (id: string): string => join(dir, `${id}.json`) + + async function readConversation(id: string): Promise { + if (!isValidConversationId(id)) return null + try { + const parsed: unknown = JSON.parse(await readFile(fileFor(id), 'utf-8')) + return isConversation(parsed) && parsed.id === id ? parsed : null + } catch { + return null + } + } + + async function renameWithRetry(from: string, to: string): Promise { + for (let attempt = 1; ; attempt++) { + try { + await renameFile(from, to) + return + } catch (err) { + const code = (err as NodeJS.ErrnoException | null)?.code + if (attempt >= RENAME_ATTEMPTS || !code || !TRANSIENT_RENAME_CODES.has(code)) throw err + await sleep(retryDelayMs * attempt) + } + } + } + + /** Best effort: temp files left by a crash between write and rename. */ + async function removeOrphanTmpFiles(files: readonly string[]): Promise { + for (const file of files) { + if (!file.endsWith('.tmp')) continue + const path = join(dir, file) + try { + if (now() - (await stat(path)).mtimeMs >= ORPHAN_TMP_AGE_MS) await rm(path, { force: true }) + } catch { + // Gone already, or locked: the next listing tries again. + } + } + } + + return { + async list(): Promise { + let files: string[] + try { + files = await readdir(dir) + } catch { + return [] + } + await removeOrphanTmpFiles(files) + const summaries: ConversationSummary[] = [] + for (const file of files) { + if (!file.endsWith('.json')) continue + const c = await readConversation(file.slice(0, -'.json'.length)) + if (c) summaries.push({ id: c.id, title: c.title, updatedAt: c.updatedAt }) + } + return sortSummaries(summaries) + }, + + read: readConversation, + + async save(conversation: unknown): Promise { + if (!isConversation(conversation)) throw new Error('Invalid conversation') + await mkdir(dir, { recursive: true }) + const target = fileFor(conversation.id) + // Write then rename: a crash mid-write never leaves a truncated file. + const tmp = `${target}.${randomUUID()}.tmp` + await writeFile(tmp, JSON.stringify(conversation), 'utf-8') + try { + await renameWithRetry(tmp, target) + } catch (err) { + await rm(tmp, { force: true }) + throw err + } + }, + + async remove(id: string): Promise { + if (!isValidConversationId(id)) throw new Error('Invalid conversation id') + await rm(fileFor(id), { force: true }) + }, + } +} diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 33a9b2409..039705780 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -50,6 +50,7 @@ import { checkSetupNeeded, markSetupDone, runFullSetup, getVenvPythonExe, ensure import { logger } from './logger' import { getProcessRunner, getPythonProcessRunner, getExtPythonExe, terminateProcessRunner, terminateAllProcessRunners } from './process-runner' import { getBuiltinExtensionsDir } from './builtin-sync' +import { createConversationStorage } from './agent-conversations' import { spawn, execFile } from 'child_process' import { EXT_INCOMPLETE_MARKER, @@ -1078,6 +1079,14 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe return result.canceled ? null : result.filePaths[0] }) + // Agent conversations: one JSON file each under /agent/conversations. + // Ids are validated in the storage layer, so no path escapes that folder. + const conversations = createConversationStorage(join(app.getPath('userData'), 'agent', 'conversations')) + ipcMain.handle('agent:conversations:list', () => conversations.list()) + ipcMain.handle('agent:conversations:read', (_, id: string) => conversations.read(id)) + ipcMain.handle('agent:conversations:save', (_, conversation: unknown) => conversations.save(conversation)) + ipcMain.handle('agent:conversations:remove', (_, id: string) => conversations.remove(id)) + // Add a local GGUF to the agent's models folder. Picking and copying both // happen here, so the renderer never hands main an arbitrary path to copy. ipcMain.handle('agent:addModel', async (): Promise<{ success: boolean; cancelled?: boolean; fileName?: string; error?: string }> => { diff --git a/electron/preload/electron-api.ts b/electron/preload/electron-api.ts index 04b50ca06..d11863beb 100644 --- a/electron/preload/electron-api.ts +++ b/electron/preload/electron-api.ts @@ -5,6 +5,10 @@ import type { AssetLibraryReadRequest, AssetLibraryReadResult, } from '../../src/shared/types/assetLibrary' +import type { + Conversation as AgentConversation, + ConversationSummary as AgentConversationSummary, +} from '../main/agent-conversations' export interface IpcRendererLike { invoke(channel: string, ...args: unknown[]): Promise @@ -120,6 +124,17 @@ export function createElectronApi(ipcRenderer: IpcRendererLike, webFrame: WebFra // Opens a file picker and copies the chosen .gguf into the agent's models folder. addModel: (): Promise<{ success: boolean; cancelled?: boolean; fileName?: string; error?: string }> => ipcRenderer.invoke('agent:addModel') as Promise<{ success: boolean; cancelled?: boolean; fileName?: string; error?: string }>, + // Persisted chat conversations (/agent/conversations/{id}.json). + conversations: { + list: (): Promise => + ipcRenderer.invoke('agent:conversations:list') as Promise, + read: (id: string): Promise => + ipcRenderer.invoke('agent:conversations:read', id) as Promise, + save: (conversation: AgentConversation): Promise => + ipcRenderer.invoke('agent:conversations:save', conversation) as Promise, + remove: (id: string): Promise => + ipcRenderer.invoke('agent:conversations:remove', id) as Promise, + }, }, // Settings diff --git a/package.json b/package.json index 22e207ac9..dd05c25e1 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/generate/components/WorkflowPanel.tsx b/src/areas/generate/components/WorkflowPanel.tsx index 14b257815..d1be5e9f7 100644 --- a/src/areas/generate/components/WorkflowPanel.tsx +++ b/src/areas/generate/components/WorkflowPanel.tsx @@ -19,9 +19,6 @@ import type { Workflow, WFNode, WFEdge, ParamSchema } from '@shared/types/electr import { PICKER_LABELS, openParamPicker, resolvePickerIntent } from '@shared/utils/paramPicker' import { FloatInput, IntInput, PickerIcon } from '@shared/components/ui' import { isMissingWeightVariant, withWeightVariantAvailability } from '@shared/utils/weightVariants' -import ChatPanel from './ChatPanel' - -type PanelMode = 'basic' | 'chat' // ─── Constants ──────────────────────────────────────────────────────────────── @@ -610,30 +607,6 @@ function EmbeddedCanvas({ workflow, allExtensions }: { ) } -// ─── Mode toggle ────────────────────────────────────────────────────────────── - -function ModeToggle({ mode, onChange }: { mode: PanelMode; onChange: (m: PanelMode) => void }): JSX.Element { - return ( -
-
- {(['basic', 'chat'] as PanelMode[]).map((m) => ( - - ))} -
-
- ) -} - // ─── Main panel ─────────────────────────────────────────────────────────────── export default function WorkflowPanel() { @@ -642,7 +615,6 @@ export default function WorkflowPanel() { const loadExtensions = useExtensionsStore((s) => s.loadExtensions) const { navigate } = useNavStore() const [selectedId, setSelectedId] = useState(activeId) - const [mode, setMode] = useState('basic') const allExtensions = useMemo( () => buildAllWorkflowExtensions(modelExtensions, processExtensions), @@ -667,56 +639,44 @@ export default function WorkflowPanel() { return (
- {/* Mode toggle */} - - - {mode === 'chat' ? ( - <> -
- - - ) : ( - <> - {/* Header */} -
-

Workflow

-
-
- -
- {selectedId && ( - - )} -
+ {/* Header */} +
+

Workflow

+
+
+
- - {/* Canvas or empty state */} - {workflow ? ( - - - - ) : ( -
-

- No workflows yet.
Create one in the Workflows tab. -

-
+ {selectedId && ( + )} - +
+
+ + {/* Canvas or empty state */} + {workflow ? ( + + + + ) : ( +
+

+ No workflows yet.
Create one in the Workflows tab. +

+
)}
) diff --git a/src/areas/generate/components/ChatPanel.tsx b/src/shared/components/agent/AgentChat.tsx similarity index 80% rename from src/areas/generate/components/ChatPanel.tsx rename to src/shared/components/agent/AgentChat.tsx index 24d76f238..a7c1bcab5 100644 --- a/src/areas/generate/components/ChatPanel.tsx +++ b/src/shared/components/agent/AgentChat.tsx @@ -8,30 +8,19 @@ import { useWorkflowRunStore } from '@areas/workflows/workflowRunStore' import { buildAllWorkflowExtensions } from '@areas/workflows/mockExtensions' import { readAgentEvents, toolStopTitle } from '@shared/utils/agentStream' import { adviceText, ollamaAdvice } from '@shared/utils/ollamaHealth' -import OllamaBanner, { useOllamaHealth } from './OllamaBanner' -import { commitSculpt } from '../sculpt/sculptStore' +import OllamaBanner, { useOllamaHealth } from '@areas/generate/components/OllamaBanner' +import { commitSculpt } from '@areas/generate/sculpt/sculptStore' +import { useAgentConversationStore } from '@shared/stores/agentConversationStore' +import { buildAgentContext } from './agentContext' +import { createClientToolDeps, createGenerationDeps, latestUserImage, runClientTool, startGenerationFromImage } from './clientTools' +import { QueuedConfirmCard, useConfirmQueue } from './ConfirmCard' // ─── Types ──────────────────────────────────────────────────────────────────── import type { ThinkingMode } from '@shared/stores/agentStore' import type { Workflow } from '@shared/types/electron.d' -import type { AgentAction, AgentActionPayload, ToolStop } from '@shared/utils/agentStream' - -interface Message { - id: string - role: 'user' | 'assistant' - content: string - thinking?: string - imageDataUrls?: string[] - actions?: ActionDone[] - stopped?: ToolStop | null - streaming?: boolean -} - -interface ActionDone extends AgentAction { - id?: string - running?: boolean -} +import type { AgentActionPayload, ToolStop } from '@shared/utils/agentStream' +import type { ActionDone, AgentMessage as Message } from '@shared/stores/agentConversationStore' // ─── Constants ──────────────────────────────────────────────────────────────── @@ -270,13 +259,18 @@ function WorkflowProgressCard({ name }: { name: string }): JSX.Element { // ─── Main component ────────────────────────────────────────────────────────── -export default function ChatPanel(): JSX.Element { +export default function AgentChat(): JSX.Element { const { provider, localModel, external, defaultThinking, toolBudget, toolTimeout } = useAgentStore() const { models: llmModels } = useLlmModels() const isLocal = provider === 'local' const externalConfig = external[provider] - const [messages, setMessages] = useState([]) + // Messages live in the conversation store: they survive page changes and + // panel close, and are saved to disk. + const messages = useAgentConversationStore((s) => s.messages) + const setMessages = useAgentConversationStore((s) => s.setMessages) + const currentId = useAgentConversationStore((s) => s.currentId) + const setBusy = useAgentConversationStore((s) => s.setBusy) const [input, setInput] = useState('') const [isLoading, setIsLoading] = useState(false) const [error, setError] = useState(null) @@ -288,6 +282,8 @@ export default function ChatPanel(): JSX.Element { const localModels = llmModels.filter((m) => m.downloaded && !(m.tags ?? []).some((t) => t === 'code' || t === 'cad')) const [showModelPicker, setShowModelPicker] = useState(false) const [pendingWorkflow, setPendingWorkflow] = useState<{ id: string; name: string } | null>(null) + // Generations the agent started; the conversation stays put until they end. + const [generations, setGenerations] = useState(0) const [attachments, setAttachments] = useState([]) // data URLs const [isDragging, setIsDragging] = useState(false) const [thinkingMode, setThinkingMode] = useState(defaultThinking) @@ -304,6 +300,10 @@ export default function ChatPanel(): JSX.Element { const applyMeshEdit = useAppStore((s) => s.applyMeshEdit) const undoMesh = useAppStore((s) => s.undoMesh) + const clientDeps = useMemo(() => createClientToolDeps(), []) + const confirm = useConfirmQueue((s) => s.confirm) + const confirmRequests = useConfirmQueue((s) => s.requests) + const isOllama = provider === 'ollama' const ollama = useOllamaHealth(isOllama, apiUrl, providerBaseUrl(provider, externalConfig), model) @@ -358,32 +358,33 @@ export default function ChatPanel(): JSX.Element { // eslint-disable-next-line react-hooks/exhaustive-deps -- fire on run-status transition; error/outputUrl read atomically }, [runState.status, pendingWorkflow]) + // The panel locks conversation switching while a turn or its workflow runs. + useEffect(() => { setBusy(isLoading || pendingWorkflow !== null || generations > 0) }, [isLoading, pendingWorkflow, generations, setBusy]) + + // Another conversation was opened: drop the view state of the previous one. + useEffect(() => { + setShowAll(false) + setError(null) + useConfirmQueue.getState().cancelAll() + useConfirmQueue.getState().clearAnswered() + }, [currentId]) + useEffect(() => { // Instant while streaming: a smooth scroll per token never catches up. endRef.current?.scrollIntoView({ behavior: isLoading ? 'auto' : 'smooth' }) }, [messages, isLoading, pendingWorkflow]) - function buildContext(): Record { - const ctx: Record = {} - // Read from the store, not the render: a sculpt commit may just have swapped the mesh. - const { currentJob, meshStats } = useAppStore.getState() - if (currentJob?.outputUrl) ctx.currentMeshPath = currentJob.outputUrl.replace('/workspace/', '') - if (meshStats?.triangles) ctx.meshTriangles = meshStats.triangles - if (workflows.length > 0) ctx.workflows = workflows.map((w) => ({ id: w.id, name: w.name })) - if (allExtensions.length > 0) ctx.extensions = allExtensions.map((e) => ({ - id: e.id, name: e.name, input: e.input, output: e.output, - })) - return ctx - } - async function callAgent(msgs: Message[], extraContext: Record = {}) { + useConfirmQueue.getState().clearAnswered() setIsLoading(true) setError(null) const id = `a-${Date.now()}` let finished = false let failure: string | null = null try { - const context = { ...buildContext(), ...extraContext } + // Read at send time: the page, the mesh (a sculpt commit may just have + // swapped it), workflows, extensions, models, folders and UI settings. + const context: Record = { ...(await buildAgentContext()), ...extraContext } // Inject workflow completion as a system hint if present const apiMessages = msgs.map((m) => { @@ -437,7 +438,14 @@ export default function ChatPanel(): JSX.Element { case 'tool_start': patch((m) => ({ ...m, actions: [...(m.actions ?? []), { id: event.id, tool: event.tool, result: '', running: true }] })) break + case 'client_tool': + // Runs in the app (stores / IPC), after a Confirm click when asked; + // the server waits for its result, then sends the usual tool_done. + // Not awaited: the stream must keep reading meanwhile. + void runClientTool(event, clientDeps, confirm) + break case 'tool_done': + useConfirmQueue.getState().cancelEvent(event.id) patch((m) => ({ ...m, actions: (m.actions ?? []).map((a) => (a.id === event.id @@ -463,6 +471,7 @@ export default function ChatPanel(): JSX.Element { // Started only now: the local LLM frees its VRAM at the end of the // turn, and the workflow needs the whole GPU. for (const action of event.actions) startWorkflow(action.payload, msgs) + for (const action of event.actions) startGeneration(action.payload, msgs) break } } @@ -479,6 +488,8 @@ export default function ChatPanel(): JSX.Element { ? { ...m, streaming: false, actions: m.actions?.map((a) => ({ ...a, running: false })) } : m))) } + // A card still waiting when the turn ends has no one listening any more. + useConfirmQueue.getState().cancelAll() setIsLoading(false) } } @@ -507,6 +518,20 @@ export default function ChatPanel(): JSX.Element { } } + // Started after `done`, like workflows: the local LLM frees the GPU first. + function startGeneration(payload: AgentActionPayload | null | undefined, msgs: Message[]) { + if (payload?.type !== 'generate_from_image') return + const deps = createGenerationDeps() + // A refused start (one is already running) is not ours to count. + const counted = !deps.isGenerating() + if (counted) setGenerations((n) => n + 1) + void startGenerationFromImage(payload, latestUserImage(msgs), deps) + .then((result) => { + setMessages((prev) => [...prev, { id: `sys-${Date.now()}`, role: 'assistant', content: result }]) + }) + .finally(() => { if (counted) setGenerations((n) => n - 1) }) + } + function startWorkflow(payload: AgentActionPayload | null | undefined, msgs: Message[]) { if (payload?.type !== 'run_workflow' || !payload.workflow_id) return const wf = workflows.find((w) => w.id === payload.workflow_id) @@ -685,6 +710,7 @@ export default function ChatPanel(): JSX.Element {
)} + {confirmRequests.map((request) => )}
@@ -692,8 +718,9 @@ export default function ChatPanel(): JSX.Element { {/* Ollama down / model not pulled */} { void ollama.check() }} /> - {/* Input bar */} -
+ {/* Input bar: a composer box (text, attach, send) with the model and + thinking controls on a quiet line under it, as in Claude. */} +
{ if (e.target.files) handleFiles(Array.from(e.target.files)); e.target.value = '' }} /> -
+
{/* Attachment previews */} {attachments.length > 0 && (
@@ -711,6 +738,7 @@ export default function ChatPanel(): JSX.Element { - {/* Thinking toggle */} + + {/* Send */} - {/* Model selector */} -
- - - {showModelPicker && ( -
- {!isLocal ? ( -

{PROVIDERS[provider].label} — change the model in Settings → Agent.

- ) : localModels.length === 0 ? ( -

No local model downloaded — open Settings → Agent.

- ) : ( - localModels.map((m) => ( - - )) - )} -
- )} -
-
+
+
+ {/* Model and thinking, under the box */} +
+
+ + {showModelPicker && ( +
+ {!isLocal ? ( +

{PROVIDERS[provider].label} — change the model in Settings → Agent.

+ ) : localModels.length === 0 ? ( +

No local model downloaded — open Settings → Agent.

+ ) : ( + localModels.map((m) => ( + + )) + )} +
+ )}
+ + {/* Thinking: auto → on → off */} +
-

Shift+Enter for new line

diff --git a/src/shared/components/agent/AgentPanel.tsx b/src/shared/components/agent/AgentPanel.tsx new file mode 100644 index 000000000..cca241cf8 --- /dev/null +++ b/src/shared/components/agent/AgentPanel.tsx @@ -0,0 +1,300 @@ +import { useEffect, useRef, useState } from 'react' +import { + AGENT_PANEL_DEFAULT_WIDTH, + AGENT_PANEL_MAX_WIDTH, + AGENT_PANEL_MIN_WIDTH, + clampPanelWidth, + useAgentPanelStore, +} from '@shared/stores/agentPanelStore' +import { useAgentConversationStore } from '@shared/stores/agentConversationStore' +import { UNTITLED, deriveTitle, relativeDate } from '@shared/utils/agentConversations' +import AgentChat from './AgentChat' +import { useConfirmQueue } from './ConfirmCard' + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +function isEditable(el: EventTarget | null): boolean { + if (!(el instanceof HTMLElement)) return false + return el.isContentEditable || el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || el.tagName === 'SELECT' +} + +function IconButton({ label, onClick, pressed, disabled, children }: { + label: string + onClick: () => void + pressed?: boolean + disabled?: boolean + children: React.ReactNode +}): JSX.Element { + return ( + + ) +} + +// ─── Conversation list ──────────────────────────────────────────────────────── + +function ConversationList({ onPicked }: { onPicked: () => void }): JSX.Element { + const list = useAgentConversationStore((s) => s.list) + const currentId = useAgentConversationStore((s) => s.currentId) + const busy = useAgentConversationStore((s) => s.busy) + const open = useAgentConversationStore((s) => s.open) + const remove = useAgentConversationStore((s) => s.remove) + // Delete takes two clicks: the first arms it. + const [armedId, setArmedId] = useState(null) + const now = Date.now() + + if (list.length === 0) { + return ( +
+

No saved conversations yet.

+
+ ) + } + + return ( +
    setArmedId(null)}> + {list.map((c) => { + const active = c.id === currentId + const locked = busy && !active + return ( +
  • + + +
    • + ) + })} +
+ ) +} + +// ─── Panel ──────────────────────────────────────────────────────────────────── + +export default function AgentPanel(): JSX.Element { + const open = useAgentPanelStore((s) => s.open) + const width = useAgentPanelStore((s) => s.width) + const setOpen = useAgentPanelStore((s) => s.setOpen) + const toggle = useAgentPanelStore((s) => s.toggle) + const setWidth = useAgentPanelStore((s) => s.setWidth) + + const messages = useAgentConversationStore((s) => s.messages) + const busy = useAgentConversationStore((s) => s.busy) + const newConversation = useAgentConversationStore((s) => s.newConversation) + + const [showList, setShowList] = useState(false) + const [dragWidth, setDragWidth] = useState(null) + const panelRef = useRef(null) + const dragRef = useRef<{ startX: number; startWidth: number } | null>(null) + + const shownWidth = dragWidth ?? width + const title = messages.length > 0 ? deriveTitle(messages) : UNTITLED + + // Load the saved conversations and reopen the latest; save before quitting. + useEffect(() => { + const { init, flush } = useAgentConversationStore.getState() + void init() + const onUnload = () => { void flush() } + window.addEventListener('beforeunload', onUnload) + return () => window.removeEventListener('beforeunload', onUnload) + }, []) + + // A confirm card is waiting for a click: show it, even if the panel was + // closed mid-turn or the conversation list covered the chat. + useEffect(() => { + const { setOnQueued } = useConfirmQueue.getState() + setOnQueued(() => { + setOpen(true) + setShowList(false) + }) + return () => setOnQueued(null) + }, [setOpen]) + + // Ctrl+L / Cmd+L toggles the panel, except while typing elsewhere. + useEffect(() => { + const handler = (e: KeyboardEvent) => { + if (!(e.ctrlKey || e.metaKey) || e.shiftKey || e.altKey || e.key.toLowerCase() !== 'l') return + const target = e.target as Node | null + if (isEditable(e.target) && !panelRef.current?.contains(target)) return + e.preventDefault() + toggle() + } + window.addEventListener('keydown', handler) + return () => window.removeEventListener('keydown', handler) + }, [toggle]) + + // Opening the panel puts the cursor in the chat input. + useEffect(() => { + if (!open || showList) return + panelRef.current?.querySelector('textarea')?.focus() + }, [open, showList]) + + // ── Resize ── + function onHandlePointerDown(e: React.PointerEvent) { + if (e.button !== 0) return + e.preventDefault() + e.currentTarget.setPointerCapture(e.pointerId) + dragRef.current = { startX: e.clientX, startWidth: width } + setDragWidth(width) + } + + function onHandlePointerMove(e: React.PointerEvent) { + const drag = dragRef.current + if (!drag) return + // The panel sits on the right: dragging left widens it. + setDragWidth(clampPanelWidth(drag.startWidth + drag.startX - e.clientX)) + } + + function onHandlePointerUp(e: React.PointerEvent) { + if (!dragRef.current) return + e.currentTarget.releasePointerCapture(e.pointerId) + dragRef.current = null + if (dragWidth !== null) setWidth(dragWidth) + setDragWidth(null) + } + + function onHandleKeyDown(e: React.KeyboardEvent) { + const step = e.shiftKey ? 64 : 16 + if (e.key === 'ArrowLeft') { e.preventDefault(); setWidth(width + step) } + if (e.key === 'ArrowRight') { e.preventDefault(); setWidth(width - step) } + } + + const dragging = dragWidth !== null + + return ( + // The slot grows from 0 to the panel width, pushing the app aside, while + // the panel slides in from the right. Closed, it stays mounted (a streaming + // turn or a running workflow keeps going) but invisible, so it takes no + // focus once the slide-out has ended. +
+ +
+ ) +} diff --git a/src/shared/components/agent/ConfirmCard.test.mjs b/src/shared/components/agent/ConfirmCard.test.mjs new file mode 100644 index 000000000..a9553ed95 --- /dev/null +++ b/src/shared/components/agent/ConfirmCard.test.mjs @@ -0,0 +1,88 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { buildSync } from 'esbuild' +import { createRequire } from 'node:module' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +// Bundle ConfirmCard.tsx with its real react/zustand dependencies; its only +// other import is type-only. Rendering is left to the app; this covers the +// confirm queue that turns confirm(description) into a Promise. +function load() { + const outfile = join(mkdtempSync(join(tmpdir(), 'modly-confirm-test-')), 'ConfirmCard.cjs') + const result = buildSync({ + entryPoints: [resolve('src/shared/components/agent/ConfirmCard.tsx')], + bundle: true, + platform: 'node', + format: 'cjs', + jsx: 'automatic', + write: false, + }) + writeFileSync(outfile, result.outputFiles[0].text, 'utf8') + return createRequire(import.meta.url)(outfile) +} + +const { useConfirmQueue, ConfirmCard } = load() +const queue = () => useConfirmQueue.getState() + +test('confirm shows a pending request and resolves true on Confirm', async () => { + const answer = queue().confirm('Uninstall the extension “TripoSR”', { tool: 'uninstall_extension', id: 'e1', callId: 'c1', destructive: true }) + const [request] = queue().requests + assert.equal(request.status, 'pending') + assert.equal(request.callId, 'c1') + assert.equal(request.destructive, true) + + queue().answer(request.key, true) + + assert.equal(await answer, true) + assert.equal(queue().requests[0].status, 'confirmed') + queue().clearAnswered() + assert.deepEqual(queue().requests, []) +}) + +test('Cancel resolves false, and a second answer is ignored', async () => { + const answer = queue().confirm('Delete the workflow “Turntable”') + const { key } = queue().requests[0] + queue().answer(key, false) + queue().answer(key, true) + + assert.equal(await answer, false) + assert.equal(queue().requests[0].status, 'cancelled') + queue().clearAnswered() +}) + +test('cancelAll declines every pending request', async () => { + const a = queue().confirm('one') + const b = queue().confirm('two') + queue().cancelAll() + assert.deepEqual(await Promise.all([a, b]), [false, false]) + assert.deepEqual(queue().requests.map((r) => r.status), ['cancelled', 'cancelled']) + queue().clearAnswered() +}) + +test('ConfirmCard is a component', () => { + assert.equal(typeof ConfirmCard, 'function') +}) + +test('cancelEvent cancels the pending card of a tool call the server gave up on', async () => { + const answer = queue().confirm('Delete the workflow “Turntable”', { tool: 'delete_workflow', id: 'e9', callId: 'c9', destructive: true }) + const other = queue().confirm('Uninstall the extension “X”', { tool: 'uninstall_extension', id: 'e10', callId: 'c10', destructive: true }) + queue().cancelEvent('e9') + assert.equal(await answer, false) + assert.deepEqual(queue().requests.map((r) => r.status), ['cancelled', 'pending']) + queue().cancelAll() + assert.equal(await other, false) + queue().clearAnswered() +}) + +test('a queued confirm calls onQueued, so the panel can open itself', async () => { + let opened = 0 + queue().setOnQueued(() => { opened++ }) + const answer = queue().confirm('Install the extension from “https://github.com/a/b”', { tool: 'install_extension', id: 'e11', callId: 'c11', destructive: false }) + assert.equal(opened, 1) + queue().cancelAll() + assert.equal(await answer, false) + queue().setOnQueued(null) + queue().clearAnswered() +}) diff --git a/src/shared/components/agent/ConfirmCard.tsx b/src/shared/components/agent/ConfirmCard.tsx new file mode 100644 index 000000000..e406b28f2 --- /dev/null +++ b/src/shared/components/agent/ConfirmCard.tsx @@ -0,0 +1,186 @@ +import { create } from 'zustand' +import type { ConfirmInfo } from './clientTools' + +// ─── Confirm queue ──────────────────────────────────────────────────────────── + +export type ConfirmStatus = 'pending' | 'confirmed' | 'cancelled' + +export interface ConfirmRequest { + /** Unique per request. */ + key: string + description: string + destructive: boolean + status: ConfirmStatus + /** The client_tool event this request belongs to, when it came from one. */ + tool?: string + eventId?: string + callId?: string +} + +interface ConfirmQueueState { + requests: ConfirmRequest[] + /** Shows a card and resolves with the user's answer. Matches runClientTool's `confirm`. */ + confirm: (description: string, info?: ConfirmInfo) => Promise + answer: (key: string, confirmed: boolean) => void + /** Cancels every pending request (stream aborted, panel closed, new conversation). */ + cancelAll: () => void + /** Forgets answered requests. */ + clearAnswered: () => void + /** Cancels the pending card of one client_tool event (the server stopped waiting for it). */ + cancelEvent: (eventId: string) => void + /** Called whenever a card is queued; the panel uses it to open itself so the card is seen. */ + onQueued: (() => void) | null + setOnQueued: (callback: (() => void) | null) => void +} + +// Resolvers stay out of the state: they are not data to render. +const resolvers = new Map void>() +let nextKey = 0 + +export const useConfirmQueue = create((set, get) => ({ + requests: [], + + confirm(description, info) { + const key = `confirm-${++nextKey}` + const request: ConfirmRequest = { + key, + description, + destructive: info?.destructive ?? true, + status: 'pending', + tool: info?.tool, + eventId: info?.id, + callId: info?.callId, + } + return new Promise((resolve) => { + resolvers.set(key, resolve) + set((s) => ({ requests: [...s.requests, request] })) + get().onQueued?.() + }) + }, + + answer(key, confirmed) { + const resolve = resolvers.get(key) + if (!resolve) return + resolvers.delete(key) + set((s) => ({ + requests: s.requests.map((r) => (r.key === key ? { ...r, status: confirmed ? 'confirmed' : 'cancelled' } : r)), + })) + resolve(confirmed) + }, + + cancelAll() { + for (const r of get().requests) if (r.status === 'pending') get().answer(r.key, false) + }, + + clearAnswered() { + set((s) => ({ requests: s.requests.filter((r) => r.status === 'pending') })) + }, + + cancelEvent(eventId) { + for (const r of get().requests) if (r.status === 'pending' && r.eventId === eventId) get().answer(r.key, false) + }, + + onQueued: null, + setOnQueued: (callback) => set({ onQueued: callback }), +})) + +// ─── Card ───────────────────────────────────────────────────────────────────── + +export interface ConfirmCardProps { + description: string + destructive?: boolean + status: ConfirmStatus + onConfirm: () => void + onCancel: () => void +} + +/** The Confirm / Cancel card shown in the chat before a sensitive action. */ +export function ConfirmCard({ description, destructive = true, status, onConfirm, onCancel }: ConfirmCardProps): JSX.Element { + const pending = status === 'pending' + + function handleKeyDown(e: React.KeyboardEvent) { + if (!pending) return + if (e.key === 'Escape') { + e.preventDefault() + onCancel() + } else if (e.key === 'Enter' && !(e.target as HTMLElement).dataset.confirmCancel) { + // Enter on the Cancel button keeps its native meaning. + e.preventDefault() + onConfirm() + } + } + + return ( +
+
+ + + + +

{description}

+
+ +
+ {pending ? ( + <> + + + + ) : ( + + {status === 'confirmed' ? ( + + + + ) : ( + + + + )} + {status === 'confirmed' ? 'Confirmed' : 'Cancelled'} + + )} +
+
+ ) +} + +/** A ConfirmCard bound to one request of the queue. */ +export function QueuedConfirmCard({ request }: { request: ConfirmRequest }): JSX.Element { + const answer = useConfirmQueue((s) => s.answer) + return ( + answer(request.key, true)} + onCancel={() => answer(request.key, false)} + /> + ) +} diff --git a/src/shared/components/agent/agentContext.test.ts b/src/shared/components/agent/agentContext.test.ts new file mode 100644 index 000000000..93ff023e2 --- /dev/null +++ b/src/shared/components/agent/agentContext.test.ts @@ -0,0 +1,81 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { assembleAgentContext, buildAgentContext } from './agentContext' +import type { AgentContextInputs } from './agentContext' +import type { ModelExtension, ProcessExtension } from '../../types/electron.d' +import { useAppStore } from '../../stores/appStore' +import { useNavStore } from '../../stores/navStore' +import { useWorkflowsStore } from '../../stores/workflowsStore' +import { useExtensionsStore } from '../../stores/extensionsStore' + +const node = (id: string, input: string, output: string) => ({ id, name: id.toUpperCase(), input, output, paramsSchema: [] }) + +const modelExt = { + type: 'model', id: 'triposr', name: 'TripoSR', trusted: true, builtin: false, nodes: [node('main', 'image', 'mesh')], +} as unknown as ModelExtension +const processExt = { + type: 'process', id: 'mesh-ops', name: 'Mesh Ops', trusted: true, builtin: true, entry: 'x', nodes: [node('decimate', 'mesh', 'mesh')], +} as unknown as ProcessExtension +const STORAGE = { modelsDir: 'M', workspaceDir: 'W', workflowsDir: 'F', extensionsDir: 'E' } + +const inputs = (over: Partial = {}): AgentContextInputs => ({ + page: 'models', + currentMeshUrl: '/workspace/Default/mesh.glb', + meshTriangles: 1200, + workflows: [{ id: 'wf1', name: 'Turntable' }], + modelExtensions: [modelExt], + processExtensions: [processExt], + models: [{ id: 'triposr/main', name: 'TripoSR' }], + storage: STORAGE, + ui: { uiScale: 'large', useAtkinsonFont: true, showRamIndicator: false }, + ...over, +}) + +test('the context carries the page, mesh, workflows, extensions, models, storage and UI options', () => { + assert.deepEqual(assembleAgentContext(inputs()), { + page: 'models', + currentMeshPath: 'Default/mesh.glb', + meshTriangles: 1200, + workflows: [{ id: 'wf1', name: 'Turntable' }], + extensions: [ + { id: 'mesh-ops/decimate', name: 'DECIMATE', input: 'mesh', output: 'mesh' }, + { id: 'triposr/main', name: 'MAIN', input: 'image', output: 'mesh' }, + ], + models: [{ id: 'triposr/main', name: 'TripoSR' }], + storage: STORAGE, + ui: { uiScale: 'large', useAtkinsonFont: true, showRamIndicator: false }, + }) +}) + +test('empty or unreadable parts are left out, as buildContext did', () => { + const ctx = assembleAgentContext(inputs({ + currentMeshUrl: undefined, meshTriangles: null, workflows: [], modelExtensions: [], processExtensions: [], models: null, storage: null, + })) + assert.deepEqual(Object.keys(ctx), ['page', 'ui']) +}) + +test('storage keeps only the four folders, never the HF token', () => { + const ctx = assembleAgentContext(inputs({ storage: { ...STORAGE, hfToken: 'secret' } as typeof STORAGE })) + assert.deepEqual(ctx.storage, STORAGE) +}) + +test('buildAgentContext reads the stores and the async sources', async () => { + useNavStore.setState({ currentPage: 'workflows' }) + useAppStore.setState({ currentJob: null, meshStats: null, uiScale: 'small', useAtkinsonFont: false, showRamIndicator: true }) + useWorkflowsStore.setState({ workflows: [{ id: 'w', name: 'W', description: '', nodes: [], edges: [], createdAt: '', updatedAt: '' }] }) + useExtensionsStore.setState({ modelExtensions: [modelExt], processExtensions: [] }) + + const ctx = await buildAgentContext({ + fetchModels: async () => [{ id: 'triposr/main', name: 'TripoSR' }], + getStorage: async () => { throw new Error('IPC down') }, + }) + + assert.deepEqual(ctx, { + page: 'workflows', + workflows: [{ id: 'w', name: 'W' }], + extensions: [{ id: 'triposr/main', name: 'MAIN', input: 'image', output: 'mesh' }], + models: [{ id: 'triposr/main', name: 'TripoSR' }], + ui: { uiScale: 'small', useAtkinsonFont: false, showRamIndicator: true }, + }) +}) diff --git a/src/shared/components/agent/agentContext.ts b/src/shared/components/agent/agentContext.ts new file mode 100644 index 000000000..a13a06de8 --- /dev/null +++ b/src/shared/components/agent/agentContext.ts @@ -0,0 +1,84 @@ +import type { ModelExtension, ProcessExtension } from '../../types/electron.d' +import type { Workflow } from '../../types/electron.d' +import type { UiScale } from '../../stores/appStore' +import type { Page } from '../../stores/navStore' +import { useAppStore } from '../../stores/appStore' +import { useExtensionsStore } from '../../stores/extensionsStore' +import { useWorkflowsStore } from '../../stores/workflowsStore' +import { useNavStore } from '../../stores/navStore' +import { buildAllWorkflowExtensions } from '../../../areas/workflows/mockExtensions' + +// The `context` object sent with every agent request: what the app shows and +// holds right now, so the agent can act without asking. + +export interface AgentContextInputs { + page: Page + currentMeshUrl?: string | null + meshTriangles?: number | null + workflows: Pick[] + modelExtensions: ModelExtension[] + processExtensions: ProcessExtension[] + /** Downloaded generation models; null when they could not be read. */ + models: { id: string; name: string }[] | null + /** Null when the settings could not be read. */ + storage: { modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string } | null + ui: { uiScale: UiScale; useAtkinsonFont: boolean; showRamIndicator: boolean } +} + +/** Pure assembly of the context from already-read state. */ +export function assembleAgentContext(inputs: AgentContextInputs): Record { + const ctx: Record = { page: inputs.page } + if (inputs.currentMeshUrl) ctx.currentMeshPath = inputs.currentMeshUrl.replace('/workspace/', '') + if (inputs.meshTriangles) ctx.meshTriangles = inputs.meshTriangles + if (inputs.workflows.length > 0) ctx.workflows = inputs.workflows.map((w) => ({ id: w.id, name: w.name })) + const extensions = buildAllWorkflowExtensions(inputs.modelExtensions, inputs.processExtensions) + if (extensions.length > 0) ctx.extensions = extensions.map((e) => ({ + id: e.id, name: e.name, input: e.input, output: e.output, + })) + if (inputs.models) ctx.models = inputs.models.map((m) => ({ id: m.id, name: m.name })) + if (inputs.storage) { + const { modelsDir, workspaceDir, workflowsDir, extensionsDir } = inputs.storage + ctx.storage = { modelsDir, workspaceDir, workflowsDir, extensionsDir } + } + ctx.ui = { uiScale: inputs.ui.uiScale, useAtkinsonFont: inputs.ui.useAtkinsonFont, showRamIndicator: inputs.ui.showRamIndicator } + return ctx +} + +export interface AgentContextSources { + /** GET /model/all, filtered to the downloaded ones, as the Generate options list does. */ + fetchModels: () => Promise<{ id: string; name: string }[]> + getStorage: () => Promise<{ modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string }> +} + +function defaultSources(): AgentContextSources { + return { + async fetchModels() { + const res = await fetch(`${useAppStore.getState().apiUrl}/model/all`) + if (!res.ok) throw new Error(`API error ${res.status}`) + const list = await res.json() as { id: string; name?: string; downloaded: boolean }[] + return list.filter((m) => m.downloaded).map((m) => ({ id: m.id, name: m.name ?? m.id })) + }, + getStorage: () => window.electron.settings.get(), + } +} + +/** Reads the stores, the downloaded models and the storage folders, then assembles the context. */ +export async function buildAgentContext(sources: AgentContextSources = defaultSources()): Promise> { + const app = useAppStore.getState() + const ext = useExtensionsStore.getState() + const [models, storage] = await Promise.all([ + sources.fetchModels().catch(() => null), + sources.getStorage().catch(() => null), + ]) + return assembleAgentContext({ + page: useNavStore.getState().currentPage, + currentMeshUrl: app.currentJob?.outputUrl, + meshTriangles: app.meshStats?.triangles, + workflows: useWorkflowsStore.getState().workflows, + modelExtensions: ext.modelExtensions, + processExtensions: ext.processExtensions, + models, + storage, + ui: { uiScale: app.uiScale, useAtkinsonFont: app.useAtkinsonFont, showRamIndicator: app.showRamIndicator }, + }) +} diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts new file mode 100644 index 000000000..e6cb9ecde --- /dev/null +++ b/src/shared/components/agent/clientTools.test.ts @@ -0,0 +1,525 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { + clientTools, + describeClientTool, + isErrorResult, + isGenerationRunning, + latestUserImage, + runClientTool, + startGenerationFromImage, +} from './clientTools' +import type { ClientToolDeps, GenerationDeps, StorageKind, ToolResultBody } from './clientTools' +import type { AnyExtension, Workflow } from '../../types/electron.d' +import type { GenerationJob, GenerationOptions, UiScale } from '../../stores/appStore' + +// ─── Fakes ──────────────────────────────────────────────────────────────────── + +const node = (id: string, extra: Record = {}) => ({ + id, name: id.toUpperCase(), input: 'image', output: 'mesh', paramsSchema: [], ...extra, +}) + +function extensions(): AnyExtension[] { + return [ + { + type: 'model', id: 'triposr', name: 'TripoSR', trusted: true, builtin: false, + nodes: [node('main', { hfRepo: 'stabilityai/TripoSR' })], + }, + { + type: 'model', id: 'hunyuan', name: 'Hunyuan3D', trusted: true, builtin: false, + nodes: [ + node('shape', { hfRepo: 'tencent/h', weightVariants: { param: 'v', default: 'full', options: [{ id: 'full', label: 'Full' }, { id: 'mini', label: 'Mini' }] } }), + node('paint', { hfRepo: 'tencent/p' }), + ], + }, + { + type: 'model', id: 'shared', name: 'Shared', trusted: true, builtin: false, + weightGroups: [{ id: 'g', dependentNodeIds: ['a'] }], + nodes: [node('a', { weightGroups: ['g'] })], + }, + { + type: 'process', id: 'mesh-ops', name: 'Mesh Ops', trusted: true, builtin: true, entry: 'x', + nodes: [node('decimate')], + }, + ] as AnyExtension[] +} + +const workflow = (id: string, name: string): Workflow => ({ + id, name, description: '', nodes: [], edges: [], createdAt: '', updatedAt: '', +}) + +interface Fake { + deps: ClientToolDeps + calls: unknown[][] + posted: ToolResultBody[] + ui: { uiScale: UiScale; useAtkinsonFont: boolean; showRamIndicator: boolean } + dirs: { modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string } + history: { undo: number; redo: number } + dirItems: Record +} + +function fake(overrides: Partial = {}): Fake { + const calls: unknown[][] = [] + const posted: ToolResultBody[] = [] + const rec = (...call: unknown[]) => { calls.push(call) } + const ok = { success: true } + const state = { + ui: { uiScale: 'medium' as UiScale, useAtkinsonFont: false, showRamIndicator: true }, + dirs: { modelsDir: 'C:\\Modly\\models', workspaceDir: 'C:\\Modly\\workspace', workflowsDir: 'C:\\Modly\\workflows', extensionsDir: 'C:\\Modly\\extensions' }, + history: { undo: 1, redo: 1 }, + dirItems: {} as Record, + } + const list = extensions() + const wfs = [workflow('wf1', 'Turntable'), workflow('wf2', 'Cleanup')] + const deps: ClientToolDeps = { + extensions: { + list: () => list, + installFromGitHub: async (url) => { + rec('install', url) + return { success: true, extensionId: 'sf3d', extension: { type: 'model', id: 'sf3d', name: 'SF3D', trusted: false, builtin: false, nodes: [node('main', { hfRepo: 'x/y' })] } as AnyExtension } + }, + uninstall: async (id) => { rec('uninstall', id); return ok }, + reload: async () => { rec('reload') }, + loadErrors: () => ({ broken: 'bad manifest' }), + }, + weights: { + download: async (id, variant) => { rec('download', id, variant); return ok }, + delete: async (id) => { rec('delete', id); return ok }, + deleteVariant: async (id, variant) => { rec('deleteVariant', id, variant); return ok }, + deleteExtensionWeights: async (id) => { rec('deleteExtensionWeights', id); return ok }, + refresh: async () => { rec('refresh') }, + }, + workflows: { + list: () => wfs, + remove: async (id) => { rec('removeWorkflow', id); return ok }, + open: (id) => { rec('openWorkflow', id) }, + reload: async () => { rec('reloadWorkflows') }, + }, + ui: { + get: () => state.ui, + setUiScale: (v) => { state.ui.uiScale = v }, + setUseAtkinsonFont: (v) => { state.ui.useAtkinsonFont = v }, + setShowRamIndicator: (v) => { state.ui.showRamIndicator = v }, + }, + storage: { + get: async () => state.dirs, + entries: async (kind: StorageKind) => ({ modly: state.dirItems[kind] ?? [], other: [] }), + checkLocation: async (path: string) => (path.startsWith('C:\\Windows') ? { ok: false, error: 'This is a system folder.' } : { ok: true }), + moveDirectory: async (args) => { rec('move', args.src, args.dest); return ok }, + apply: async (kind: StorageKind, path: string) => { rec('applyStorage', kind, path) }, + }, + mesh: { + history: () => state.history, + undo: () => { rec('undo') }, + redo: () => { rec('redo') }, + }, + navigate: (page) => { rec('navigate', page) }, + postResult: async (body) => { posted.push(body) }, + downloadSettleMs: 20, + ...overrides, + } + return { deps, calls, posted, ...state } +} + +const run = (tool: string, args: Record, f = fake()) => clientTools[tool].run(args, f.deps) + +// ─── Extensions ─────────────────────────────────────────────────────────────── + +test('install_extension installs from GitHub and names the extension', async () => { + const f = fake() + const result = await run('install_extension', { github_url: 'https://github.com/x/sf3d' }, f) + + assert.deepEqual(f.calls, [['install', 'https://github.com/x/sf3d']]) + assert.match(result, /Installed the extension “SF3D” \(id sf3d\)/) + assert.match(result, /not downloaded yet/) +}) + +test('install_extension refuses a missing or non-GitHub URL without installing', async () => { + const f = fake() + assert.match(await run('install_extension', {}, f), /^Error: github_url is required/) + assert.match(await run('install_extension', { github_url: 'https://evil.example/x' }, f), /^Error: .*github\.com/) + assert.match(await run('install_extension', { github_url: 'not a url' }, f), /^Error: .*not a valid URL/) + assert.deepEqual(f.calls, []) +}) + +test('install_extension reports the installer error', async () => { + const f = fake() + f.deps.extensions.installFromGitHub = async () => ({ success: false, error: 'No manifest.json' }) + assert.equal(await run('install_extension', { github_url: 'https://github.com/x/y' }, f), 'Error: No manifest.json') +}) + +test('uninstall_extension uninstalls by id, describes by name and refuses built-ins and unknown ids', async () => { + const f = fake() + assert.equal(describeClientTool('uninstall_extension', { extension_id: 'triposr' }, f.deps), 'Uninstall the extension “TripoSR”') + assert.match(await run('uninstall_extension', { extension_id: 'triposr' }, f), /^Uninstalled the extension “TripoSR”/) + assert.deepEqual(f.calls, [['uninstall', 'triposr']]) + + assert.match(await run('uninstall_extension', { extension_id: 'mesh-ops' }, f), /^Error: “Mesh Ops” is built into Modly/) + assert.match(await run('uninstall_extension', { extension_id: 'nope' }, f), /^Error: No installed extension has the id “nope”. Installed extensions: triposr, hunyuan, shared, mesh-ops\./) + assert.match(await run('uninstall_extension', {}, f), /^Error: extension_id is required/) + assert.equal(f.calls.length, 1) +}) + +test('uninstall_extension refuses a model id, which would remove the whole extension', async () => { + const f = fake() + const problem = clientTools.uninstall_extension.check({ extension_id: 'hunyuan/shape' }, f.deps) + assert.match(problem ?? '', /“hunyuan\/shape” names a model; pass the extension id \(“hunyuan”\)/) + let asked = false + const outcome = await runClientTool(event('uninstall_extension', { extension_id: 'hunyuan/shape' }, true), f.deps, async () => { asked = true; return true }) + assert.equal(asked, false) + assert.equal(outcome.error, true) + assert.deepEqual(f.calls, []) +}) + +test('reload_extensions reloads and reports the count and load errors', async () => { + const f = fake() + const result = await run('reload_extensions', {}, f) + assert.deepEqual(f.calls, [['reload']]) + assert.equal(result, 'Reloaded the extensions: 4 installed. 1 failed to load: broken (bad manifest).') +}) + +// ─── Weights ────────────────────────────────────────────────────────────────── + +test('download_model_weights downloads every model of the extension in turn', async () => { + const f = fake() + const result = await run('download_model_weights', { extension_id: 'hunyuan' }, f) + assert.deepEqual(f.calls, [['download', 'hunyuan/shape', undefined], ['download', 'hunyuan/paint', undefined], ['refresh']]) + assert.equal(result, 'Downloaded the model weights of “Hunyuan3D”.') +}) + +test('download_model_weights with a variant only targets the models offering it', async () => { + const f = fake() + const result = await run('download_model_weights', { extension_id: 'hunyuan', variant: 'mini' }, f) + assert.deepEqual(f.calls, [['download', 'hunyuan/shape', 'mini'], ['refresh']]) + assert.equal(result, 'Downloaded the model weights of “Hunyuan3D” (variant “mini”).') +}) + +test('download_model_weights validates the extension, model and variant', async () => { + const f = fake() + assert.match(await run('download_model_weights', { extension_id: 'hunyuan', variant: 'huge' }, f), /^Error: “Hunyuan3D” has no weight variant “huge”. Available variants: full, mini\./) + assert.match(await run('download_model_weights', { extension_id: 'triposr', variant: 'x' }, f), /^Error: “TripoSR” does not offer weight variants/) + assert.match(await run('download_model_weights', { extension_id: 'hunyuan/nope' }, f), /^Error: “Hunyuan3D” has no model “nope”/) + assert.match(await run('download_model_weights', { extension_id: 'mesh-ops' }, f), /^Error: “Mesh Ops” is a process extension/) + assert.match(await run('download_model_weights', { extension_id: 'ghost' }, f), /^Error: No installed extension/) + assert.match(await run('download_model_weights', { extension_id: 'triposr', variant: 3 }, f), /^Error: variant must be a string/) + assert.deepEqual(f.calls, []) +}) + +test('download_model_weights reports a long download as started, and an early failure as an error', async () => { + const slow = fake() + slow.deps.weights.download = () => new Promise(() => {}) + assert.match(await run('download_model_weights', { extension_id: 'triposr' }, slow), /^Started downloading the model weights of “TripoSR” \(1 model\)\. The download continues in the background/) + + const broken = fake() + broken.deps.weights.download = async () => ({ success: false, error: 'HF 401' }) + assert.equal(await run('download_model_weights', { extension_id: 'triposr/main' }, broken), 'Error: Downloading “TripoSR / MAIN” failed: triposr/main: HF 401.') +}) + +test('delete_model_weights deletes per model, per variant, or the shared extension weights', async () => { + const f = fake() + assert.equal(await run('delete_model_weights', { extension_id: 'hunyuan' }, f), 'Deleted the downloaded model weights of “Hunyuan3D”.') + assert.equal(await run('delete_model_weights', { extension_id: 'hunyuan', variant: 'full' }, f), 'Deleted the downloaded model weights of “Hunyuan3D” (variant “full”).') + await run('delete_model_weights', { extension_id: 'shared' }, f) + assert.deepEqual(f.calls, [ + ['delete', 'hunyuan/shape'], ['delete', 'hunyuan/paint'], ['refresh'], + ['deleteVariant', 'hunyuan/shape', 'full'], ['refresh'], + ['deleteExtensionWeights', 'shared'], ['refresh'], + ]) + assert.equal(describeClientTool('delete_model_weights', { extension_id: 'hunyuan/shape', variant: 'full' }, f.deps), 'Delete the downloaded model weights of “Hunyuan3D / SHAPE” (variant “full”)') +}) + +test('delete_model_weights reports a failed delete', async () => { + const f = fake() + f.deps.weights.delete = async () => ({ success: false, error: 'file locked' }) + assert.equal(await run('delete_model_weights', { extension_id: 'triposr' }, f), 'Error: Deleting the weights of “TripoSR” failed: file locked.') +}) + +// ─── Workflows ──────────────────────────────────────────────────────────────── + +test('delete_workflow deletes by id and resolves its name', async () => { + const f = fake() + assert.equal(describeClientTool('delete_workflow', { workflow_id: 'wf2' }, f.deps), 'Delete the workflow “Cleanup”') + assert.equal(await run('delete_workflow', { workflow_id: 'wf2' }, f), 'Deleted the workflow “Cleanup”.') + assert.deepEqual(f.calls, [['removeWorkflow', 'wf2']]) + assert.match(await run('delete_workflow', { workflow_id: 'wf9' }, f), /^Error: No workflow has the id “wf9”. Workflows: Turntable \(wf1\), Cleanup \(wf2\)\./) +}) + +test('open_workflow opens the tab without navigating', async () => { + const f = fake() + const result = await run('open_workflow', { workflow_id: 'wf1' }, f) + assert.deepEqual(f.calls, [['openWorkflow', 'wf1']]) + assert.match(result, /“Turntable”.*did not change/) + assert.match(await run('open_workflow', {}, f), /^Error: workflow_id is required/) +}) + +// ─── Settings ───────────────────────────────────────────────────────────────── + +test('set_setting sets each UI option and validates key and value', async () => { + const f = fake() + assert.equal(await run('set_setting', { key: 'uiScale', value: 'large' }, f), 'Set the interface scale to "large".') + assert.equal(await run('set_setting', { key: 'useAtkinsonFont', value: true }, f), 'Set the Atkinson Hyperlegible font to true.') + assert.equal(await run('set_setting', { key: 'showRamIndicator', value: 'false' }, f), 'Set the RAM indicator to false.') + assert.deepEqual(f.ui, { uiScale: 'large', useAtkinsonFont: true, showRamIndicator: false }) + + assert.match(await run('set_setting', { key: 'uiScale', value: 'huge' }, f), /^Error: uiScale must be one of: small, medium, large, very-large/) + assert.match(await run('set_setting', { key: 'showRamIndicator', value: 1 }, f), /^Error: showRamIndicator must be true or false/) + assert.match(await run('set_setting', { key: 'hfToken', value: 'x' }, f), /^Error: key must be one of/) +}) + +test('set_storage_folder moves existing files, then saves the folder', async () => { + const f = fake() + f.dirItems.models = ['triposr', 'hunyuan'] + const result = await run('set_storage_folder', { kind: 'models', path: 'D:\\AI\\models' }, f) + assert.deepEqual(f.calls, [['move', 'C:\\Modly\\models', 'D:\\AI\\models'], ['applyStorage', 'models', 'D:\\AI\\models']]) + assert.equal(result, 'Moved 2 items from “C:\\Modly\\models” and set the models folder to “D:\\AI\\models”.') +}) + +test('set_storage_folder moves a workflows folder that holds only files', async () => { + const f = fake() + f.dirItems.workflows = ['turntable.json'] + const result = await run('set_storage_folder', { kind: 'workflows', path: 'D:/flows' }, f) + assert.deepEqual(f.calls, [['move', 'C:\\Modly\\workflows', 'D:/flows'], ['applyStorage', 'workflows', 'D:/flows']]) + assert.match(result, /^Moved 1 item from/) +}) + +test('set_storage_folder refuses a location Settings would refuse', async () => { + const f = fake() + assert.equal(await run('set_storage_folder', { kind: 'models', path: 'C:\\Windows\\m' }, f), 'Error: This is a system folder. Nothing changed.') + assert.deepEqual(f.calls, []) +}) + +test('set_storage_folder says which items stayed behind', async () => { + const f = fake() + f.dirItems.models = ['a', 'b'] + f.deps.storage.moveDirectory = async (args) => { f.calls.push(['move', args.src, args.dest]); return { success: true, kept: ['b'] } } + assert.match(await run('set_storage_folder', { kind: 'models', path: 'D:/m' }, f), /^Moved 1 item from .*; 1 item could not be moved and stayed in “C:\\Modly\\models”\.$/) +}) + +test('set_storage_folder with an empty folder only saves it', async () => { + const f = fake() + assert.equal(await run('set_storage_folder', { kind: 'workflows', path: 'D:/flows' }, f), 'Set the workflows folder to “D:/flows”.') + assert.deepEqual(f.calls, [['applyStorage', 'workflows', 'D:/flows']]) +}) + +test('set_storage_folder refuses invalid locations and keeps everything on a failed move', async () => { + const f = fake() + assert.match(await run('set_storage_folder', { kind: 'cache', path: 'D:/x' }, f), /^Error: kind must be one of/) + assert.match(await run('set_storage_folder', { kind: 'models', path: 'models' }, f), /^Error: “models” is not an absolute path/) + assert.match(await run('set_storage_folder', { kind: 'models', path: 'c:/modly/models/sub' }, f), /^Error: .*overlaps the current models folder/) + assert.match(await run('set_storage_folder', { kind: 'models', path: 'C:\\Modly' }, f), /^Error: .*overlaps the current models folder/) + assert.match(await run('set_storage_folder', { kind: 'models', path: 'C:\\Modly\\workspace\\m' }, f), /^Error: .*overlaps another Modly folder/) + assert.match(await run('set_storage_folder', { kind: 'models', path: 'C:/Modly/models/' }, f), /already/) + assert.deepEqual(f.calls, []) + + f.dirItems.workspace = ['a.glb'] + f.deps.storage.moveDirectory = async () => ({ success: false, error: 'EACCES' }) + assert.equal(await run('set_storage_folder', { kind: 'workspace', path: 'E:/ws' }, f), 'Error: Could not move the workspace files to “E:/ws”: EACCES. Nothing changed.') + assert.deepEqual(f.calls, []) +}) + +// ─── Mesh, navigation ───────────────────────────────────────────────────────── + +test('undo_mesh and redo_mesh step through the history, and say when there is nothing to do', async () => { + const f = fake() + assert.match(await run('undo_mesh', {}, f), /^Undid/) + assert.match(await run('redo_mesh', {}, f), /^Redid/) + assert.deepEqual(f.calls, [['undo'], ['redo']]) + + f.history.undo = 0 + f.history.redo = 0 + assert.equal(await run('undo_mesh', {}, f), 'There is no mesh change to undo.') + assert.equal(await run('redo_mesh', {}, f), 'There is no mesh change to redo.') + assert.equal(f.calls.length, 2) +}) + +test('navigate changes page and validates it', async () => { + const f = fake() + assert.equal(await run('navigate', { page: 'models' }, f), 'Opened the models page.') + assert.deepEqual(f.calls, [['navigate', 'models']]) + assert.match(await run('navigate', { page: 'home' }, f), /^Error: page must be one of: generate, workflows, models, settings/) +}) + +test('a throwing dependency comes back as an error text', async () => { + const f = fake() + f.deps.extensions.reload = async () => { throw new Error('IPC gone') } + assert.equal(await run('reload_extensions', {}, f), 'Error: IPC gone') +}) + +// ─── runClientTool ──────────────────────────────────────────────────────────── + +const event = (tool: string, args: Record, confirm = false) => ({ + type: 'client_tool' as const, id: 'evt-1', call_id: 'call-7', tool, arguments: args, confirm, +}) + +test('runClientTool runs the tool and posts the result with its call_id', async () => { + const f = fake() + const outcome = await runClientTool(event('navigate', { page: 'settings' }), f.deps, async () => { throw new Error('not asked') }) + assert.deepEqual(f.posted, [{ call_id: 'call-7', result: 'Opened the settings page.' }]) + assert.deepEqual(outcome, { description: 'Go to the settings page', result: 'Opened the settings page.', error: false, declined: false, posted: true }) +}) + +test('runClientTool asks first when confirm is set, and runs on Confirm', async () => { + const f = fake() + const asked: unknown[] = [] + await runClientTool(event('uninstall_extension', { extension_id: 'triposr' }, true), f.deps, async (description, info) => { + asked.push(description, info) + return true + }) + assert.deepEqual(asked, ['Uninstall the extension “TripoSR”', { tool: 'uninstall_extension', id: 'evt-1', callId: 'call-7', destructive: true }]) + assert.deepEqual(f.calls, [['uninstall', 'triposr']]) + assert.equal(f.posted[0].call_id, 'call-7') +}) + +test('runClientTool tells the server about the Confirm click before running, and not on a decline', async () => { + const order: string[] = [] + const f = fake({ + postConfirmed: async (callId) => { order.push(`confirmed ${callId}`) }, + postResult: async (body) => { order.push(`result ${body.call_id}`) }, + }) + f.deps.extensions.uninstall = async () => { order.push('uninstall'); return { success: true } } + await runClientTool(event('uninstall_extension', { extension_id: 'triposr' }, true), f.deps, async () => true) + assert.deepEqual(order, ['confirmed call-7', 'uninstall', 'result call-7']) + + order.length = 0 + await runClientTool(event('uninstall_extension', { extension_id: 'triposr' }, true), f.deps, async () => false) + assert.deepEqual(order, ['result call-7']) + + order.length = 0 + await runClientTool(event('navigate', { page: 'models' }), f.deps, async () => true) + assert.deepEqual(order, ['result call-7']) +}) + +test('runClientTool still runs the tool when the confirmed notice fails', async () => { + const f = fake({ postConfirmed: async () => { throw new Error('offline') } }) + const outcome = await runClientTool(event('delete_workflow', { workflow_id: 'wf1' }, true), f.deps, async () => true) + assert.deepEqual(f.calls, [['removeWorkflow', 'wf1']]) + assert.equal(outcome.error, false) +}) + +test('runClientTool reports a declined confirmation without running the tool', async () => { + const f = fake() + const outcome = await runClientTool(event('delete_workflow', { workflow_id: 'wf1' }, true), f.deps, async () => false) + assert.deepEqual(f.calls, []) + assert.deepEqual(f.posted, [{ call_id: 'call-7', result: 'The user declined: Delete the workflow “Turntable”.' }]) + assert.equal(outcome.declined, true) +}) + +test('runClientTool asks for code installs and destructive tools even when the event says no confirm', async () => { + for (const [tool, args] of [ + ['install_extension', { github_url: 'https://github.com/a/b' }], + ['delete_workflow', { workflow_id: 'wf1' }], + ] as const) { + const f = fake() + let asked = false + const outcome = await runClientTool(event(tool, { ...args }, false), f.deps, async () => { asked = true; return false }) + assert.equal(asked, true, tool) + assert.equal(outcome.declined, true, tool) + assert.deepEqual(f.calls, [], tool) + } +}) + +test('runClientTool does not ask about a call that cannot run, and flags errors', async () => { + const f = fake() + let asked = false + const outcome = await runClientTool(event('delete_workflow', { workflow_id: 'nope' }, true), f.deps, async () => { asked = true; return true }) + assert.equal(asked, false) + assert.equal(outcome.error, true) + assert.equal(f.posted[0].error, true) + assert.match(f.posted[0].result, /^Error: No workflow has the id “nope”/) +}) + +test('runClientTool answers an unknown tool with an error', async () => { + const f = fake() + const outcome = await runClientTool(event('format_disk', {}), f.deps, async () => true) + assert.match(outcome.result, /^Error: Unknown client tool “format_disk”. Client tools: install_extension,/) + assert.deepEqual(f.posted, [{ call_id: 'call-7', result: outcome.result, error: true }]) +}) + +test('runClientTool survives a failed post', async () => { + const f = fake({ postResult: async () => { throw new Error('offline') } }) + const outcome = await runClientTool(event('reload_extensions', {}), f.deps, async () => true) + assert.equal(outcome.posted, false) + assert.equal(isErrorResult(outcome.result), false) +}) + +// ─── generate_from_image ────────────────────────────────────────────────────── + +const OPTIONS: GenerationOptions = { modelId: 'triposr/main', remesh: 'quad', enableTexture: false, textureResolution: 512, modelParams: { steps: 30 } } + +function generation(statuses: { status: string; progress?: number; outputUrl?: string; error?: string }[]) { + const calls: unknown[][] = [] + let job: GenerationJob | null = null + const deps: GenerationDeps = { + isGenerating: () => isGenerationRunning(job), + api: { + generateFromImage: async (path, options, data) => { calls.push(['submit', path, options, data]); return { jobId: 'j1' } }, + pollJobStatus: async () => ({ progress: 0, ...statuses.shift()! }) as never, + cancelJob: async () => { calls.push(['cancel']) }, + }, + store: { + setCurrentJob: (j) => { job = j }, + updateCurrentJob: (patch) => { if (job) job = { ...job, ...patch } }, + pushMeshUrl: (url) => { calls.push(['push', url]) }, + clearMeshHistory: () => { calls.push(['clear']) }, + }, + options: () => OPTIONS, + notifyComplete: () => {}, + notifyError: () => {}, + wait: async () => {}, + } + return { deps, calls, job: () => job } +} + +test('startGenerationFromImage generates from the attached image into the current job', async () => { + const g = generation([{ status: 'running', progress: 50 }, { status: 'done', outputUrl: '/workspace/out.glb' }]) + const result = await startGenerationFromImage({ type: 'generate_from_image', model_id: 'sf3d/main' }, 'data:image/jpeg;base64,QUJD', g.deps) + + assert.equal(result, 'Generated a mesh with sf3d/main: /workspace/out.glb.') + const submit = g.calls.find((c) => c[0] === 'submit')! + assert.equal(submit[3], 'QUJD') + assert.deepEqual(submit[2], { ...OPTIONS, modelId: 'sf3d/main', modelParams: {} }) + assert.deepEqual(g.calls.at(-1), ['push', '/workspace/out.glb']) + assert.equal(g.job()?.status, 'done') + assert.equal(g.job()?.outputUrl, '/workspace/out.glb') +}) + +test('startGenerationFromImage keeps the selected model and its params when none is given', async () => { + const g = generation([{ status: 'done', outputUrl: '/workspace/o.glb' }]) + await startGenerationFromImage({ type: 'generate_from_image' }, 'QUJD', g.deps) + assert.deepEqual(g.calls.find((c) => c[0] === 'submit')![2], OPTIONS) +}) + +test('startGenerationFromImage reports a missing image, a wrong payload and a failed job', async () => { + const g = generation([{ status: 'error', error: 'CUDA OOM' }]) + assert.match(await startGenerationFromImage({ type: 'generate_from_image' }, undefined, g.deps), /^Error: No image/) + assert.match(await startGenerationFromImage({ type: 'mesh_update' }, 'QUJD', g.deps), /^Error: Not a generate_from_image payload/) + assert.equal(await startGenerationFromImage({ type: 'generate_from_image' }, 'QUJD', g.deps), 'Error: The generation failed: CUDA OOM') + assert.equal(g.job()?.status, 'error') +}) + +test('startGenerationFromImage refuses while a generation is running, without touching the current job', async () => { + for (const status of ['uploading', 'generating'] as const) { + const g = generation([{ status: 'done', outputUrl: '/workspace/o.glb' }]) + const running = { id: 'busy', imageFile: 'x.png', status, progress: 40, createdAt: 1 } as GenerationJob + g.deps.store.setCurrentJob(running) + const result = await startGenerationFromImage({ type: 'generate_from_image' }, 'QUJD', g.deps) + assert.equal(result, 'Error: A generation is already running. Wait for it to finish.') + assert.deepEqual(g.calls, [], status) + assert.equal(g.job(), running) + } + for (const status of ['done', 'error', 'idle'] as const) { + assert.equal(isGenerationRunning({ status }), false, status) + } + assert.equal(isGenerationRunning(null), false) +}) + +test('latestUserImage takes the most recent user message with an image', () => { + assert.equal(latestUserImage([ + { role: 'user', imageDataUrls: ['data:old'] }, + { role: 'assistant', imageDataUrls: ['data:assistant'] }, + { role: 'user', imageDataUrls: ['data:new', 'data:second'] }, + { role: 'user' }, + ]), 'data:new') + assert.equal(latestUserImage([{ role: 'user' }]), undefined) +}) diff --git a/src/shared/components/agent/clientTools.ts b/src/shared/components/agent/clientTools.ts new file mode 100644 index 000000000..526873e64 --- /dev/null +++ b/src/shared/components/agent/clientTools.ts @@ -0,0 +1,821 @@ +import type { AnyExtension, ExtensionNode, Workflow } from '../../types/electron.d' +import type { AgentActionPayload, ClientToolEvent } from '../../utils/agentStream' +import type { GenerationOptions, UiScale } from '../../stores/appStore' +import type { Page } from '../../stores/navStore' +import type { GenerationApi, GenerationStoreActions } from '../../utils/generationRunner' +import { createGenerationApi, runGeneration } from '../../utils/generationRunner' +import { useAppStore } from '../../stores/appStore' +import { useExtensionsStore } from '../../stores/extensionsStore' +import { useWorkflowsStore } from '../../stores/workflowsStore' +import { useNavStore } from '../../stores/navStore' +import { redoDepth, undoDepth } from '../../stores/meshHistory' +import { showCompletionNotification, showRunErrorNotification } from '../../utils/notification' + +// Tools the agent asks the frontend to run, because they need IPC or the +// renderer stores. Every handler returns the text sent back to the LLM and +// never throws: failures come back as text starting with "Error: ". + +// ─── Dependencies ───────────────────────────────────────────────────────────── + +type OpResult = { success: boolean; error?: string } + +export type StorageKind = 'models' | 'workspace' | 'workflows' + +export interface StorageDirs { + modelsDir: string + workspaceDir: string + workflowsDir: string + extensionsDir: string +} + +export interface UiSettings { + uiScale: UiScale + useAtkinsonFont: boolean + showRamIndicator: boolean +} + +export interface ToolResultBody { + call_id: string + result: string + error?: boolean +} + +export interface ClientToolDeps { + extensions: { + list: () => AnyExtension[] + installFromGitHub: (url: string) => Promise + uninstall: (extensionId: string) => Promise + reload: () => Promise + loadErrors: () => Record + } + weights: { + download: (modelId: string, variantId?: string) => Promise + delete: (modelId: string) => Promise + deleteVariant: (modelId: string, variantId: string) => Promise + deleteExtensionWeights: (extensionId: string) => Promise + /** Re-reads what is installed, so the Models page stays accurate. */ + refresh: () => Promise + } + workflows: { + list: () => Workflow[] + remove: (id: string) => Promise + /** Opens the workflow as the active editor tab, without changing page. */ + open: (id: string) => void + reload: () => Promise + } + ui: { + get: () => UiSettings + setUiScale: (value: UiScale) => void + setUseAtkinsonFont: (value: boolean) => void + setShowRamIndicator: (value: boolean) => void + } + storage: { + get: () => Promise + /** What Modly created in the current folder (moved) and what it did not (left alone), as Settings sees it. */ + entries: (kind: StorageKind) => Promise<{ modly: string[]; other: string[] }> + /** Settings' guard: refuses a drive root, a system folder or an ancestor of home/app data. */ + checkLocation: (path: string) => Promise<{ ok: boolean; error?: string }> + moveDirectory: (args: { src: string; dest: string }) => Promise + /** Saves the new folder, the way Settings > Storage does once files are moved. */ + apply: (kind: StorageKind, path: string) => Promise + } + mesh: { + /** Steps available each way on the mesh undo stack. */ + history: () => { undo: number; redo: number } + undo: () => void + redo: () => void + } + navigate: (page: Page) => void + postResult: (body: ToolResultBody) => Promise + /** + * Tells the server the user clicked Confirm, so it stops waiting on the click + * and gives the tool its own timeout. Optional: without it the server keeps + * waiting under the confirmation cap. + */ + postConfirmed?: (callId: string) => Promise + /** How long a download is watched for an early failure before it is reported as started. */ + downloadSettleMs?: number +} + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +type Args = Record + +const ERROR_PREFIX = 'Error: ' + +const fail = (message: string): string => ERROR_PREFIX + message + +/** True for a result text that reports a failure. */ +export function isErrorResult(result: string): boolean { + return result.startsWith(ERROR_PREFIX) +} + +const quote = (s: string): string => `“${s}”` + +function errorText(err: unknown): string { + return err instanceof Error ? err.message : String(err) +} + +function str(args: Args, key: string): string | null { + const value = args[key] + return typeof value === 'string' && value.trim() ? value.trim() : null +} + +function nodeHasManagedWeights(node: ExtensionNode): boolean { + return Boolean(node.hfRepo || node.hasModelSources || node.weightGroups?.length) +} + +function listOf(items: string[]): string { + return items.length ? items.join(', ') : 'none' +} + +function findExtension(id: string, deps: ClientToolDeps | undefined): AnyExtension | undefined { + return deps?.extensions.list().find((e) => e.id === id.split('/')[0]) +} + +function extensionLabel(id: string, deps: ClientToolDeps | undefined): string { + const ext = findExtension(id, deps) + if (!ext) return quote(id) + const nodeId = id.includes('/') ? id.slice(id.indexOf('/') + 1) : null + const node = nodeId ? ext.nodes.find((n) => n.id === nodeId) : undefined + return quote(node ? `${ext.name} / ${node.name}` : ext.name) +} + +function workflowLabel(id: string, deps: ClientToolDeps | undefined): string { + return quote(deps?.workflows.list().find((w) => w.id === id)?.name ?? id) +} + +function lookupExtension(args: Args, deps: ClientToolDeps): { ext: AnyExtension; id: string } | string { + const id = str(args, 'extension_id') + if (!id) return 'extension_id is required.' + const ext = findExtension(id, deps) + if (!ext) { + const installed = deps.extensions.list().map((e) => e.id) + return `No installed extension has the id ${quote(id)}. Installed extensions: ${listOf(installed)}.` + } + return { ext, id } +} + +function lookupWorkflow(args: Args, deps: ClientToolDeps): Workflow | string { + const id = str(args, 'workflow_id') + if (!id) return 'workflow_id is required.' + const workflows = deps.workflows.list() + const wf = workflows.find((w) => w.id === id) + if (!wf) { + return `No workflow has the id ${quote(id)}. Workflows: ${listOf(workflows.map((w) => `${w.name} (${w.id})`))}.` + } + return wf +} + +interface WeightTarget { + ext: AnyExtension + /** Set when the id named one model ("ext/node") rather than the whole extension. */ + nodeId: string | null + variant: string | null + nodes: { fullId: string; node: ExtensionNode }[] +} + +function lookupWeights(args: Args, deps: ClientToolDeps): WeightTarget | string { + const found = lookupExtension(args, deps) + if (typeof found === 'string') return found + const { ext, id } = found + if (args.variant !== undefined && args.variant !== null && typeof args.variant !== 'string') { + return 'variant must be a string.' + } + const variant = str(args, 'variant') + if (ext.type !== 'model') return `${quote(ext.name)} is a process extension; it has no model weights.` + + const nodeId = id.includes('/') ? id.slice(id.indexOf('/') + 1) : null + let nodes = ext.nodes.filter(nodeHasManagedWeights) + if (nodeId) { + nodes = nodes.filter((n) => n.id === nodeId) + if (!nodes.length) { + const models = ext.nodes.filter(nodeHasManagedWeights).map((n) => `${ext.id}/${n.id}`) + return `${quote(ext.name)} has no model ${quote(nodeId)} with weights. Models with weights: ${listOf(models)}.` + } + } + if (!nodes.length) return `${quote(ext.name)} has no downloadable model weights.` + if (variant) { + const offered = [...new Set(nodes.flatMap((n) => n.weightVariants?.options.map((o) => o.id) ?? []))] + nodes = nodes.filter((n) => n.weightVariants?.options.some((o) => o.id === variant)) + if (!nodes.length) { + return offered.length + ? `${quote(ext.name)} has no weight variant ${quote(variant)}. Available variants: ${listOf(offered)}.` + : `${quote(ext.name)} does not offer weight variants.` + } + } + return { ext, nodeId, variant, nodes: nodes.map((node) => ({ fullId: `${ext.id}/${node.id}`, node })) } +} + +function weightsLabel(args: Args, deps: ClientToolDeps | undefined): string { + const id = typeof args.extension_id === 'string' ? args.extension_id : '?' + const variant = typeof args.variant === 'string' && args.variant ? ` (variant ${quote(args.variant)})` : '' + return `${extensionLabel(id, deps)}${variant}` +} + +const sleep = (ms: number): Promise => new Promise((r) => setTimeout(() => r(null), ms)) + +// ─── Paths ──────────────────────────────────────────────────────────────────── + +const STORAGE_KINDS: StorageKind[] = ['models', 'workspace', 'workflows'] + +const STORAGE_KEY: Record = { + models: 'modelsDir', + workspace: 'workspaceDir', + workflows: 'workflowsDir', +} + +function isAbsolutePath(p: string): boolean { + return /^[A-Za-z]:[\\/]/.test(p) || p.startsWith('\\\\') || p.startsWith('/') +} + +/** Comparable form: forward slashes, no trailing slash, Windows paths lower-cased. */ +function normPath(p: string): string { + const s = p.replace(/\\/g, '/').replace(/\/+$/, '') + return /^[A-Za-z]:/.test(s) || s.startsWith('//') ? s.toLowerCase() : s +} + +function isInside(child: string, parent: string): boolean { + return child === parent || child.startsWith(`${parent}/`) +} + +// ─── Registry ───────────────────────────────────────────────────────────────── + +export interface ClientTool { + /** Destructive: the confirm card shows a red Confirm button. */ + destructive: boolean + /** + * Always asks before running, whatever the server's `confirm` says: true for + * destructive tools and for installs (they run code downloaded from GitHub). + */ + requiresConfirm: boolean + /** The human sentence for the confirm card and the action list. */ + describe: (args: Args, deps?: ClientToolDeps) => string + /** Why the arguments cannot be run, or null when they can. */ + check: (args: Args, deps: ClientToolDeps) => string | null + /** Runs the tool; returns the result text for the LLM and never throws. */ + run: (args: Args, deps: ClientToolDeps) => Promise +} + +interface ToolSpec { + destructive?: boolean + /** Ask even when not destructive (see ClientTool.requiresConfirm). */ + requiresConfirm?: boolean + describe: (args: Args, deps?: ClientToolDeps) => string + check: (args: Args, deps: ClientToolDeps) => string | null + execute: (args: Args, deps: ClientToolDeps) => Promise +} + +function defineTool(spec: ToolSpec): ClientTool { + return { + destructive: spec.destructive ?? false, + requiresConfirm: (spec.destructive ?? false) || (spec.requiresConfirm ?? false), + describe: spec.describe, + check: (args, deps) => { + try { + return spec.check(args, deps) + } catch (err) { + return errorText(err) + } + }, + run: async (args, deps) => { + try { + const problem = spec.check(args, deps) + if (problem) return fail(problem) + return await spec.execute(args, deps) + } catch (err) { + return fail(errorText(err)) + } + }, + } +} + +const checked = (value: T | string): string | null => (typeof value === 'string' ? value : null) + +export const clientTools: Record = { + install_extension: defineTool({ + requiresConfirm: true, + describe: (args) => `Install the extension from ${quote(String(args.github_url ?? '?'))}`, + check: (args) => { + const url = str(args, 'github_url') + if (!url) return 'github_url is required.' + let host: string + try { + host = new URL(url).hostname.toLowerCase() + } catch { + return `${quote(url)} is not a valid URL.` + } + return host === 'github.com' || host === 'www.github.com' ? null : 'github_url must be a github.com repository URL.' + }, + async execute(args, deps) { + const result = await deps.extensions.installFromGitHub(str(args, 'github_url')!) + if (!result.success) return fail(result.error ?? 'Installation failed.') + const ext = result.extension + const id = ext?.id ?? result.extensionId ?? '?' + const needsWeights = ext?.type === 'model' && ext.nodes.some(nodeHasManagedWeights) + return `Installed the extension ${quote(ext?.name ?? id)} (id ${id}).` + + (needsWeights ? ' Its model weights are not downloaded yet; download_model_weights fetches them.' : '') + }, + }), + + uninstall_extension: defineTool({ + destructive: true, + describe: (args, deps) => `Uninstall the extension ${extensionLabel(String(args.extension_id ?? '?'), deps)}`, + check: (args, deps) => { + // "ext/node" names one model, but uninstalling removes the whole extension: + // the card must not read as if only that model goes. + const id = str(args, 'extension_id') + if (id?.includes('/')) { + return `${quote(id)} names a model; pass the extension id (${quote(id.split('/')[0])}) to uninstall the whole extension.` + } + const found = lookupExtension(args, deps) + if (typeof found === 'string') return found + return found.ext.builtin ? `${quote(found.ext.name)} is built into Modly and cannot be uninstalled.` : null + }, + async execute(args, deps) { + const { ext } = lookupExtension(args, deps) as { ext: AnyExtension } + const result = await deps.extensions.uninstall(ext.id) + if (!result.success) return fail(result.error ?? `Could not uninstall ${quote(ext.name)}.`) + return `Uninstalled the extension ${quote(ext.name)}. Its downloaded model weights, if any, were left on disk.` + }, + }), + + reload_extensions: defineTool({ + describe: () => 'Reload the installed extensions', + check: () => null, + async execute(_args, deps) { + await deps.extensions.reload() + const count = deps.extensions.list().length + const errors = Object.entries(deps.extensions.loadErrors()) + const tail = errors.length + ? ` ${errors.length} failed to load: ${errors.map(([id, err]) => `${id} (${err})`).join('; ')}.` + : '' + return `Reloaded the extensions: ${count} installed.${tail}` + }, + }), + + download_model_weights: defineTool({ + describe: (args, deps) => `Download the model weights of ${weightsLabel(args, deps)}`, + check: (args, deps) => checked(lookupWeights(args, deps)), + async execute(args, deps) { + const target = lookupWeights(args, deps) as WeightTarget + const variant = target.variant ?? undefined + // Downloads run one after the other, as "Install all" does on the Models page. + const queue = (async () => { + const outcomes: { fullId: string; result: OpResult & { paused?: boolean; cancelled?: boolean } }[] = [] + for (const { fullId } of target.nodes) { + let result: OpResult & { paused?: boolean; cancelled?: boolean } + try { + result = await deps.weights.download(fullId, variant) + } catch (err) { + result = { success: false, error: errorText(err) } + } + outcomes.push({ fullId, result }) + } + await deps.weights.refresh().catch(() => {}) + return outcomes + })() + const outcomes = await Promise.race([queue, sleep(deps.downloadSettleMs ?? 1500)]) + const label = weightsLabel(args, deps) + if (!outcomes) { + return `Started downloading the model weights of ${label} (${target.nodes.length} model${target.nodes.length > 1 ? 's' : ''}). ` + + 'The download continues in the background; its progress shows on the Models page.' + } + const failed = outcomes.filter((o) => !o.result.success && !o.result.paused && !o.result.cancelled) + if (failed.length) { + return fail(`Downloading ${label} failed: ${failed.map((o) => `${o.fullId}: ${o.result.error ?? 'unknown error'}`).join('; ')}.`) + } + if (outcomes.some((o) => o.result.paused || o.result.cancelled)) { + return `The download of ${label} was paused or cancelled before it finished.` + } + return `Downloaded the model weights of ${label}.` + }, + }), + + delete_model_weights: defineTool({ + destructive: true, + describe: (args, deps) => `Delete the downloaded model weights of ${weightsLabel(args, deps)}`, + check: (args, deps) => checked(lookupWeights(args, deps)), + async execute(args, deps) { + const target = lookupWeights(args, deps) as WeightTarget + const label = weightsLabel(args, deps) + const results: OpResult[] = [] + if (target.variant) { + for (const { fullId } of target.nodes) results.push(await deps.weights.deleteVariant(fullId, target.variant)) + } else if (!target.nodeId && target.ext.type === 'model' && target.ext.weightGroups?.length) { + // Shared weight groups: the whole-extension delete, as the uninstall dialog does. + results.push(await deps.weights.deleteExtensionWeights(target.ext.id)) + } else { + for (const { fullId } of target.nodes) results.push(await deps.weights.delete(fullId)) + } + await deps.weights.refresh().catch(() => {}) + const errors = results.filter((r) => !r.success).map((r) => r.error ?? 'unknown error') + if (errors.length) return fail(`Deleting the weights of ${label} failed: ${errors.join('; ')}.`) + return `Deleted the downloaded model weights of ${label}.` + }, + }), + + delete_workflow: defineTool({ + destructive: true, + describe: (args, deps) => `Delete the workflow ${workflowLabel(String(args.workflow_id ?? '?'), deps)}`, + check: (args, deps) => checked(lookupWorkflow(args, deps)), + async execute(args, deps) { + const wf = lookupWorkflow(args, deps) as Workflow + const result = await deps.workflows.remove(wf.id) + if (!result.success) return fail(result.error ?? `Could not delete the workflow ${quote(wf.name)}.`) + return `Deleted the workflow ${quote(wf.name)}.` + }, + }), + + open_workflow: defineTool({ + describe: (args, deps) => `Open the workflow ${workflowLabel(String(args.workflow_id ?? '?'), deps)}`, + check: (args, deps) => checked(lookupWorkflow(args, deps)), + async execute(args, deps) { + const wf = lookupWorkflow(args, deps) as Workflow + deps.workflows.open(wf.id) + return `Opened the workflow ${quote(wf.name)} as the active tab of the Workflows editor. The current page did not change.` + }, + }), + + set_setting: defineTool({ + describe: (args) => `Set ${SETTING_LABELS[String(args.key)] ?? String(args.key ?? '?')} to ${JSON.stringify(args.value)}`, + check: (args) => { + const key = args.key + if (typeof key !== 'string' || !(key in SETTING_LABELS)) { + return `key must be one of: ${Object.keys(SETTING_LABELS).join(', ')}.` + } + if (key === 'uiScale') { + return UI_SCALES.includes(args.value as UiScale) ? null : `uiScale must be one of: ${UI_SCALES.join(', ')}.` + } + return toBool(args.value) === null ? `${key} must be true or false.` : null + }, + async execute(args, deps) { + const key = args.key as keyof UiSettings + if (key === 'uiScale') deps.ui.setUiScale(args.value as UiScale) + else if (key === 'useAtkinsonFont') deps.ui.setUseAtkinsonFont(toBool(args.value)!) + else deps.ui.setShowRamIndicator(toBool(args.value)!) + return `Set ${SETTING_LABELS[key]} to ${JSON.stringify(deps.ui.get()[key])}.` + }, + }), + + set_storage_folder: defineTool({ + destructive: true, + describe: (args) => `Change the ${String(args.kind ?? '?')} folder to ${quote(String(args.path ?? '?'))}, moving its current files there`, + check: (args) => { + if (!STORAGE_KINDS.includes(args.kind as StorageKind)) return `kind must be one of: ${STORAGE_KINDS.join(', ')}.` + const path = str(args, 'path') + if (!path) return 'path is required.' + return isAbsolutePath(path) ? null : `${quote(path)} is not an absolute path.` + }, + async execute(args, deps) { + const kind = args.kind as StorageKind + const path = str(args, 'path')! + const dirs = await deps.storage.get() + const current = dirs[STORAGE_KEY[kind]] + const next = normPath(path) + if (current && normPath(current) === next) return `The ${kind} folder is already ${quote(current)}; nothing changed.` + if (current && (isInside(next, normPath(current)) || isInside(normPath(current), next))) { + return fail(`${quote(path)} overlaps the current ${kind} folder ${quote(current)}; choose a folder outside it. Nothing changed.`) + } + for (const [key, dir] of Object.entries(dirs) as [keyof StorageDirs, string][]) { + if (key === STORAGE_KEY[kind] || !dir) continue + if (isInside(next, normPath(dir)) || isInside(normPath(dir), next)) { + return fail(`${quote(path)} overlaps another Modly folder (${quote(dir)}); choose a separate folder. Nothing changed.`) + } + } + + const location = await deps.storage.checkLocation(path).catch((err) => ({ ok: false, error: errorText(err) })) + if (!location.ok) return fail(`${location.error ?? `${quote(path)} cannot hold Modly data.`} Nothing changed.`) + + // Files count too: a workflows folder holds only JSON files. + const items = current ? (await deps.storage.entries(kind).catch(() => ({ modly: [] as string[] }))).modly : [] + let kept: string[] = [] + if (items.length > 0) { + const moved = await deps.storage.moveDirectory({ src: current, dest: path }) + if (!moved.success) { + return fail(`Could not move the ${kind} files to ${quote(path)}: ${moved.error ?? 'unknown error'}. Nothing changed.`) + } + kept = moved.kept ?? [] + } + try { + await deps.storage.apply(kind, path) + } catch (err) { + return fail(items.length + ? `Moved the ${kind} files to ${quote(path)}, but saving the new folder failed: ${errorText(err)}. Set it in Settings > Storage.` + : `Saving the new ${kind} folder failed: ${errorText(err)}. Nothing changed.`) + } + if (!items.length) return `Set the ${kind} folder to ${quote(path)}.` + const count = (n: number) => `${n} item${n > 1 ? 's' : ''}` + const movedCount = items.length - kept.length + return kept.length + ? `Moved ${count(movedCount)} from ${quote(current)} and set the ${kind} folder to ${quote(path)}; ${count(kept.length)} could not be moved and stayed in ${quote(current)}.` + : `Moved ${count(movedCount)} from ${quote(current)} and set the ${kind} folder to ${quote(path)}.` + }, + }), + + undo_mesh: defineTool({ + describe: () => 'Undo the last mesh change', + check: () => null, + async execute(_args, deps) { + if (deps.mesh.history().undo <= 0) return 'There is no mesh change to undo.' + deps.mesh.undo() + return 'Undid the last mesh change; the viewer shows the previous mesh.' + }, + }), + + redo_mesh: defineTool({ + describe: () => 'Redo the mesh change', + check: () => null, + async execute(_args, deps) { + if (deps.mesh.history().redo <= 0) return 'There is no mesh change to redo.' + deps.mesh.redo() + return 'Redid the mesh change; the viewer shows the next mesh.' + }, + }), + + navigate: defineTool({ + describe: (args) => `Go to the ${String(args.page ?? '?')} page`, + check: (args) => (PAGES.includes(args.page as Page) ? null : `page must be one of: ${PAGES.join(', ')}.`), + async execute(args, deps) { + deps.navigate(args.page as Page) + return `Opened the ${String(args.page)} page.` + }, + }), +} + +const SETTING_LABELS: Record = { + uiScale: 'the interface scale', + useAtkinsonFont: 'the Atkinson Hyperlegible font', + showRamIndicator: 'the RAM indicator', +} + +const UI_SCALES: UiScale[] = ['small', 'medium', 'large', 'very-large'] + +const PAGES: Page[] = ['generate', 'workflows', 'models', 'settings'] + +function toBool(value: unknown): boolean | null { + if (typeof value === 'boolean') return value + if (value === 'true') return true + if (value === 'false') return false + return null +} + +/** The sentence for a tool call; falls back to the tool name for an unknown tool. */ +export function describeClientTool(tool: string, args: Args, deps?: ClientToolDeps): string { + const entry = clientTools[tool] + if (!entry) return tool.replace(/_/g, ' ') + try { + return entry.describe(args ?? {}, deps) + } catch { + return tool.replace(/_/g, ' ') + } +} + +// ─── Running a client_tool event ────────────────────────────────────────────── + +export interface ConfirmInfo { + tool: string + /** The `client_tool` event id, which the later `tool_done` carries too. */ + id: string + callId: string + destructive: boolean +} + +export type ConfirmFn = (description: string, info?: ConfirmInfo) => Promise + +export interface ClientToolOutcome { + description: string + result: string + error: boolean + declined: boolean + /** False when the result could not be sent back to the server. */ + posted: boolean +} + +/** + * Runs one `client_tool` event: asks for confirmation when the server says so, + * runs the tool, then posts the result to `/agent/tool-result`. Never throws. + */ +export async function runClientTool( + event: Pick, + deps: ClientToolDeps, + confirm: ConfirmFn, +): Promise { + const tool = clientTools[event.tool] + const args: Args = event.arguments && typeof event.arguments === 'object' && !Array.isArray(event.arguments) + ? event.arguments + : {} + const description = describeClientTool(event.tool, args, deps) + let result: string + let declined = false + + if (!tool) { + result = fail(`Unknown client tool ${quote(event.tool)}. Client tools: ${Object.keys(clientTools).join(', ')}.`) + } else { + const problem = tool.check(args, deps) + if (problem) { + // Nothing to confirm: the call cannot run as given. + result = fail(problem) + } else { + let ok = true + const asked = event.confirm || tool.requiresConfirm + if (asked) { + try { + ok = await confirm(description, { tool: event.tool, id: event.id, callId: event.call_id, destructive: tool.destructive }) + } catch { + ok = false + } + } + if (ok) { + if (asked && deps.postConfirmed) { + // From here the server waits on the tool, not on the click. + await deps.postConfirmed(event.call_id).catch(() => {}) + } + result = await tool.run(args, deps) + } else { + declined = true + result = `The user declined: ${description}.` + } + } + } + + const error = isErrorResult(result) + let posted = true + try { + await deps.postResult({ call_id: event.call_id, result, ...(error ? { error: true } : {}) }) + } catch { + posted = false + } + return { description, result, error, declined, posted } +} + +// ─── generate_from_image ────────────────────────────────────────────────────── + +export interface GenerationDeps { + /** True while a generation is uploading or running (the Generate page then disables its upload too). */ + isGenerating: () => boolean + api: GenerationApi + store: GenerationStoreActions + /** The Generate page's current options (model, remesh, texture, params). */ + options: () => GenerationOptions + notifyComplete: (body: string) => void | Promise + notifyError: (error: string | undefined, modelId: string) => void | Promise + isCancelled?: () => boolean + wait?: () => Promise +} + +export const GENERATION_BUSY = 'A generation is already running. Wait for it to finish.' + +/** Same condition as the Generate page's upload and options lock. */ +export function isGenerationRunning(job: { status: string } | null | undefined): boolean { + return job?.status === 'uploading' || job?.status === 'generating' +} + +/** The first image of the most recent user message that has one. */ +export function latestUserImage(messages: { role: string; imageDataUrls?: string[] }[]): string | undefined { + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i] + if (m.role === 'user' && m.imageDataUrls?.length) return m.imageDataUrls[0] + } + return undefined +} + +/** + * Runs a `generate_from_image` payload through the Generate page's own path, + * so the result appears in its viewer. `image` is a data URL or bare base64. + * Resolves once the generation ends, with a sentence describing the outcome. + */ +export async function startGenerationFromImage( + payload: AgentActionPayload | null | undefined, + image: string | undefined, + deps: GenerationDeps, +): Promise { + if (payload?.type !== 'generate_from_image') return fail('Not a generate_from_image payload.') + const base64 = image?.startsWith('data:') ? image.slice(image.indexOf(',') + 1) : image + if (!base64) return fail('No image is attached to the conversation; attach one and ask again.') + // Checked synchronously right before runGeneration claims currentJob, so two + // calls cannot both pass it. + if (deps.isGenerating()) return fail(GENERATION_BUSY) + + const current = deps.options() + const modelId = payload.model_id || current.modelId + if (!modelId) return fail('No generation model is selected or downloaded.') + const options: GenerationOptions = { + ...current, + modelId, + // Saved params belong to the selected model; another model starts from its defaults. + modelParams: { ...(modelId === current.modelId ? current.modelParams : {}), ...(payload.params ?? {}) }, + } + + try { + const outcome = await runGeneration('agent-image.png', options, base64, { + api: deps.api, + store: deps.store, + notifyComplete: deps.notifyComplete, + notifyError: deps.notifyError, + isCancelled: deps.isCancelled ?? (() => false), + wait: deps.wait, + }) + if (outcome.status === 'done') return `Generated a mesh with ${modelId}${outcome.outputUrl ? `: ${outcome.outputUrl}` : ''}.` + if (outcome.status === 'cancelled') return 'The generation was cancelled.' + return fail(`The generation failed: ${outcome.error}`) + } catch (err) { + return fail(`The generation failed: ${errorText(err)}`) + } +} + +// ─── App wiring ─────────────────────────────────────────────────────────────── + +/** The real dependencies: stores, IPC and the local API. */ +export function createClientToolDeps(): ClientToolDeps { + const ext = () => useExtensionsStore.getState() + const app = () => useAppStore.getState() + const wfs = () => useWorkflowsStore.getState() + return { + extensions: { + list: () => [...ext().modelExtensions, ...ext().processExtensions], + installFromGitHub: (url) => ext().installFromGitHub(url), + uninstall: (id) => ext().uninstall(id), + reload: () => ext().reload(), + loadErrors: () => ext().loadErrors, + }, + weights: { + download: (modelId, variantId) => window.electron.model.download(modelId, variantId), + delete: (modelId) => window.electron.model.delete(modelId), + deleteVariant: (modelId, variantId) => window.electron.model.deleteWeightVariant(modelId, variantId), + deleteExtensionWeights: (extensionId) => window.electron.model.deleteExtensionWeights(extensionId), + refresh: () => ext().refreshInstalledWeightVariants(), + }, + workflows: { + list: () => wfs().workflows, + remove: (id) => wfs().remove(id), + open: (id) => wfs().openWorkflow(id), + reload: () => wfs().load(), + }, + ui: { + get: () => ({ uiScale: app().uiScale, useAtkinsonFont: app().useAtkinsonFont, showRamIndicator: app().showRamIndicator }), + setUiScale: (v) => app().setUiScale(v), + setUseAtkinsonFont: (v) => app().setUseAtkinsonFont(v), + setShowRamIndicator: (v) => app().setShowRamIndicator(v), + }, + storage: { + get: () => window.electron.settings.get(), + entries: (kind) => window.electron.fs.storageEntries(kind), + checkLocation: (path) => window.electron.fs.checkStorageLocation(path), + moveDirectory: (args) => window.electron.fs.moveDirectory(args), + async apply(kind, path) { + // Same calls as Settings > Storage. + if (kind === 'models') { + await window.electron.settings.set({ modelsDir: path }) + await window.electron.api.updatePaths({ modelsDir: path }) + } else if (kind === 'workspace') { + await window.electron.settings.set({ workspaceDir: path }) + await window.electron.api.updatePaths({ workspaceDir: path }) + } else { + await window.electron.settings.set({ workflowsDir: path }) + await wfs().load() + } + }, + }, + mesh: { + history: () => ({ undo: undoDepth(app().meshHistory), redo: redoDepth(app().meshHistory) }), + undo: () => app().undoMesh(), + redo: () => app().redoMesh(), + }, + navigate: (page) => useNavStore.getState().navigate(page), + async postResult(body) { + const res = await fetch(`${app().apiUrl}/agent/tool-result`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + if (!res.ok) throw new Error(`API error ${res.status}`) + }, + async postConfirmed(callId) { + const res = await fetch(`${app().apiUrl}/agent/tool-confirmed`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ call_id: callId }), + }) + if (!res.ok) throw new Error(`API error ${res.status}`) + }, + } +} + +/** The real dependencies of startGenerationFromImage. */ +export function createGenerationDeps(): GenerationDeps { + const app = () => useAppStore.getState() + return { + isGenerating: () => isGenerationRunning(app().currentJob), + api: createGenerationApi(app().apiUrl), + store: { + setCurrentJob: (job) => app().setCurrentJob(job), + updateCurrentJob: (patch) => app().updateCurrentJob(patch), + pushMeshUrl: (url) => app().pushMeshUrl(url), + clearMeshHistory: () => app().clearMeshHistory(), + }, + options: () => app().generationOptions, + notifyComplete: (body) => showCompletionNotification(body), + notifyError: (error, modelId) => showRunErrorNotification({ title: 'Generation failed', name: modelId, error }), + } +} diff --git a/src/shared/components/layout/MainLayout.tsx b/src/shared/components/layout/MainLayout.tsx index da5bcd9c5..71a0cccac 100644 --- a/src/shared/components/layout/MainLayout.tsx +++ b/src/shared/components/layout/MainLayout.tsx @@ -1,6 +1,7 @@ import TopBar from './TopBar' import Sidebar from './Sidebar' import Router from '@shared/router/Router' +import AgentPanel from '@shared/components/agent/AgentPanel' export default function MainLayout(): JSX.Element { @@ -11,9 +12,11 @@ export default function MainLayout(): JSX.Element {
-
+
+ +
) diff --git a/src/shared/components/layout/TopBar.tsx b/src/shared/components/layout/TopBar.tsx index c73fd02a0..056ddf8af 100644 --- a/src/shared/components/layout/TopBar.tsx +++ b/src/shared/components/layout/TopBar.tsx @@ -1,10 +1,13 @@ import { useEffect, useState } from 'react' import { useAppStore } from '@shared/stores/appStore' +import { useAgentPanelStore } from '@shared/stores/agentPanelStore' import MemoryIndicator from './MemoryIndicator' export default function TopBar(): JSX.Element { const { patchUpdateReady, platform, showRamIndicator } = useAppStore() const isMac = platform === 'darwin' + const agentOpen = useAgentPanelStore((s) => s.open) + const toggleAgent = useAgentPanelStore((s) => s.toggle) const [isMaximized, setIsMaximized] = useState(false) @@ -45,6 +48,27 @@ export default function TopBar(): JSX.Element { {/* Memory indicator */} {showRamIndicator && } + {/* Agent panel toggle */} + + {/* Patch update badge */} {patchUpdateReady && (
diff --git a/src/shared/hooks/useApi.ts b/src/shared/hooks/useApi.ts index 4b24b7870..ebd62ca72 100644 --- a/src/shared/hooks/useApi.ts +++ b/src/shared/hooks/useApi.ts @@ -1,5 +1,6 @@ import axios from 'axios' -import { useAppStore, GenerationOptions } from '@shared/stores/appStore' +import { useAppStore } from '@shared/stores/appStore' +import { createGenerationApi } from '../utils/generationRunner' import type { MeshOpDescriptor, MeshOpResponse } from '@areas/generate/meshEditPanel' import type { VertexPatch } from '@areas/generate/sculpt/sculptLogic' @@ -8,43 +9,7 @@ export function useApi() { const client = axios.create({ baseURL: apiUrl }) - async function generateFromImage( - imagePath: string, - options: GenerationOptions, - imageData?: string, - signal?: AbortSignal, - ): Promise<{ jobId: string }> { - // Use provided base64 (drag & drop) or read from disk via IPC - const base64 = imageData ?? await window.electron.fs.readFileBase64(imagePath) - const byteArray = Uint8Array.from(atob(base64), (c) => c.charCodeAt(0)) - const blob = new Blob([byteArray], { type: 'image/png' }) - const filename = imagePath.split(/[\\/]/).pop() ?? 'image.png' - - const formData = new FormData() - formData.append('image', blob, filename) - formData.append('model_id', options.modelId) - formData.append('remesh', options.remesh) - formData.append('enable_texture', String(options.enableTexture)) - formData.append('texture_resolution', String(options.textureResolution)) - formData.append('params', JSON.stringify(options.modelParams)) - const { data } = await client.post<{ job_id: string }>('/generate/from-image', formData, { - headers: { 'Content-Type': 'multipart/form-data' }, - signal, - }) - - return { jobId: data.job_id } - } - - async function pollJobStatus(jobId: string): Promise<{ - status: 'pending' | 'running' | 'done' | 'error' | 'cancelled' - progress: number - step?: string - outputUrl?: string - error?: string - }> { - const { data } = await client.get(`/generate/status/${jobId}`) - return { ...data, outputUrl: data.output_url } - } + const { generateFromImage, pollJobStatus, cancelJob } = createGenerationApi(apiUrl) async function getModelStatus(): Promise<{ downloaded: boolean @@ -98,10 +63,6 @@ export function useApi() { return { url: data.url, faceCount: data.face_count } } - async function cancelJob(jobId: string): Promise { - await client.post(`/generate/cancel/${jobId}`).catch(() => {}) - } - async function smoothMesh( path: string, iterations: number, diff --git a/src/shared/hooks/useGeneration.ts b/src/shared/hooks/useGeneration.ts index a2b0f9cf3..550927d3e 100644 --- a/src/shared/hooks/useGeneration.ts +++ b/src/shared/hooks/useGeneration.ts @@ -2,6 +2,7 @@ import { useCallback, useRef } from 'react' import { useAppStore } from '@shared/stores/appStore' import { useApi } from './useApi' import { showCompletionNotification, showRunErrorNotification } from '@shared/utils/notification' +import { runGeneration } from '@shared/utils/generationRunner' export function useGeneration() { const { currentJob, setCurrentJob, updateCurrentJob, generationOptions, selectedImageData, pushMeshUrl, clearMeshHistory } = useAppStore() @@ -13,90 +14,19 @@ export function useGeneration() { async (imagePath: string) => { cancelledRef.current = false abortControllerRef.current = new AbortController() - clearMeshHistory() - const job = { - id: crypto.randomUUID(), - imageFile: imagePath, - status: 'uploading' as const, - progress: 0, - createdAt: Date.now(), - modelId: generationOptions.modelId, - generationOptions, - } - setCurrentJob(job) - - try { - const { jobId } = await generateFromImage(imagePath, generationOptions, selectedImageData ?? undefined, abortControllerRef.current.signal) - - if (cancelledRef.current) { - await cancelJob(jobId) - setCurrentJob(null) - return - } - - updateCurrentJob({ status: 'generating', progress: 0 }) - - await pollUntilDone(jobId, job.modelId) - } catch (err) { - if (cancelledRef.current) { - setCurrentJob(null) - return - } - let errorMessage: string - if (err && typeof err === 'object' && 'response' in err) { - const axiosErr = err as { response?: { data?: { detail?: string } }; message: string } - errorMessage = axiosErr.response?.data?.detail ?? axiosErr.message - } else { - errorMessage = err instanceof Error ? err.message : String(err) - } - updateCurrentJob({ - status: 'error', - error: errorMessage - }) - void showRunErrorNotification({ title: 'Generation failed', name: job.modelId, error: errorMessage }) - } + await runGeneration(imagePath, generationOptions, selectedImageData ?? undefined, { + api: { generateFromImage, pollJobStatus, cancelJob }, + store: { setCurrentJob, updateCurrentJob, pushMeshUrl, clearMeshHistory }, + notifyComplete: (body) => showCompletionNotification(body), + notifyError: (error, modelId) => showRunErrorNotification({ title: 'Generation failed', name: modelId, error, cancelled: cancelledRef.current }), + isCancelled: () => cancelledRef.current, + signal: abortControllerRef.current.signal, + }) }, // eslint-disable-next-line react-hooks/exhaustive-deps -- useApi re-creates its fns each render, so this re-memoizes anyway (values stay fresh) [generateFromImage, pollJobStatus, cancelJob, setCurrentJob, updateCurrentJob] ) - const pollUntilDone = async (jobId: string, modelId: string) => { - while (true) { - await new Promise((r) => setTimeout(r, 1000)) - - if (cancelledRef.current) { - await cancelJob(jobId) - setCurrentJob(null) - break - } - - const result = await pollJobStatus(jobId) - - if (result.status === 'cancelled') { - setCurrentJob(null) - break - } - - if (result.status === 'done') { - updateCurrentJob({ status: 'done', progress: 100, outputUrl: result.outputUrl, originalOutputUrl: result.outputUrl }) - if (result.outputUrl) pushMeshUrl(result.outputUrl) - void showCompletionNotification('Generation complete') - break - } - - if (result.status === 'error') { - updateCurrentJob({ status: 'error', error: result.error }) - void showRunErrorNotification({ title: 'Generation failed', name: modelId, error: result.error, cancelled: cancelledRef.current }) - break - } - - updateCurrentJob({ - progress: result.progress, - step: result.step, - }) - } - } - const cancelGeneration = useCallback(() => { cancelledRef.current = true abortControllerRef.current?.abort() diff --git a/src/shared/stores/agentConversationStore.ts b/src/shared/stores/agentConversationStore.ts new file mode 100644 index 000000000..621f5e1a4 --- /dev/null +++ b/src/shared/stores/agentConversationStore.ts @@ -0,0 +1,220 @@ +import { create } from 'zustand' +import type { AgentAction, ToolStop } from '@shared/utils/agentStream' +// Relative so the store loads under `node --test` (no path aliases there). +import { + createSerialQueue, + deriveTitle, + settleMessages, + upsertSummary, + type Conversation, + type ConversationSummary, +} from '../utils/agentConversations' + +// ─── Message types (AgentChat) ──────────────────────────────────────────────── + +export interface ActionDone extends AgentAction { + id?: string + running?: boolean +} + +export interface AgentMessage { + id: string + role: 'user' | 'assistant' + content: string + thinking?: string + /** Attached images, inline as data URLs (persisted as-is). */ + imageDataUrls?: string[] + actions?: ActionDone[] + stopped?: ToolStop | null + streaming?: boolean +} + +// ─── Store ──────────────────────────────────────────────────────────────────── + +const SAVE_DEBOUNCE_MS = 500 +/** Unreadable conversations init skips before giving up on reopening one. */ +const MAX_INIT_ATTEMPTS = 5 + +interface AgentConversationState { + /** Id of the conversation on screen; not on disk until it has a message. */ + currentId: string + createdAt: number + messages: AgentMessage[] + /** Saved conversations, newest first. */ + list: ConversationSummary[] + /** A turn is streaming or a workflow it started is running: switching would misroute its output. */ + busy: boolean + + setMessages: (update: AgentMessage[] | ((prev: AgentMessage[]) => AgentMessage[])) => void + setBusy: (busy: boolean) => void + newConversation: () => Promise + open: (id: string) => Promise + remove: (id: string) => Promise + refreshList: () => Promise + /** Loads the list and opens the most recent readable conversation. Runs once, retried if it failed. */ + init: () => Promise + /** Writes a pending debounced save now. */ + flush: () => Promise +} + +type OpenResult = 'opened' | 'unreadable' | 'skipped' + +function newId(): string { + return crypto.randomUUID() +} + +let saveTimer: ReturnType | null = null +// Unsaved changes: opening or switching away must not bump an untouched conversation. +let dirty = false +// Bumped on every edit: a save clears `dirty` only if nothing changed while it was writing. +let edits = 0 +// Deleted conversations: a save finishing late must not bring one back into the list. +const removedIds = new Set() +let initPromise: Promise | null = null +// Every disk operation runs through this queue, one at a time: a save in +// flight is always awaited, and a delete never races the write of its file. +const enqueue = createSerialQueue() + +export const useAgentConversationStore = create()((set, get) => { + /** + * Writes the current conversation if it has unsaved changes. Queued tasks only. + * False when the write failed: the changes stay unsaved and switching away must wait. + */ + async function saveNow(): Promise { + if (saveTimer) { clearTimeout(saveTimer); saveTimer = null } + const { currentId, createdAt, messages } = get() + // An empty conversation never reaches the disk. + if (!dirty || messages.length === 0) return true + const editsAtStart = edits + const conversation: Conversation = { + id: currentId, + title: deriveTitle(messages), + createdAt, + updatedAt: Date.now(), + messages, + } + try { + await window.electron.agent.conversations.save(conversation) + } catch (err) { + // Still dirty: the next change or switch tries again. + console.error('[agent] Failed to save conversation:', err) + return false + } + if (get().currentId === conversation.id && edits === editsAtStart) dirty = false + if (!removedIds.has(conversation.id)) { + set((s) => ({ list: upsertSummary(s.list, { id: conversation.id, title: conversation.title, updatedAt: conversation.updatedAt }) })) + } + return true + } + + function save(): Promise { + return enqueue(saveNow) + } + + function scheduleSave(): void { + if (saveTimer) clearTimeout(saveTimer) + saveTimer = setTimeout(() => { void save() }, SAVE_DEBOUNCE_MS) + } + + /** Queued tasks only. */ + async function openNow(id: string): Promise { + if (get().busy || id === get().currentId) return 'skipped' + // Leaving a conversation whose save failed would lose it. + if (!(await saveNow())) return 'skipped' + let conversation: Conversation | null + try { + conversation = await window.electron.agent.conversations.read(id) + } catch (err) { + console.error('[agent] Failed to read conversation:', err) + return 'skipped' + } + if (!conversation) { + // Gone or corrupted on disk: drop it from the list. + set((s) => ({ list: s.list.filter((c) => c.id !== id) })) + return 'unreadable' + } + set({ + currentId: conversation.id, + createdAt: conversation.createdAt, + messages: settleMessages(conversation.messages) as AgentMessage[], + }) + dirty = false + return 'opened' + } + + return { + currentId: newId(), + createdAt: Date.now(), + messages: [], + list: [], + busy: false, + + setMessages: (update) => { + set((s) => ({ messages: typeof update === 'function' ? update(s.messages) : update })) + dirty = true + edits++ + scheduleSave() + }, + + setBusy: (busy) => set({ busy }), + + newConversation: () => enqueue(async () => { + if (get().busy) return + // Leaving a conversation whose save failed would lose it. + if (!(await saveNow())) return + set({ currentId: newId(), createdAt: Date.now(), messages: [] }) + dirty = false + }), + + open: async (id) => { await enqueue(() => openNow(id)) }, + + remove: (id) => enqueue(async () => { + const isCurrent = id === get().currentId + if (isCurrent && get().busy) return + try { + await window.electron.agent.conversations.remove(id) + } catch (err) { + // Still on disk, and its pending changes still get saved. + console.error('[agent] Failed to delete conversation:', err) + return + } + removedIds.add(id) + if (isCurrent) { + if (saveTimer) { clearTimeout(saveTimer); saveTimer = null } + dirty = false + } + set((s) => ({ + list: s.list.filter((c) => c.id !== id), + ...(isCurrent ? { currentId: newId(), createdAt: Date.now(), messages: [] } : {}), + })) + }), + + refreshList: () => enqueue(async () => { + try { + set({ list: await window.electron.agent.conversations.list() }) + } catch (err) { + console.error('[agent] Failed to list conversations:', err) + } + }), + + init: () => { + initPromise ??= (async () => { + // Throws on failure, so a failed init is forgotten and the next call retries. + const list = await enqueue(() => window.electron.agent.conversations.list()) + set({ list }) + // Open the most recent readable one; unreadable files drop out of the list. + for (const { id } of list.slice(0, MAX_INIT_ATTEMPTS)) { + // The user may already have started typing a new conversation. + if (get().messages.length > 0 || get().busy) return + if ((await enqueue(() => openNow(id))) !== 'unreadable') return + } + })().catch((err: unknown) => { + console.error('[agent] Failed to load conversations:', err) + initPromise = null + }) + return initPromise + }, + + flush: async () => { await save() }, + } +}) diff --git a/src/shared/stores/agentPanelStore.ts b/src/shared/stores/agentPanelStore.ts new file mode 100644 index 000000000..f82997087 --- /dev/null +++ b/src/shared/stores/agentPanelStore.ts @@ -0,0 +1,43 @@ +import { create } from 'zustand' +import { persist } from 'zustand/middleware' + +export const AGENT_PANEL_MIN_WIDTH = 320 +export const AGENT_PANEL_MAX_WIDTH = 640 +export const AGENT_PANEL_DEFAULT_WIDTH = 380 + +export function clampPanelWidth(width: number): number { + if (!Number.isFinite(width)) return AGENT_PANEL_DEFAULT_WIDTH + return Math.round(Math.min(AGENT_PANEL_MAX_WIDTH, Math.max(AGENT_PANEL_MIN_WIDTH, width))) +} + +interface AgentPanelState { + open: boolean + width: number + setOpen: (open: boolean) => void + toggle: () => void + setWidth: (width: number) => void +} + +export const useAgentPanelStore = create()( + persist( + (set) => ({ + open: false, + width: AGENT_PANEL_DEFAULT_WIDTH, + setOpen: (open) => set({ open }), + toggle: () => set((s) => ({ open: !s.open })), + setWidth: (width) => set({ width: clampPanelWidth(width) }), + }), + { + name: 'modly-agent-panel', + partialize: (s) => ({ open: s.open, width: s.width }), + merge: (persisted, current) => { + const p = (persisted ?? {}) as Partial + return { + ...current, + open: typeof p.open === 'boolean' ? p.open : current.open, + width: typeof p.width === 'number' ? clampPanelWidth(p.width) : current.width, + } + }, + }, + ), +) diff --git a/src/shared/types/electron.d.ts b/src/shared/types/electron.d.ts index 2be5a5dc1..675827f62 100644 --- a/src/shared/types/electron.d.ts +++ b/src/shared/types/electron.d.ts @@ -8,6 +8,7 @@ import type { AssetLibraryReadRequest, AssetLibraryReadResult, } from './assetLibrary' +import type { Conversation, ConversationSummary } from '../utils/agentConversations' // ─── Extension types ────────────────────────────────────────────────────────── @@ -228,6 +229,16 @@ declare global { agent: { /** Opens a file picker and copies the chosen .gguf into the agent's models folder. */ addModel: () => Promise<{ success: boolean; cancelled?: boolean; fileName?: string; error?: string }> + /** Persisted chat conversations, one JSON file each in /agent/conversations. */ + conversations: { + /** Newest first. */ + list: () => Promise + /** null when the id is invalid, missing or the file is corrupted. */ + read: (id: string) => Promise + /** Rejects on an invalid id or shape. */ + save: (conversation: Conversation) => Promise + remove: (id: string) => Promise + } } cache: { clear: () => Promise<{ success: boolean; error?: string }> diff --git a/src/shared/utils/agentConversations.test.ts b/src/shared/utils/agentConversations.test.ts new file mode 100644 index 000000000..1362f54c9 --- /dev/null +++ b/src/shared/utils/agentConversations.test.ts @@ -0,0 +1,252 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { + TITLE_MAX_LENGTH, + UNTITLED, + createSerialQueue, + deriveTitle, + relativeDate, + settleMessages, + sortSummaries, + upsertSummary, +} from './agentConversations' +import { useAgentConversationStore } from '../stores/agentConversationStore' + +test('title is the first user message, collapsed to one line', () => { + const title = deriveTitle([ + { role: 'assistant', content: 'Hello' }, + { role: 'user', content: ' make the\n\nmesh smoother ' }, + { role: 'user', content: 'second' }, + ]) + assert.equal(title, 'make the mesh smoother') +}) + +test('a long first message is cut to the title limit', () => { + const title = deriveTitle([{ role: 'user', content: 'x'.repeat(200) }]) + assert.equal(title.length, TITLE_MAX_LENGTH) + assert.ok(title.endsWith('…')) + assert.equal(deriveTitle([{ role: 'user', content: 'y'.repeat(TITLE_MAX_LENGTH) }]), 'y'.repeat(TITLE_MAX_LENGTH)) +}) + +test('no usable user message gives the placeholder title', () => { + assert.equal(deriveTitle([]), UNTITLED) + assert.equal(deriveTitle([{ role: 'assistant', content: 'hi' }, { role: 'user', content: ' ' }, null]), UNTITLED) +}) + +test('summaries sort newest first without mutating the input', () => { + const list = [ + { id: 'a', title: 'A', updatedAt: 10 }, + { id: 'b', title: 'B', updatedAt: 30 }, + { id: 'c', title: 'C', updatedAt: 20 }, + ] + assert.deepEqual(sortSummaries(list).map((s) => s.id), ['b', 'c', 'a']) + assert.deepEqual(list.map((s) => s.id), ['a', 'b', 'c']) +}) + +test('upsertSummary replaces an entry and moves it to the top', () => { + const list = [ + { id: 'b', title: 'B', updatedAt: 30 }, + { id: 'a', title: 'A', updatedAt: 10 }, + ] + assert.deepEqual(upsertSummary(list, { id: 'a', title: 'A2', updatedAt: 40 }), [ + { id: 'a', title: 'A2', updatedAt: 40 }, + { id: 'b', title: 'B', updatedAt: 30 }, + ]) + assert.equal(upsertSummary(list, { id: 'c', title: 'C', updatedAt: 1 }).length, 3) +}) + +test('settleMessages clears streaming and running flags', () => { + const settled = settleMessages([ + { id: 'u', role: 'user', content: 'hi' }, + { id: 'a', role: 'assistant', content: 'part', streaming: true, actions: [{ id: 't', tool: 'x', running: true }] }, + ]) + assert.deepEqual(settled, [ + { id: 'u', role: 'user', content: 'hi' }, + { id: 'a', role: 'assistant', content: 'part', streaming: false, actions: [{ id: 't', tool: 'x', running: false }] }, + ]) +}) + +test('relativeDate buckets by age', () => { + const now = Date.UTC(2026, 10, 12, 12, 0, 0) + assert.equal(relativeDate(now - 5_000, now), 'just now') + assert.equal(relativeDate(now - 5 * 60_000, now), '5m ago') + assert.equal(relativeDate(now - 3 * 3_600_000, now), '3h ago') + assert.equal(relativeDate(now - 2 * 86_400_000, now), '2d ago') + assert.equal(relativeDate(now + 60_000, now), 'just now') + assert.match(relativeDate(now - 30 * 86_400_000, now), /^Oct \d+$/) + assert.match(relativeDate(Date.UTC(2024, 0, 15), now), /2024/) +}) + +test('the serial queue runs tasks one at a time, and a failure does not stop it', async () => { + const enqueue = createSerialQueue() + const log: string[] = [] + const first = deferred() + const a = enqueue(async () => { log.push('a start'); const v = await first.promise; log.push('a end'); return v }) + const b = enqueue(async () => { log.push('b'); throw new Error('b failed') }) + const c = enqueue(async () => { log.push('c'); return 'c' }) + await tick() + assert.deepEqual(log, ['a start']) + first.resolve('a') + assert.equal(await a, 'a') + await assert.rejects(b, /b failed/) + assert.equal(await c, 'c') + assert.deepEqual(log, ['a start', 'a end', 'b', 'c']) +}) + +// ─── Store: disk operations against a fake IPC ─────────────────────────────── + +interface Deferred { promise: Promise; resolve: (v: T) => void; reject: (e: unknown) => void } + +function deferred(): Deferred { + let resolve!: (v: T) => void + let reject!: (e: unknown) => void + const promise = new Promise((res, rej) => { resolve = res; reject = rej }) + return { promise, resolve, reject } +} + +const tick = () => new Promise((resolve) => setTimeout(resolve, 0)) + +type Ipc = { + list: () => Promise + read: (id: string) => Promise + save: (c: { id: string; messages: unknown[] }) => Promise + remove: (id: string) => Promise +} +const ipc: Ipc = { + list: async () => [], + read: async () => null, + save: async () => {}, + remove: async () => {}, +} +const calls: string[] = [] +;(globalThis as Record).window = { + electron: { + agent: { + conversations: { + list: () => { calls.push('list'); return ipc.list() }, + read: (id: string) => { calls.push(`read ${id}`); return ipc.read(id) }, + save: (c: { id: string; messages: unknown[] }) => { calls.push(`save ${c.id}`); return ipc.save(c) }, + remove: (id: string) => { calls.push(`remove ${id}`); return ipc.remove(id) }, + }, + }, + }, +} + +const store = useAgentConversationStore +const msg = (content: string) => ({ id: content, role: 'user' as const, content }) +const stored = (id: string, updatedAt: number) => ({ id, title: id, createdAt: 1, updatedAt, messages: [msg(id)] }) + +function resetIpc(): void { + ipc.list = async () => [] + ipc.read = async () => null + ipc.save = async () => {} + ipc.remove = async () => {} + calls.length = 0 +} + +// Runs first: init is memoized once it succeeds. +test('store: a failed init is retried, and an unreadable latest conversation falls back to the next', async () => { + resetIpc() + const quietError = console.error + console.error = () => {} + try { + ipc.list = async () => { throw new Error('IPC down') } + await store.getState().init() + assert.deepEqual(store.getState().list, []) + + ipc.list = async () => [ + { id: 'broken', title: 'broken', updatedAt: 30 }, + { id: 'good', title: 'good', updatedAt: 20 }, + { id: 'older', title: 'older', updatedAt: 10 }, + ] + ipc.read = async (id) => (id === 'good' ? stored('good', 20) : null) + await store.getState().init() + assert.equal(store.getState().currentId, 'good') + assert.deepEqual(store.getState().list.map((s) => s.id), ['good', 'older']) + assert.deepEqual(calls, ['list', 'list', 'read broken', 'read good']) + + // Memoized now. + await store.getState().init() + assert.equal(calls.filter((c) => c === 'list').length, 2) + } finally { + console.error = quietError + } +}) + +test('store: switching away waits for a save in flight, and stays put when it fails', async () => { + resetIpc() + await store.getState().newConversation() + const id = store.getState().currentId + const quietError = console.error + console.error = () => {} + try { + const first = deferred() + ipc.save = () => first.promise + store.getState().setMessages([msg('hello')]) + const flushing = store.getState().flush() + await tick() + const switching = store.getState().newConversation() + await tick() + // Only the first write has started; the switch is waiting for it. + assert.deepEqual(calls, [`save ${id}`]) + + ipc.save = async () => { throw new Error('disk full') } + first.reject(new Error('disk full')) + await flushing + await switching + // The switch retried the write, which failed too: nothing is lost. + assert.deepEqual(calls, [`save ${id}`, `save ${id}`]) + assert.equal(store.getState().currentId, id) + assert.deepEqual(store.getState().messages, [msg('hello')]) + + ipc.save = async () => {} + await store.getState().newConversation() + assert.notEqual(store.getState().currentId, id) + assert.equal(store.getState().list[0]?.id, id) + } finally { + console.error = quietError + } +}) + +test('store: deleting the current conversation waits for its save and does not come back', async () => { + resetIpc() + await store.getState().newConversation() + const id = store.getState().currentId + const write = deferred() + ipc.save = () => write.promise + store.getState().setMessages([msg('to delete')]) + const flushing = store.getState().flush() + await tick() + const removing = store.getState().remove(id) + await tick() + assert.deepEqual(calls, [`save ${id}`]) + + write.resolve() + await flushing + await removing + assert.deepEqual(calls, [`save ${id}`, `remove ${id}`]) + assert.equal(store.getState().list.some((s) => s.id === id), false) + assert.notEqual(store.getState().currentId, id) + assert.deepEqual(store.getState().messages, []) +}) + +test('store: a failed delete keeps the pending changes', async () => { + resetIpc() + await store.getState().newConversation() + const id = store.getState().currentId + const quietError = console.error + console.error = () => {} + try { + ipc.remove = async () => { throw new Error('locked') } + store.getState().setMessages([msg('keep me')]) + await store.getState().remove(id) + assert.equal(store.getState().currentId, id) + + await store.getState().flush() + assert.deepEqual(calls, [`remove ${id}`, `save ${id}`]) + assert.equal(store.getState().list[0]?.id, id) + } finally { + console.error = quietError + } +}) diff --git a/src/shared/utils/agentConversations.ts b/src/shared/utils/agentConversations.ts new file mode 100644 index 000000000..6f97943bd --- /dev/null +++ b/src/shared/utils/agentConversations.ts @@ -0,0 +1,94 @@ +// Pure renderer-side logic for persisted agent conversations. File storage +// and id validation live in electron/main/agent-conversations.ts. + +export interface ConversationSummary { + id: string + title: string + /** Epoch milliseconds. */ + updatedAt: number +} + +export interface Conversation { + id: string + title: string + /** Epoch milliseconds. */ + createdAt: number + /** Epoch milliseconds. */ + updatedAt: number + /** AgentChat messages; attached images stay inline as data URLs. */ + messages: unknown[] +} + +export const TITLE_MAX_LENGTH = 60 +export const UNTITLED = 'New conversation' + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +/** The first user message, on one line, cut to TITLE_MAX_LENGTH characters. */ +export function deriveTitle(messages: readonly unknown[]): string { + for (const m of messages) { + if (!isRecord(m) || m.role !== 'user' || typeof m.content !== 'string') continue + const text = m.content.replace(/\s+/g, ' ').trim() + if (!text) continue + return text.length > TITLE_MAX_LENGTH ? `${text.slice(0, TITLE_MAX_LENGTH - 1).trimEnd()}…` : text + } + return UNTITLED +} + +/** Newest first; ties keep a stable order by id. */ +export function sortSummaries(list: readonly T[]): T[] { + return [...list].sort((a, b) => (b.updatedAt - a.updatedAt) || a.id.localeCompare(b.id)) +} + +/** Replace (or add) one summary and keep the list sorted. */ +export function upsertSummary(list: readonly ConversationSummary[], summary: ConversationSummary): ConversationSummary[] { + return sortSummaries([...list.filter((s) => s.id !== summary.id), summary]) +} + +/** + * Runs async tasks one at a time, in call order. A failed task does not stop + * the ones queued after it; its caller still gets the rejection. + */ +export function createSerialQueue(): (task: () => Promise) => Promise { + let tail: Promise = Promise.resolve() + return (task: () => Promise): Promise => { + const run = tail.then(() => task()) + tail = run.catch(() => undefined) + return run + } +} + +/** + * A conversation saved mid-turn holds a streaming message and running tools. + * Nothing is streaming once it is loaded again, so settle them. + */ +export function settleMessages(messages: readonly unknown[]): unknown[] { + return messages.map((m) => { + if (!isRecord(m)) return m + const settled: Record = { ...m } + if (settled.streaming) settled.streaming = false + if (Array.isArray(settled.actions)) { + settled.actions = settled.actions.map((a) => (isRecord(a) && a.running ? { ...a, running: false } : a)) + } + return settled + }) +} + +const MINUTE = 60_000 +const HOUR = 60 * MINUTE +const DAY = 24 * HOUR + +/** "just now", "5m ago", "3h ago", "2d ago", then a short date. */ +export function relativeDate(ts: number, now: number = Date.now()): string { + const diff = Math.max(0, now - ts) + if (diff < MINUTE) return 'just now' + if (diff < HOUR) return `${Math.floor(diff / MINUTE)}m ago` + if (diff < DAY) return `${Math.floor(diff / HOUR)}h ago` + if (diff < 7 * DAY) return `${Math.floor(diff / DAY)}d ago` + const d = new Date(ts) + const opts: Intl.DateTimeFormatOptions = { month: 'short', day: 'numeric' } + if (d.getFullYear() !== new Date(now).getFullYear()) opts.year = 'numeric' + return d.toLocaleDateString('en-US', opts) +} diff --git a/src/shared/utils/agentStream.test.ts b/src/shared/utils/agentStream.test.ts index 8eda242dc..9f6ede1b6 100644 --- a/src/shared/utils/agentStream.test.ts +++ b/src/shared/utils/agentStream.test.ts @@ -38,6 +38,35 @@ test('malformed and non-data lines are skipped', () => { ]) }) +test('a client_tool event passes through with its call id, arguments and confirm flag', () => { + const parser = createSseParser() + const event = { + type: 'client_tool', + id: 'call_1', + call_id: 'c-42', + tool: 'uninstall_extension', + arguments: { extension_id: 'triposr' }, + confirm: true, + } + const done = { type: 'tool_done', id: 'call_1', tool: 'uninstall_extension', result: 'ok', payload: null } + + const events = parser.push(line(event) + line(done)) + + assert.deepEqual(events, [event, done]) + const [parsed] = events + assert.ok(parsed.type === 'client_tool' && parsed.call_id === 'c-42' && parsed.confirm) +}) + +test('a generate_from_image payload keeps its model id', () => { + const parser = createSseParser() + const payload = { type: 'generate_from_image', model_id: 'sf3d/main' } + + const [event] = parser.push(line({ type: 'tool_done', id: 't', tool: 'generate_from_image', result: 'queued', payload })) + + assert.ok(event.type === 'tool_done') + assert.deepEqual(event.payload, payload) +}) + test('a response body is read into events, multi-byte characters intact', async () => { const bytes = new TextEncoder().encode(line({ type: 'token', text: 'é→' }) + line({ type: 'done', message: 'é→', actions: [], thinking: null, error: false })) // Split inside the multi-byte "é" so the decoder has to carry it over. diff --git a/src/shared/utils/agentStream.ts b/src/shared/utils/agentStream.ts index 3772ca5d6..6d80903d0 100644 --- a/src/shared/utils/agentStream.ts +++ b/src/shared/utils/agentStream.ts @@ -9,6 +9,10 @@ export interface AgentActionPayload { workflow_id?: string workflow_name?: string workflow?: Omit + /** `generate_from_image`: model to generate with; the Generate page's selection when absent. */ + model_id?: string + /** `generate_from_image`: model parameters overriding the defaults. */ + params?: Record } /** Why a guardrail ended the tool loop; `message` says what to do next. */ @@ -26,11 +30,26 @@ export interface AgentAction { error?: ToolStop | null } +/** + * A tool the frontend must run (it needs IPC or renderer stores). The server + * waits for `POST /agent/tool-result {call_id, result, error?}`, then sends the + * usual `tool_done` for the same `id`. `confirm` asks the user first. + */ +export interface ClientToolEvent { + type: 'client_tool' + id: string + call_id: string + tool: string + arguments: Record + confirm: boolean +} + export type AgentEvent = | { type: 'token'; text: string } | { type: 'thinking'; text: string } | { type: 'tool_start'; id: string; tool: string; arguments: Record } | { type: 'tool_done'; id: string; tool: string; result: string; payload: AgentActionPayload | null; error?: ToolStop | null } + | ClientToolEvent | { type: 'done'; message: string; actions: AgentAction[]; thinking: string | null; error: boolean; stopped?: ToolStop | null } /** Bounds of the request's `tool_budget` (calls per turn) and `tool_timeout` (seconds per call). */ diff --git a/src/shared/utils/generationRunner.ts b/src/shared/utils/generationRunner.ts new file mode 100644 index 000000000..5e50dc93a --- /dev/null +++ b/src/shared/utils/generationRunner.ts @@ -0,0 +1,172 @@ +import axios from 'axios' +import type { GenerationJob, GenerationOptions } from '../stores/appStore' + +// The single-image generation path (POST /generate/from-image, then poll +// /generate/status), free of React so the agent can start one outside a hook. +// useGeneration and useApi both go through it. + +export interface JobStatus { + status: 'pending' | 'running' | 'done' | 'error' | 'cancelled' + progress: number + step?: string + outputUrl?: string + error?: string +} + +export interface GenerationApi { + generateFromImage: (imagePath: string, options: GenerationOptions, imageData?: string, signal?: AbortSignal) => Promise<{ jobId: string }> + pollJobStatus: (jobId: string) => Promise + cancelJob: (jobId: string) => Promise +} + +export function createGenerationApi(apiUrl: string): GenerationApi { + const client = axios.create({ baseURL: apiUrl }) + + async function generateFromImage( + imagePath: string, + options: GenerationOptions, + imageData?: string, + signal?: AbortSignal, + ): Promise<{ jobId: string }> { + // Use provided base64 (drag & drop) or read from disk via IPC + const base64 = imageData ?? await window.electron.fs.readFileBase64(imagePath) + const byteArray = Uint8Array.from(atob(base64), (c) => c.charCodeAt(0)) + const blob = new Blob([byteArray], { type: 'image/png' }) + const filename = imagePath.split(/[\\/]/).pop() ?? 'image.png' + + const formData = new FormData() + formData.append('image', blob, filename) + formData.append('model_id', options.modelId) + formData.append('remesh', options.remesh) + formData.append('enable_texture', String(options.enableTexture)) + formData.append('texture_resolution', String(options.textureResolution)) + formData.append('params', JSON.stringify(options.modelParams)) + const { data } = await client.post<{ job_id: string }>('/generate/from-image', formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + signal, + }) + + return { jobId: data.job_id } + } + + async function pollJobStatus(jobId: string): Promise { + const { data } = await client.get(`/generate/status/${jobId}`) + return { ...data, outputUrl: data.output_url } + } + + async function cancelJob(jobId: string): Promise { + await client.post(`/generate/cancel/${jobId}`).catch(() => {}) + } + + return { generateFromImage, pollJobStatus, cancelJob } +} + +export interface GenerationStoreActions { + setCurrentJob: (job: GenerationJob | null) => void + updateCurrentJob: (patch: Partial) => void + pushMeshUrl: (url: string) => void + clearMeshHistory: () => void +} + +export interface GenerationRunDeps { + api: GenerationApi + store: GenerationStoreActions + notifyComplete: (body: string) => void | Promise + /** Same input as showRunErrorNotification: the error text and the model that failed. */ + notifyError: (error: string | undefined, modelId: string) => void | Promise + isCancelled: () => boolean + signal?: AbortSignal + /** Delay between two status polls; tests pass a fast one. */ + wait?: () => Promise +} + +export type GenerationOutcome = + | { status: 'done'; outputUrl?: string } + | { status: 'error'; error: string } + | { status: 'cancelled' } + +/** Runs one generation into `currentJob`, so the Generate viewer shows it. */ +export async function runGeneration( + imagePath: string, + options: GenerationOptions, + imageData: string | undefined, + deps: GenerationRunDeps, +): Promise { + const { api, store } = deps + const wait = deps.wait ?? (() => new Promise((r) => setTimeout(r, 1000))) + store.clearMeshHistory() + store.setCurrentJob({ + id: crypto.randomUUID(), + imageFile: imagePath, + status: 'uploading', + progress: 0, + createdAt: Date.now(), + modelId: options.modelId, + generationOptions: options, + }) + + try { + const { jobId } = await api.generateFromImage(imagePath, options, imageData, deps.signal) + + if (deps.isCancelled()) { + await api.cancelJob(jobId) + store.setCurrentJob(null) + return { status: 'cancelled' } + } + + store.updateCurrentJob({ status: 'generating', progress: 0 }) + + for (;;) { + await wait() + + if (deps.isCancelled()) { + await api.cancelJob(jobId) + store.setCurrentJob(null) + return { status: 'cancelled' } + } + + const result = await api.pollJobStatus(jobId) + + if (result.status === 'cancelled') { + store.setCurrentJob(null) + return { status: 'cancelled' } + } + + if (result.status === 'done') { + store.updateCurrentJob({ status: 'done', progress: 100, outputUrl: result.outputUrl, originalOutputUrl: result.outputUrl }) + if (result.outputUrl) store.pushMeshUrl(result.outputUrl) + void deps.notifyComplete('Generation complete') + return { status: 'done', outputUrl: result.outputUrl } + } + + if (result.status === 'error') { + store.updateCurrentJob({ status: 'error', error: result.error }) + void deps.notifyError(result.error, options.modelId) + return { status: 'error', error: result.error ?? 'Unknown error' } + } + + store.updateCurrentJob({ + progress: result.progress, + step: result.step, + }) + } + } catch (err) { + if (deps.isCancelled()) { + store.setCurrentJob(null) + return { status: 'cancelled' } + } + let errorMessage: string + if (err && typeof err === 'object' && 'response' in err) { + const axiosErr = err as { response?: { data?: { detail?: string } }; message: string } + errorMessage = axiosErr.response?.data?.detail ?? axiosErr.message + } else { + errorMessage = err instanceof Error ? err.message : String(err) + } + store.updateCurrentJob({ + status: 'error', + error: errorMessage, + }) + void deps.notifyError(errorMessage, options.modelId) + return { status: 'error', error: errorMessage } + } +}