From b6df969a9f737ede4ce82951793b7c15a1076f48 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 23 Sep 2026 15:42:39 -0700 Subject: [PATCH 1/3] fix(browser): keep snapshot ids valid after an observed action is refused An observed click refused before dispatch (for example, covered by an overlay) never starts its observation, yet it invalidated every id in the snapshot, so the agent could not click the overlay's own close button without re-observing. A failed observed action with no dispatched outcome now only cancels an in-flight capture, which clears the refs itself when it starts. --- .../src/main/browser-agent/driver.test.ts | 18 +++++++++++++++++ apps/desktop/src/main/browser-agent/driver.ts | 20 ++++++++++++------- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/apps/desktop/src/main/browser-agent/driver.test.ts b/apps/desktop/src/main/browser-agent/driver.test.ts index a471258d051..0d0fe73e58c 100644 --- a/apps/desktop/src/main/browser-agent/driver.test.ts +++ b/apps/desktop/src/main/browser-agent/driver.test.ts @@ -3425,6 +3425,24 @@ describe('credential protection', () => { expect(second).not.toMatchObject({ result: { notices: expect.anything() } }) }) + it('keeps element ids valid when an observed action is refused before dispatch', async () => { + const contents = await openPage() + respondWith(contents, { clickElement: { error: 'obstructed', blocker: 'IMG' } }) + + const refused = await driver.executeTool('chat-test', 'browser_click', { + elementId: 0, + observe: {}, + }) + respondWith(contents, {}) + const retried = await driver.executeTool('chat-test', 'browser_click', { elementId: 0 }) + + expect(refused).toEqual({ + ok: false, + error: expect.stringContaining('That element is covered by IMG'), + }) + expect(retried).toMatchObject({ ok: true, result: { dispatched: true } }) + }) + it('invalidates element ids when the active tab changes', async () => { await openPage() await driver.executeTool('chat-test', 'browser_open_tab', {}) diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index 9a049a8fcb0..f33a4977394 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -313,6 +313,14 @@ function createDriverScopeState(): DriverScopeState { function invalidateSnapshot(state = driverScopeState()): void { state.snapshotTabId = null state.snapshotTargets.clear() + cancelPendingSnapshotCapture(state) +} + +/** + * Keeps the current refs but stops any in-flight capture from committing its own. A capture + * clears the refs when it starts, so this is all a failed action with a pending observation needs. + */ +function cancelPendingSnapshotCapture(state: DriverScopeState): void { state.snapshotCaptureEpoch++ } @@ -5002,7 +5010,10 @@ export async function executeTool( try { observedResult = await guardedExecution } catch (error) { - if (!actionOutcome) throw error + if (!actionOutcome) { + if (params.observe !== undefined) cancelPendingSnapshotCapture(state) + throw error + } invalidateSnapshot(state) observedResult = actionOutcome.status === 'pending' @@ -5049,12 +5060,7 @@ export async function executeTool( // The watchdog cannot cancel an in-flight renderer promise. Invalidate its // capture token before releasing the queue so a late snapshot cannot // overwrite refs belonging to a newer tab or snapshot. - if ( - tool === 'browser_snapshot' || - tool === 'browser_open_url' || - tool === 'browser_find' || - params.observe !== undefined - ) { + if (tool === 'browser_snapshot' || tool === 'browser_open_url' || tool === 'browser_find') { invalidateSnapshot(state) } const message = String(sanitizeBrowserResult(getErrorMessage(error), undefined, 0, 'error')) From 4548dfadf48ecb4484453650d3496f3b0fbc24c6 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 23 Sep 2026 15:51:51 -0700 Subject: [PATCH 2/3] feat(browser): run planned page actions in order with browser_batch The desktop runs 2-8 single-page interactions from one authorized call, in order, stopping at the first failed action or after an action navigates, switches tabs, or invalidates the snapshot's element ids. A batch stopped by a failure reports an error status; one stopped by a page change does not. Once an action has run, a cancelled or timed-out batch reports an unknown outcome so the agent does not repeat it. --- .../src/main/browser-agent/driver.test.ts | 148 +++++++++++++++++- apps/desktop/src/main/browser-agent/driver.ts | 108 ++++++++++++- .../browser-agent/post-action-observation.ts | 1 + .../home/components/message-content/utils.ts | 1 + .../mothership/generated/tool-catalog-v1.ts | 94 +++++++++++ .../mothership/generated/tool-schemas-v1.ts | 95 +++++++++++ .../client/browser-tool-execution.test.ts | 35 +++++ .../tools/client/browser-tool-execution.ts | 17 +- apps/sim/lib/mothership/tools/tool-display.ts | 1 + packages/browser-protocol/src/index.ts | 2 + 10 files changed, 492 insertions(+), 10 deletions(-) diff --git a/apps/desktop/src/main/browser-agent/driver.test.ts b/apps/desktop/src/main/browser-agent/driver.test.ts index 0d0fe73e58c..60e30a53810 100644 --- a/apps/desktop/src/main/browser-agent/driver.test.ts +++ b/apps/desktop/src/main/browser-agent/driver.test.ts @@ -1,4 +1,5 @@ import { BROWSER_TOOL_QUEUE_WAIT_TIMEOUT_MS } from '@sim/browser-protocol' +import { toRecord } from '@sim/utils/object' import type { MenuItemConstructorOptions, WebContents } from 'electron' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -2097,13 +2098,20 @@ describe('credential protection', () => { for (const [fnName, value] of Object.entries(replies)) { if (isPageCall(expression, fnName)) return Promise.resolve(value) } - if (isPageCall(expression, 'clickElement')) { - return Promise.resolve({ dispatched: false, x: 24, y: 48, element: 'Test' }) - } + if (isPageCall(expression, 'clickElement')) return Promise.resolve(CLICK_TARGET) return Promise.resolve(undefined) }) } + function mousePresses(contents: Awaited>): number { + return cdpCalls(contents, 'Input.dispatchMouseEvent').filter( + ([, params]) => toRecord(params).type === 'mousePressed' + ).length + } + + /** What the page reports for an ordinary click target before native dispatch. */ + const CLICK_TARGET = { dispatched: false, x: 24, y: 48, element: 'Test' } + function cdpCalls(contents: Awaited>, method: string): unknown[][] { return vi .mocked(contents.debugger.sendCommand) @@ -3425,6 +3433,140 @@ describe('credential protection', () => { expect(second).not.toMatchObject({ result: { notices: expect.anything() } }) }) + it('runs batched actions in order and returns each result', async () => { + const contents = await openPage() + respondWith(contents, {}) + + const result = await driver.executeTool('chat-test', 'browser_batch', { + actions: [ + { tool: 'browser_click', args: { elementId: 0 } }, + { tool: 'browser_click', args: { elementId: 0 } }, + ], + }) + + expect(result).toMatchObject({ + ok: true, + result: { + completed: true, + completedCount: 2, + results: [ + { index: 0, tool: 'browser_click', result: { dispatched: true } }, + { index: 1, tool: 'browser_click', result: { dispatched: true } }, + ], + }, + }) + }) + + it('stops a batch at the first failed action and keeps earlier results', async () => { + const contents = await openPage() + vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => { + if (!isPageCall(expression, 'clickElement')) return Promise.resolve(undefined) + return Promise.resolve( + mousePresses(contents) > 0 ? { error: 'obstructed', blocker: 'IMG' } : CLICK_TARGET + ) + }) + + const result = await driver.executeTool('chat-test', 'browser_batch', { + actions: [ + { tool: 'browser_click', args: { elementId: 0 } }, + { tool: 'browser_click', args: { elementId: 0 } }, + { tool: 'browser_click', args: { elementId: 0 } }, + ], + }) + + expect(mousePresses(contents)).toBe(1) + expect(result).toMatchObject({ + ok: true, + result: { + completed: false, + completedCount: 1, + stoppedIndex: 1, + stoppedBy: 'failure', + error: expect.stringContaining('covered by IMG'), + }, + }) + }) + + it('stops a batch after an action navigates the page', async () => { + const contents = await openPage() + respondWith(contents, {}) + const sendCommand = vi.mocked(contents.debugger.sendCommand) + const dispatch = sendCommand.getMockImplementation() + sendCommand.mockImplementation((method, params) => { + if (method === 'Input.dispatchMouseEvent' && toRecord(params).type === 'mouseReleased') { + emitContentsEvent(contents, 'did-navigate') + } + return dispatch?.(method, params) ?? Promise.resolve(undefined) + }) + + const result = await driver.executeTool('chat-test', 'browser_batch', { + actions: [ + { tool: 'browser_click', args: { elementId: 0 } }, + { tool: 'browser_click', args: { elementId: 0 } }, + ], + }) + + expect(mousePresses(contents)).toBe(1) + expect(result).toMatchObject({ + ok: true, + result: { completed: false, completedCount: 1, stoppedIndex: 1, stoppedBy: 'page-change' }, + }) + }) + + it('reports a batch cancelled after an action ran as an unknown outcome', async () => { + const contents = await openPage() + respondWith(contents, {}) + const sendCommand = vi.mocked(contents.debugger.sendCommand) + const dispatch = sendCommand.getMockImplementation() + sendCommand.mockImplementation((method, params) => + method === 'Input.dispatchKeyEvent' + ? new Promise(() => {}) + : (dispatch?.(method, params) ?? Promise.resolve(undefined)) + ) + + const pending = driver.executeTool( + 'chat-test', + 'browser_batch', + { + actions: [ + { tool: 'browser_click', args: { elementId: 0 } }, + { tool: 'browser_press_key', args: { key: 'Enter' } }, + ], + }, + 'batch-call' + ) + await vi.waitFor(() => expect(cdpCalls(contents, 'Input.dispatchKeyEvent')).toHaveLength(1)) + driver.cancelTool('chat-test', 'batch-call') + + await expect(pending).resolves.toMatchObject({ + ok: true, + result: { outcomeUnknown: true, doNotRetry: true }, + }) + }) + + it('rejects batches that name non-action tools or observe per action', async () => { + await openPage() + + const navigation = await driver.executeTool('chat-test', 'browser_batch', { + actions: [ + { tool: 'browser_navigate', args: { url: 'https://example.com' } }, + { tool: 'browser_click', args: { elementId: 0 } }, + ], + }) + const observed = await driver.executeTool('chat-test', 'browser_batch', { + actions: [ + { tool: 'browser_click', args: { elementId: 0, observe: {} } }, + { tool: 'browser_click', args: { elementId: 0 } }, + ], + }) + + expect(navigation).toMatchObject({ + ok: false, + error: expect.stringContaining('Batch action 0'), + }) + expect(observed).toMatchObject({ ok: false, error: expect.stringContaining('cannot observe') }) + }) + it('keeps element ids valid when an observed action is refused before dispatch', async () => { const contents = await openPage() respondWith(contents, { clickElement: { error: 'obstructed', blocker: 'IMG' } }) diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index f33a4977394..35aef9c4c7f 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -144,6 +144,53 @@ type FormField = | { elementId: number; kind: 'select'; value: string } | { elementId: number; kind: 'checked'; checked: boolean } +const MAX_BATCH_ACTIONS = 8 +/** Single-page interactions a batch may run; navigation, observation, and file tools stay separate. */ +const BATCH_ACTION_TOOLS: ReadonlySet = new Set([ + 'browser_click', + 'browser_click_at', + 'browser_type', + 'browser_insert_text', + 'browser_press_key', + 'browser_scroll', + 'browser_select_option', + 'browser_set_checked', + 'browser_hover', +]) + +interface BatchAction { + tool: BrowserToolName + args: Record +} + +function isBatchActionTool(value: unknown): value is BrowserToolName { + return BATCH_ACTION_TOOLS.has(value as BrowserToolName) +} + +function parseBatchActions(params: Record): BatchAction[] { + if (Object.keys(params).some((key) => key !== 'actions')) { + throw new ToolError('A batch accepts only actions; pass observe on the batch itself.') + } + if ( + !Array.isArray(params.actions) || + params.actions.length < 2 || + params.actions.length > MAX_BATCH_ACTIONS + ) { + throw new ToolError(`A batch requires between 2 and ${MAX_BATCH_ACTIONS} actions.`) + } + return params.actions.map((action, index): BatchAction => { + if (!isRecordLike(action) || !isBatchActionTool(action.tool) || !isRecordLike(action.args)) { + throw new ToolError( + `Batch action ${index} must be {tool, args} with tool one of ${[...BATCH_ACTION_TOOLS].join(', ')}.` + ) + } + if ('observe' in action.args) { + throw new ToolError(`Batch action ${index} cannot observe; pass observe on the batch itself.`) + } + return { tool: action.tool, args: action.args } + }) +} + function parseFormFields(params: Record): FormField[] { if (Object.keys(params).some((key) => key !== 'fields')) { throw new ToolError('Form filling accepts only fields; submitting is not supported.') @@ -1046,7 +1093,8 @@ export function browserToolWatchdogMs( tool === 'browser_open_tab' || tool === 'browser_switch_tab' || tool === 'browser_upload_file' || - tool === 'browser_save_download' + tool === 'browser_save_download' || + tool === 'browser_batch' ) { return BROWSER_NAVIGATION_NATIVE_WATCHDOG_MS } @@ -3389,6 +3437,64 @@ async function executeToolInner( } } + case 'browser_batch': { + const actions = parseBatchActions(params) + const results: { index: number; tool: BrowserToolName; result: unknown }[] = [] + const stopped = ( + stoppedIndex: number, + stoppedBy: 'failure' | 'page-change', + error: string + ) => ({ + completed: false, + completedCount: results.length, + stoppedIndex, + stoppedBy, + error, + results, + }) + for (const [index, action] of actions.entries()) { + let tab: ReturnType + let epoch: number + let snapshotValid: boolean + let result: unknown + // Once an action has run, a cancelled or timed-out batch must not read as never started. + if (index > 0) onActionOutcome?.({ status: 'pending' }) + try { + tab = session.requireAutomationTab() + epoch = navigationEpoch(tab.view.webContents) + snapshotValid = driverScopeState().snapshotTabId === tab.id + result = await executeToolInner( + action.tool, + action.args, + assertCurrentExecution, + executionDeadline, + invocationEpoch, + signal + ) + } catch (error) { + return stopped( + index, + 'failure', + `Action ${index} (${action.tool}) failed: ${getErrorMessage(error)} Earlier actions already took effect.` + ) + } + results.push({ index, tool: action.tool, result }) + if (index === actions.length - 1) break + if ( + session.automationTab()?.id !== tab.id || + navigationEpoch(tab.view.webContents) !== epoch || + (snapshotValid && driverScopeState().snapshotTabId !== tab.id) + ) { + return stopped( + index + 1, + 'page-change', + `Action ${index} (${action.tool}) changed the page, so the remaining actions did not run. Inspect the page before continuing.` + ) + } + } + return { completed: true, completedCount: results.length, results } + } + case 'browser_fill_form': { const fields = parseFormFields(params) const contents = session.requireAutomationTab().view.webContents diff --git a/apps/desktop/src/main/browser-agent/post-action-observation.ts b/apps/desktop/src/main/browser-agent/post-action-observation.ts index 91e8aa4b616..b00b94f7b2b 100644 --- a/apps/desktop/src/main/browser-agent/post-action-observation.ts +++ b/apps/desktop/src/main/browser-agent/post-action-observation.ts @@ -10,6 +10,7 @@ const OBSERVABLE_ACTIONS: ReadonlySet = new Set([ 'browser_fill_form', 'browser_scroll', 'browser_hover', + 'browser_batch', ]) /** Preserves a dispatched action when its acknowledgement or observation is interrupted. */ diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts index e5da95feb55..5357f638a08 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts @@ -39,6 +39,7 @@ export const TOOL_ICONS: Readonly> = { apply_file_edit: File, auth: Integration, browser: Globe, + browser_batch: Cursor, browser_click: Cursor, browser_click_at: Cursor, browser_close_tab: Cursor, diff --git a/apps/sim/lib/mothership/generated/tool-catalog-v1.ts b/apps/sim/lib/mothership/generated/tool-catalog-v1.ts index 01ddc25be53..30392883f50 100644 --- a/apps/sim/lib/mothership/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/mothership/generated/tool-catalog-v1.ts @@ -15,6 +15,7 @@ export interface ToolCatalogEntry { | 'apply_file_edit' | 'auth' | 'browser' + | 'browser_batch' | 'browser_click' | 'browser_click_at' | 'browser_close_tab' @@ -159,6 +160,7 @@ export interface ToolCatalogEntry { | 'apply_file_edit' | 'auth' | 'browser' + | 'browser_batch' | 'browser_click' | 'browser_click_at' | 'browser_close_tab' @@ -393,6 +395,97 @@ export const Browser: ToolCatalogEntry = { internal: true, } +export const BrowserBatch: ToolCatalogEntry = { + id: 'browser_batch', + name: 'browser_batch', + route: 'client', + mode: 'async', + parameters: { + type: 'object', + properties: { + actions: { + type: 'array', + description: + "Ordered list of 2–8 actions. Each names one action tool and gives exactly that tool's parameters in args, without observe.", + items: { + type: 'object', + properties: { + args: { + type: 'object', + description: + 'That tool\'s own parameters, for example {"elementId": 12} for browser_click or {"key": "Enter"} for browser_press_key.', + }, + tool: { + type: 'string', + description: 'The action tool to run.', + enum: [ + 'browser_click', + 'browser_click_at', + 'browser_type', + 'browser_insert_text', + 'browser_press_key', + 'browser_scroll', + 'browser_select_option', + 'browser_set_checked', + 'browser_hover', + ], + }, + }, + required: ['tool', 'args'], + }, + }, + observe: { + type: 'object', + description: + 'Observe immediately after this action in the same call. Use {} for a fresh snapshot or {query: string} for matching element refs only. Returned refs replace prior refs. A failed observation does not mean the action failed; inspect its result before retrying.', + properties: { + query: { + type: 'string', + description: + 'Case-insensitive text to find in the resulting page. Omit for the full snapshot. Maximum 4096 characters.', + }, + }, + }, + }, + required: ['actions'], + }, + resultSchema: { + type: 'object', + properties: { + completed: { type: 'boolean', description: 'True when every action ran and succeeded.' }, + completedCount: { + type: 'number', + description: 'Number of actions that ran and succeeded, from the start of the list.', + }, + error: { + type: 'string', + description: + 'Why the batch stopped early. Earlier actions already took effect; do not repeat them.', + }, + results: { + type: 'array', + description: 'Results of the actions that ran, in order.', + items: { + type: 'object', + properties: { + index: { type: 'number' }, + result: { type: 'object', description: "The action tool's own result." }, + tool: { type: 'string' }, + }, + required: ['index', 'tool', 'result'], + }, + }, + stoppedIndex: { + type: 'number', + description: + 'Zero-based index of the first action that did not run or failed, when the batch stopped early.', + }, + }, + required: ['completed', 'completedCount', 'results'], + }, + clientExecutable: true, +} + export const BrowserClick: ToolCatalogEntry = { id: 'browser_click', name: 'browser_click', @@ -8132,6 +8225,7 @@ export const TOOL_CATALOG: Record = { [ApplyFileEdit.id]: ApplyFileEdit, [Auth.id]: Auth, [Browser.id]: Browser, + [BrowserBatch.id]: BrowserBatch, [BrowserClick.id]: BrowserClick, [BrowserClickAt.id]: BrowserClickAt, [BrowserCloseTab.id]: BrowserCloseTab, diff --git a/apps/sim/lib/mothership/generated/tool-schemas-v1.ts b/apps/sim/lib/mothership/generated/tool-schemas-v1.ts index 54ac6909ade..081c0ac672b 100644 --- a/apps/sim/lib/mothership/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/mothership/generated/tool-schemas-v1.ts @@ -81,6 +81,101 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, + browser_batch: { + parameters: { + type: 'object', + properties: { + actions: { + type: 'array', + description: + "Ordered list of 2–8 actions. Each names one action tool and gives exactly that tool's parameters in args, without observe.", + items: { + type: 'object', + properties: { + args: { + type: 'object', + description: + 'That tool\'s own parameters, for example {"elementId": 12} for browser_click or {"key": "Enter"} for browser_press_key.', + }, + tool: { + type: 'string', + description: 'The action tool to run.', + enum: [ + 'browser_click', + 'browser_click_at', + 'browser_type', + 'browser_insert_text', + 'browser_press_key', + 'browser_scroll', + 'browser_select_option', + 'browser_set_checked', + 'browser_hover', + ], + }, + }, + required: ['tool', 'args'], + }, + }, + observe: { + type: 'object', + description: + 'Observe immediately after this action in the same call. Use {} for a fresh snapshot or {query: string} for matching element refs only. Returned refs replace prior refs. A failed observation does not mean the action failed; inspect its result before retrying.', + properties: { + query: { + type: 'string', + description: + 'Case-insensitive text to find in the resulting page. Omit for the full snapshot. Maximum 4096 characters.', + }, + }, + }, + }, + required: ['actions'], + }, + resultSchema: { + type: 'object', + properties: { + completed: { + type: 'boolean', + description: 'True when every action ran and succeeded.', + }, + completedCount: { + type: 'number', + description: 'Number of actions that ran and succeeded, from the start of the list.', + }, + error: { + type: 'string', + description: + 'Why the batch stopped early. Earlier actions already took effect; do not repeat them.', + }, + results: { + type: 'array', + description: 'Results of the actions that ran, in order.', + items: { + type: 'object', + properties: { + index: { + type: 'number', + }, + result: { + type: 'object', + description: "The action tool's own result.", + }, + tool: { + type: 'string', + }, + }, + required: ['index', 'tool', 'result'], + }, + }, + stoppedIndex: { + type: 'number', + description: + 'Zero-based index of the first action that did not run or failed, when the batch stopped early.', + }, + }, + required: ['completed', 'completedCount', 'results'], + }, + }, browser_click: { parameters: { type: 'object', diff --git a/apps/sim/lib/mothership/tools/client/browser-tool-execution.test.ts b/apps/sim/lib/mothership/tools/client/browser-tool-execution.test.ts index 1e8d4eab57c..f9dfa10fa2e 100644 --- a/apps/sim/lib/mothership/tools/client/browser-tool-execution.test.ts +++ b/apps/sim/lib/mothership/tools/client/browser-tool-execution.test.ts @@ -110,6 +110,41 @@ describe('executeBrowserToolOnClient', () => { expect(mockExecuteBrowserTool).toHaveBeenCalledTimes(1) }) + it('reports a batch as failed only when one of its actions failed', async () => { + const actions = [ + { tool: 'browser_click', args: { elementId: 1 } }, + { tool: 'browser_click', args: { elementId: 2 } }, + ] + const partial = { + completed: false, + completedCount: 1, + stoppedIndex: 1, + results: [{ index: 0, tool: 'browser_click', result: { dispatched: true } }], + } + const failed = { ...partial, stoppedBy: 'failure', error: 'Action 1 failed' } + const pageChanged = { ...partial, stoppedBy: 'page-change', error: 'Action 0 changed the page' } + const failedCallId = nextToolCallId() + const changedCallId = nextToolCallId() + + mockExecuteBrowserTool.mockResolvedValueOnce(failed).mockResolvedValueOnce(pageChanged) + executeBrowserToolOnClient(failedCallId, 'browser_batch', { actions }, CHAT_SCOPE) + executeBrowserToolOnClient(changedCallId, 'browser_batch', { actions }, CHAT_SCOPE) + await flush() + + expect(mockReportCompletion).toHaveBeenCalledWith( + failedCallId, + 'error', + 'A batched browser action failed; inspect the partial result', + failed + ) + expect(mockReportCompletion).toHaveBeenCalledWith( + changedCallId, + 'success', + 'Browser action completed', + pageChanged + ) + }) + it('preserves every executed completion when a guard result arrives at retention capacity', async () => { const replayClaim = vi .spyOn(BrowserToolReplayLedger.prototype, 'claim') diff --git a/apps/sim/lib/mothership/tools/client/browser-tool-execution.ts b/apps/sim/lib/mothership/tools/client/browser-tool-execution.ts index 23b9587ca58..eb65edd1e11 100644 --- a/apps/sim/lib/mothership/tools/client/browser-tool-execution.ts +++ b/apps/sim/lib/mothership/tools/client/browser-tool-execution.ts @@ -78,6 +78,7 @@ const OBSERVATION_ONLY_BROWSER_TOOLS = { browser_click_at: false, browser_type: false, browser_fill_form: false, + browser_batch: false, browser_insert_text: false, browser_press_key: false, browser_scroll: false, @@ -937,21 +938,25 @@ async function doExecuteBrowserTool( if (cancelled) return const outcomeUnknown = isRecordLike(result) && result.outcomeUnknown === true const effectUnconfirmed = isRecordLike(result) && result.effectObserved === false - const formStopped = + const stoppedMessage = toolName === 'browser_fill_form' && isRecordLike(result) && result.completed === false + ? 'Form filling stopped; inspect the partial result' + : toolName === 'browser_batch' && isRecordLike(result) && result.stoppedBy === 'failure' + ? 'A batched browser action failed; inspect the partial result' + : undefined reportTerminalCompletion( { status: - outcomeUnknown || formStopped + stoppedMessage || outcomeUnknown ? ASYNC_TOOL_CONFIRMATION_STATUS.error : ASYNC_TOOL_CONFIRMATION_STATUS.success, - message: formStopped - ? 'Form filling stopped; inspect the partial result' - : outcomeUnknown + message: + stoppedMessage ?? + (outcomeUnknown ? 'Browser action outcome is unconfirmed; inspect the page before repeating it.' : effectUnconfirmed ? 'Browser input completed; its effect is unconfirmed. Inspect the current state before retrying.' - : 'Browser action completed', + : 'Browser action completed'), data: sanitizeBrowserToolResultForModel(toolName, result), }, 'Failed to report browser tool completion' diff --git a/apps/sim/lib/mothership/tools/tool-display.ts b/apps/sim/lib/mothership/tools/tool-display.ts index b5120c3a177..a390b8873d7 100644 --- a/apps/sim/lib/mothership/tools/tool-display.ts +++ b/apps/sim/lib/mothership/tools/tool-display.ts @@ -676,6 +676,7 @@ const TOOL_TITLES: Record = { browser_drag: 'Dragging element', browser_select_option: 'Selecting option', browser_fill_form: 'Filling form', + browser_batch: 'Running page actions', browser_set_checked: 'Updating control', browser_hover: 'Hovering element', browser_zoom: 'Changing page zoom', diff --git a/packages/browser-protocol/src/index.ts b/packages/browser-protocol/src/index.ts index 926b8b7bc66..aae31c887e3 100644 --- a/packages/browser-protocol/src/index.ts +++ b/packages/browser-protocol/src/index.ts @@ -43,6 +43,7 @@ export const CURRENT_BROWSER_TOOL_NAMES = [ 'browser_click_at', 'browser_type', 'browser_fill_form', + 'browser_batch', 'browser_insert_text', 'browser_press_key', 'browser_scroll', @@ -118,6 +119,7 @@ export function browserToolRendererTimeoutMs( case 'browser_switch_tab': case 'browser_upload_file': case 'browser_save_download': + case 'browser_batch': return BROWSER_NAVIGATION_RENDERER_TIMEOUT_MS case 'browser_wait_for': return ( From d9295b2d751663091b8b38f194f1c2d3876b4633 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 23 Sep 2026 17:04:51 -0700 Subject: [PATCH 3/3] fix(browser): mark batched actions pending and stop batches on same-document URL changes A batch now reports its outcome as pending before every action, since an action can dispatch input before it returns; a cancelled or timed-out batch therefore never reads as not started. It also stops after an action changes the tab URL within the document, not only on cross-document navigation. --- .../src/main/browser-agent/driver.test.ts | 59 +++++++++++++++++++ apps/desktop/src/main/browser-agent/driver.ts | 8 ++- .../mothership/generated/tool-catalog-v1.ts | 20 +++++-- .../mothership/generated/tool-schemas-v1.ts | 20 +++++-- 4 files changed, 95 insertions(+), 12 deletions(-) diff --git a/apps/desktop/src/main/browser-agent/driver.test.ts b/apps/desktop/src/main/browser-agent/driver.test.ts index 60e30a53810..9f3a4daf86a 100644 --- a/apps/desktop/src/main/browser-agent/driver.test.ts +++ b/apps/desktop/src/main/browser-agent/driver.test.ts @@ -3513,6 +3513,65 @@ describe('credential protection', () => { }) }) + it('stops a batch after an action changes the URL within the document', async () => { + const contents = await openPage() + respondWith(contents, {}) + const url = contents.getURL() + const sendCommand = vi.mocked(contents.debugger.sendCommand) + const dispatch = sendCommand.getMockImplementation() + sendCommand.mockImplementation((method, params) => { + if (method === 'Input.dispatchMouseEvent' && toRecord(params).type === 'mouseReleased') { + vi.mocked(contents.getURL).mockReturnValue(`${url}#next`) + emitContentsEvent(contents, 'did-navigate-in-page') + } + return dispatch?.(method, params) ?? Promise.resolve(undefined) + }) + + const result = await driver.executeTool('chat-test', 'browser_batch', { + actions: [ + { tool: 'browser_click', args: { elementId: 0 } }, + { tool: 'browser_click', args: { elementId: 0 } }, + ], + }) + + expect(mousePresses(contents)).toBe(1) + expect(result).toMatchObject({ + ok: true, + result: { completed: false, completedCount: 1, stoppedIndex: 1, stoppedBy: 'page-change' }, + }) + }) + + it('reports a batch cancelled during its first action as an unknown outcome', async () => { + const contents = await openPage() + respondWith(contents, {}) + const sendCommand = vi.mocked(contents.debugger.sendCommand) + const dispatch = sendCommand.getMockImplementation() + sendCommand.mockImplementation((method, params) => + method === 'Input.dispatchKeyEvent' + ? new Promise(() => {}) + : (dispatch?.(method, params) ?? Promise.resolve(undefined)) + ) + + const pending = driver.executeTool( + 'chat-test', + 'browser_batch', + { + actions: [ + { tool: 'browser_press_key', args: { key: 'Enter' } }, + { tool: 'browser_click', args: { elementId: 0 } }, + ], + }, + 'batch-first-call' + ) + await vi.waitFor(() => expect(cdpCalls(contents, 'Input.dispatchKeyEvent')).toHaveLength(1)) + driver.cancelTool('chat-test', 'batch-first-call') + + await expect(pending).resolves.toMatchObject({ + ok: true, + result: { outcomeUnknown: true, doNotRetry: true }, + }) + }) + it('reports a batch cancelled after an action ran as an unknown outcome', async () => { const contents = await openPage() respondWith(contents, {}) diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index 35aef9c4c7f..2ba47c613f0 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -3455,13 +3455,16 @@ async function executeToolInner( for (const [index, action] of actions.entries()) { let tab: ReturnType let epoch: number + let url: string let snapshotValid: boolean let result: unknown - // Once an action has run, a cancelled or timed-out batch must not read as never started. - if (index > 0) onActionOutcome?.({ status: 'pending' }) + // An action may dispatch input before it returns, so a cancelled or timed-out batch must + // never read as not started once any action has begun. + onActionOutcome?.({ status: 'pending' }) try { tab = session.requireAutomationTab() epoch = navigationEpoch(tab.view.webContents) + url = tab.view.webContents.getURL() snapshotValid = driverScopeState().snapshotTabId === tab.id result = await executeToolInner( action.tool, @@ -3483,6 +3486,7 @@ async function executeToolInner( if ( session.automationTab()?.id !== tab.id || navigationEpoch(tab.view.webContents) !== epoch || + tab.view.webContents.getURL() !== url || (snapshotValid && driverScopeState().snapshotTabId !== tab.id) ) { return stopped( diff --git a/apps/sim/lib/mothership/generated/tool-catalog-v1.ts b/apps/sim/lib/mothership/generated/tool-catalog-v1.ts index 30392883f50..6b3546050e0 100644 --- a/apps/sim/lib/mothership/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/mothership/generated/tool-catalog-v1.ts @@ -401,22 +401,20 @@ export const BrowserBatch: ToolCatalogEntry = { route: 'client', mode: 'async', parameters: { - type: 'object', + additionalProperties: false, properties: { actions: { - type: 'array', description: "Ordered list of 2–8 actions. Each names one action tool and gives exactly that tool's parameters in args, without observe.", items: { - type: 'object', + additionalProperties: false, properties: { args: { - type: 'object', description: 'That tool\'s own parameters, for example {"elementId": 12} for browser_click or {"key": "Enter"} for browser_press_key.', + type: 'object', }, tool: { - type: 'string', description: 'The action tool to run.', enum: [ 'browser_click', @@ -429,10 +427,15 @@ export const BrowserBatch: ToolCatalogEntry = { 'browser_set_checked', 'browser_hover', ], + type: 'string', }, }, required: ['tool', 'args'], + type: 'object', }, + maxItems: 8, + minItems: 2, + type: 'array', }, observe: { type: 'object', @@ -448,6 +451,7 @@ export const BrowserBatch: ToolCatalogEntry = { }, }, required: ['actions'], + type: 'object', }, resultSchema: { type: 'object', @@ -475,6 +479,12 @@ export const BrowserBatch: ToolCatalogEntry = { required: ['index', 'tool', 'result'], }, }, + stoppedBy: { + type: 'string', + description: + 'failure when an action failed; page-change when an action changed the page and the rest were skipped.', + enum: ['failure', 'page-change'], + }, stoppedIndex: { type: 'number', description: diff --git a/apps/sim/lib/mothership/generated/tool-schemas-v1.ts b/apps/sim/lib/mothership/generated/tool-schemas-v1.ts index 081c0ac672b..c17380c3a42 100644 --- a/apps/sim/lib/mothership/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/mothership/generated/tool-schemas-v1.ts @@ -83,22 +83,20 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, browser_batch: { parameters: { - type: 'object', + additionalProperties: false, properties: { actions: { - type: 'array', description: "Ordered list of 2–8 actions. Each names one action tool and gives exactly that tool's parameters in args, without observe.", items: { - type: 'object', + additionalProperties: false, properties: { args: { - type: 'object', description: 'That tool\'s own parameters, for example {"elementId": 12} for browser_click or {"key": "Enter"} for browser_press_key.', + type: 'object', }, tool: { - type: 'string', description: 'The action tool to run.', enum: [ 'browser_click', @@ -111,10 +109,15 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { 'browser_set_checked', 'browser_hover', ], + type: 'string', }, }, required: ['tool', 'args'], + type: 'object', }, + maxItems: 8, + minItems: 2, + type: 'array', }, observe: { type: 'object', @@ -130,6 +133,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, }, required: ['actions'], + type: 'object', }, resultSchema: { type: 'object', @@ -167,6 +171,12 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['index', 'tool', 'result'], }, }, + stoppedBy: { + type: 'string', + description: + 'failure when an action failed; page-change when an action changed the page and the rest were skipped.', + enum: ['failure', 'page-change'], + }, stoppedIndex: { type: 'number', description: