diff --git a/apps/desktop/src/main/browser-agent/cdp.test.ts b/apps/desktop/src/main/browser-agent/cdp.test.ts index a579b13855f..4b24e3484df 100644 --- a/apps/desktop/src/main/browser-agent/cdp.test.ts +++ b/apps/desktop/src/main/browser-agent/cdp.test.ts @@ -15,10 +15,13 @@ import { captureScreenshot, clickAt, consumeAgentContextMenu, + dragPointer, ensureInstrumented, evaluateInIsolatedFrame, insertText, + movePointer, PRIMARY_CLICK, + pointerPathSteps, releaseFileInput, resolveFileInput, setColorScheme, @@ -258,6 +261,122 @@ describe('browser-agent CDP instrumentation', () => { ]) }) + it('stops a pointer route as soon as it is aborted', async () => { + const contents = new WebContentsView().webContents + const moves = () => + vi + .mocked(contents.debugger.sendCommand) + .mock.calls.filter(([method]) => String(method).startsWith('Input.dispatchMouseEvent')) + const aborted = new AbortController() + aborted.abort() + await expect( + movePointer(contents, { via: [], durationMs: null }, { x: 5, y: 5 }, aborted.signal) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(moves()).toHaveLength(0) + + vi.useFakeTimers() + try { + const controller = new AbortController() + const route = movePointer( + contents, + { via: [{ x: 0, y: 0 }], durationMs: 5_000 }, + { x: 500, y: 0 }, + controller.signal + ) + const settled = expect(route).rejects.toMatchObject({ name: 'AbortError' }) + await vi.advanceTimersByTimeAsync(100) + const sentBeforeAbort = moves().length + controller.abort() + await settled + expect(moves()).toHaveLength(sentBeforeAbort) + } finally { + vi.useRealTimers() + } + }) + + it('sends nothing for an already-aborted drag and cancels one aborted while it settles', async () => { + const contents = new WebContentsView().webContents + const mouse = () => + vi + .mocked(contents.debugger.sendCommand) + .mock.calls.filter(([method]) => method === 'Input.dispatchMouseEvent') + .map(([, params]) => toRecord(params).type) + const aborted = new AbortController() + aborted.abort() + await expect( + dragPointer(contents, { x: 0, y: 0 }, { x: 50, y: 0 }, undefined, aborted.signal) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(contents.debugger.sendCommand).not.toHaveBeenCalled() + + vi.useFakeTimers() + try { + const controller = new AbortController() + const drag = dragPointer( + contents, + { x: 0, y: 0 }, + { x: 50, y: 0 }, + undefined, + controller.signal + ) + const settled = expect(drag).rejects.toMatchObject({ name: 'AbortError' }) + // The default route takes 13 moves 20 ms apart, then a 120 ms settle hold. + await vi.advanceTimersByTimeAsync(300) + controller.abort() + await settled + expect(mouse().at(-1)).toBe('mouseReleased') + } finally { + vi.useRealTimers() + } + }) + + it('releases the button when a timed drag is aborted mid-route', async () => { + const contents = new WebContentsView().webContents + const types = () => + vi + .mocked(contents.debugger.sendCommand) + .mock.calls.filter(([method]) => method === 'Input.dispatchMouseEvent') + .map(([, params]) => toRecord(params).type) + vi.useFakeTimers() + try { + const controller = new AbortController() + const drag = dragPointer( + contents, + { x: 0, y: 0 }, + { x: 500, y: 0 }, + { via: [], durationMs: 5_000 }, + controller.signal + ) + const settled = expect(drag).rejects.toMatchObject({ name: 'AbortError' }) + await vi.advanceTimersByTimeAsync(100) + controller.abort() + await settled + expect(types().at(-1)).toBe('mouseReleased') + expect(types().filter((type) => type === 'mouseMoved').length).toBeLessThan(20) + } finally { + vi.useRealTimers() + } + }) + + it('keeps the default drag pace and lands exactly on every via point', () => { + const direct = pointerPathSteps({ x: 0, y: 0 }, { via: [], durationMs: null }, { x: 120, y: 0 }) + expect(direct.stepDelayMs).toBe(20) + expect(direct.points).toHaveLength(12) + expect(direct.points[0]).toEqual({ x: 10, y: 0 }) + expect(direct.points[11]).toEqual({ x: 120, y: 0 }) + + const routed = pointerPathSteps( + { x: 0, y: 0 }, + { via: [{ x: 100, y: 0 }], durationMs: 800 }, + { x: 100, y: 300 } + ) + expect(routed.points).toContainEqual({ x: 100, y: 0 }) + expect(routed.points[routed.points.length - 1]).toEqual({ x: 100, y: 300 }) + expect(routed.points.length * routed.stepDelayMs).toBeCloseTo(800) + // The longer second segment gets about three times the steps of the first. + const corner = routed.points.findIndex((point) => point.x === 100 && point.y === 0) + expect(routed.points.length - 1 - corner).toBeGreaterThan(corner * 2) + }) + it('holds the button down for holdMs before releasing it', async () => { const contents = new WebContentsView().webContents const types = () => diff --git a/apps/desktop/src/main/browser-agent/cdp.ts b/apps/desktop/src/main/browser-agent/cdp.ts index e5b2b4143d1..6bf9f9fbc2d 100644 --- a/apps/desktop/src/main/browser-agent/cdp.ts +++ b/apps/desktop/src/main/browser-agent/cdp.ts @@ -11,7 +11,7 @@ import type { BrowserTheme } from '@sim/browser-protocol' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { interruptibleSleep, sleep } from '@sim/utils/helpers' +import { interruptibleSleep } from '@sim/utils/helpers' import { isRecordLike } from '@sim/utils/object' import type { NativeImage, WebContents, WebFrameMain } from 'electron' @@ -948,6 +948,91 @@ export async function moveMouse(contents: WebContents, x: number, y: number): Pr }) } +/** A point in CSS viewport pixels. */ +export interface ViewportPoint { + x: number + y: number +} + +/** The points a pointer passes through on its way, and how long the whole movement takes. */ +export interface PointerPath { + via: ViewportPoint[] + /** Total movement time; null keeps the default brisk pace. */ + durationMs: number | null +} + +export const DIRECT_PATH: PointerPath = { via: [], durationMs: null } + +const DEFAULT_PATH_STEPS = 12 +const DEFAULT_PATH_STEP_MS = 20 +/** One display frame, so a timed movement looks continuous to animation-driven pages. */ +const TIMED_PATH_STEP_MS = 16 + +/** + * The moves from `from` through `path.via` to `to`, and the pause after each. A direct path + * keeps the default 12 moves 20 ms apart. A path with via points or a duration moves one frame + * at a time, shares the steps across segments by length, and lands exactly on every via point. + */ +export function pointerPathSteps( + from: ViewportPoint, + path: PointerPath, + to: ViewportPoint +): { points: ViewportPoint[]; stepDelayMs: number } { + const lerp = (a: ViewportPoint, b: ViewportPoint, t: number): ViewportPoint => ({ + x: a.x + (b.x - a.x) * t, + y: a.y + (b.y - a.y) * t, + }) + if (path.via.length === 0 && path.durationMs === null) { + const points: ViewportPoint[] = [] + for (let step = 1; step <= DEFAULT_PATH_STEPS; step++) { + points.push(lerp(from, to, step / DEFAULT_PATH_STEPS)) + } + return { points, stepDelayMs: DEFAULT_PATH_STEP_MS } + } + const vertices = [from, ...path.via, to] + const durationMs = path.durationMs ?? DEFAULT_PATH_STEPS * DEFAULT_PATH_STEP_MS + const lengths = vertices + .slice(1) + .map((vertex, index) => Math.hypot(vertex.x - vertices[index].x, vertex.y - vertices[index].y)) + const totalLength = lengths.reduce((sum, length) => sum + length, 0) + const totalSteps = Math.max(lengths.length, Math.round(durationMs / TIMED_PATH_STEP_MS)) + const points: ViewportPoint[] = [] + lengths.forEach((length, index) => { + const share = totalLength > 0 ? length / totalLength : 1 / lengths.length + const segmentSteps = Math.max(1, Math.round(totalSteps * share)) + for (let step = 1; step <= segmentSteps; step++) { + points.push(lerp(vertices[index], vertices[index + 1], step / segmentSteps)) + } + }) + return { points, stepDelayMs: durationMs / points.length } +} + +/** + * Moves the pointer with no button pressed through `path.via` to `to`, starting from the first + * via point (or `to` itself for a direct move), for hover effects that follow the cursor. + */ +export async function movePointer( + contents: WebContents, + path: PointerPath, + to: ViewportPoint, + signal?: AbortSignal +): Promise { + const [start, ...rest] = [...path.via, to] + signal?.throwIfAborted() + await moveMouse(contents, start.x, start.y) + if (rest.length === 0) return + const { points, stepDelayMs } = pointerPathSteps( + start, + { via: rest.slice(0, -1), durationMs: path.durationMs }, + to + ) + for (const point of points) { + await interruptibleSleep(stepDelayMs, signal) + signal?.throwIfAborted() + await moveMouse(contents, point.x, point.y) + } +} + /** One trusted click gesture: which button, how many presses, and held modifiers. */ export interface PointerClick { button: 'left' | 'right' | 'middle' @@ -1075,11 +1160,13 @@ export async function clickAt( */ export async function dragPointer( contents: WebContents, - from: { x: number; y: number }, - to: { x: number; y: number }, - steps = 12, - stepDelayMs = 20 + from: ViewportPoint, + to: ViewportPoint, + path: PointerPath = DIRECT_PATH, + signal?: AbortSignal ): Promise<{ nativeDragIntercepted: boolean }> { + signal?.throwIfAborted() + const { points, stepDelayMs } = pointerPathSteps(from, path, to) const interception: DragInterception = { intercepted: false, data: null } dragInterceptionsByContents.set(contents, interception) let interceptEnabled = false @@ -1124,17 +1211,19 @@ export async function dragPointer( }) // Small first nudge so libraries with a start threshold (commonly 3-8px) // register the drag before the pointer sweeps across the page. - await dragMove(from.x + Math.sign(to.x - from.x || 1) * 4, from.y + 2) - await sleep(stepDelayMs) - const stepCount = Math.max(2, steps) - for (let step = 1; step <= stepCount; step++) { - const progress = step / stepCount - await dragMove(from.x + (to.x - from.x) * progress, from.y + (to.y - from.y) * progress) - await sleep(stepDelayMs) + const heading = points[0] ?? to + await dragMove(from.x + Math.sign(heading.x - from.x || 1) * 4, from.y + 2) + await interruptibleSleep(stepDelayMs, signal) + for (const point of points) { + signal?.throwIfAborted() + await dragMove(point.x, point.y) + await interruptibleSleep(stepDelayMs, signal) } + signal?.throwIfAborted() // Hold over the target so drop zones running enter/over animations settle // before the release lands. - await sleep(120) + await interruptibleSleep(120, signal) + signal?.throwIfAborted() if (interception.intercepted && interception.data) { await sendInput(contents, 'Input.dispatchDragEvent', { type: 'drop', diff --git a/apps/desktop/src/main/browser-agent/driver.test.ts b/apps/desktop/src/main/browser-agent/driver.test.ts index 84290af3c75..fa402b3ef61 100644 --- a/apps/desktop/src/main/browser-agent/driver.test.ts +++ b/apps/desktop/src/main/browser-agent/driver.test.ts @@ -3683,6 +3683,33 @@ describe('credential protection', () => { expect(retried).toMatchObject({ ok: true, result: { dispatched: true } }) }) + it('names the overlay and nested controls a refused click can use next', async () => { + const contents = await openPage() + respondWith(contents, { + clickElement: { + error: 'obstructed', + blocker: 'We use cookies', + blockerControls: [{ id: 4, name: 'Accept all' }], + }, + }) + const covered = await driver.executeTool('chat-test', 'browser_click', { elementId: 0 }) + respondWith(contents, { + clickElement: { error: 'nested-control', blocker: 'Delete channel', controlId: 3 }, + }) + const nested = await driver.executeTool('chat-test', 'browser_click', { elementId: 0 }) + + expect(covered).toEqual({ + ok: false, + error: expect.stringContaining( + '[ref=4] "Accept all". Dismiss it with one of those, then retry the same id.' + ), + }) + expect(nested).toEqual({ + ok: false, + error: expect.stringContaining('Delete channel [ref=3]'), + }) + }) + it('invalidates element ids when the active tab changes', async () => { await openPage() await driver.executeTool('chat-test', 'browser_open_tab', {}) @@ -3756,6 +3783,36 @@ describe('credential protection', () => { expect(cdpCalls(contents, 'Input.dispatchKeyEvent').length).toBeGreaterThan(0) }) + it('aborts a coordinate hover when a cross-document navigation lands mid-flight', async () => { + const contents = await openPage() + let navigated = false + vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => { + if (isPageCall(expression, 'describePointTarget')) return Promise.resolve({ found: true }) + if (isPageCall(expression, 'readActiveElementState')) return Promise.resolve({}) + if (isPageCall(expression, 'readPageActionState')) { + if (!navigated) { + navigated = true + emitContentsEvent(contents, 'did-navigate') + } + return Promise.resolve({ + url: 'https://example.com/login', + title: 'Example', + focus: 'body', + mutationRevision: 0, + dialogs: [], + scroll: [0], + }) + } + return Promise.resolve(undefined) + }) + + const result = await driver.executeTool('chat-test', 'browser_hover', { x: 40, y: 50 }) + + expect(result.ok).toBe(false) + expect(result.error).toMatch(/active tab or page changed/) + expect(cdpCalls(contents, 'Input.dispatchMouseEvent')).toHaveLength(0) + }) + it('aborts a keypress when a cross-document navigation lands mid-flight', async () => { const contents = await openPage() let navigated = false @@ -4596,6 +4653,107 @@ describe('credential protection', () => { expect(cdpCalls(contents, 'Input.setInterceptDrags').length).toBeGreaterThan(0) }) + it('drags through via points at the requested pace', async () => { + const contents = await openPage() + respondWith(contents, { + describePointTarget: { found: true, element: 'div "Card"' }, + readActiveElementState: {}, + readPageActionState: {}, + }) + + const result = await driver.executeTool('chat-test', 'browser_drag', { + fromX: 40, + fromY: 50, + toX: 200, + toY: 260, + via: [{ x: 300, y: 50 }], + durationMs: 320, + }) + + expect(result).toMatchObject({ ok: true, result: { dispatched: true } }) + const moves = cdpCalls(contents, 'Input.dispatchMouseEvent') + .map(([, event]) => event as { type?: string; x?: number; y?: number; buttons?: number }) + .filter((event) => event.type === 'mouseMoved' && event.buttons === 1) + expect(moves.some((event) => event.x === 300 && event.y === 50)).toBe(true) + expect(moves.length).toBeGreaterThanOrEqual(20) + }) + + it('moves the pointer along a coordinate hover path with no button pressed', async () => { + const contents = await openPage() + respondWith(contents, { + describePointTarget: { found: true, element: 'canvas "Field"' }, + readActiveElementState: {}, + readPageActionState: {}, + }) + + const result = await driver.executeTool('chat-test', 'browser_hover', { + x: 400, + y: 300, + via: [ + { x: 100, y: 300 }, + { x: 250, y: 150 }, + ], + }) + + expect(result).toMatchObject({ ok: true, result: { hovered: true, x: 400, y: 300 } }) + const events = cdpCalls(contents, 'Input.dispatchMouseEvent').map( + ([, event]) => event as { type?: string; x?: number; y?: number; button?: string } + ) + expect(events.every((event) => event.type === 'mouseMoved' && event.button === 'none')).toBe( + true + ) + expect(events[0]).toMatchObject({ x: 100, y: 300 }) + expect(events.some((event) => event.x === 250 && event.y === 150)).toBe(true) + expect(events[events.length - 1]).toMatchObject({ x: 400, y: 300 }) + }) + + it('rejects pointer paths where they cannot apply', async () => { + await openPage() + + const elementHover = await driver.executeTool('chat-test', 'browser_hover', { + elementId: 0, + via: [{ x: 1, y: 2 }], + }) + const unroutedPace = await driver.executeTool('chat-test', 'browser_hover', { + x: 10, + y: 10, + durationMs: 800, + }) + const badPoint = await driver.executeTool('chat-test', 'browser_hover', { + x: 10, + y: 10, + via: [{ x: 'left', y: 2 }], + }) + const tooLong = await driver.executeTool('chat-test', 'browser_drag', { + fromX: 1, + fromY: 1, + toX: 50, + toY: 50, + durationMs: 10_001, + }) + const batched = await driver.executeTool('chat-test', 'browser_batch', { + actions: [ + { tool: 'browser_click', args: { elementId: 0 } }, + { tool: 'browser_hover', args: { x: 5, y: 5, durationMs: 500 } }, + ], + }) + + expect(elementHover).toMatchObject({ + ok: false, + error: expect.stringContaining('coordinate hover'), + }) + expect(unroutedPace).toMatchObject({ + ok: false, + error: expect.stringContaining('paces a hover route'), + }) + expect(badPoint).toMatchObject({ ok: false, error: expect.stringContaining('via point') }) + expect(tooLong).toMatchObject({ ok: false, error: expect.stringContaining('durationMs') }) + expect(batched).toMatchObject({ + ok: false, + error: expect.stringContaining('timed pointer path'), + }) + }) + it('drags from a snapshot element to a coordinate target', async () => { const contents = await openPage() respondWith(contents, { diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index 7bf2eff96f5..4e2dbe9c126 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -33,7 +33,7 @@ import type { BrowserDownloadsState, BrowserToolbarCommand } from '@sim/desktop- import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' -import { isRecordLike, omit, toRecord } from '@sim/utils/object' +import { isRecordLike, omit, toArray, toRecord } from '@sim/utils/object' import type { BrowserWindow, MenuItemConstructorOptions, WebContents, WebFrameMain } from 'electron' import { Menu } from 'electron' import * as cdp from '@/main/browser-agent/cdp' @@ -191,6 +191,11 @@ function parseBatchActions(params: Record): BatchAction[] { if (num(action.args, 'holdMs')) { throw new ToolError(`Batch action ${index} cannot press and hold; run it as its own click.`) } + if ('via' in action.args || 'durationMs' in action.args) { + throw new ToolError( + `Batch action ${index} cannot follow a timed pointer path; run it as its own action.` + ) + } return { tool: action.tool, args: action.args } }) } @@ -1157,6 +1162,44 @@ function pointerClick(params: Record): cdp.PointerClick { } } +const NOTHING_RENDERED_AT_POINT = + "Nothing is rendered at that point. Coordinates are CSS pixels in the current viewport — when reading them off a browser_screenshot, follow its caption's X/Y coordinate mapping and crop origin." + +const MAX_POINTER_PATH_POINTS = 20 +/** Longest timed pointer movement; well inside the pointer tools' watchdog. */ +const MAX_POINTER_PATH_MS = 10_000 + +/** The optional pointer route shared by `browser_drag` and coordinate `browser_hover`. */ +function pointerPath(params: Record): cdp.PointerPath { + const rawVia = params.via ?? [] + if (!Array.isArray(rawVia) || rawVia.length > MAX_POINTER_PATH_POINTS) { + throw new ToolError( + `via must be a list of at most ${MAX_POINTER_PATH_POINTS} {x, y} viewport points.` + ) + } + const via = rawVia.map((point) => { + const x = isRecordLike(point) ? point.x : undefined + const y = isRecordLike(point) ? point.y : undefined + if (typeof x !== 'number' || typeof y !== 'number' || !Number.isFinite(x + y)) { + throw new ToolError('Each via point must be {x, y} in CSS viewport pixels.') + } + return { x, y } + }) + const durationMs = params.durationMs ?? null + if ( + durationMs !== null && + (typeof durationMs !== 'number' || + !Number.isInteger(durationMs) || + durationMs < 0 || + durationMs > MAX_POINTER_PATH_MS) + ) { + throw new ToolError( + `durationMs must be a whole number of milliseconds from 0 to ${MAX_POINTER_PATH_MS}.` + ) + } + return { via, durationMs } +} + /** The exact client tool call executing now; the app binds file transfers to it. */ function requireActiveToolCallId(): string { const toolCallId = driverScopeState().activeToolCallId @@ -1440,14 +1483,23 @@ function unwrapPageResult(result: unknown): unknown { } if (code === 'obstructed') { const blocker = String((result as { blocker?: unknown }).blocker || 'another element') + const controls = toArray((result as { blockerControls?: unknown }).blockerControls) + .map(toRecord) + .filter((control) => typeof control.id === 'number') + .map((control) => `[ref=${control.id}] "${String(control.name ?? '')}"`) throw new ToolError( - `That element is covered by ${blocker}. Close or move the overlay, then take a fresh browser_snapshot.` + controls.length > 0 + ? `That element is covered by ${blocker}. The overlay's controls in the current snapshot: ${controls.join(', ')}. Dismiss it with one of those, then retry the same id.` + : `That element is covered by ${blocker}. Close or move the overlay, then take a fresh browser_snapshot.` ) } if (code === 'nested-control') { const blocker = String((result as { blocker?: unknown }).blocker || 'a nested control') + const controlId = (result as { controlId?: unknown }).controlId throw new ToolError( - `The point you targeted lands on ${blocker}, which is its own control inside that element — nothing is covering it. Take a fresh browser_snapshot and use the id of the control you actually want.` + typeof controlId === 'number' + ? `The point you targeted lands on ${blocker} [ref=${controlId}], which is its own control inside that element — nothing is covering it. Use id ${controlId} if that is the control you want; otherwise target the element through a part that is not a separate control.` + : `The point you targeted lands on ${blocker}, which is its own control inside that element — nothing is covering it. Take a fresh browser_snapshot and use the id of the control you actually want.` ) } if (code === 'suggestions-open') { @@ -4456,6 +4508,59 @@ async function executeToolInner( case 'browser_hover': { const contents = session.requireAutomationTab().view.webContents + if (params.elementId === undefined) { + const hoverNavigationEpoch = navigationEpoch(contents) + const x = requireNum(params, 'x') + const y = requireNum(params, 'y') + const path = pointerPath(params) + if (path.via.length === 0 && path.durationMs !== null) { + throw new ToolError( + 'durationMs paces a hover route; pass via points for the pointer to travel through.' + ) + } + assertCurrentExecution() + assertActiveContents(contents, hoverNavigationEpoch) + const pointTarget = unwrapPageResult( + await execInPage(contents, describePointTarget, [x, y], false, executionDeadline) + ) + if (!isRecordLike(pointTarget) || pointTarget.found !== true) { + throw new ToolError(NOTHING_RENDERED_AT_POINT) + } + const beforePage = await pageActionState(contents, true) + const beforeElement = await activeElementState(contents) + assertCurrentExecution() + assertActiveContents(contents, hoverNavigationEpoch) + await cdp.movePointer(contents, path, { x, y }, signal) + await sleep(150) + const afterElement = await activeElementState(contents) + const afterPage = await pageActionState(contents) + const observation = pageEffect(beforePage, afterPage, beforeElement, afterElement) + const effectObserved = + observation.effect.urlChanged || + observation.effect.dialogChanged || + observation.effect.popupChanged + return { + hovered: true, + x, + y, + trusted: true, + effect: observation.effect, + possibleEffectObserved: observation.possibleEffectObserved, + effectObserved, + ...(!effectObserved + ? { + note: observation.possibleEffectObserved + ? 'The page changed while the pointer moved; confirm the intended effect with browser_snapshot or browser_screenshot.' + : 'No tooltip, menu, or other strong hover effect was observed.', + } + : {}), + } + } + if ('via' in params || 'durationMs' in params) { + throw new ToolError( + 'via and durationMs apply to a coordinate hover; pass x and y instead of elementId.' + ) + } const elementId = requireNum(params, 'elementId') const target = pageTargetForElement(contents, elementId) const targetFrame = frameExecutionTarget(target, contents) @@ -4636,9 +4741,7 @@ async function executeToolInner( await execInPage(contents, describePointTarget, [x, y], false, executionDeadline) ) if (!isRecordLike(pointTarget) || pointTarget.found !== true) { - throw new ToolError( - "Nothing is rendered at that point. Coordinates are CSS pixels in the current viewport — when reading them off a browser_screenshot, follow its caption's X/Y coordinate mapping and crop origin." - ) + throw new ToolError(NOTHING_RENDERED_AT_POINT) } if (pointTarget.fileInput === true) { throw new ToolError(FILE_INPUT_REFUSAL) @@ -4829,6 +4932,7 @@ async function executeToolInner( const draggedTab = session.requireAutomationTab() const contents = draggedTab.view.webContents const dragNavigationEpoch = navigationEpoch(contents) + const path = pointerPath(params) const resolveEndpoint = async ( which: 'from' | 'to' @@ -4894,7 +4998,7 @@ async function executeToolInner( assertActiveContents(contents) const from = await resolveEndpoint('from') const to = await resolveEndpoint('to') - if (Math.abs(from.x - to.x) < 1 && Math.abs(from.y - to.y) < 1) { + if (path.via.length === 0 && Math.abs(from.x - to.x) < 1 && Math.abs(from.y - to.y) < 1) { throw new ToolError('The drag source and target are the same point; nothing to drag.') } const beforePage = await pageActionState(contents, true) @@ -4903,7 +5007,7 @@ async function executeToolInner( assertActiveContents(contents, dragNavigationEpoch) let interception: { nativeDragIntercepted: boolean } try { - interception = await cdp.dragPointer(contents, from, to) + interception = await cdp.dragPointer(contents, from, to, path, signal) } catch (error) { throw new ToolError( `Native drag dispatch failed (${getErrorMessage(error)}). The pointer may have been mid-drag; take a fresh snapshot to see the page's current state before retrying.` diff --git a/apps/desktop/src/main/browser-agent/page-functions.test.ts b/apps/desktop/src/main/browser-agent/page-functions.test.ts index a8e61266783..3a10100b097 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.test.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.test.ts @@ -427,6 +427,7 @@ describe('combobox typing surfaces', () => { expect(focusElementForTyping(0)).toEqual({ error: 'obstructed', blocker: 'Mondu', + blockerControls: [], }) expect(clickElement(0, false)).toMatchObject({ @@ -448,6 +449,7 @@ describe('combobox typing surfaces', () => { expect(focusElementForTyping(0)).toEqual({ error: 'obstructed', blocker: 'Mondu', + blockerControls: [], }) }) @@ -643,6 +645,52 @@ describe('collectSnapshot', () => { expect(clicked).toBe(true) }) + it('names the covering dialog controls so the agent can dismiss it and retry', () => { + document.body.innerHTML = ` + +

We use cookies

+ +
` + for (const element of Array.from(document.body.querySelectorAll('*'))) visible(element) + const outline = outlineOf(collectSnapshot()) + const ref = refFor(outline, 'Checkout') + const notice = document.querySelector('p') as HTMLParagraphElement + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => notice }) + + expect(clickElement(ref, false)).toEqual({ + error: 'obstructed', + blocker: 'We use cookies', + blockerControls: [ + { id: refFor(outline, 'Accept all'), name: 'Accept all' }, + { id: refFor(outline, 'Close notice'), name: 'Close notice' }, + ], + }) + }) + + it('names the controls of an overlay built from a web component', () => { + document.body.innerHTML = ` + + ` + const host = document.getElementById('banner') as HTMLDivElement + const shadow = host.attachShadow({ mode: 'open' }) + shadow.innerHTML = '

We use cookies

' + for (const element of [ + ...Array.from(document.body.querySelectorAll('*')), + ...Array.from(shadow.querySelectorAll('*')), + ]) { + visible(element) + } + const outline = outlineOf(collectSnapshot()) + const ref = refFor(outline, 'Checkout') + const notice = shadow.querySelector('p') as HTMLParagraphElement + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => notice }) + + expect(clickElement(ref, false)).toMatchObject({ + error: 'obstructed', + blockerControls: [{ id: refFor(outline, 'Close banner'), name: 'Close banner' }], + }) + }) + it('refuses a coordinate click when an overlay owns every hit point', () => { document.body.innerHTML = '
' @@ -658,6 +706,7 @@ describe('collectSnapshot', () => { expect(clickElement(ref, false)).toEqual({ error: 'obstructed', blocker: 'Confirmation overlay', + blockerControls: [], }) }) @@ -682,6 +731,7 @@ describe('collectSnapshot', () => { expect(clickElement(ref, false)).toEqual({ error: 'nested-control', blocker: 'Delete channel', + controlId: 1, }) }) diff --git a/apps/desktop/src/main/browser-agent/page-functions.ts b/apps/desktop/src/main/browser-agent/page-functions.ts index bc2089e70ab..3cfa32e0a8a 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.ts @@ -47,6 +47,10 @@ declare global { __simAgentShownElements?: WeakSet /** Installed by {@link installPageHelpers} before every page function; cached per call. */ __simAgentIsExemptModal: (element: Element) => boolean + /** Installed by {@link installPageHelpers}: the element's id in the current snapshot. */ + __simAgentRefOf: (element: Element | null) => number | null + /** Installed by {@link installPageHelpers}: the snapshot controls of the overlay a blocker belongs to. */ + __simAgentOverlayControls: (blocker: Element | null) => Array<{ id: number; name: string }> /** Why the last __simAgentResolveElement call returned null — read by the * shared stale-error producers so a refusal names its cause instead of * the blanket "the page changed". Cleared on every successful resolve. */ @@ -84,6 +88,78 @@ export function installPageHelpers(): void { } return modal === element } + window.__simAgentRefOf = (element: Element | null): number | null => { + const index = element ? (window.__simAgentElements ?? []).indexOf(element) : -1 + return index >= 0 ? index : null + } + /** + * A refusal that names the overlay's own controls lets the agent dismiss it and retry the + * same id without another snapshot. Only a real overlay qualifies — a dialog, a modal, or a + * fixed or sticky layer — so an ordinary element in the way never lists unrelated page controls. + */ + window.__simAgentOverlayControls = (blocker: Element | null) => { + /** The parent across shadow boundaries, so overlays built from web components qualify. */ + const composedParent = (element: Element): Element | null => { + if (element.parentElement) return element.parentElement + const root = element.getRootNode() + return 'host' in root ? (root.host as Element) : null + } + const tagOf = (element: Element): string => String(element.tagName).toUpperCase() + let overlay: Element | null = null + for (let current = blocker; current && !overlay; current = composedParent(current)) { + const role = current.getAttribute('role') + const position = current.ownerDocument.defaultView?.getComputedStyle(current).position + if ( + role === 'dialog' || + role === 'alertdialog' || + current.getAttribute('aria-modal') === 'true' || + tagOf(current) === 'DIALOG' || + position === 'fixed' || + position === 'sticky' + ) { + overlay = current + } + } + if (!overlay) return [] + const controls: Array<{ id: number; name: string }> = [] + const registry = window.__simAgentElements ?? [] + for (let id = 0; id < registry.length && controls.length < 4; id++) { + const element = registry[id] + if (!element) continue + let inOverlay = false + for ( + let current: Element | null = element; + current && !inOverlay; + current = composedParent(current) + ) { + inOverlay = current === overlay + } + if (!inOverlay) continue + const role = element.getAttribute('role') + const tag = tagOf(element) + if ( + tag !== 'BUTTON' && + tag !== 'A' && + role !== 'button' && + role !== 'link' && + !(tag === 'INPUT' && ['button', 'submit'].includes((element as HTMLInputElement).type)) + ) { + continue + } + const name = ( + element.getAttribute('aria-label') || + (element as HTMLElement).innerText || + (element as HTMLInputElement).value || + element.getAttribute('title') || + '' + ) + .replace(/\s+/g, ' ') + .trim() + .slice(0, 60) + controls.push({ id, name }) + } + return controls + } function findExemptModal(doc: Document): Element | null { const rendered: Array<{ modal: Element; hidden: Element[] }> = [] for (const modal of Array.from(doc.querySelectorAll('[aria-modal="true"], dialog[open]'))) { @@ -1361,9 +1437,21 @@ export function clickElement( } } if (nested) { - return { error: 'nested-control', blocker: blockerLabel(blocker) } + let control = blocker + while (control && control !== el && !isIndependentInteractive(control)) { + control = composedParent(control) + } + return { + error: 'nested-control', + blocker: blockerLabel(blocker), + controlId: control && control !== el ? window.__simAgentRefOf(control) : null, + } + } + return { + error: 'obstructed', + blocker: blockerLabel(blocker), + blockerControls: window.__simAgentOverlayControls(blocker), } - return { error: 'obstructed', blocker: blockerLabel(blocker) } } let pageX = clientX @@ -1395,7 +1483,11 @@ export function clickElement( if (!scrollToTarget) { return clickElement(id, dispatchSynthetic, focusForKeyboard, allowDisabled, true) } - return { error: 'obstructed', blocker: blockerLabel(parentHit) } + return { + error: 'obstructed', + blocker: blockerLabel(parentHit), + blockerControls: window.__simAgentOverlayControls(parentHit), + } } } ownerView = frame.ownerDocument.defaultView @@ -1775,7 +1867,11 @@ export function focusElementForTyping(id: number, moveFocus = true): unknown { } } if (!chosenPoint) { - return { error: 'obstructed', blocker: blockerLabel(firstBlocker) } + return { + error: 'obstructed', + blocker: blockerLabel(firstBlocker), + blockerControls: window.__simAgentOverlayControls(firstBlocker), + } } return { diff --git a/apps/desktop/src/main/browser-agent/panel.ts b/apps/desktop/src/main/browser-agent/panel.ts index 93ad3538169..242e41378a6 100644 --- a/apps/desktop/src/main/browser-agent/panel.ts +++ b/apps/desktop/src/main/browser-agent/panel.ts @@ -103,6 +103,14 @@ interface OccludablePanelFrame { } /** Geometry of the painted frame that is currently allowed to replace the view. */ let occludableFrame: OccludablePanelFrame | null = null +/** + * Each chat's agent tab, kept in a window even while no panel shows it. A view that is in no + * window has no compositor surface: CDP input on it never acknowledges and captures never + * complete. Parked invisibly, it renders exactly like the hidden panel view does. + */ +const agentViews = new Map() +/** Agent views parked invisibly, with the window each one is parked in. */ +const parkedViews = new Map() /** The host window whose `resize` currently drives {@link layout}, if any. */ let resizeBoundWindow: BrowserWindow | null = null /** Captures nothing, so one instance serves every window it is bound to. */ @@ -114,6 +122,8 @@ export function initPanel(panelHost: PanelHost): void { // by the next session: a stale owner window that rejects legitimate panel // updates, a lease timer polling for a panel that no longer exists, a // `lastApplied*` value that dedupes away the first layout of the new one. + agentViews.clear() + for (const view of [...parkedViews.keys()]) unparkView(view) detachAttachedView() resetOcclusion() if (leaseTimer !== null) { @@ -145,6 +155,12 @@ export function activatePanelScope(scopeId: string | null): void { /** Retags an active pending scope without tearing down the compositor. */ export function migratePanelScope(fromScopeId: string, toScopeId: string): void { + const agentView = agentViews.get(fromScopeId) + if (agentView) { + agentViews.delete(fromScopeId) + if (!agentViews.has(toScopeId)) agentViews.set(toScopeId, agentView) + else if (!isAgentView(agentView)) unparkView(agentView) + } if (activePanelScopeId !== fromScopeId) return activePanelScopeId = toScopeId panelCaptureGeneration++ @@ -299,12 +315,19 @@ function detachAttachedView(): void { if (!view || !win) return try { if (win.isDestroyed() || view.webContents.isDestroyed()) return + // An agent view stays in the main window, which outlives any secondary window. + if (isAgentView(view) && win === host.getMainWindow()) { + view.setVisible(false) + parkedViews.set(view, win) + return + } win.contentView.removeChildView(view) } catch (error) { logger.warn('Could not detach embedded browser view', { error: getErrorMessage(error, 'unknown'), }) } + if (isAgentView(view)) parkAgentViews() } /** Reveals the native view and invalidates every frame captured for its old state. */ @@ -342,9 +365,71 @@ function hideAttachedView(): void { * tab must not pull the visible tab out of the window. */ export function detachIfAttached(view: WebContentsView): void { + for (const [scopeId, agentView] of agentViews) { + if (agentView === view) agentViews.delete(scopeId) + } if (attachedView === view) { detachAttachedView() } + unparkView(view) +} + +/** + * Registers the tab a chat's agent drives, so it stays composited while the panel shows + * another tab or no panel is open. Null releases the chat's previous agent tab. + */ +export function setAgentView(scopeId: string, view: WebContentsView | null): void { + const previous = agentViews.get(scopeId) + if (view) agentViews.set(scopeId, view) + else agentViews.delete(scopeId) + if (previous && previous !== view && !isAgentView(previous)) unparkView(previous) + parkAgentViews() +} + +function isAgentView(view: WebContentsView): boolean { + for (const agentView of agentViews.values()) { + if (agentView === view) return true + } + return false +} + +/** Parks every agent view that no window holds, keeping renderer focus where it was. */ +function parkAgentViews(): void { + const win = host.getMainWindow() + if (!win || win.isDestroyed()) return + for (const view of agentViews.values()) { + if (view === attachedView || view.webContents.isDestroyed()) continue + const parkedIn = parkedViews.get(view) + if (parkedIn && !parkedIn.isDestroyed()) continue + // addChildView hands keyboard focus to the parked view; give it back to whichever of the + // Sim renderer or the visible browser page held it. + const focused = [win.webContents, attachedView?.webContents].find( + (contents) => contents && !contents.isDestroyed() && contents.isFocused() + ) + try { + view.setVisible(false) + win.contentView.addChildView(view) + parkedViews.set(view, win) + } catch (error) { + logger.warn('Could not park the agent browser view', { + error: getErrorMessage(error, 'unknown'), + }) + } + focused?.focus() + } +} + +function unparkView(view: WebContentsView): void { + const win = parkedViews.get(view) + if (!win) return + parkedViews.delete(view) + try { + if (!win.isDestroyed() && !view.webContents.isDestroyed()) win.contentView.removeChildView(view) + } catch (error) { + logger.warn('Could not unpark the agent browser view', { + error: getErrorMessage(error, 'unknown'), + }) + } } /** @@ -393,22 +478,31 @@ export function layout(): void { } if (!showing || !active || !win || panelBounds === null) { hideAttachedView() + parkAgentViews() return } if (attachedView !== active.view) { - // addChildView hands keyboard focus to the newly attached WebContentsView. - // Agent-driven attaches happen while the user may be typing in the chat - // composer, so if the renderer held focus before the attach, give it back — - // automation drives the page over CDP and never needs OS focus. - const rendererHadFocus = !win.webContents.isDestroyed() && win.webContents.isFocused() - win.contentView.addChildView(active.view) + // A parked agent view already sits in this window; adopting it in place avoids the + // blank repaint a remove-and-add costs. + const parkedIn = parkedViews.get(active.view) + parkedViews.delete(active.view) + if (parkedIn !== win) { + if (parkedIn && !parkedIn.isDestroyed()) parkedIn.contentView.removeChildView(active.view) + // addChildView hands keyboard focus to the newly attached WebContentsView. + // Agent-driven attaches happen while the user may be typing in the chat + // composer, so if the renderer held focus before the attach, give it back — + // automation drives the page over CDP and never needs OS focus. + const rendererHadFocus = !win.webContents.isDestroyed() && win.webContents.isFocused() + win.contentView.addChildView(active.view) + if (rendererHadFocus) { + win.webContents.focus() + } + } hostedWindow = win attachedView = active.view - if (rendererHadFocus) { - win.webContents.focus() - } } + parkAgentViews() bindHostResize(win) const zoom = win.webContents.getZoomFactor() const [contentWidth, contentHeight] = win.getContentSize() diff --git a/apps/desktop/src/main/browser-agent/session.test.ts b/apps/desktop/src/main/browser-agent/session.test.ts index 02a4f52bb10..58bb9854c12 100644 --- a/apps/desktop/src/main/browser-agent/session.test.ts +++ b/apps/desktop/src/main/browser-agent/session.test.ts @@ -522,6 +522,20 @@ describe('browser-agent session', () => { expect(session.migrateBrowserScope('chat-real', 'occupied')).toBe(false) }) + it('carries the agent tab registration through a scope migration', () => { + const first = session.withBrowserScope('pending:workspace', () => session.ensureTab()) + expect(session.migrateBrowserScope('pending:workspace', 'chat-real')).toBe(true) + const removeChildView = ( + win as unknown as { contentView: { removeChildView: ReturnType } } + ).contentView.removeChildView + removeChildView.mockClear() + + // The migrated chat moves its agent to a new tab: the first stops being an agent tab. + session.withBrowserScope('chat-real', () => session.addAutomationTab()) + + expect(removeChildView).toHaveBeenCalledWith(first.view) + }) + it('retains a migrated provisional alias until the durable scope is disposed', () => { const tab = session.withBrowserScope('pending:workspace', () => session.ensureTab()) expect(session.migrateBrowserScope('pending:workspace', 'chat-real')).toBe(true) @@ -2398,6 +2412,7 @@ describe('browser-agent session', () => { it('gives background automation a viewport without taking panel ownership', () => { const tab = session.withBrowserScope('background-chat', () => session.ensureTab()) + const view = tab.view as unknown as MockView expect(tab.view.setBounds).toHaveBeenCalledWith({ x: 0, @@ -2405,7 +2420,10 @@ describe('browser-agent session', () => { width: 1180, height: 850, }) - expect(win.contentView.addChildView).not.toHaveBeenCalledWith(tab.view) + // Parked invisibly so input and captures on it complete; it never shows. + expect(win.contentView.addChildView).toHaveBeenCalledWith(tab.view) + expect(view.setVisible).toHaveBeenCalledWith(false) + expect(view.setVisible).not.toHaveBeenCalledWith(true) expect(session.getActiveBrowserScopeId()).toBe('chat-test') }) @@ -2422,11 +2440,16 @@ describe('browser-agent session', () => { const content = (win as unknown as { contentView: { addChildView: ReturnType } }) .contentView - // No bounds yet: the view is not attached to the window. - expect(content.addChildView).not.toHaveBeenCalledWith(tab.view) + // No bounds yet: the agent's view is parked invisibly, never shown. + expect(content.addChildView).toHaveBeenCalledWith(tab.view) + expect(view.setVisible).toHaveBeenCalledWith(false) + expect(view.setVisible).not.toHaveBeenCalledWith(true) + // Bounds arrive: the parked view is adopted in place, not re-added. + content.addChildView.mockClear() panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) - expect(content.addChildView).toHaveBeenCalledWith(tab.view) + expect(content.addChildView).not.toHaveBeenCalled() + expect(view.setVisible).toHaveBeenLastCalledWith(true) expect(view.setBounds).toHaveBeenCalledWith({ x: 100, y: 50, width: 800, height: 600 }) // Panel hidden: the view stops painting but stays attached. Detaching @@ -2467,11 +2490,81 @@ describe('browser-agent session', () => { const second = session.addTab() - // Hiding keeps a view attached, but a tab switch still has to detach: - // two native views stacked in the window would composite over each other. - expect(content.removeChildView).toHaveBeenCalledWith(first.view) + // A tab switch hides the previous view so two native views never composite over + // each other; the agent still drives the first tab, so it stays parked, not removed. + expect(content.removeChildView).not.toHaveBeenCalledWith(first.view) + expect(first.view.setVisible).toHaveBeenLastCalledWith(false) expect(content.addChildView).toHaveBeenCalledWith(second.view) expect(second.view.webContents.invalidate).toHaveBeenCalledOnce() + + // A tab the agent does not drive is removed when another tab takes over. + content.removeChildView.mockClear() + session.switchTab(first.id) + expect(content.removeChildView).toHaveBeenCalledWith(second.view) + }) + + it('gives keyboard focus back to the visible page when it parks the agent tab', () => { + const visibleTab = session.ensureTab() + panel.setPanelBounds({ x: 0, y: 0, width: 800, height: 600 }) + const visibleContents = visibleTab.view.webContents as unknown as MockView['webContents'] + visibleContents.isFocused.mockReturnValue(true) + vi.mocked(visibleContents.focus).mockClear() + + session.addAutomationTab() + + expect(visibleContents.focus).toHaveBeenCalled() + }) + + it('parks an agent tab in the main window when a secondary window stops showing it', () => { + const otherWindow = mainWindowMock() as unknown as { + contentView: { + addChildView: ReturnType + removeChildView: ReturnType + } + } + const agentTab = session.ensureTab() + panel.setPanelBounds( + { x: 0, y: 0, width: 800, height: 600 }, + otherWindow as unknown as BrowserWindow + ) + expect(otherWindow.contentView.addChildView).toHaveBeenCalledWith(agentTab.view) + vi.mocked(win.contentView.addChildView).mockClear() + + session.addTab() + + expect(otherWindow.contentView.removeChildView).toHaveBeenCalledWith(agentTab.view) + expect(win.contentView.addChildView).toHaveBeenCalledWith(agentTab.view) + }) + + it('keeps the agent tab composited while the user views another tab, and releases it on close', () => { + const agentTab = session.ensureTab() + panel.setPanelBounds({ x: 0, y: 0, width: 800, height: 600 }) + const content = ( + win as unknown as { + contentView: { + addChildView: ReturnType + removeChildView: ReturnType + } + } + ).contentView + const userTab = session.addTab() + const agentView = agentTab.view as unknown as MockView + + // The user's tab is visible; the agent's tab is parked, still in the window. + expect(content.removeChildView).not.toHaveBeenCalledWith(agentTab.view) + expect(agentView.setVisible).toHaveBeenLastCalledWith(false) + + // Switching back adopts the parked view in place. + content.addChildView.mockClear() + session.switchTab(agentTab.id) + expect(content.addChildView).not.toHaveBeenCalledWith(agentTab.view) + expect(agentView.setVisible).toHaveBeenLastCalledWith(true) + + // Closing the agent tab while the user views another tab removes its parked view. + session.switchTab(userTab.id) + content.removeChildView.mockClear() + session.closeTab(agentTab.id) + expect(content.removeChildView).toHaveBeenCalledWith(agentTab.view) }) // The measured report is the sole writer of bounds. A main-process diff --git a/apps/desktop/src/main/browser-agent/session.ts b/apps/desktop/src/main/browser-agent/session.ts index b709510f0e1..d1643b12dbe 100644 --- a/apps/desktop/src/main/browser-agent/session.ts +++ b/apps/desktop/src/main/browser-agent/session.ts @@ -62,6 +62,7 @@ import { migratePanelScope, panelUpdateAllowed, panelWindow, + setAgentView, } from '@/main/browser-agent/panel' import { agentAppOrigin, @@ -2155,7 +2156,7 @@ function createFreshTabView(appSession: BrowserAppSession | undefined): WebConte preload: join(__dirname, 'browser-preload.cjs'), // Throttled by default: a hidden tab should idle. The one exception is // the active tab while a tool waits on it, applied explicitly by - // applyActiveTabThrottling — never blanket across every tab. + // applyAutomationTabPolicy — never blanket across every tab. backgroundThrottling: true, spellcheck: false, // The default every origin this tab visits starts at; a per-origin zoom @@ -2476,7 +2477,7 @@ export function hasSession(): boolean { export function setAutomationActive(active: boolean): void { if (currentScope.automationActive === active) return currentScope.automationActive = active - applyActiveTabThrottling() + applyAutomationTabPolicy() events?.onTabsChanged() } @@ -2486,26 +2487,27 @@ export function setAutomationNeedsAttention(needsAttention: boolean): void { events?.onTabsChanged() } -/** - * Unthrottles the active tab while automation is active, and throttles every - * other tab. Call after anything that changes which tab is active, so the - * exemption follows the active tab rather than being stranded on the old one. - */ /** * Re-applies the tab throttling policy after a caller temporarily suspended it * (the panel's reveal pulse). Exempts the automation-active tab exactly as the * internal policy does. */ export function reassertTabThrottling(): void { - applyActiveTabThrottling() + applyAutomationTabPolicy() } -function applyActiveTabThrottling(): void { +/** + * Unthrottles the automation tab while automation is active, throttles every other tab, and + * keeps the automation tab composited while no panel shows it. Call after anything that changes + * which tab the agent drives, so neither follows a stale tab. + */ +function applyAutomationTabPolicy(): void { for (const tab of tabs) { if (tab.view.webContents.isDestroyed()) continue const exempt = currentScope.automationActive && tab.id === currentScope.automationTabId tab.view.webContents.setBackgroundThrottling(!exempt) } + setAgentView(getBrowserScopeId(), automationTab()?.view ?? null) } /** A closed target must not transfer its activity marker to a replacement tab. */ @@ -2643,7 +2645,7 @@ function addTabInternal({ revokeTabMediaPermissions(previousActiveTab, false) } currentScope.activeTabId = tab.id - applyActiveTabThrottling() + applyAutomationTabPolicy() if (!currentScope.restoring) layout() if (transferBrowserFocus) currentScope.focusedBrowserTabId = tab.id if (notify && !currentScope.restoring) events?.onActiveTabChanged(tab.view.webContents) @@ -2923,7 +2925,7 @@ export function tabForNavigation( detachIfAttached(tab.view) tab.view = view contents.close() - applyActiveTabThrottling() + applyAutomationTabPolicy() layout() events?.onActiveTabChanged(view.webContents) return view.webContents @@ -3058,13 +3060,13 @@ export function restoreBrowserSession(): void { state.lastPersistedSnapshot = previousState.lastPersistedSnapshot if (previousDownloads) browserDownloadsByScope.set(scopeId, previousDownloads) else browserDownloadsByScope.delete(scopeId) - applyActiveTabThrottling() + applyAutomationTabPolicy() throw error } finally { state.restoring = false } - applyActiveTabThrottling() + applyAutomationTabPolicy() const restoredActive = restoredLoads.find(({ tab }) => tab.id === state.activeTabId) if (restoredActive) { pendingForegroundTabRestores.push( @@ -3099,7 +3101,7 @@ export function addAutomationTab(url?: string, popup?: PopupWindowOptions): Agen restoreBrowserSession() const tab = addTabInternal({ activate: false, notify: false, url, popup }) currentScope.automationTabId = tab.id - applyActiveTabThrottling() + applyAutomationTabPolicy() persistBrowserSession() events?.onTabsChanged() return tab @@ -3113,7 +3115,7 @@ export function ensureAutomationTab(): AgentTab { tab = activeTab() if (tab) { currentScope.automationTabId = tab.id - applyActiveTabThrottling() + applyAutomationTabPolicy() events?.onTabsChanged() return tab } @@ -3174,7 +3176,7 @@ export function switchTab(tabId: string, { claim = true }: { claim?: boolean } = promotePendingTabRestore(tab) // Visible selection does not move the automation exemption; the user may // inspect another page while a tool continues in its background tab. - applyActiveTabThrottling() + applyAutomationTabPolicy() layout() if (transferBrowserFocus) currentScope.focusedBrowserTabId = tab.id persistBrowserSession() @@ -3189,7 +3191,7 @@ export function switchAutomationTab(tabId: string): AgentTab { const tab = tabs.find((entry) => entry.id === tabId) if (!tab) throw new SessionError(`No tab with id ${tabId} — call browser_list_tabs.`) currentScope.automationTabId = tab.id - applyActiveTabThrottling() + applyAutomationTabPolicy() events?.onTabsChanged() return tab } @@ -3271,7 +3273,7 @@ function removeTab( const opener = tabs.find((entry) => entry.id === tab.openerTabId) currentScope.automationTabId = opener?.id ?? (adoptNeighborForAgent ? ((tabs[index] ?? tabs[index - 1])?.id ?? null) : null) - applyActiveTabThrottling() + applyAutomationTabPolicy() } if (transferBrowserFocus) currentScope.focusedBrowserTabId = currentScope.activeTabId persistBrowserSession() diff --git a/apps/sim/lib/mothership/generated/tool-catalog-v1.ts b/apps/sim/lib/mothership/generated/tool-catalog-v1.ts index c93d196cc8e..f5dadaedb4b 100644 --- a/apps/sim/lib/mothership/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/mothership/generated/tool-catalog-v1.ts @@ -830,6 +830,13 @@ export const BrowserDrag: ToolCatalogEntry = { parameters: { type: 'object', properties: { + durationMs: { + type: 'integer', + description: + 'Optional total movement time in milliseconds, 0 to 10000. Use it for slow, smooth movement; the default is a brisk move.', + minimum: 0, + maximum: 10000, + }, fromElementId: { type: 'number', description: 'Drag source element id from the latest snapshot. Alternative to fromX/fromY.', @@ -849,6 +856,17 @@ export const BrowserDrag: ToolCatalogEntry = { description: 'Drop target X in CSS viewport pixels (paired with toY).', }, toY: { type: 'number', description: 'Drop target Y in CSS viewport pixels.' }, + via: { + type: 'array', + description: + 'Optional viewport points in CSS pixels, at most 20, that the pointer passes through in order before the end point — to route around obstacles, sweep a surface, or trace a shape.', + items: { + type: 'object', + properties: { x: { type: 'number' }, y: { type: 'number' } }, + required: ['x', 'y'], + }, + maxItems: 20, + }, }, }, resultSchema: { @@ -1246,6 +1264,13 @@ export const BrowserHover: ToolCatalogEntry = { parameters: { type: 'object', properties: { + durationMs: { + type: 'integer', + description: + 'Optional total movement time in milliseconds, 0 to 10000. Use it for slow, smooth movement; the default is a brisk move.', + minimum: 0, + maximum: 10000, + }, elementId: { type: 'number', description: @@ -1263,8 +1288,24 @@ export const BrowserHover: ToolCatalogEntry = { }, }, }, + via: { + type: 'array', + description: + 'Optional viewport points in CSS pixels, at most 20, that the pointer passes through in order before the end point — to route around obstacles, sweep a surface, or trace a shape.', + items: { + type: 'object', + properties: { x: { type: 'number' }, y: { type: 'number' } }, + required: ['x', 'y'], + }, + maxItems: 20, + }, + x: { + type: 'number', + description: + 'Hover at this X in CSS viewport pixels instead of an element id (paired with y), for canvas and map surfaces.', + }, + y: { type: 'number', description: 'Hover Y in CSS viewport pixels.' }, }, - required: ['elementId'], }, resultSchema: { type: 'object', diff --git a/apps/sim/lib/mothership/generated/tool-schemas-v1.ts b/apps/sim/lib/mothership/generated/tool-schemas-v1.ts index ae16ddb0653..7269af03ca5 100644 --- a/apps/sim/lib/mothership/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/mothership/generated/tool-schemas-v1.ts @@ -576,6 +576,13 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { parameters: { type: 'object', properties: { + durationMs: { + type: 'integer', + description: + 'Optional total movement time in milliseconds, 0 to 10000. Use it for slow, smooth movement; the default is a brisk move.', + minimum: 0, + maximum: 10000, + }, fromElementId: { type: 'number', description: @@ -602,6 +609,24 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { type: 'number', description: 'Drop target Y in CSS viewport pixels.', }, + via: { + type: 'array', + description: + 'Optional viewport points in CSS pixels, at most 20, that the pointer passes through in order before the end point — to route around obstacles, sweep a surface, or trace a shape.', + items: { + type: 'object', + properties: { + x: { + type: 'number', + }, + y: { + type: 'number', + }, + }, + required: ['x', 'y'], + }, + maxItems: 20, + }, }, }, resultSchema: { @@ -1075,6 +1100,13 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { parameters: { type: 'object', properties: { + durationMs: { + type: 'integer', + description: + 'Optional total movement time in milliseconds, 0 to 10000. Use it for slow, smooth movement; the default is a brisk move.', + minimum: 0, + maximum: 10000, + }, elementId: { type: 'number', description: @@ -1092,8 +1124,34 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, }, }, + via: { + type: 'array', + description: + 'Optional viewport points in CSS pixels, at most 20, that the pointer passes through in order before the end point — to route around obstacles, sweep a surface, or trace a shape.', + items: { + type: 'object', + properties: { + x: { + type: 'number', + }, + y: { + type: 'number', + }, + }, + required: ['x', 'y'], + }, + maxItems: 20, + }, + x: { + type: 'number', + description: + 'Hover at this X in CSS viewport pixels instead of an element id (paired with y), for canvas and map surfaces.', + }, + y: { + type: 'number', + description: 'Hover Y in CSS viewport pixels.', + }, }, - required: ['elementId'], }, resultSchema: { type: 'object',