From a782d589afc0025ab693c627e96a796c8cf14894 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 11 Sep 2026 08:32:01 -0700 Subject: [PATCH] feat(mcp): add --image-responses=only to return images without the text part Fixes: https://github.com/microsoft/playwright/issues/42554 --- .../src/tools/backend/context.ts | 2 +- .../src/tools/backend/response.ts | 21 ++++------ .../playwright-core/src/tools/mcp/config.d.ts | 5 ++- .../playwright-core/src/tools/mcp/config.ts | 4 +- .../playwright-core/src/tools/mcp/program.ts | 2 +- tests/mcp/screenshot.spec.ts | 38 +++++++++++++++++++ 6 files changed, 53 insertions(+), 19 deletions(-) diff --git a/packages/playwright-core/src/tools/backend/context.ts b/packages/playwright-core/src/tools/backend/context.ts index aefe85839427c..05b22548a3059 100644 --- a/packages/playwright-core/src/tools/backend/context.ts +++ b/packages/playwright-core/src/tools/backend/context.ts @@ -42,7 +42,7 @@ export type ContextConfig = { capabilities?: ToolCapability[]; codegen?: 'typescript' | 'python' | 'java' | 'csharp' | 'none'; console?: { level?: 'error' | 'warning' | 'info' | 'debug' }; - imageResponses?: 'allow' | 'omit'; + imageResponses?: 'allow' | 'omit' | 'only'; network?: { allowedOrigins?: string[]; blockedOrigins?: string[]; diff --git a/packages/playwright-core/src/tools/backend/response.ts b/packages/playwright-core/src/tools/backend/response.ts index b2c849419f524..1c34ee11ce257 100644 --- a/packages/playwright-core/src/tools/backend/response.ts +++ b/packages/playwright-core/src/tools/backend/response.ts @@ -171,10 +171,10 @@ export class Response { const rawSections = ['Error', 'Result', 'Snapshot'] as const; const sections = this._raw ? allSections.filter(section => rawSections.includes(section.title as typeof rawSections[number])) : allSections; + const isError = sections.some(section => section.isError); let serializedText: string; if (this._json) { const payload: Record = {}; - const isError = sections.some(section => section.isError); if (isError) payload.isError = true; for (const section of sections) { @@ -214,23 +214,18 @@ export class Response { serializedText = text.join('\n'); } - const content: (TextContent | ImageContent)[] = [ - { - type: 'text', - text: sanitizeUnicode(this._context.redactSecrets(serializedText)), - } + const imageResponses = this._context.config.imageResponses ?? 'allow'; + const images: ImageContent[] = imageResponses === 'omit' ? [] : this._imageResults.map(imageResult => ({ type: 'image', data: imageResult.data.toString('base64'), mimeType: `image/${imageResult.imageType}` })); + const imagesOnly = imageResponses === 'only' && images.length > 0 && !isError; + const content: (TextContent | ImageContent)[] = imagesOnly ? images : [ + { type: 'text', text: sanitizeUnicode(this._context.redactSecrets(serializedText)) }, + ...images, ]; - // Image attachments. - if (this._context.config.imageResponses !== 'omit') { - for (const imageResult of this._imageResults) - content.push({ type: 'image', data: imageResult.data.toString('base64'), mimeType: `image/${imageResult.imageType}` }); - } - return { content, ...(this._isClose ? { isClose: true } : {}), - ...(sections.some(section => section.isError) ? { isError: true } : {}), + ...(isError ? { isError: true } : {}), }; } diff --git a/packages/playwright-core/src/tools/mcp/config.d.ts b/packages/playwright-core/src/tools/mcp/config.d.ts index b446dbd4241f0..4ffe96ecbfc3f 100644 --- a/packages/playwright-core/src/tools/mcp/config.d.ts +++ b/packages/playwright-core/src/tools/mcp/config.d.ts @@ -225,9 +225,10 @@ export type Config = { }; /** - * Whether to send image responses to the client. Can be "allow", "omit", or "auto". Defaults to "auto", which sends images if the client can display them. + * Whether to send image responses to the client. Can be "allow", "omit", or "only". Defaults to "allow". + * With "only", a response that carries an image consists of the image parts alone, without the text part. */ - imageResponses?: 'allow' | 'omit'; + imageResponses?: 'allow' | 'omit' | 'only'; snapshot?: { /** diff --git a/packages/playwright-core/src/tools/mcp/config.ts b/packages/playwright-core/src/tools/mcp/config.ts index adf45552336ff..b8e3577a85056 100644 --- a/packages/playwright-core/src/tools/mcp/config.ts +++ b/packages/playwright-core/src/tools/mcp/config.ts @@ -57,7 +57,7 @@ export type CLIOptions = { initScript?: string[]; initPage?: string[]; isolated?: boolean; - imageResponses?: 'allow' | 'omit'; + imageResponses?: 'allow' | 'omit' | 'only'; mobile?: boolean; sandbox?: boolean; outputDir?: string; @@ -439,7 +439,7 @@ export function configFromEnv(env?: NodeJS.ProcessEnv): Config & { configFile?: options.initScript = [initScript]; options.isolated = envToBoolean(e.PLAYWRIGHT_MCP_ISOLATED); if (e.PLAYWRIGHT_MCP_IMAGE_RESPONSES) - options.imageResponses = enumParser<'allow' | 'omit'>('--image-responses', ['allow', 'omit'], e.PLAYWRIGHT_MCP_IMAGE_RESPONSES); + options.imageResponses = enumParser<'allow' | 'omit' | 'only'>('--image-responses', ['allow', 'omit', 'only'], e.PLAYWRIGHT_MCP_IMAGE_RESPONSES); options.mobile = envToBoolean(e.PLAYWRIGHT_MCP_MOBILE); options.sandbox = envToBoolean(e.PLAYWRIGHT_MCP_SANDBOX); options.outputDir = envToString(e.PLAYWRIGHT_MCP_OUTPUT_DIR); diff --git a/packages/playwright-core/src/tools/mcp/program.ts b/packages/playwright-core/src/tools/mcp/program.ts index 476282b1df868..258920a3001d6 100644 --- a/packages/playwright-core/src/tools/mcp/program.ts +++ b/packages/playwright-core/src/tools/mcp/program.ts @@ -58,7 +58,7 @@ export function decorateMCPCommand(command: Command) { .option('--init-page ', 'path to TypeScript file to evaluate on Playwright page object') .option('--init-script ', 'path to JavaScript file to add as an initialization script. The script will be evaluated in every page before any of the page\'s scripts. Can be specified multiple times.') .option('--isolated', 'keep the browser profile in memory, do not save it to disk.') - .option('--image-responses ', 'whether to send image responses to the client. Can be "allow" or "omit", Defaults to "allow".', enumParser.bind(null, '--image-responses', ['allow', 'omit'])) + .option('--image-responses ', 'whether to send image responses to the client. Can be "allow", "omit" or "only". With "only", a response that carries an image consists of the image parts alone, without the text part. Defaults to "allow".', enumParser.bind(null, '--image-responses', ['allow', 'omit', 'only'])) .option('--no-sandbox', 'disable the sandbox for all process types that are normally sandboxed.') .option('--output-dir ', 'path to the directory for automatically named output files, for example a screenshot taken without an explicit file name. Files with an explicit name are resolved against the workspace root instead and are not affected by this option.') .option('--output-max-size ', 'Threshold for evicting old output files, in bytes.', numberParser) diff --git a/tests/mcp/screenshot.spec.ts b/tests/mcp/screenshot.spec.ts index b9f6ad7d2e69f..ee7692432d8db 100644 --- a/tests/mcp/screenshot.spec.ts +++ b/tests/mcp/screenshot.spec.ts @@ -303,6 +303,44 @@ test('browser_take_screenshot (imageResponses=omit)', async ({ startClient, serv }); }); +test('browser_take_screenshot (imageResponses=only)', async ({ startClient, server }, testInfo) => { + const outputDir = testInfo.outputPath('output'); + const { client } = await startClient({ + args: ['--image-responses=only'], + config: { outputDir }, + }); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.HELLO_WORLD }, + }); + + expect(await client.callTool({ + name: 'browser_take_screenshot', + })).toEqual({ + content: [ + { + data: expect.any(String), + mimeType: 'image/png', + type: 'image', + }, + ], + }); + expect(fs.readdirSync(outputDir).filter(f => f.endsWith('.png'))).toHaveLength(1); + + expect(await client.callTool({ + name: 'browser_take_screenshot', + arguments: { filename: 'screenshot.png' }, + })).toEqual({ + content: [ + { + text: expect.stringContaining('[Screenshot of viewport](./screenshot.png)'), + type: 'text', + }, + ], + }); +}); + test('browser_take_screenshot (fullPage: true)', async ({ startClient, server }, testInfo) => { const { client } = await startClient({ config: { outputDir: testInfo.outputPath('output') },