Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore(mcp): cap image size (#37856)
Pavel Feldman1532d51db567
diff --git a/packages/playwright-core/src/server/utils/comparators.ts b/packages/playwright-core/src/server/utils/comparators.tsindex 747abb2ecf..77426a2d29 100644--- a/packages/playwright-core/src/server/utils/comparators.ts+++ b/packages/playwright-core/src/server/utils/comparators.ts@@ -21,6 +21,9 @@ import pixelmatch from '../../third_party/pixelmatch';import { jpegjs } from '../../utilsBundle';import { colors, diff } from '../../utilsBundle';import { PNG } from '../../utilsBundle';+import { padImageToSize } from './imageUtils';++import type { ImageData } from './imageUtils';export type ImageComparatorOptions = { threshold?: number, maxDiffPixels?: number, maxDiffPixelRatio?: number, comparator?: string };export type ComparatorResult = { diff?: Buffer; errorMessage: string; } | null;@@ -48,8 +51,6 @@ export function compareBuffersOrStrings(actualBuffer: Buffer | string, expectedBreturn null;}-type ImageData = { width: number, height: number, data: Buffer };-function compareImages(mimeType: string, actualBuffer: Buffer | string, expectedBuffer: Buffer, options: ImageComparatorOptions = {}): ComparatorResult {if (!actualBuffer || !(actualBuffer instanceof Buffer))return { errorMessage: 'Actual result should be a Buffer.' };@@ -61,8 +62,8 @@ function compareImages(mimeType: string, actualBuffer: Buffer | string, expectedlet sizesMismatchError = '';if (expected.width !== actual.width || expected.height !== actual.height) {sizesMismatchError = `Expected an image ${expected.width}px by ${expected.height}px, received ${actual.width}px by ${actual.height}px. `;- actual = resizeImage(actual, size);- expected = resizeImage(expected, size);+ actual = padImageToSize(actual, size);+ expected = padImageToSize(expected, size);}const diff = new PNG({ width: size.width, height: size.height });let count;@@ -131,27 +132,3 @@ function compareText(actual: Buffer | string, expectedBuffer: Buffer): Comparatoconst errorMessage = coloredLines.join('\n');return { errorMessage };}--function resizeImage(image: ImageData, size: { width: number, height: number }): ImageData {- if (image.width === size.width && image.height === size.height)- return image;- const buffer = new Uint8Array(size.width * size.height * 4);- for (let y = 0; y < size.height; y++) {- for (let x = 0; x < size.width; x++) {- const to = (y * size.width + x) * 4;- if (y < image.height && x < image.width) {- const from = (y * image.width + x) * 4;- buffer[to] = image.data[from];- buffer[to + 1] = image.data[from + 1];- buffer[to + 2] = image.data[from + 2];- buffer[to + 3] = image.data[from + 3];- } else {- buffer[to] = 0;- buffer[to + 1] = 0;- buffer[to + 2] = 0;- buffer[to + 3] = 0;- }- }- }- return { data: Buffer.from(buffer), width: size.width, height: size.height };-}diff --git a/packages/playwright-core/src/server/utils/imageUtils.ts b/packages/playwright-core/src/server/utils/imageUtils.tsnew file mode 100644index 0000000000..1801851115--- /dev/null+++ b/packages/playwright-core/src/server/utils/imageUtils.ts@@ -0,0 +1,135 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++export type ImageData = { width: number, height: number, data: Buffer };++export function padImageToSize(image: ImageData, size: { width: number, height: number }): ImageData {+ if (image.width === size.width && image.height === size.height)+ return image;+ const buffer = new Uint8Array(size.width * size.height * 4);+ for (let y = 0; y < size.height; y++) {+ for (let x = 0; x < size.width; x++) {+ const to = (y * size.width + x) * 4;+ if (y < image.height && x < image.width) {+ const from = (y * image.width + x) * 4;+ buffer[to] = image.data[from];+ buffer[to + 1] = image.data[from + 1];+ buffer[to + 2] = image.data[from + 2];+ buffer[to + 3] = image.data[from + 3];+ } else {+ buffer[to] = 0;+ buffer[to + 1] = 0;+ buffer[to + 2] = 0;+ buffer[to + 3] = 0;+ }+ }+ }+ return { data: Buffer.from(buffer), width: size.width, height: size.height };+}++export function scaleImageToSize(image: ImageData, size: { width: number; height: number }): ImageData {+ const { data: src, width: w1, height: h1 } = image;+ const w2 = size.width | 0, h2 = size.height | 0;+ if (w1 === w2 && h1 === h2)+ return image;++ const clamp = (v: number, lo: number, hi: number) => (v < lo ? lo : v > hi ? hi : v);++ // Catmull–Rom weights+ const weights = (t: number, o: Float32Array) => {+ const t2 = t * t, t3 = t2 * t;+ o[0] = -0.5 * t + 1.0 * t2 - 0.5 * t3;+ o[1] = 1.0 - 2.5 * t2 + 1.5 * t3;+ o[2] = 0.5 * t + 2.0 * t2 - 1.5 * t3;+ o[3] = -0.5 * t2 + 0.5 * t3;+ };++ const srcRowStride = w1 * 4;+ const dstRowStride = w2 * 4;++ // Precompute X: indices, weights, and byte offsets (idx*4)+ const xIdx = new Int32Array(w2 * 4);+ const xOff = new Int32Array(w2 * 4); // byte offsets = xIdx*4+ const xW = new Float32Array(w2 * 4);+ const wx = new Float32Array(4);+ const xScale = w1 / w2;+ for (let x = 0; x < w2; x++) {+ const sx = (x + 0.5) * xScale - 0.5;+ const sxi = Math.floor(sx);+ const t = sx - sxi;+ weights(t, wx);+ const b = x * 4;+ const i0 = clamp(sxi - 1, 0, w1 - 1);+ const i1 = clamp(sxi + 0, 0, w1 - 1);+ const i2 = clamp(sxi + 1, 0, w1 - 1);+ const i3 = clamp(sxi + 2, 0, w1 - 1);+ xIdx[b + 0] = i0; xIdx[b + 1] = i1; xIdx[b + 2] = i2; xIdx[b + 3] = i3;+ xOff[b + 0] = i0 << 2; xOff[b + 1] = i1 << 2; xOff[b + 2] = i2 << 2; xOff[b + 3] = i3 << 2;+ xW[b + 0] = wx[0]; xW[b + 1] = wx[1]; xW[b + 2] = wx[2]; xW[b + 3] = wx[3];+ }++ // Precompute Y: indices, weights, and row-base byte offsets (y*rowStride)+ const yIdx = new Int32Array(h2 * 4);+ const yRow = new Int32Array(h2 * 4); // row base in bytes+ const yW = new Float32Array(h2 * 4);+ const wy = new Float32Array(4);+ const yScale = h1 / h2;+ for (let y = 0; y < h2; y++) {+ const sy = (y + 0.5) * yScale - 0.5;+ const syi = Math.floor(sy);+ const t = sy - syi;+ weights(t, wy);+ const b = y * 4;+ const j0 = clamp(syi - 1, 0, h1 - 1);+ const j1 = clamp(syi + 0, 0, h1 - 1);+ const j2 = clamp(syi + 1, 0, h1 - 1);+ const j3 = clamp(syi + 2, 0, h1 - 1);+ yIdx[b + 0] = j0; yIdx[b + 1] = j1; yIdx[b + 2] = j2; yIdx[b + 3] = j3;+ yRow[b + 0] = j0 * srcRowStride;+ yRow[b + 1] = j1 * srcRowStride;+ yRow[b + 2] = j2 * srcRowStride;+ yRow[b + 3] = j3 * srcRowStride;+ yW[b + 0] = wy[0]; yW[b + 1] = wy[1]; yW[b + 2] = wy[2]; yW[b + 3] = wy[3];+ }++ const dst = new Uint8Array(w2 * h2 * 4);++ for (let y = 0; y < h2; y++) {+ const yb = y * 4;+ const rb0 = yRow[yb + 0], rb1 = yRow[yb + 1], rb2 = yRow[yb + 2], rb3 = yRow[yb + 3];+ const wy0 = yW[yb + 0], wy1 = yW[yb + 1], wy2 = yW[yb + 2], wy3 = yW[yb + 3];+ const dstBase = y * dstRowStride;++ for (let x = 0; x < w2; x++) {+ const xb = x * 4;+ const xo0 = xOff[xb + 0], xo1 = xOff[xb + 1], xo2 = xOff[xb + 2], xo3 = xOff[xb + 3];+ const wx0 = xW[xb + 0], wx1 = xW[xb + 1], wx2 = xW[xb + 2], wx3 = xW[xb + 3];+ const di = dstBase + (x << 2);++ // unrolled RGBA+ for (let c = 0; c < 4; c++) {+ const r0 = src[rb0 + xo0 + c] * wx0 + src[rb0 + xo1 + c] * wx1 + src[rb0 + xo2 + c] * wx2 + src[rb0 + xo3 + c] * wx3;+ const r1 = src[rb1 + xo0 + c] * wx0 + src[rb1 + xo1 + c] * wx1 + src[rb1 + xo2 + c] * wx2 + src[rb1 + xo3 + c] * wx3;+ const r2 = src[rb2 + xo0 + c] * wx0 + src[rb2 + xo1 + c] * wx1 + src[rb2 + xo2 + c] * wx2 + src[rb2 + xo3 + c] * wx3;+ const r3 = src[rb3 + xo0 + c] * wx0 + src[rb3 + xo1 + c] * wx1 + src[rb3 + xo2 + c] * wx2 + src[rb3 + xo3 + c] * wx3;+ const v = r0 * wy0 + r1 * wy1 + r2 * wy2 + r3 * wy3;+ dst[di + c] = v < 0 ? 0 : v > 255 ? 255 : v | 0;+ }+ }+ }++ return { data: Buffer.from(dst), width: w2, height: h2 };+}diff --git a/packages/playwright-core/src/utils.ts b/packages/playwright-core/src/utils.tsindex db049937ce..78cd6eb6a2 100644--- a/packages/playwright-core/src/utils.ts+++ b/packages/playwright-core/src/utils.ts@@ -43,6 +43,7 @@ export * from './server/utils/expectUtils';export * from './server/utils/fileUtils';export * from './server/utils/hostPlatform';export * from './server/utils/httpServer';+export * from './server/utils/imageUtils';export * from './server/utils/network';export * from './server/utils/nodePlatform';export * from './server/utils/processLauncher';diff --git a/packages/playwright/src/mcp/browser/tools/screenshot.ts b/packages/playwright/src/mcp/browser/tools/screenshot.tsindex 18335b0b09..0b357a7dc2 100644--- a/packages/playwright/src/mcp/browser/tools/screenshot.ts+++ b/packages/playwright/src/mcp/browser/tools/screenshot.ts@@ -14,6 +14,11 @@* limitations under the License.*/+import fs from 'fs';++import { mkdirIfNeeded, scaleImageToSize } from 'playwright-core/lib/utils';+import { jpegjs, PNG } from 'playwright-core/lib/utilsBundle';+import { z } from '../../sdk/bundle';import { defineTabTool } from './tool';import * as javascript from '../codegen';@@ -51,7 +56,6 @@ const screenshot = defineTabTool({type: fileType,quality: fileType === 'png' ? undefined : 90,scale: 'css',- path: fileName,...(params.fullPage !== undefined && { fullPage: params.fullPage })};const isElementScreenshot = params.element && params.ref;@@ -68,19 +72,36 @@ const screenshot = defineTabTool({response.addCode(`await page.screenshot(${javascript.formatObject(options)});`);const buffer = ref ? await ref.locator.screenshot(options) : await tab.page.screenshot(options);++ await mkdirIfNeeded(fileName);+ await fs.promises.writeFile(fileName, buffer);+response.addResult(`Took the ${screenshotTarget} screenshot and saved it as ${fileName}`);- // https://github.com/microsoft/playwright-mcp/issues/817- // Never return large images to LLM, saving them to the file system is enough.- if (!params.fullPage) {- response.addImage({- contentType: fileType === 'png' ? 'image/png' : 'image/jpeg',- data: buffer- });- }+ response.addImage({+ contentType: fileType === 'png' ? 'image/png' : 'image/jpeg',+ data: scaleImageToFitMessage(buffer, fileType)+ });}});+export function scaleImageToFitMessage(buffer: Buffer, imageType: 'png' | 'jpeg'): Buffer {+ // https://docs.claude.com/en/docs/build-with-claude/vision#evaluate-image-size+ // Not more than 1.15 megapixel, linear size not more than 1568.++ const image = imageType === 'png' ? PNG.sync.read(buffer) : jpegjs.decode(buffer, { maxMemoryUsageInMB: 512 });+ const pixels = image.width * image.height;++ const shrink = Math.min(1568 / image.width, 1568 / image.height, Math.sqrt(1.15 * 1024 * 1024 / pixels));+ if (shrink > 1)+ return buffer;++ const width = image.width * shrink | 0;+ const height = image.height * shrink | 0;+ const scaledImage = scaleImageToSize(image, { width, height });+ return imageType === 'png' ? PNG.sync.write(scaledImage as any) : jpegjs.encode(scaledImage, 80).data;+}+export default [screenshot,];diff --git a/tests/mcp/screenshot.spec.ts b/tests/mcp/screenshot.spec.tsindex 50b2eaeaff..18dbc5dbbc 100644--- a/tests/mcp/screenshot.spec.ts+++ b/tests/mcp/screenshot.spec.ts@@ -17,6 +17,7 @@import fs from 'fs';import { test, expect } from './fixtures';+import { jpegjs, PNG } from 'packages/playwright-core/lib/utilsBundle';test('browser_take_screenshot (viewport)', async ({ startClient, server }, testInfo) => {const { client } = await startClient({@@ -264,11 +265,53 @@ test('browser_take_screenshot (fullPage: true)', async ({ startClient, server },{text: expect.stringContaining('fullPage: true'),type: 'text',- }+ },+ {+ data: expect.any(String),+ mimeType: 'image/png',+ type: 'image',+ },],});});+test('browser_take_screenshot size cap', async ({ startClient, server, mcpBrowser }, testInfo) => {+ test.skip(!['chrome', 'msedge', 'chromium'].includes(mcpBrowser ?? ''), 'Non-chrome has unusual full page size');++ const { client } = await startClient({+ config: { outputDir: testInfo.outputPath('output') },+ });++ const expectations = [+ { title: '2000x500', pageWidth: 2000, pageHeight: 500, expectedWidth: 1568, expectedHeight: 500 * 1568 / 2000 | 0 },+ { title: '2000x2000', pageWidth: 2000, pageHeight: 2000, expectedWidth: 1098, expectedHeight: 1098 },+ { title: '1280x800', pageWidth: 1280, pageHeight: 800, expectedWidth: 1280, expectedHeight: 800 },+ ];++ for (const expectation of expectations) {+ await test.step(expectation.title, async () => {+ server.setContent('/', `<body style="width: ${expectation.pageWidth}px; height: ${expectation.pageHeight}px; background: red; margin: 0;"></body>`, 'text/html');+ await client.callTool({ name: 'browser_navigate', arguments: { url: server.PREFIX } });++ const pngResult = await client.callTool({+ name: 'browser_take_screenshot',+ arguments: { fullPage: true },+ });+ const png = PNG.sync.read(Buffer.from(pngResult.content?.[1]?.data, 'base64'));+ expect(png.width).toBe(expectation.expectedWidth);+ expect(png.height).toBe(expectation.expectedHeight);++ const jpegResult = await client.callTool({+ name: 'browser_take_screenshot',+ arguments: { fullPage: true, type: 'jpeg' },+ });+ const jpeg = jpegjs.decode(Buffer.from(jpegResult.content?.[1]?.data, 'base64'));+ expect(jpeg.width).toBe(expectation.expectedWidth);+ expect(jpeg.height).toBe(expectation.expectedHeight);+ });+ }+});+test('browser_take_screenshot (fullPage with element should error)', async ({ startClient, server }, testInfo) => {const { client } = await startClient({config: { outputDir: testInfo.outputPath('output') },