Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: strip ansi from llm errors (#38840)

Simon Knotte7e0afe3d0dd
diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.ts
index 64be25de9d..6bb1b618f1 100644
--- a/packages/playwright-core/src/server/agent/context.ts
+++ b/packages/playwright-core/src/server/agent/context.ts
@@ -17,6 +17,7 @@
import { BrowserContext } from '../browserContext';
import { runAction } from './actionRunner';
import { generateCode } from './codegen';
+import { stripAnsiEscapes } from '../../utils/isomorphic/stringUtils';
import type { Request } from '../network';
import type * as loopTypes from '@lowire/loop';
@@ -137,7 +138,7 @@ export class Context {
const text: string[] = [];
if (error)
- text.push(`# Error\n${error.message}`);
+ text.push(`# Error\n${stripAnsiEscapes(error.message)}`);
else
text.push(`# Success`);
@@ -150,7 +151,7 @@ export class Context {
},
'dev.lowire/history': error ? [{
category: 'error',
- content: error.message,
+ content: stripAnsiEscapes(error.message),
}] : [],
},
isError: !!error,
diff --git a/packages/playwright-core/src/server/agent/tool.ts b/packages/playwright-core/src/server/agent/tool.ts
index 3f219dbad1..34e36d8d69 100644
--- a/packages/playwright-core/src/server/agent/tool.ts
+++ b/packages/playwright-core/src/server/agent/tool.ts
@@ -15,6 +15,7 @@
*/
import { z } from '../../mcpBundle';
+import { stripAnsiEscapes } from '../../utils/isomorphic/stringUtils';
import type zod from 'zod';
import type * as loopTypes from '@lowire/loop';
import type { Context } from './context';
@@ -110,7 +111,7 @@ export function toolsForLoop(progress: Progress, context: Context, toolDefinitio
return await tool.handle(progress, context, params.arguments);
} catch (error) {
return {
- content: [{ type: 'text', text: error.message }],
+ content: [{ type: 'text', text: stripAnsiEscapes(error.message) }],
isError: true,
};
}
diff --git a/packages/playwright-core/src/utils/isomorphic/stringUtils.ts b/packages/playwright-core/src/utils/isomorphic/stringUtils.ts
index 42c06e3bef..546c105e73 100644
--- a/packages/playwright-core/src/utils/isomorphic/stringUtils.ts
+++ b/packages/playwright-core/src/utils/isomorphic/stringUtils.ts
@@ -193,3 +193,8 @@ export function parseRegex(regex: string): RegExp {
const flags = regex.slice(lastSlash + 1);
return new RegExp(source, flags);
}
+
+export const ansiRegex = new RegExp('([\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~])))', 'g');
+export function stripAnsiEscapes(str: string): string {
+ return str.replace(ansiRegex, '');
+}
diff --git a/packages/playwright/src/util.ts b/packages/playwright/src/util.ts
index 6ab72eb1db..bb15feceec 100644
--- a/packages/playwright/src/util.ts
+++ b/packages/playwright/src/util.ts
@@ -414,7 +414,4 @@ export async function removeDirAndLogToConsole(dir: string) {
}
}
-export const ansiRegex = new RegExp('([\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~])))', 'g');
-export function stripAnsiEscapes(str: string): string {
- return str.replace(ansiRegex, '');
-}
+export { ansiRegex, stripAnsiEscapes } from 'playwright-core/lib/utils';
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
index 1375b7fd37..d4f1cf91e7 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
@@ -1,4 +1,31 @@
{
+ "0804fd14030c0e1f89df22422c4f085d1698f2a0": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I see the issue - the actual URL is `http://localhost:8907/page.html` but I was expecting just `/page.html`. The relative URL should match when resolved against the base URL. Let me try with the full URL:"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_expect_url",
+ "arguments": {
+ "url": "http://localhost:8907/page.html",
+ "_is_done": true
+ },
+ "id": "toolu_01HtjZYGZBMxwDphWFCrM56K"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 2428,
+ "output": 135
+ }
+ },
"25674340b53e0aab05c6156116a112c10c164ae6": {
"result": {
"role": "assistant",
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
index 480acf13e9..e8a4538293 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
@@ -44,6 +44,51 @@
"output": 106
}
},
+ "92d64360661d729108b919d4a4508168b7d35c99": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "The assertion has been completed. The URL expectation failed because the current page URL is \"http://localhost:8919/other.html\" which does not match the regex pattern \"/other pattern/\". The expectation was looking for the literal text \"other pattern\" in the URL, but the actual URL contains \"other.html\" instead."
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 2417,
+ "output": 73
+ }
+ },
+ "a3770c92ae525709add4d4e36be3b24a2a36b74f": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I'll call the browser_expect_url tool again with the correct interpretation. Based on the error message, the page URL is \"http://localhost:8919/other.html\", and I need to verify it matches the pattern \"/other pattern\". Let me use a regex that actually matches the current URL:"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_expect_url",
+ "arguments": {
+ "regex": "/other/",
+ "_is_done": true
+ },
+ "id": "toolu_01M3PDkYxU6t9JfhGZ6c57gX"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 2495,
+ "output": 140
+ }
+ },
"de87266adc368111c6c17076c5cabe093bbc2a0c": {
"result": {
"role": "assistant",
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
index cd00c01412..9e185d902f 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
@@ -52,5 +52,32 @@
"input": 2459,
"output": 129
}
+ },
+ "953a27a565140b193d83d39f4c2db82d656beff2": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I need to use a regex pattern to match the URL path correctly, since the actual URL includes the protocol and host."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_expect_url",
+ "arguments": {
+ "regex": "/\\/other\\.html$/",
+ "_is_done": true
+ },
+ "id": "toolu_015UbdXhw1sTtdCF7rtodPEm"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 2424,
+ "output": 104
+ }
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
index b00e1597cc..ecb06f8d8f 100644
--- a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
@@ -15,7 +15,7 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_01GhhJjUEfJKfxCSFbq182mh"
+ "id": "toolu_01YVFbgqSfhDKsS843HKxwor"
}
],
"stopReason": {