Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
test: server port is unstable (#38846)
Simon Knott56c7836a9772
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.jsonindex d4f1cf91e7..453653504d 100644--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json@@ -1,20 +1,20 @@{- "0804fd14030c0e1f89df22422c4f085d1698f2a0": {+ "74935765e910e2883dc5925fd5f947bb528f174d": {"result": {"role": "assistant","content": [{"type": "text",- "text": "I see the issue - the actual URL is `http://localhost:8907/page.html` but I was expecting just `/page.html`. The relative URL should match when resolved against the base URL. Let me try with the full URL:"+ "text": "I see the issue - the URL contains a server prefix. Let me use a regex pattern to match the URL ending with /counter.html."},{"type": "tool_call","name": "browser_expect_url","arguments": {- "url": "http://localhost:8907/page.html",+ "regex": "/counter\\.html$/","_is_done": true},- "id": "toolu_01HtjZYGZBMxwDphWFCrM56K"+ "id": "toolu_018XaEFPSm6aLzqA1BBqRNhu"}],"stopReason": {@@ -22,26 +22,26 @@}},"usage": {- "input": 2428,- "output": 135+ "input": 2447,+ "output": 108}},- "25674340b53e0aab05c6156116a112c10c164ae6": {+ "ee1b825529efd3c64fff821e9ce6f2e06433f6d3": {"result": {"role": "assistant","content": [{"type": "text",- "text": "I need to verify that the page URL is `/page.html`. Let me check the current URL against this expectation."+ "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool."},{"type": "tool_call","name": "browser_expect_url","arguments": {- "url": "/page.html",+ "url": "/counter.html","_is_done": true},- "id": "toolu_01EX6MnTTtTzSY7hMT8Z59ei"+ "id": "toolu_01PRTpTFD78UKubB5EzwTAii"}],"stopReason": {@@ -49,35 +49,8 @@}},"usage": {- "input": 2230,- "output": 103- }- },- "2aadf7e119b88ed7fdbcdaabb4d862edd2988581": {- "result": {- "role": "assistant",- "content": [- {- "type": "text",- "text": "I see the issue - the actual URL is the full URL `http://localhost:8927/page.html` while I was checking for just `/page.html`. According to the documentation, relative URLs should be resolved against the baseURL. Let me try with the full URL pattern using regex to match the path."- },- {- "type": "tool_call",- "name": "browser_expect_url",- "arguments": {- "regex": "/page\\.html$/",- "_is_done": true- },- "id": "toolu_01KZFwWZi2w5zL1b6wnutPDM"- }- ],- "stopReason": {- "code": "ok"- }- },- "usage": {- "input": 2463,- "output": 144+ "input": 2243,+ "output": 99}}}\ No newline at end of filediff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.jsonindex e8a4538293..7e0cc5425c 100644--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json@@ -1,83 +1,20 @@{- "33311bb11c7224b910aded4180fd27537282fe86": {+ "37de212b569c4a2d8412505c647ce7ad1f824387": {"result": {"role": "assistant","content": [{"type": "text",- "text": "The assertion completed. The page URL (`http://localhost:8907/other.html`) does not match the expected pattern `/other pattern/`. The expectation failed as shown in the error output."- }- ],- "stopReason": {- "code": "ok"- }- },- "usage": {- "input": 2452,- "output": 44- }- },- "8abfa22f63f07079f6ac52e46a771fdbabdfc67a": {- "result": {- "role": "assistant",- "content": [- {- "type": "text",- "text": "I need to verify that the page URL matches the pattern \"/other pattern\". Let me correct my regex pattern to properly match URLs containing \"other\":"- },- {- "type": "tool_call",- "name": "browser_expect_url",- "arguments": {- "regex": "/other/",- "_is_done": true- },- "id": "toolu_01GSSYMo7XYGRQMoKmRemr3k"- }- ],- "stopReason": {- "code": "ok"- }- },- "usage": {- "input": 2501,- "output": 106- }- },- "92d64360661d729108b919d4a4508168b7d35c99": {- "result": {- "role": "assistant",- "content": [- {- "type": "text",- "text": "The assertion has been completed. The URL expectation failed because the current page URL is \"http://localhost:8919/other.html\" which does not match the regex pattern \"/other pattern/\". The expectation was looking for the literal text \"other pattern\" in the URL, but the actual URL contains \"other.html\" instead."- }- ],- "stopReason": {- "code": "ok"- }- },- "usage": {- "input": 2417,- "output": 73- }- },- "a3770c92ae525709add4d4e36be3b24a2a36b74f": {- "result": {- "role": "assistant",- "content": [- {- "type": "text",- "text": "I'll call the browser_expect_url tool again with the correct interpretation. Based on the error message, the page URL is \"http://localhost:8919/other.html\", and I need to verify it matches the pattern \"/other pattern\". Let me use a regex that actually matches the current URL:"+ "text": "I need to verify that the page URL matches the `/counter` pattern. I'll use the `browser_expect_url` tool with a regex pattern."},{"type": "tool_call","name": "browser_expect_url","arguments": {- "regex": "/other/",+ "regex": "/counter","_is_done": true},- "id": "toolu_01M3PDkYxU6t9JfhGZ6c57gX"+ "id": "toolu_01NFN4Fz1V3Bek9pKTbuEsBg"}],"stopReason": {@@ -85,26 +22,26 @@}},"usage": {- "input": 2495,- "output": 140+ "input": 2242,+ "output": 108}},- "de87266adc368111c6c17076c5cabe093bbc2a0c": {+ "a786ca3af437fcb35753c1f7ceb7f9a90bc8deeb": {"result": {"role": "assistant","content": [{"type": "text",- "text": "I'll verify that the page URL matches the pattern \"/other pattern\"."+ "text": "I need to fix the regex pattern format. It should end with a `/` and optional flags."},{"type": "tool_call","name": "browser_expect_url","arguments": {- "regex": "/other pattern/",+ "regex": "/counter/","_is_done": true},- "id": "toolu_01Chezp7seC7dNLCrf84DnHE"+ "id": "toolu_01SLaSETGg54hyKNATWckQVw"}],"stopReason": {@@ -112,8 +49,8 @@}},"usage": {- "input": 2229,- "output": 92+ "input": 2440,+ "output": 97}}}\ No newline at end of filediff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.jsonindex 78586ad0d5..dd48e0d5fd 100644--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json@@ -1,20 +1,20 @@{- "20ce06d6e961a709aad1b0f42701fd6aa6cceac4": {+ "37de212b569c4a2d8412505c647ce7ad1f824387": {"result": {"role": "assistant","content": [{"type": "text",- "text": "I need to properly format the regex pattern with the closing slash. Let me fix that:"+ "text": "I need to verify that the page URL matches the /counter pattern using a browser_expect assertion tool."},{"type": "tool_call","name": "browser_expect_url","arguments": {- "regex": "/page/",+ "regex": "/counter/","_is_done": true},- "id": "toolu_01UyMaCDi7iQdvyfXVnCDxfX"+ "id": "toolu_015RtjMDCtGsTUpu36HUk5fC"}],"stopReason": {@@ -22,35 +22,8 @@}},"usage": {- "input": 2414,- "output": 95- }- },- "55679bf3e65a5bf754774c639528ad14d8c037c8": {- "result": {- "role": "assistant",- "content": [- {- "type": "text",- "text": "I need to verify that the page URL matches the pattern \"/page\". I'll use the browser_expect_url tool with the regex parameter to check this."- },- {- "type": "tool_call",- "name": "browser_expect_url",- "arguments": {- "regex": "/page",- "_is_done": true- },- "id": "toolu_015xQfJBhmAcKKgddECYdQtP"- }- ],- "stopReason": {- "code": "ok"- }- },- "usage": {- "input": 2229,- "output": 108+ "input": 2242,+ "output": 98}}}\ No newline at end of filediff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.jsonindex 9e185d902f..353b806e09 100644--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json@@ -1,20 +1,20 @@{- "118e114326db2240e7997e7f2fa044ce856fc8e6": {+ "688127ac19026114a89d4e621b86bdeaa699418e": {"result": {"role": "assistant","content": [{"type": "text",- "text": "I'll verify that the page URL is /other.html using the browser_expect_url tool."+ "text": "I see the issue - the actual URL includes the server prefix. Let me use a regex pattern to match the URL ending with /counter.html."},{"type": "tool_call","name": "browser_expect_url","arguments": {- "url": "/other.html",+ "regex": "/\\/counter\\.html$/","_is_done": true},- "id": "toolu_01Tn7U997Ds6NuXKg1jtgM4N"+ "id": "toolu_01V6sRugMvahkP21AHaffA11"}],"stopReason": {@@ -22,26 +22,26 @@}},"usage": {- "input": 2230,- "output": 99+ "input": 2447,+ "output": 110}},- "15b04448ca67e533376e7333fb87419c02007a33": {+ "ee1b825529efd3c64fff821e9ce6f2e06433f6d3": {"result": {"role": "assistant","content": [{"type": "text",- "text": "I see the issue. The actual URL is the full URL `http://localhost:8919/other.html`, not just the path `/other.html`. Let me use a regex pattern to match just the path portion of the URL."+ "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool."},{"type": "tool_call","name": "browser_expect_url","arguments": {- "regex": "/\\/other\\.html$/",+ "url": "/counter.html","_is_done": true},- "id": "toolu_012xLUJb4md8YUyuBMVpxSUB"+ "id": "toolu_014UE4ajKxYwk8kM6MsURpks"}],"stopReason": {@@ -49,35 +49,8 @@}},"usage": {- "input": 2459,- "output": 129- }- },- "953a27a565140b193d83d39f4c2db82d656beff2": {- "result": {- "role": "assistant",- "content": [- {- "type": "text",- "text": "I need to use a regex pattern to match the URL path correctly, since the actual URL includes the protocol and host."- },- {- "type": "tool_call",- "name": "browser_expect_url",- "arguments": {- "regex": "/\\/other\\.html$/",- "_is_done": true- },- "id": "toolu_015UbdXhw1sTtdCF7rtodPEm"- }- ],- "stopReason": {- "code": "ok"- }- },- "usage": {- "input": 2424,- "output": 104+ "input": 2243,+ "output": 99}}}\ No newline at end of filediff --git a/tests/library/agent-expect.spec.ts b/tests/library/agent-expect.spec.tsindex 863c9eaccc..c7befd887a 100644--- a/tests/library/agent-expect.spec.ts+++ b/tests/library/agent-expect.spec.ts@@ -221,78 +221,90 @@ Call log:});test('expectURL success', async ({ context, server }) => {+ const secrets = {+ SERVER: server.PREFIX+ };{- const { page, agent } = await generateAgent(context);- await page.goto(server.PREFIX + '/page.html');- await agent.expect('page URL is /page.html');+ const { page, agent } = await generateAgent(context, { secrets });+ await page.goto(server.PREFIX + '/counter.html');+ await agent.expect('page URL is /counter.html');}expect(await cacheObject()).toEqual({- 'page URL is /page.html': {+ 'page URL is /counter.html': {actions: [expect.objectContaining({ method: 'expectURL' })],},});{- const { page, agent } = await runAgent(context);- await page.goto(server.PREFIX + '/page.html');- await agent.expect('page URL is /page.html');+ const { page, agent } = await runAgent(context, { secrets });+ await page.goto(server.PREFIX + '/counter.html');+ await agent.expect('page URL is /counter.html');}});test('expectURL wrong URL error', async ({ context, server }) => {+ const secrets = {+ SERVER: server.PREFIX+ };{- const { page, agent } = await generateAgent(context);- await page.goto(server.PREFIX + '/other.html');- await agent.expect('page URL is /other.html');+ const { page, agent } = await generateAgent(context, { secrets });+ await page.goto(server.PREFIX + '/counter.html');+ await agent.expect('page URL is /counter.html');}expect(await cacheObject()).toEqual({- 'page URL is /other.html': {+ 'page URL is /counter.html': {actions: [expect.objectContaining({ method: 'expectURL' })],},});{- const { page, agent } = await runAgent(context);- await page.goto(server.PREFIX + '/page.html');- const error = await agent.expect('page URL is /other.html').catch(e => e);+ const { page, agent } = await runAgent(context, { secrets });+ await page.goto(server.PREFIX + '/empty.html');+ const error = await agent.expect('page URL is /counter.html').catch(e => e);expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(page).toHaveURL(expected) failed`);- expect(stripAnsi(error.message)).toContain(`Received: ${server.PREFIX}/page.html`);+ expect(stripAnsi(error.message)).toContain(`Received: ${server.PREFIX}/empty.html`);}});test('expectURL with regex', async ({ context, server }) => {+ const secrets = {+ SERVER: server.PREFIX+ };{- const { page, agent } = await generateAgent(context);- await page.goto(server.PREFIX + '/page.html');- await agent.expect('page URL matches /page pattern');+ const { page, agent } = await generateAgent(context, { secrets });+ await page.goto(server.PREFIX + '/counter.html');+ await agent.expect('page URL matches /counter pattern');}expect(await cacheObject()).toEqual({- 'page URL matches /page pattern': {- actions: [expect.objectContaining({ method: 'expectURL' })],+ 'page URL matches /counter pattern': {+ actions: [expect.objectContaining({ method: 'expectURL', regex: expect.any(String) })],},});{- const { page, agent } = await runAgent(context);- await page.goto(server.PREFIX + '/page.html');- await agent.expect('page URL matches /page pattern');+ const { page, agent } = await runAgent(context, { secrets });+ await page.goto(server.PREFIX + '/counter.html');+ await agent.expect('page URL matches /counter pattern');}});test('expectURL with regex error', async ({ context, server }) => {+ const secrets = {+ SERVER: server.PREFIX+ };{- const { page, agent } = await generateAgent(context);- await page.goto(server.PREFIX + '/other.html');- await agent.expect('page URL matches /other pattern');+ const { page, agent } = await generateAgent(context, { secrets });+ await page.goto(server.PREFIX + '/counter.html');+ await agent.expect('page URL matches /counter pattern');}expect(await cacheObject()).toEqual({- 'page URL matches /other pattern': {- actions: [expect.objectContaining({ method: 'expectURL' })],+ 'page URL matches /counter pattern': {+ actions: [expect.objectContaining({ method: 'expectURL', regex: expect.any(String) })],},});{- const { page, agent } = await runAgent(context);- await page.goto(server.PREFIX + '/page.html');- const error = await agent.expect('page URL matches /other pattern').catch(e => e);+ const { page, agent } = await runAgent(context, { secrets });+ await page.goto(server.PREFIX + '/empty.html');+ const error = await agent.expect('page URL matches /counter pattern').catch(e => e);expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(page).toHaveURL(expected) failed`);- expect(stripAnsi(error.message)).toContain(`Received: ${server.PREFIX}/page.html`);+ expect(stripAnsi(error.message)).toContain(`Received: ${server.PREFIX}/empty.html`);}});