Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

test: server port is unstable (#38846)

Simon Knott56c7836a9772
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
index d4f1cf91e7..453653504d 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
@@ -1,20 +1,20 @@
{
- "0804fd14030c0e1f89df22422c4f085d1698f2a0": {
+ "74935765e910e2883dc5925fd5f947bb528f174d": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I see the issue - the actual URL is `http://localhost:8907/page.html` but I was expecting just `/page.html`. The relative URL should match when resolved against the base URL. Let me try with the full URL:"
+ "text": "I see the issue - the URL contains a server prefix. Let me use a regex pattern to match the URL ending with /counter.html."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "url": "http://localhost:8907/page.html",
+ "regex": "/counter\\.html$/",
"_is_done": true
},
- "id": "toolu_01HtjZYGZBMxwDphWFCrM56K"
+ "id": "toolu_018XaEFPSm6aLzqA1BBqRNhu"
}
],
"stopReason": {
@@ -22,26 +22,26 @@
}
},
"usage": {
- "input": 2428,
- "output": 135
+ "input": 2447,
+ "output": 108
}
},
- "25674340b53e0aab05c6156116a112c10c164ae6": {
+ "ee1b825529efd3c64fff821e9ce6f2e06433f6d3": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to verify that the page URL is `/page.html`. Let me check the current URL against this expectation."
+ "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "url": "/page.html",
+ "url": "/counter.html",
"_is_done": true
},
- "id": "toolu_01EX6MnTTtTzSY7hMT8Z59ei"
+ "id": "toolu_01PRTpTFD78UKubB5EzwTAii"
}
],
"stopReason": {
@@ -49,35 +49,8 @@
}
},
"usage": {
- "input": 2230,
- "output": 103
- }
- },
- "2aadf7e119b88ed7fdbcdaabb4d862edd2988581": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I see the issue - the actual URL is the full URL `http://localhost:8927/page.html` while I was checking for just `/page.html`. According to the documentation, relative URLs should be resolved against the baseURL. Let me try with the full URL pattern using regex to match the path."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/page\\.html$/",
- "_is_done": true
- },
- "id": "toolu_01KZFwWZi2w5zL1b6wnutPDM"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2463,
- "output": 144
+ "input": 2243,
+ "output": 99
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
index e8a4538293..7e0cc5425c 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
@@ -1,83 +1,20 @@
{
- "33311bb11c7224b910aded4180fd27537282fe86": {
+ "37de212b569c4a2d8412505c647ce7ad1f824387": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "The assertion completed. The page URL (`http://localhost:8907/other.html`) does not match the expected pattern `/other pattern/`. The expectation failed as shown in the error output."
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2452,
- "output": 44
- }
- },
- "8abfa22f63f07079f6ac52e46a771fdbabdfc67a": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I need to verify that the page URL matches the pattern \"/other pattern\". Let me correct my regex pattern to properly match URLs containing \"other\":"
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/other/",
- "_is_done": true
- },
- "id": "toolu_01GSSYMo7XYGRQMoKmRemr3k"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2501,
- "output": 106
- }
- },
- "92d64360661d729108b919d4a4508168b7d35c99": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "The assertion has been completed. The URL expectation failed because the current page URL is \"http://localhost:8919/other.html\" which does not match the regex pattern \"/other pattern/\". The expectation was looking for the literal text \"other pattern\" in the URL, but the actual URL contains \"other.html\" instead."
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2417,
- "output": 73
- }
- },
- "a3770c92ae525709add4d4e36be3b24a2a36b74f": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I'll call the browser_expect_url tool again with the correct interpretation. Based on the error message, the page URL is \"http://localhost:8919/other.html\", and I need to verify it matches the pattern \"/other pattern\". Let me use a regex that actually matches the current URL:"
+ "text": "I need to verify that the page URL matches the `/counter` pattern. I'll use the `browser_expect_url` tool with a regex pattern."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/other/",
+ "regex": "/counter",
"_is_done": true
},
- "id": "toolu_01M3PDkYxU6t9JfhGZ6c57gX"
+ "id": "toolu_01NFN4Fz1V3Bek9pKTbuEsBg"
}
],
"stopReason": {
@@ -85,26 +22,26 @@
}
},
"usage": {
- "input": 2495,
- "output": 140
+ "input": 2242,
+ "output": 108
}
},
- "de87266adc368111c6c17076c5cabe093bbc2a0c": {
+ "a786ca3af437fcb35753c1f7ceb7f9a90bc8deeb": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll verify that the page URL matches the pattern \"/other pattern\"."
+ "text": "I need to fix the regex pattern format. It should end with a `/` and optional flags."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/other pattern/",
+ "regex": "/counter/",
"_is_done": true
},
- "id": "toolu_01Chezp7seC7dNLCrf84DnHE"
+ "id": "toolu_01SLaSETGg54hyKNATWckQVw"
}
],
"stopReason": {
@@ -112,8 +49,8 @@
}
},
"usage": {
- "input": 2229,
- "output": 92
+ "input": 2440,
+ "output": 97
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
index 78586ad0d5..dd48e0d5fd 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
@@ -1,20 +1,20 @@
{
- "20ce06d6e961a709aad1b0f42701fd6aa6cceac4": {
+ "37de212b569c4a2d8412505c647ce7ad1f824387": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to properly format the regex pattern with the closing slash. Let me fix that:"
+ "text": "I need to verify that the page URL matches the /counter pattern using a browser_expect assertion tool."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/page/",
+ "regex": "/counter/",
"_is_done": true
},
- "id": "toolu_01UyMaCDi7iQdvyfXVnCDxfX"
+ "id": "toolu_015RtjMDCtGsTUpu36HUk5fC"
}
],
"stopReason": {
@@ -22,35 +22,8 @@
}
},
"usage": {
- "input": 2414,
- "output": 95
- }
- },
- "55679bf3e65a5bf754774c639528ad14d8c037c8": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I need to verify that the page URL matches the pattern \"/page\". I'll use the browser_expect_url tool with the regex parameter to check this."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/page",
- "_is_done": true
- },
- "id": "toolu_015xQfJBhmAcKKgddECYdQtP"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2229,
- "output": 108
+ "input": 2242,
+ "output": 98
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
index 9e185d902f..353b806e09 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
@@ -1,20 +1,20 @@
{
- "118e114326db2240e7997e7f2fa044ce856fc8e6": {
+ "688127ac19026114a89d4e621b86bdeaa699418e": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll verify that the page URL is /other.html using the browser_expect_url tool."
+ "text": "I see the issue - the actual URL includes the server prefix. Let me use a regex pattern to match the URL ending with /counter.html."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "url": "/other.html",
+ "regex": "/\\/counter\\.html$/",
"_is_done": true
},
- "id": "toolu_01Tn7U997Ds6NuXKg1jtgM4N"
+ "id": "toolu_01V6sRugMvahkP21AHaffA11"
}
],
"stopReason": {
@@ -22,26 +22,26 @@
}
},
"usage": {
- "input": 2230,
- "output": 99
+ "input": 2447,
+ "output": 110
}
},
- "15b04448ca67e533376e7333fb87419c02007a33": {
+ "ee1b825529efd3c64fff821e9ce6f2e06433f6d3": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I see the issue. The actual URL is the full URL `http://localhost:8919/other.html`, not just the path `/other.html`. Let me use a regex pattern to match just the path portion of the URL."
+ "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/\\/other\\.html$/",
+ "url": "/counter.html",
"_is_done": true
},
- "id": "toolu_012xLUJb4md8YUyuBMVpxSUB"
+ "id": "toolu_014UE4ajKxYwk8kM6MsURpks"
}
],
"stopReason": {
@@ -49,35 +49,8 @@
}
},
"usage": {
- "input": 2459,
- "output": 129
- }
- },
- "953a27a565140b193d83d39f4c2db82d656beff2": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I need to use a regex pattern to match the URL path correctly, since the actual URL includes the protocol and host."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/\\/other\\.html$/",
- "_is_done": true
- },
- "id": "toolu_015UbdXhw1sTtdCF7rtodPEm"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2424,
- "output": 104
+ "input": 2243,
+ "output": 99
}
}
}
\ No newline at end of file
diff --git a/tests/library/agent-expect.spec.ts b/tests/library/agent-expect.spec.ts
index 863c9eaccc..c7befd887a 100644
--- a/tests/library/agent-expect.spec.ts
+++ b/tests/library/agent-expect.spec.ts
@@ -221,78 +221,90 @@ Call log:
});
test('expectURL success', async ({ context, server }) => {
+ const secrets = {
+ SERVER: server.PREFIX
+ };
{
- const { page, agent } = await generateAgent(context);
- await page.goto(server.PREFIX + '/page.html');
- await agent.expect('page URL is /page.html');
+ const { page, agent } = await generateAgent(context, { secrets });
+ await page.goto(server.PREFIX + '/counter.html');
+ await agent.expect('page URL is /counter.html');
}
expect(await cacheObject()).toEqual({
- 'page URL is /page.html': {
+ 'page URL is /counter.html': {
actions: [expect.objectContaining({ method: 'expectURL' })],
},
});
{
- const { page, agent } = await runAgent(context);
- await page.goto(server.PREFIX + '/page.html');
- await agent.expect('page URL is /page.html');
+ const { page, agent } = await runAgent(context, { secrets });
+ await page.goto(server.PREFIX + '/counter.html');
+ await agent.expect('page URL is /counter.html');
}
});
test('expectURL wrong URL error', async ({ context, server }) => {
+ const secrets = {
+ SERVER: server.PREFIX
+ };
{
- const { page, agent } = await generateAgent(context);
- await page.goto(server.PREFIX + '/other.html');
- await agent.expect('page URL is /other.html');
+ const { page, agent } = await generateAgent(context, { secrets });
+ await page.goto(server.PREFIX + '/counter.html');
+ await agent.expect('page URL is /counter.html');
}
expect(await cacheObject()).toEqual({
- 'page URL is /other.html': {
+ 'page URL is /counter.html': {
actions: [expect.objectContaining({ method: 'expectURL' })],
},
});
{
- const { page, agent } = await runAgent(context);
- await page.goto(server.PREFIX + '/page.html');
- const error = await agent.expect('page URL is /other.html').catch(e => e);
+ const { page, agent } = await runAgent(context, { secrets });
+ await page.goto(server.PREFIX + '/empty.html');
+ const error = await agent.expect('page URL is /counter.html').catch(e => e);
expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(page).toHaveURL(expected) failed`);
- expect(stripAnsi(error.message)).toContain(`Received: ${server.PREFIX}/page.html`);
+ expect(stripAnsi(error.message)).toContain(`Received: ${server.PREFIX}/empty.html`);
}
});
test('expectURL with regex', async ({ context, server }) => {
+ const secrets = {
+ SERVER: server.PREFIX
+ };
{
- const { page, agent } = await generateAgent(context);
- await page.goto(server.PREFIX + '/page.html');
- await agent.expect('page URL matches /page pattern');
+ const { page, agent } = await generateAgent(context, { secrets });
+ await page.goto(server.PREFIX + '/counter.html');
+ await agent.expect('page URL matches /counter pattern');
}
expect(await cacheObject()).toEqual({
- 'page URL matches /page pattern': {
- actions: [expect.objectContaining({ method: 'expectURL' })],
+ 'page URL matches /counter pattern': {
+ actions: [expect.objectContaining({ method: 'expectURL', regex: expect.any(String) })],
},
});
{
- const { page, agent } = await runAgent(context);
- await page.goto(server.PREFIX + '/page.html');
- await agent.expect('page URL matches /page pattern');
+ const { page, agent } = await runAgent(context, { secrets });
+ await page.goto(server.PREFIX + '/counter.html');
+ await agent.expect('page URL matches /counter pattern');
}
});
test('expectURL with regex error', async ({ context, server }) => {
+ const secrets = {
+ SERVER: server.PREFIX
+ };
{
- const { page, agent } = await generateAgent(context);
- await page.goto(server.PREFIX + '/other.html');
- await agent.expect('page URL matches /other pattern');
+ const { page, agent } = await generateAgent(context, { secrets });
+ await page.goto(server.PREFIX + '/counter.html');
+ await agent.expect('page URL matches /counter pattern');
}
expect(await cacheObject()).toEqual({
- 'page URL matches /other pattern': {
- actions: [expect.objectContaining({ method: 'expectURL' })],
+ 'page URL matches /counter pattern': {
+ actions: [expect.objectContaining({ method: 'expectURL', regex: expect.any(String) })],
},
});
{
- const { page, agent } = await runAgent(context);
- await page.goto(server.PREFIX + '/page.html');
- const error = await agent.expect('page URL matches /other pattern').catch(e => e);
+ const { page, agent } = await runAgent(context, { secrets });
+ await page.goto(server.PREFIX + '/empty.html');
+ const error = await agent.expect('page URL matches /counter pattern').catch(e => e);
expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(page).toHaveURL(expected) failed`);
- expect(stripAnsi(error.message)).toContain(`Received: ${server.PREFIX}/page.html`);
+ expect(stripAnsi(error.message)).toContain(`Received: ${server.PREFIX}/empty.html`);
}
});