Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(playwright): report the URL the browser actually landed on (#4862)

Gergő Móriczbce6a2e6431a
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index b787a2c89..a14d00ea6 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -340,6 +340,71 @@ describe("Scrape tests", () => {
scrapeTimeout,
);
+ concurrentIf(TEST_SELF_HOST && HAS_PLAYWRIGHT)(
+ "playwright reports the landed URL after a cross-hostname redirect",
+ async () => {
+ // google.com 301s to www.google.com — the same stable redirect the
+ // threat-protection suite relies on. Before the playwright-service
+ // reported page.url(), metadata.url echoed the requested URL, so a page
+ // fetched from another host was indistinguishable from a real one.
+ const response = await scrape(
+ {
+ url: "https://google.com/",
+ waitFor: 100,
+ },
+ identity,
+ );
+
+ expect(response.metadata.sourceURL).toBe("https://google.com/");
+ expect(response.metadata.url).toBeDefined();
+ // Not pinned to www.google.com: the target is region-dependent from
+ // whichever network the runner sits on. That the landed host is no
+ // longer the requested one is the whole invariant under test.
+ expect(new URL(response.metadata.url!).hostname).not.toBe(
+ new URL("https://google.com/").hostname,
+ );
+ },
+ scrapeTimeout,
+ );
+
+ concurrentIf(TEST_SELF_HOST && HAS_PLAYWRIGHT && ALLOW_TEST_SUITE_WEBSITE)(
+ "playwright reports the requested URL when there is no redirect",
+ async () => {
+ // The other half of the contract: reporting the landed URL must not
+ // invent a redirect where there is none.
+ const response = await scrape(
+ {
+ url: TEST_SUITE_WEBSITE,
+ waitFor: 100,
+ },
+ identity,
+ );
+
+ expect(response.metadata.sourceURL).toBe(TEST_SUITE_WEBSITE);
+ expect(response.metadata.url).toBeDefined();
+ expect(new URL(response.metadata.url!).hostname).toBe(
+ new URL(TEST_SUITE_WEBSITE).hostname,
+ );
+ },
+ scrapeTimeout,
+ );
+
+ concurrentIf(TEST_SELF_HOST && HAS_PLAYWRIGHT && ALLOW_TEST_SUITE_WEBSITE)(
+ "playwright reports the landed URL after a client-side redirect",
+ async () => {
+ const url = `${TEST_SUITE_WEBSITE}/client-redirect.html`;
+ const response = await scrape({ url, waitFor: 1000 }, identity);
+
+ expect(response.metadata.sourceURL).toBe(url);
+ expect(response.metadata.url).toBeDefined();
+ expect(new URL(response.metadata.url!).pathname.replace(/\/$/, "")).toBe(
+ "/about",
+ );
+ expect(response.markdown).not.toContain("Redirecting to the about page");
+ },
+ scrapeTimeout,
+ );
+
concurrentIf(TEST_PRODUCTION || (HAS_PLAYWRIGHT && ALLOW_TEST_SUITE_WEBSITE))(
"waitFor works",
async () => {
diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
index 1acb185b9..e641db355 100644
--- a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
@@ -27,6 +27,8 @@ export async function scrapeURLWithPlaywright(
pageStatusCode: z.number(),
pageError: z.string().optional(),
contentType: z.string().optional(),
+ // Optional: older playwright-service builds don't send it.
+ url: z.string().optional(),
}),
mock: meta.mock,
abort: meta.abort.asSignal(),
@@ -37,7 +39,10 @@ export async function scrapeURLWithPlaywright(
}
return {
- url: meta.rewrittenUrl ?? meta.url, // TODO: impove redirect following
+ // The landed URL reported by the service. Redirect threat and blocklist
+ // re-checks compare it against the requested URL. Falls back to the
+ // requested URL for older services or a non-http(s) value (about:blank).
+ url: landedHttpUrl(response.url) ?? meta.rewrittenUrl ?? meta.url,
html: response.content,
statusCode: response.pageStatusCode,
error: response.pageError,
@@ -47,6 +52,16 @@ export async function scrapeURLWithPlaywright(
};
}
+function landedHttpUrl(url: string | undefined): string | undefined {
+ if (!url) return undefined;
+ try {
+ const { protocol } = new URL(url);
+ return protocol === "http:" || protocol === "https:" ? url : undefined;
+ } catch {
+ return undefined;
+ }
+}
+
export function playwrightMaxReasonableTime(meta: Meta): number {
return (meta.options.waitFor ?? 0) + 30000;
}
diff --git a/apps/playwright-service-ts/api.ts b/apps/playwright-service-ts/api.ts
index 73dc1f1e2..1d0fa7f12 100644
--- a/apps/playwright-service-ts/api.ts
+++ b/apps/playwright-service-ts/api.ts
@@ -320,7 +320,8 @@ const scrapePage = async (
}
let headers = null,
- content = await page.content();
+ content = await page.content(),
+ landedUrl = page.url();
let ct: string | undefined = undefined;
if (response) {
headers = await response.allHeaders();
@@ -333,11 +334,16 @@ const scrapePage = async (
ct.toLowerCase().includes('text/plain'))
) {
content = (await response.body()).toString('utf8'); // TODO: determine real encoding
+ landedUrl = response.url();
}
}
return {
content,
+ // Where the returned content came from: page.url() for rendered HTML
+ // (covers HTTP 3xx and client-side redirects), response.url() for raw
+ // JSON/text bodies, which come from the page.goto response.
+ url: landedUrl,
status: response ? response.status() : null,
headers,
contentType: ct,
@@ -537,6 +543,7 @@ app.post('/scrape', async (req: Request, res: Response) => {
content: result.content,
pageStatusCode: result.status,
contentType: result.contentType,
+ url: result.url,
...(pageError && { pageError }),
});
} catch (error) {
diff --git a/apps/test-site/public/client-redirect.html b/apps/test-site/public/client-redirect.html
new file mode 100644
index 000000000..ada91b23a
--- /dev/null
+++ b/apps/test-site/public/client-redirect.html
@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8" />
+ <title>Client-side redirect</title>
+ <script>
+ window.location.replace("/about/");
+ </script>
+ </head>
+ <body>
+ <p>Redirecting to the about page.</p>
+ </body>
+</html>