Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(playwright): report the URL the browser actually landed on (#4862)
Gergő Móriczbce6a2e6431a
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex b787a2c89..a14d00ea6 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -340,6 +340,71 @@ describe("Scrape tests", () => {scrapeTimeout,);+ concurrentIf(TEST_SELF_HOST && HAS_PLAYWRIGHT)(+ "playwright reports the landed URL after a cross-hostname redirect",+ async () => {+ // google.com 301s to www.google.com — the same stable redirect the+ // threat-protection suite relies on. Before the playwright-service+ // reported page.url(), metadata.url echoed the requested URL, so a page+ // fetched from another host was indistinguishable from a real one.+ const response = await scrape(+ {+ url: "https://google.com/",+ waitFor: 100,+ },+ identity,+ );++ expect(response.metadata.sourceURL).toBe("https://google.com/");+ expect(response.metadata.url).toBeDefined();+ // Not pinned to www.google.com: the target is region-dependent from+ // whichever network the runner sits on. That the landed host is no+ // longer the requested one is the whole invariant under test.+ expect(new URL(response.metadata.url!).hostname).not.toBe(+ new URL("https://google.com/").hostname,+ );+ },+ scrapeTimeout,+ );++ concurrentIf(TEST_SELF_HOST && HAS_PLAYWRIGHT && ALLOW_TEST_SUITE_WEBSITE)(+ "playwright reports the requested URL when there is no redirect",+ async () => {+ // The other half of the contract: reporting the landed URL must not+ // invent a redirect where there is none.+ const response = await scrape(+ {+ url: TEST_SUITE_WEBSITE,+ waitFor: 100,+ },+ identity,+ );++ expect(response.metadata.sourceURL).toBe(TEST_SUITE_WEBSITE);+ expect(response.metadata.url).toBeDefined();+ expect(new URL(response.metadata.url!).hostname).toBe(+ new URL(TEST_SUITE_WEBSITE).hostname,+ );+ },+ scrapeTimeout,+ );++ concurrentIf(TEST_SELF_HOST && HAS_PLAYWRIGHT && ALLOW_TEST_SUITE_WEBSITE)(+ "playwright reports the landed URL after a client-side redirect",+ async () => {+ const url = `${TEST_SUITE_WEBSITE}/client-redirect.html`;+ const response = await scrape({ url, waitFor: 1000 }, identity);++ expect(response.metadata.sourceURL).toBe(url);+ expect(response.metadata.url).toBeDefined();+ expect(new URL(response.metadata.url!).pathname.replace(/\/$/, "")).toBe(+ "/about",+ );+ expect(response.markdown).not.toContain("Redirecting to the about page");+ },+ scrapeTimeout,+ );+concurrentIf(TEST_PRODUCTION || (HAS_PLAYWRIGHT && ALLOW_TEST_SUITE_WEBSITE))("waitFor works",async () => {diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.tsindex 1acb185b9..e641db355 100644--- a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts@@ -27,6 +27,8 @@ export async function scrapeURLWithPlaywright(pageStatusCode: z.number(),pageError: z.string().optional(),contentType: z.string().optional(),+ // Optional: older playwright-service builds don't send it.+ url: z.string().optional(),}),mock: meta.mock,abort: meta.abort.asSignal(),@@ -37,7 +39,10 @@ export async function scrapeURLWithPlaywright(}return {- url: meta.rewrittenUrl ?? meta.url, // TODO: impove redirect following+ // The landed URL reported by the service. Redirect threat and blocklist+ // re-checks compare it against the requested URL. Falls back to the+ // requested URL for older services or a non-http(s) value (about:blank).+ url: landedHttpUrl(response.url) ?? meta.rewrittenUrl ?? meta.url,html: response.content,statusCode: response.pageStatusCode,error: response.pageError,@@ -47,6 +52,16 @@ export async function scrapeURLWithPlaywright(};}+function landedHttpUrl(url: string | undefined): string | undefined {+ if (!url) return undefined;+ try {+ const { protocol } = new URL(url);+ return protocol === "http:" || protocol === "https:" ? url : undefined;+ } catch {+ return undefined;+ }+}+export function playwrightMaxReasonableTime(meta: Meta): number {return (meta.options.waitFor ?? 0) + 30000;}diff --git a/apps/playwright-service-ts/api.ts b/apps/playwright-service-ts/api.tsindex 73dc1f1e2..1d0fa7f12 100644--- a/apps/playwright-service-ts/api.ts+++ b/apps/playwright-service-ts/api.ts@@ -320,7 +320,8 @@ const scrapePage = async (}let headers = null,- content = await page.content();+ content = await page.content(),+ landedUrl = page.url();let ct: string | undefined = undefined;if (response) {headers = await response.allHeaders();@@ -333,11 +334,16 @@ const scrapePage = async (ct.toLowerCase().includes('text/plain'))) {content = (await response.body()).toString('utf8'); // TODO: determine real encoding+ landedUrl = response.url();}}return {content,+ // Where the returned content came from: page.url() for rendered HTML+ // (covers HTTP 3xx and client-side redirects), response.url() for raw+ // JSON/text bodies, which come from the page.goto response.+ url: landedUrl,status: response ? response.status() : null,headers,contentType: ct,@@ -537,6 +543,7 @@ app.post('/scrape', async (req: Request, res: Response) => {content: result.content,pageStatusCode: result.status,contentType: result.contentType,+ url: result.url,...(pageError && { pageError }),});} catch (error) {diff --git a/apps/test-site/public/client-redirect.html b/apps/test-site/public/client-redirect.htmlnew file mode 100644index 000000000..ada91b23a--- /dev/null+++ b/apps/test-site/public/client-redirect.html@@ -0,0 +1,13 @@+<!doctype html>+<html lang="en">+ <head>+ <meta charset="utf-8" />+ <title>Client-side redirect</title>+ <script>+ window.location.replace("/about/");+ </script>+ </head>+ <body>+ <p>Redirecting to the about page.</p>+ </body>+</html>