Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(playwright-service): seed Cookie into the context jar so it survives redirects (#3726)
Oscar Jiang6092fda6b560
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex a287bf11c..2cc13bc0f 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -715,6 +715,28 @@ describe("Scrape tests", () => {scrapeTimeout * 2 + 1 * indexCooldown,);+ // Gated to the playwright engine (where cookies are seeded into the jar);+ // a Cookie passed as an extra request header is dropped on redirect hops.+ concurrentIf(HAS_PLAYWRIGHT && !HAS_FIRE_ENGINE)(+ "forwards cookies across redirects",+ async () => {+ // httpbin's /cookies echoes the cookies it received. The cookie only+ // survives the 302 hop if it was seeded into the browser cookie jar.+ const response = await scrape(+ {+ url: "https://httpbin.org/redirect-to?url=https%3A%2F%2Fhttpbin.org%2Fcookies&status_code=302",+ headers: { Cookie: "fc_cookie_redirect_test=1" },+ formats: ["rawHtml"],+ waitFor: 1000,+ },+ identity,+ );++ expect(response.rawHtml).toContain("fc_cookie_redirect_test");+ },+ scrapeTimeout,+ );+it.concurrent("respects mobile",async () => {diff --git a/apps/playwright-service-ts/api.ts b/apps/playwright-service-ts/api.tsindex 4e0f3b88a..6f5b6ceee 100644--- a/apps/playwright-service-ts/api.ts+++ b/apps/playwright-service-ts/api.ts@@ -414,10 +414,61 @@ app.post('/scrape', async (req: Request, res: Response) => {page = await requestContext.newPage();if (headers) {- // Remove the user-agent key before calling setExtraHTTPHeaders since- // we already forwarded it to the context-level userAgent option.+ // A Cookie header passed through setExtraHTTPHeaders is sent on the first+ // request but DROPPED on any redirect hop (the browser regenerates the+ // redirected request from its cookie jar, which is empty). Authenticated+ // sites that 302 (e.g. to /signin when the session looks absent) then+ // land on the login page. Seed the cookie jar instead so Chromium re-sends+ // it on every request, including redirects — matching what a raw HTTP+ // client does.+ const cookieHeader = Object.entries(headers).find(([k]) => k.toLowerCase() === 'cookie')?.[1];+ if (cookieHeader) {+ // Scope cookies to the registrable domain (e.g. ".example.com"), not+ // host-only. Authenticated pages often 302 across sibling subdomains+ // (example.com -> app.example.com); a host-only cookie set for the+ // original host would not be sent to the redirect target, leaving the+ // request unauthenticated. The Cookie header carries no domain info, so+ // we apply the eTLD+1 — broad enough to follow the redirect, and these+ // are first-party cookies being returned to their own origin anyway.+ let cookieDomain: string | undefined;+ try {+ const host = new URL(url).hostname;+ const labels = host.split('.');+ cookieDomain = labels.length > 2 ? labels.slice(-2).join('.') : host;+ } catch {+ cookieDomain = undefined;+ }+ type SeedCookie = { name: string; value: string; url?: string; domain?: string; path?: string };+ const cookies = cookieHeader+ .split(';')+ .map(pair => pair.trim())+ .filter(Boolean)+ .map((pair): SeedCookie | null => {+ const eq = pair.indexOf('=');+ if (eq === -1) return null;+ const name = pair.slice(0, eq).trim();+ const value = pair.slice(eq + 1).trim();+ return cookieDomain+ ? { name, value, domain: `.${cookieDomain}`, path: '/' }+ : { name, value, url };+ })+ .filter((c): c is SeedCookie => c !== null);+ if (cookies.length > 0) {+ try {+ await requestContext.addCookies(cookies);+ } catch (error) {+ console.warn('Failed to seed cookies from Cookie header:', error);+ }+ }+ }++ // Remove user-agent (already applied at the context level) and cookie+ // (now seeded into the jar) before forwarding the rest verbatim.const filteredHeaders = Object.fromEntries(- Object.entries(headers).filter(([k]) => k.toLowerCase() !== 'user-agent')+ Object.entries(headers).filter(([k]) => {+ const lower = k.toLowerCase();+ return lower !== 'user-agent' && lower !== 'cookie';+ }));if (Object.keys(filteredHeaders).length > 0) {await page.setExtraHTTPHeaders(filteredHeaders);