Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(browser-crawler): persist session cookies after requestHandler (#3939)

Felix-Ayushfb392227a9b9
diff --git a/docs/public-api/crawlee-browser.api.md b/docs/public-api/crawlee-browser.api.md
index 2e82b9f38..4b8d6c76c 100644
--- a/docs/public-api/crawlee-browser.api.md
+++ b/docs/public-api/crawlee-browser.api.md
@@ -107,6 +107,7 @@ export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Respo
statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
id: StringPredicate & BasePredicate<string | undefined>;
};
+ protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
}
// @public (undocumented)
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md
index a84fdcad1..2b9d132ac 100644
--- a/docs/upgrading/upgrading_v4.md
+++ b/docs/upgrading/upgrading_v4.md
@@ -764,7 +764,13 @@ const cookieHeader2 = await session.cookieJar.getCookieString(url);
## `persistCookiesPerSession` renamed to `saveResponseCookies`
-The `persistCookiesPerSession` crawler option has been renamed to `saveResponseCookies` on both `HttpCrawler` (and its subclasses like `CheerioCrawler`, `JSDOMCrawler`, etc.) and `BrowserCrawler`. The behavior is unchanged - when enabled (the default), response `Set-Cookie` headers are stored in the session's cookie jar so they're sent on subsequent requests using the same session. Rename the option in your crawler constructor options to migrate.
+The `persistCookiesPerSession` crawler option has been renamed to `saveResponseCookies` on both `HttpCrawler` (and its subclasses like `CheerioCrawler`, `JSDOMCrawler`, etc.) and `BrowserCrawler`. When enabled (the default), response cookies are stored in the session's cookie jar so they're sent on subsequent requests using the same session. Rename the option in your crawler constructor options to migrate.
+
+## Browser cookies are also persisted after `requestHandler`
+
+Previously, `BrowserCrawler` with `saveResponseCookies` (formerly `persistCookiesPerSession`) only copied cookies from the page into the session after navigation and **before** `requestHandler` ran. Cookies set during the handler — login flows, `page.setCookie()`, or XHR/`fetch` `Set-Cookie` responses — were not stored on the session for later requests.
+
+In v4, when `saveResponseCookies` is enabled (the default), browser cookies are also re-read and stored in the session cookie jar **after** `requestHandler` completes. If you relied on handler-set cookies staying page-local and not affecting later requests on the same session, set `saveResponseCookies: false` or clear/overwrite cookies on the session explicitly.
## Internal KVS keys renamed
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 8d7dcc633..530f42b3d 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -655,23 +655,55 @@ export abstract class BrowserCrawler<
await this.processResponse(response, crawlingContext);
tryCancel();
- // TODO: Should we save the cookies also after/only the handle page?
- if (this.saveResponseCookies && crawlingContext.session) {
- const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
- tryCancel();
- const url = crawlingContext.request.loadedUrl!;
- for (const cookie of cookies) {
+ // Persist cookies from the navigation response before the user handler runs.
+ // Cookies set during `requestHandler` are saved again afterwards.
+ await this.persistCookiesFromPage(crawlingContext);
+
+ return { request: crawlingContext.request as LoadedRequest<Request> } as Partial<Context>;
+ }
+
+ /**
+ * Copies cookies from the live browser page into the session cookie jar.
+ */
+ private async persistCookiesFromPage(
+ crawlingContext: Pick<BrowserCrawlingContext<Page, Response>, 'session' | 'page' | 'request'>,
+ ): Promise<void> {
+ if (!this.saveResponseCookies || !crawlingContext.session) {
+ return;
+ }
+
+ const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
+ tryCancel();
+ // Prefer the live page URL — the handler may have navigated after the initial load.
+ const url =
+ (await crawlingContext.page.url()) || crawlingContext.request.loadedUrl || crawlingContext.request.url;
+ for (const cookie of cookies) {
+ try {
+ crawlingContext.session.cookieJar.setCookieSync(browserPoolCookieToToughCookie(cookie), url, {
+ ignoreError: false,
+ });
+ } catch (e) {
+ this.log.debug(`Could not set cookie: ${(e as Error).message}`);
+ }
+ }
+ }
+
+ /**
+ * Runs the user request handler, then re-reads browser cookies so login flows /
+ * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
+ */
+ protected override async runRequestHandler(crawlingContext: ExtendedContext): Promise<void> {
+ try {
+ await super.runRequestHandler(crawlingContext);
+ } finally {
+ if (!crawlingContext.request.skipNavigation) {
try {
- crawlingContext.session.cookieJar.setCookieSync(browserPoolCookieToToughCookie(cookie), url, {
- ignoreError: false,
- });
- } catch (e) {
- this.log.debug(`Could not set cookie: ${(e as Error).message}`);
+ await this.persistCookiesFromPage(crawlingContext);
+ } catch {
+ // Page may already be closed on some failure paths; ignore.
}
}
}
-
- return { request: crawlingContext.request as LoadedRequest<Request> } as Partial<Context>;
}
private async handleBlockedRequestByContent(crawlingContext: BrowserCrawlingContext<Page, Response>) {
diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts
index 4532ae84f..45b7930d3 100644
--- a/test/core/crawlers/browser_crawler.test.ts
+++ b/test/core/crawlers/browser_crawler.test.ts
@@ -642,6 +642,49 @@ describe('BrowserCrawler', () => {
});
});
+ test('should persist cookies set during requestHandler for the next request', async () => {
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ const requestList = await RequestList.open(null, [
+ { url: `${serverAddress}/?q=cookie-1` },
+ { url: `${serverAddress}/?q=cookie-2` },
+ ]);
+
+ const cookieStrings: string[] = [];
+ const crawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ saveResponseCookies: true,
+ sessionPool: new SessionPool({
+ maxPoolSize: 1,
+ }),
+ requestHandler: async ({ page, session, request }) => {
+ cookieStrings.push(session.cookieJar.getCookieStringSync(request.url));
+
+ if (request.url.includes('cookie-1')) {
+ const hostname = new URL(request.loadedUrl || request.url).hostname;
+ await page.setCookie({
+ name: 'HANDLER_COOKIE',
+ value: 'from-request-handler',
+ domain: hostname,
+ path: '/',
+ expires: Date.now() / 1000 + 3600,
+ });
+ }
+ },
+ });
+
+ await crawler.run();
+
+ expect(cookieStrings).toHaveLength(2);
+ // First request has no handler cookie yet (it is set during that handler).
+ expect(cookieStrings[0]).not.toContain('HANDLER_COOKIE=from-request-handler');
+ // Second request must see the cookie persisted after the first requestHandler.
+ expect(cookieStrings[1]).toContain('HANDLER_COOKIE=from-request-handler');
+ });
+
test.concurrent('should throw on "blocked" status codes', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);