Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix inconsistency between puppeteer and sessionPool cookies, add tests (#943)
Andrey Bykovf7f38edf70d5
diff --git a/src/crawlers/browser_crawler.js b/src/crawlers/browser_crawler.jsindex 715b03012..2b52093a1 100644--- a/src/crawlers/browser_crawler.js+++ b/src/crawlers/browser_crawler.js@@ -261,6 +261,7 @@ class BrowserCrawler extends BasicCrawler {...basicCrawlerOptions} = options;+ // Cookies should be persisted per session only if session pool is usedif (!useSessionPool && persistCookiesPerSession) {throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');}@@ -285,7 +286,7 @@ class BrowserCrawler extends BasicCrawler {this.postNavigationHooks = postNavigationHooks;if (useSessionPool) {- this.persistCookiesPerSession = persistCookiesPerSession || true;+ this.persistCookiesPerSession = persistCookiesPerSession !== undefined ? persistCookiesPerSession : true;this.sessionPool = new SessionPool({...sessionPoolOptions,@@ -294,6 +295,8 @@ class BrowserCrawler extends BasicCrawler {// Assuming there are not more than 20 browsers running at once;this.sessionPool.setMaxListeners(20);+ } else {+ this.persistCookiesPerSession = false;}const { preLaunchHooks = [], postLaunchHooks = [], ...rest } = browserPoolOptions;@@ -326,9 +329,9 @@ class BrowserCrawler extends BasicCrawler {const { request, session } = crawlingContext;- if (this.persistCookiesPerSession) {- const cookies = crawlingContext.session.getPuppeteerCookies(request.url);- await crawlingContext.browserController.setCookies(page, cookies);+ const sessionCookies = crawlingContext.session.getPuppeteerCookies(request.url);+ if (sessionCookies.length) {+ await crawlingContext.browserController.setCookies(page, sessionCookies);}try {diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 8fb21c643..a72bdc712 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -231,7 +231,7 @@ const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {* If you're not sure, just keep the default value and the concurrency will scale up automatically.* @property {number} [maxConcurrency=1000]* Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.- * @property {boolean} [useSessionPool=false]+ * @property {boolean} [useSessionPool=true]* If set to true Crawler will automatically use Session Pool. It will automatically retire sessions on 403, 401 and 429 status codes.* It also marks Session as bad after a request timeout.* @property {SessionPoolOptions} [sessionPoolOptions]@@ -376,7 +376,7 @@ class CheerioCrawler extends BasicCrawler {proxyConfiguration,prepareRequestFunction,postResponseFunction,- persistCookiesPerSession = false,+ persistCookiesPerSession,// BasicCrawlerautoscaledPoolOptions = CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS,@@ -393,8 +393,9 @@ class CheerioCrawler extends BasicCrawler {handleRequestTimeoutSecs: (requestTimeoutSecs + handlePageTimeoutSecs * 2) + 5,});- if (persistCookiesPerSession && !this.useSessionPool) {- throw new Error('Cannot use "options.persistCookiesPerSession" without "options.useSessionPool"');+ // Cookies should be persisted per session only if session pool is used+ if (!this.useSessionPool && persistCookiesPerSession) {+ throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');}this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);@@ -412,7 +413,12 @@ class CheerioCrawler extends BasicCrawler {this.prepareRequestFunction = prepareRequestFunction;this.postResponseFunction = postResponseFunction;this.proxyConfiguration = proxyConfiguration;- this.persistCookiesPerSession = persistCookiesPerSession;++ if (this.useSessionPool) {+ this.persistCookiesPerSession = persistCookiesPerSession !== undefined ? persistCookiesPerSession : true;+ } else {+ this.persistCookiesPerSession = false;+ }}/**@@ -538,7 +544,7 @@ class CheerioCrawler extends BasicCrawler {// Using the streaming API of Request to be able to// handle the response based on headers receieved.- if (this.persistCookiesPerSession) {+ if (this.useSessionPool) {const { headers } = request;headers.Cookie = session.getCookieString(request.url);}diff --git a/src/session_pool/session.js b/src/session_pool/session.jsindex 2d11c1902..1bc2855b7 100644--- a/src/session_pool/session.js+++ b/src/session_pool/session.js@@ -305,7 +305,7 @@ export class Session {* @private*/_puppeteerCookieToTough(puppeteerCookie) {- const isExpiresValid = puppeteerCookie.expires && typeof puppeteerCookie.expires === 'number';+ const isExpiresValid = puppeteerCookie.expires && typeof puppeteerCookie.expires === 'number' && puppeteerCookie.expires > 0;const expires = isExpiresValid ? new Date(puppeteerCookie.expires * 1000) : this._getDefaultCookieExpirationDate(this.maxAgeSecs);const domain = typeof puppeteerCookie.domain === 'string' && puppeteerCookie.domain.startsWith('.')? puppeteerCookie.domain.slice(1)@@ -331,7 +331,8 @@ export class Session {return {name: toughCookie.key,value: toughCookie.value,- expires: new Date(toughCookie.expires).getTime(),+ // Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms+ expires: new Date(toughCookie.expires).getTime() / 1000,domain: toughCookie.domain,path: toughCookie.path,secure: toughCookie.secure,diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex 117abb3a4..405d38ea4 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -738,7 +738,7 @@ describe('CheerioCrawler', () => {},});} catch (e) {- expect(e.message).toEqual('Cannot use "options.persistCookiesPerSession" without "options.useSessionPool"');+ expect(e.message).toEqual('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');}});diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.jsindex c5769e92a..938597201 100644--- a/test/crawlers/puppeteer_crawler.test.js+++ b/test/crawlers/puppeteer_crawler.test.js@@ -200,4 +200,39 @@ describe('PuppeteerCrawler', () => {expect(loadedUserAgent).toEqual(opts.userAgent);});++ test('should set cookies assigned to session to page', async () => {+ const cookies = [+ {+ name: 'example_cookie_name',+ domain: '.example.com',+ value: 'example_cookie_value',+ expires: -1,+ },+ ];++ let pageCookies;+ let sessionCookies;++ const puppeteerCrawler = new Apify.PuppeteerCrawler({+ requestList,+ useSessionPool: true,+ persistCookiesPerSession: true,+ sessionPoolOptions: {+ createSessionFunction: (sessionPool) => {+ const session = new Apify.Session({ sessionPool });+ session.setPuppeteerCookies(cookies, 'http://www.example.com');+ return session;+ },+ },+ handlePageFunction: async ({ page, session }) => {+ pageCookies = await page.cookies().then((cks) => cks.map((c) => `${c.name}=${c.value}`).join('; '));+ sessionCookies = session.getCookieString('http://www.example.com');+ },+ });++ await puppeteerCrawler.run();++ expect(pageCookies).toEqual(sessionCookies);+ });});diff --git a/test/session_pool/session.test.js b/test/session_pool/session.test.jsindex c3071a4ec..70fdfe562 100644--- a/test/session_pool/session.test.js+++ b/test/session_pool/session.test.js@@ -202,6 +202,17 @@ describe('Session - testing session behaviour ', () => {expect(session.getCookieString(url)).toBe('cookie1=my-cookie; cookie2=your-cookie');});+ test('setPuppeteerCookies should work for session (with expiration date: -1) cookies', () => {+ const url = 'https://example.com';+ const cookies = [+ { name: 'session_cookie', value: 'session-cookie-value', expires: -1 },+ ];++ session = new Session({ sessionPool });+ session.setPuppeteerCookies(cookies, url);+ expect(session.getCookieString(url)).toBe('session_cookie=session-cookie-value');+ });+test('setPuppeteerCookies works with leading dots in domains', () => {const url = 'https://www.example.com';const cookies = [