Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactored session API.
Petr Patek68a034a40be6
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 629d359b2..6c589d2c9 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -403,7 +403,7 @@ class CheerioCrawler {}if (this.persistCookiesPerSession) {- session.putResponse(response);+ session.setCookiesFromResponse(response);}request.loadedUrl = response.url;diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex d5825e38b..8834b17cc 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -290,9 +290,9 @@ class PuppeteerCrawler {const browser = page.browser();session = browser[BROWSER_SESSION_KEY_NAME];- // setting cookies for page+ // setting cookies to pageif (this.persistCookiesPerSession) {- await page.setCookie(...session.getPuppeteerCookies(request.url));+ await page.setCookie(...session.getCookies(request.url));}}@@ -304,7 +304,7 @@ class PuppeteerCrawler {// save cookiesif (this.persistCookiesPerSession) {const cookies = await page.cookies(request.loadedUrl);- session.putPuppeteerCookies(cookies, request.loadedUrl);+ session.setCookies(cookies, request.loadedUrl);}await addTimeoutToPromise(diff --git a/src/session_pool/session.js b/src/session_pool/session.jsindex 207a503d4..6eece4e7c 100644--- a/src/session_pool/session.js+++ b/src/session_pool/session.js@@ -8,6 +8,9 @@ import { getCookiesFromResponse } from './session_utils';const { Cookie } = tough;+const PUPPETEER_COOKIE_TYPE = 1;+const TOUGH_COOKIE_TYPE = 2;+/*** Class aggregating data for session.@@ -52,6 +55,7 @@ export class Session {} = options;const { expiresAt = new Date(Date.now() + (maxAgeSecs * 1000)) } = options;+ const type = options.type ? parseInt(options.type, 10) : TOUGH_COOKIE_TYPE;// ValidationcheckParamOrThrow(id, 'options.id', 'String');@@ -64,6 +68,7 @@ export class Session {checkParamOrThrow(errorScore, 'options.errorScore', 'Number');checkParamOrThrow(maxUsageCount, 'options.maxUsageCount', 'Number');checkParamOrThrow(sessionPool, 'options.sessionPool', 'Object');+ checkParamOrThrow(type, 'options.type', 'Number');// sessionPool must be instance of SessionPool.if (sessionPool.constructor.name !== 'SessionPool') {@@ -86,6 +91,7 @@ export class Session {this.errorScore = errorScore; // indicates number of markBaded request with the sessionthis.maxUsageCount = maxUsageCount;this.sessionPool = sessionPool;+ this.type = type;}/**@@ -153,6 +159,7 @@ export class Session {createdAt: this.createdAt.toISOString(),usageCount: this.usageCount,errorScore: this.errorScore,+ type: this.type,};}@@ -196,46 +203,56 @@ export class Session {/*** Sets cookies from response to the cookieJar.+ * Accepts responses with headers accessible by `response.headers` or `response.headers()`.* Parses cookies from `set-cookie` header and sets them to `Session.cookieJar`.* @param response*/- putResponse(response) {+ setCookiesFromResponse(response) {const cookies = getCookiesFromResponse(response).filter(c => c);this.setCookies(cookies, response.url);}- /**- * Persists puppeteer cookies to session for reuse.- * @param puppeteerCookies - cookie from puppeteer `page.cookies` method.- * @param url - Loaded url from page function.- */- putPuppeteerCookies(puppeteerCookies, url) {- const cookies = puppeteerCookies.map(puppeteerCookie => this._transformPuppeteerCookie(puppeteerCookie));-- this.setCookies(cookies, url);- }-/*** Set cookies to session cookieJar.- * Cookies array should be compatible with tough-cookie.+ * Cookies array should be [tough-cookie](https://github.com/salesforce/tough-cookie)+ * or [puppeteer](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls) cookie compatible.* @param cookies {Array<Cookie>}* @param url {String}*/setCookies(cookies, url) {+ const isPuppeteerCookies = !!cookies[0].name;+for (const cookie of cookies) {- this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });+ let cookieToSave;++ if (isPuppeteerCookies) {+ cookieToSave = this._puppeteerCookieToTough(cookie);+ this.type = PUPPETEER_COOKIE_TYPE;+ } else {+ cookieToSave = cookie;+ this.type = TOUGH_COOKIE_TYPE;+ }++ this.cookieJar.setCookieSync(cookieToSave, url, { ignoreError: false });}}/*** Get cookies.- * Gets a array of `tough-cookie` Cookie instances.+ * Gets a array of cookies in the format corresponding to the type property.* @param url {String}* @return {Array<Object>}*/getCookies(url) {- return this.cookieJar.getCookiesSync(url);+ const shouldTransformToPuppeteerCookie = this.type === PUPPETEER_COOKIE_TYPE;+ const cookies = this.cookieJar.getCookiesSync(url);++ if (!shouldTransformToPuppeteerCookie) {+ return cookies;+ }++ return cookies.map(this._toughCookieToPuppeteer);}/**@@ -247,16 +264,6 @@ export class Session {return this.cookieJar.getCookieStringSync(url, {});}- /**- * Gets cookies in format ready for puppeteer.- * @param url {String}- * @return {*}- */- getPuppeteerCookies(url) {- const cookies = this.getCookies(url);- return cookies.map(cookie => this._transformToughCookie(cookie));- }-/*** Transforms puppeteer cookie to tough-cookie.@@ -264,7 +271,7 @@ export class Session {* @return {Cookie}* @private*/- _transformPuppeteerCookie(puppeteerCookie) {+ _puppeteerCookieToTough(puppeteerCookie) {return new Cookie({key: puppeteerCookie.name,value: puppeteerCookie.value,@@ -282,7 +289,7 @@ export class Session {* @return {Object} - puppeteer cookie* @private*/- _transformToughCookie(toughCookie) {+ _toughCookieToPuppeteer(toughCookie) {return {name: toughCookie.key,value: toughCookie.value,diff --git a/src/session_pool/session_utils.js b/src/session_pool/session_utils.jsindex cb3d40db2..625814e9b 100644--- a/src/session_pool/session_utils.js+++ b/src/session_pool/session_utils.js@@ -6,7 +6,7 @@ import { Cookie } from 'tough-cookie';* @return {undefined|Array}*/export const getCookiesFromResponse = (response) => {- const { headers } = response;+ const headers = typeof response.headers === 'function' ? response.headers() : response.headers;let cookies;if (Array.isArray(headers['set-cookie'])) {diff --git a/test/session_pool/session.test.js b/test/session_pool/session.test.jsindex 15b622e75..2958c9233 100644--- a/test/session_pool/session.test.js+++ b/test/session_pool/session.test.js@@ -153,13 +153,13 @@ describe('Session - testing session behaviour ', () => {];const newSession = new Session({ sessionPool: new SessionPool() });const url = 'https://example.com';- newSession.putResponse({ headers, url });+ newSession.setCookiesFromResponse({ headers, url });let cookies = newSession.getCookieString(url);expect(cookies).toEqual('CSRF=e8b667; id=a3fWa');const newCookie = 'ABCD=1231231213; Domain=example.com; Secure';- newSession.putResponse({ headers: { 'set-cookie': newCookie }, url });+ newSession.setCookiesFromResponse({ headers: { 'set-cookie': newCookie }, url });cookies = newSession.getCookieString(url);expect(cookies).toEqual('CSRF=e8b667; id=a3fWa; ABCD=1231231213');});@@ -174,7 +174,7 @@ describe('Session - testing session behaviour ', () => {];const newSession = new Session({ sessionPool: new SessionPool() });const url = 'https://example.com';- newSession.putResponse({ headers, url });+ newSession.setCookiesFromResponse({ headers, url });const old = newSession.getState();