Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
finalized the API.
Petr Patekd713ca5beb3a
diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex 3f98aa20f..16ef95f6d 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -300,7 +300,7 @@ class PuppeteerCrawler {// setting cookies to pageif (this.persistCookiesPerSession) {- await page.setCookie(...session.getCookies(request.url));+ await page.setCookie(...session.getPuppeteerCookies(request.url));}}@@ -312,7 +312,7 @@ class PuppeteerCrawler {// save cookiesif (this.persistCookiesPerSession) {const cookies = await page.cookies(request.loadedUrl);- session.setCookies(cookies, request.loadedUrl);+ session.setPuppeteerCookies(cookies, request.loadedUrl);}await addTimeoutToPromise(diff --git a/src/session_pool/session.js b/src/session_pool/session.jsindex c3407bc76..0ce019b77 100644--- a/src/session_pool/session.js+++ b/src/session_pool/session.js@@ -8,9 +8,6 @@ import { getCookiesFromResponse } from './session_utils';const { Cookie } = tough;-const PUPPETEER_COOKIE_TYPE = 1;-const TOUGH_COOKIE_TYPE = 2;-/*** Class aggregating data for session.@@ -55,7 +52,6 @@ export class Session {} = options;const { expiresAt = new Date(Date.now() + (maxAgeSecs * 1000)) } = options;- const type = options.type ? parseInt(options.type, 10) : TOUGH_COOKIE_TYPE;// ValidationcheckParamOrThrow(id, 'options.id', 'String');@@ -68,7 +64,6 @@ export class Session {checkParamOrThrow(errorScore, 'options.errorScore', 'Number');checkParamOrThrow(maxUsageCount, 'options.maxUsageCount', 'Number');checkParamOrThrow(sessionPool, 'options.sessionPool', 'Object');- checkParamOrThrow(type, 'options.type', 'Number');// sessionPool must be instance of SessionPool.if (sessionPool.constructor.name !== 'SessionPool') {@@ -91,7 +86,6 @@ export class Session {this.errorScore = errorScore; // indicates number of markBaded request with the sessionthis.maxUsageCount = maxUsageCount;this.sessionPool = sessionPool;- this.type = type;}/**@@ -159,7 +153,6 @@ export class Session {createdAt: this.createdAt.toISOString(),usageCount: this.usageCount,errorScore: this.errorScore,- type: this.type,};}@@ -203,7 +196,6 @@ export class Session {/*** Sets cookies from response to the cookieJar.- * Accepts responses with headers accessible by `response.headers` or `response.headers()`.* Parses cookies from `set-cookie` header and sets them to `Session.cookieJar`.* @param response*/@@ -211,7 +203,7 @@ export class Session {try {const cookies = getCookiesFromResponse(response).filter(c => c);- this.setCookies(cookies, response.url);+ this._setCookies(cookies, response.url);} catch (e) {// if invalid Cookie header is provided just log the exception.log.exception(e, 'Session: Could not get cookies from response');@@ -220,43 +212,27 @@ export class Session {/*** Set cookies to session cookieJar.- * Cookies array should be [tough-cookie](https://github.com/salesforce/tough-cookie)- * or [puppeteer](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls) cookie compatible.- * @param cookies {Array<Cookie>}+ * Cookies array should be [puppeteer](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls) cookie compatible.+ * @param cookies {Array<Object>}* @param url {String}*/- setCookies(cookies, url) {- const isPuppeteerCookies = !!cookies[0] && !!cookies[0].name;-- for (const cookie of cookies) {- let cookieToSave;-- if (isPuppeteerCookies) {- cookieToSave = this._puppeteerCookieToTough(cookie);- this.type = PUPPETEER_COOKIE_TYPE;- } else {- cookieToSave = cookie;- this.type = TOUGH_COOKIE_TYPE;- }-- this.cookieJar.setCookieSync(cookieToSave, url, { ignoreError: false });+ setPuppeteerCookies(cookies, url) {+ try {+ this._setCookies(cookies.map(this._puppeteerCookieToTough), url);+ } catch (e) {+ // if invalid cookies are provided just log the exception. No need to retry the request automatically.+ log.exception(e, 'Session: Could not set cookies in puppeteer format.');}}/**- * Get cookies.- * Gets a array of cookies in the format corresponding to the type property.- * @param url {String}+ * Gets cookies in puppeteer ready to be used with `page.setCookie`.+ * @param url {String} - website url. Only cookies stored for this url will be returned* @return {Array<Object>}*/- getCookies(url) {- const shouldTransformToPuppeteerCookie = this.type === PUPPETEER_COOKIE_TYPE;+ getPuppeteerCookies(url) {const cookies = this.cookieJar.getCookiesSync(url);- if (!shouldTransformToPuppeteerCookie) {- return cookies;- }-return cookies.map(this._toughCookieToPuppeteer);}@@ -305,4 +281,16 @@ export class Session {httpOnly: toughCookie.httpOnly,};}++ /**+ * Sets cookies.+ * @param cookies+ * @param url+ * @private+ */+ _setCookies(cookies, url) {+ for (const cookie of cookies) {+ this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });+ }+ }}