Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
CheerioCrawler SessionPool implementation and Basic CookieJar per session logic.
Petr Patek69d261615e80
diff --git a/package.json b/package.jsonindex 801d64ac2..3d03824b4 100644--- a/package.json+++ b/package.json@@ -65,6 +65,7 @@"rimraf": "^2.7.1","semver": "^6.3.0","socket.io": "^2.2.0",+ "tough-cookie": "^3.0.1","underscore": "^1.9.1","ws": "^6.2.1","xregexp": "^4.2.4"diff --git a/src/constants.js b/src/constants.jsindex fd0ed84d1..be3bf2d16 100644--- a/src/constants.js+++ b/src/constants.js@@ -123,3 +123,5 @@ export const APIFY_API_BASE_URL = 'https://api.apify.com/v2';export const BASIC_CRAWLER_TIMEOUT_MULTIPLIER = 10;export const COUNTRY_CODE_REGEX = /^[A-Z]{2}$/;++export const STATUS_CODES_BLOCKED = [403, 429, 401];diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 4e6ca9cca..62aaad45d 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -8,9 +8,9 @@ import { checkParamOrThrow } from 'apify-client/build/utils';import contentTypeParser from 'content-type';import { readStreamToString, concatStreamToBuffer } from 'apify-shared/streams_utilities';import BasicCrawler from './basic_crawler';-import { addTimeoutToPromise, parseContentTypeFromResponse } from '../utils';+import { addTimeoutToPromise, parseContentTypeFromResponse, getCookiesFromResponse, getCookieHeader } from '../utils';import { getApifyProxyUrl } from '../actor';-import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';+import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER, STATUS_CODES_BLOCKED } from '../constants';import { requestAsBrowser } from '../utils_request';import { TimeoutError } from '../errors';@@ -39,6 +39,9 @@ const DEFAULT_OPTIONS = {},},additionalMimeTypes: [],+ useSessionPool: false,+ sessionPoolOptions: {},+ persistCookiesPerSession: false,};/**@@ -284,6 +287,9 @@ class CheerioCrawler {handleFailedRequestFunction,autoscaledPoolOptions,prepareRequestFunction,+ useSessionPool,+ sessionPoolOptions,+ persistCookiesPerSession, // @TODO: Better naming} = _.defaults({}, options, DEFAULT_OPTIONS);checkParamOrThrow(handlePageFunction, 'options.handlePageFunction', 'Function');@@ -297,9 +303,15 @@ class CheerioCrawler {checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe [String]');checkParamOrThrow(prepareRequestFunction, 'options.prepareRequestFunction', 'Maybe Function');checkParamOrThrow(additionalMimeTypes, 'options.additionalMimeTypes', '[String]');+ checkParamOrThrow(useSessionPool, 'options.useSessionPool', 'Boolean');+ checkParamOrThrow(sessionPoolOptions, 'options.sessionPoolOptions', 'Object');+ checkParamOrThrow(persistCookiesPerSession, 'options.persistCookiesPerSession', 'Boolean');// Enforce valid proxy configurationif (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');if (useApifyProxy && proxyUrls) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrls"!');+ if (persistCookiesPerSession && !useSessionPool) {+ throw new Error('Cannot use "options.persistCookiesPerSession" without "options.useSessionPool"');+ }this.supportedMimeTypes = new Set(DEFAULT_MIME_TYPES);if (additionalMimeTypes.length) this._extendSupportedMimeTypes(additionalMimeTypes);@@ -315,6 +327,7 @@ class CheerioCrawler {this.proxyUrls = _.shuffle(proxyUrls);this.lastUsedProxyUrlIndex = 0;this.prepareRequestFunction = prepareRequestFunction;+ this.persistCookiesPerSession = persistCookiesPerSession;this.basicCrawler = new BasicCrawler({// Basic crawler options.@@ -330,6 +343,10 @@ class CheerioCrawler {minConcurrency,maxConcurrency,autoscaledPoolOptions,++ // Session pool options+ sessionPoolOptions,+ useSessionPool,});this.isRunningPromise = null;@@ -352,14 +369,18 @@ class CheerioCrawler {** @ignore*/- async _handleRequestFunction({ request, autoscaledPool }) {- if (this.prepareRequestFunction) await this.prepareRequestFunction({ request });+ async _handleRequestFunction({ request, autoscaledPool, session }) {+ if (this.prepareRequestFunction) await this.prepareRequestFunction({ request, session });const { dom, isXmlOrHtml, body, contentType, responseStream: response } = await addTimeoutToPromise(- this._requestFunction({ request }),+ this._requestFunction({ request, session }),this.requestTimeoutMillis,`CheerioCrawler: request timed out after ${this.requestTimeoutMillis / 1000} seconds.`,);+ if (this.persistCookiesPerSession) {+ session = this._updateSessionCookies(session, response);+ }+request.loadedUrl = response.url;const $ = dom ? cheerio.load(dom, { xmlMode: isXmlOrHtml }) : null;@@ -386,6 +407,7 @@ class CheerioCrawler {request,response,autoscaledPool,+ session,};return addTimeoutToPromise(this.handlePageFunction(context),@@ -400,9 +422,15 @@ class CheerioCrawler {* received content type matches text/html, application/xml, application/xhtml+xml.* @ignore*/- async _requestFunction({ request }) {+ async _requestFunction({ request, session }) {// Using the streaming API of Request to be able to// handle the response based on headers receieved.++ if (this.persistCookiesPerSession) {+ const { headers = {} } = request;+ headers.Cookie = getCookieHeader(session.cookies);+ }+const opts = this._getRequestOptions(request);let responseStream;@@ -432,6 +460,8 @@ class CheerioCrawler {// It's not a JSON so it's probably some text. Get the first 100 chars of it.throw new Error(`CheerioCrawler: ${statusCode} - Internal Server Error: ${body.substr(0, 100)}`);+ } else if (STATUS_CODES_BLOCKED.includes(statusCode)) {+ this._handleBlockedRequest(session, statusCode);} else if (type === 'text/html' || type === 'application/xhtml+xml' || type === 'application/xml') {const dom = await this._parseHtmlToDom(responseStream);return ({ dom, isXmlOrHtml: type.includes('xml'), responseStream, contentType });@@ -527,6 +557,41 @@ class CheerioCrawler {}});}++ /**+ * @param session {Session} - `Session` instance to be updated.+ * @param response+ * @return {Session}+ * @private+ */+ _updateSessionCookies(session, response) {+ // update cookies for session+ const newCookies = getCookiesFromResponse(response);+ const { cookies: oldCookies } = session;++ for (const cookie of newCookies) {+ const cookieIndex = oldCookies.find(c => c.name === cookie.name);+ if (cookieIndex) {+ oldCookies[cookieIndex] = cookie;+ } else {+ oldCookies.push(cookie);+ }+ }+ session.cookies = oldCookies;++ return session;+ }++ /**+ * Handles blocked request+ * @param session {Session}+ * @param statusCode {Number}+ * @private+ */+ _handleBlockedRequest(session, statusCode) {+ if (session) session.retire();+ throw new Error(`CheerioCrawler: Request blocked - received ${statusCode} status code`);+ }}diff --git a/src/utils.js b/src/utils.jsindex c65df32e6..c93b6cd46 100644--- a/src/utils.js+++ b/src/utils.js@@ -17,6 +17,7 @@ import { checkParamOrThrow } from 'apify-client/build/utils';import { version as apifyClientVersion } from 'apify-client/package.json';import mime from 'mime-types';import path from 'path';+import { Cookie } from 'tough-cookie';import { URL } from 'url';import { version as apifyVersion } from '../package.json';import { USER_AGENT_LIST } from './constants';@@ -666,6 +667,25 @@ export const parseContentTypeFromResponse = (response) => {return parsedContentType;};+export const getCookiesFromResponse = (response) => {+ const { headers } = response;+ let cookies;++ if (Array.isArray(headers['set-cookie'])) {+ cookies = headers['set-cookie'].map(Cookie.parse);+ } else {+ cookies = [Cookie.parse(headers['set-cookie'])];+ }++ return cookies;+};++export const getCookieHeader = (cookies) => {+ if (cookies.length < 1) {+ return;+ }+ return cookies.map(cookie => cookie && cookie.cookieString()).join(';');+};/*** A namespace that contains various utilities.*diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex 7c2dc96ee..9af10de33 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -9,6 +9,9 @@ import express from 'express';import bodyParser from 'body-parser';import Apify from '../../build';import { sleep } from '../../build/utils';+import { Session } from '../../build/session_pool/session';+import { STATUS_CODES_BLOCKED } from '../../build/constants';+import { emptyLocalStorageSubdir, LOCAL_STORAGE_DIR } from '../_helper';// Add common props to mocked request responses.const responseMock = {@@ -791,4 +794,85 @@ describe('CheerioCrawler', () => {});});});++ describe('SessionPool', () => {+ const sources = ['http://example.com/'];+ let requestList;++ beforeEach(async () => {+ process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;+ requestList = await Apify.openRequestList('test', sources);+ });++ afterEach(() => {+ emptyLocalStorageSubdir('key_value_stores/default');+ });++ test('should work', async () => {+ const crawler = new Apify.CheerioCrawler({+ requestList,+ useSessionPool: true,+ persistCookiesPerSession: false,+ handlePageFunction: async ({ session }) => {+ expect(session).toBeInstanceOf(Session);+ },+ });+ await crawler.run();+ });++ test('should retire session on "blocked" status codes', async () => {+ for (const code of STATUS_CODES_BLOCKED) {+ const failed = [];+ const sessions = [];+ const crawler = new Apify.CheerioCrawler({+ requestList: await getRequestListForMock({+ statusCode: code,+ error: false,+ headers: { 'Content-type': 'text/html' },+ }),+ useSessionPool: true,+ persistCookiesPerSession: false,+ maxRequestRetries: 0,+ handlePageFunction: ({ session }) => {+ sessions.push(session);+ },+ handleFailedRequestFunction: async ({ request }) => {+ failed.push(request);+ },+ });+ const oldCall = crawler._handleBlockedRequest;+ crawler._handleBlockedRequest = (session, statusCode) => {+ sessions.push(session);+ return oldCall(session, statusCode);+ };+ await crawler.run();++ sessions.forEach((session) => {+ expect(session.errorScore).toBeGreaterThanOrEqual(session.maxErrorScore);+ });++ failed.forEach((request) => {+ expect(request.errorMessages[0].includes(`Request blocked - received ${code} status code`)).toBeTruthy();+ });+ }+ });++ test('should throw when "options.useSessionPool" false and "options.persistCookiesPerSession" is true', async () => {+ try {+ new Apify.CheerioCrawler({+ requestList: await getRequestListForMock({++ }),+ useSessionPool: false,+ persistCookiesPerSession: true,+ maxRequestRetries: 0,+ handlePageFunction: () => {+ },+ });+ } catch (e) {+ expect(e.message).toEqual('Cannot use "options.persistCookiesPerSession" without "options.useSessionPool"');+ }+ });+ // @TODO: TEST COKIE PERSISTANCE.+ });});diff --git a/test/utils.test.js b/test/utils.test.jsindex 9c507e400..f4b339e4a 100644--- a/test/utils.test.js+++ b/test/utils.test.js@@ -9,6 +9,7 @@ import requestPromise from 'request-promise-native';import LruCache from 'apify-shared/lru_cache';import log from 'apify-shared/log';import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';+import { Cookie } from 'tough-cookie';import * as utils from '../build/utils';import Apify from '../build/index';@@ -963,3 +964,48 @@ describe('utils.parseContentTypeFromResponse', () => {expect(parsedReallyBad.parameters.charset).toBe('utf-8');});});++describe('utils.getCookiesFromResponse & utils.getCookieHeader', () => {+ test('should parse cookies if set-cookie is array', () => {+ const headers = {};+ const dummyCookies = ['CSRF=e8b667; Domain=example.com; Secure', 'id=a3fWa; Expires=Wed, 21 Oct 2015 07:28:00 GMT'];+ headers['set-cookie'] = dummyCookies;+ const cookies = utils.getCookiesFromResponse({ headers });++ cookies.forEach((cookie) => {+ expect(cookie).toBeInstanceOf(Cookie);+ });++ expect(dummyCookies[0]).toEqual(cookies[0].toString());+ expect(dummyCookies[1]).toEqual(cookies[1].toString());++ const parsed = utils.getCookieHeader(cookies);+ expect(parsed).toEqual('CSRF=e8b667;id=a3fWa');+ });++ test('should parse cookies if set-cookie is string', () => {+ const headers = {};+ const dummyCookie = 'CSRF=e8b667; Domain=example.com; Secure';+ headers['set-cookie'] = dummyCookie;+ const cookies = utils.getCookiesFromResponse({ headers });++ expect(cookies).toHaveLength(1);+ expect(dummyCookie).toEqual(cookies[0].toString());+ expect(cookies[0]).toBeInstanceOf(Cookie);+ const parsed = utils.getCookieHeader(cookies);+ expect(parsed).toEqual('CSRF=e8b667');+ });++ test('should not throw error on parsing invalid cookie', () => {+ const headers = {};+ const dummyCookie = 'totally Invalid Cookie $@$@#$**';+ headers['set-cookie'] = dummyCookie;+ const cookies = utils.getCookiesFromResponse({ headers });++ expect(cookies).toHaveLength(1);+ expect(cookies[0]).toBeUndefined();++ const parsed = utils.getCookieHeader(cookies);+ expect(parsed).toBeFalsy();+ });+});