Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Improved code.
Petr Patekf34b84d4decb
diff --git a/src/constants.js b/src/constants.jsindex be3bf2d16..15ffef8d6 100644--- a/src/constants.js+++ b/src/constants.js@@ -124,4 +124,4 @@ export const BASIC_CRAWLER_TIMEOUT_MULTIPLIER = 10;export const COUNTRY_CODE_REGEX = /^[A-Z]{2}$/;-export const STATUS_CODES_BLOCKED = [403, 429, 401];+export const STATUS_CODES_BLOCKED = [401, 403, 429];diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 01136c872..ef3f87078 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -9,9 +9,9 @@ import { checkParamOrThrow } from 'apify-client/build/utils';import contentTypeParser from 'content-type';import { readStreamToString, concatStreamToBuffer } from 'apify-shared/streams_utilities';import BasicCrawler from './basic_crawler';-import { addTimeoutToPromise, parseContentTypeFromResponse, updateSessionCookies, getCookieHeader } from '../utils';+import { addTimeoutToPromise, parseContentTypeFromResponse } from '../utils';import { getApifyProxyUrl } from '../actor';-import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER, STATUS_CODES_BLOCKED } from '../constants';+import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';import { requestAsBrowser } from '../utils_request';import { TimeoutError } from '../errors';@@ -342,6 +342,7 @@ class CheerioCrawler {this.lastUsedProxyUrlIndex = 0;this.prepareRequestFunction = prepareRequestFunction;this.persistCookiesPerSession = persistCookiesPerSession;+ this.useSessionPool = useSessionPool;this.basicCrawler = new BasicCrawler({// Basic crawler options.@@ -391,8 +392,12 @@ class CheerioCrawler {`CheerioCrawler: request timed out after ${this.requestTimeoutMillis / 1000} seconds.`,);+ if (this.useSessionPool) {+ this._handleBlockedRequest(session, response.statusCode);+ }+if (this.persistCookiesPerSession) {- session = updateSessionCookies(session, response);+ session.setCookiesToJar(response);}request.loadedUrl = response.url;@@ -442,7 +447,7 @@ class CheerioCrawler {if (this.persistCookiesPerSession) {const { headers = {} } = request;- headers.Cookie = getCookieHeader(session.cookies);+ headers.Cookie = session.getCookieString(request.url);}const opts = this._getRequestOptions(request);@@ -477,8 +482,6 @@ class CheerioCrawler {// It's not a JSON so it's probably some text. Get the first 100 chars of it.throw new Error(`CheerioCrawler: ${statusCode} - Internal Server Error: ${body.substr(0, 100)}`);- } else if (STATUS_CODES_BLOCKED.includes(statusCode)) {- this._handleBlockedRequest(session, statusCode);} else if (type === 'text/html' || type === 'application/xhtml+xml' || type === 'application/xml') {const dom = await this._parseHtmlToDom(response);return ({ dom, isXml: type.includes('xml'), response, contentType });@@ -602,8 +605,11 @@ class CheerioCrawler {* @private*/_handleBlockedRequest(session, statusCode) {- if (session) session.retire();- throw new Error(`CheerioCrawler: Request blocked - received ${statusCode} status code`);+ const isBlocked = session.checkStatus(statusCode);++ if (isBlocked) {+ throw new Error(`CheerioCrawler: Request blocked - received ${statusCode} status code`);+ }}/**diff --git a/src/session_pool/session.js b/src/session_pool/session.jsindex af81792d0..633396ce3 100644--- a/src/session_pool/session.js+++ b/src/session_pool/session.js@@ -1,7 +1,10 @@import { cryptoRandomObjectId } from 'apify-shared/utilities';import { checkParamOrThrow } from 'apify-client/build/utils';+import tough from 'tough-cookie';import EVENTS from './events';+import { STATUS_CODES_BLOCKED } from '../constants';+import { getCookiesFromResponse } from './session_utils';/**@@ -33,7 +36,8 @@ export class Session {constructor(options = {}) {const {id = `session_${cryptoRandomObjectId(10)}`,- cookies = [],+ cookies = [], // @TODO: Delete, deprecate or leave it as custom cookie persistance?+ cookieJar = new tough.CookieJar(),maxAgeSecs = 3000,userData = {},maxErrorScore = 3,@@ -67,6 +71,7 @@ export class Session {// Configurablethis.id = id;this.cookies = cookies;+ this.cookieJar = cookieJar.setCookie ? cookieJar : tough.CookieJar.fromJSON(JSON.stringify(cookieJar));this.maxAgeSecs = maxAgeSecs;this.userData = userData;this.maxErrorScore = maxErrorScore;@@ -138,6 +143,7 @@ export class Session {return {id: this.id,cookies: this.cookies,+ cookieJar: this.cookieJar.toJSON(),userData: this.userData,maxErrorScore: this.maxErrorScore,errorScoreDecrement: this.errorScoreDecrement,@@ -172,4 +178,40 @@ export class Session {this.errorScore += 1;this.usageCount += 1;}++ /**+ * Retires session based on status code.+ * @param statusCode {Number} - HTTP status code+ * @return {boolean} - whether the session was retired+ */+ checkStatus(statusCode) {+ const isBlocked = STATUS_CODES_BLOCKED.includes(statusCode);+ if (isBlocked) {+ this.retire();+ }+ return isBlocked;+ }++ /**+ * Sets cookies from response to the cookieJar.+ * Parses cookies from "set-cookie" header and sets them to "Session.cookieJar".+ * @param response+ */+ setCookiesToJar(response) {+ const cookies = getCookiesFromResponse(response).filter(c => c);++ for (const cookie of cookies) {+ this.cookieJar.setCookieSync(cookie, response.url, { ignoreError: true });+ }+ }++ /**+ * Wrapper around "tough" cookie jar `getCookieString` method.+ * @param url+ * @return {String};+ */+ getCookieString(url) {+ console.log(this.cookieJar);+ return this.cookieJar.getCookieStringSync(url, {});+ }}diff --git a/src/session_pool/session_utils.js b/src/session_pool/session_utils.jsnew file mode 100644index 000000000..cb3d40db2--- /dev/null+++ b/src/session_pool/session_utils.js@@ -0,0 +1,19 @@+import { Cookie } from 'tough-cookie';++/**+ *+ * @param response+ * @return {undefined|Array}+ */+export const getCookiesFromResponse = (response) => {+ const { headers } = response;+ let cookies;++ if (Array.isArray(headers['set-cookie'])) {+ cookies = headers['set-cookie'].map(Cookie.parse);+ } else {+ cookies = [Cookie.parse(headers['set-cookie'])];+ }++ return cookies;+};diff --git a/src/utils.js b/src/utils.jsindex eb86d2699..cd12ffc25 100644--- a/src/utils.js+++ b/src/utils.js@@ -17,7 +17,6 @@ import { checkParamOrThrow } from 'apify-client/build/utils';import { version as apifyClientVersion } from 'apify-client/package.json';import mime from 'mime-types';import path from 'path';-import { Cookie } from 'tough-cookie';import { URL } from 'url';import { version as apifyVersion } from '../package.json';import { USER_AGENT_LIST } from './constants';@@ -670,60 +669,6 @@ export const parseContentTypeFromResponse = (response) => {};};-/**- *- * @param response- * @return {undefined|Array}- */-export const getCookiesFromResponse = (response) => {- const { headers } = response;- let cookies;-- if (Array.isArray(headers['set-cookie'])) {- cookies = headers['set-cookie'].map(Cookie.parse);- } else {- cookies = [Cookie.parse(headers['set-cookie'])];- }-- return cookies;-};--/**- * Gets cookie string for the "Cookie" header- * @param cookies {Array}- * @return {String|undefined}- */-export const getCookieHeader = (cookies) => {- if (cookies.length < 1) {- return;- }- return cookies.map(cookie => cookie && cookie.cookieString()).join(';');-};--/**- * Updates cookies from response to the `Session` instance.- * @param session {Session} - `Session` instance to be updated.- * @param response- * @return {Session}- *- */-export const updateSessionCookies = (session, response) => {- const newCookies = getCookiesFromResponse(response);- const { cookies: oldCookies } = session;-- for (const cookie of newCookies) {- const cookieIndex = oldCookies.findIndex(c => c.key === cookie.key);-- if (cookieIndex >= 0) {- oldCookies[cookieIndex] = cookie;- } else {- oldCookies.push(cookie);- }- }- session.cookies = oldCookies;-- return session;-};/*** A namespace that contains various utilities.*diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex 15c1d2f8e..50a55b502 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -859,7 +859,7 @@ describe('CheerioCrawler', () => {useSessionPool: true,persistCookiesPerSession: false,maxRequestRetries: 0,- handlePageFunction: ({ session }) => {+ handlePageFunction: async ({ session }) => {sessions.push(session);},handleFailedRequestFunction: async ({ request }) => {@@ -900,33 +900,6 @@ describe('CheerioCrawler', () => {}});- test('should save cookies to session', async () => {- const cookie = 'SESSID=abcd123';- const sessions = [];- const requests = [];- const crawler = new Apify.CheerioCrawler({- requestList: await getRequestListForMock({- headers: { 'set-cookie': cookie, 'content-type': 'text/html' },- statusCode: 200,- }),- useSessionPool: true,- persistCookiesPerSession: true,- maxRequestRetries: 1,- handlePageFunction: async ({ session, request }) => {- sessions.push(session);- requests.push(request);- },-- });-- await crawler.run();- expect(sessions.length).toBeGreaterThan(1);- sessions.forEach((session) => {- expect(session.cookies).toHaveLength(1);- expect(session.cookies[0].toString()).toEqual(cookie);- });- });-test('should send cookies', async () => {const cookie = 'SESSID=abcd123';const requests = [];diff --git a/test/session_pool/session.test.js b/test/session_pool/session.test.jsindex 47f551a02..0af4616cb 100644--- a/test/session_pool/session.test.js+++ b/test/session_pool/session.test.js@@ -1,6 +1,7 @@import { Session } from '../../build/session_pool/session';import { SessionPool } from '../../build/session_pool/session_pool';import EVENTS from '../../build/session_pool/events';+import { STATUS_CODES_BLOCKED } from '../../build/constants';import Apify from '../../build';import { getApifyProxyUrl } from '../../build/actor';@@ -86,20 +87,23 @@ describe('Session - testing session behaviour ', () => {test('should get state', () => {const state = session.getState();- expect(state.id).toBeDefined(); // eslint-disable-line- expect(state.cookies).toBeDefined(); // eslint-disable-line- expect(state.userData).toBeDefined(); // eslint-disable-line- expect(state.maxErrorScore).toBeDefined(); // eslint-disable-line- expect(state.errorScoreDecrement).toBeDefined(); // eslint-disable-line- expect(state.expiresAt).toBeDefined(); // eslint-disable-line- expect(state.createdAt).toBeDefined(); // eslint-disable-line- expect(state.usageCount).toBeDefined(); // eslint-disable-line- expect(state.errorScore).toBeDefined(); // eslint-disable-line+ expect(state.id).toBeDefined();+ expect(state.cookies).toBeDefined();+ expect(state.cookieJar).toBeDefined();+ expect(state.userData).toBeDefined();+ expect(state.maxErrorScore).toBeDefined();+ expect(state.errorScoreDecrement).toBeDefined();+ expect(state.expiresAt).toBeDefined();+ expect(state.createdAt).toBeDefined();+ expect(state.usageCount).toBeDefined();+ expect(state.errorScore).toBeDefined();Object.entries(state).forEach(([key, value]) => {if (session[key] instanceof Date) {expect(session[key].toISOString()).toEqual(value);+ } else if (key === 'cookieJar') {+ expect(value).toEqual(session[key].toJSON());} else {expect(session[key]).toEqual(value);}@@ -117,4 +121,66 @@ describe('Session - testing session behaviour ', () => {expect(error).toBeUndefined();});++ test('should use cookieJar', () => {+ session = new Session({ sessionPool });+ expect(session.cookieJar.setCookie).toBeDefined();+ });++ test('should checkStatus work', () => {+ session = new Session({ sessionPool });+ expect(session.checkStatus(100)).toBeFalsy();+ expect(session.checkStatus(200)).toBeFalsy();+ expect(session.checkStatus(400)).toBeFalsy();+ expect(session.checkStatus(500)).toBeFalsy();+ STATUS_CODES_BLOCKED.forEach((status) => {+ const sess = new Session({ sessionPool });+ let isCalled;+ const call = () => { isCalled = true; };+ sess.retire = call;+ expect(sess.checkStatus(status)).toBeTruthy();+ expect(isCalled).toBeTruthy();+ });+ });++ describe('.setCookiesToJar & .getCookieString', () => {+ test('should set and update cookies from "set-cookie" header', () => {+ const headers = {};++ headers['set-cookie'] = [+ 'CSRF=e8b667; Domain=example.com; Secure ',+ 'id=a3fWa; Expires=Wed, Domain=example.com; 21 Oct 2015 07:28:00 GMT',+ ];+ const newSession = new Session({ sessionPool: new SessionPool() });+ const url = 'https://example.com';+ newSession.setCookiesToJar({ headers, url });+ let cookies = newSession.getCookieString(url);+ expect(cookies).toEqual('CSRF=e8b667; id=a3fWa');++ const newCookie = 'ABCD=1231231213; Domain=example.com; Secure';++ newSession.setCookiesToJar({ headers: { 'set-cookie': newCookie }, url });+ cookies = newSession.getCookieString(url);+ expect(cookies).toEqual('CSRF=e8b667; id=a3fWa; ABCD=1231231213');+ });+ });++ test('should correctly persist and init cookieJar', () => {+ const headers = {};++ headers['set-cookie'] = [+ 'CSRF=e8b667; Domain=example.com; Secure ',+ 'id=a3fWa; Expires=Wed, Domain=example.com; 21 Oct 2015 07:28:00 GMT',+ ];+ const newSession = new Session({ sessionPool: new SessionPool() });+ const url = 'https://example.com';+ newSession.setCookiesToJar({ headers, url });++ const old = newSession.getState();++ old.createdAt = new Date(old.createdAt);+ old.expiresAt = new Date(old.expiresAt);+ const reinitializedSession = new Session({ sessionPool, ...old });+ expect(reinitializedSession.getCookieString(url)).toEqual('CSRF=e8b667; id=a3fWa');+ });});diff --git a/test/session_pool/session_pool.test.js b/test/session_pool/session_pool.test.jsindex e95b2bf22..178f431ef 100644--- a/test/session_pool/session_pool.test.js+++ b/test/session_pool/session_pool.test.js@@ -151,6 +151,8 @@ describe('SessionPool - testing session pool', () => {Object.entries(session).forEach(([key, value]) => {if (sessionPool.sessions[index][key] instanceof Date) {expect(value).toEqual(sessionPool.sessions[index][key].toISOString());+ } else if (key === 'cookieJar') {+ expect(value).toEqual(sessionPool.sessions[index][key].toJSON());} else {expect(sessionPool.sessions[index][key]).toEqual(value);}diff --git a/test/session_pool/session_utils.test.js b/test/session_pool/session_utils.test.jsnew file mode 100644index 000000000..972eacf42--- /dev/null+++ b/test/session_pool/session_utils.test.js@@ -0,0 +1,39 @@+import { Cookie } from 'tough-cookie';+import { getCookiesFromResponse } from '../../build/session_pool/session_utils';++describe('getCookiesFromResponse ', () => {+ test('should parse cookies if set-cookie is array', () => {+ const headers = {};+ const dummyCookies = ['CSRF=e8b667; Domain=example.com; Secure', 'id=a3fWa; Expires=Wed, 21 Oct 2015 07:28:00 GMT'];+ headers['set-cookie'] = dummyCookies;+ const cookies = getCookiesFromResponse({ headers });++ cookies.forEach((cookie) => {+ expect(cookie).toBeInstanceOf(Cookie);+ });++ expect(dummyCookies[0]).toEqual(cookies[0].toString());+ expect(dummyCookies[1]).toEqual(cookies[1].toString());+ });++ test('should parse cookies if set-cookie is string', () => {+ const headers = {};+ const dummyCookie = 'CSRF=e8b667; Domain=example.com; Secure';+ headers['set-cookie'] = dummyCookie;+ const cookies = getCookiesFromResponse({ headers });++ expect(cookies).toHaveLength(1);+ expect(dummyCookie).toEqual(cookies[0].toString());+ expect(cookies[0]).toBeInstanceOf(Cookie);+ });++ test('should not throw error on parsing invalid cookie', () => {+ const headers = {};+ const dummyCookie = 'totally Invalid Cookie $@$@#$**';+ headers['set-cookie'] = dummyCookie;+ const cookies = getCookiesFromResponse({ headers });++ expect(cookies).toHaveLength(1);+ expect(cookies[0]).toBeUndefined();+ });+});diff --git a/test/utils.test.js b/test/utils.test.jsindex e48f9914d..da175b6af 100644--- a/test/utils.test.js+++ b/test/utils.test.js@@ -966,75 +966,3 @@ describe('utils.parseContentTypeFromResponse', () => {expect(parsedReallyBad.charset).toBe('utf-8');});});--describe('utils.getCookiesFromResponse & utils.getCookieHeader', () => {- test('should parse cookies if set-cookie is array', () => {- const headers = {};- const dummyCookies = ['CSRF=e8b667; Domain=example.com; Secure', 'id=a3fWa; Expires=Wed, 21 Oct 2015 07:28:00 GMT'];- headers['set-cookie'] = dummyCookies;- const cookies = utils.getCookiesFromResponse({ headers });-- cookies.forEach((cookie) => {- expect(cookie).toBeInstanceOf(Cookie);- });-- expect(dummyCookies[0]).toEqual(cookies[0].toString());- expect(dummyCookies[1]).toEqual(cookies[1].toString());-- const parsed = utils.getCookieHeader(cookies);- expect(parsed).toEqual('CSRF=e8b667;id=a3fWa');- });-- test('should parse cookies if set-cookie is string', () => {- const headers = {};- const dummyCookie = 'CSRF=e8b667; Domain=example.com; Secure';- headers['set-cookie'] = dummyCookie;- const cookies = utils.getCookiesFromResponse({ headers });-- expect(cookies).toHaveLength(1);- expect(dummyCookie).toEqual(cookies[0].toString());- expect(cookies[0]).toBeInstanceOf(Cookie);- const parsed = utils.getCookieHeader(cookies);- expect(parsed).toEqual('CSRF=e8b667');- });-- test('should not throw error on parsing invalid cookie', () => {- const headers = {};- const dummyCookie = 'totally Invalid Cookie $@$@#$**';- headers['set-cookie'] = dummyCookie;- const cookies = utils.getCookiesFromResponse({ headers });-- expect(cookies).toHaveLength(1);- expect(cookies[0]).toBeUndefined();-- const parsed = utils.getCookieHeader(cookies);- expect(parsed).toBeFalsy();- });-});--describe('utils.updateSessionCookies', () => {- test('should set and update cookies from "set-cookie" header', () => {- const headers = {};- const dummyCookies = ['CSRF=e8b667; Domain=example.com; Secure', 'id=a3fWa; Expires=Wed, 21 Oct 2015 07:28:00 GMT'];- headers['set-cookie'] = dummyCookies;- let session = new Session({ sessionPool: new SessionPool() });- session = updateSessionCookies(session, { headers });-- expect(session.cookies).toHaveLength(2);- expect(session.cookies[0].toString()).toEqual(dummyCookies[0]);- expect(session.cookies[1].toString()).toEqual(dummyCookies[1]);-- const newDummyCookies = ['CSRF=123121; Domain=example.cz; Secure', 'id=abcdef; Expires=Wed, 21 Oct 2015 07:28:00 GMT'];- headers['set-cookie'] = newDummyCookies;- session = updateSessionCookies(session, { headers });-- expect(session.cookies).toHaveLength(2);- expect(session.cookies[0].toString()).toEqual(newDummyCookies[0]);- expect(session.cookies[1].toString()).toEqual(newDummyCookies[1]);-- const newCookie = 'ABCD=1231231213; Domain=example-site.cz; Secure';-- session = updateSessionCookies(session, { headers: { 'set-cookie': newCookie } });- expect(session.cookies).toHaveLength(3);- });-});