Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: allow configuring what status codes will cause session retirement (#1423)
Martin Adámekcd3795f8b94d
diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.tsdeleted file mode 100644index 49747e44f..000000000--- a/packages/core/src/constants.ts+++ /dev/null@@ -1,2 +0,0 @@-/** @internal */-export const STATUS_CODES_BLOCKED = [401, 403, 429];diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex a493e2fa8..3154c0bef 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -1,7 +1,6 @@export * from './errors';export * from './autoscaling';export * from './configuration';-export * from './constants';export * from './crawlers';export * from './enqueue_links';export * from './events';diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.tsindex 7b4c8c273..1fe984787 100644--- a/packages/core/src/session_pool/session.ts+++ b/packages/core/src/session_pool/session.ts@@ -1,19 +1,15 @@import type { Log } from '@apify/log';import { cryptoRandomObjectId } from '@apify/utilities';-import type { Dictionary, Cookie as CookieObject, BrowserLikeResponse } from '@crawlee/types';+import type { BrowserLikeResponse, Cookie as CookieObject, Dictionary } from '@crawlee/types';import type { IncomingMessage } from 'node:http';import { EventEmitter } from 'node:events';import ow from 'ow';import type { Cookie } from 'tough-cookie';import { CookieJar } from 'tough-cookie';-import { STATUS_CODES_BLOCKED } from '../constants';import { log as defaultLog } from '../log';import { EVENT_SESSION_RETIRED } from './events';import { browserPoolCookieToToughCookie, getCookiesFromResponse, getDefaultCookieExpirationDate, toughCookieToBrowserPoolCookie } from '../cookie_utils';-// CONSTANTS-const DEFAULT_SESSION_MAX_AGE_SECS = 3000;-/*** Persistable {@link Session} state.*/@@ -134,7 +130,7 @@ export class Session {sessionPool,id = `session_${cryptoRandomObjectId(10)}`,cookieJar = new CookieJar(),- maxAgeSecs = DEFAULT_SESSION_MAX_AGE_SECS,+ maxAgeSecs = 3000,userData = {},maxErrorScore = 3,errorScoreDecrement = 0.5,@@ -264,11 +260,28 @@ export class Session {* by retiring the session when such code is received. Optionally the default status* codes can be extended in the second parameter.* @param statusCode HTTP status code.- * @param [blockedStatusCodes] Custom HTTP status codes that means blocking on particular website.* @returns Whether the session was retired.*/- retireOnBlockedStatusCodes(statusCode: number, blockedStatusCodes: number[] = []): boolean {- const isBlocked = STATUS_CODES_BLOCKED.concat(blockedStatusCodes).includes(statusCode);+ retireOnBlockedStatusCodes(statusCode: number): boolean;++ /**+ * With certain status codes: `401`, `403` or `429` we can be certain+ * that the target website is blocking us. This function helps to do this conveniently+ * by retiring the session when such code is received. Optionally the default status+ * codes can be extended in the second parameter.+ * @param statusCode HTTP status code.+ * @param [additionalBlockedStatusCodes]+ * Custom HTTP status codes that means blocking on particular website.+ *+ * **This parameter is deprecated and will be removed in next major version.**+ * @returns Whether the session was retired.+ * @deprecated The parameter `additionalBlockedStatusCodes` is deprecated and will be removed in next major version.+ */+ retireOnBlockedStatusCodes(statusCode: number, additionalBlockedStatusCodes?: number[]): boolean;++ retireOnBlockedStatusCodes(statusCode: number, additionalBlockedStatusCodes: number[] = []): boolean {+ // @ts-expect-error+ const isBlocked = this.sessionPool.blockedStatusCodes.concat(additionalBlockedStatusCodes).includes(statusCode);if (isBlocked) {this.retire();}diff --git a/packages/core/src/session_pool/session_pool.ts b/packages/core/src/session_pool/session_pool.tsindex d6a9b6fa0..e3e02cfee 100644--- a/packages/core/src/session_pool/session_pool.ts+++ b/packages/core/src/session_pool/session_pool.ts@@ -47,6 +47,13 @@ export interface SessionPoolOptions {*/createSessionFunction?: CreateSession;+ /**+ * Specifies which response status codes are considered as blocked.+ * Session connected to such request will be marked as retired.+ * @default [401, 403, 429]+ */+ blockedStatusCodes?: number[];+/** @internal */log?: Log;}@@ -110,17 +117,18 @@ export interface SessionPoolOptions {* @category Scaling*/export class SessionPool extends EventEmitter {- log: Log;- maxPoolSize: number;- createSessionFunction: CreateSession;- keyValueStore!: KeyValueStore;- sessions: Session[] = [];- sessionMap = new Map<string, Session>();- sessionOptions: SessionOptions;- persistStateKeyValueStoreId?: string;- persistStateKey: string;- private _listener!: () => Promise<void>;- private events: EventManager;+ protected log: Log;+ protected maxPoolSize: number;+ protected createSessionFunction: CreateSession;+ protected keyValueStore!: KeyValueStore;+ protected sessions: Session[] = [];+ protected sessionMap = new Map<string, Session>();+ protected sessionOptions: SessionOptions;+ protected persistStateKeyValueStoreId?: string;+ protected persistStateKey: string;+ protected _listener!: () => Promise<void>;+ protected events: EventManager;+ protected readonly blockedStatusCodes: number[];/*** @internal@@ -134,22 +142,22 @@ export class SessionPool extends EventEmitter {persistStateKey: ow.optional.string,createSessionFunction: ow.optional.function,sessionOptions: ow.optional.object,+ blockedStatusCodes: ow.optional.array.ofType(ow.number),log: ow.optional.object,}));const {maxPoolSize = 1000,-persistStateKeyValueStoreId,persistStateKey = 'SDK_SESSION_POOL_STATE',-createSessionFunction,sessionOptions = {},-+ blockedStatusCodes = [401, 403, 429],log = defaultLog,} = options;this.config = config;+ this.blockedStatusCodes = blockedStatusCodes;this.events = config.getEventManager();this.log = log.child({ prefix: 'SessionPool' });diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 8b82ef52a..aac59d04d 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -970,6 +970,7 @@ describe('BasicCrawler', () => {});await crawler.run();+ // @ts-expect-error private symbolexpect(crawler.sessionPool.maxPoolSize).toEqual(10);});@@ -998,6 +999,7 @@ describe('BasicCrawler', () => {await crawler.run();expect(events.listenerCount(EventType.PERSIST_STATE)).toEqual(0);+ // @ts-expect-error private symbolexpect(crawler.sessionPool.maxPoolSize).toEqual(10);});});diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.tsindex 29a87a57c..7b140cb9d 100644--- a/test/core/crawlers/browser_crawler.test.ts+++ b/test/core/crawlers/browser_crawler.test.ts@@ -17,7 +17,6 @@ import {Request,RequestList,Session,- STATUS_CODES_BLOCKED,} from '@crawlee/puppeteer';import { gotScraping } from 'got-scraping';import { sleep } from '@crawlee/utils';@@ -390,7 +389,7 @@ describe('BrowserCrawler', () => {test('should throw on "blocked" status codes', async () => {const baseUrl = 'https://example.com/';- const sources = STATUS_CODES_BLOCKED.map((statusCode) => {+ const sources = [401, 403, 429].map((statusCode) => {return {url: baseUrl + statusCode,userData: { statusCode },@@ -423,7 +422,7 @@ describe('BrowserCrawler', () => {await crawler.run();- expect(failedRequests.length).toBe(STATUS_CODES_BLOCKED.length);+ expect(failedRequests.length).toBe(3);failedRequests.forEach((fr) => {const [msg] = fr.errorMessages;expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);@@ -433,7 +432,7 @@ describe('BrowserCrawler', () => {test('should throw on "blocked" status codes (retire session)', async () => {const baseUrl = 'https://example.com/';- const sources = STATUS_CODES_BLOCKED.map((statusCode) => {+ const sources = [401, 403, 429].map((statusCode) => {return {url: baseUrl + statusCode,userData: { statusCode },@@ -466,7 +465,7 @@ describe('BrowserCrawler', () => {await crawler.run();- expect(failedRequests.length).toBe(STATUS_CODES_BLOCKED.length);+ expect(failedRequests.length).toBe(3);failedRequests.forEach((fr) => {const [msg] = fr.errorMessages;expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex 42082bf0a..68e055947 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -19,7 +19,6 @@ import {Request,RequestList,Session,- STATUS_CODES_BLOCKED,} from '@crawlee/cheerio';import express from 'express';import fs from 'fs';@@ -858,6 +857,7 @@ describe('CheerioCrawler', () => {await cheerioCrawler.run();+ // @ts-expect-error private symbolconst { sessions } = cheerioCrawler.sessionPool;expect(sessions.length).toBe(4);sessions.forEach((session) => {@@ -878,7 +878,7 @@ describe('CheerioCrawler', () => {});test('should retire session on "blocked" status codes', async () => {- for (const code of STATUS_CODES_BLOCKED) {+ for (const code of [401, 403, 429]) {const failed: Request[] = [];const sessions: Session[] = [];const crawler = new CheerioCrawler({@@ -899,7 +899,9 @@ describe('CheerioCrawler', () => {});await crawler.run();+ // @ts-expect-error private symbolexpect(crawler.sessionPool.sessions.length).toBe(4);+ // @ts-expect-error private symbol// eslint-disable-next-line no-loop-funccrawler.sessionPool.sessions.forEach((session) => {// @ts-expect-error Accessing private propdiff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.tsindex 93ced1082..e2be4069b 100644--- a/test/core/session_pool/session.test.ts+++ b/test/core/session_pool/session.test.ts@@ -1,4 +1,4 @@-import { EVENT_SESSION_RETIRED, ProxyConfiguration, Session, SessionPool, STATUS_CODES_BLOCKED } from '@crawlee/core';+import { EVENT_SESSION_RETIRED, ProxyConfiguration, Session, SessionPool } from '@crawlee/core';import type { Dictionary } from '@crawlee/utils';import { entries, sleep } from '@crawlee/utils';@@ -178,7 +178,8 @@ describe('Session - testing session behaviour ', () => {expect(session.retireOnBlockedStatusCodes(200)).toBeFalsy();expect(session.retireOnBlockedStatusCodes(400)).toBeFalsy();expect(session.retireOnBlockedStatusCodes(500)).toBeFalsy();- STATUS_CODES_BLOCKED.forEach((status) => {+ // @ts-expect-error+ sessionPool.blockedStatusCodes.forEach((status) => {const sess = new Session({ sessionPool });let isCalled;const call = () => { isCalled = true; };diff --git a/test/core/session_pool/session_pool.test.ts b/test/core/session_pool/session_pool.test.tsindex 96ccc651a..9ee340522 100644--- a/test/core/session_pool/session_pool.test.ts+++ b/test/core/session_pool/session_pool.test.ts@@ -22,11 +22,15 @@ describe('SessionPool - testing session pool', () => {});test('should initialize with default values for first time', async () => {+ // @ts-expect-error private symbolexpect(sessionPool.sessions.length).toBeDefined();+ // @ts-expect-error private symbolexpect(sessionPool.maxPoolSize).toBeDefined();+ // @ts-expect-error private symbolexpect(sessionPool.sessionOptions).toBeDefined();+ // @ts-expect-error private symbolexpect(sessionPool.persistStateKey).toBeDefined();- // @ts-expect-error Accessing private property+ // @ts-expect-error private symbolexpect(sessionPool.createSessionFunction).toEqual(sessionPool._defaultCreateSessionFunction);});@@ -52,6 +56,7 @@ describe('SessionPool - testing session pool', () => {expect(sessionPool[key]).toEqual(value);});// log is appended to sessionOptions after sessionPool instantiation+ // @ts-expect-error private symbolexpect(sessionPool.sessionOptions).toEqual({ ...opts.sessionOptions, log: expect.any(Log) });});@@ -77,6 +82,7 @@ describe('SessionPool - testing session pool', () => {expect(sessionPool[key]).toEqual(value);});// log is appended to sessionOptions after sessionPool instantiation+ // @ts-expect-error private symbolexpect(sessionPool.sessionOptions).toEqual({ ...opts.sessionOptions, log: expect.any(Log) });});@@ -84,6 +90,7 @@ describe('SessionPool - testing session pool', () => {test('should retrieve session with correct shape', async () => {sessionPool = await SessionPool.open({ sessionOptions: { maxAgeSecs: 100, maxUsageCount: 10 } });const session = await sessionPool.getSession();+ // @ts-expect-error private symbolexpect(sessionPool.sessions.length).toBe(1);expect(session.id).toBeDefined();// @ts-expect-error Accessing private property@@ -95,6 +102,7 @@ describe('SessionPool - testing session pool', () => {});test('should pick session when pool is full', async () => {+ // @ts-expect-error private symbolsessionPool.maxPoolSize = 2;await sessionPool.getSession();await sessionPool.getSession();@@ -114,17 +122,21 @@ describe('SessionPool - testing session pool', () => {});test('should delete picked session when it is unusable and create a new one', async () => {+ // @ts-expect-error private symbolsessionPool.maxPoolSize = 1;await sessionPool.addSession();const session = await sessionPool.getSession();+ // @ts-expect-error private symbolexpect(sessionPool.sessions[0].id === session.id).toBe(true);// @ts-expect-error Overriding private propertysession.errorScore += session.maxErrorScore;await sessionPool.getSession();+ // @ts-expect-error private symbolexpect(sessionPool.sessions[0].id === session.id).toBe(false);+ // @ts-expect-error private symbolexpect(sessionPool.sessions).toHaveLength(1);});});@@ -151,6 +163,7 @@ describe('SessionPool - testing session pool', () => {await sessionPool.persistState();const kvStore = await KeyValueStore.open();+ // @ts-expect-error private symbolconst sessionPoolSaved = await kvStore.getValue<ReturnType<SessionPool['getState']>>(sessionPool.persistStateKey);entries(sessionPoolSaved).forEach(([key, value]) => {@@ -159,15 +172,20 @@ describe('SessionPool - testing session pool', () => {}});+ // @ts-expect-error private symbolexpect(sessionPoolSaved.sessions.length).toEqual(sessionPool.sessions.length);sessionPoolSaved.sessions.forEach((session, index) => {entries(session).forEach(([key, value]) => {+ // @ts-expect-error private symbolif (sessionPool.sessions[index][key] instanceof Date) {+ // @ts-expect-error private symbolexpect(value).toEqual((sessionPool.sessions[index][key] as Date).toISOString());} else if (key === 'cookieJar') {+ // @ts-expect-error private symbolexpect(value).toEqual(sessionPool.sessions[index][key].toJSON());} else {+ // @ts-expect-error private symbolexpect(sessionPool.sessions[index][key]).toEqual(value);}});@@ -176,24 +194,31 @@ describe('SessionPool - testing session pool', () => {const loadedSessionPool = new SessionPool();await loadedSessionPool.initialize();+ // @ts-expect-error private symbolexpect(sessionPool.sessions).toHaveLength(loadedSessionPool.sessions.length);+ // @ts-expect-error private symbolexpect(sessionPool.maxPoolSize).toEqual(loadedSessionPool.maxPoolSize);+ // @ts-expect-error private symbolexpect(sessionPool.persistStateKey).toEqual(loadedSessionPool.persistStateKey);await sessionPool.teardown();});test('should create only maxPoolSize number of sessions', async () => {+ // @ts-expect-error private symbolconst max = sessionPool.maxPoolSize;for (let i = 0; i < max + 100; i++) {await sessionPool.getSession();}+ // @ts-expect-error private symbolexpect(sessionPool.sessions.length).toEqual(sessionPool.maxPoolSize);});test('should create session', async () => {- // @ts-expect-error Accessing private property+ // @ts-expect-error private symbolawait sessionPool._createSession();+ // @ts-expect-error private symbolexpect(sessionPool.sessions.length).toBe(1);+ // @ts-expect-error private symbolexpect(sessionPool.sessions[0].id).toBeDefined();});@@ -212,12 +237,14 @@ describe('SessionPool - testing session pool', () => {test('on persist event', async () => {await sessionPool.getSession();+ // @ts-expect-error private symbolexpect(sessionPool.sessions.length).toBe(1);events.emit(EventType.PERSIST_STATE);await new Promise<void>((resolve) => {const interval = setInterval(async () => {+ // @ts-expect-error private symbolconst state = await sessionPool.keyValueStore.getValue(sessionPool.persistStateKey);if (state) {resolve();@@ -226,6 +253,7 @@ describe('SessionPool - testing session pool', () => {}, 100);});+ // @ts-expect-error private symbolconst state = await sessionPool.keyValueStore.getValue(sessionPool.persistStateKey);expect(sessionPool.getState()).toEqual(state);@@ -233,9 +261,11 @@ describe('SessionPool - testing session pool', () => {});test('should remove retired sessions', async () => {+ // @ts-expect-error private symbolsessionPool.maxPoolSize = 1;await sessionPool.getSession();+ // @ts-expect-error private symbolconst session = sessionPool.sessions[0];// @ts-expect-error Overriding private propertysession.errorScore += session.maxErrorScore;@@ -243,6 +273,7 @@ describe('SessionPool - testing session pool', () => {await sessionPool.getSession();+ // @ts-expect-error private symbolexpect(sessionPool.sessions.find((s) => s.id === retiredSessionId)).toEqual(undefined);});@@ -263,6 +294,7 @@ describe('SessionPool - testing session pool', () => {const newSessionPool = new SessionPool();await newSessionPool.initialize();+ // @ts-expect-error private symbolexpect(newSessionPool.sessions).toHaveLength(10 - invalidSessionsCount);await newSessionPool.teardown();@@ -293,10 +325,14 @@ describe('SessionPool - testing session pool', () => {await newSessionPool.teardown();+ // @ts-expect-error private symbolconst kvStore = await KeyValueStore.open(newSessionPool.persistStateKeyValueStoreId);+ // @ts-expect-error private symbolconst state = await kvStore.getValue(newSessionPool.persistStateKey);+ // @ts-expect-error private symbolexpect(newSessionPool.persistStateKeyValueStoreId).toBeDefined();+ // @ts-expect-error private symbolexpect(newSessionPool.persistStateKey).toBeDefined();expect(state).toBeDefined();expect(state).toBeInstanceOf(Object);@@ -326,6 +362,7 @@ describe('SessionPool - testing session pool', () => {test('should be able to create session with provided id', async () => {await sessionPool.addSession({ id: 'test-session' });+ // @ts-expect-error private symbolconst session = sessionPool.sessions[0];expect(session.id).toBe('test-session');});@@ -360,16 +397,21 @@ describe('SessionPool - testing session pool', () => {});test('should correctly populate session array and session map', async () => {+ // @ts-expect-error private symbolsessionPool.maxPoolSize = 10;for (let i = 0; i < 20; i++) await sessionPool.getSession();+ // @ts-expect-error private symbolexpect(sessionPool.sessions.length).toEqual(10);+ // @ts-expect-error private symbolexpect(sessionPool.sessionMap.size).toEqual(10);+ // @ts-expect-error private symbolexpect(sessionPool.sessions.length).toEqual(sessionPool.sessionMap.size);});test('should correctly remove retired sessions both from array and session map', async () => {+ // @ts-expect-error private symbolsessionPool.maxPoolSize = 10;for (let i = 0; i < 10; i++) {@@ -381,8 +423,11 @@ describe('SessionPool - testing session pool', () => {await sessionPool.getSession();+ // @ts-expect-error private symbolexpect(sessionPool.sessions.length).toEqual(1);+ // @ts-expect-error private symbolexpect(sessionPool.sessionMap.size).toEqual(1);+ // @ts-expect-error private symbolexpect(sessionPool.sessions.length).toEqual(sessionPool.sessionMap.size);});});