Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: allow configuring what status codes will cause session retirement (#1423)

Martin Adámekcd3795f8b94d
diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts
deleted file mode 100644
index 49747e44f..000000000
--- a/packages/core/src/constants.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-/** @internal */
-export const STATUS_CODES_BLOCKED = [401, 403, 429];
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index a493e2fa8..3154c0bef 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -1,7 +1,6 @@
export * from './errors';
export * from './autoscaling';
export * from './configuration';
-export * from './constants';
export * from './crawlers';
export * from './enqueue_links';
export * from './events';
diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.ts
index 7b4c8c273..1fe984787 100644
--- a/packages/core/src/session_pool/session.ts
+++ b/packages/core/src/session_pool/session.ts
@@ -1,19 +1,15 @@
import type { Log } from '@apify/log';
import { cryptoRandomObjectId } from '@apify/utilities';
-import type { Dictionary, Cookie as CookieObject, BrowserLikeResponse } from '@crawlee/types';
+import type { BrowserLikeResponse, Cookie as CookieObject, Dictionary } from '@crawlee/types';
import type { IncomingMessage } from 'node:http';
import { EventEmitter } from 'node:events';
import ow from 'ow';
import type { Cookie } from 'tough-cookie';
import { CookieJar } from 'tough-cookie';
-import { STATUS_CODES_BLOCKED } from '../constants';
import { log as defaultLog } from '../log';
import { EVENT_SESSION_RETIRED } from './events';
import { browserPoolCookieToToughCookie, getCookiesFromResponse, getDefaultCookieExpirationDate, toughCookieToBrowserPoolCookie } from '../cookie_utils';
-// CONSTANTS
-const DEFAULT_SESSION_MAX_AGE_SECS = 3000;
-
/**
* Persistable {@link Session} state.
*/
@@ -134,7 +130,7 @@ export class Session {
sessionPool,
id = `session_${cryptoRandomObjectId(10)}`,
cookieJar = new CookieJar(),
- maxAgeSecs = DEFAULT_SESSION_MAX_AGE_SECS,
+ maxAgeSecs = 3000,
userData = {},
maxErrorScore = 3,
errorScoreDecrement = 0.5,
@@ -264,11 +260,28 @@ export class Session {
* by retiring the session when such code is received. Optionally the default status
* codes can be extended in the second parameter.
* @param statusCode HTTP status code.
- * @param [blockedStatusCodes] Custom HTTP status codes that means blocking on particular website.
* @returns Whether the session was retired.
*/
- retireOnBlockedStatusCodes(statusCode: number, blockedStatusCodes: number[] = []): boolean {
- const isBlocked = STATUS_CODES_BLOCKED.concat(blockedStatusCodes).includes(statusCode);
+ retireOnBlockedStatusCodes(statusCode: number): boolean;
+
+ /**
+ * With certain status codes: `401`, `403` or `429` we can be certain
+ * that the target website is blocking us. This function helps to do this conveniently
+ * by retiring the session when such code is received. Optionally the default status
+ * codes can be extended in the second parameter.
+ * @param statusCode HTTP status code.
+ * @param [additionalBlockedStatusCodes]
+ * Custom HTTP status codes that means blocking on particular website.
+ *
+ * **This parameter is deprecated and will be removed in next major version.**
+ * @returns Whether the session was retired.
+ * @deprecated The parameter `additionalBlockedStatusCodes` is deprecated and will be removed in next major version.
+ */
+ retireOnBlockedStatusCodes(statusCode: number, additionalBlockedStatusCodes?: number[]): boolean;
+
+ retireOnBlockedStatusCodes(statusCode: number, additionalBlockedStatusCodes: number[] = []): boolean {
+ // @ts-expect-error
+ const isBlocked = this.sessionPool.blockedStatusCodes.concat(additionalBlockedStatusCodes).includes(statusCode);
if (isBlocked) {
this.retire();
}
diff --git a/packages/core/src/session_pool/session_pool.ts b/packages/core/src/session_pool/session_pool.ts
index d6a9b6fa0..e3e02cfee 100644
--- a/packages/core/src/session_pool/session_pool.ts
+++ b/packages/core/src/session_pool/session_pool.ts
@@ -47,6 +47,13 @@ export interface SessionPoolOptions {
*/
createSessionFunction?: CreateSession;
+ /**
+ * Specifies which response status codes are considered as blocked.
+ * Session connected to such request will be marked as retired.
+ * @default [401, 403, 429]
+ */
+ blockedStatusCodes?: number[];
+
/** @internal */
log?: Log;
}
@@ -110,17 +117,18 @@ export interface SessionPoolOptions {
* @category Scaling
*/
export class SessionPool extends EventEmitter {
- log: Log;
- maxPoolSize: number;
- createSessionFunction: CreateSession;
- keyValueStore!: KeyValueStore;
- sessions: Session[] = [];
- sessionMap = new Map<string, Session>();
- sessionOptions: SessionOptions;
- persistStateKeyValueStoreId?: string;
- persistStateKey: string;
- private _listener!: () => Promise<void>;
- private events: EventManager;
+ protected log: Log;
+ protected maxPoolSize: number;
+ protected createSessionFunction: CreateSession;
+ protected keyValueStore!: KeyValueStore;
+ protected sessions: Session[] = [];
+ protected sessionMap = new Map<string, Session>();
+ protected sessionOptions: SessionOptions;
+ protected persistStateKeyValueStoreId?: string;
+ protected persistStateKey: string;
+ protected _listener!: () => Promise<void>;
+ protected events: EventManager;
+ protected readonly blockedStatusCodes: number[];
/**
* @internal
@@ -134,22 +142,22 @@ export class SessionPool extends EventEmitter {
persistStateKey: ow.optional.string,
createSessionFunction: ow.optional.function,
sessionOptions: ow.optional.object,
+ blockedStatusCodes: ow.optional.array.ofType(ow.number),
log: ow.optional.object,
}));
const {
maxPoolSize = 1000,
-
persistStateKeyValueStoreId,
persistStateKey = 'SDK_SESSION_POOL_STATE',
-
createSessionFunction,
sessionOptions = {},
-
+ blockedStatusCodes = [401, 403, 429],
log = defaultLog,
} = options;
this.config = config;
+ this.blockedStatusCodes = blockedStatusCodes;
this.events = config.getEventManager();
this.log = log.child({ prefix: 'SessionPool' });
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 8b82ef52a..aac59d04d 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -970,6 +970,7 @@ describe('BasicCrawler', () => {
});
await crawler.run();
+ // @ts-expect-error private symbol
expect(crawler.sessionPool.maxPoolSize).toEqual(10);
});
@@ -998,6 +999,7 @@ describe('BasicCrawler', () => {
await crawler.run();
expect(events.listenerCount(EventType.PERSIST_STATE)).toEqual(0);
+ // @ts-expect-error private symbol
expect(crawler.sessionPool.maxPoolSize).toEqual(10);
});
});
diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts
index 29a87a57c..7b140cb9d 100644
--- a/test/core/crawlers/browser_crawler.test.ts
+++ b/test/core/crawlers/browser_crawler.test.ts
@@ -17,7 +17,6 @@ import {
Request,
RequestList,
Session,
- STATUS_CODES_BLOCKED,
} from '@crawlee/puppeteer';
import { gotScraping } from 'got-scraping';
import { sleep } from '@crawlee/utils';
@@ -390,7 +389,7 @@ describe('BrowserCrawler', () => {
test('should throw on "blocked" status codes', async () => {
const baseUrl = 'https://example.com/';
- const sources = STATUS_CODES_BLOCKED.map((statusCode) => {
+ const sources = [401, 403, 429].map((statusCode) => {
return {
url: baseUrl + statusCode,
userData: { statusCode },
@@ -423,7 +422,7 @@ describe('BrowserCrawler', () => {
await crawler.run();
- expect(failedRequests.length).toBe(STATUS_CODES_BLOCKED.length);
+ expect(failedRequests.length).toBe(3);
failedRequests.forEach((fr) => {
const [msg] = fr.errorMessages;
expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);
@@ -433,7 +432,7 @@ describe('BrowserCrawler', () => {
test('should throw on "blocked" status codes (retire session)', async () => {
const baseUrl = 'https://example.com/';
- const sources = STATUS_CODES_BLOCKED.map((statusCode) => {
+ const sources = [401, 403, 429].map((statusCode) => {
return {
url: baseUrl + statusCode,
userData: { statusCode },
@@ -466,7 +465,7 @@ describe('BrowserCrawler', () => {
await crawler.run();
- expect(failedRequests.length).toBe(STATUS_CODES_BLOCKED.length);
+ expect(failedRequests.length).toBe(3);
failedRequests.forEach((fr) => {
const [msg] = fr.errorMessages;
expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index 42082bf0a..68e055947 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -19,7 +19,6 @@ import {
Request,
RequestList,
Session,
- STATUS_CODES_BLOCKED,
} from '@crawlee/cheerio';
import express from 'express';
import fs from 'fs';
@@ -858,6 +857,7 @@ describe('CheerioCrawler', () => {
await cheerioCrawler.run();
+ // @ts-expect-error private symbol
const { sessions } = cheerioCrawler.sessionPool;
expect(sessions.length).toBe(4);
sessions.forEach((session) => {
@@ -878,7 +878,7 @@ describe('CheerioCrawler', () => {
});
test('should retire session on "blocked" status codes', async () => {
- for (const code of STATUS_CODES_BLOCKED) {
+ for (const code of [401, 403, 429]) {
const failed: Request[] = [];
const sessions: Session[] = [];
const crawler = new CheerioCrawler({
@@ -899,7 +899,9 @@ describe('CheerioCrawler', () => {
});
await crawler.run();
+ // @ts-expect-error private symbol
expect(crawler.sessionPool.sessions.length).toBe(4);
+ // @ts-expect-error private symbol
// eslint-disable-next-line no-loop-func
crawler.sessionPool.sessions.forEach((session) => {
// @ts-expect-error Accessing private prop
diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.ts
index 93ced1082..e2be4069b 100644
--- a/test/core/session_pool/session.test.ts
+++ b/test/core/session_pool/session.test.ts
@@ -1,4 +1,4 @@
-import { EVENT_SESSION_RETIRED, ProxyConfiguration, Session, SessionPool, STATUS_CODES_BLOCKED } from '@crawlee/core';
+import { EVENT_SESSION_RETIRED, ProxyConfiguration, Session, SessionPool } from '@crawlee/core';
import type { Dictionary } from '@crawlee/utils';
import { entries, sleep } from '@crawlee/utils';
@@ -178,7 +178,8 @@ describe('Session - testing session behaviour ', () => {
expect(session.retireOnBlockedStatusCodes(200)).toBeFalsy();
expect(session.retireOnBlockedStatusCodes(400)).toBeFalsy();
expect(session.retireOnBlockedStatusCodes(500)).toBeFalsy();
- STATUS_CODES_BLOCKED.forEach((status) => {
+ // @ts-expect-error
+ sessionPool.blockedStatusCodes.forEach((status) => {
const sess = new Session({ sessionPool });
let isCalled;
const call = () => { isCalled = true; };
diff --git a/test/core/session_pool/session_pool.test.ts b/test/core/session_pool/session_pool.test.ts
index 96ccc651a..9ee340522 100644
--- a/test/core/session_pool/session_pool.test.ts
+++ b/test/core/session_pool/session_pool.test.ts
@@ -22,11 +22,15 @@ describe('SessionPool - testing session pool', () => {
});
test('should initialize with default values for first time', async () => {
+ // @ts-expect-error private symbol
expect(sessionPool.sessions.length).toBeDefined();
+ // @ts-expect-error private symbol
expect(sessionPool.maxPoolSize).toBeDefined();
+ // @ts-expect-error private symbol
expect(sessionPool.sessionOptions).toBeDefined();
+ // @ts-expect-error private symbol
expect(sessionPool.persistStateKey).toBeDefined();
- // @ts-expect-error Accessing private property
+ // @ts-expect-error private symbol
expect(sessionPool.createSessionFunction).toEqual(sessionPool._defaultCreateSessionFunction);
});
@@ -52,6 +56,7 @@ describe('SessionPool - testing session pool', () => {
expect(sessionPool[key]).toEqual(value);
});
// log is appended to sessionOptions after sessionPool instantiation
+ // @ts-expect-error private symbol
expect(sessionPool.sessionOptions).toEqual({ ...opts.sessionOptions, log: expect.any(Log) });
});
@@ -77,6 +82,7 @@ describe('SessionPool - testing session pool', () => {
expect(sessionPool[key]).toEqual(value);
});
// log is appended to sessionOptions after sessionPool instantiation
+ // @ts-expect-error private symbol
expect(sessionPool.sessionOptions).toEqual({ ...opts.sessionOptions, log: expect.any(Log) });
});
@@ -84,6 +90,7 @@ describe('SessionPool - testing session pool', () => {
test('should retrieve session with correct shape', async () => {
sessionPool = await SessionPool.open({ sessionOptions: { maxAgeSecs: 100, maxUsageCount: 10 } });
const session = await sessionPool.getSession();
+ // @ts-expect-error private symbol
expect(sessionPool.sessions.length).toBe(1);
expect(session.id).toBeDefined();
// @ts-expect-error Accessing private property
@@ -95,6 +102,7 @@ describe('SessionPool - testing session pool', () => {
});
test('should pick session when pool is full', async () => {
+ // @ts-expect-error private symbol
sessionPool.maxPoolSize = 2;
await sessionPool.getSession();
await sessionPool.getSession();
@@ -114,17 +122,21 @@ describe('SessionPool - testing session pool', () => {
});
test('should delete picked session when it is unusable and create a new one', async () => {
+ // @ts-expect-error private symbol
sessionPool.maxPoolSize = 1;
await sessionPool.addSession();
const session = await sessionPool.getSession();
+ // @ts-expect-error private symbol
expect(sessionPool.sessions[0].id === session.id).toBe(true);
// @ts-expect-error Overriding private property
session.errorScore += session.maxErrorScore;
await sessionPool.getSession();
+ // @ts-expect-error private symbol
expect(sessionPool.sessions[0].id === session.id).toBe(false);
+ // @ts-expect-error private symbol
expect(sessionPool.sessions).toHaveLength(1);
});
});
@@ -151,6 +163,7 @@ describe('SessionPool - testing session pool', () => {
await sessionPool.persistState();
const kvStore = await KeyValueStore.open();
+ // @ts-expect-error private symbol
const sessionPoolSaved = await kvStore.getValue<ReturnType<SessionPool['getState']>>(sessionPool.persistStateKey);
entries(sessionPoolSaved).forEach(([key, value]) => {
@@ -159,15 +172,20 @@ describe('SessionPool - testing session pool', () => {
}
});
+ // @ts-expect-error private symbol
expect(sessionPoolSaved.sessions.length).toEqual(sessionPool.sessions.length);
sessionPoolSaved.sessions.forEach((session, index) => {
entries(session).forEach(([key, value]) => {
+ // @ts-expect-error private symbol
if (sessionPool.sessions[index][key] instanceof Date) {
+ // @ts-expect-error private symbol
expect(value).toEqual((sessionPool.sessions[index][key] as Date).toISOString());
} else if (key === 'cookieJar') {
+ // @ts-expect-error private symbol
expect(value).toEqual(sessionPool.sessions[index][key].toJSON());
} else {
+ // @ts-expect-error private symbol
expect(sessionPool.sessions[index][key]).toEqual(value);
}
});
@@ -176,24 +194,31 @@ describe('SessionPool - testing session pool', () => {
const loadedSessionPool = new SessionPool();
await loadedSessionPool.initialize();
+ // @ts-expect-error private symbol
expect(sessionPool.sessions).toHaveLength(loadedSessionPool.sessions.length);
+ // @ts-expect-error private symbol
expect(sessionPool.maxPoolSize).toEqual(loadedSessionPool.maxPoolSize);
+ // @ts-expect-error private symbol
expect(sessionPool.persistStateKey).toEqual(loadedSessionPool.persistStateKey);
await sessionPool.teardown();
});
test('should create only maxPoolSize number of sessions', async () => {
+ // @ts-expect-error private symbol
const max = sessionPool.maxPoolSize;
for (let i = 0; i < max + 100; i++) {
await sessionPool.getSession();
}
+ // @ts-expect-error private symbol
expect(sessionPool.sessions.length).toEqual(sessionPool.maxPoolSize);
});
test('should create session', async () => {
- // @ts-expect-error Accessing private property
+ // @ts-expect-error private symbol
await sessionPool._createSession();
+ // @ts-expect-error private symbol
expect(sessionPool.sessions.length).toBe(1);
+ // @ts-expect-error private symbol
expect(sessionPool.sessions[0].id).toBeDefined();
});
@@ -212,12 +237,14 @@ describe('SessionPool - testing session pool', () => {
test('on persist event', async () => {
await sessionPool.getSession();
+ // @ts-expect-error private symbol
expect(sessionPool.sessions.length).toBe(1);
events.emit(EventType.PERSIST_STATE);
await new Promise<void>((resolve) => {
const interval = setInterval(async () => {
+ // @ts-expect-error private symbol
const state = await sessionPool.keyValueStore.getValue(sessionPool.persistStateKey);
if (state) {
resolve();
@@ -226,6 +253,7 @@ describe('SessionPool - testing session pool', () => {
}, 100);
});
+ // @ts-expect-error private symbol
const state = await sessionPool.keyValueStore.getValue(sessionPool.persistStateKey);
expect(sessionPool.getState()).toEqual(state);
@@ -233,9 +261,11 @@ describe('SessionPool - testing session pool', () => {
});
test('should remove retired sessions', async () => {
+ // @ts-expect-error private symbol
sessionPool.maxPoolSize = 1;
await sessionPool.getSession();
+ // @ts-expect-error private symbol
const session = sessionPool.sessions[0];
// @ts-expect-error Overriding private property
session.errorScore += session.maxErrorScore;
@@ -243,6 +273,7 @@ describe('SessionPool - testing session pool', () => {
await sessionPool.getSession();
+ // @ts-expect-error private symbol
expect(sessionPool.sessions.find((s) => s.id === retiredSessionId)).toEqual(undefined);
});
@@ -263,6 +294,7 @@ describe('SessionPool - testing session pool', () => {
const newSessionPool = new SessionPool();
await newSessionPool.initialize();
+ // @ts-expect-error private symbol
expect(newSessionPool.sessions).toHaveLength(10 - invalidSessionsCount);
await newSessionPool.teardown();
@@ -293,10 +325,14 @@ describe('SessionPool - testing session pool', () => {
await newSessionPool.teardown();
+ // @ts-expect-error private symbol
const kvStore = await KeyValueStore.open(newSessionPool.persistStateKeyValueStoreId);
+ // @ts-expect-error private symbol
const state = await kvStore.getValue(newSessionPool.persistStateKey);
+ // @ts-expect-error private symbol
expect(newSessionPool.persistStateKeyValueStoreId).toBeDefined();
+ // @ts-expect-error private symbol
expect(newSessionPool.persistStateKey).toBeDefined();
expect(state).toBeDefined();
expect(state).toBeInstanceOf(Object);
@@ -326,6 +362,7 @@ describe('SessionPool - testing session pool', () => {
test('should be able to create session with provided id', async () => {
await sessionPool.addSession({ id: 'test-session' });
+ // @ts-expect-error private symbol
const session = sessionPool.sessions[0];
expect(session.id).toBe('test-session');
});
@@ -360,16 +397,21 @@ describe('SessionPool - testing session pool', () => {
});
test('should correctly populate session array and session map', async () => {
+ // @ts-expect-error private symbol
sessionPool.maxPoolSize = 10;
for (let i = 0; i < 20; i++) await sessionPool.getSession();
+ // @ts-expect-error private symbol
expect(sessionPool.sessions.length).toEqual(10);
+ // @ts-expect-error private symbol
expect(sessionPool.sessionMap.size).toEqual(10);
+ // @ts-expect-error private symbol
expect(sessionPool.sessions.length).toEqual(sessionPool.sessionMap.size);
});
test('should correctly remove retired sessions both from array and session map', async () => {
+ // @ts-expect-error private symbol
sessionPool.maxPoolSize = 10;
for (let i = 0; i < 10; i++) {
@@ -381,8 +423,11 @@ describe('SessionPool - testing session pool', () => {
await sessionPool.getSession();
+ // @ts-expect-error private symbol
expect(sessionPool.sessions.length).toEqual(1);
+ // @ts-expect-error private symbol
expect(sessionPool.sessionMap.size).toEqual(1);
+ // @ts-expect-error private symbol
expect(sessionPool.sessions.length).toEqual(sessionPool.sessionMap.size);
});
});