Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat(`session-pool`): add `sessionReuseStrategy` option (#3587)
Jindřich Bär9047dc2890f5
diff --git a/docs/guides/session_management.mdx b/docs/guides/session_management.mdxindex ad0eca80e..4bd069b14 100644--- a/docs/guides/session_management.mdx+++ b/docs/guides/session_management.mdx@@ -24,8 +24,8 @@ The main benefit of using Session pool is that we can filter out blocked or non-so our actor does not retry requests over known blocked/non-working proxies.Another benefit of using SessionPool is that we can store information tied tightly to an IP address,such as cookies, auth tokens, and particular headers. Having our cookies and other identifiers used only with a specific IP will reduce the chance of being blocked.-The last but not least benefit is the even rotation of IP addresses - SessionPool picks the session randomly,-which should prevent burning out a small pool of available IPs.+The last but not least benefit is the even rotation of IP addresses - by default, SessionPool picks sessions randomly,+which should prevent burning out a small pool of available IPs. The selection strategy is configurable — see below.Check out the [avoid blocking guide](./avoid-blocking) for more information about blocking.@@ -79,3 +79,19 @@ Now let's take a look at the examples of how to use Session pool:These are the basics of configuring SessionPool.Please, bear in mind that a Session pool needs time to find working IPs and build up the pool,so we will probably see a lot of errors until it becomes stabilized.++## Session reuse strategy++The `sessionReuseStrategy` option controls how sessions are picked from the pool. Three strategies are available:++**`random`** (default) — the pool fills up to `maxPoolSize` by creating a new session for every request, then picks randomly from the available sessions. Best for maximising fingerprint and IP diversity.++**`round-robin`** — like `random`, the pool fills up to `maxPoolSize` first, then cycles through the sessions in order instead of picking randomly. Best when you want even load distribution across sessions, for example when combined with `maxUsageCount` to ensure all sessions expire at roughly the same time.++**`use-until-failure`** — the same session is returned on every call until it is retired, then the next available one is used. Best for proxy session stickiness, where you want to keep the same IP for as long as possible before rotating.++```js+const sessionPool = new SessionPool({+ sessionReuseStrategy: 'use-until-failure',+});+```diff --git a/packages/core/src/session_pool/session_pool.ts b/packages/core/src/session_pool/session_pool.tsindex 0e040bf1d..047d094b7 100644--- a/packages/core/src/session_pool/session_pool.ts+++ b/packages/core/src/session_pool/session_pool.ts@@ -14,6 +14,9 @@ import { MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';import type { SessionOptions } from './session.js';import { Session } from './session.js';+const SESSION_REUSE_STRATEGIES = ['random', 'round-robin', 'use-until-failure'] as const;+export type SessionReuseStrategy = (typeof SESSION_REUSE_STRATEGIES)[number];+/*** Factory user-function which creates customized {@apilink Session} instances.*/@@ -58,6 +61,15 @@ export interface SessionPoolOptions {*/createSessionFunction?: CreateSession;+ /**+ * Strategy for picking sessions from the pool.+ * - `'random'` (default): fills the pool up to `maxPoolSize`, then picks a random usable session+ * - `'round-robin'`: fills the pool up to `maxPoolSize`, then reuses sessions cycling through them in order+ * - `'use-until-failure'`: always reuses the same session until it is retired, then moves to the next one+ * @default 'random'+ */+ sessionReuseStrategy?: SessionReuseStrategy;+/** @internal */log?: CrawleeLogger;@@ -136,9 +148,11 @@ export class SessionPool extends EventEmitter {protected _listener?: () => Promise<void>;protected events: EventManager;protected persistenceOptions: PersistenceOptions;+ protected sessionReuseStrategy: SessionReuseStrategy;private initPromise?: Promise<void>;private queue = new AsyncQueue();+ private roundRobinIndex = 0;constructor(options: SessionPoolOptions = {}) {super();@@ -154,6 +168,7 @@ export class SessionPool extends EventEmitter {sessionOptions: ow.optional.object,log: ow.optional.object,persistenceOptions: ow.optional.object,+ sessionReuseStrategy: ow.optional.string.oneOf([...SESSION_REUSE_STRATEGIES]),}),);@@ -168,9 +183,11 @@ export class SessionPool extends EventEmitter {persistenceOptions = {enable: true,},+ sessionReuseStrategy = 'random',} = options;this.id = id != null ? String(id) : String(SessionPool.nextId++);+ this.sessionReuseStrategy = sessionReuseStrategy;this.events = serviceLocator.getEventManager();this.log = log.child({ prefix: 'SessionPool' });this.persistenceOptions = persistenceOptions;@@ -317,15 +334,13 @@ export class SessionPool extends EventEmitter {return undefined;}+ const pickedSession = this._pickSession();+ if (pickedSession) return pickedSession;+if (this._hasSpaceForSession()) {return await this._createSession();}- const pickedSession = this._pickSession();- if (pickedSession.isUsable()) {- return pickedSession;- }-this._removeRetiredSessions();return await this._createSession();} finally {@@ -475,11 +490,26 @@ export class SessionPool extends EventEmitter {}/**- * Picks random session from the `SessionPool`.- * @returns Picked `Session`.+ * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.+ * Returns `undefined` when no session should be reused and a new one should be created instead.*/- protected _pickSession(): Session {- return this.sessions[this._getRandomIndex()]; // Or maybe we should let the developer to customize the picking algorithm+ protected _pickSession(): Session | undefined {+ if (this.sessionReuseStrategy !== 'use-until-failure' && this._hasSpaceForSession()) return undefined;++ if (this.sessionReuseStrategy === 'use-until-failure') {+ return this.sessions.find((session) => session.isUsable());+ }++ let picked: Session;+ if (this.sessionReuseStrategy === 'round-robin') {+ const index = this.roundRobinIndex % this.sessions.length;+ this.roundRobinIndex = index + 1;+ picked = this.sessions[index];+ } else {+ picked = this.sessions[this._getRandomIndex()];+ }++ return picked.isUsable() ? picked : undefined;}/**diff --git a/test/core/session_pool/session_pool.test.ts b/test/core/session_pool/session_pool.test.tsindex 7eb522f56..fcb596f18 100644--- a/test/core/session_pool/session_pool.test.ts+++ b/test/core/session_pool/session_pool.test.ts@@ -425,6 +425,92 @@ describe('SessionPool - testing session pool', () => {expect(sessionPool.sessions.length).toEqual(sessionPool.sessionMap.size);});+ describe('sessionReuseStrategy', () => {+ test('random should fill pool before reusing sessions', async () => {+ sessionPool = new SessionPool({ sessionReuseStrategy: 'random', maxPoolSize: 3 });++ const s1 = await sessionPool.getSession();+ const s2 = await sessionPool.getSession();+ const s3 = await sessionPool.getSession();++ expect(new Set([s1.id, s2.id, s3.id]).size).toBe(3);++ const s4 = await sessionPool.getSession();+ expect([s1.id, s2.id, s3.id]).toContain(s4.id);+ });++ test('round-robin should fill pool before cycling', async () => {+ sessionPool = new SessionPool({ sessionReuseStrategy: 'round-robin', maxPoolSize: 3 });++ const s1 = await sessionPool.getSession();+ const s2 = await sessionPool.getSession();+ const s3 = await sessionPool.getSession();++ expect(new Set([s1.id, s2.id, s3.id]).size).toBe(3);++ const ids: string[] = [];+ for (let i = 0; i < 6; i++) {+ ids.push((await sessionPool.getSession()).id);+ }++ expect(ids).toEqual([s1.id, s2.id, s3.id, s1.id, s2.id, s3.id]);+ });++ test('round-robin should create a new session when all existing ones are retired', async () => {+ sessionPool = new SessionPool({ sessionReuseStrategy: 'round-robin', maxPoolSize: 1 });++ const s1 = await sessionPool.getSession();+ s1.retire();++ const s2 = await sessionPool.getSession();+ expect(s2.id).not.toBe(s1.id);+ });++ test.each(['random', 'round-robin'] as const)(+ '%s should evict a retired session from a full pool and replenish',+ async (strategy) => {+ sessionPool = new SessionPool({ sessionReuseStrategy: strategy, maxPoolSize: 3 });++ const s1 = await sessionPool.getSession();+ await sessionPool.getSession();+ await sessionPool.getSession();++ s1.retire();++ // @ts-expect-error private symbol+ expect(sessionPool.sessions).toHaveLength(3);++ for (let i = 0; i < 50; i++) await sessionPool.getSession();++ // @ts-expect-error private symbol+ expect(sessionPool.sessions).toHaveLength(3);+ // @ts-expect-error private symbol+ expect(sessionPool.sessions.find((s) => s.id === s1.id)).toBeUndefined();+ },+ );++ test('use-until-failure should keep reusing the same session', async () => {+ sessionPool = new SessionPool({ sessionReuseStrategy: 'use-until-failure' });++ const s1 = await sessionPool.getSession();+ const s2 = await sessionPool.getSession();+ const s3 = await sessionPool.getSession();++ expect(s1.id).toBe(s2.id);+ expect(s2.id).toBe(s3.id);+ });++ test('use-until-failure should switch to a new session after the current one is retired', async () => {+ sessionPool = new SessionPool({ sessionReuseStrategy: 'use-until-failure' });++ const s1 = await sessionPool.getSession();+ s1.retire();++ const s2 = await sessionPool.getSession();+ expect(s2.id).not.toBe(s1.id);+ });+ });+describe('multiple SessionPool instances isolation', () => {test('should use unique persist keys by default', async () => {const pool1 = new SessionPool();