Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: share `SessionPool` instance between crawlers (#3499)

Jindřich Bärcb1b9848b68b
diff --git a/docs/guides/session_management_basic.ts b/docs/guides/session_management_basic.ts
index 5dd90c209..3a48a36c4 100644
--- a/docs/guides/session_management_basic.ts
+++ b/docs/guides/session_management_basic.ts
@@ -1,4 +1,4 @@
-import { BasicCrawler, ProxyConfiguration } from 'crawlee';
+import { BasicCrawler, ProxyConfiguration, SessionPool } from 'crawlee';
import { Impit } from 'impit';
import { Cookie } from 'tough-cookie';
@@ -8,7 +8,7 @@ const proxyConfiguration = new ProxyConfiguration({
const crawler = new BasicCrawler({
// Overrides default Session pool configuration.
- sessionPoolOptions: { maxPoolSize: 100 },
+ sessionPool: new SessionPool({ maxPoolSize: 100 }),
async requestHandler({ request, session }) {
const { url } = request;
const client = new Impit({
diff --git a/docs/guides/session_management_cheerio.ts b/docs/guides/session_management_cheerio.ts
index c819b4dd1..a41b5c67a 100644
--- a/docs/guides/session_management_cheerio.ts
+++ b/docs/guides/session_management_cheerio.ts
@@ -1,4 +1,4 @@
-import { CheerioCrawler, ProxyConfiguration } from 'crawlee';
+import { CheerioCrawler, ProxyConfiguration, SessionPool } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({
/* opts */
@@ -8,7 +8,7 @@ const crawler = new CheerioCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxyConfiguration,
// Overrides default Session pool configuration.
- sessionPoolOptions: { maxPoolSize: 100 },
+ sessionPool: new SessionPool({ maxPoolSize: 100 }),
// Set to true if you want the crawler to save cookies per session,
// and set the cookie header to request automatically (default is true).
persistCookiesPerSession: true,
diff --git a/docs/guides/session_management_http.ts b/docs/guides/session_management_http.ts
index 3aae84268..28da56f5c 100644
--- a/docs/guides/session_management_http.ts
+++ b/docs/guides/session_management_http.ts
@@ -1,4 +1,4 @@
-import { HttpCrawler, ProxyConfiguration } from 'crawlee';
+import { HttpCrawler, ProxyConfiguration, SessionPool } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({
/* opts */
@@ -8,7 +8,7 @@ const crawler = new HttpCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxyConfiguration,
// Overrides default Session pool configuration.
- sessionPoolOptions: { maxPoolSize: 100 },
+ sessionPool: new SessionPool({ maxPoolSize: 100 }),
// Set to true if you want the crawler to save cookies per session,
// and set the cookie header to request automatically (default is true).
persistCookiesPerSession: true,
diff --git a/docs/guides/session_management_jsdom.ts b/docs/guides/session_management_jsdom.ts
index fe68d7cea..e96bf3144 100644
--- a/docs/guides/session_management_jsdom.ts
+++ b/docs/guides/session_management_jsdom.ts
@@ -1,4 +1,4 @@
-import { JSDOMCrawler, ProxyConfiguration } from 'crawlee';
+import { JSDOMCrawler, ProxyConfiguration, SessionPool } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({
/* opts */
@@ -8,7 +8,7 @@ const crawler = new JSDOMCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxyConfiguration,
// Overrides default Session pool configuration.
- sessionPoolOptions: { maxPoolSize: 100 },
+ sessionPool: new SessionPool({ maxPoolSize: 100 }),
// Set to true if you want the crawler to save cookies per session,
// and set the cookie header to request automatically (default is true).
persistCookiesPerSession: true,
diff --git a/docs/guides/session_management_playwright.ts b/docs/guides/session_management_playwright.ts
index aa9852290..e610540ec 100644
--- a/docs/guides/session_management_playwright.ts
+++ b/docs/guides/session_management_playwright.ts
@@ -1,4 +1,4 @@
-import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
+import { PlaywrightCrawler, ProxyConfiguration, SessionPool } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({
/* opts */
@@ -8,7 +8,7 @@ const crawler = new PlaywrightCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxyConfiguration,
// Overrides default Session pool configuration
- sessionPoolOptions: { maxPoolSize: 100 },
+ sessionPool: new SessionPool({ maxPoolSize: 100 }),
// Set to true if you want the crawler to save cookies per session,
// and set the cookies to page before navigation automatically (default is true).
persistCookiesPerSession: true,
diff --git a/docs/guides/session_management_puppeteer.ts b/docs/guides/session_management_puppeteer.ts
index 7504e0c6c..3542d344f 100644
--- a/docs/guides/session_management_puppeteer.ts
+++ b/docs/guides/session_management_puppeteer.ts
@@ -1,4 +1,4 @@
-import { PuppeteerCrawler, ProxyConfiguration } from 'crawlee';
+import { PuppeteerCrawler, ProxyConfiguration, SessionPool } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({
/* opts */
@@ -8,7 +8,7 @@ const crawler = new PuppeteerCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxyConfiguration,
// Overrides default Session pool configuration
- sessionPoolOptions: { maxPoolSize: 100 },
+ sessionPool: new SessionPool({ maxPoolSize: 100 }),
// Set to true if you want the crawler to save cookies per session,
// and set the cookies to page before navigation automatically (default is true).
persistCookiesPerSession: true,
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md
index c61c28501..f53dd8c46 100644
--- a/docs/upgrading/upgrading_v4.md
+++ b/docs/upgrading/upgrading_v4.md
@@ -109,6 +109,23 @@ const state = await sessionPool.getState();
`Session.retireOnBlockedStatusCodes` is removed. Blocked status code handling is now internal to the crawler. Configure blocked status codes via the `blockedStatusCodes` crawler option (moved from `sessionPoolOptions`).
+## `useSessionPool` and `sessionPoolOptions` are removed
+
+The `useSessionPool` and `sessionPoolOptions` options have been removed from the `BasicCrawler` constructor. Every crawler now uses a `SessionPool` by default. Instead of passing `sessionPoolOptions`, create a `SessionPool` instance directly and pass it via the `sessionPool` option.
+
+```typescript
+import { SessionPool } from '@crawlee/core';
+
+const crawler = new BasicCrawler({
+ // The old parameters won't work anymore
+ // useSessionPool: true,
+ // sessionPoolOptions: { maxUsageCount: 5 },
+ sessionPool: new SessionPool({
+ maxUsageCount: 5,
+ }),
+});
+```
+
## Remove `experimentalContainers` option
This experimental option relied on an outdated manifest version for browser extensions, it is not possible to achieve this with the currently supported versions.
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 4aba93db6..7e4800965 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -22,7 +22,6 @@ import type {
RouterHandler,
RouterRoutes,
Session,
- SessionPoolOptions,
SkippedRequestCallback,
Source,
StatisticsOptions,
@@ -315,9 +314,11 @@ export interface BasicCrawlerOptions<
keepAlive?: boolean;
/**
- * The configuration options for {@apilink SessionPool} to use.
+ * An existing {@apilink SessionPool} instance to use. When provided, the crawler will use this
+ * pool directly instead of creating a new one, enabling session sharing across multiple crawlers.
+ * The crawler will not tear down a shared pool — the caller is responsible for its lifecycle.
*/
- sessionPoolOptions?: SessionPoolOptions;
+ sessionPool?: SessionPool;
/**
* Defines the length of the interval for calling the `setStatusMessage` in seconds.
@@ -574,6 +575,11 @@ export class BasicCrawler<
*/
sessionPool: SessionPool;
+ /**
+ * Indicates whether the crawler owns the session pool (it was not passed from the outside using the `sessionPool` constructor option).
+ */
+ private ownsSessionPool: boolean;
+
/**
* A reference to the underlying {@apilink AutoscaledPool} class that manages the concurrency of the crawler.
* > *NOTE:* This property is only initialized after calling the {@apilink BasicCrawler.run|`crawler.run()`} function.
@@ -659,7 +665,6 @@ export class BasicCrawler<
protected statusMessageLoggingInterval: number;
protected statusMessageCallback?: StatusMessageCallback;
- protected sessionPoolOptions?: SessionPoolOptions;
protected blockedStatusCodes = new Set<number>();
protected additionalHttpErrorStatusCodes: Set<number>;
protected ignoreHttpErrorStatusCodes: Set<number>;
@@ -699,7 +704,7 @@ export class BasicCrawler<
maxRequestsPerCrawl: ow.optional.number,
maxCrawlDepth: ow.optional.number,
autoscaledPoolOptions: ow.optional.object,
- sessionPoolOptions: ow.optional.object,
+ sessionPool: ow.optional.object.instanceOf(SessionPool),
proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
statusMessageLoggingInterval: ow.optional.number,
@@ -753,7 +758,7 @@ export class BasicCrawler<
maxCrawlDepth,
autoscaledPoolOptions = {},
keepAlive,
- sessionPoolOptions = {},
+ sessionPool,
proxyConfiguration,
additionalHttpErrorStatusCodes = [],
@@ -882,11 +887,11 @@ export class BasicCrawler<
...(this.hasExplicitId ? { id: this.crawlerId } : {}),
...statisticsOptions,
});
- this.sessionPoolOptions = {
- ...sessionPoolOptions,
- log: this.log,
- };
- this.sessionPool = new SessionPool(this.sessionPoolOptions);
+
+ this.sessionPool = sessionPool ?? new SessionPool();
+ this.sessionPool.setMaxListeners(20);
+
+ this.ownsSessionPool = !sessionPool;
this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);
@@ -1294,7 +1299,9 @@ export class BasicCrawler<
this.stats.reset();
await this.stats.resetStore();
- await this.sessionPool?.resetStore();
+ if (this.ownsSessionPool) {
+ await this.sessionPool.resetStore();
+ }
}
this.unexpectedStop = false;
@@ -1730,7 +1737,6 @@ export class BasicCrawler<
// so that the caller can get a reference to it before awaiting the promise returned from run()
// (otherwise there would be no way)
this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);
- this.sessionPool.setMaxListeners(20);
await this.initializeRequestManager();
}
@@ -2305,13 +2311,19 @@ export class BasicCrawler<
serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });
}
- await this.sessionPool?.teardown({ persistState: this._ownsEventManager });
+ if (this.ownsSessionPool) {
+ await this.sessionPool.teardown({ persistState: this._ownsEventManager });
+ }
if (this._ownsEventManager) {
await serviceLocator.getEventManager().close();
}
+ if (this.ownsSessionPool) {
+ await this.sessionPool.teardown();
+ }
+
await this.autoscaledPool?.abort();
}
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index c87a8babd..daf45afc7 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -325,7 +325,6 @@ export abstract class BrowserCrawler<
launchContext: ow.optional.object,
headless: ow.optional.any(ow.boolean, ow.string),
browserPoolOptions: ow.object,
- sessionPoolOptions: ow.optional.object,
persistCookiesPerSession: ow.optional.boolean,
proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
};
diff --git a/packages/core/src/session_pool/session_pool.ts b/packages/core/src/session_pool/session_pool.ts
index 544230f60..eda376c6e 100644
--- a/packages/core/src/session_pool/session_pool.ts
+++ b/packages/core/src/session_pool/session_pool.ts
@@ -26,6 +26,13 @@ export interface CreateSession {
}
export interface SessionPoolOptions {
+ /**
+ * Unique identifier for this session pool instance. Used to generate a unique
+ * persistence key when `persistStateKey` is not provided.
+ * If not provided, an auto-incrementing ID is used.
+ */
+ id?: string | number;
+
/**
* Maximum size of the pool. Indicates how many sessions are rotated.
* @default 1000
@@ -39,8 +46,8 @@ export interface SessionPoolOptions {
persistStateKeyValueStoreId?: string;
/**
- * Session pool persists it's state under this key in Key value store.
- * @default SESSION_POOL_STATE
+ * Session pool persists its state under this key in Key value store.
+ * @default SDK_SESSION_POOL_STATE_{id}
*/
persistStateKey?: string;
@@ -114,6 +121,9 @@ export interface SessionPoolOptions {
* @category Scaling
*/
export class SessionPool extends EventEmitter {
+ private static nextId = 0;
+
+ readonly id: string;
protected log: CrawleeLogger;
protected maxPoolSize: number;
protected createSessionFunction: CreateSession;
@@ -136,6 +146,7 @@ export class SessionPool extends EventEmitter {
ow(
options,
ow.object.exactShape({
+ id: ow.optional.any(ow.number, ow.string),
maxPoolSize: ow.optional.number,
persistStateKeyValueStoreId: ow.optional.string,
persistStateKey: ow.optional.string,
@@ -147,9 +158,10 @@ export class SessionPool extends EventEmitter {
);
const {
+ id,
maxPoolSize = MAX_POOL_SIZE,
persistStateKeyValueStoreId,
- persistStateKey = PERSIST_STATE_KEY,
+ persistStateKey,
createSessionFunction,
sessionOptions = {},
log = serviceLocator.getLogger(),
@@ -158,6 +170,7 @@ export class SessionPool extends EventEmitter {
},
} = options;
+ this.id = id != null ? String(id) : String(SessionPool.nextId++);
this.events = serviceLocator.getEventManager();
this.log = log.child({ prefix: 'SessionPool' });
this.persistenceOptions = persistenceOptions;
@@ -176,7 +189,7 @@ export class SessionPool extends EventEmitter {
// Session keyValueStore
this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
- this.persistStateKey = persistStateKey;
+ this.persistStateKey = persistStateKey ?? `${PERSIST_STATE_KEY}_${this.id}`;
}
/**
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index f2cce3807..4510a003d 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -19,6 +19,7 @@ import {
RequestValidationError,
Router,
serviceLocator,
+ SessionPool,
} from '@crawlee/basic';
import { RequestState, SessionPool, Statistics } from '@crawlee/core';
import type { Dictionary } from '@crawlee/utils';
@@ -1520,10 +1521,10 @@ describe('BasicCrawler', () => {
requestList,
requestHandlerTimeoutSecs: 0.01,
maxRequestRetries: 1,
- sessionPoolOptions: {
+ sessionPool: new SessionPool({
maxPoolSize: 10,
persistStateKey: 'POOL',
- },
+ }),
requestHandler: async ({ session }) => {
expect(session.constructor.name).toEqual('Session');
expect(session.id).toBeDefined();
@@ -1546,10 +1547,10 @@ describe('BasicCrawler', () => {
requestList,
requestHandlerTimeoutSecs: 0.01,
maxRequestRetries: 1,
- sessionPoolOptions: {
+ sessionPool: new SessionPool({
maxPoolSize: 10,
persistStateKey: 'POOL',
- },
+ }),
requestHandler: async () => {},
failedRequestHandler: async () => {},
});
@@ -1559,19 +1560,16 @@ describe('BasicCrawler', () => {
expect(crawler.sessionPool.maxPoolSize).toEqual(10);
});
- it('should destroy Session pool after it is finished', async () => {
+ it('should accept a pre-initialized SessionPool instance', async () => {
const url = 'https://example.com';
const requestList = await RequestList.open({ sources: [{ url }] });
- serviceLocator.getEventManager().off(EventType.PERSIST_STATE);
+ const sharedPool = new SessionPool({ maxPoolSize: 25 });
const crawler = new BasicCrawler({
requestList,
- requestHandlerTimeoutSecs: 0.01,
- maxRequestRetries: 1,
- sessionPoolOptions: {
- maxPoolSize: 10,
- },
- requestHandler: async () => {
+ sessionPool: sharedPool,
+ requestHandler: async ({ session }) => {
+ expect(session).toBeDefined();
expect(crawler.sessionPool).toBeDefined();
expect(serviceLocator.getEventManager().listenerCount(EventType.PERSIST_STATE)).toEqual(1);
},
@@ -1579,9 +1577,60 @@ describe('BasicCrawler', () => {
});
await crawler.run();
- expect(serviceLocator.getEventManager().listenerCount(EventType.PERSIST_STATE)).toEqual(0);
- // @ts-expect-error private symbol
- expect(crawler.sessionPool.maxPoolSize).toEqual(10);
+
+ expect(crawler.sessionPool).toBe(sharedPool);
+ await sharedPool.teardown();
+ });
+
+ it('should not tear down an injected SessionPool', async () => {
+ const url = 'https://example.com';
+ const requestList = await RequestList.open({ sources: [{ url }] });
+ const sharedPool = new SessionPool({ maxPoolSize: 25 });
+ const teardownSpy = vitest.spyOn(sharedPool, 'teardown');
+
+ const crawler = new BasicCrawler({
+ requestList,
+ sessionPool: sharedPool,
+ requestHandler: async () => {},
+ });
+ await crawler.run();
+
+ expect(teardownSpy).not.toHaveBeenCalled();
+ await sharedPool.teardown();
+ });
+
+ it('should share sessions across crawlers using the same SessionPool', async () => {
+ const sharedPool = new SessionPool({ maxPoolSize: 5 });
+ const crawler1Sessions = new Set<string>();
+ const crawler2Sessions = new Set<string>();
+
+ const requestList1 = await RequestList.open({ sources: [{ url: 'https://example.com' }] });
+ const crawler1 = new BasicCrawler({
+ requestList: requestList1,
+ sessionPool: sharedPool,
+ requestHandler: async ({ session }) => {
+ crawler1Sessions.add(session.id);
+ },
+ });
+ await crawler1.run();
+
+ expect(crawler1Sessions.size).toBeGreaterThan(0);
+ const poolSizeAfterCrawler1 = sharedPool.usableSessionsCount;
+
+ const requestList2 = await RequestList.open({ sources: [{ url: 'https://example.com' }] });
+ const crawler2 = new BasicCrawler({
+ requestList: requestList2,
+ sessionPool: sharedPool,
+ requestHandler: async ({ session }) => {
+ crawler2Sessions.add(session.id);
+ },
+ });
+ await crawler2.run();
+
+ expect(crawler1.sessionPool).toBe(crawler2.sessionPool);
+ // crawler2 should reuse sessions created by crawler1, not grow the pool further
+ expect(sharedPool.usableSessionsCount).toBe(poolSizeAfterCrawler1);
+ await sharedPool.teardown();
});
});
diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts
index 78320bf42..f290da6c1 100644
--- a/test/core/crawlers/browser_crawler.test.ts
+++ b/test/core/crawlers/browser_crawler.test.ts
@@ -1,7 +1,7 @@
import type { Server } from 'node:http';
import { BROWSER_POOL_EVENTS, OperatingSystemsName, PuppeteerPlugin } from '@crawlee/browser-pool';
-import { bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ServiceLocator } from '@crawlee/core';
+import { bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ServiceLocator, SessionPool } from '@crawlee/core';
import type { PuppeteerGoToOptions } from '@crawlee/puppeteer';
import { EnqueueStrategy, ProxyConfiguration, Request, RequestList, RequestState, Session } from '@crawlee/puppeteer';
import { sleep } from '@crawlee/utils';
@@ -523,19 +523,19 @@ describe('BrowserCrawler', () => {
},
persistCookiesPerSession: false,
- sessionPoolOptions: {
+ sessionPool: new SessionPool({
sessionOptions: {
maxUsageCount: 1,
},
persistStateKeyValueStoreId: 'abc',
- },
+ }),
requestHandler: async () => {},
});
// @ts-expect-error Accessing private prop
- expect(crawler.sessionPoolOptions.sessionOptions.maxUsageCount).toBe(1);
+ expect(crawler.sessionPool.sessionOptions.maxUsageCount).toBe(1);
// @ts-expect-error Accessing private prop
- expect(crawler.sessionPoolOptions.persistStateKeyValueStoreId).toBe('abc');
+ expect(crawler.sessionPool.persistStateKeyValueStoreId).toBe('abc');
} finally {
await localStorageEmulator.destroy();
}
@@ -842,10 +842,9 @@ describe('BrowserCrawler', () => {
browserPoolOptions: {
browserPlugins: [puppeteerPlugin],
},
-
- sessionPoolOptions: {
+ sessionPool: new SessionPool({
maxPoolSize: 1,
- },
+ }),
requestHandler: async ({ session }) => {
sessionUsageHistory.push(session!.usageCount);
},
@@ -900,88 +899,6 @@ describe('BrowserCrawler', () => {
});
describe('proxy', () => {
- // TODO move to actor sdk tests before splitting the repos
- // test('browser should launch with correct proxyUrl', async () => {
- // process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
- // const status = { connected: true };
- // const fakeCall = async () => {
- // return { body: status } as never;
- // };
- //
- // // @ts-expect-error FIXME
- // const stub = gotScrapingSpy.mockImplementation(fakeCall);
- // const proxyConfiguration = await Actor.createProxyConfiguration();
- // const generatedProxyUrl = new URL(await proxyConfiguration.newUrl()).href.slice(0, -1);
- // let browserProxy;
- //
- // const browserCrawler = new BrowserCrawlerTest({
- // browserPoolOptions: {
- // browserPlugins: [puppeteerPlugin],
- // postLaunchHooks: [(pageId, browserController) => {
- // browserProxy = browserController.launchContext.proxyUrl;
- // }],
- // },
- // useSessionPool: false,
- // persistCookiesPerSession: false,
- // navigationTimeoutSecs: 1,
- // requestList,
- // maxRequestsPerCrawl: 1,
- // maxRequestRetries: 0,
- // requestHandler: async () => {},
- // proxyConfiguration,
- // });
- // await browserCrawler.run();
- // delete process.env[ENV_VARS.PROXY_PASSWORD];
- //
- // expect(browserProxy).toEqual(generatedProxyUrl);
- //
- // stub.mockClear();
- // });
-
- // TODO move to actor sdk tests before splitting the repos
- // test('requestHandler should expose the proxyInfo object with sessions correctly', async () => {
- // process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
- // const status = { connected: true };
- // const fakeCall = async () => {
- // return { body: status } as never;
- // };
- //
- // // @ts-expect-error FIXME
- // const stub = gotScrapingSpy.mockImplementation(fakeCall);
- //
- // const proxyConfiguration = await Actor.createProxyConfiguration();
- // const proxies: ProxyInfo[] = [];
- // const sessions: Session[] = [];
- // const requestHandler = async ({ session, proxyInfo }: BrowserCrawlingContext) => {
- // proxies.push(proxyInfo);
- // sessions.push(session);
- // };
- //
- // const browserCrawler = new BrowserCrawlerTest({
- // browserPoolOptions: {
- // browserPlugins: [puppeteerPlugin],
- // },
- // requestList,
- // requestHandler,
- //
- // proxyConfiguration,
- // useSessionPool: true,
- // sessionPoolOptions: {
- // maxPoolSize: 1,
- // },
- // });
- //
- // await browserCrawler.run();
- //
- // expect(proxies[0].sessionId).toEqual(sessions[0].id);
- // expect(proxies[1].sessionId).toEqual(sessions[1].id);
- // expect(proxies[2].sessionId).toEqual(sessions[2].id);
- // expect(proxies[3].sessionId).toEqual(sessions[3].id);
- //
- // delete process.env[ENV_VARS.PROXY_PASSWORD];
- // stub.mockClear();
- // });
-
// This test manipulates environment variables, so it must NOT be run concurrently
test('browser should launch with rotated custom proxy', async () => {
const localStorageEmulator = new MemoryStorageEmulator();
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index 6efba0b0c..d1631e7e8 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -11,7 +11,7 @@ import {
RequestList,
Session,
} from '@crawlee/cheerio';
-import { BaseCrawleeLogger } from '@crawlee/core';
+import { BaseCrawleeLogger, SessionPool } from '@crawlee/core';
import { ImpitHttpClient } from '@crawlee/impit-client';
import type { ProxyInfo } from '@crawlee/types';
import type { Dictionary } from '@crawlee/utils';
@@ -893,18 +893,18 @@ describe('CheerioCrawler', () => {
requestList,
persistCookiesPerSession: false,
- sessionPoolOptions: {
+ sessionPool: new SessionPool({
sessionOptions: {
maxUsageCount: 1,
},
persistStateKeyValueStoreId: 'abc',
- },
+ }),
requestHandler: () => {},
});
// @ts-expect-error Accessing private prop
- expect(crawler.sessionPoolOptions.sessionOptions.maxUsageCount).toBe(1);
+ expect(crawler.sessionPool.sessionOptions.maxUsageCount).toBe(1);
// @ts-expect-error Accessing private prop
- expect(crawler.sessionPoolOptions.persistStateKeyValueStoreId).toBe('abc');
+ expect(crawler.sessionPool.persistStateKeyValueStoreId).toBe('abc');
});
test('should markBad sessions after request timeout', async () => {
@@ -1000,9 +1000,9 @@ describe('CheerioCrawler', () => {
),
persistCookiesPerSession: true,
- sessionPoolOptions: {
+ sessionPool: new SessionPool({
maxPoolSize: 1,
- },
+ }),
maxRequestRetries: 1,
maxConcurrency: 1,
requestHandler: ({ request }) => {
@@ -1029,10 +1029,10 @@ describe('CheerioCrawler', () => {
headers: { cookie: 'foo=bar2; baz=123' },
},
]),
-
- sessionPoolOptions: {
+ persistCookiesPerSession: false,
+ sessionPool: new SessionPool({
maxPoolSize: 1,
- },
+ }),
preNavigationHooks: [
({ session, request }) => {
// this should get overriden by the server
@@ -1068,9 +1068,9 @@ describe('CheerioCrawler', () => {
]),
persistCookiesPerSession: false,
- sessionPoolOptions: {
+ sessionPool: new SessionPool({
maxPoolSize: 1,
- },
+ }),
requestHandler: ({ json }) => {
responses.push(json);
},
@@ -1177,10 +1177,10 @@ describe('CheerioCrawler', () => {
const crawler = new CheerioCrawler({
requestList: await RequestList.open(null, [{ url: `${serverAddress}/special/get-cookies` }]),
- sessionPoolOptions: {
+ sessionPool: new SessionPool({
// Even with multiple available sessions, the preNavigationHook should use the same one as the main request
maxPoolSize: 10,
- },
+ }),
preNavigationHooks: [
async ({ sendRequest }) => {
await sendRequest({
diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts
index b6db33a13..85ad5954b 100644
--- a/test/core/crawlers/http_crawler.test.ts
+++ b/test/core/crawlers/http_crawler.test.ts
@@ -2,7 +2,7 @@ import http from 'node:http';
import type { AddressInfo } from 'node:net';
import { Readable } from 'node:stream';
-import { HttpCrawler } from '@crawlee/http';
+import { HttpCrawler, SessionPool } from '@crawlee/http';
import { ResponseWithUrl } from '@crawlee/http-client';
import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator.js';
@@ -218,9 +218,9 @@ test('handles cookies from redirects', async () => {
const results: string[] = [];
const crawler = new HttpCrawler({
- sessionPoolOptions: {
+ sessionPool: new SessionPool({
maxPoolSize: 1,
- },
+ }),
requestHandler: async ({ body }) => {
results.push(JSON.parse(body.toString()));
},
@@ -255,9 +255,9 @@ test('handles cookies from redirects - no empty cookie header', async () => {
const results: string[] = [];
const crawler = new HttpCrawler({
- sessionPoolOptions: {
+ sessionPool: new SessionPool({
maxPoolSize: 1,
- },
+ }),
requestHandler: async ({ body }) => {
const str = body.toString();
@@ -276,9 +276,9 @@ test('no empty cookie header', async () => {
const results: string[] = [];
const crawler = new HttpCrawler({
- sessionPoolOptions: {
+ sessionPool: new SessionPool({
maxPoolSize: 1,
- },
+ }),
requestHandler: async ({ body }) => {
const str = body.toString();
diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.ts
index 487e07bee..157f7bec5 100644
--- a/test/core/crawlers/puppeteer_crawler.test.ts
+++ b/test/core/crawlers/puppeteer_crawler.test.ts
@@ -22,6 +22,7 @@ import {
RequestQueue,
RequestValidationError,
Session,
+ SessionPool,
} from '@crawlee/puppeteer';
import type { Cookie } from '@crawlee/types';
import { sleep } from '@crawlee/utils';
@@ -332,13 +333,13 @@ describe('PuppeteerCrawler', () => {
requestList,
persistCookiesPerSession: true,
- sessionPoolOptions: {
+ sessionPool: new SessionPool({
createSessionFunction: (sessionPool) => {
const session = new Session({ sessionPool });
session.setCookies(cookies, serverUrl);
return session;
},
- },
+ }),
requestHandler: async ({ page, session }) => {
pageCookies = await page.cookies().then((cks) => cks.map((c) => `${c.name}=${c.value}`).join('; '));
sessionCookies = session!.getCookieString(serverUrl);
@@ -365,11 +366,11 @@ describe('PuppeteerCrawler', () => {
},
},
maxConcurrency: 1,
- sessionPoolOptions: {
+ sessionPool: new SessionPool({
sessionOptions: {
maxUsageCount: 1,
},
- },
+ }),
proxyConfiguration,
requestHandler: async ({ proxyInfo, session }) => {
proxies.add(proxyInfo!.url);
diff --git a/test/core/session_pool/session_pool.test.ts b/test/core/session_pool/session_pool.test.ts
index 8129f3bfa..3a092d78b 100644
--- a/test/core/session_pool/session_pool.test.ts
+++ b/test/core/session_pool/session_pool.test.ts
@@ -131,6 +131,9 @@ describe('SessionPool - testing session pool', () => {
});
test('should persist state and recreate it from storage', async () => {
+ const persistStateKey = 'PERSIST_TEST';
+ sessionPool = new SessionPool({ persistStateKey });
+
await sessionPool.getSession();
await sessionPool.persistState();
@@ -163,7 +166,7 @@ describe('SessionPool - testing session pool', () => {
});
});
- const loadedSessionPool = new SessionPool();
+ const loadedSessionPool = new SessionPool({ persistStateKey });
// @ts-expect-error Accessing protected method
await loadedSessionPool.ensureInitialized();
// @ts-expect-error private symbol
@@ -251,6 +254,9 @@ describe('SessionPool - testing session pool', () => {
});
test('should recreate only usable sessions', async () => {
+ const persistStateKey = 'RECREATE_TEST';
+ sessionPool = new SessionPool({ persistStateKey });
+
let invalidSessionsCount = 0;
for (let i = 0; i < 10; i++) {
const session = await sessionPool.getSession();
@@ -265,7 +271,7 @@ describe('SessionPool - testing session pool', () => {
await sessionPool.persistState();
- const newSessionPool = new SessionPool();
+ const newSessionPool = new SessionPool({ persistStateKey });
// @ts-expect-error Accessing protected method
await newSessionPool.ensureInitialized();
// @ts-expect-error Accessing private property
@@ -275,10 +281,19 @@ describe('SessionPool - testing session pool', () => {
});
test('should restore persisted maxUsageCount of recreated sessions', async () => {
- sessionPool = new SessionPool({ maxPoolSize: 1, sessionOptions: { maxUsageCount: 66 } });
+ const persistStateKey = 'MAX_USAGE_TEST';
+ sessionPool = new SessionPool({
+ maxPoolSize: 1,
+ sessionOptions: { maxUsageCount: 66 },
+ persistStateKey,
+ });
await sessionPool.getSession();
await sessionPool.persistState();
- const loadedSessionPool = new SessionPool({ maxPoolSize: 1, sessionOptions: { maxUsageCount: 88 } });
+ const loadedSessionPool = new SessionPool({
+ maxPoolSize: 1,
+ sessionOptions: { maxUsageCount: 88 },
+ persistStateKey,
+ });
const recreatedSession = await loadedSessionPool.getSession();
@@ -409,4 +424,68 @@ describe('SessionPool - testing session pool', () => {
// @ts-expect-error private symbol
expect(sessionPool.sessions.length).toEqual(sessionPool.sessionMap.size);
});
+
+ describe('multiple SessionPool instances isolation', () => {
+ test('should use unique persist keys by default', async () => {
+ const pool1 = new SessionPool();
+ const pool2 = new SessionPool();
+
+ // @ts-expect-error private symbol
+ expect(pool1.persistStateKey).not.toEqual(pool2.persistStateKey);
+
+ await pool1.teardown();
+ await pool2.teardown();
+ });
+
+ test("should not overwrite each other's persisted state", async () => {
+ const pool1 = new SessionPool({ maxPoolSize: 5 });
+ const pool2 = new SessionPool({ maxPoolSize: 5 });
+
+ for (let i = 0; i < 3; i++) await pool1.getSession();
+ for (let i = 0; i < 5; i++) await pool2.getSession();
+
+ await pool1.persistState();
+ await pool2.persistState();
+
+ const pool1Reloaded = new SessionPool({
+ // @ts-expect-error private symbol
+ persistStateKey: pool1.persistStateKey,
+ });
+ const pool2Reloaded = new SessionPool({
+ // @ts-expect-error private symbol
+ persistStateKey: pool2.persistStateKey,
+ });
+
+ // @ts-expect-error -- we're reading the private sessions field, public methods initialize the instance automatically
+ await pool1Reloaded.ensureInitialized();
+ // @ts-expect-error
+ await pool2Reloaded.ensureInitialized();
+
+ // @ts-expect-error private symbol
+ expect(pool1Reloaded.sessions).toHaveLength(3);
+ // @ts-expect-error private symbol
+ expect(pool2Reloaded.sessions).toHaveLength(5);
+
+ await pool1.teardown();
+ await pool2.teardown();
+ await pool1Reloaded.teardown();
+ await pool2Reloaded.teardown();
+ });
+
+ test('retiring sessions in one pool should not affect another', async () => {
+ const pool1 = new SessionPool({ maxPoolSize: 2 });
+ const pool2 = new SessionPool({ maxPoolSize: 2 });
+
+ const session1 = await pool1.getSession();
+ await pool2.getSession();
+
+ session1.retire();
+
+ expect(await pool1.retiredSessionsCount()).toBe(1);
+ expect(await pool2.retiredSessionsCount()).toBe(0);
+
+ await pool1.teardown();
+ await pool2.teardown();
+ });
+ });
});