Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: lazy-initialize `SessionPool`, drop `SessionPool.open()` (#3513)
Jindřich Bära23e79bc2edd
diff --git a/docs/guides/session_management_standalone.ts b/docs/guides/session_management_standalone.tsindex c6fa33d82..5062a8a87 100644--- a/docs/guides/session_management_standalone.ts+++ b/docs/guides/session_management_standalone.ts@@ -5,8 +5,7 @@ const sessionPoolOptions = {maxPoolSize: 100,};-// Open Session Pool.-const sessionPool = await SessionPool.open(sessionPoolOptions);+const sessionPool = new SessionPool(sessionPoolOptions);// Get session.const session = await sessionPool.getSession();diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex 4a0a4fc1e..862f8023b 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -84,6 +84,26 @@ const crawler = new CheerioCrawler({Previously, the crawling context extended a `Record` type, allowing to access any property. This was changed to a strict type, which means that you can only access properties that are defined in the context.+## `SessionPool` is now lazy-initialized++`SessionPool.open()` static factory method is removed. Create instances with `new SessionPool(options)` instead — all public methods automatically initialize the pool on first use.++`SessionPool.usableSessionsCount` and `SessionPool.retiredSessionsCount` are now async methods instead of synchronous getters. `SessionPool.getState()` is also async now.++**Before:**+```typescript+const sessionPool = await SessionPool.open({ maxPoolSize: 100 });+const count = sessionPool.usableSessionsCount;+const state = sessionPool.getState();+```++**After:**+```typescript+const sessionPool = new SessionPool({ maxPoolSize: 100 });+const count = await sessionPool.usableSessionsCount();+const state = await sessionPool.getState();+```+## `retireOnBlockedStatusCodes` is removed from `Session``Session.retireOnBlockedStatusCodes` is removed. Blocked status code handling is now internal to the crawler. Configure blocked status codes via the `blockedStatusCodes` crawler option (moved from `sessionPoolOptions`).diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 7ab97aeff..0377cd3a0 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -571,7 +571,7 @@ export class BasicCrawler</*** A reference to the underlying {@apilink SessionPool} class that manages the crawler's {@apilink Session|sessions}.*/- sessionPool?: SessionPool;+ sessionPool: SessionPool;/*** A reference to the underlying {@apilink AutoscaledPool} class that manages the concurrency of the crawler.@@ -658,7 +658,7 @@ export class BasicCrawler<protected statusMessageLoggingInterval: number;protected statusMessageCallback?: StatusMessageCallback;- protected sessionPoolOptions: SessionPoolOptions;+ protected sessionPoolOptions?: SessionPoolOptions;protected blockedStatusCodes = new Set<number>();protected additionalHttpErrorStatusCodes: Set<number>;protected ignoreHttpErrorStatusCodes: Set<number>;@@ -885,6 +885,8 @@ export class BasicCrawler<...sessionPoolOptions,log: this.log,};+ this.sessionPool = new SessionPool(this.sessionPoolOptions);+this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);const maxSignedInteger = 2 ** 31 - 1;@@ -1725,8 +1727,6 @@ export class BasicCrawler<// so that the caller can get a reference to it before awaiting the promise returned from run()// (otherwise there would be no way)this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);-- this.sessionPool = await SessionPool.open(this.sessionPoolOptions);this.sessionPool.setMaxListeners(20);await this.initializeRequestManager();diff --git a/packages/core/src/session_pool/session_pool.ts b/packages/core/src/session_pool/session_pool.tsindex c6e310276..2e2ceb8ce 100644--- a/packages/core/src/session_pool/session_pool.ts+++ b/packages/core/src/session_pool/session_pool.ts@@ -66,9 +66,8 @@ export interface SessionPoolOptions {* When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).* Learn more in the {@doclink guides/session-management | Session management guide}.*- * You can create one by calling the {@apilink SessionPool.open} function.- ** Session pool is already integrated into crawlers and is always active.+ * All public methods are lazy-initialized — the pool initializes itself on first use.** You can configure the pool with many options. See the {@apilink SessionPoolOptions}.* Session pool is by default persisted in default {@apilink KeyValueStore}.@@ -78,7 +77,7 @@ export interface SessionPoolOptions {* **Advanced usage:**** ```javascript- * const sessionPool = await SessionPool.open({+ * const sessionPool = new SessionPool({* maxPoolSize: 25,* sessionOptions:{* maxAgeSecs: 10,@@ -118,22 +117,19 @@ export class SessionPool extends EventEmitter {protected log: CrawleeLogger;protected maxPoolSize: number;protected createSessionFunction: CreateSession;- protected keyValueStore!: KeyValueStore;+ protected keyValueStore?: KeyValueStore;protected sessions: Session[] = [];protected sessionMap = new Map<string, Session>();protected sessionOptions: SessionOptions;protected persistStateKeyValueStoreId?: string;protected persistStateKey: string;- protected _listener!: () => Promise<void>;+ protected _listener?: () => Promise<void>;protected events: EventManager;protected persistenceOptions: PersistenceOptions;- protected isInitialized = false;+ private initPromise?: Promise<void>;private queue = new AsyncQueue();- /**- * @internal- */constructor(options: SessionPoolOptions = {}) {super();@@ -186,33 +182,38 @@ export class SessionPool extends EventEmitter {/*** Gets count of usable sessions in the pool.*/- get usableSessionsCount(): number {+ async usableSessionsCount(): Promise<number> {+ await this.ensureInitialized();return this.sessions.filter((session) => session.isUsable()).length;}/*** Gets count of retired sessions in the pool.*/- get retiredSessionsCount(): number {+ async retiredSessionsCount(): Promise<number> {+ await this.ensureInitialized();return this.sessions.filter((session) => !session.isUsable()).length;}/*** Starts periodic state persistence and potentially loads SessionPool state from {@apilink KeyValueStore}.- * It is called automatically by the {@apilink SessionPool.open} function.+ * Called automatically on first use of any public method.*/- async initialize(): Promise<void> {- if (this.isInitialized) {+ protected async ensureInitialized(): Promise<void> {+ if (!this.initPromise) {+ this.initPromise = this.setupPool();+ }+ return this.initPromise;+ }++ private async setupPool(): Promise<void> {+ if (!this.persistenceOptions.enable) {return;}this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId, {config: serviceLocator.getConfiguration(),});- if (!this.persistenceOptions.enable) {- this.isInitialized = true;- return;- }if (!this.persistStateKeyValueStoreId) {this.log.debug(@@ -224,9 +225,7 @@ export class SessionPool extends EventEmitter {await this._maybeLoadSessionPool();this._listener = this.persistState.bind(this);-this.events.on(EventType.PERSIST_STATE, this._listener);- this.isInitialized = true;}/**@@ -236,7 +235,7 @@ export class SessionPool extends EventEmitter {* @param [options] The configuration options for the session being added to the session pool.*/async addSession(options: Session | SessionOptions = {}): Promise<void> {- this._throwIfNotInitialized();+ await this.ensureInitialized();const { id } = options;if (id) {const sessionExists = this.sessionMap.has(id);@@ -263,7 +262,7 @@ export class SessionPool extends EventEmitter {* @param [options] The configuration options for the session being added to the session pool.*/async newSession(sessionOptions?: SessionOptions): Promise<Session> {- this._throwIfNotInitialized();+ await this.ensureInitialized();const newSession = await this.createSessionFunction(this, { sessionOptions });this._addSession(newSession);@@ -292,11 +291,10 @@ export class SessionPool extends EventEmitter {* @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.*/async getSession(sessionId?: string): Promise<Session | undefined> {- await this.queue.wait();+ await this.ensureInitialized();+ await this.queue.wait();try {- this._throwIfNotInitialized();-if (sessionId) {const session = this.sessionMap.get(sessionId);if (session && session.isUsable()) return session;@@ -327,6 +325,7 @@ export class SessionPool extends EventEmitter {return;}+ await this.ensureInitialized();await this.keyValueStore?.setValue(this.persistStateKey, null);}@@ -334,10 +333,11 @@ export class SessionPool extends EventEmitter {* Returns an object representing the internal state of the `SessionPool` instance.* Note that the object's fields can change in future releases.*/- getState() {+ async getState() {+ await this.ensureInitialized();return {- usableSessionsCount: this.usableSessionsCount,- retiredSessionsCount: this.retiredSessionsCount,+ usableSessionsCount: await this.usableSessionsCount(),+ retiredSessionsCount: await this.retiredSessionsCount(),sessions: this.sessions.map((session) => session.getState()),};}@@ -352,6 +352,8 @@ export class SessionPool extends EventEmitter {return;}+ await this.ensureInitialized();+this.log.debug('Persisting state', {persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,persistStateKey: this.persistStateKey,@@ -361,7 +363,7 @@ export class SessionPool extends EventEmitter {const persistStateIntervalMillis = serviceLocator.getConfiguration().get('persistStateIntervalMillis')!;const timeoutSecs = persistStateIntervalMillis / 2_000;await this.keyValueStore- .setValue(this.persistStateKey, this.getState(), {+ ?.setValue(this.persistStateKey, await this.getState(), {timeoutSecs,doNotRetryTimeouts: true,})@@ -376,19 +378,16 @@ export class SessionPool extends EventEmitter {* @param options - Set `persistState` to false when the final state was already persisted by the event manager.*/async teardown({ persistState = true }: { persistState?: boolean } = {}): Promise<void> {- this.events.off(EventType.PERSIST_STATE, this._listener);+ if (!this.initPromise) return;+ await this.ensureInitialized();+ if (this._listener) {+ this.events.off(EventType.PERSIST_STATE, this._listener);+ }if (persistState) {await this.persistState();}}- /**- * SessionPool should not work before initialization.- */- protected _throwIfNotInitialized() {- if (!this.isInitialized) throw new Error('SessionPool is not initialized.');- }-/*** Removes retired `Session` instances from `SessionPool`.*/@@ -472,7 +471,7 @@ export class SessionPool extends EventEmitter {* If the state was persisted it loads the `SessionPool` from the persisted state.*/protected async _maybeLoadSessionPool(): Promise<void> {- const loadedSessionPool = await this.keyValueStore.getValue<{ sessions: Dictionary[] }>(this.persistStateKey);+ const loadedSessionPool = await this.keyValueStore?.getValue<{ sessions: Dictionary[] }>(this.persistStateKey);if (!loadedSessionPool) return;@@ -493,18 +492,6 @@ export class SessionPool extends EventEmitter {}}- this.log.debug(`${this.usableSessionsCount} active sessions loaded from KeyValueStore`);- }-- /**- * Opens a SessionPool and returns a promise resolving to an instance- * of the {@apilink SessionPool} class that is already initialized.- *- * For more details and code examples, see the {@apilink SessionPool} class.- */- static async open(options?: SessionPoolOptions): Promise<SessionPool> {- const sessionPool = new SessionPool(options);- await sessionPool.initialize();- return sessionPool;+ this.log.debug(`${this.sessions.length} active sessions loaded from KeyValueStore`);}}diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 5b9d2e862..89990482e 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -1571,16 +1571,13 @@ describe('BasicCrawler', () => {sessionPoolOptions: {maxPoolSize: 10,},- requestHandler: async () => {},+ requestHandler: async () => {+ expect(crawler.sessionPool).toBeDefined();+ expect(serviceLocator.getEventManager().listenerCount(EventType.PERSIST_STATE)).toEqual(1);+ },failedRequestHandler: async () => {},});- // @ts-expect-error Accessing private prop- crawler._loadHandledRequestCount = () => {- expect(crawler.sessionPool).toBeDefined();- expect(serviceLocator.getEventManager().listenerCount(EventType.PERSIST_STATE)).toEqual(1);- };-await crawler.run();expect(serviceLocator.getEventManager().listenerCount(EventType.PERSIST_STATE)).toEqual(0);// @ts-expect-error private symboldiff --git a/test/core/session_pool/session_pool.test.ts b/test/core/session_pool/session_pool.test.tsindex aa28bd2d7..3801b31ac 100644--- a/test/core/session_pool/session_pool.test.ts+++ b/test/core/session_pool/session_pool.test.ts@@ -8,7 +8,7 @@ describe('SessionPool - testing session pool', () => {beforeEach(async () => {await localStorageEmulator.init();- sessionPool = await SessionPool.open();+ sessionPool = new SessionPool();});afterEach(async () => {@@ -46,34 +46,6 @@ describe('SessionPool - testing session pool', () => {createSessionFunction: () => ({}) as never,};sessionPool = new SessionPool(opts);- await sessionPool.initialize();- await sessionPool.teardown();-- entries(opts)- .filter(([key]) => key !== 'sessionOptions')- .forEach(([key, value]) => {- expect(sessionPool[key]).toEqual(value);- });- // log is appended to sessionOptions after sessionPool instantiation- // @ts-expect-error private symbol- expect(sessionPool.sessionOptions).toEqual({ ...opts.sessionOptions, log: expect.any(BaseCrawleeLogger) });- });-- test('should work using SessionPool.open', async () => {- const opts = {- maxPoolSize: 3000,-- sessionOptions: {- maxAgeSecs: 100,- maxUsageCount: 1,- },-- persistStateKeyValueStoreId: 'TEST',- persistStateKey: 'SESSION_POOL_STATE2',-- createSessionFunction: () => ({}) as never,- };- sessionPool = await SessionPool.open(opts);await sessionPool.teardown();entries(opts)@@ -88,7 +60,7 @@ describe('SessionPool - testing session pool', () => {describe('should retrieve session', () => {test('should retrieve session with correct shape', async () => {- sessionPool = await SessionPool.open({ sessionOptions: { maxAgeSecs: 100, maxUsageCount: 10 } });+ sessionPool = new SessionPool({ sessionOptions: { maxAgeSecs: 100, maxUsageCount: 10 } });const session = await sessionPool.getSession();// @ts-expect-error private symbolexpect(sessionPool.sessions.length).toBe(1);@@ -151,7 +123,7 @@ describe('SessionPool - testing session pool', () => {const newSession = await sessionPool.getSession();newSession.setCookies(cookies, url);- const state = sessionPool.getState();+ const state = await sessionPool.getState();expect(state).toBeInstanceOf(Object);expect(state).toHaveProperty('usableSessionsCount');expect(state).toHaveProperty('retiredSessionsCount');@@ -163,16 +135,14 @@ describe('SessionPool - testing session pool', () => {await sessionPool.persistState();const kvStore = await KeyValueStore.open();- const sessionPoolSaved = await kvStore.getValue<ReturnType<SessionPool['getState']>>(+ const sessionPoolSaved = await kvStore.getValue<Awaited<ReturnType<SessionPool['getState']>>>(// @ts-expect-error private symbolsessionPool.persistStateKey,);- entries(sessionPoolSaved!).forEach(([key, value]) => {- if (key !== 'sessions') {- expect(value).toEqual(sessionPool[key]);- }- });+ const currentState = await sessionPool.getState();+ expect(sessionPoolSaved!.usableSessionsCount).toEqual(currentState.usableSessionsCount);+ expect(sessionPoolSaved!.retiredSessionsCount).toEqual(currentState.retiredSessionsCount);// @ts-expect-error private symbolexpect(sessionPoolSaved.sessions.length).toEqual(sessionPool.sessions.length);@@ -194,8 +164,8 @@ describe('SessionPool - testing session pool', () => {});const loadedSessionPool = new SessionPool();-- await loadedSessionPool.initialize();+ // @ts-expect-error Accessing protected method+ await loadedSessionPool.ensureInitialized();// @ts-expect-error private symbolexpect(sessionPool.sessions).toHaveLength(loadedSessionPool.sessions.length);// @ts-expect-error private symbol@@ -216,6 +186,8 @@ describe('SessionPool - testing session pool', () => {});test('should create session', async () => {+ // @ts-expect-error Accessing protected method+ await sessionPool.ensureInitialized();// @ts-expect-error private symbolawait sessionPool._createSession();// @ts-expect-error private symbol@@ -229,7 +201,6 @@ describe('SessionPool - testing session pool', () => {beforeEach(async () => {sessionPool = new SessionPool({ persistStateKeyValueStoreId: KV_STORE });- await sessionPool.initialize();});afterEach(async () => {@@ -258,7 +229,7 @@ describe('SessionPool - testing session pool', () => {// @ts-expect-error private symbolconst state = await sessionPool.keyValueStore.getValue(sessionPool.persistStateKey);- expect(sessionPool.getState()).toEqual(state);+ expect(await sessionPool.getState()).toEqual(state);});});@@ -290,24 +261,24 @@ describe('SessionPool - testing session pool', () => {invalidSessionsCount += 1;}}- expect(sessionPool.retiredSessionsCount).toEqual(invalidSessionsCount);+ expect(await sessionPool.retiredSessionsCount()).toEqual(invalidSessionsCount);await sessionPool.persistState();const newSessionPool = new SessionPool();- await newSessionPool.initialize();- // @ts-expect-error private symbol+ // @ts-expect-error Accessing protected method+ await newSessionPool.ensureInitialized();+ // @ts-expect-error Accessing private propertyexpect(newSessionPool.sessions).toHaveLength(10 - invalidSessionsCount);await newSessionPool.teardown();});test('should restore persisted maxUsageCount of recreated sessions', async () => {- sessionPool = await SessionPool.open({ maxPoolSize: 1, sessionOptions: { maxUsageCount: 66 } });+ sessionPool = new SessionPool({ maxPoolSize: 1, sessionOptions: { maxUsageCount: 66 } });await sessionPool.getSession();await sessionPool.persistState();const loadedSessionPool = new SessionPool({ maxPoolSize: 1, sessionOptions: { maxUsageCount: 88 } });- await loadedSessionPool.initialize();const recreatedSession = await loadedSessionPool.getSession();@@ -318,11 +289,13 @@ describe('SessionPool - testing session pool', () => {const persistStateKey = 'TEST-KEY';const persistStateKeyValueStoreId = 'TEST-VALUE-STORE';- const newSessionPool = await SessionPool.open({+ const newSessionPool = new SessionPool({maxPoolSize: 1,persistStateKeyValueStoreId,persistStateKey,});+ // @ts-expect-error Accessing protected method+ await newSessionPool.ensureInitialized();await newSessionPool.teardown();@@ -349,7 +322,7 @@ describe('SessionPool - testing session pool', () => {expect(sessionPool2 instanceof SessionPool).toBe(true);return new Session({ sessionPool: sessionPool2 });};- const newSessionPool = await SessionPool.open({ createSessionFunction });+ const newSessionPool = new SessionPool({ createSessionFunction });const session = await newSessionPool.getSession();expect(isCalled).toBe(true);expect(session.constructor.name).toBe('Session');@@ -357,6 +330,8 @@ describe('SessionPool - testing session pool', () => {it('should remove persist state event listener', async () => {const events = serviceLocator.getEventManager();+ // @ts-expect-error Accessing protected method+ await sessionPool.ensureInitialized();expect(events.listenerCount(EventType.PERSIST_STATE)).toEqual(1);await sessionPool.teardown();expect(events.listenerCount(EventType.PERSIST_STATE)).toEqual(0);