Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Feature/session pool persist state on teardown (#615)
Jan Bártac0507c08fb8b
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex bde1f92b7..ab617d04f 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -307,7 +307,7 @@ class BasicCrawler {await this.isRunningPromise;} finally {if (this.useSessionPool) {- this.sessionPool.teardown();+ await this.sessionPool.teardown();}this.stats.stopLogging();diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex 92d361a1e..66750cc6e 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -311,9 +311,9 @@ class PuppeteerCrawler {await this.isRunningPromise;} finally {if (this.useSessionPool) {- this.sessionPool.teardown();+ await this.sessionPool.teardown();}- this.puppeteerPool.destroy();+ await this.puppeteerPool.destroy();}}diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.jsindex b72bb453b..23a93079c 100644--- a/src/session_pool/session_pool.js+++ b/src/session_pool/session_pool.js@@ -202,8 +202,9 @@ export class SessionPool extends EventEmitter {* Removes listener from `persistState` event.* This function should be called after you are done with using the `SessionPool` instance.*/- teardown() {+ async teardown() {events.removeListener(ACTOR_EVENT_NAMES_EX.PERSIST_STATE, this._listener);+ await this.persistState();}/**diff --git a/test/session_pool/session_pool.test.js b/test/session_pool/session_pool.test.jsindex 37bc76772..b249139e8 100644--- a/test/session_pool/session_pool.test.js+++ b/test/session_pool/session_pool.test.js@@ -55,7 +55,7 @@ describe('SessionPool - testing session pool', () => {};sessionPool = new SessionPool(opts);await sessionPool.initialize();- sessionPool.teardown();+ await sessionPool.teardown();Object.entries(opts).forEach(([key, value]) => {expect(sessionPool[key]).toEqual(value);@@ -80,7 +80,7 @@ describe('SessionPool - testing session pool', () => {};sessionPool = await openSessionPool(opts);- sessionPool.teardown();+ await sessionPool.teardown();Object.entries(opts).forEach(([key, value]) => {@@ -176,7 +176,7 @@ describe('SessionPool - testing session pool', () => {expect(sessionPool.sessions).toHaveLength(loadedSessionPool.sessions.length);expect(sessionPool.maxPoolSize).toEqual(loadedSessionPool.maxPoolSize);expect(sessionPool.persistStateKey).toEqual(loadedSessionPool.persistStateKey);- sessionPool.teardown();+ await sessionPool.teardown();});test('should create only maxPoolSize number of sessions', async () => {@@ -202,7 +202,7 @@ describe('SessionPool - testing session pool', () => {});afterEach(async () => {- sessionPool.teardown();+ await sessionPool.teardown();});test('on persist event', async () => {@@ -255,7 +255,31 @@ describe('SessionPool - testing session pool', () => {await newSessionPool.initialize();expect(newSessionPool.sessions).toHaveLength(10 - invalidSessionsCount);- newSessionPool.teardown();+ await newSessionPool.teardown();+ });++ test('should persist state on teardown', async () => {+ const persistStateKey = 'TEST-KEY';+ const persistStateKeyValueStoreId = 'TEST-VALUE-STORE';++ const newSessionPool = await Apify.openSessionPool({+ maxPoolSize: 1,+ persistStateKeyValueStoreId,+ persistStateKey,+ });++ await newSessionPool.teardown();++ const kvStore = await Apify.openKeyValueStore(newSessionPool.persistStateKeyValueStoreId);+ const state = await kvStore.getValue(newSessionPool.persistStateKey);++ expect(newSessionPool.persistStateKeyValueStoreId).toBeDefined();+ expect(newSessionPool.persistStateKey).toBeDefined();+ expect(state).toBeDefined();+ expect(state).toBeInstanceOf(Object);+ expect(state).toHaveProperty('usableSessionsCount');+ expect(state).toHaveProperty('retiredSessionsCount');+ expect(state).toHaveProperty('sessions');});test('should createSessionFunction work', async () => {@@ -271,9 +295,9 @@ describe('SessionPool - testing session pool', () => {expect(session.constructor.name).toBe("Session") // eslint-disable-line});- it('should remove persist state event listener', () => {+ it('should remove persist state event listener', async () => {expect(events.listenerCount(ACTOR_EVENT_NAMES_EX.PERSIST_STATE)).toEqual(1);- sessionPool.teardown();+ await sessionPool.teardown();expect(events.listenerCount(ACTOR_EVENT_NAMES_EX.PERSIST_STATE)).toEqual(0);});});diff --git a/types/session_pool/session_pool.d.ts b/types/session_pool/session_pool.d.tsindex 353c12ca8..555a00ea5 100644--- a/types/session_pool/session_pool.d.ts+++ b/types/session_pool/session_pool.d.ts@@ -122,7 +122,7 @@ export class SessionPool extends EventEmitter {* Removes listener from `persistState` event.* This function should be called after you are done with using the `SessionPool` instance.*/- teardown(): void;+ teardown(): Promise<void>;/*** Removes `Session` instance from `SessionPool`.* @param {Session} session - Session to be removed