Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Feature/session pool persist state on teardown (#615)

Jan Bártac0507c08fb8b
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.js
index bde1f92b7..ab617d04f 100644
--- a/src/crawlers/basic_crawler.js
+++ b/src/crawlers/basic_crawler.js
@@ -307,7 +307,7 @@ class BasicCrawler {
await this.isRunningPromise;
} finally {
if (this.useSessionPool) {
- this.sessionPool.teardown();
+ await this.sessionPool.teardown();
}
this.stats.stopLogging();
diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.js
index 92d361a1e..66750cc6e 100644
--- a/src/crawlers/puppeteer_crawler.js
+++ b/src/crawlers/puppeteer_crawler.js
@@ -311,9 +311,9 @@ class PuppeteerCrawler {
await this.isRunningPromise;
} finally {
if (this.useSessionPool) {
- this.sessionPool.teardown();
+ await this.sessionPool.teardown();
}
- this.puppeteerPool.destroy();
+ await this.puppeteerPool.destroy();
}
}
diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.js
index b72bb453b..23a93079c 100644
--- a/src/session_pool/session_pool.js
+++ b/src/session_pool/session_pool.js
@@ -202,8 +202,9 @@ export class SessionPool extends EventEmitter {
* Removes listener from `persistState` event.
* This function should be called after you are done with using the `SessionPool` instance.
*/
- teardown() {
+ async teardown() {
events.removeListener(ACTOR_EVENT_NAMES_EX.PERSIST_STATE, this._listener);
+ await this.persistState();
}
/**
diff --git a/test/session_pool/session_pool.test.js b/test/session_pool/session_pool.test.js
index 37bc76772..b249139e8 100644
--- a/test/session_pool/session_pool.test.js
+++ b/test/session_pool/session_pool.test.js
@@ -55,7 +55,7 @@ describe('SessionPool - testing session pool', () => {
};
sessionPool = new SessionPool(opts);
await sessionPool.initialize();
- sessionPool.teardown();
+ await sessionPool.teardown();
Object.entries(opts).forEach(([key, value]) => {
expect(sessionPool[key]).toEqual(value);
@@ -80,7 +80,7 @@ describe('SessionPool - testing session pool', () => {
};
sessionPool = await openSessionPool(opts);
- sessionPool.teardown();
+ await sessionPool.teardown();
Object.entries(opts).forEach(([key, value]) => {
@@ -176,7 +176,7 @@ describe('SessionPool - testing session pool', () => {
expect(sessionPool.sessions).toHaveLength(loadedSessionPool.sessions.length);
expect(sessionPool.maxPoolSize).toEqual(loadedSessionPool.maxPoolSize);
expect(sessionPool.persistStateKey).toEqual(loadedSessionPool.persistStateKey);
- sessionPool.teardown();
+ await sessionPool.teardown();
});
test('should create only maxPoolSize number of sessions', async () => {
@@ -202,7 +202,7 @@ describe('SessionPool - testing session pool', () => {
});
afterEach(async () => {
- sessionPool.teardown();
+ await sessionPool.teardown();
});
test('on persist event', async () => {
@@ -255,7 +255,31 @@ describe('SessionPool - testing session pool', () => {
await newSessionPool.initialize();
expect(newSessionPool.sessions).toHaveLength(10 - invalidSessionsCount);
- newSessionPool.teardown();
+ await newSessionPool.teardown();
+ });
+
+ test('should persist state on teardown', async () => {
+ const persistStateKey = 'TEST-KEY';
+ const persistStateKeyValueStoreId = 'TEST-VALUE-STORE';
+
+ const newSessionPool = await Apify.openSessionPool({
+ maxPoolSize: 1,
+ persistStateKeyValueStoreId,
+ persistStateKey,
+ });
+
+ await newSessionPool.teardown();
+
+ const kvStore = await Apify.openKeyValueStore(newSessionPool.persistStateKeyValueStoreId);
+ const state = await kvStore.getValue(newSessionPool.persistStateKey);
+
+ expect(newSessionPool.persistStateKeyValueStoreId).toBeDefined();
+ expect(newSessionPool.persistStateKey).toBeDefined();
+ expect(state).toBeDefined();
+ expect(state).toBeInstanceOf(Object);
+ expect(state).toHaveProperty('usableSessionsCount');
+ expect(state).toHaveProperty('retiredSessionsCount');
+ expect(state).toHaveProperty('sessions');
});
test('should createSessionFunction work', async () => {
@@ -271,9 +295,9 @@ describe('SessionPool - testing session pool', () => {
expect(session.constructor.name).toBe("Session") // eslint-disable-line
});
- it('should remove persist state event listener', () => {
+ it('should remove persist state event listener', async () => {
expect(events.listenerCount(ACTOR_EVENT_NAMES_EX.PERSIST_STATE)).toEqual(1);
- sessionPool.teardown();
+ await sessionPool.teardown();
expect(events.listenerCount(ACTOR_EVENT_NAMES_EX.PERSIST_STATE)).toEqual(0);
});
});
diff --git a/types/session_pool/session_pool.d.ts b/types/session_pool/session_pool.d.ts
index 353c12ca8..555a00ea5 100644
--- a/types/session_pool/session_pool.d.ts
+++ b/types/session_pool/session_pool.d.ts
@@ -122,7 +122,7 @@ export class SessionPool extends EventEmitter {
* Removes listener from `persistState` event.
* This function should be called after you are done with using the `SessionPool` instance.
*/
- teardown(): void;
+ teardown(): Promise<void>;
/**
* Removes `Session` instance from `SessionPool`.
* @param {Session} session - Session to be removed