Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Added SessionPool teardown. Fixed tests.

Petr Patek813bea9b7e14
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.js
index 57b4e694b..0643a81de 100644
--- a/src/crawlers/basic_crawler.js
+++ b/src/crawlers/basic_crawler.js
@@ -286,6 +286,10 @@ class BasicCrawler {
try {
await this.isRunningPromise;
} finally {
+ if (this.useSessionPool) {
+ this.sessionPool.teardown();
+ }
+
this.stats.stopLogging();
const finalStats = this.stats.getCurrent();
log.info('Crawler final request statistics:', finalStats);
diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.js
index f137f2425..3bfe1028a 100644
--- a/src/session_pool/session_pool.js
+++ b/src/session_pool/session_pool.js
@@ -96,7 +96,6 @@ export class SessionPool extends EventEmitter {
this.createSessionFunction = createSessionFunction || this._defaultCreateSessionFunction;
// Session configuration
- // @TODO: Maybe options.sessionOptions / this.sessionOptions?
this.maxSessionAgeSecs = maxSessionAgeSecs;
this.maxSessionUsageCount = maxSessionUsageCount;
@@ -137,7 +136,9 @@ export class SessionPool extends EventEmitter {
// in case of migration happened and SessionPool state should be restored from the keyValueStore.
await this._maybeLoadSessionPool();
- events.on(ACTOR_EVENT_NAMES_EX.PERSIST_STATE, this.persistState.bind(this));
+ this._listener = this.persistState.bind(this);
+
+ events.on(ACTOR_EVENT_NAMES_EX.PERSIST_STATE, this._listener);
}
/**
@@ -192,6 +193,14 @@ export class SessionPool extends EventEmitter {
await this.keyValueStore.setValue(this.persistStateKey, this.getState());
}
+ /**
+ * Removes listener from `persistState` event.
+ * This function should be called after you are done with using the `SessionPool` instance.
+ */
+ teardown() {
+ events.removeListener(ACTOR_EVENT_NAMES_EX.PERSIST_STATE, this._listener);
+ }
+
/**
* Removes `Session` instance from `SessionPool`.
* @param session {Session} - Session to be removed
@@ -296,7 +305,7 @@ export class SessionPool extends EventEmitter {
}
log.debug(`SessionPool: Loaded ${this.sessions.length} Sessions from KeyValueStore`);
- log.debug(`SessionPool: Active sessions ${this.activeSessionsCount} Sessions from KeyValueStore`);
+ log.debug(`SessionPool: Active sessions ${this.usableSessionsCount} Sessions from KeyValueStore`);
}
}
diff --git a/test/crawlers/basic_crawler.js b/test/crawlers/basic_crawler.js
index 707cb5e0b..d300d4e37 100644
--- a/test/crawlers/basic_crawler.js
+++ b/test/crawlers/basic_crawler.js
@@ -8,6 +8,7 @@ import * as Apify from '../../build';
import * as keyValueStore from '../../build/key_value_store';
import { RequestQueue, RequestQueueLocal } from '../../build/request_queue';
import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir } from '../_helper';
+import events from '../../build/events';
describe('BasicCrawler', () => {
let logLevel;
@@ -15,8 +16,8 @@ describe('BasicCrawler', () => {
before(() => {
logLevel = log.getLevel();
log.setLevel(log.LEVELS.OFF);
+ process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;
emptyLocalStorageSubdir('key_value_stores/default');
-
});
after(() => {
@@ -631,7 +632,7 @@ describe('BasicCrawler', () => {
expect(results).length(0);
});
- xit('should use pass options to sessionPool', async () => {
+ it('should use pass options to sessionPool', async () => {
const url = 'https://example.com';
const requestList = new Apify.RequestList({ sources: [{ url }] });
await requestList.initialize();
@@ -653,6 +654,34 @@ describe('BasicCrawler', () => {
expect(crawler.sessionPool.maxPoolSize).to.be.eql(10);
});
+ it('should teardown Session pool after it is finished', async () => {
+ const url = 'https://example.com';
+ const requestList = new Apify.RequestList({ sources: [{ url }] });
+ await requestList.initialize();
+ events.removeAllListeners(ACTOR_EVENT_NAMES.PERSIST_STATE);
+
+ const crawler = new Apify.BasicCrawler({
+ requestList,
+ handleRequestTimeoutSecs: 0.01,
+ maxRequestRetries: 1,
+ useSessionPool: true,
+ sessionPoolOptions: {
+ maxPoolSize: 10,
+ },
+ handleRequestFunction: async () => {},
+ handleFailedRequestFunction: () => {},
+ });
+
+ crawler._loadHandledRequestCount = () => { // eslint-disable-line
+ expect(crawler.sessionPool).to.exist; // eslint-disable-line
+ expect(events.listenerCount(ACTOR_EVENT_NAMES.PERSIST_STATE)).to.be.eql(1);
+ };
+
+ await crawler.run();
+ expect(events.listenerCount(ACTOR_EVENT_NAMES.PERSIST_STATE)).to.be.eql(0);
+ expect(crawler.sessionPool.maxPoolSize).to.be.eql(10);
+ });
+
it('should not use SessionPool by default', async () => {
const url = 'https://example.com';
const requestList = new Apify.RequestList({ sources: [{ url }] });
diff --git a/test/session_pool/session_pool.js b/test/session_pool/session_pool.js
index 304bb6f84..d362fce87 100644
--- a/test/session_pool/session_pool.js
+++ b/test/session_pool/session_pool.js
@@ -15,10 +15,14 @@ describe('SessionPool - testing session pool', () => {
});
beforeEach(async () => {
- await emptyLocalStorageSubdir('key_value_stores/default');
+ emptyLocalStorageSubdir('key_value_stores/default');
sessionPool = await Apify.openSessionPool();
});
+ afterEach(() => {
+ events.removeAllListeners(ACTOR_EVENT_NAMES_EX.PERSIST_STATE);
+ });
+
it('should initialize with default values for first time', async () => {
expect(sessionPool.sessions.length).to.exist; // eslint-disable-line
expect(sessionPool.maxPoolSize).to.exist; // eslint-disable-line
@@ -42,6 +46,7 @@ describe('SessionPool - testing session pool', () => {
};
sessionPool = new SessionPool(opts);
await sessionPool.initialize();
+ sessionPool.teardown();
Object.entries(opts).forEach(([key, value]) => {
expect(sessionPool[key]).to.be.eql(value);
@@ -61,6 +66,8 @@ describe('SessionPool - testing session pool', () => {
};
sessionPool = await openSessionPool(opts);
+ sessionPool.teardown();
+
Object.entries(opts).forEach(([key, value]) => {
expect(sessionPool[key]).to.be.eql(value);
@@ -146,6 +153,7 @@ describe('SessionPool - testing session pool', () => {
await loadedSessionPool.initialize();
expect(loadedSessionPool).to.have.all.keys(Object.keys(sessionPool));
+ sessionPool.teardown();
});
it('should create only maxPoolSize number of sessions', async () => {
@@ -172,6 +180,7 @@ describe('SessionPool - testing session pool', () => {
afterEach(async () => {
await emptyLocalStorageSubdir(`key_value_stores/${KV_STORE}`);
+ sessionPool.teardown();
});
it('on persist event', async () => {
@@ -222,8 +231,8 @@ describe('SessionPool - testing session pool', () => {
const newSessionPool = new SessionPool();
await newSessionPool.initialize();
-
expect(newSessionPool.sessions).to.be.length(10 - invalidSessionsCount);
+ newSessionPool.teardown();
});
it('should createSessionFunction work', async () => {
@@ -231,7 +240,6 @@ describe('SessionPool - testing session pool', () => {
const createSessionFunction = (sessionPool2) => {
isCalled = true;
expect(sessionPool2 instanceof SessionPool).to.be.true; // eslint-disable-line
- console.log(sessionPool2 instanceof SessionPool);
return new Session({ sessionPool: sessionPool2 });
};
const newSessionPool = await Apify.openSessionPool({ createSessionFunction });
@@ -239,4 +247,10 @@ describe('SessionPool - testing session pool', () => {
expect(isCalled).to.be.true; // eslint-disable-line
expect(session.constructor.name).to.be.eql("Session") // eslint-disable-line
});
+
+ it('should remove persist state event listener', () => {
+ expect(events.listenerCount(ACTOR_EVENT_NAMES_EX.PERSIST_STATE)).to.be.eq(1);
+ sessionPool.teardown();
+ expect(events.listenerCount(ACTOR_EVENT_NAMES_EX.PERSIST_STATE)).to.be.eq(0);
+ });
});