Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

added tests

Petr Patek65c7e4cc8bcd
diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.js
index 47cc7c60c..b8b9c93ba 100644
--- a/src/session_pool/session_pool.js
+++ b/src/session_pool/session_pool.js
@@ -63,10 +63,6 @@ export default class SessionPool extends EventEmitter {
this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
this.persistStateKey = persistStateKey;
- // Statistics
- this.activeSessions = 0;
- this.blockedSessions = 0;
-
// Operative states
this.keyValueStore = null;
this.sessions = [];
@@ -74,6 +70,22 @@ export default class SessionPool extends EventEmitter {
// Maybe we can add onSessionRetired function to configuration ?
}
+ /**
+ * Gets number of active sessions in the pool.
+ * @return {number}
+ */
+ get activeSessionsCount() {
+ return this.sessions.filter(session => session.isUsable()).length;
+ }
+
+ /**
+ * Gets number of blocked sessions in the pool.
+ * @return {number}
+ */
+ get blockedSessionsCount() {
+ return this.sessions.filter(session => !session.isUsable()).length;
+ }
+
/**
* Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.`
* This function must be called before you can start using the instance in a meaningful way.
@@ -84,7 +96,7 @@ export default class SessionPool extends EventEmitter {
this.keyValueStore = await openKeyValueStore(this.persistStateKeyValueStoreId);
// in case of migration happened and SessionPool state should be restored from the keyValueStore.
- this._maybeRecreateSessionPool();
+ await this._maybeRecreateSessionPool();
events.on(ACTOR_EVENT_NAMES_EX.PERSIST_STATE, this.persistState.bind(this));
}
@@ -121,12 +133,12 @@ export default class SessionPool extends EventEmitter {
* Gets SessionPool statistics.
* These function could be use in the statistics logging in actor run to know how much is the website blocking
*
- * @return {{blockedSessions: number, activeSessions: number}}
+ * @return {{blockedSessionsCount: number, activeSessionsCount: number}}
*/
getStats() {
return {
- activeSessions: this.activeSessions,
- blockedSessions: this.blockedSessions,
+ activeSessionsCount: this.activeSessionsCount,
+ blockedSessionsCount: this.blockedSessionsCount,
};
}
@@ -254,7 +266,6 @@ export default class SessionPool extends EventEmitter {
}
log.debug(`SessionPool: Loaded ${this.sessions.length} Sessions from KeyValueStore`);
- this.activeSessions = loadedSessionPool.activeSessions;
- this.blockedSessions = loadedSessionPool.blockedSessions;
+ log.debug(`SessionPool: Active sessions ${this.activeSessionsCount} Sessions from KeyValueStore`);
}
}
diff --git a/test/session_pool/session.js b/test/session_pool/session.js
index 4c5e24737..fe7fff77f 100644
--- a/test/session_pool/session.js
+++ b/test/session_pool/session.js
@@ -2,6 +2,7 @@ import { expect } from 'chai';
import Session from '../../build/session_pool/session';
import SessionPool from '../../build/session_pool/session_pool';
+import EVENTS from '../../build/session_pool/events';
import Apify from '../../build';
@@ -26,6 +27,17 @@ describe('Session - testing session behaviour ', async () => {
expect(session.errorScore).to.be.eql(0.5);
});
+ it('should throw error when param sessionPool is not EventEmitter instance', () => {
+ let err;
+ try {
+ const session = new Session({ sessionPool: {} }); // eslint-disable-line
+ } catch (e) {
+ err = e;
+ }
+ expect(err).to.exist; // eslint-disable-line
+ expect(err.message.includes('Session: sessionPool must be instance of SessionPool')).to.be.true; // eslint-disable-line
+ });
+
it('should mark session failed', () => {
session.fail();
expect(session.errorScore).to.be.eql(1);
@@ -52,6 +64,23 @@ describe('Session - testing session behaviour ', async () => {
expect(session.isUsable()).to.be.eql(false);
});
+ it('should reclaim session', () => {
+ session.reclaim();
+ expect(session.usageCount).to.be.eql(1);
+ expect(session.isUsable()).to.be.eql(true);
+ });
+
+ it('should retire session', () => {
+ let discarded = false;
+ sessionPool.on(EVENTS.DISCARDED, (ses) => {
+ expect(ses instanceof Session).to.be.eql(true);
+ discarded = true;
+ });
+ session.retire();
+ expect(discarded).to.be.true; // eslint-disable-line
+ expect(session.usageCount).to.be.eql(1);
+ });
+
it('should get state', () => {
const state = session.getState();
diff --git a/test/session_pool/session_pool.js b/test/session_pool/session_pool.js
index 71f0ce3d1..eab7946c7 100644
--- a/test/session_pool/session_pool.js
+++ b/test/session_pool/session_pool.js
@@ -1,9 +1,12 @@
import { expect } from 'chai';
+import log from 'apify-shared/log';
import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir } from '../_helper';
import SessionPool from '../../build/session_pool/session_pool';
import Apify from '../../build';
+import events from '../../build/events';
+
+import { ACTOR_EVENT_NAMES_EX } from '../../build/constants';
-// TODO: Add more tests
describe('SessionPool - testing session pool', async () => {
let sessionPool;
@@ -49,17 +52,58 @@ describe('SessionPool - testing session pool', async () => {
});
});
- it('should retrieve session', async () => {
- const session = await sessionPool.retrieveSession();
- expect(sessionPool.sessions.length).to.be.eql(1);
- expect(session.id).to.exist; // eslint-disable-line
- expect(session.cookies.length).to.be.eql(0);
- expect(session.fingerprintSeed).to.exist; // eslint-disable-line
- expect(session.maxAgeSecs).to.eql(sessionPool.maxSessionAgeSecs);
- expect(session.maxAgeSecs).to.eql(sessionPool.maxSessionAgeSecs);
- expect(session.sessionPool).to.eql(sessionPool);
+ describe('should retrieve session', () => {
+ it('should retrieve session with correct shape', async () => {
+ const session = await sessionPool.retrieveSession();
+ expect(sessionPool.sessions.length).to.be.eql(1);
+ expect(session.id).to.exist; // eslint-disable-line
+ expect(session.cookies.length).to.be.eql(0);
+ expect(session.fingerprintSeed).to.exist; // eslint-disable-line
+ expect(session.maxAgeSecs).to.eql(sessionPool.maxSessionAgeSecs);
+ expect(session.maxAgeSecs).to.eql(sessionPool.maxSessionAgeSecs);
+ expect(session.sessionPool).to.eql(sessionPool);
+ });
+
+ it('should pick session when pool is full', async () => {
+ sessionPool.maxPoolSize = 2;
+ await sessionPool.retrieveSession();
+ await sessionPool.retrieveSession();
+ let isCalled = false;
+ const oldPick = sessionPool._pickSession; //eslint-disable-line
+
+ sessionPool._pickSession = () => { //eslint-disable-line
+ isCalled = true;
+ return oldPick.bind(sessionPool)();
+ };
+
+ await sessionPool.retrieveSession();
+
+ expect(isCalled).to.be.true; //eslint-disable-line
+ });
+
+ it('should delete picked session when it is usable a create a new one', async () => {
+ sessionPool.maxPoolSize = 1;
+ await sessionPool.retrieveSession();
+ const session = sessionPool.sessions[0];
+
+ session.errorScore += session.maxErrorScore;
+ let isCalled = false;
+ const oldRemove = sessionPool._removeSession; //eslint-disable-line
+
+ sessionPool._removeSession = (session) => { //eslint-disable-line
+ isCalled = true;
+ return oldRemove.bind(sessionPool)(session);
+ };
+
+ await sessionPool.retrieveSession();
+
+ expect(isCalled).to.be.true; //eslint-disable-line
+ expect(sessionPool.sessions[0].id === session.id).to.be.false; //eslint-disable-line
+ expect(sessionPool.sessions).to.be.length(1);
+ });
});
+
it('should persist state and recreate it from storage', async () => {
await sessionPool.retrieveSession();
await sessionPool.persistState();
@@ -93,4 +137,74 @@ describe('SessionPool - testing session pool', async () => {
}
expect(sessionPool.sessions.length).to.be.eql(sessionPool.maxPoolSize);
});
+
+ it('should create session', async () => {
+ await sessionPool._createSession(); // eslint-disable-line
+ expect(sessionPool.sessions.length).to.be.eql(1);
+ expect(sessionPool.sessions[0].id).to.exist; // eslint-disable-line
+ });
+
+ describe('should persist state', () => {
+ const KV_STORE = 'SESSION-TEST';
+
+ beforeEach(async () => {
+ sessionPool = new SessionPool({ persistStateKeyValueStoreId: KV_STORE });
+ await sessionPool.initialize();
+ });
+
+ afterEach(async () => {
+ await emptyLocalStorageSubdir(`key_value_stores/${KV_STORE}`);
+ });
+
+ it('on persist event', async () => {
+ await sessionPool.retrieveSession();
+
+ expect(sessionPool.sessions.length).to.be.eql(1);
+
+ events.emit(ACTOR_EVENT_NAMES_EX.PERSIST_STATE);
+
+ await new Promise((resolve) => {
+ const interval = setInterval(async () => {
+ const state = await sessionPool.keyValueStore.getValue(sessionPool.persistStateKey);
+ if (state) {
+ resolve();
+ clearInterval(interval);
+ }
+ }, 100);
+ });
+
+ const state = await sessionPool.keyValueStore.getValue(sessionPool.persistStateKey);
+
+ expect(sessionPool.getState()).to.be.eql(state);
+ });
+ });
+
+ it('should remove session', async () => {
+ for (let i = 0; i < 10; i++) {
+ await sessionPool.retrieveSession();
+ }
+ const picked = sessionPool.retrieveSession();
+ sessionPool._removeSession(picked); // eslint-disable-line
+ expect(sessionPool.sessions.find(s => s.id === picked.id)).to.be.eql(undefined);
+ });
+
+ it('should recreate only usable sessions', async () => {
+ let invalidSessionsCount = 0;
+ for (let i = 0; i < 10; i++) {
+ const session = await sessionPool.retrieveSession();
+
+ if (i % 2 === 0) {
+ session.errorScore += session.maxErrorScore;
+ invalidSessionsCount += 1;
+ }
+ }
+ expect(sessionPool.blockedSessionsCount).to.be.eql(invalidSessionsCount);
+
+ await sessionPool.persistState();
+
+ const newSessionPool = new SessionPool();
+ await newSessionPool.initialize();
+
+ expect(newSessionPool.sessions).to.be.length(10 - invalidSessionsCount);
+ });
});