Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

added comments and session tests.

Petr Patek9ff861e78280
diff --git a/src/session_pool/session.js b/src/session_pool/session.js
index 35902fcb3..f56631122 100644
--- a/src/session_pool/session.js
+++ b/src/session_pool/session.js
@@ -1,13 +1,19 @@
-import { checkParamPrototypeOrThrow, cryptoRandomObjectId } from 'apify-shared/utilities';
+import EventEmitter from 'events';
+
+import { cryptoRandomObjectId } from 'apify-shared/utilities';
import moment from 'moment';
+import { checkParamOrThrow } from 'apify-client/build/utils';
import EVENTS from './events';
-// TODO: Validation
+/**
+ * Class aggregating data for `Session`.
+ * Session internal state can be enriched with custom user data for example some authorization tokens.
+ */
export default class Session {
- constructor(
- {
+ constructor(options = {}) {
+ const {
id = cryptoRandomObjectId(),
cookies = [],
fingerprintSeed = cryptoRandomObjectId(),
@@ -15,15 +21,39 @@ export default class Session {
userData = {},
maxErrorScore = 3,
errorScoreDecrement = 0.5,
- expiresAt = moment().add(3000, 'seconds').toISOString(),
createdAt = moment().toISOString(),
usageCount = 0,
errorScore = 0,
maxSessionUsageCount = 50,
sessionPool,
+ } = options;
+
+ let { expiresAt } = options;
+
+ // Validation
+ checkParamOrThrow(id, 'options.id', 'Maybe String');
+ checkParamOrThrow(cookies, 'options.cookies', 'Maybe Array');
+ checkParamOrThrow(fingerprintSeed, 'options.fingerprintSeed', 'Maybe String');
+ checkParamOrThrow(maxAgeSecs, 'options.maxAgeSecs', 'Maybe Number');
+ checkParamOrThrow(userData, 'options.userData', 'Maybe Object');
+ checkParamOrThrow(maxErrorScore, 'options.maxErrorScore', 'Maybe Number');
+ checkParamOrThrow(expiresAt, 'options.expiresAt', 'Maybe String');
+ checkParamOrThrow(createdAt, 'options.createdAt', 'Maybe String');
+ checkParamOrThrow(usageCount, 'options.usageCount', 'Maybe Number');
+ checkParamOrThrow(errorScore, 'options.errorScore', 'Maybe Number');
+ checkParamOrThrow(maxSessionUsageCount, 'options.maxSessionUsageCount', 'Maybe Number');
+ checkParamOrThrow(sessionPool, 'options.sessionPool', 'Object');
+
+ // sessionPool must be at least instance of EvenEmitter.
+ // That way we can allow custom implementation of SessionPool in the future (It should not be needed).
+ if (!(sessionPool instanceof EventEmitter)) {
+ throw new Error('Session: sessionPool must be instance of SessionPool');
+ }
+
+ if (!expiresAt) {
+ expiresAt = moment().add(maxAgeSecs, 'seconds').toISOString();
+ }
- },
- ) {
// Configurable
this.id = id;
this.cookies = cookies;
@@ -42,31 +72,68 @@ export default class Session {
this.sessionPool = sessionPool;
}
+ /**
+ * Decides whether the `Session` is blocked.
+ * `Session` is blocked once it reached the maximum error count.
+ * @return {boolean}
+ */
isBlocked() {
return this.errorScore >= this.maxErrorScore;
}
+ /**
+ * Decides whether the `Session` is expired.
+ * @return {boolean}
+ */
isExpired() {
const now = moment();
- return moment(this.expiresAt).isSameOrAfter(now);
- }
-
- isUsable() {
- return !(this.isBlocked() && this.isExpired() && this.isMaxUseCountReached());
+ return moment(this.expiresAt).isSameOrBefore(now);
}
+ /**
+ * Decides whether the `Session` is used maximum number of times.
+ * @return {boolean}
+ */
isMaxUseCountReached() {
return this.usageCount >= this.maxSessionUsageCount;
}
+ /**
+ * Decides whether the `Session` can be used for next requests.
+ * @return {boolean}
+ */
+ isUsable() {
+ return !this.isBlocked() && !this.isExpired() && !this.isMaxUseCountReached();
+ }
+
+ /**
+ * Marks the `Session` after successful request.
+ * Increases usage count and if the `Session` had failed in the previous requests it lowers the errorScore to 'heel' itself.
+ */
reclaim() {
this.usageCount += 1;
- // I should probably lower the errorScore
- this.errorScore -= this.errorScoreDecrement;
+ // We should probably lower the errorScore.
+ if (this.errorScore > 0) {
+ this.errorScore -= this.errorScoreDecrement;
+ }
}
+ /**
+ * Gets `Session` state for persistence in KeyValueStore.
+ * @return {{
+ * createdAt: string | *,
+ * userData: {},
+ * errorScoreDecrement: number,
+ * maxErrorScore: number,
+ * id: String,
+ * cookies: ([]|Array),
+ * expiresAt: string | *,
+ * usageCount: number,
+ * errorScore: number
+ * }}
+ */
getState() {
return {
id: this.id,
@@ -82,7 +149,7 @@ export default class Session {
}
/**
- * Marks session as blocked
+ * Marks session as blocked and emits event on the `SessionPool`
*/
retire() {
// mark it as an invalid by increasing the error score count.
@@ -94,7 +161,7 @@ export default class Session {
}
/**
- * Increases and usage error count
+ * Increases usage and error count
* Should be used unsuccessful request/use with the session
*/
fail() {
diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.js
index a8905ff59..47cc7c60c 100644
--- a/src/session_pool/session_pool.js
+++ b/src/session_pool/session_pool.js
@@ -1,4 +1,7 @@
import EventEmitter from 'events';
+import log from 'apify-shared/log';
+import { checkParamOrThrow } from 'apify-client/build/utils';
+
import { openKeyValueStore } from '../key_value_store';
import Session from './session';
import events from '../events';
@@ -6,35 +9,47 @@ import { ACTOR_EVENT_NAMES_EX } from '../constants';
/**
- * Handles the session rotation, creation and persistence.
+ * Handles the sessions rotation, creation and persistence.
+ * Creates a pool of `Session` instances, that are randomly rotated.
+ * When some `Session` is marked as blocked. It is removed and new one is created instead.
*
*/
// TODO: We should probably add some debug loging
// TODO: Validation
+// TODO: Class docs will be filled once the integration is done.
export default class SessionPool extends EventEmitter {
/**
*
- * @param [options]
- * @param options.initialPoolSize {Number}
- * @param options.maxPoolSize {Number}
+ * @param options
+ * @param options.maxPoolSize {Number} - Maximum size of the pool
* @param options.maxSessionAgeSecs {Number}
* @param options.maxSessionReuseCount {Number}
* @param options.persistStateKeyValueStoreId {String}
*/
- constructor({
- maxPoolSize = 1000,
- maxSessionAgeSecs = 3000,
- maxSessionReuseCount = 50,
+ constructor(options = {}) {
+ const {
+ maxPoolSize = 1000,
+ maxSessionAgeSecs = 3000,
+ maxSessionReuseCount = 50,
+
+ persistStateKeyValueStoreId = null,
+ persistStateKey = 'SESSION_POOL_STATE',
- persistStateKeyValueStoreId = null,
- persistStateKey = 'SESSION_POOL_STATE',
+ createSessionFunction = null,
- createSessionFunction = null,
+ } = options;
- }) {
super();
+ // Validation
+ checkParamOrThrow(maxPoolSize, 'options.maxPoolSize', 'Maybe Number');
+ checkParamOrThrow(maxSessionAgeSecs, 'options.maxSessionAgeSecs', 'Maybe Number');
+ checkParamOrThrow(maxSessionReuseCount, 'options.maxSessionReuseCount', 'Maybe Number');
+ checkParamOrThrow(persistStateKeyValueStoreId, 'options.persistStateKeyValueStoreId', 'Maybe String');
+ checkParamOrThrow(persistStateKey, 'options.persistStateKey', 'Maybe String');
+ checkParamOrThrow(createSessionFunction, 'options.createSessionFunction', 'Maybe Function');
+
// Pool Configuration
this.maxPoolSize = maxPoolSize;
this.createSessionFunction = createSessionFunction || this._defaultCreateSessionFunction;
@@ -44,7 +59,7 @@ export default class SessionPool extends EventEmitter {
this.maxSessionAgeSecs = maxSessionAgeSecs;
this.maxSessionReuseCount = maxSessionReuseCount;
- // Session storage
+ // Session keyValueStore
this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
this.persistStateKey = persistStateKey;
@@ -53,31 +68,35 @@ export default class SessionPool extends EventEmitter {
this.blockedSessions = 0;
// Operative states
- this.storage = null;
+ this.keyValueStore = null;
this.sessions = [];
// Maybe we can add onSessionRetired function to configuration ?
}
-
+ /**
+ * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.`
+ * This function must be called before you can start using the instance in a meaningful way.
+ *
+ * @return {Promise<void>}
+ */
async initialize() {
- this.storage = await openKeyValueStore(this.persistStateKeyValueStoreId);
+ this.keyValueStore = await openKeyValueStore(this.persistStateKeyValueStoreId);
- // Load sessions from storage
- const loadedSessions = this.storage.getValue(this.persistStateKey) || {};
-
- // Invalidate old sessions and load active sessions only
- for (const [sessionName, sessionObject] of Object.entries(loadedSessions)) {
- const session = new Session(sessionObject);
-
- if (session.isUsable()) {
- this.sessions[sessionName] = session;
- }
- }
+ // in case of migration happened and SessionPool state should be restored from the keyValueStore.
+ this._maybeRecreateSessionPool();
events.on(ACTOR_EVENT_NAMES_EX.PERSIST_STATE, this.persistState.bind(this));
}
+ /**
+ * Gets `Session`.
+ * If there is space for new `Session`, it creates and return new `Session`.
+ * If the `SessionPool` is full, it picks a `Session` from the pool,
+ * If the picked `Session` is usable it is returned, otherwise it creates and returns a new `Session`.
+ *
+ * @return {Promise<Session>}
+ */
async retrieveSession() {
// If we have enough space for session. Return newly created session.
if (this._hasSpaceForSession()) {
@@ -95,9 +114,15 @@ export default class SessionPool extends EventEmitter {
// otherwise remove old session and return newly created session
this._removeSession(pickedSession);
- this._createSession();
+ return this._createSession();
}
+ /**
+ * Gets SessionPool statistics.
+ * These function could be use in the statistics logging in actor run to know how much is the website blocking
+ *
+ * @return {{blockedSessions: number, activeSessions: number}}
+ */
getStats() {
return {
activeSessions: this.activeSessions,
@@ -105,6 +130,12 @@ export default class SessionPool extends EventEmitter {
};
}
+ /**
+ * Returns an object representing the internal state of the `SessionPool` instance.
+ * Note that the object's fields can change in future releases.
+ *
+ * @returns {Object}
+ */
getState() {
return {
...this.getStats(),
@@ -112,42 +143,118 @@ export default class SessionPool extends EventEmitter {
};
}
+ /**
+ * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
+ * The state is persisted automatically in regular intervals, but calling this method manually
+ *
+ * @return {Promise}
+ */
async persistState() {
- return this.storage.setValue(this.persistStateKey, this.getState());
+ log.debug('SessionPool: Persisting state');
+ log.debug(`SessionPool: persistStateKeyValueStoreId: ${this.persistStateKeyValueStoreId}, persistStateKey: ${this.persistStateKey} `);
+ return this.keyValueStore.setValue(this.persistStateKey, this.getState());
}
+ /**
+ * Removes `Session` instance from `SessionPool`.
+ * @param session {Session} - Session to be removed
+ * @private
+ */
_removeSession(session) {
const sessionIndex = this.sessions.findIndex(storedSession => storedSession.id === session.id);
- this.sessions.splice(sessionIndex, 1);
+ const removedSession = this.sessions.splice(sessionIndex, 1);
+ log.debug(`SessionPool: Removed Session - ${removedSession.id}`);
}
+ /**
+ * Adds `Session` instance to `SessionPool`.
+ * @param newSession {Session} - `Session` instance to be added.
+ * @private
+ */
_addSession(newSession) {
this.sessions.push(newSession);
}
+ /**
+ * Gets random index.
+ * @return {number}
+ * @private
+ */
_getRandomIndex() {
return Math.floor(Math.random() * this.sessions.length);
}
- _defaultCreateSessionFunction(sessionPool) {
+ /**
+ * Creates new `Session` without any extra behavior.
+ * @return {Session} - New session.
+ * @private
+ */
+ _defaultCreateSessionFunction() {
return new Session({
maxSessionAgeSecs: this.maxSessionAgeSecs,
maxSessionReuseCount: this.maxSessionReuseCount,
- sessionPool,
+ sessionPool: this,
});
}
+ /**
+ * Creates new `Session` and adds it to the pool.
+ * @return {Promise<Session>} - Newly created `Session` instance.
+ * @private
+ */
async _createSession() {
const newSession = await this.createSessionFunction();
this._addSession(newSession);
+
+ log.debug(`SessionPool: Created new Session - ${newSession.id}`);
+
+ return newSession;
}
+ /**
+ * Decides whether there is enough space for creating new `Session`.
+ * @return {boolean}
+ * @private
+ */
_hasSpaceForSession() {
return this.sessions.length < this.maxPoolSize;
}
+ /**
+ * Picks random `Session` from the `SessionPool`.
+ * @return {Session} - Picked `Session`
+ * @private
+ */
_pickSession() {
return this.sessions[this._getRandomIndex()]; // Or maybe we should let the developer to customize the picking algorithm
}
+
+ /**
+ * Potentially recreates `SessionPool`.
+ * If the state was persisted it recreates the `SessionPool` from the persisted state.
+ * @return {Promise<void>}
+ * @private
+ */
+ async _maybeRecreateSessionPool() {
+ const loadedSessionPool = await this.keyValueStore.getValue(this.persistStateKey);
+
+ if (!loadedSessionPool) return;
+ // Invalidate old sessions and load active sessions only
+ log.debug('SessionPool: Recreating state from KeyValueStore');
+ log.debug(`SessionPool: persistStateKeyValueStoreId: ${this.persistStateKeyValueStoreId}, persistStateKey: ${this.persistStateKey} `);
+
+ for (const sessionObject of loadedSessionPool.sessions) {
+ sessionObject.sessionPool = this;
+ const recreatedSession = new Session(sessionObject);
+
+ if (recreatedSession.isUsable()) {
+ this._addSession(recreatedSession);
+ }
+ }
+
+ log.debug(`SessionPool: Loaded ${this.sessions.length} Sessions from KeyValueStore`);
+ this.activeSessions = loadedSessionPool.activeSessions;
+ this.blockedSessions = loadedSessionPool.blockedSessions;
+ }
}
diff --git a/test/session_pool/session.js b/test/session_pool/session.js
new file mode 100644
index 000000000..4c5e24737
--- /dev/null
+++ b/test/session_pool/session.js
@@ -0,0 +1,73 @@
+import { expect } from 'chai';
+
+import Session from '../../build/session_pool/session';
+import SessionPool from '../../build/session_pool/session_pool';
+
+import Apify from '../../build';
+
+
+describe('Session - testing session behaviour ', async () => {
+ let sessionPool;
+ let session;
+
+ beforeEach(() => {
+ sessionPool = new SessionPool();
+ session = new Session({ sessionPool });
+ });
+
+ it('should reclaim session and lower the errorScore', () => {
+ expect(session.usageCount).to.be.eql(0);
+ expect(session.errorScore).to.be.eql(0);
+ session.reclaim();
+ expect(session.usageCount).to.be.eql(1);
+ expect(session.errorScore).to.be.eql(0);
+ session.errorScore = 1;
+ session.reclaim();
+ expect(session.errorScore).to.be.eql(0.5);
+ });
+
+ it('should mark session failed', () => {
+ session.fail();
+ expect(session.errorScore).to.be.eql(1);
+ expect(session.usageCount).to.be.eql(1);
+ });
+
+ it('should expire session', async () => {
+ session = new Session({ maxAgeSecs: 1 / 100, sessionPool });
+ await Apify.utils.sleep(101);
+ expect(session.isExpired()).to.be.eql(true);
+ expect(session.isUsable()).to.be.eql(false);
+ });
+
+ it('should max out session usage', () => {
+ session.maxSessionUsageCount = 1;
+ session.reclaim();
+ expect(session.isMaxUseCountReached()).to.be.eql(true);
+ expect(session.isUsable()).to.be.eql(false);
+ });
+
+ it('should block session', () => {
+ session.errorScore += session.maxErrorScore;
+ expect(session.isBlocked()).to.be.eql(true);
+ expect(session.isUsable()).to.be.eql(false);
+ });
+
+ it('should get state', () => {
+ const state = session.getState();
+
+ expect(state.id).to.exist; // eslint-disable-line
+ expect(state.cookies).to.exist; // eslint-disable-line
+ expect(state.userData).to.exist; // eslint-disable-line
+ expect(state.maxErrorScore).to.exist; // eslint-disable-line
+ expect(state.errorScoreDecrement).to.exist; // eslint-disable-line
+ expect(state.expiresAt).to.exist; // eslint-disable-line
+ expect(state.createdAt).to.exist; // eslint-disable-line
+ expect(state.usageCount).to.exist; // eslint-disable-line
+ expect(state.errorScore).to.exist; // eslint-disable-line
+
+
+ Object.entries(state).forEach(([key, value]) => {
+ expect(session[key]).to.be.eql(value);
+ });
+ });
+});
diff --git a/test/session_pool/session_pool.js b/test/session_pool/session_pool.js
index 9a06be8a4..71f0ce3d1 100644
--- a/test/session_pool/session_pool.js
+++ b/test/session_pool/session_pool.js
@@ -1,27 +1,32 @@
import { expect } from 'chai';
+import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir } from '../_helper';
+import SessionPool from '../../build/session_pool/session_pool';
+import Apify from '../../build';
-import SessionPool, { SESSION_POOL_DEFAULTS } from '../../src/session_pool/session_pool';
-
+// TODO: Add more tests
describe('SessionPool - testing session pool', async () => {
let sessionPool;
before(() => {
- process.env.APIFY_LOCAL_STORAGE_DIR = '../../';
+ process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;
});
beforeEach(async () => {
- sessionPool = new SessionPool({ persistStateKeyValueStoreId: 'default' });
+ sessionPool = new SessionPool();
await sessionPool.initialize();
});
+ afterEach(async () => {
+ await emptyLocalStorageSubdir('key_value_stores/default');
+ });
+
it('should initialize with default values for first time', async () => {
- expect(sessionPool.sessions.length).to.be.eql(0);
- expect(sessionPool.maxPoolSize).to.be.eql(SESSION_POOL_DEFAULTS.maxPoolSize);
- expect(sessionPool.maxSessionAgeSecs).to.be.eql(SESSION_POOL_DEFAULTS.maxSessionAgeSecs);
- expect(sessionPool.maxSessionReuseCount).to.be.eql(SESSION_POOL_DEFAULTS.maxSessionReuseCount);
- expect(sessionPool.persistStateKeyValueStoreId).to.be.eql(SESSION_POOL_DEFAULTS.persistStateKeyValueStoreId);
- expect(sessionPool.persistStateKey).to.be.eql(SESSION_POOL_DEFAULTS.persistStateKey);
- expect(sessionPool.createSessionFunction).to.be.eql(sessionPool._createSessionFunction); // eslint-disable-line
+ expect(sessionPool.sessions.length).to.exist; // eslint-disable-line
+ expect(sessionPool.maxPoolSize).to.exist; // eslint-disable-line
+ expect(sessionPool.maxSessionAgeSecs).to.exist; // eslint-disable-line
+ expect(sessionPool.maxSessionReuseCount).to.exist; // eslint-disable-line
+ expect(sessionPool.persistStateKey).to.exist; // eslint-disable-line
+ expect(sessionPool.createSessionFunction).to.be.eql(sessionPool._defaultCreateSessionFunction); // eslint-disable-line
});
it('should override default values', async () => {
@@ -47,31 +52,38 @@ describe('SessionPool - testing session pool', async () => {
it('should retrieve session', async () => {
const session = await sessionPool.retrieveSession();
expect(sessionPool.sessions.length).to.be.eql(1);
- expect(session.name).to.exist;
+ expect(session.id).to.exist; // eslint-disable-line
expect(session.cookies.length).to.be.eql(0);
- expect(session.fingerPrintSeed).to.exist;
+ expect(session.fingerprintSeed).to.exist; // eslint-disable-line
expect(session.maxAgeSecs).to.eql(sessionPool.maxSessionAgeSecs);
expect(session.maxAgeSecs).to.eql(sessionPool.maxSessionAgeSecs);
expect(session.sessionPool).to.eql(sessionPool);
});
- xit('should persist state and recreate it from storage', async () => {
- const session = await sessionPool.retrieveSession();
- let key;
- let value;
-
- const mySetValue = (k, v) => {
- key = k;
- value = v;
- };
- sessionPool.storage.setValue = mySetValue;
+ it('should persist state and recreate it from storage', async () => {
+ await sessionPool.retrieveSession();
await sessionPool.persistState();
- expect(key).to.be.eql(SESSION_POOL_DEFAULTS.persistStateKey);
- expect(value).to.be.eql(sessionPool.getState());
- const loadedSessionPool = new SessionPool({ persistStateKeyValueStoreId: 'default' });
+ const kvStore = await Apify.openKeyValueStore();
+ const sessionPoolSaved = await kvStore.getValue(sessionPool.persistStateKey);
+
+ Object.entries(sessionPoolSaved).forEach(([key, value]) => {
+ if (key !== 'sessions') {
+ expect(value).to.be.eql(sessionPool[key]);
+ }
+ });
+ expect(sessionPoolSaved.sessions.length).to.be.eql(sessionPool.sessions.length);
+ sessionPoolSaved.sessions.forEach((session, index) => {
+ Object.entries(session).forEach(([key, value]) => {
+ expect(value).to.be.eql(sessionPool.sessions[index][key]);
+ });
+ });
+
+
+ const loadedSessionPool = new SessionPool();
+
await loadedSessionPool.initialize();
- console.log('SESSION');
+ expect(loadedSessionPool).to.have.all.keys(Object.keys(sessionPool));
});
it('should create only maxPoolSize number of sessions', async () => {