Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Session Pool implementation proposal.
Petr Pateke8a0b3417d42
diff --git a/package.json b/package.jsonindex 7916a0ead..6985c321b 100644--- a/package.json+++ b/package.json@@ -60,6 +60,7 @@"htmlparser2": "^3.10.1","jquery": "^3.4.1","mime": "^2.3.1",+ "moment": "^2.24.0","proxy-chain": "^0.2.7","request": "^2.88.0","request-promise-native": "^1.0.5",diff --git a/src/session_pool/events.js b/src/session_pool/events.jsnew file mode 100644index 000000000..1f7f6cf47--- /dev/null+++ b/src/session_pool/events.js@@ -0,0 +1,3 @@+export default {+ DISCARD: 'DISCARD',+};diff --git a/src/session_pool/session.js b/src/session_pool/session.jsnew file mode 100644index 000000000..80165d359--- /dev/null+++ b/src/session_pool/session.js@@ -0,0 +1,98 @@+import { checkParamPrototypeOrThrow, cryptoRandomObjectId } from 'apify-shared/utilities';+import _ from 'underscore';+import moment from 'moment';++import EVENTS from './events';++export const SESSION_DEFAULT_OPTIONS = {+ name: cryptoRandomObjectId(),+ cookies: [],+ fingerPrintSeed: cryptoRandomObjectId(),+ maxAgeSecs: 3000,+ maxReuseCount: 50,+ userData: {},+ maxErrorScore: 3,+ errorScoreDecrement: 0.5,+ expiresAt: moment().add(3000, 'seconds').toISOString(),+ createdAt: moment().toISOString(),+ usedCount: 0,+ errorScore: 0,++};++// TODO: Validation+export default class Session {+ constructor(options) {+ const opts = _.defaults({}, options, SESSION_DEFAULT_OPTIONS);++ // Configurable+ this.name = opts.name;+ this.cookies = opts.cookies;+ this.fingerPrintSeed = opts.fingerPrintSeed;+ this.maxAgeSecs = opts.maxAgeSecs;+ this.userData = opts.userData;+ this.maxErrorScore = opts.maxErrorScore;+ this.errorScoreDecrement = opts.errorScoreDecrement; // TODO: Better Naming++ // Internal+ this.expiresAt = opts.expiresAt;+ this.createdAt = opts.createdAt;+ this.usedCount = opts.usedCount;+ this.errorScore = opts.errorScoreDecrement;+ this.sessionPool = opts.sessionPool;+ }++ static recreateSession(sessionObject) {+ return new Session(sessionObject);+ }++ isBlocked() {+ return this.errorScore >= this.maxErrorScore;+ }++ isExpired() {+ const now = moment();++ return moment(this.expiresAt).isSameOrAfter(now);+ }++ isUsable() {+ return !(this.isBlocked() && this.isExpired());+ }++ reclaim() {+ this.usedCount += 1;++ // I should probably lower the errorScore+ this.errorScore -= this.errorScoreDecrement;+ }++ getState() {+ return {+ name: this.name,+ cookies: this.cookies,+ userData: this.userData,+ maxErrorScore: this.maxErrorScore,+ errorScoreDecrement: this.errorScoreDecrement,+ expiresAt: this.expiresAt,+ createdAt: this.createdAt,+ usedCount: this.usedCount,+ errorScore: this.errorScore,+ };+ }+++ retire(discard = false) {+ if (discard) {+ // mark it as an invalid by increasing the error score count.+ this.errorScore += this.maxErrorScore;++ // emit event so we can retire browser in puppeteer pool+ this.sessionPool.emit(EVENTS.DISCARD, this);+ } else {+ this.errorScore += 1;+ }++ this.usedCount += 1;+ }+}diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.jsnew file mode 100644index 000000000..e1942c7b5--- /dev/null+++ b/src/session_pool/session_pool.js@@ -0,0 +1,148 @@+import EventEmitter from 'events';+import _ from 'underscore';+import { openKeyValueStore } from '../key_value_store';+import Session from './session';++export const SESSION_POOL_DEFAULTS = {+ maxPoolSize: 1000,+ maxSessionAgeSecs: 3000,+ maxSessionReuseCount: 50,++ persistStateKeyValueStoreId: null,+ persistStateKey: 'SESSION_POOL_STATE',++ createSessionFunction: null,++};++/**+ * Handles the session rotation, creation and persistence.+ *+ */++// TODO: We should probably add some debug loging+// TODO: Validation+export default class SessionPool extends EventEmitter {+ /**+ *+ * @param [options]+ * @param options.initialPoolSize {Number}+ * @param options.maxPoolSize {Number}+ * @param options.maxSessionAgeSecs {Number}+ * @param options.maxSessionReuseCount {Number}+ * @param options.persistStateKeyValueStoreId {String}+ */+ constructor(options) {+ super();++ const opts = _.defaults({}, options, SESSION_POOL_DEFAULTS);++ // Pool Configuration+ this.maxPoolSize = opts.maxPoolSize;+ this.createSessionFunction = opts.createSessionFunction || this._createSessionFunction;++ // Session configuration+ // @TODO: Maybe options.sessionOptions / this.sessionOptions?+ this.maxSessionAgeSecs = opts.maxSessionAgeSecs;+ this.maxSessionReuseCount = opts.maxSessionReuseCount;++ // Session storage+ this.persistStateKeyValueStoreId = opts.persistStateKeyValueStoreId;+ this.persistStateKey = opts.persistStateKey;++ // Statistics+ this.activeSessions = 0;+ this.blockedSessions = 0;++ // Operative states+ this.storage = null;+ this.sessions = [];++ // Maybe we can add onSessionRetired function to configuration ?+ }+++ async initialize() {+ this.storage = await openKeyValueStore(this.persistStateKeyValueStoreId);++ // Load sessions from storage+ const loadedSessions = this.storage.getValue(this.persistStateKey) || [];++ // Invalidate old sessions and load active sessions only+ for (const [sessionName, sessionObject] of Object.entries(loadedSessions)) {+ const session = Session.recreateSession(sessionObject);+ const isValid = !session.isExpired() || !session.isBlocked() || session.usedCount >= session.maxReuseCount;++ if (isValid) {+ this.sessions[sessionName] = session;+ }+ }+ }++ async retrieveSession() {+ // If we have enough space for session. Return newly created session.+ if (this.sessions.length < this.maxPoolSize) {+ const newSession = await this.createSessionFunction(this);+ this._addSession(newSession);+ return newSession;+ }+++ // For example that developer can plug different picking algorithms such as the Lukášův+ // Maybe a should have pickSession function to be customizable.+ const pickedSession = this.sessions[this._getRandomIndex()];++ // If session can be used return the session+ if (pickedSession.isUsable()) {+ return pickedSession;+ }++ // otherwise remove old session and return newly created session+ this._removeSession(pickedSession);++ const newSession = this.createSessionFunction(this);+ this._addSession(newSession);+ return newSession;+ }++ getStats() {+ return {+ activeSessions: this.activeSessions,+ blockedSessions: this.blockedSessions,+ };+ }++ getState() {+ return {+ ...this.getStats(),+ sessions: this.sessions.map(session => session.getState()),+ };+ }++ async persistState() {+ return this.storage.setValue(this.persistStateKey, this.getState());+ }++ _removeSession(session) {+ const sessionIndex = this.sessions.findIndex(storedSession => storedSession.name === session.name);++ this.sessions.splice(sessionIndex, 1);+ }++ _addSession(newSession) {+ this.sessions.push(newSession);+ }++ _getRandomIndex() {+ return Math.floor(Math.random() * this.sessions.length);+ }++ _createSessionFunction(sessionPool) {+ return new Session({+ maxSessionAgeSecs: this.maxSessionAgeSecs,+ maxSessionReuseCount: this.maxSessionReuseCount,+ sessionPool,++ });+ }+}diff --git a/test/session_pool/session_pool.js b/test/session_pool/session_pool.jsnew file mode 100644index 000000000..9a06be8a4--- /dev/null+++ b/test/session_pool/session_pool.js@@ -0,0 +1,84 @@+import { expect } from 'chai';++import SessionPool, { SESSION_POOL_DEFAULTS } from '../../src/session_pool/session_pool';++describe('SessionPool - testing session pool', async () => {+ let sessionPool;++ before(() => {+ process.env.APIFY_LOCAL_STORAGE_DIR = '../../';+ });++ beforeEach(async () => {+ sessionPool = new SessionPool({ persistStateKeyValueStoreId: 'default' });+ await sessionPool.initialize();+ });++ it('should initialize with default values for first time', async () => {+ expect(sessionPool.sessions.length).to.be.eql(0);+ expect(sessionPool.maxPoolSize).to.be.eql(SESSION_POOL_DEFAULTS.maxPoolSize);+ expect(sessionPool.maxSessionAgeSecs).to.be.eql(SESSION_POOL_DEFAULTS.maxSessionAgeSecs);+ expect(sessionPool.maxSessionReuseCount).to.be.eql(SESSION_POOL_DEFAULTS.maxSessionReuseCount);+ expect(sessionPool.persistStateKeyValueStoreId).to.be.eql(SESSION_POOL_DEFAULTS.persistStateKeyValueStoreId);+ expect(sessionPool.persistStateKey).to.be.eql(SESSION_POOL_DEFAULTS.persistStateKey);+ expect(sessionPool.createSessionFunction).to.be.eql(sessionPool._createSessionFunction); // eslint-disable-line+ });++ it('should override default values', async () => {+ const opts = {+ maxPoolSize: 3000,+ maxSessionAgeSecs: 100,+ maxSessionReuseCount: 1,++ persistStateKeyValueStoreId: 'TEST',+ persistStateKey: 'SESSION_POOL_STATE2',++ createSessionFunction: () => ({}),++ };+ sessionPool = new SessionPool(opts);+ await sessionPool.initialize();++ Object.entries(opts).forEach(([key, value]) => {+ expect(sessionPool[key]).to.be.eql(value);+ });+ });++ it('should retrieve session', async () => {+ const session = await sessionPool.retrieveSession();+ expect(sessionPool.sessions.length).to.be.eql(1);+ expect(session.name).to.exist;+ expect(session.cookies.length).to.be.eql(0);+ expect(session.fingerPrintSeed).to.exist;+ expect(session.maxAgeSecs).to.eql(sessionPool.maxSessionAgeSecs);+ expect(session.maxAgeSecs).to.eql(sessionPool.maxSessionAgeSecs);+ expect(session.sessionPool).to.eql(sessionPool);+ });++ xit('should persist state and recreate it from storage', async () => {+ const session = await sessionPool.retrieveSession();+ let key;+ let value;++ const mySetValue = (k, v) => {+ key = k;+ value = v;+ };+ sessionPool.storage.setValue = mySetValue;+ await sessionPool.persistState();+ expect(key).to.be.eql(SESSION_POOL_DEFAULTS.persistStateKey);+ expect(value).to.be.eql(sessionPool.getState());++ const loadedSessionPool = new SessionPool({ persistStateKeyValueStoreId: 'default' });+ await loadedSessionPool.initialize();+ console.log('SESSION');+ });++ it('should create only maxPoolSize number of sessions', async () => {+ const max = sessionPool.maxPoolSize;+ for (let i = 0; i < max + 100; i++) {+ await sessionPool.retrieveSession();+ }+ expect(sessionPool.sessions.length).to.be.eql(sessionPool.maxPoolSize);+ });+});