Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Refactored and improved code.

Petr Patekdde9f81cc71a
diff --git a/src/session_pool/events.js b/src/session_pool/events.js
index 1f7f6cf47..7d20af038 100644
--- a/src/session_pool/events.js
+++ b/src/session_pool/events.js
@@ -1,3 +1,3 @@
export default {
- DISCARD: 'DISCARD',
+ DISCARDED: 'discarded',
};
diff --git a/src/session_pool/session.js b/src/session_pool/session.js
index 80165d359..35902fcb3 100644
--- a/src/session_pool/session.js
+++ b/src/session_pool/session.js
@@ -1,49 +1,45 @@
import { checkParamPrototypeOrThrow, cryptoRandomObjectId } from 'apify-shared/utilities';
-import _ from 'underscore';
import moment from 'moment';
import EVENTS from './events';
-export const SESSION_DEFAULT_OPTIONS = {
- name: cryptoRandomObjectId(),
- cookies: [],
- fingerPrintSeed: cryptoRandomObjectId(),
- maxAgeSecs: 3000,
- maxReuseCount: 50,
- userData: {},
- maxErrorScore: 3,
- errorScoreDecrement: 0.5,
- expiresAt: moment().add(3000, 'seconds').toISOString(),
- createdAt: moment().toISOString(),
- usedCount: 0,
- errorScore: 0,
-
-};
// TODO: Validation
export default class Session {
- constructor(options) {
- const opts = _.defaults({}, options, SESSION_DEFAULT_OPTIONS);
-
+ constructor(
+ {
+ id = cryptoRandomObjectId(),
+ cookies = [],
+ fingerprintSeed = cryptoRandomObjectId(),
+ maxAgeSecs = 3000,
+ userData = {},
+ maxErrorScore = 3,
+ errorScoreDecrement = 0.5,
+ expiresAt = moment().add(3000, 'seconds').toISOString(),
+ createdAt = moment().toISOString(),
+ usageCount = 0,
+ errorScore = 0,
+ maxSessionUsageCount = 50,
+ sessionPool,
+
+ },
+ ) {
// Configurable
- this.name = opts.name;
- this.cookies = opts.cookies;
- this.fingerPrintSeed = opts.fingerPrintSeed;
- this.maxAgeSecs = opts.maxAgeSecs;
- this.userData = opts.userData;
- this.maxErrorScore = opts.maxErrorScore;
- this.errorScoreDecrement = opts.errorScoreDecrement; // TODO: Better Naming
+ this.id = id;
+ this.cookies = cookies;
+ this.fingerprintSeed = fingerprintSeed;
+ this.maxAgeSecs = maxAgeSecs;
+ this.userData = userData;
+ this.maxErrorScore = maxErrorScore;
+ this.errorScoreDecrement = errorScoreDecrement; // TODO: Better Naming
// Internal
- this.expiresAt = opts.expiresAt;
- this.createdAt = opts.createdAt;
- this.usedCount = opts.usedCount;
- this.errorScore = opts.errorScoreDecrement;
- this.sessionPool = opts.sessionPool;
- }
-
- static recreateSession(sessionObject) {
- return new Session(sessionObject);
+ this.expiresAt = expiresAt;
+ this.createdAt = createdAt;
+ this.usageCount = usageCount;
+ this.errorScore = errorScore;
+ this.maxSessionUsageCount = maxSessionUsageCount;
+ this.sessionPool = sessionPool;
}
isBlocked() {
@@ -57,11 +53,15 @@ export default class Session {
}
isUsable() {
- return !(this.isBlocked() && this.isExpired());
+ return !(this.isBlocked() && this.isExpired() && this.isMaxUseCountReached());
+ }
+
+ isMaxUseCountReached() {
+ return this.usageCount >= this.maxSessionUsageCount;
}
reclaim() {
- this.usedCount += 1;
+ this.usageCount += 1;
// I should probably lower the errorScore
this.errorScore -= this.errorScoreDecrement;
@@ -69,30 +69,36 @@ export default class Session {
getState() {
return {
- name: this.name,
+ id: this.id,
cookies: this.cookies,
userData: this.userData,
maxErrorScore: this.maxErrorScore,
errorScoreDecrement: this.errorScoreDecrement,
expiresAt: this.expiresAt,
createdAt: this.createdAt,
- usedCount: this.usedCount,
+ usageCount: this.usageCount,
errorScore: this.errorScore,
};
}
+ /**
+ * Marks session as blocked
+ */
+ retire() {
+ // mark it as an invalid by increasing the error score count.
+ this.errorScore += this.maxErrorScore;
+ this.usageCount += 1;
- retire(discard = false) {
- if (discard) {
- // mark it as an invalid by increasing the error score count.
- this.errorScore += this.maxErrorScore;
-
- // emit event so we can retire browser in puppeteer pool
- this.sessionPool.emit(EVENTS.DISCARD, this);
- } else {
- this.errorScore += 1;
- }
+ // emit event so we can retire browser in puppeteer pool
+ this.sessionPool.emit(EVENTS.DISCARDED, this);
+ }
- this.usedCount += 1;
+ /**
+ * Increases and usage error count
+ * Should be used unsuccessful request/use with the session
+ */
+ fail() {
+ this.errorScore += 1;
+ this.usageCount += 1;
}
}
diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.js
index e1942c7b5..6b4fa02ec 100644
--- a/src/session_pool/session_pool.js
+++ b/src/session_pool/session_pool.js
@@ -1,19 +1,9 @@
import EventEmitter from 'events';
-import _ from 'underscore';
import { openKeyValueStore } from '../key_value_store';
import Session from './session';
+import events from '../events';
+import { ACTOR_EVENT_NAMES_EX } from '../constants';
-export const SESSION_POOL_DEFAULTS = {
- maxPoolSize: 1000,
- maxSessionAgeSecs: 3000,
- maxSessionReuseCount: 50,
-
- persistStateKeyValueStoreId: null,
- persistStateKey: 'SESSION_POOL_STATE',
-
- createSessionFunction: null,
-
-};
/**
* Handles the session rotation, creation and persistence.
@@ -32,23 +22,31 @@ export default class SessionPool extends EventEmitter {
* @param options.maxSessionReuseCount {Number}
* @param options.persistStateKeyValueStoreId {String}
*/
- constructor(options) {
- super();
+ constructor({
+ maxPoolSize = 1000,
+ maxSessionAgeSecs = 3000,
+ maxSessionReuseCount = 50,
+
+ persistStateKeyValueStoreId = null,
+ persistStateKey = 'SESSION_POOL_STATE',
- const opts = _.defaults({}, options, SESSION_POOL_DEFAULTS);
+ createSessionFunction = null,
+
+ }) {
+ super();
// Pool Configuration
- this.maxPoolSize = opts.maxPoolSize;
- this.createSessionFunction = opts.createSessionFunction || this._createSessionFunction;
+ this.maxPoolSize = maxPoolSize;
+ this.createSessionFunction = createSessionFunction || this._defaultCreateSessionFunction;
// Session configuration
// @TODO: Maybe options.sessionOptions / this.sessionOptions?
- this.maxSessionAgeSecs = opts.maxSessionAgeSecs;
- this.maxSessionReuseCount = opts.maxSessionReuseCount;
+ this.maxSessionAgeSecs = maxSessionAgeSecs;
+ this.maxSessionReuseCount = maxSessionReuseCount;
// Session storage
- this.persistStateKeyValueStoreId = opts.persistStateKeyValueStoreId;
- this.persistStateKey = opts.persistStateKey;
+ this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
+ this.persistStateKey = persistStateKey;
// Statistics
this.activeSessions = 0;
@@ -70,27 +68,25 @@ export default class SessionPool extends EventEmitter {
// Invalidate old sessions and load active sessions only
for (const [sessionName, sessionObject] of Object.entries(loadedSessions)) {
- const session = Session.recreateSession(sessionObject);
- const isValid = !session.isExpired() || !session.isBlocked() || session.usedCount >= session.maxReuseCount;
+ const session = new Session(sessionObject);
- if (isValid) {
+ if (session.isUsable()) {
this.sessions[sessionName] = session;
}
}
+
+ events.on(ACTOR_EVENT_NAMES_EX.PERSIST_STATE, this.persistState.bind(this));
}
async retrieveSession() {
// If we have enough space for session. Return newly created session.
- if (this.sessions.length < this.maxPoolSize) {
- const newSession = await this.createSessionFunction(this);
- this._addSession(newSession);
- return newSession;
+ if (this._isSpaceForSession()) {
+ return this._makeSession();
}
-
// For example that developer can plug different picking algorithms such as the Lukášův
// Maybe a should have pickSession function to be customizable.
- const pickedSession = this.sessions[this._getRandomIndex()];
+ const pickedSession = this._pickSession();
// If session can be used return the session
if (pickedSession.isUsable()) {
@@ -99,10 +95,7 @@ export default class SessionPool extends EventEmitter {
// otherwise remove old session and return newly created session
this._removeSession(pickedSession);
-
- const newSession = this.createSessionFunction(this);
- this._addSession(newSession);
- return newSession;
+ this._makeSession();
}
getStats() {
@@ -124,7 +117,7 @@ export default class SessionPool extends EventEmitter {
}
_removeSession(session) {
- const sessionIndex = this.sessions.findIndex(storedSession => storedSession.name === session.name);
+ const sessionIndex = this.sessions.findIndex(storedSession => storedSession.id === session.id);
this.sessions.splice(sessionIndex, 1);
}
@@ -137,12 +130,24 @@ export default class SessionPool extends EventEmitter {
return Math.floor(Math.random() * this.sessions.length);
}
- _createSessionFunction(sessionPool) {
+ _defaultCreateSessionFunction(sessionPool) {
return new Session({
maxSessionAgeSecs: this.maxSessionAgeSecs,
maxSessionReuseCount: this.maxSessionReuseCount,
sessionPool,
-
});
}
+
+ async _makeSession() {
+ const newSession = await this.createSessionFunction();
+ this._addSession(newSession);
+ }
+
+ _isSpaceForSession() {
+ return this.sessions.length < this.maxPoolSize;
+ }
+
+ _pickSession() {
+ return this.sessions[this._getRandomIndex()]; // Or maybe we should let the developer to customize the picking algorithm
+ }
}