Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Refactored and improved code.
Petr Patekdde9f81cc71a
diff --git a/src/session_pool/events.js b/src/session_pool/events.jsindex 1f7f6cf47..7d20af038 100644--- a/src/session_pool/events.js+++ b/src/session_pool/events.js@@ -1,3 +1,3 @@export default {- DISCARD: 'DISCARD',+ DISCARDED: 'discarded',};diff --git a/src/session_pool/session.js b/src/session_pool/session.jsindex 80165d359..35902fcb3 100644--- a/src/session_pool/session.js+++ b/src/session_pool/session.js@@ -1,49 +1,45 @@import { checkParamPrototypeOrThrow, cryptoRandomObjectId } from 'apify-shared/utilities';-import _ from 'underscore';import moment from 'moment';import EVENTS from './events';-export const SESSION_DEFAULT_OPTIONS = {- name: cryptoRandomObjectId(),- cookies: [],- fingerPrintSeed: cryptoRandomObjectId(),- maxAgeSecs: 3000,- maxReuseCount: 50,- userData: {},- maxErrorScore: 3,- errorScoreDecrement: 0.5,- expiresAt: moment().add(3000, 'seconds').toISOString(),- createdAt: moment().toISOString(),- usedCount: 0,- errorScore: 0,--};// TODO: Validationexport default class Session {- constructor(options) {- const opts = _.defaults({}, options, SESSION_DEFAULT_OPTIONS);-+ constructor(+ {+ id = cryptoRandomObjectId(),+ cookies = [],+ fingerprintSeed = cryptoRandomObjectId(),+ maxAgeSecs = 3000,+ userData = {},+ maxErrorScore = 3,+ errorScoreDecrement = 0.5,+ expiresAt = moment().add(3000, 'seconds').toISOString(),+ createdAt = moment().toISOString(),+ usageCount = 0,+ errorScore = 0,+ maxSessionUsageCount = 50,+ sessionPool,++ },+ ) {// Configurable- this.name = opts.name;- this.cookies = opts.cookies;- this.fingerPrintSeed = opts.fingerPrintSeed;- this.maxAgeSecs = opts.maxAgeSecs;- this.userData = opts.userData;- this.maxErrorScore = opts.maxErrorScore;- this.errorScoreDecrement = opts.errorScoreDecrement; // TODO: Better Naming+ this.id = id;+ this.cookies = cookies;+ this.fingerprintSeed = fingerprintSeed;+ this.maxAgeSecs = maxAgeSecs;+ this.userData = userData;+ this.maxErrorScore = maxErrorScore;+ this.errorScoreDecrement = errorScoreDecrement; // TODO: Better Naming// Internal- this.expiresAt = opts.expiresAt;- this.createdAt = opts.createdAt;- this.usedCount = opts.usedCount;- this.errorScore = opts.errorScoreDecrement;- this.sessionPool = opts.sessionPool;- }-- static recreateSession(sessionObject) {- return new Session(sessionObject);+ this.expiresAt = expiresAt;+ this.createdAt = createdAt;+ this.usageCount = usageCount;+ this.errorScore = errorScore;+ this.maxSessionUsageCount = maxSessionUsageCount;+ this.sessionPool = sessionPool;}isBlocked() {@@ -57,11 +53,15 @@ export default class Session {}isUsable() {- return !(this.isBlocked() && this.isExpired());+ return !(this.isBlocked() && this.isExpired() && this.isMaxUseCountReached());+ }++ isMaxUseCountReached() {+ return this.usageCount >= this.maxSessionUsageCount;}reclaim() {- this.usedCount += 1;+ this.usageCount += 1;// I should probably lower the errorScorethis.errorScore -= this.errorScoreDecrement;@@ -69,30 +69,36 @@ export default class Session {getState() {return {- name: this.name,+ id: this.id,cookies: this.cookies,userData: this.userData,maxErrorScore: this.maxErrorScore,errorScoreDecrement: this.errorScoreDecrement,expiresAt: this.expiresAt,createdAt: this.createdAt,- usedCount: this.usedCount,+ usageCount: this.usageCount,errorScore: this.errorScore,};}+ /**+ * Marks session as blocked+ */+ retire() {+ // mark it as an invalid by increasing the error score count.+ this.errorScore += this.maxErrorScore;+ this.usageCount += 1;- retire(discard = false) {- if (discard) {- // mark it as an invalid by increasing the error score count.- this.errorScore += this.maxErrorScore;-- // emit event so we can retire browser in puppeteer pool- this.sessionPool.emit(EVENTS.DISCARD, this);- } else {- this.errorScore += 1;- }+ // emit event so we can retire browser in puppeteer pool+ this.sessionPool.emit(EVENTS.DISCARDED, this);+ }- this.usedCount += 1;+ /**+ * Increases and usage error count+ * Should be used unsuccessful request/use with the session+ */+ fail() {+ this.errorScore += 1;+ this.usageCount += 1;}}diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.jsindex e1942c7b5..6b4fa02ec 100644--- a/src/session_pool/session_pool.js+++ b/src/session_pool/session_pool.js@@ -1,19 +1,9 @@import EventEmitter from 'events';-import _ from 'underscore';import { openKeyValueStore } from '../key_value_store';import Session from './session';+import events from '../events';+import { ACTOR_EVENT_NAMES_EX } from '../constants';-export const SESSION_POOL_DEFAULTS = {- maxPoolSize: 1000,- maxSessionAgeSecs: 3000,- maxSessionReuseCount: 50,-- persistStateKeyValueStoreId: null,- persistStateKey: 'SESSION_POOL_STATE',-- createSessionFunction: null,--};/*** Handles the session rotation, creation and persistence.@@ -32,23 +22,31 @@ export default class SessionPool extends EventEmitter {* @param options.maxSessionReuseCount {Number}* @param options.persistStateKeyValueStoreId {String}*/- constructor(options) {- super();+ constructor({+ maxPoolSize = 1000,+ maxSessionAgeSecs = 3000,+ maxSessionReuseCount = 50,++ persistStateKeyValueStoreId = null,+ persistStateKey = 'SESSION_POOL_STATE',- const opts = _.defaults({}, options, SESSION_POOL_DEFAULTS);+ createSessionFunction = null,++ }) {+ super();// Pool Configuration- this.maxPoolSize = opts.maxPoolSize;- this.createSessionFunction = opts.createSessionFunction || this._createSessionFunction;+ this.maxPoolSize = maxPoolSize;+ this.createSessionFunction = createSessionFunction || this._defaultCreateSessionFunction;// Session configuration// @TODO: Maybe options.sessionOptions / this.sessionOptions?- this.maxSessionAgeSecs = opts.maxSessionAgeSecs;- this.maxSessionReuseCount = opts.maxSessionReuseCount;+ this.maxSessionAgeSecs = maxSessionAgeSecs;+ this.maxSessionReuseCount = maxSessionReuseCount;// Session storage- this.persistStateKeyValueStoreId = opts.persistStateKeyValueStoreId;- this.persistStateKey = opts.persistStateKey;+ this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;+ this.persistStateKey = persistStateKey;// Statisticsthis.activeSessions = 0;@@ -70,27 +68,25 @@ export default class SessionPool extends EventEmitter {// Invalidate old sessions and load active sessions onlyfor (const [sessionName, sessionObject] of Object.entries(loadedSessions)) {- const session = Session.recreateSession(sessionObject);- const isValid = !session.isExpired() || !session.isBlocked() || session.usedCount >= session.maxReuseCount;+ const session = new Session(sessionObject);- if (isValid) {+ if (session.isUsable()) {this.sessions[sessionName] = session;}}++ events.on(ACTOR_EVENT_NAMES_EX.PERSIST_STATE, this.persistState.bind(this));}async retrieveSession() {// If we have enough space for session. Return newly created session.- if (this.sessions.length < this.maxPoolSize) {- const newSession = await this.createSessionFunction(this);- this._addSession(newSession);- return newSession;+ if (this._isSpaceForSession()) {+ return this._makeSession();}-// For example that developer can plug different picking algorithms such as the Lukášův// Maybe a should have pickSession function to be customizable.- const pickedSession = this.sessions[this._getRandomIndex()];+ const pickedSession = this._pickSession();// If session can be used return the sessionif (pickedSession.isUsable()) {@@ -99,10 +95,7 @@ export default class SessionPool extends EventEmitter {// otherwise remove old session and return newly created sessionthis._removeSession(pickedSession);-- const newSession = this.createSessionFunction(this);- this._addSession(newSession);- return newSession;+ this._makeSession();}getStats() {@@ -124,7 +117,7 @@ export default class SessionPool extends EventEmitter {}_removeSession(session) {- const sessionIndex = this.sessions.findIndex(storedSession => storedSession.name === session.name);+ const sessionIndex = this.sessions.findIndex(storedSession => storedSession.id === session.id);this.sessions.splice(sessionIndex, 1);}@@ -137,12 +130,24 @@ export default class SessionPool extends EventEmitter {return Math.floor(Math.random() * this.sessions.length);}- _createSessionFunction(sessionPool) {+ _defaultCreateSessionFunction(sessionPool) {return new Session({maxSessionAgeSecs: this.maxSessionAgeSecs,maxSessionReuseCount: this.maxSessionReuseCount,sessionPool,-});}++ async _makeSession() {+ const newSession = await this.createSessionFunction();+ this._addSession(newSession);+ }++ _isSpaceForSession() {+ return this.sessions.length < this.maxPoolSize;+ }++ _pickSession() {+ return this.sessions[this._getRandomIndex()]; // Or maybe we should let the developer to customize the picking algorithm+ }}