Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

improved code and docs

Petr Patek0b97439defba
diff --git a/docs/api/SessionPool.md b/docs/api/SessionPool.md
index d110662bc..fcf5ede33 100644
--- a/docs/api/SessionPool.md
+++ b/docs/api/SessionPool.md
@@ -103,7 +103,8 @@ It useful, when you know the site rate-limits, so you can retire the session bef
<td><code>options.createSessionFunction</code></td><td><code>function</code></td>
</tr>
<tr>
-<td colspan="3"><p>Custom function that should return <code>Session</code> instance.</p>
+<td colspan="3"><p>Custom function that should return <code>Session</code> instance.
+Function receives <code>SessionPool</code> instance as a parameter</p>
</td></tr></tbody>
</table>
<a name="SessionPool+usableSessionsCount"></a>
@@ -116,7 +117,7 @@ Gets count of usable sessions in the pool.
## `sessionPool.retiredSessionsCount` ⇒ `number`
-Gets count of blocked sessions in the pool.
+Gets count of retired sessions in the pool.
<a name="SessionPool+initialize"></a>
diff --git a/src/session_pool/session.js b/src/session_pool/session.js
index a078f172f..41e1d204b 100644
--- a/src/session_pool/session.js
+++ b/src/session_pool/session.js
@@ -1,9 +1,8 @@
-import EventEmitter from 'events';
-
import { cryptoRandomObjectId } from 'apify-shared/utilities';
import { checkParamOrThrow } from 'apify-client/build/utils';
import EVENTS from './events';
+import { SessionPool } from './session_pool';
/**
@@ -46,7 +45,7 @@ export class Session {
sessionPool,
} = options;
- let { expiresAt } = options;
+ const { expiresAt = new Date(Date.now() + (maxAgeSecs * 1000)) } = options;
// Validation
checkParamOrThrow(id, 'options.id', 'String');
@@ -60,16 +59,11 @@ export class Session {
checkParamOrThrow(maxSessionUsageCount, 'options.maxSessionUsageCount', 'Number');
checkParamOrThrow(sessionPool, 'options.sessionPool', 'Object');
- // sessionPool must be at least instance of EvenEmitter.
- // That way we can allow custom implementation of SessionPool in the future (It should not be needed).
- if (!(sessionPool instanceof EventEmitter)) {
+ // sessionPool must be instance of SessionPool.
+ if (sessionPool.constructor.name !== 'SessionPool') {
throw new Error('Session: sessionPool must be instance of SessionPool');
}
- if (!expiresAt) {
- expiresAt = new Date((new Date()).getTime() + (maxAgeSecs * 1000));
- }
-
// Configurable
this.id = id;
this.cookies = cookies;
diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.js
index c5d9596aa..99bca1832 100644
--- a/src/session_pool/session_pool.js
+++ b/src/session_pool/session_pool.js
@@ -51,6 +51,7 @@ import { ACTOR_EVENT_NAMES_EX } from '../constants';
* session3.retire()
*
* ```
+ * @hideconstructor
*/
export class SessionPool extends EventEmitter {
/**
@@ -64,6 +65,7 @@ export class SessionPool extends EventEmitter {
* @param options.persistStateKeyValueStoreId {String} - Name or Id of `KeyValueStore` where is the `SessionPool` state stored.
* @param options.persistStateKey {String} - Session pool persists it's state under this key in Key value store.
* @param options.createSessionFunction {function} - Custom function that should return `Session` instance.
+ * Function receives `SessionPool` instance as a parameter
*/
constructor(options = {}) {
const {
@@ -115,7 +117,7 @@ export class SessionPool extends EventEmitter {
}
/**
- * Gets count of blocked sessions in the pool.
+ * Gets count of retired sessions in the pool.
* @return {number}
*/
get retiredSessionsCount() {
@@ -146,21 +148,16 @@ export class SessionPool extends EventEmitter {
* @return {Promise<Session>}
*/
async retrieveSession() {
- // If we have enough space for session. Return newly created session.
if (this._hasSpaceForSession()) {
return this._createSession();
}
- // For example that developer can plug different picking algorithms such as the Lukášův
- // Maybe a should have pickSession function to be customizable.
const pickedSession = this._pickSession();
- // If session can be used return the session
if (pickedSession.isUsable()) {
return pickedSession;
}
- // otherwise remove old session and return newly created session
this._removeSession(pickedSession);
return this._createSession();
}
@@ -188,7 +185,7 @@ export class SessionPool extends EventEmitter {
async persistState() {
log.debug('SessionPool: Persisting state');
log.debug(`SessionPool: persistStateKeyValueStoreId: ${this.persistStateKeyValueStoreId}, persistStateKey: ${this.persistStateKey} `);
- return this.keyValueStore.setValue(this.persistStateKey, this.getState());
+ await this.keyValueStore.setValue(this.persistStateKey, this.getState());
}
/**
@@ -223,14 +220,15 @@ export class SessionPool extends EventEmitter {
/**
* Creates new session without any extra behavior.
+ * @param sessionPool
* @return {Session} - New session.
* @private
*/
- _defaultCreateSessionFunction() {
+ _defaultCreateSessionFunction(sessionPool) {
return new Session({
maxSessionAgeSecs: this.maxSessionAgeSecs,
maxSessionUsageCount: this.maxSessionUsageCount,
- sessionPool: this,
+ sessionPool,
});
}
@@ -240,7 +238,7 @@ export class SessionPool extends EventEmitter {
* @private
*/
async _createSession() {
- const newSession = await this.createSessionFunction();
+ const newSession = await this.createSessionFunction(this);
this._addSession(newSession);
log.debug(`SessionPool: Created new Session - ${newSession.id}`);
diff --git a/test/key_value_store.js b/test/key_value_store.js
index 62f824f0c..45bf819a6 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -512,7 +512,7 @@ describe('KeyValueStore', () => {
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
delete process.env[ENV_VARS.INPUT_KEY];
- defaultStore.getValue = oldGet.bind(defaultStore);
+ defaultStore.getValue = oldGet;
});
});
});
diff --git a/test/session_pool/session_pool.js b/test/session_pool/session_pool.js
index 1a8e6817b..b7ee2b9f3 100644
--- a/test/session_pool/session_pool.js
+++ b/test/session_pool/session_pool.js
@@ -5,6 +5,7 @@ import Apify from '../../build';
import events from '../../build/events';
import { ACTOR_EVENT_NAMES_EX } from '../../build/constants';
+import { Session } from '../../src/session_pool/session';
describe('SessionPool - testing session pool', async () => {
let sessionPool;
@@ -15,8 +16,7 @@ describe('SessionPool - testing session pool', async () => {
beforeEach(async () => {
await emptyLocalStorageSubdir('key_value_stores/default');
- sessionPool = new SessionPool();
- await sessionPool.initialize();
+ sessionPool = await Apify.openSessionPool();
});
it('should initialize with default values for first time', async () => {
@@ -225,4 +225,18 @@ describe('SessionPool - testing session pool', async () => {
expect(newSessionPool.sessions).to.be.length(10 - invalidSessionsCount);
});
+
+ it('should createSessionFunction work', async () => {
+ let isCalled;
+ const createSessionFunction = (sessionPool2) => {
+ isCalled = true;
+ expect(sessionPool2 instanceof SessionPool).to.be.true; // eslint-disable-line
+ console.log(sessionPool2 instanceof SessionPool);
+ return new Session({ sessionPool: sessionPool2 });
+ };
+ const newSessionPool = await Apify.openSessionPool({ createSessionFunction });
+ const session = await newSessionPool.retrieveSession();
+ expect(isCalled).to.be.true; // eslint-disable-line
+ expect(session.constructor.name).to.be.eql("Session") // eslint-disable-line
+ });
});