Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Implementation improved. Docs Added.

Petr Patek37d236969550
diff --git a/docs/api/BasicCrawler.md b/docs/api/BasicCrawler.md
index 9b642a394..fc848a313 100644
--- a/docs/api/BasicCrawler.md
+++ b/docs/api/BasicCrawler.md
@@ -55,12 +55,12 @@ await crawler.run();
```
- [BasicCrawler](basiccrawler)
- - [`new BasicCrawler(options)`](#new_BasicCrawler_new)
+ - [`new BasicCrawler(options, sessionPoolOptions)`](#new_BasicCrawler_new)
- [`.run()`](#BasicCrawler+run) ⇒ `Promise`
<a name="new_BasicCrawler_new"></a>
-## `new BasicCrawler(options)`
+## `new BasicCrawler(options, sessionPoolOptions)`
<table>
<thead>
@@ -157,6 +157,17 @@ await crawler.run();
</tr>
<tr>
<td colspan="3"><p>Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding <a href="autoscaledpool"><code>AutoscaledPool</code></a> option.</p>
+</td></tr><tr>
+<td><code>[options.useSessionPool]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If set to true. Basic crawler will initialize the <a href="sessionpool"><code>SessionPool</code></a> with the corresponding <code>sessionPoolOptions</code>.
+ The session instance will be than available in the <code>handleRequestFunction</code>.</p>
+</td></tr><tr>
+<td><code>sessionPoolOptions</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>The <a href="sessionpool#new_SessionPool_new"><code>new SessionPool</code></a> options</p>
</td></tr></tbody>
</table>
<a name="BasicCrawler+run"></a>
diff --git a/docs/api/SessionPool.md b/docs/api/SessionPool.md
index d4da806af..7c5c5246c 100644
--- a/docs/api/SessionPool.md
+++ b/docs/api/SessionPool.md
@@ -5,8 +5,8 @@ title: SessionPool
<a name="SessionPool"></a>
-Handles the sessions rotation, creation and persistence. Creates a pool of [`Session`](session) instances, that are randomly rotated. When some
-session is marked as blocked. It is removed and new one is created instead.
+Experimental feature and might change in the future releases. Handles the sessions rotation, creation and persistence. Creates a pool of
+[`Session`](session) instances, that are randomly rotated. When some session is marked as blocked. It is removed and new one is created instead.
Session pool is by default persisted in default [`KeyValueStore`](keyvaluestore). If you want to have one pool for all runs you have to specify
`persistStateKeyValueStoreId`.
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.js
index 9bb752781..57b4e694b 100644
--- a/src/crawlers/basic_crawler.js
+++ b/src/crawlers/basic_crawler.js
@@ -33,6 +33,7 @@ const DEFAULT_OPTIONS = {
},
autoscaledPoolOptions: {},
sessionPoolOptions: {}, // We could add sessionPool true/false config to use/not use SessionPool.
+ useSessionPool: false,
};
/**
@@ -159,6 +160,10 @@ const DEFAULT_OPTIONS = {
* If you're not sure, just keep the default value and the concurrency will scale up automatically.
* @param {Object} [options.maxConcurrency=1000]
* Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
+ * @param {Boolean} [options.useSessionPool=false]
+ * If set to true. Basic crawler will initialize the {@link SessionPool} with the corresponding `sessionPoolOptions`.
+ * The session instance will be than available in the `handleRequestFunction`.
+ * @param sessionPoolOptions {Object} The [`new SessionPool`](sessionpool#new_SessionPool_new) options
*/
class BasicCrawler {
constructor(options) {
@@ -172,6 +177,7 @@ class BasicCrawler {
maxRequestsPerCrawl,
autoscaledPoolOptions,
sessionPoolOptions,
+ useSessionPool,
// AutoscaledPool shorthands
minConcurrency,
@@ -187,6 +193,7 @@ class BasicCrawler {
checkParamOrThrow(maxRequestsPerCrawl, 'options.maxRequestsPerCrawl', 'Maybe Number');
checkParamOrThrow(autoscaledPoolOptions, 'options.autoscaledPoolOptions', 'Object');
checkParamOrThrow(sessionPoolOptions, 'options.sessionPoolOptions', 'Object');
+ checkParamOrThrow(useSessionPool, 'options.useSessionPool', 'Boolean');
if (!requestList && !requestQueue) {
throw new Error('At least one of the parameters "options.requestList" and "options.requestQueue" must be provided!');
@@ -201,6 +208,7 @@ class BasicCrawler {
this.handledRequestsCount = 0;
this.stats = new Statistics({ logMessage: 'Crawler request statistics:' });
this.sessionPoolOptions = sessionPoolOptions;
+ this.useSessionPool = useSessionPool;
let shouldLogMaxPagesExceeded = true;
const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;
@@ -267,7 +275,10 @@ class BasicCrawler {
// (otherwise there would be no way)
this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);
- this.sessionPool = await openSessionPool(this.sessionPoolOptions);
+ if (this.useSessionPool) {
+ this.sessionPool = await openSessionPool(this.sessionPoolOptions);
+ }
+
await this._loadHandledRequestCount();
this.isRunningPromise = this.autoscaledPool.run();
@@ -344,9 +355,15 @@ class BasicCrawler {
async _runTaskFunction() {
const source = this.requestQueue || this.requestList;
- // Maybe we can use Promise.all() to speed it up.
- const request = await this._fetchNextRequest();
- const session = await this.sessionPool.getSession();
+ let request;
+ let session;
+
+ if (this.useSessionPool) {
+ [request, session] = await Promise.all([this._fetchNextRequest(), this.sessionPool.getSession()]);
+ } else {
+ request = await this._fetchNextRequest();
+ }
+
if (!request) return;
// Reset loadedUrl so an old one is not carried over to retries.
@@ -365,7 +382,7 @@ class BasicCrawler {
this.handledRequestsCount++;
// reclaim session if request finishes successfully
- session.markGood();
+ if (session) session.markGood();
} catch (err) {
try {
await this._requestFunctionErrorHandler(err, request, source);
diff --git a/src/session_pool/session.js b/src/session_pool/session.js
index 250c7fa9c..dcd4e3edf 100644
--- a/src/session_pool/session.js
+++ b/src/session_pool/session.js
@@ -124,7 +124,6 @@ export class Session {
markGood() {
this.usageCount += 1;
- // We should probably lower the errorScore.
if (this.errorScore > 0) {
this.errorScore -= this.errorScoreDecrement;
}
diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.js
index e121abbb5..f137f2425 100644
--- a/src/session_pool/session_pool.js
+++ b/src/session_pool/session_pool.js
@@ -9,6 +9,7 @@ import { ACTOR_EVENT_NAMES_EX } from '../constants';
/**
+ * Experimental feature and might change in the future releases.
* Handles the sessions rotation, creation and persistence.
* Creates a pool of {@link Session} instances, that are randomly rotated.
* When some session is marked as blocked. It is removed and new one is created instead.
@@ -183,8 +184,11 @@ export class SessionPool extends EventEmitter {
* @return {Promise}
*/
async persistState() {
- log.debug('SessionPool: Persisting state');
- log.debug(`SessionPool: persistStateKeyValueStoreId: ${this.persistStateKeyValueStoreId}, persistStateKey: ${this.persistStateKey} `);
+ log.debug('SessionPool: Persisting state',
+ {
+ persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
+ persistStateKey: this.persistStateKey,
+ });
await this.keyValueStore.setValue(this.persistStateKey, this.getState());
}
@@ -275,9 +279,11 @@ export class SessionPool extends EventEmitter {
if (!loadedSessionPool) return;
// Invalidate old sessions and load active sessions only
- log.debug('SessionPool: Recreating state from KeyValueStore');
- log.debug(`SessionPool: persistStateKeyValueStoreId: ${this.persistStateKeyValueStoreId}, persistStateKey: ${this.persistStateKey} `);
-
+ log.debug('SessionPool: Recreating state from KeyValueStore',
+ {
+ persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
+ persistStateKey: this.persistStateKey,
+ });
for (const sessionObject of loadedSessionPool.sessions) {
sessionObject.sessionPool = this;
sessionObject.createdAt = new Date(sessionObject.createdAt);
diff --git a/test/crawlers/basic_crawler.js b/test/crawlers/basic_crawler.js
index aa2778364..9a3d2a776 100644
--- a/test/crawlers/basic_crawler.js
+++ b/test/crawlers/basic_crawler.js
@@ -598,20 +598,69 @@ describe('BasicCrawler', () => {
expect(results[0].url).to.be.eql(url);
results[0].errorMessages.forEach(msg => expect(msg).to.include('handleRequestFunction timed out'));
});
+ describe('Uses SessionPool', () => {
+ before(() => {
+ process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;
+ });
- it('should pass session to handleRequestFunction ', async () => {
- const url = 'https://example.com';
- const requestList = new Apify.RequestList({ sources: [{ url }] });
- await requestList.initialize();
+ it('should use SessionPool when useSessionPool is true ', async () => {
+ const url = 'https://example.com';
+ const requestList = new Apify.RequestList({ sources: [{ url }] });
+ await requestList.initialize();
+ const results = [];
+
+ const crawler = new Apify.BasicCrawler({
+ requestList,
+ handleRequestTimeoutSecs: 0.01,
+ maxRequestRetries: 1,
+ useSessionPool: true,
+ handleRequestFunction: async ({ session }) => {
+ expect(session.constructor.name).to.be.eql('Session');
+ expect(session.id).to.exist // eslint-disable-line
+ },
+ handleFailedRequestFunction: ({ request }) => results.push(request),
+ });
- const crawler = new Apify.BasicCrawler({
- requestList,
- handleRequestTimeoutSecs: 0.01,
- maxRequestRetries: 1,
- handleRequestFunction: ({ session }) => expect(session.constructor.name).to.be.eql('Session'),
- handleFailedRequestFunction: ({ request }) => results.push(request),
+ await crawler.run();
+ expect(crawler.sessionPool).to.exist; //eslint-disable-line
+ expect(results).length(0);
});
- await crawler.run();
+ it('should use pass options to sessionPool', async () => {
+ const url = 'https://example.com';
+ const requestList = new Apify.RequestList({ sources: [{ url }] });
+ await requestList.initialize();
+
+ const crawler = new Apify.BasicCrawler({
+ requestList,
+ handleRequestTimeoutSecs: 0.01,
+ maxRequestRetries: 1,
+ useSessionPool: true,
+ sessionPoolOptions: {
+ maxPoolSize: 10,
+ },
+ handleRequestFunction: async () => {},
+ handleFailedRequestFunction: () => {},
+ });
+ await crawler.run();
+
+ expect(crawler.sessionPool.maxPoolSize).to.be.eql(10);
+ });
+
+ it('should not use SessionPool by default', async () => {
+ const url = 'https://example.com?a=1';
+ const requestList = await Apify.openRequestList('my', [url]);
+
+ const crawler = new Apify.BasicCrawler({
+ requestList,
+ handleRequestTimeoutSecs: 0.01,
+ maxRequestRetries: 1,
+ handleRequestFunction: async () => {},
+ handleFailedRequestFunction: () => {},
+ });
+ await crawler.run();
+
+ expect(crawler.sessionPool).to.be.undefined; // eslint-disable-line
+ });
});
});