Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: broken session pool (#970)

Ondra Urban088959fed745
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index a0558b6a0..d1292854c 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -15,7 +15,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- node-version: [10, 12, 14]
+ node-version: [10, 12, 14, 15]
steps:
-
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index e4c2e3300..b01a5b6e9 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -18,7 +18,7 @@ jobs:
strategy:
matrix:
- node-version: [10, 12, 14]
+ node-version: [10, 12, 14, 15]
steps:
-
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0aee7d542..c7887a9a6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,8 +1,12 @@
1.1.0 / 2021/03/XX
====================
-- Add `sessionPool.addSession()` `async` function to add a new session to the session pool (possibly with the provided options, e.g. with specific session id).
+- Add `sessionPool.addSession()` function to add a new session to the session pool (possibly with the provided options, e.g. with specific session id).
- Add optional parameter `sessionId` to `sessionPool.getSession()` to be able to retrieve a session from the session pool with the specific session id.
+- Fix `SessionPool` not working properly in both `PuppeteerCrawler` and `PlaywrightCrawler`.
- Fix `Apify.call()` and `Apify.callTask()` output - make it backwards compatible with previous versions of the client.
+- Improve handling of browser executable paths when using the official SDK Docker images.
+- Update `browser-pool` to fix issues with failing hooks causing browsers to get stuck in limbo.
+- Removed `proxy-chain` dependency because now it's covered in `browser-pool`.
1.0.2 / 2021/03/05
====================
diff --git a/package.json b/package.json
index b9064b414..a55646b15 100644
--- a/package.json
+++ b/package.json
@@ -62,7 +62,7 @@
"@types/tough-cookie": "^4.0.0",
"apify-client": "^1.0.3",
"apify-shared": "^0.6.0",
- "browser-pool": "^1.0.2",
+ "browser-pool": "^1.1.2",
"cheerio": "1.0.0-rc.3",
"content-type": "^1.0.4",
"express": "^4.17.1",
@@ -72,7 +72,6 @@
"jquery": "^3.5.1",
"mime-types": "^2.1.27",
"ow": "^0.23.0",
- "proxy-chain": "^1.0.0",
"rimraf": "^3.0.2",
"semver": "^7.3.2",
"socket.io": "^3.0.3",
diff --git a/src/constants.js b/src/constants.js
index a4639b086..3cbb89191 100644
--- a/src/constants.js
+++ b/src/constants.js
@@ -32,12 +32,12 @@ export const ACTOR_EVENT_NAMES_EX = { ...ACTOR_EVENT_NAMES, PERSIST_STATE: 'pers
export const APIFY_API_BASE_URL = 'https://api.apify.com/v2';
/**
- * Multiplier used in CheerioCrawler and PuppeteerCrawler to set a reasonable
- * handleRequestTimeoutSecs in BasicCrawler that would not impare functionality.
+ * Additional number of seconds used in CheerioCrawler and BrowserCrawler to set a reasonable
+ * handleRequestTimeoutSecs for BasicCrawler that would not impare functionality (not timeout before crawlers).
*
* @type {number}
*/
-export const BASIC_CRAWLER_TIMEOUT_MULTIPLIER = 10;
+export const BASIC_CRAWLER_TIMEOUT_BUFFER_SECS = 10;
export const COUNTRY_CODE_REGEX = /^[A-Z]{2}$/;
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.js
index e42477eca..79b7cf215 100644
--- a/src/crawlers/basic_crawler.js
+++ b/src/crawlers/basic_crawler.js
@@ -364,6 +364,8 @@ class BasicCrawler {
if (this.useSessionPool) {
this.sessionPool = await openSessionPool(this.sessionPoolOptions);
+ // Assuming there are not more than 20 browsers running at once;
+ this.sessionPool.setMaxListeners(20);
}
await this._loadHandledRequestCount();
diff --git a/src/crawlers/browser_crawler.js b/src/crawlers/browser_crawler.js
index bb11c7166..80a80d84a 100644
--- a/src/crawlers/browser_crawler.js
+++ b/src/crawlers/browser_crawler.js
@@ -1,7 +1,6 @@
import ow from 'ow';
import { BrowserPool, BrowserController } from 'browser-pool'; // eslint-disable-line import/no-duplicates,no-unused-vars
-import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';
-import { SessionPool } from '../session_pool/session_pool'; // eslint-disable-line import/no-duplicates
+import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS } from '../constants';
import EVENTS from '../session_pool/events'; // eslint-disable-line import/no-duplicates
import { addTimeoutToPromise } from '../utils';
import { validators } from '../validators';
@@ -263,7 +262,9 @@ class BrowserCrawler extends BasicCrawler {
handlePageFunction: ow.function,
gotoFunction: ow.optional.function,
- handlePageTimeoutSecs: ow.optional.number,
+ gotoTimeoutSecs: ow.optional.number.greaterThan(0),
+ navigationTimeoutSecs: ow.optional.number.greaterThan(0),
+ handlePageTimeoutSecs: ow.optional.number.greaterThan(0),
preNavigationHooks: ow.optional.array,
postNavigationHooks: ow.optional.array,
@@ -283,9 +284,10 @@ class BrowserCrawler extends BasicCrawler {
const {
handlePageFunction,
handlePageTimeoutSecs = 60,
- gotoFunction,
+ navigationTimeoutSecs = 60,
+ gotoFunction, // deprecated
+ gotoTimeoutSecs, // deprecated
persistCookiesPerSession,
- useSessionPool = true,
sessionPoolOptions,
proxyConfiguration,
browserPoolOptions,
@@ -294,24 +296,30 @@ class BrowserCrawler extends BasicCrawler {
...basicCrawlerOptions
} = options;
- // Cookies should be persisted per session only if session pool is used
- if (!useSessionPool && persistCookiesPerSession) {
- throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
- }
-
super({
...basicCrawlerOptions,
- useSessionPool: false,
handleRequestFunction: (...args) => this._handleRequestFunction(...args),
- handleRequestTimeoutSecs: handlePageTimeoutSecs * BASIC_CRAWLER_TIMEOUT_MULTIPLIER,
+ handleRequestTimeoutSecs: navigationTimeoutSecs + handlePageTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
});
+ // Cookies should be persisted per session only if session pool is used
+ if (!this.useSessionPool && persistCookiesPerSession) {
+ throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
+ }
+
+ if (gotoTimeoutSecs) {
+ this.log.deprecated('Option "gotoTimeoutSecs" is deprecated. Use "navigationTimeoutSecs" instead.');
+ }
+
this.handlePageFunction = handlePageFunction;
this.handlePageTimeoutSecs = handlePageTimeoutSecs;
this.handlePageTimeoutMillis = this.handlePageTimeoutSecs * 1000;
+ this.navigationTimeoutMillis = (gotoTimeoutSecs || navigationTimeoutSecs) * 1000;
this.gotoFunction = gotoFunction;
- this.defaultGotoOptions = {};
+ this.defaultGotoOptions = {
+ timeout: this.navigationTimeoutMillis,
+ };
this.proxyConfiguration = proxyConfiguration;
@@ -320,16 +328,8 @@ class BrowserCrawler extends BasicCrawler {
/** @type {Array<Hook>} */
this.postNavigationHooks = postNavigationHooks;
- if (useSessionPool) {
+ if (this.useSessionPool) {
this.persistCookiesPerSession = persistCookiesPerSession !== undefined ? persistCookiesPerSession : true;
-
- this.sessionPool = new SessionPool({
- ...sessionPoolOptions,
- log: this.log,
- });
-
- // Assuming there are not more than 20 browsers running at once;
- this.sessionPool.setMaxListeners(20);
} else {
this.persistCookiesPerSession = false;
}
@@ -363,9 +363,11 @@ class BrowserCrawler extends BasicCrawler {
const { request, session } = crawlingContext;
- const sessionCookies = crawlingContext.session.getPuppeteerCookies(request.url);
- if (sessionCookies.length) {
- await crawlingContext.browserController.setCookies(page, sessionCookies);
+ if (this.useSessionPool) {
+ const sessionCookies = session.getPuppeteerCookies(request.url);
+ if (sessionCookies.length) {
+ await crawlingContext.browserController.setCookies(page, sessionCookies);
+ }
}
try {
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
index 996d033ad..533efc748 100644
--- a/src/crawlers/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -7,6 +7,7 @@ import { WritableStream } from 'htmlparser2/lib/WritableStream';
import * as iconv from 'iconv-lite';
import ow from 'ow';
import * as util from 'util';
+import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS } from '../constants';
import { TimeoutError } from '../errors';
import { addTimeoutToPromise, parseContentTypeFromResponse } from '../utils';
import * as utilsRequest from '../utils_request'; // eslint-disable-line import/no-duplicates
@@ -391,7 +392,7 @@ class CheerioCrawler extends BasicCrawler {
autoscaledPoolOptions,
// We need to add some time for internal functions to finish,
// but not too much so that we would stall the crawler.
- handleRequestTimeoutSecs: (requestTimeoutSecs + handlePageTimeoutSecs * 2) + 5,
+ handleRequestTimeoutSecs: requestTimeoutSecs + handlePageTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
});
// Cookies should be persisted per session only if session pool is used
diff --git a/src/crawlers/playwright_crawler.js b/src/crawlers/playwright_crawler.js
index 33e587cb7..1f407cac6 100644
--- a/src/crawlers/playwright_crawler.js
+++ b/src/crawlers/playwright_crawler.js
@@ -263,8 +263,6 @@ class PlaywrightCrawler extends BrowserCrawler {
...BrowserCrawler.optionsShape,
browserPoolOptions: ow.optional.object,
launcher: ow.optional.object,
- gotoTimeoutSecs: ow.optional.number,
- navigationTimeoutSecs: ow.optional.number,
launchContext: ow.optional.object,
}
@@ -277,8 +275,6 @@ class PlaywrightCrawler extends BrowserCrawler {
const {
launchContext = {},
- gotoTimeoutSecs,
- navigationTimeoutSecs,
browserPoolOptions = {},
...browserCrawlerOptions
} = options;
@@ -298,16 +294,7 @@ class PlaywrightCrawler extends BrowserCrawler {
browserPoolOptions,
});
- if (gotoTimeoutSecs) {
- this.log.deprecated('Option "gotoTimeoutSecs" is deprecated. Use "navigationTimeoutSecs" instead.');
- }
-
- this.navigationTimeoutMillis = (navigationTimeoutSecs || gotoTimeoutSecs) * 1000;
this.launchContext = launchContext;
-
- this.defaultGotoOptions = {
- timeout: this.navigationTimeoutMillis,
- };
}
/**
diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.js
index 839397866..d83f4ee66 100644
--- a/src/crawlers/puppeteer_crawler.js
+++ b/src/crawlers/puppeteer_crawler.js
@@ -249,8 +249,6 @@ class PuppeteerCrawler extends BrowserCrawler {
static optionsShape = {
...BrowserCrawler.optionsShape,
browserPoolOptions: ow.optional.object,
- gotoTimeoutSecs: ow.optional.number,
- navigationTimeoutSecs: ow.optional.number,
launchContext: ow.optional.object,
}
@@ -263,8 +261,6 @@ class PuppeteerCrawler extends BrowserCrawler {
const {
launchContext = {}, // @TODO: should not launcher be inside launchContext
- gotoTimeoutSecs,
- navigationTimeoutSecs,
browserPoolOptions = {},
proxyConfiguration,
...browserCrawlerOptions
@@ -290,14 +286,6 @@ class PuppeteerCrawler extends BrowserCrawler {
browserPoolOptions,
});
- if (gotoTimeoutSecs) {
- this.log.deprecated('Option "gotoTimeoutSecs" is deprecated. Use "navigationTimeoutSecs" instead.');
- }
-
- if (gotoTimeoutSecs) {
- this.log.deprecated('Option "gotoTimeoutSecs" is deprecated. Use "navigationTimeoutSecs" instead.');
- }
-
if (stealth) {
this.browserPool.postLaunchHooks.push(async (pageId, browserController) => {
// @TODO: We can do this better now. It is not necessary to override the page.
@@ -307,12 +295,7 @@ class PuppeteerCrawler extends BrowserCrawler {
});
}
- this.navigationTimeoutMillis = (navigationTimeoutSecs || gotoTimeoutSecs) * 1000;
this.launchContext = launchContext;
-
- this.defaultGotoOptions = {
- timeout: this.navigationTimeoutMillis,
- };
}
/**
diff --git a/src/request_list.js b/src/request_list.js
index 1d0c6d0c8..f7af09c37 100644
--- a/src/request_list.js
+++ b/src/request_list.js
@@ -840,7 +840,7 @@ export class RequestList {
export const openRequestList = async (listName, sources, options = {}) => {
ow(listName, ow.any(ow.string, ow.null));
ow(sources, ow.array);
- // options will get validated in RequestList constructor
+ ow(options, ow.object.is((v) => !Array.isArray(v)));
const rl = new RequestList({
...options,
diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.js
index c722e684e..21e4ab72b 100644
--- a/src/session_pool/session_pool.js
+++ b/src/session_pool/session_pool.js
@@ -177,6 +177,7 @@ export class SessionPool extends EventEmitter {
* @param {Session|SessionOptions} [options] - The configuration options for the session being added to the session pool.
*/
async addSession(options = {}) {
+ this._throwIfNotInitialized();
const { id } = options;
if (id) {
const sessionExists = this.sessionMap.has(id);
@@ -207,6 +208,7 @@ export class SessionPool extends EventEmitter {
* @return {Promise<Session>}
*/
async getSession(sessionId) {
+ this._throwIfNotInitialized();
if (sessionId) {
const session = this.sessionMap.get(sessionId);
if (session && session.isUsable()) return session;
@@ -262,6 +264,16 @@ export class SessionPool extends EventEmitter {
await this.persistState();
}
+ /**
+ * SessionPool should not work before initialization.
+ * @ignore
+ * @protected
+ * @internal
+ */
+ _throwIfNotInitialized() {
+ if (!this._listener) throw new Error('SessionPool is not initialized.');
+ }
+
/**
* Removes retired `Session` instances from `SessionPool`.
* @ignore
diff --git a/test/request_list.test.js b/test/request_list.test.js
index 613ed87c5..535d6318c 100644
--- a/test/request_list.test.js
+++ b/test/request_list.test.js
@@ -7,16 +7,24 @@ import Apify from '../build/index';
import * as keyValueStore from '../build/storages/key_value_store';
import * as utils from '../build/utils';
import * as requestUtils from '../build/utils_request';
+import LocalStorageDirEmulator from './local_storage_dir_emulator';
describe('Apify.RequestList', () => {
let ll;
+ let localStorageEmulator;
beforeAll(() => {
ll = log.getLevel();
log.setLevel(log.LEVELS.ERROR);
+ localStorageEmulator = new LocalStorageDirEmulator();
});
- afterAll(() => {
+ beforeEach(async () => {
+ await localStorageEmulator.init();
+ });
+
+ afterAll(async () => {
log.setLevel(ll);
+ await localStorageEmulator.destroy();
});
test('should not accept to pages with same uniqueKey', async () => {
diff --git a/website/versioned_docs/version-0.22.4/typedefs/BrowserCrawlerOptions.md b/website/versioned_docs/version-0.22.4/typedefs/BrowserCrawlerOptions.md
deleted file mode 100644
index a3164e0c6..000000000
--- a/website/versioned_docs/version-0.22.4/typedefs/BrowserCrawlerOptions.md
+++ /dev/null
@@ -1,261 +0,0 @@
----
-id: version-0.22.4-browser-crawler-options
-title: BrowserCrawlerOptions
-original_id: browser-crawler-options
----
-
-<a name="browsercrawleroptions"></a>
-
-## Properties
-
-### `handlePageFunction`
-
-**Type**: `function`
-
-Function that is called to process each request. It is passed an object with the following fields:
-
-```
-{
- request: Request,
- response: Response,
- page: Page,
- session: Session,
- browserController: BrowserController,
- proxyInfo: ProxyInfo,
- crawler: BrowserCrawler,
-}
-```
-
-`request` is an instance of the [`Request`](../api/request) object with details about the URL to open, HTTP method etc. `page` is an instance of the
-`Puppeteer` [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) or `Playwright` [`Page`](https://playwright.dev/docs/api/class-page)
-`browserPool` is an instance of the [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool), `browserController` is an instance of the
-[`BrowserController`](https://github.com/apify/browser-pool#browsercontroller), `response` is an instance of the `Puppeteer`
-[`Response`](https://pptr.dev/#?product=Puppeteer&show=api-class-response) or `Playwright`
-[`Response`](https://playwright.dev/docs/api/class-response), which is the main resource response as returned by `page.goto(request.url)`. The
-function must return a promise, which is then awaited by the crawler.
-
-If the function throws an exception, the crawler will try to re-crawl the request later, up to `option.maxRequestRetries` times. If all the retries
-fail, the crawler calls the function provided to the `handleFailedRequestFunction` parameter. To make this work, you should **always** let your
-function throw exceptions rather than catch them. The exceptions are logged to the request using the
-[`Request.pushErrorMessage()`](../api/request#pusherrormessage) function.
-
----
-
-### `handlePageTimeoutSecs`
-
-**Type**: `number` <code> = 60</code>
-
-Timeout in which the function passed as `handlePageFunction` needs to finish, in seconds.
-
----
-
-### `gotoFunction`
-
-**Type**: `function`
-
-Navigation function for corresponding library. `page.goto(url)` is supported by both `playwright` and `puppeteer`.
-
----
-
-### `handleFailedRequestFunction`
-
-**Type**: [`HandleFailedRequest`](../typedefs/handle-failed-request)
-
-A function to handle requests that failed more than `option.maxRequestRetries` times.
-
-The function receives the following object as an argument:
-
-```
-{
- request: Request,
- response: Response,
- page: Page,
- browserPool: BrowserPool,
- autoscaledPool: AutoscaledPool,
- session: Session,
- browserController: BrowserController,
- proxyInfo: ProxyInfo,
-}
-```
-
-Where the [`Request`](../api/request) instance corresponds to the failed request, and the `Error` instance represents the last error thrown during
-processing of the request.
-
----
-
-### `browserPoolOptions`
-
-**Type**: `BrowserPoolOptions`
-
-Custom options passed to the underlying [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool) constructor. You can tweak those to
-fine-tune browser management.
-
----
-
-### `persistCookiesPerSession`
-
-**Type**: `boolean` <code> = false</code>
-
-Automatically saves cookies to Session. Works only if Session Pool is used.
-
----
-
-### `proxyConfiguration`
-
-**Type**: [`ProxyConfiguration`](../api/proxy-configuration)
-
-If set, `PuppeteerCrawler` will be configured for all connections to use [Apify Proxy](https://my.apify.com/proxy) or your own Proxy URLs provided and
-rotated according to the configuration. For more information, see the [documentation](https://docs.apify.com/proxy).
-
----
-
-### `preNavigationHooks`
-
-**Type**: `Array<function()>`
-
-Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies or browser properties before navigation.
-The function accepts two parameters, `crawlingContext` and `gotoOptions`, which are passed to the `gotoFunction` the crawler calls to navigate.
-Example:
-
-```
-preNavigationHooks: [
- async (crawlingContext, gotoOptions) => {
- await page.evaluate((attr) => { window.foo = attr; }, 'bar');
- }
-]
-```
-
----
-
-### `postNavigationHooks`
-
-**Type**: `Array<function()>`
-
-Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful. The function accepts
-`crawlingContext` as an only parameter. Example:
-
-```
-postNavigationHooks: [
- async (crawlingContext) => {
- const { page } = crawlingContext;
- if (hasCaptcha(page)) {
- await solveCaptcha (page);
- }
- };
-]
-```
-
----
-
-### `requestList`
-
-**Type**: [`RequestList`](../api/request-list)
-
-Static list of URLs to be processed. Either `requestList` or `requestQueue` option must be provided (or both).
-
----
-
-### `requestQueue`
-
-**Type**: [`RequestQueue`](../api/request-queue)
-
-Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites. Either `requestList` or `requestQueue` option must be
-provided (or both).
-
----
-
-### `handleRequestTimeoutSecs`
-
-**Type**: `number` <code> = 60</code>
-
-Timeout in which the function passed as `handleRequestFunction` needs to finish, in seconds.
-
----
-
-### `handleFailedRequestFunction`
-
-**Type**: [`HandleFailedRequest`](../typedefs/handle-failed-request)
-
-A function to handle requests that failed more than `option.maxRequestRetries` times.
-
-The function receives the following object as an argument:
-
-```
-{
- request: Request,
- error: Error,
- session: Session,
- crawler: BrowserCrawler,
-}
-```
-
-where the [`Request`](../api/request) instance corresponds to the failed request, and the `Error` instance represents the last error thrown during
-processing of the request.
-
-See [source code](https://github.com/apify/apify-js/blob/master/src/crawlers/basic_crawler.js#L11) for the default implementation of this function.
-
----
-
-### `maxRequestRetries`
-
-**Type**: `number` <code> = 3</code>
-
-Indicates how many times the request is retried if
-[`BasicCrawlerOptions.handleRequestFunction`](../typedefs/basic-crawler-options#handlerequestfunction) fails.
-
----
-
-### `maxRequestsPerCrawl`
-
-**Type**: `number`
-
-Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached. Always set this value in order to prevent infinite
-loops in misconfigured crawlers. Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
-
----
-
-### `autoscaledPoolOptions`
-
-**Type**: [`AutoscaledPoolOptions`](../typedefs/autoscaled-pool-options)
-
-Custom options passed to the underlying [`AutoscaledPool`](../api/autoscaled-pool) constructor. Note that the `runTaskFunction` and
-`isTaskReadyFunction` options are provided by `BasicCrawler` and cannot be overridden. However, you can provide a custom implementation of
-`isFinishedFunction`.
-
----
-
-### `minConcurrency`
-
-**Type**: `number` <code> = 1</code>
-
-Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding [`AutoscaledPool`](../api/autoscaled-pool) option.
-
-_WARNING:_ If you set this value too high with respect to the available system memory and CPU, your crawler will run extremely slow or crash. If
-you're not sure, just keep the default value and the concurrency will scale up automatically.
-
----
-
-### `maxConcurrency`
-
-**Type**: `number` <code> = 1000</code>
-
-Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding [`AutoscaledPool`](../api/autoscaled-pool) option.
-
----
-
-### `useSessionPool`
-
-**Type**: `boolean` <code> = false</code>
-
-If set to true. Basic crawler will initialize the [`SessionPool`](../api/session-pool) with the corresponding `sessionPoolOptions`. The session
-instance will be than available in the `handleRequestFunction`.
-
----
-
-### `sessionPoolOptions`
-
-**Type**: [`SessionPoolOptions`](../typedefs/session-pool-options)
-
-The configuration options for [`SessionPool`](../api/session-pool) to use.
-
----