Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: drop `useSessionPool` option, make `SessionPool` required in all crawlers (#3478)

Jindřich Bär2e1e450d1cfc
diff --git a/docs/guides/proxy_management_session_cheerio.ts b/docs/guides/proxy_management_session_cheerio.ts
index bb19a5b88..440d10cf8 100644
--- a/docs/guides/proxy_management_session_cheerio.ts
+++ b/docs/guides/proxy_management_session_cheerio.ts
@@ -5,7 +5,6 @@ const proxyConfiguration = new ProxyConfiguration({
});
const crawler = new CheerioCrawler({
- useSessionPool: true,
persistCookiesPerSession: true,
proxyConfiguration,
// ...
diff --git a/docs/guides/proxy_management_session_http.ts b/docs/guides/proxy_management_session_http.ts
index c8c289de4..96d7e1058 100644
--- a/docs/guides/proxy_management_session_http.ts
+++ b/docs/guides/proxy_management_session_http.ts
@@ -5,7 +5,6 @@ const proxyConfiguration = new ProxyConfiguration({
});
const crawler = new HttpCrawler({
- useSessionPool: true,
persistCookiesPerSession: true,
proxyConfiguration,
// ...
diff --git a/docs/guides/proxy_management_session_jsdom.ts b/docs/guides/proxy_management_session_jsdom.ts
index 98e71d904..7bc94da7b 100644
--- a/docs/guides/proxy_management_session_jsdom.ts
+++ b/docs/guides/proxy_management_session_jsdom.ts
@@ -5,7 +5,6 @@ const proxyConfiguration = new ProxyConfiguration({
});
const crawler = new JSDOMCrawler({
- useSessionPool: true,
persistCookiesPerSession: true,
proxyConfiguration,
// ...
diff --git a/docs/guides/proxy_management_session_playwright.ts b/docs/guides/proxy_management_session_playwright.ts
index 70edcb79a..c3d22d55b 100644
--- a/docs/guides/proxy_management_session_playwright.ts
+++ b/docs/guides/proxy_management_session_playwright.ts
@@ -5,7 +5,6 @@ const proxyConfiguration = new ProxyConfiguration({
});
const crawler = new PlaywrightCrawler({
- useSessionPool: true,
persistCookiesPerSession: true,
proxyConfiguration,
// ...
diff --git a/docs/guides/proxy_management_session_puppeteer.ts b/docs/guides/proxy_management_session_puppeteer.ts
index fcd1e1442..a09e68b14 100644
--- a/docs/guides/proxy_management_session_puppeteer.ts
+++ b/docs/guides/proxy_management_session_puppeteer.ts
@@ -5,7 +5,6 @@ const proxyConfiguration = new ProxyConfiguration({
});
const crawler = new PuppeteerCrawler({
- useSessionPool: true,
persistCookiesPerSession: true,
proxyConfiguration,
// ...
diff --git a/docs/guides/session_management_basic.ts b/docs/guides/session_management_basic.ts
index 38b65ee9d..1b76177d8 100644
--- a/docs/guides/session_management_basic.ts
+++ b/docs/guides/session_management_basic.ts
@@ -7,8 +7,6 @@ const proxyConfiguration = new ProxyConfiguration({
});
const crawler = new BasicCrawler({
- // Activates the Session pool (default is true).
- useSessionPool: true,
// Overrides default Session pool configuration.
sessionPoolOptions: { maxPoolSize: 100 },
async requestHandler({ request, session }) {
diff --git a/docs/guides/session_management_cheerio.ts b/docs/guides/session_management_cheerio.ts
index 7f8b2f90a..c819b4dd1 100644
--- a/docs/guides/session_management_cheerio.ts
+++ b/docs/guides/session_management_cheerio.ts
@@ -7,8 +7,6 @@ const proxyConfiguration = new ProxyConfiguration({
const crawler = new CheerioCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxyConfiguration,
- // Activates the Session pool (default is true).
- useSessionPool: true,
// Overrides default Session pool configuration.
sessionPoolOptions: { maxPoolSize: 100 },
// Set to true if you want the crawler to save cookies per session,
diff --git a/docs/guides/session_management_http.ts b/docs/guides/session_management_http.ts
index 9c684bcb0..5a556ea73 100644
--- a/docs/guides/session_management_http.ts
+++ b/docs/guides/session_management_http.ts
@@ -7,8 +7,6 @@ const proxyConfiguration = new ProxyConfiguration({
const crawler = new HttpCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxyConfiguration,
- // Activates the Session pool (default is true).
- useSessionPool: true,
// Overrides default Session pool configuration.
sessionPoolOptions: { maxPoolSize: 100 },
// Set to true if you want the crawler to save cookies per session,
diff --git a/docs/guides/session_management_jsdom.ts b/docs/guides/session_management_jsdom.ts
index ef55b6632..fe68d7cea 100644
--- a/docs/guides/session_management_jsdom.ts
+++ b/docs/guides/session_management_jsdom.ts
@@ -7,8 +7,6 @@ const proxyConfiguration = new ProxyConfiguration({
const crawler = new JSDOMCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxyConfiguration,
- // Activates the Session pool (default is true).
- useSessionPool: true,
// Overrides default Session pool configuration.
sessionPoolOptions: { maxPoolSize: 100 },
// Set to true if you want the crawler to save cookies per session,
diff --git a/docs/guides/session_management_playwright.ts b/docs/guides/session_management_playwright.ts
index f4f2f7c80..aa9852290 100644
--- a/docs/guides/session_management_playwright.ts
+++ b/docs/guides/session_management_playwright.ts
@@ -7,8 +7,6 @@ const proxyConfiguration = new ProxyConfiguration({
const crawler = new PlaywrightCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxyConfiguration,
- // Activates the Session pool (default is true).
- useSessionPool: true,
// Overrides default Session pool configuration
sessionPoolOptions: { maxPoolSize: 100 },
// Set to true if you want the crawler to save cookies per session,
diff --git a/docs/guides/session_management_puppeteer.ts b/docs/guides/session_management_puppeteer.ts
index 63b342146..7504e0c6c 100644
--- a/docs/guides/session_management_puppeteer.ts
+++ b/docs/guides/session_management_puppeteer.ts
@@ -7,8 +7,6 @@ const proxyConfiguration = new ProxyConfiguration({
const crawler = new PuppeteerCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxyConfiguration,
- // Activates the Session pool (default is true).
- useSessionPool: true,
// Overrides default Session pool configuration
sessionPoolOptions: { maxPoolSize: 100 },
// Set to true if you want the crawler to save cookies per session,
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 211169937..93f999559 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -312,12 +312,6 @@ export interface BasicCrawlerOptions<
*/
keepAlive?: boolean;
- /**
- * Basic crawler will initialize the {@apilink SessionPool} with the corresponding {@apilink SessionPoolOptions|`sessionPoolOptions`}.
- * The session instance will be than available in the {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}.
- */
- useSessionPool?: boolean;
-
/**
* The configuration options for {@apilink SessionPool} to use.
*/
@@ -647,7 +641,6 @@ export class BasicCrawler<
protected statusMessageLoggingInterval: number;
protected statusMessageCallback?: StatusMessageCallback;
protected sessionPoolOptions: SessionPoolOptions;
- protected useSessionPool: boolean;
protected autoscaledPoolOptions: AutoscaledPoolOptions;
protected httpClient: BaseHttpClient;
protected retryOnBlocked: boolean;
@@ -685,7 +678,6 @@ export class BasicCrawler<
maxCrawlDepth: ow.optional.number,
autoscaledPoolOptions: ow.optional.object,
sessionPoolOptions: ow.optional.object,
- useSessionPool: ow.optional.boolean,
proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
statusMessageLoggingInterval: ow.optional.number,
@@ -736,7 +728,6 @@ export class BasicCrawler<
autoscaledPoolOptions = {},
keepAlive,
sessionPoolOptions = {},
- useSessionPool = true,
proxyConfiguration,
// Service locator options
@@ -870,8 +861,6 @@ export class BasicCrawler<
);
}
}
- this.useSessionPool = useSessionPool;
-
const maxSignedInteger = 2 ** 31 - 1;
if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
this.log.warning(
@@ -1082,25 +1071,23 @@ export class BasicCrawler<
}
private async resolveSession({ request }: { request: Request }) {
- const session = this.useSessionPool
- ? await this._timeoutAndRetry(
- async () => {
- return await this.sessionPool!.newSession({
- proxyInfo: await this.proxyConfiguration?.newProxyInfo({
- request: request ?? undefined,
- }),
- maxUsageCount: 1,
- });
- },
- this.internalTimeoutMillis,
- `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,
- )
- : undefined;
-
- return { session, proxyInfo: session?.proxyInfo };
+ const session = await this._timeoutAndRetry(
+ async () => {
+ return await this.sessionPool!.newSession({
+ proxyInfo: await this.proxyConfiguration?.newProxyInfo({
+ request: request ?? undefined,
+ }),
+ maxUsageCount: 1,
+ });
+ },
+ this.internalTimeoutMillis,
+ `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,
+ );
+
+ return { session, proxyInfo: session.proxyInfo };
}
- private async createContextHelpers({ request, session }: { request: Request; session?: Session }) {
+ private async createContextHelpers({ request, session }: { request: Request; session: Session }) {
const enqueueLinksWrapper: CrawlingContext['enqueueLinks'] = async (options) => {
const requestQueue = await this.getRequestQueue();
@@ -1700,11 +1687,8 @@ export class BasicCrawler<
// (otherwise there would be no way)
this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);
- if (this.useSessionPool) {
- this.sessionPool = await SessionPool.open(this.sessionPoolOptions);
- // Assuming there are not more than 20 browsers running at once;
- this.sessionPool.setMaxListeners(20);
- }
+ this.sessionPool = await SessionPool.open(this.sessionPoolOptions);
+ this.sessionPool.setMaxListeners(20);
await this.initializeRequestManager();
}
@@ -1904,7 +1888,7 @@ export class BasicCrawler<
// reclaim session if request finishes successfully
request.state = RequestState.DONE;
- crawlingContext.session?.markGood();
+ crawlingContext.session.markGood();
} catch (rawError) {
const err = this.unwrapError(rawError);
@@ -1942,7 +1926,7 @@ export class BasicCrawler<
throw unwrappedSecondaryError;
}
// decrease the session score if the request fails (but the error handler did not throw)
- crawlingContext.session?.markBad();
+ crawlingContext.session.markBad();
} finally {
// Safety net - release the lock if nobody managed to do it before
if (isRequestLocked && requestSource instanceof RequestProvider) {
@@ -2086,7 +2070,7 @@ export class BasicCrawler<
request.sessionRotationCount ??= 0;
request.sessionRotationCount++;
- crawlingContext.session?.retire();
+ crawlingContext.session.retire();
}
/**
@@ -2273,9 +2257,8 @@ export class BasicCrawler<
serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });
}
- if (this.useSessionPool) {
- await this.sessionPool!.teardown({ persistState: this._ownsEventManager });
- }
+ await this.sessionPool?.teardown({ persistState: this._ownsEventManager });
+
if (this._ownsEventManager) {
await serviceLocator.getEventManager().close();
@@ -2358,7 +2341,7 @@ export class BasicCrawler<
export interface CreateContextOptions {
request: Request;
- session?: Session;
+ session: Session;
proxyInfo?: ProxyInfo;
}
diff --git a/packages/basic-crawler/src/internals/send-request.ts b/packages/basic-crawler/src/internals/send-request.ts
index 4e826ef05..26d0b7a4d 100644
--- a/packages/basic-crawler/src/internals/send-request.ts
+++ b/packages/basic-crawler/src/internals/send-request.ts
@@ -9,11 +9,7 @@ import type { BaseHttpClient, HttpRequestOptions, SendRequestOptions } from '@cr
* @param originRequest The crawling request being processed.
* @param session The user session associated with the current request.
*/
-export function createSendRequest(
- httpClient: BaseHttpClient,
- originRequest: CrawleeRequest,
- session: Session | undefined,
-) {
+export function createSendRequest(httpClient: BaseHttpClient, originRequest: CrawleeRequest, session: Session) {
return async (
overrideRequest: Partial<HttpRequestOptions> = {},
overrideOptions: SendRequestOptions = {},
@@ -37,7 +33,7 @@ export function createSendRequest(
return httpClient.sendRequest(request, {
session,
- cookieJar: overrideOptions?.cookieJar ?? session?.cookieJar,
+ cookieJar: overrideOptions?.cookieJar ?? session.cookieJar,
timeoutMillis: overrideOptions.timeoutMillis,
signal: overrideOptions.signal,
});
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 04dc6909b..9a12b1a31 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -222,8 +222,7 @@ export interface BrowserCrawlerOptions<
navigationTimeoutSecs?: number;
/**
- * Defines whether the cookies should be persisted for sessions.
- * This can only be used when `useSessionPool` is set to `true`.
+ * Defines whether the cookies should be persisted for sessions. Enabled by default.
*/
persistCookiesPerSession?: boolean;
@@ -328,7 +327,6 @@ export abstract class BrowserCrawler<
browserPoolOptions: ow.object,
sessionPoolOptions: ow.optional.object,
persistCookiesPerSession: ow.optional.boolean,
- useSessionPool: ow.optional.boolean,
proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
};
@@ -350,7 +348,7 @@ export abstract class BrowserCrawler<
ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape));
const {
navigationTimeoutSecs = 60,
- persistCookiesPerSession,
+ persistCookiesPerSession = true,
launchContext = {},
browserPoolOptions,
preNavigationHooks = [],
@@ -374,11 +372,6 @@ export abstract class BrowserCrawler<
extendContext: extendContext as (context: Context) => Awaitable<ContextExtension>,
});
- // Cookies should be persisted per session only if session pool is used
- if (!this.useSessionPool && persistCookiesPerSession) {
- throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
- }
-
this.launchContext = launchContext;
this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
this.proxyConfiguration = proxyConfiguration;
@@ -392,11 +385,7 @@ export abstract class BrowserCrawler<
(this.launchContext.launchOptions as Dictionary).headless = headless;
}
- if (this.useSessionPool) {
- this.persistCookiesPerSession = persistCookiesPerSession !== undefined ? persistCookiesPerSession : true;
- } else {
- this.persistCookiesPerSession = false;
- }
+ this.persistCookiesPerSession = persistCookiesPerSession;
if (launchContext?.userAgent) {
if (browserPoolOptions.useFingerprints)
@@ -716,6 +705,8 @@ export abstract class BrowserCrawler<
protected async _extendLaunchContext(_pageId: string, launchContext: LaunchContext): Promise<void> {
const launchContextExtends: { session?: Session; proxyInfo?: ProxyInfo } = {};
+ // Hacky access from `AdaptivePlaywrightCrawler` calls this without calling `.init()`.
+ // This is the only case where this.sessionPool is accessed without being initialized.
if (this.sessionPool) {
launchContextExtends.session = await this.sessionPool.newSession({
proxyInfo: await this.proxyConfiguration?.newProxyInfo({
diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.ts
index 6b29864ea..aec214e59 100644
--- a/packages/core/src/crawlers/crawler_commons.ts
+++ b/packages/core/src/crawlers/crawler_commons.ts
@@ -28,7 +28,7 @@ export type LoadedContext<Context extends RestrictedCrawlingContext> =
export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> {
id: string;
- session?: Session;
+ session: Session;
/**
* An object with information about currently used proxy by the crawler
diff --git a/packages/core/src/session_pool/session_pool.ts b/packages/core/src/session_pool/session_pool.ts
index 673779dfe..3c8210e31 100644
--- a/packages/core/src/session_pool/session_pool.ts
+++ b/packages/core/src/session_pool/session_pool.ts
@@ -75,18 +75,7 @@ export interface SessionPoolOptions {
*
* You can create one by calling the {@apilink SessionPool.open} function.
*
- * Session pool is already integrated into crawlers, and it can significantly improve your scraper
- * performance with just 2 lines of code.
- *
- * **Example usage:**
- *
- * ```javascript
- * const crawler = new CheerioCrawler({
- * useSessionPool: true,
- * persistCookiesPerSession: true,
- * // ...
- * })
- * ```
+ * Session pool is already integrated into crawlers and is always active.
*
* You can configure the pool with many options. See the {@apilink SessionPoolOptions}.
* Session pool is by default persisted in default {@apilink KeyValueStore}.
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index ad73a5fa1..8cf405b9b 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -140,7 +140,7 @@ export interface HttpCrawlerOptions<
forceResponseEncoding?: string;
/**
- * Automatically saves cookies to Session. Works only if Session Pool is used.
+ * Automatically saves cookies to Session. Enabled by default.
*
* It parses cookie from response "set-cookie" header saves or updates cookies for session and once the session is used for next request.
* It passes the "Cookie" header to the request with the session cookies.
@@ -361,7 +361,7 @@ export class HttpCrawler<
additionalMimeTypes = [],
suggestResponseEncoding,
forceResponseEncoding,
- persistCookiesPerSession,
+ persistCookiesPerSession = true,
preNavigationHooks = [],
postNavigationHooks = [],
additionalHttpErrorStatusCodes = [],
@@ -381,11 +381,6 @@ export class HttpCrawler<
(() => this.buildContextPipeline() as ContextPipeline<CrawlingContext, Context>),
});
- // Cookies should be persisted per session only if session pool is used
- if (!this.useSessionPool && persistCookiesPerSession) {
- throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
- }
-
this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
if (additionalMimeTypes.length) this._extendSupportedMimeTypes(additionalMimeTypes);
@@ -407,11 +402,7 @@ export class HttpCrawler<
...postNavigationHooks,
];
- if (this.useSessionPool) {
- this.persistCookiesPerSession = persistCookiesPerSession ?? true;
- } else {
- this.persistCookiesPerSession = false;
- }
+ this.persistCookiesPerSession = persistCookiesPerSession;
}
protected override buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext> {
@@ -523,12 +514,10 @@ export class HttpCrawler<
return $;
};
- if (this.useSessionPool) {
- this._throwOnBlockedRequest(crawlingContext.session!, response.status!);
- }
+ this._throwOnBlockedRequest(crawlingContext.session, response.status!);
if (this.persistCookiesPerSession) {
- crawlingContext.session!.setCookiesFromResponse(response);
+ crawlingContext.session.setCookiesFromResponse(response);
}
return {
@@ -586,7 +575,7 @@ export class HttpCrawler<
preHookCookies: string,
postHookCookies: string,
): string {
- const sessionCookie = session?.getCookieString(request.url) ?? '';
+ const sessionCookie = session.getCookieString(request.url);
const sourceCookies = [sessionCookie, preHookCookies, postHookCookies];
@@ -671,13 +660,13 @@ export class HttpCrawler<
/**
* Combines the provided `requestOptions` with mandatory (non-overridable) values.
*/
- protected _getRequestOptions(request: CrawleeRequest, session?: Session, proxyUrl?: string) {
+ protected _getRequestOptions(request: CrawleeRequest, session: Session, proxyUrl?: string) {
const requestOptions = {
url: request.url,
method: request.method,
proxyUrl,
timeout: this.navigationTimeoutMillis,
- cookieJar: this.persistCookiesPerSession ? session?.cookieJar : undefined,
+ cookieJar: this.persistCookiesPerSession ? session.cookieJar : undefined,
sessionToken: session,
headers: request.headers,
https: {
@@ -690,7 +679,7 @@ export class HttpCrawler<
Reflect.deleteProperty(requestOptions.headers!, 'cookie');
// Disable SSL verification for MITM proxies
- if (session?.proxyInfo?.ignoreTlsErrors) {
+ if (session.proxyInfo?.ignoreTlsErrors) {
requestOptions.https = {
...requestOptions.https,
rejectUnauthorized: false,
@@ -769,8 +758,8 @@ export class HttpCrawler<
/**
* Handles timeout request
*/
- protected _handleRequestTimeout(session?: Session) {
- session?.markBad();
+ protected _handleRequestTimeout(session: Session) {
+ session.markBad();
throw new Error(`request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
}
@@ -795,7 +784,7 @@ export class HttpCrawler<
/**
* @internal wraps public utility for mocking purposes
*/
- private _requestAsBrowser = async (options: Dictionary<any>, session?: Session, cookieString?: string) => {
+ private _requestAsBrowser = async (options: Dictionary<any>, session: Session, cookieString?: string) => {
const opts = processHttpRequestOptions({
...(options as any),
cookieJar: options.cookieJar,
@@ -828,7 +817,7 @@ export class HttpCrawler<
interface RequestFunctionOptions {
request: CrawleeRequest;
- session?: Session;
+ session: Session;
proxyUrl?: string;
cookieString?: string;
}
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index 0b390b2f6..bbb3e6f12 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -361,7 +361,6 @@ export class AdaptivePlaywrightCrawler<
}
const staticCrawler = new CheerioCrawler({
...rest,
- useSessionPool: false,
statisticsOptions: {
persistenceOptions: { enable: false },
},
@@ -383,7 +382,6 @@ export class AdaptivePlaywrightCrawler<
const browserCrawler = new PlaywrightCrawler({
...rest,
- useSessionPool: false,
statisticsOptions: {
persistenceOptions: { enable: false },
},
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index d039a30f9..5b9d2e862 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -1478,17 +1478,12 @@ describe('BasicCrawler', () => {
it('persists statistics and the session pool once when finishing', async () => {
const persistStatistics = vitest.spyOn(Statistics.prototype, 'persistState');
const persistSessionPool = vitest.spyOn(SessionPool.prototype, 'persistState');
- const config = new Configuration();
- const crawler = new BasicCrawler(
- {
- useSessionPool: true,
- requestHandler: async () => {
- persistStatistics.mockClear();
- persistSessionPool.mockClear();
- },
+ const crawler = new BasicCrawler({
+ requestHandler: async () => {
+ persistStatistics.mockClear();
+ persistSessionPool.mockClear();
},
- config,
- );
+ });
await crawler.run(['https://example.com']);
@@ -1499,29 +1494,24 @@ describe('BasicCrawler', () => {
it('persists the session pool once with a shared event manager', async () => {
const persistStatistics = vitest.spyOn(Statistics.prototype, 'persistState');
const persistSessionPool = vitest.spyOn(SessionPool.prototype, 'persistState');
- const config = new Configuration();
- await config.getEventManager().init();
- const crawler = new BasicCrawler(
- {
- useSessionPool: true,
- requestHandler: async () => {
- persistStatistics.mockClear();
- persistSessionPool.mockClear();
- },
+ await serviceLocator.getEventManager().init();
+ const crawler = new BasicCrawler({
+ requestHandler: async () => {
+ persistStatistics.mockClear();
+ persistSessionPool.mockClear();
},
- config,
- );
+ });
await crawler.run(['https://example.com']);
- expect(config.getEventManager().listenerCount(EventType.PERSIST_STATE)).toBe(0);
+ expect(serviceLocator.getEventManager().listenerCount(EventType.PERSIST_STATE)).toBe(0);
expect(persistSessionPool).toHaveBeenCalledTimes(1);
- await config.getEventManager().close();
+ await serviceLocator.getEventManager().close();
expect(persistStatistics).toHaveBeenCalledTimes(1);
expect(persistSessionPool).toHaveBeenCalledTimes(1);
});
- it('should use SessionPool when useSessionPool is true ', async () => {
+ it('should use SessionPool', async () => {
const url = 'https://example.com';
const requestList = await RequestList.open({ sources: [{ url }] });
const results: Request[] = [];
@@ -1530,14 +1520,13 @@ describe('BasicCrawler', () => {
requestList,
requestHandlerTimeoutSecs: 0.01,
maxRequestRetries: 1,
- useSessionPool: true,
sessionPoolOptions: {
maxPoolSize: 10,
persistStateKey: 'POOL',
},
requestHandler: async ({ session }) => {
- expect(session!.constructor.name).toEqual('Session');
- expect(session!.id).toBeDefined();
+ expect(session.constructor.name).toEqual('Session');
+ expect(session.id).toBeDefined();
},
failedRequestHandler: async ({ request }) => {
results.push(request);
@@ -1557,7 +1546,6 @@ describe('BasicCrawler', () => {
requestList,
requestHandlerTimeoutSecs: 0.01,
maxRequestRetries: 1,
- useSessionPool: true,
sessionPoolOptions: {
maxPoolSize: 10,
persistStateKey: 'POOL',
@@ -1580,7 +1568,6 @@ describe('BasicCrawler', () => {
requestList,
requestHandlerTimeoutSecs: 0.01,
maxRequestRetries: 1,
- useSessionPool: true,
sessionPoolOptions: {
maxPoolSize: 10,
},
@@ -1647,35 +1634,6 @@ describe('BasicCrawler', () => {
const requestList = await RequestList.open(null, [url]);
const crawler = new BasicCrawler({
- useSessionPool: true,
- requestList,
- async requestHandler({ sendRequest }) {
- const response = await sendRequest();
-
- responses.push({
- statusCode: response.status,
- body: await response.text(),
- });
- },
- });
-
- await crawler.run();
-
- expect(responses).toStrictEqual([
- {
- statusCode: 200,
- body: html,
- },
- ]);
- });
-
- test('works without session', async () => {
- const requestList = await RequestList.open(null, [url]);
-
- const responses: { statusCode: number; body: string }[] = [];
-
- const crawler = new BasicCrawler({
- useSessionPool: false,
requestList,
async requestHandler({ sendRequest }) {
const response = await sendRequest();
@@ -1699,7 +1657,6 @@ describe('BasicCrawler', () => {
test('proxyUrl TypeScript support', async () => {
const crawler = new BasicCrawler({
- useSessionPool: true,
async requestHandler({ sendRequest }) {
await sendRequest({
proxyUrl: 'http://example.com',
diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts
index cd3018f08..9596bfd24 100644
--- a/test/core/crawlers/browser_crawler.test.ts
+++ b/test/core/crawlers/browser_crawler.test.ts
@@ -116,7 +116,7 @@ describe('BrowserCrawler', () => {
browserPlugins: [puppeteerPlugin],
},
requestList,
- useSessionPool: true,
+
requestHandler: async () => {},
maxRequestRetries: 1,
});
@@ -165,7 +165,7 @@ describe('BrowserCrawler', () => {
browserPlugins: [puppeteerPlugin],
},
requestList,
- useSessionPool: true,
+
requestHandler: async () => {},
maxRequestRetries: 1,
});
@@ -196,7 +196,7 @@ describe('BrowserCrawler', () => {
browserPlugins: [puppeteerPlugin],
},
requestList,
- useSessionPool: true,
+
requestHandler: async () => {},
maxRequestRetries: 0,
preNavigationHooks: [hook],
@@ -229,7 +229,7 @@ describe('BrowserCrawler', () => {
browserPlugins: [puppeteerPlugin],
},
requestList,
- useSessionPool: true,
+
requestHandler: async () => {},
maxRequestRetries: 0,
postNavigationHooks: [hook],
@@ -389,7 +389,7 @@ describe('BrowserCrawler', () => {
browserPlugins: [puppeteerPlugin],
},
requestList,
- useSessionPool: true,
+
requestHandler: async () => {},
maxRequestRetries: 0,
preNavigationHooks: [
@@ -496,7 +496,7 @@ describe('BrowserCrawler', () => {
browserPlugins: [puppeteerPlugin],
},
requestList,
- useSessionPool: false,
+
requestHandler: async () => {},
});
@@ -521,7 +521,7 @@ describe('BrowserCrawler', () => {
browserPoolOptions: {
browserPlugins: [puppeteerPlugin],
},
- useSessionPool: true,
+
persistCookiesPerSession: false,
sessionPoolOptions: {
sessionOptions: {
@@ -563,10 +563,9 @@ describe('BrowserCrawler', () => {
browserPlugins: [puppeteerPlugin],
},
requestList,
- useSessionPool: true,
persistCookiesPerSession: true,
requestHandler: async ({ session, request }) => {
- loadedCookies.push(session!.getCookieString(request.url));
+ loadedCookies.push(session.getCookieString(request.url));
return Promise.resolve();
},
preNavigationHooks: [
@@ -621,7 +620,7 @@ describe('BrowserCrawler', () => {
browserPlugins: [puppeteerPlugin],
},
requestList,
- useSessionPool: true,
+
persistCookiesPerSession: false,
maxRequestRetries: 0,
requestHandler: async () => {
@@ -756,7 +755,7 @@ describe('BrowserCrawler', () => {
browserPlugins: [puppeteerPlugin],
},
requestList,
- useSessionPool: true,
+
persistCookiesPerSession: false,
maxRequestRetries: 0,
requestHandler: async () => {
@@ -816,7 +815,7 @@ describe('BrowserCrawler', () => {
browserPlugins: [puppeteerPlugin],
},
requestList,
- useSessionPool: true,
+
requestHandler: async () => {
await retirementPromise;
},
@@ -843,7 +842,7 @@ describe('BrowserCrawler', () => {
browserPoolOptions: {
browserPlugins: [puppeteerPlugin],
},
- useSessionPool: true,
+
sessionPoolOptions: {
maxPoolSize: 1,
},
@@ -887,7 +886,7 @@ describe('BrowserCrawler', () => {
},
},
requestList,
- useSessionPool: false,
+
requestHandler: async ({ browserController }) => {
expect(browserController.launchContext.fingerprint).toBeDefined();
},
@@ -1077,7 +1076,7 @@ describe('BrowserCrawler', () => {
requestList,
maxRequestRetries: 0,
maxConcurrency: 1,
- useSessionPool: true,
+
proxyConfiguration,
requestHandler,
});
@@ -1257,7 +1256,7 @@ describe('BrowserCrawler', () => {
requestList,
maxRequestRetries: 0,
maxConcurrency: 1,
- useSessionPool: true,
+
requestHandler,
failedRequestHandler,
});
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index aa51a4f40..856f5d72e 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -738,7 +738,6 @@ describe('CheerioCrawler', () => {
requestList,
requestHandler,
proxyConfiguration,
- useSessionPool: true,
});
await crawler.run();
@@ -772,7 +771,7 @@ describe('CheerioCrawler', () => {
})({
maxSessionRotations: 2,
maxConcurrency: 1,
- useSessionPool: true,
+
proxyConfiguration,
requestHandler: () => {},
});
@@ -856,7 +855,7 @@ describe('CheerioCrawler', () => {
expect.assertions(1);
const crawler = new CheerioCrawler({
requestList,
- useSessionPool: true,
+
persistCookiesPerSession: false,
requestHandler: ({ session }) => {
expect(session).toBeInstanceOf(Session);
@@ -868,7 +867,7 @@ describe('CheerioCrawler', () => {
test('should correctly set session pool options', async () => {
const crawler = new CheerioCrawler({
requestList,
- useSessionPool: true,
+
persistCookiesPerSession: false,
sessionPoolOptions: {
sessionOptions: {
@@ -894,7 +893,7 @@ describe('CheerioCrawler', () => {
maxRequestRetries: 1,
navigationTimeoutSecs: 1,
maxConcurrency: 1,
- useSessionPool: true,
+
requestHandler: async () => {
await sleep(1);
},
@@ -930,7 +929,7 @@ describe('CheerioCrawler', () => {
error: false,
headers: { 'Content-type': 'text/html' },
}),
- useSessionPool: true,
+
persistCookiesPerSession: false,
maxRequestRetries: 0,
requestHandler: ({ session }) => {
@@ -960,23 +959,6 @@ describe('CheerioCrawler', () => {
}
});
- test('should throw when "options.useSessionPool" false and "options.persistCookiesPerSession" is true', async () => {
- try {
- // eslint-disable-next-line no-new
- new CheerioCrawler({
- requestList: await getRequestListForMock({}),
- useSessionPool: false,
- persistCookiesPerSession: true,
- maxRequestRetries: 0,
- requestHandler: () => {},
- });
- } catch (e) {
- expect((e as Error).message).toEqual(
- 'You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.',
- );
- }
- });
-
test('should send cookies', async () => {
const cookie = 'SESSID=abcd123';
const requests: Request[] = [];
@@ -988,7 +970,7 @@ describe('CheerioCrawler', () => {
},
'/getRawHeaders',
),
- useSessionPool: true,
+
persistCookiesPerSession: true,
sessionPoolOptions: {
maxPoolSize: 1,
@@ -1019,7 +1001,7 @@ describe('CheerioCrawler', () => {
headers: { cookie: 'foo=bar2; baz=123' },
},
]),
- useSessionPool: true,
+
persistCookiesPerSession: false,
sessionPoolOptions: {
maxPoolSize: 1,
@@ -1049,7 +1031,7 @@ describe('CheerioCrawler', () => {
headers: { cookie: 'foo=bar2; baz=123' },
},
]),
- useSessionPool: true,
+
persistCookiesPerSession: false,
sessionPoolOptions: {
maxPoolSize: 1,
@@ -1084,7 +1066,7 @@ describe('CheerioCrawler', () => {
url: `${serverAddress}/special/headers`,
},
]),
- useSessionPool: true,
+
requestHandler: async ({ json, request }) => {
responses.push(json);
requests.push(request);
@@ -1159,7 +1141,7 @@ describe('CheerioCrawler', () => {
const crawler = new CheerioCrawler({
requestList: await RequestList.open(null, [{ url: `${serverAddress}/special/get-cookies` }]),
- useSessionPool: true,
+
sessionPoolOptions: {
// Even with multiple available sessions, the preNavigationHook should use the same one as the main request
maxPoolSize: 10,
@@ -1243,7 +1225,7 @@ describe('CheerioCrawler', () => {
requestList,
maxRequestRetries: 0,
maxConcurrency: 1,
- useSessionPool: true,
+
preNavigationHooks: [preNavigationHook],
requestHandler,
failedRequestHandler,
diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.ts
index e2b869e22..db36c0ac3 100644
--- a/test/core/crawlers/puppeteer_crawler.test.ts
+++ b/test/core/crawlers/puppeteer_crawler.test.ts
@@ -335,7 +335,7 @@ describe('PuppeteerCrawler', () => {
const puppeteerCrawler = new PuppeteerCrawler({
requestList,
- useSessionPool: true,
+
persistCookiesPerSession: true,
sessionPoolOptions: {
createSessionFunction: (sessionPool) => {
@@ -420,7 +420,7 @@ describe('PuppeteerCrawler', () => {
const puppeteerCrawler = new PuppeteerCrawler({
requestList: requestListLarge,
- useSessionPool: true,
+
launchContext: {
useIncognitoPages: true,
},