Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: export `SessionPool` interface, minimize `ISession`, allow custom implementations (#3644)
Jindřich Bärdcf8d6bcb62f
diff --git a/docs/guides/session_management_basic.ts b/docs/guides/session_management_basic.tsindex 3a48a36c4..948f75ea0 100644--- a/docs/guides/session_management_basic.ts+++ b/docs/guides/session_management_basic.ts@@ -17,7 +17,7 @@ const crawler = new BasicCrawler({headers: {// If you want to use the cookieJar.// This way you get the Cookie headers string from session.- Cookie: session?.getCookieString(url) ?? '',+ Cookie: session?.cookieJar.getCookieStringSync(url) ?? '',},});let response;diff --git a/docs/guides/session_management_standalone.ts b/docs/guides/session_management_standalone.tsindex 5062a8a87..8ac133d95 100644--- a/docs/guides/session_management_standalone.ts+++ b/docs/guides/session_management_standalone.ts@@ -11,10 +11,10 @@ const sessionPool = new SessionPool(sessionPoolOptions);const session = await sessionPool.getSession();// Increase the errorScore.-session.markBad();+session?.markBad();// Throw away the session.-session.retire();+session?.retire();// Lower the errorScore and mark the session good.-session.markGood();+session?.markGood();diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex e821304de..e2ca2c4a1 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -136,10 +136,63 @@ new SessionPool({## `Session` no longer requires a `sessionPool` reference-`Session` no longer holds a back-reference to its `SessionPool` and no longer emits a `sessionRetired` event when retired. The `sessionPool` constructor option is gone, `SessionPool` is no longer an `EventEmitter`, and the `EVENT_SESSION_RETIRED` constant is no longer exported.+`Session` no longer holds a back-reference to its `SessionPool` and no longer emits a `sessionRetired` event when retired. The `sessionPool` constructor option is gone, `SessionPool` is no longer an `EventEmitter`, and the `EVENT_SESSION_RETIRED` constant is no longer exported. Custom `createSessionFunction` implementations that constructed `Session` instances manually should drop the `sessionPool` argument.++**Before:**+```typescript+new SessionPool({+ createSessionFunction: async (pool, opts) =>+ new Session({ ...opts?.sessionOptions, sessionPool: pool }),+});+```++**After:**+```typescript+new SessionPool({+ createSessionFunction: async (opts) =>+ new Session({ ...opts?.sessionOptions }),+});+```If you previously subscribed to `sessionRetired` on the pool to clean up resources tied to a session, perform the cleanup at the end of your request handler (or via a context-pipeline cleanup hook) by checking `session.isUsable()` instead. `Session.retire()` is now a terminal state — once retired, `isUsable()` returns `false` permanently and cannot be undone by a subsequent `markGood()`.+## Custom `SessionPool` implementations via the `ISessionPool` interface++Crawlers now accept any object implementing the new `ISessionPool` interface as their `sessionPool` option, not just instances of the built-in `SessionPool`. The contract is intentionally tiny — a single method, `getSession()` / `getSession(id)`, that hands out a `Session` for a request. Lifecycle (reset, teardown) is the responsibility of whoever owns the pool: a custom pool you construct yourself is never owned by the crawler, so the crawler never tears it down. This makes it straightforward to plug in a remote, shared, or otherwise customized session-management strategy without subclassing `SessionPool` or copying its internals.++```typescript+import { BasicCrawler, Session, type ISessionPool } from '@crawlee/core';++class MySessionPool implements ISessionPool {+ private readonly sessions = new Map<string, Session>();++ async getSession(): Promise<Session>;+ async getSession(sessionId: string): Promise<Session | undefined>;+ async getSession(sessionId?: string): Promise<Session | undefined> {+ if (sessionId) {+ const existing = this.sessions.get(sessionId);+ return existing?.isUsable() ? existing : undefined;+ }++ const usable = [...this.sessions.values()].find((s) => s.isUsable());+ if (usable) return usable;++ const fresh = new Session();+ this.sessions.set(fresh.id, fresh);+ return fresh;+ }+}++const crawler = new BasicCrawler({+ sessionPool: new MySessionPool(),+ requestHandler: async ({ session }) => {+ // session is a Session instance, use it as usual+ },+});+```++The returned objects must be `Session` instances — the rest of the crawler relies on `session.markGood()`, `session.cookieJar`, `session.proxyInfo`, and the rest of the concrete `Session` API.+## `retireOnBlockedStatusCodes` is removed from `Session``Session.retireOnBlockedStatusCodes` is removed. Blocked status code handling is now internal to the crawler. Configure blocked status codes via the `blockedStatusCodes` crawler option (moved from `sessionPoolOptions`).diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 37e6b7346..16ac49615 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -72,6 +72,8 @@ import type {BaseHttpClient,BatchAddRequestsResult,Dictionary,+ ISession,+ ISessionPool,ProxyInfo,SetStatusMessageOptions,StorageClient,@@ -304,11 +306,14 @@ export interface BasicCrawlerOptions<keepAlive?: boolean;/**- * An existing {@apilink SessionPool} instance to use. When provided, the crawler will use this- * pool directly instead of creating a new one, enabling session sharing across multiple crawlers.- * The crawler will not tear down a shared pool — the caller is responsible for its lifecycle.+ * An existing session pool instance to use. When provided, the crawler will use this pool directly instead of+ * creating a new one, enabling session sharing across multiple crawlers. The crawler will not tear down a shared+ * pool — the caller is responsible for its lifecycle.+ *+ * Accepts the built-in {@apilink SessionPool} or any object implementing the {@apilink ISessionPool} interface,+ * so custom session-management strategies can be plugged in.*/- sessionPool?: SessionPool;+ sessionPool?: ISessionPool;/*** Defines the length of the interval for calling the `setStatusMessage` in seconds.@@ -561,14 +566,18 @@ export class BasicCrawler<protected requestManager?: IRequestManager;/**- * A reference to the underlying {@apilink SessionPool} class that manages the crawler's {@apilink Session|sessions}.+ * A reference to the underlying session pool that manages the crawler's {@apilink Session|sessions}. Typed as+ * {@apilink ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.*/- sessionPool: SessionPool;+ sessionPool: ISessionPool;/**- * Indicates whether the crawler owns the session pool (it was not passed from the outside using the `sessionPool` constructor option).+ * Set when the crawler constructed its own {@apilink SessionPool} (no `sessionPool` option was provided).+ * Holds the same instance as `sessionPool`, but typed as the concrete class so the crawler can call+ * lifecycle methods (`resetStore`, `teardown`) that aren't part of {@apilink ISessionPool}. A user-supplied+ * pool is never owned and never torn down by the crawler.*/- private ownsSessionPool: boolean;+ private ownedSessionPool?: SessionPool;/*** A reference to the underlying {@apilink AutoscaledPool} class that manages the concurrency of the crawler.@@ -692,7 +701,7 @@ export class BasicCrawler<maxRequestsPerCrawl: ow.optional.number,maxCrawlDepth: ow.optional.number,autoscaledPoolOptions: ow.optional.object,- sessionPool: ow.optional.object.instanceOf(SessionPool),+ sessionPool: ow.optional.object.validate(validators.sessionPool),proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),statusMessageLoggingInterval: ow.optional.number,@@ -883,9 +892,10 @@ export class BasicCrawler<);}- this.sessionPool =- sessionPool ??- new SessionPool({+ if (sessionPool) {+ this.sessionPool = sessionPool;+ } else {+ this.ownedSessionPool = new SessionPool({createSessionFunction: async (opts) =>new Session({...opts?.sessionOptions,@@ -893,8 +903,8 @@ export class BasicCrawler<opts?.sessionOptions?.proxyInfo ?? (await this.proxyConfiguration?.newProxyInfo()),}),});-- this.ownsSessionPool = !sessionPool;+ this.sessionPool = this.ownedSessionPool;+ }this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);@@ -1128,26 +1138,22 @@ export class BasicCrawler<private async resolveSession({ request }: { request: Request }) {const session = await this._timeoutAndRetry(async () => {- if (request.sessionId) {- const existingSession = await this.sessionPool!.getSession(request.sessionId);+ const existingSession = await this.sessionPool.getSession(request.sessionId);- if (!existingSession) {- throw new ContextPipelineInitializationError(new MissingSessionError(request.sessionId));- }-- return existingSession;+ if (!existingSession) {+ throw new ContextPipelineInitializationError(new MissingSessionError(request.sessionId));}- return await this.sessionPool!.getSession();+ return existingSession;},this.internalTimeoutMillis,`Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,);- return { session, proxyInfo: session.proxyInfo };+ return { session, proxyInfo: session?.proxyInfo };}- private async createContextHelpers({ request, session }: { request: Request; session: Session }) {+ private async createContextHelpers({ request, session }: { request: Request; session: ISession }) {const enqueueLinksWrapper: CrawlingContext['enqueueLinks'] = async (options) => {const requestQueue = await this.getRequestQueue();@@ -1312,9 +1318,7 @@ export class BasicCrawler<this.stats.reset();await this.stats.resetStore();- if (this.ownsSessionPool) {- await this.sessionPool.resetStore();- }+ await this.ownedSessionPool?.resetStore();}this.unexpectedStop = false;@@ -2327,9 +2331,7 @@ export class BasicCrawler<await serviceLocator.getEventManager().close();}- if (this.ownsSessionPool) {- await this.sessionPool.teardown();- }+ await this.ownedSessionPool?.teardown();await this.autoscaledPool?.abort();}@@ -2423,7 +2425,7 @@ export class BasicCrawler<export interface CreateContextOptions {request: Request;- session: Session;+ session: ISession;proxyInfo?: ProxyInfo;}diff --git a/packages/basic-crawler/src/internals/send-request.ts b/packages/basic-crawler/src/internals/send-request.tsindex 26d0b7a4d..629c26451 100644--- a/packages/basic-crawler/src/internals/send-request.ts+++ b/packages/basic-crawler/src/internals/send-request.ts@@ -1,5 +1,5 @@-import type { Request as CrawleeRequest, Session } from '@crawlee/core';-import type { BaseHttpClient, HttpRequestOptions, SendRequestOptions } from '@crawlee/types';+import type { Request as CrawleeRequest } from '@crawlee/core';+import type { BaseHttpClient, HttpRequestOptions, ISession, SendRequestOptions } from '@crawlee/types';/*** Prepares a function to be used as the `sendRequest` context helper.@@ -9,7 +9,7 @@ import type { BaseHttpClient, HttpRequestOptions, SendRequestOptions } from '@cr* @param originRequest The crawling request being processed.* @param session The user session associated with the current request.*/-export function createSendRequest(httpClient: BaseHttpClient, originRequest: CrawleeRequest, session: Session) {+export function createSendRequest(httpClient: BaseHttpClient, originRequest: CrawleeRequest, session: ISession) {return async (overrideRequest: Partial<HttpRequestOptions> = {},overrideOptions: SendRequestOptions = {},diff --git a/packages/basic-crawler/test/batch-add-requests.test.ts b/packages/basic-crawler/test/batch-add-requests.test.tsindex 88788edf1..3b3dee9c4 100644--- a/packages/basic-crawler/test/batch-add-requests.test.ts+++ b/packages/basic-crawler/test/batch-add-requests.test.ts@@ -1,6 +1,7 @@import { BasicCrawler } from '@crawlee/basic';import { MemoryStorageEmulator } from '../../../test/shared/MemoryStorageEmulator.js';+import { SessionPool } from '@crawlee/core';describe('BasicCrawler#addRequests with big batch sizes', () => {const localStorageEmulator = new MemoryStorageEmulator();@@ -74,17 +75,20 @@ describe('BasicCrawler - request.sessionId', () => {});test('uses the session matching request.sessionId from the session pool', async () => {+ const REQUESTED_SESSION_ID = 'my-session';let resolvedSessionId: string | undefined;+ const sessionPool = new SessionPool();+ sessionPool.addSession({ id: REQUESTED_SESSION_ID });+const crawler = new BasicCrawler({requestHandler({ session }) {resolvedSessionId = session.id;},+ sessionPool,});- const session = await crawler.sessionPool.newSession({ id: 'my-session' });-- await crawler.run([{ url: 'http://localhost', sessionId: session.id }]);+ await crawler.run([{ url: 'http://localhost', sessionId: REQUESTED_SESSION_ID }]);expect(resolvedSessionId).toBe('my-session');});@@ -104,7 +108,7 @@ describe('BasicCrawler - request.sessionId', () => {expect(errors).toHaveLength(1);expect(errors[0].message).toContain(- "The current SessionPool instance doesn't contain the requested sessionId: nonexistent",+ "The current SessionPool instance couldn't find a valid session for the following id: nonexistent",);});});diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex a2f902ff2..6440a862f 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -10,11 +10,11 @@ import type {Request,RequestHandler,RequestProvider,- Session,SkippedRequestCallback,} from '@crawlee/basic';import {BasicCrawler,+ browserPoolCookieToToughCookie,ContextPipeline,cookieStringToToughCookie,enqueueLinks,@@ -23,6 +23,7 @@ import {RequestState,resolveBaseUrlForEnqueueLinksFiltering,SessionError,+ toughCookieToBrowserPoolCookie,tryAbsoluteURL,validators,} from '@crawlee/basic';@@ -36,7 +37,7 @@ import type {LaunchContext,} from '@crawlee/browser-pool';import { BrowserPool } from '@crawlee/browser-pool';-import type { BatchAddRequestsResult, Cookie as CookieObject } from '@crawlee/types';+import type { BatchAddRequestsResult, Cookie as CookieObject, ISession } from '@crawlee/types';import type { RobotsTxtFile } from '@crawlee/utils';import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';import ow from 'ow';@@ -402,7 +403,7 @@ export abstract class BrowserCrawler<action: this.preparePage.bind(this),cleanup: async (context: {page: Page;- session: Session;+ session: ISession;browserController: ProvidedController;registerDeferredCleanup: BasicCrawlingContext['registerDeferredCleanup'];}) => {@@ -578,10 +579,19 @@ export abstract class BrowserCrawler<// save cookies// TODO: Should we save the cookies also after/only the handle page?- if (this.saveResponseCookies) {+ if (this.saveResponseCookies && crawlingContext.session) {const cookies = await crawlingContext.browserController.getCookies(crawlingContext.page);tryCancel();- crawlingContext.session?.setCookies(cookies, crawlingContext.request.loadedUrl!);+ const url = crawlingContext.request.loadedUrl!;+ for (const cookie of cookies) {+ try {+ crawlingContext.session.cookieJar.setCookieSync(browserPoolCookieToToughCookie(cookie), url, {+ ignoreError: false,+ });+ } catch (e) {+ this.log.debug(`Could not set cookie: ${(e as Error).message}`);+ }+ }}if (response !== undefined) {@@ -617,7 +627,7 @@ export abstract class BrowserCrawler<preHooksCookies: string,postHooksCookies: string,) {- const sessionCookie = session?.getCookies(request.url) ?? [];+ const sessionCookie = session?.cookieJar.getCookiesSync(request.url).map(toughCookieToBrowserPoolCookie) ?? [];const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));diff --git a/packages/core/src/cookie_utils.ts b/packages/core/src/cookie_utils.tsindex b7f764853..53aeb9130 100644--- a/packages/core/src/cookie_utils.ts+++ b/packages/core/src/cookie_utils.ts@@ -59,19 +59,24 @@ export function toughCookieToBrowserPoolCookie(toughCookie: Cookie): CookieObjec/*** Transforms browser-pool cookie to tough-cookie.* @param cookieObject Cookie object (for instance from the `page.cookies` method).+ * @param maxAgeSecs Fallback expiration in seconds when the cookie itself has no `expires`.+ * When omitted, such a cookie is stored as a session cookie (no automatic expiration).* @internal*/-export function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs: number) {+export function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs?: number) {const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;- const expires = isExpiresValid- ? new Date(cookieObject.expires! * 1000)- : getDefaultCookieExpirationDate(maxAgeSecs);+ let expires: Date | 'Infinity' | undefined;+ if (isExpiresValid) {+ expires = new Date(cookieObject.expires! * 1000);+ } else if (maxAgeSecs != null) {+ expires = getDefaultCookieExpirationDate(maxAgeSecs);+ }const domainHasLeadingDot = cookieObject.domain?.startsWith?.('.');const domain = domainHasLeadingDot ? cookieObject.domain?.slice?.(1) : cookieObject.domain;return new Cookie({key: cookieObject.name,value: cookieObject.value,- expires,+ ...(expires !== undefined && { expires }),domain,path: cookieObject.path,secure: cookieObject.secure,diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.tsindex 4fb4d2925..6c95d24c3 100644--- a/packages/core/src/crawlers/crawler_commons.ts+++ b/packages/core/src/crawlers/crawler_commons.ts@@ -1,11 +1,10 @@-import type { Dictionary, HttpRequestOptions, ProxyInfo, SendRequestOptions } from '@crawlee/types';+import type { Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';import type { ReadonlyDeep, SetRequired } from 'type-fest';import type { Configuration } from '../configuration.js';import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';import type { CrawleeLogger } from '../log.js';import type { Request, Source } from '../request.js';-import type { Session } from '../session_pool/session.js';import type { Dataset } from '../storages/dataset.js';import { KeyValueStore, type RecordOptions } from '../storages/key_value_store.js';import type { RequestQueueOperationOptions } from '../storages/request_provider.js';@@ -29,7 +28,7 @@ export type LoadedContext<Context extends RestrictedCrawlingContext> =export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> {id: string;- session: Session;+ session: ISession;/*** An object with information about currently used proxy by the crawlerdiff --git a/packages/core/src/crawlers/crawler_utils.ts b/packages/core/src/crawlers/crawler_utils.tsindex 6bba99b25..992ef595e 100644--- a/packages/core/src/crawlers/crawler_utils.ts+++ b/packages/core/src/crawlers/crawler_utils.ts@@ -1,12 +1,12 @@-import { TimeoutError } from '@apify/timeout';+import type { ISession } from '@crawlee/types';-import type { Session } from '../session_pool/session.js';+import { TimeoutError } from '@apify/timeout';/*** Handles timeout request* @internal*/-export function handleRequestTimeout({ session, errorMessage }: { session?: Session; errorMessage: string }) {+export function handleRequestTimeout({ session, errorMessage }: { session?: ISession; errorMessage: string }) {session?.markBad();const timeoutMillis = /(\d+)\s?ms/.exec(errorMessage)?.[1]; // first capturing groupconst timeoutSecs = Number(timeoutMillis) / 1000;diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.tsindex ae2f26775..92a30dbe6 100644--- a/packages/core/src/errors.ts+++ b/packages/core/src/errors.ts@@ -68,8 +68,10 @@ export class SessionError extends Error {* Thrown when a requested session is not found in the referenced SessionPool.*/export class MissingSessionError extends Error {- constructor(sessionId: string) {- super(`The current SessionPool instance doesn't contain the requested sessionId: ${sessionId}`);+ constructor(sessionId?: string) {+ super(+ `The current SessionPool instance couldn't find a valid session${sessionId ? ` for the following id: ${sessionId}.` : '.'}`,+ );}}diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.tsindex 4d5092927..0a14ecfa7 100644--- a/packages/core/src/session_pool/session.ts+++ b/packages/core/src/session_pool/session.ts@@ -1,16 +1,10 @@-import type { Cookie as CookieObject, Dictionary, ISession, ProxyInfo, SessionState } from '@crawlee/types';+import type { Dictionary, ISession, ProxyInfo, SessionState } from '@crawlee/types';import ow from 'ow';-import type { Cookie } from 'tough-cookie';import { CookieJar } from 'tough-cookie';import { cryptoRandomObjectId } from '@apify/utilities';-import {- browserPoolCookieToToughCookie,- getCookiesFromResponse,- getDefaultCookieExpirationDate,- toughCookieToBrowserPoolCookie,-} from '../cookie_utils.js';+import { getDefaultCookieExpirationDate } from '../cookie_utils.js';import type { CrawleeLogger } from '../log.js';import { serviceLocator } from '../service_locator.js';@@ -84,7 +78,6 @@ export interface SessionOptions {*/export class Session implements ISession {readonly id: string;- private maxAgeSecs: number;userData: Dictionary;private _maxErrorScore: number;private _errorScoreDecrement: number;@@ -189,7 +182,6 @@ export class Session implements ISession {this._cookieJar = (cookieJar.setCookie as unknown) ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));this._proxyInfo = proxyInfo;this.id = id;- this.maxAgeSecs = maxAgeSecs;this.userData = userData;this._maxErrorScore = maxErrorScore;this._errorScoreDecrement = errorScoreDecrement;@@ -296,51 +288,6 @@ export class Session implements ISession {this._maybeSelfRetire();}- /**- * Saves cookies from an HTTP response to be used with the session.- * It expects an object with a `headers` property that's either an `Object`- * (typical Node.js responses) or a `Function` (Puppeteer Response).- *- * It then parses and saves the cookies from the `set-cookie` header, if available.- */- setCookiesFromResponse(response: Response) {- try {- const cookies = getCookiesFromResponse(response).filter((c) => c);- this._setCookies(cookies, response.url);- } catch (e) {- const err = e as Error;- // if invalid Cookie header is provided just log the exception.- this.log.exception(err, 'Could not get cookies from response');- }- }-- /**- * Saves an array with cookie objects to be used with the session.- * The objects should be in the format that- * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),- * but you can also use this function to set cookies manually:- *- * ```- * [- * { name: 'cookie1', value: 'my-cookie' },- * { name: 'cookie2', value: 'your-cookie' }- * ]- * ```- */- setCookies(cookies: CookieObject[], url: string) {- const normalizedCookies = cookies.map((c) => browserPoolCookieToToughCookie(c, this.maxAgeSecs));- this._setCookies(normalizedCookies, url);- }-- /**- * Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.- * @param url website url. Only cookies stored for this url will be returned- */- getCookies(url: string): CookieObject[] {- const cookies = this.cookieJar.getCookiesSync(url);- return cookies.map((c) => toughCookieToBrowserPoolCookie(c));- }-/*** Returns cookies saved with the session in the typical* key1=value1; key2=value2 format, ready to be used in@@ -362,27 +309,6 @@ export class Session implements ISession {}}- /**- * Sets cookies.- */- protected _setCookies(cookies: Cookie[], url: string): void {- const errorMessages: string[] = [];-- for (const cookie of cookies) {- try {- this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });- } catch (e) {- const err = e as Error;- errorMessages.push(err.message);- }- }-- // if invalid cookies are provided just log the exception. No need to retry the request automatically.- if (errorMessages.length) {- this.log.warning('Could not set cookies.', { errorMessages });- }- }-/*** Checks if session is not usable. if it is not retires the session.*/diff --git a/packages/core/src/session_pool/session_pool.ts b/packages/core/src/session_pool/session_pool.tsindex 0c399f34c..fb06659d5 100644--- a/packages/core/src/session_pool/session_pool.ts+++ b/packages/core/src/session_pool/session_pool.ts@@ -1,4 +1,4 @@-import type { Dictionary } from '@crawlee/types';+import type { Dictionary, ISessionPool } from '@crawlee/types';import { AsyncQueue } from '@sapphire/async-queue';import ow from 'ow';@@ -129,7 +129,7 @@ export interface SessionPoolOptions {** @category Scaling*/-export class SessionPool {+export class SessionPool implements ISessionPool {private static nextId = 0;readonly id: string;@@ -297,19 +297,6 @@ export class SessionPool {return newSession;}- /**- * Gets session.- * If there is space for new session, it creates and returns new session.- * If the session pool is full, it picks a session from the pool,- * If the picked session is usable it is returned, otherwise it creates and returns a new one.- */- async getSession(): Promise<Session>;-- /**- * Gets session based on the provided session id or `undefined.- */- async getSession(sessionId: string): Promise<Session>;-/*** Gets session.* If there is space for new session, it creates and returns new session.diff --git a/packages/core/src/validators.ts b/packages/core/src/validators.tsindex 3a1103e71..da5c15d11 100644--- a/packages/core/src/validators.ts+++ b/packages/core/src/validators.ts@@ -20,4 +20,9 @@ export const validators = {validator: ow.isValid(value, ow.object.hasKeys('fetchNextRequest', 'addRequest')),message: (label: string) => `Expected argument '${label}' to be a RequestQueue, got something else.`,}),+ sessionPool: (value: Dictionary) => ({+ validator: ow.isValid(value, ow.object.hasKeys('getSession')),+ message: (label: string) =>+ `Expected argument '${label}' to implement the ISessionPool interface (missing 'getSession'), got something else.`,+ }),};diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 455e5d198..6a9733c19 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -14,7 +14,6 @@ import type {RouterRoutes,RouteSchemas,RoutesFromSchemas,- Session,} from '@crawlee/basic';import {BasicCrawler,@@ -24,9 +23,9 @@ import {Router,SessionError,} from '@crawlee/basic';-import type { LoadedRequest } from '@crawlee/core';+import { type LoadedRequest, getCookiesFromResponse } from '@crawlee/core';import { ResponseWithUrl } from '@crawlee/http-client';-import type { Awaitable, Dictionary } from '@crawlee/types';+import type { Awaitable, Dictionary, ISession } from '@crawlee/types';import { type CheerioRoot, RETRY_CSS_SELECTORS } from '@crawlee/utils';import * as cheerio from 'cheerio';import type { RequestLike, ResponseLike } from 'content-type';@@ -501,7 +500,18 @@ export class HttpCrawler<this._throwOnBlockedRequest(response.status);if (this.saveResponseCookies) {- crawlingContext.session.setCookiesFromResponse(response);+ try {+ for (const cookie of getCookiesFromResponse(response)) {+ if (!cookie) continue;+ try {+ crawlingContext.session.cookieJar.setCookieSync(cookie, response.url, { ignoreError: false });+ } catch (e) {+ this.log.debug(`Could not set cookie: ${(e as Error).message}`);+ }+ }+ } catch (e) {+ this.log.exception(e as Error, 'Could not get cookies from response');+ }}return {@@ -613,7 +623,7 @@ export class HttpCrawler</*** Combines the provided `requestOptions` with mandatory (non-overridable) values.*/- protected _getRequestOptions(request: CrawleeRequest, session: Session, proxyUrl?: string) {+ protected _getRequestOptions(request: CrawleeRequest, session: ISession, proxyUrl?: string) {const requestOptions = {url: request.url,method: request.method,@@ -714,7 +724,7 @@ export class HttpCrawler</*** Handles timeout request*/- protected _handleRequestTimeout(session: Session) {+ protected _handleRequestTimeout(session: ISession) {session.markBad();throw new Error(`request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);}@@ -737,7 +747,7 @@ export class HttpCrawler</*** @internal wraps public utility for mocking purposes*/- private _requestAsBrowser = async (options: Dictionary<any>, session: Session) => {+ private _requestAsBrowser = async (options: Dictionary<any>, session: ISession) => {const opts = processHttpRequestOptions({...(options as any),responseType: 'text',@@ -769,7 +779,7 @@ export class HttpCrawler<interface RequestFunctionOptions {request: CrawleeRequest;- session: Session;+ session: ISession;proxyUrl?: string;}diff --git a/packages/types/src/session.ts b/packages/types/src/session.tsindex 1059bfb24..0a67617fb 100644--- a/packages/types/src/session.ts+++ b/packages/types/src/session.ts@@ -1,8 +1,5 @@import type { CookieJar, SerializedCookieJar } from 'tough-cookie';-import type { Cookie } from './browser.js';-import type { Dictionary } from './utility-types.js';-/*** The main purpose of the ProxyInfo object is to provide information* about the current proxy connection used by the crawler for the request.@@ -90,56 +87,22 @@ export interface SessionState {*/export interface ISession {readonly id: string;- userData: Dictionary;- errorScore: number;- usageCount: number;- maxErrorScore: number;- errorScoreDecrement: number;- expiresAt: Date;- createdAt: Date;- maxUsageCount: number;cookieJar: CookieJar;proxyInfo?: ProxyInfo;- /**- * Indicates whether the session is blocked.- * Session is blocked once it reaches the `maxErrorScore`.- */- isBlocked(): boolean;-- /**- * Indicates whether the session is expired.- * Session expiration is determined by the `maxAgeSecs`.- * Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.- */- isExpired(): boolean;-- /**- * Indicates whether the session is used maximum number of times.- * Session maximum usage count can be changed by `maxUsageCount` parameter.- */- isMaxUsageCountReached(): boolean;-/*** Indicates whether the session can be used for next requests.- * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.+ * Session is usable when it is not expired, not blocked and the maximum usage count has not been reached.*/isUsable(): boolean;/*** This method should be called after a successful session usage.- * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.*/markGood(): void;/**- * Gets session state for persistence in KeyValueStore.- * @returns Represents session internal state.- */- getState(): SessionState;-- /**- * Marks session as blocked and emits event on the `SessionPool`+ * Marks session as blocked.* This method should be used if the session usage was unsuccessful* and you are sure that it is because of the session configuration and not any external matters.* For example when server returns 403 status code.@@ -152,34 +115,30 @@ export interface ISession {* Should be used when the session has been used unsuccessfully. For example because of timeouts.*/markBad(): void;+}+/**+ * Minimal contract that any object passed to a crawler as its `sessionPool` option must satisfy.+ *+ * Crawlers only depend on a single method of the built-in `SessionPool`: `getSession()` /+ * `getSession(id)` to hand out an {@apilink ISession} for a request. Lifecycle (reset, teardown)+ * is the responsibility of whoever owns the pool — since a user-supplied pool is never owned by+ * the crawler, the crawler never tears it down.+ *+ * Implement this interface to plug a custom session-management strategy into any Crawlee crawler —+ * for example a remote, multi-process pool, a database-backed pool, or a thin wrapper around the+ * built-in `SessionPool` with different rotation rules.+ *+ * @category Scaling+ */+export interface ISessionPool {/**- * Saves cookies from an HTTP response to be used with the session.- * It expects an object with a `headers` property that's either an `Object`- * (typical Node.js responses) or a `Function` (Puppeteer Response).- *- * It then parses and saves the cookies from the `set-cookie` header, if available.- */- setCookiesFromResponse(response: Response): void;-- /**- * Saves an array with cookie objects to be used with the session.- * The objects should be in the format that- * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),- * but you can also use this function to set cookies manually:+ * Returns a usable {@apilink ISession}. Without an id, the pool decides which session to return+ * (creating a new one when appropriate). With an id, the pool returns the matching session if+ * it is still usable.*- * ```- * [- * { name: 'cookie1', value: 'my-cookie' },- * { name: 'cookie2', value: 'your-cookie' }- * ]- * ```- */- setCookies(cookies: Cookie[], url: string): void;-- /**- * Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.- * @param url website url. Only cookies stored for this url will be returned+ * In case the `SessionPool` cannot provide a usable session given the configuration,+ * this method may return `undefined`.*/- getCookies(url: string): Cookie[];+ getSession(sessionId?: string): Promise<ISession | undefined>;}diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 5f00c015b..61f87277d 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -24,7 +24,7 @@ import {} from '@crawlee/basic';import { RequestState, SessionPool, Statistics } from '@crawlee/core';import { MemoryStorage } from '@crawlee/memory-storage';-import type { ProxyInfo } from '@crawlee/types';+import type { ISession, ProxyInfo } from '@crawlee/types';import type { Dictionary } from '@crawlee/utils';import { RobotsTxtFile, sleep } from '@crawlee/utils';import express from 'express';@@ -1627,7 +1627,7 @@ describe('BasicCrawler', () => {const proxyUrls = [0, 1, 2].map((n) => `http://proxy.example.com:${1000 + n}`);const proxyConfiguration = new ProxyConfiguration({ proxyUrls });- const sessions: Session[] = [];+ const sessions: ISession[] = [];const proxyInfos: (ProxyInfo | undefined)[] = [];const crawler = new BasicCrawler({@@ -1660,7 +1660,7 @@ describe('BasicCrawler', () => {const crawler = new BasicCrawler({sessionPool: new SessionPool({ maxPoolSize: 1 }),requestHandler: async ({ session, proxyInfo }) => {- sessions.push(session);+ sessions.push(session as Session);proxyInfos.push(proxyInfo);},});diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.tsindex 599221c4a..24017204c 100644--- a/test/core/crawlers/browser_crawler.test.ts+++ b/test/core/crawlers/browser_crawler.test.ts@@ -17,6 +17,7 @@ import log from '@apify/log';import type { TestCrawlingContext } from './basic_browser_crawler.js';import { BrowserCrawlerTest } from './basic_browser_crawler.js';+import { ISession } from '@crawlee/types';describe('BrowserCrawler', () => {let prevEnvHeadless: string;@@ -147,7 +148,7 @@ describe('BrowserCrawler', () => {});class TimeoutError extends Error {}let markBadCalled = false;- let sessionGoto!: Session;+ let sessionGoto!: ISession;const browserCrawler = new (class extends BrowserCrawlerTest {protected override async _navigationHandler(ctx: TestCrawlingContext,@@ -565,7 +566,7 @@ describe('BrowserCrawler', () => {requestList,saveResponseCookies: true,requestHandler: async ({ session, request }) => {- loadedCookies.push(session.getCookieString(request.url));+ loadedCookies.push(session.cookieJar.getCookieStringSync(request.url));return Promise.resolve();},preNavigationHooks: [@@ -833,7 +834,7 @@ describe('BrowserCrawler', () => {maxPoolSize: 1,}),requestHandler: async ({ session }) => {- sessionUsageHistory.push(session!.usageCount);+ sessionUsageHistory.push((session as Session).usageCount);},});diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex a62f25c5e..ddb1ecbaa 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -13,7 +13,7 @@ import {} from '@crawlee/cheerio';import { BaseCrawleeLogger, SessionPool } from '@crawlee/core';import { ImpitHttpClient } from '@crawlee/impit-client';-import type { ProxyInfo } from '@crawlee/types';+import type { ISession, ProxyInfo } from '@crawlee/types';import type { Dictionary } from '@crawlee/utils';import { sleep } from '@crawlee/utils';import iconv from 'iconv-lite';@@ -751,7 +751,7 @@ describe('CheerioCrawler', () => {});const proxies: ProxyInfo[] = [];- const sessions: Session[] = [];+ const sessions: ISession[] = [];const requestHandler = ({ session, proxyInfo }: CheerioCrawlingContext) => {proxies.push(proxyInfo!);sessions.push(session!);@@ -927,7 +927,7 @@ describe('CheerioCrawler', () => {await cheerioCrawler.run();// @ts-expect-error private symbol- const sessions = cheerioCrawler.sessionPool!.sessions;+ const sessions: Session[] = cheerioCrawler.sessionPool!.sessions;expect(sessions.length).toBe(4);sessions.forEach((session) => {// TODO this test is flaky in CI and we need some more info to debug why.@@ -947,7 +947,7 @@ describe('CheerioCrawler', () => {test('should retire session on "blocked" status codes', async () => {for (const code of [401, 403, 429]) {const failed: Request[] = [];- const sessions: Session[] = [];+ const sessions: ISession[] = [];const maxRequestRetries = 5;const crawler = new CheerioCrawler({requestList: await getRequestListForMock({@@ -959,7 +959,7 @@ describe('CheerioCrawler', () => {saveResponseCookies: false,maxRequestRetries,requestHandler: ({ session }) => {- sessions.push(session!);+ sessions.push(session);},failedRequestHandler: ({ request }) => {failed.push(request);@@ -1036,8 +1036,8 @@ describe('CheerioCrawler', () => {preNavigationHooks: [({ session, request }) => {// this should get overriden by the server- session.setCookie('foo=bar1', request.url);- session.setCookie('other=cookie1', request.url);+ session.cookieJar.setCookieSync('foo=bar1', request.url);+ session.cookieJar.setCookieSync('other=cookie1', request.url);request.headers ??= {};request.headers.cookie += '; coo=kie';@@ -1287,7 +1287,7 @@ describe('CheerioCrawler', () => {},]),requestHandler: async ({ session, request }) => {- sessionCookies.push(session.getCookieString(request.url));+ sessionCookies.push(session.cookieJar.getCookieStringSync(request.url));},});@@ -1305,7 +1305,7 @@ describe('CheerioCrawler', () => {requestList: await RequestList.open(null, [`${serverAddress}/special/get-cookies`]),preNavigationHooks: [({ session, request }) => {- session.setCookie('manual=fromHook', request.url);+ session.cookieJar.setCookieSync('manual=fromHook', request.url);},],requestHandler: ({ json }) => {diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.tsindex c0a2955bc..bd644497e 100644--- a/test/core/crawlers/puppeteer_crawler.test.ts+++ b/test/core/crawlers/puppeteer_crawler.test.ts@@ -24,7 +24,6 @@ import {Session,SessionPool,} from '@crawlee/puppeteer';-import type { Cookie } from '@crawlee/types';import { sleep } from '@crawlee/utils';import type { Server as ProxyChainServer } from 'proxy-chain';import { z } from 'zod';@@ -317,15 +316,6 @@ describe('PuppeteerCrawler', () => {});test('should set cookies assigned to session to page', async () => {- const cookies: Cookie[] = [- {- name: 'example_cookie_name',- domain: '127.0.0.1',- value: 'example_cookie_value',- expires: -1,- } as never,- ];-let pageCookies;let sessionCookies;@@ -336,13 +326,13 @@ describe('PuppeteerCrawler', () => {sessionPool: new SessionPool({createSessionFunction: () => {const session = new Session();- session.setCookies(cookies, serverUrl);+ session.cookieJar.setCookieSync('example_cookie_name=example_cookie_value', serverUrl);return session;},}),requestHandler: async ({ page, session }) => {pageCookies = await page.cookies().then((cks) => cks.map((c) => `${c.name}=${c.value}`).join('; '));- sessionCookies = session!.getCookieString(serverUrl);+ sessionCookies = session!.cookieJar.getCookieStringSync(serverUrl);},});diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.tsindex ae3ce6509..1cf8a79d5 100644--- a/test/core/session_pool/session.test.ts+++ b/test/core/session_pool/session.test.ts@@ -1,7 +1,5 @@-import { log, Session } from '@crawlee/core';-import { ResponseWithUrl } from '@crawlee/http-client';+import { Session } from '@crawlee/core';import { entries, sleep } from '@crawlee/utils';-import { CookieJar } from 'tough-cookie';describe('Session - testing session behaviour', () => {let session: Session;@@ -49,23 +47,6 @@ describe('Session - testing session behaviour', () => {expect(session.isBlocked()).toBe(true);expect(session.isUsable()).toBe(false);});- test('should not throw on invalid Cookie header', () => {- let error;-- try {- session.setCookiesFromResponse(- new ResponseWithUrl('', {- headers: { Cookie: 'invaldi*{*{*{*-----***@s' },- url: 'http://localhost:1337',- }),- );- } catch (e) {- error = e;- }-- expect(error).toBeUndefined();- });-test('should markGood session', () => {session.markGood();expect(session.usageCount).toBe(1);@@ -162,151 +143,11 @@ describe('Session - testing session behaviour', () => {expect(session.cookieJar.setCookie).toBeDefined();});- test('setCookies should work', () => {- const url = 'https://example.com';- const cookies = [- { name: 'cookie1', value: 'my-cookie' },- { name: 'cookie2', value: 'your-cookie' },- ];-- session = new Session();- session.setCookies(cookies, url);- expect(session.getCookieString(url)).toBe('cookie1=my-cookie; cookie2=your-cookie');- });-- test('setCookies should work for session (with expiration date: -1) cookies', () => {- const url = 'https://example.com';- const cookies = [{ name: 'session_cookie', value: 'session-cookie-value', expires: -1 }];-- session = new Session();- session.setCookies(cookies, url);- expect(session.getCookieString(url)).toBe('session_cookie=session-cookie-value');- });-- test('setCookies works with leading dots in domains', () => {- const url = 'https://www.example.com';- const cookies = [- { name: 'cookie1', value: 'my-cookie', domain: 'abc.example.com' },- { name: 'cookie2', value: 'your-cookie', domain: '.example.com' },- ];-- session = new Session();- session.setCookies(cookies, url);- expect(session.getCookieString(url)).toBe('cookie2=your-cookie');- });-- test('setCookies will log warning (not throw) on invalid cookies', () => {- const url = 'https://www.example.com';- // domain 'abc.different.domain' does not match the request URL, so tough-cookie rejects it- const cookies = [{ name: 'cookie1', value: 'my-cookie', domain: 'abc.different.domain' }];-- const mockedLog = vitest.mockObject(log, {- spy: true,- });-- session = new Session({ log: mockedLog } as any);- session.setCookies(cookies, url);- expect(session.getCookieString(url)).toBe('');- expect(mockedLog.warning).toHaveBeenCalledOnce();- });-test('setCookie does not throw on malformed raw cookie string', () => {session = new Session();expect(() => session.setCookie('garbled!!!@#$%nonsense', 'https://www.example.com')).not.toThrow();});- test('setCookies works with hostOnly cookies', () => {- const url = 'https://www.example.com';- const cookies = [- { name: 'cookie1', value: 'my-cookie', domain: 'abc.example.com' },- { name: 'cookie2', value: 'your-cookie', domain: 'example.com' },- ];-- session = new Session();- session.setCookies(cookies, url);- expect(session.getCookieString(url)).toBe('');- expect(session.getCookieString('https://example.com')).toBe('cookie2=your-cookie');- });-- test('getCookies should work', () => {- const url = 'https://www.example.com';-- session = new Session({- cookieJar: CookieJar.fromJSON(- JSON.stringify({- cookies: [- {- 'key': 'foo',- 'value': 'bar',- 'domain': 'example.com',- 'path': '/',- 'hostOnly': false,- },- ],- }),- ),- });-- expect(session.getCookies(url)).to.containSubset([- {- name: 'foo',- value: 'bar',- domain: '.example.com',- },- ]);- expect(session.getCookies(url)).to.deep.equal(session.getCookies('https://example.com'));- });-- test('getCookies should work with hostOnly cookies', () => {- const url = 'https://www.example.com';-- session = new Session({- cookieJar: CookieJar.fromJSON(- JSON.stringify({- cookies: [- {- 'key': 'foo',- 'value': 'bar',- 'domain': 'example.com',- 'path': '/',- 'hostOnly': true,- },- ],- }),- ),- });-- expect(session.getCookies(url)).toHaveLength(0);- expect(session.getCookies('https://example.com')).to.containSubset([- {- name: 'foo',- value: 'bar',- domain: 'example.com',- },- ]);- });-- describe('.putResponse & .getCookieString', () => {- test('should set and update cookies from "set-cookie" header', () => {- const headers = new Headers();-- headers.append('set-cookie', 'CSRF=e8b667; Domain=example.com; Secure ');- headers.append('set-cookie', 'id=a3fWa; Expires=Wed, Domain=example.com; 21 Oct 2015 07:28:00 GMT');-- const newSession = new Session();- const url = 'https://example.com';- newSession.setCookiesFromResponse(new ResponseWithUrl('', { headers, url }));- let cookies = newSession.getCookieString(url);- expect(cookies).toEqual('CSRF=e8b667; id=a3fWa');-- const newCookie = 'ABCD=1231231213; Domain=example.com; Secure';-- newSession.setCookiesFromResponse(new ResponseWithUrl('', { headers: { 'set-cookie': newCookie }, url }));- cookies = newSession.getCookieString(url);- expect(cookies).toEqual('CSRF=e8b667; id=a3fWa; ABCD=1231231213');- });- });-test('retired state survives a getState() / new Session() round-trip', () => {session.retire();@@ -328,14 +169,10 @@ describe('Session - testing session behaviour', () => {});test('should correctly persist and init cookieJar', () => {- const headers = new Headers();-- headers.append('set-cookie', 'CSRF=e8b667; Domain=example.com; Secure ');- headers.append('set-cookie', 'id=a3fWa; Expires=Wed, Domain=example.com; 21 Oct 2015 07:28:00 GMT');-const newSession = new Session();const url = 'https://example.com';- newSession.setCookiesFromResponse(new ResponseWithUrl('', { headers, url }));+ newSession.cookieJar.setCookieSync('CSRF=e8b667; Domain=example.com; Secure', url);+ newSession.cookieJar.setCookieSync('id=a3fWa; Expires=Wed, 21 Oct 2099 07:28:00 GMT; Domain=example.com', url);const old = newSession.getState();diff --git a/test/core/session_pool/session_pool.test.ts b/test/core/session_pool/session_pool.test.tsindex d1bfa91da..08a6d7184 100644--- a/test/core/session_pool/session_pool.test.ts+++ b/test/core/session_pool/session_pool.test.ts@@ -64,11 +64,13 @@ describe('SessionPool - testing session pool', () => {const session = await sessionPool.getSession();// @ts-expect-error private symbolexpect(sessionPool.sessions.length).toBe(1);- expect(session.id).toBeDefined();- // @ts-expect-error Accessing private property- expect(session.maxAgeSecs).toEqual(sessionPool.sessionOptions.maxAgeSecs);- // @ts-expect-error Accessing private property- expect(session.maxUsageCount).toEqual(sessionPool.sessionOptions.maxUsageCount);+ expect(session?.id).toBeDefined();+ expect(session!.expiresAt.getTime() - session!.createdAt.getTime()).toEqual(+ // @ts-expect-error Accessing protected property+ (sessionPool.sessionOptions.maxAgeSecs as number) * 1000,+ );+ // @ts-expect-error Accessing protected property+ expect(session?.maxUsageCount).toEqual(sessionPool.sessionOptions.maxUsageCount);});test('should pick session when pool is full', async () => {@@ -113,13 +115,9 @@ describe('SessionPool - testing session pool', () => {test('get state should work', async () => {const url = 'https://example.com';- const cookies = [- { name: 'cookie1', value: 'my-cookie' },- { name: 'cookie2', value: 'your-cookie' },- ];-const newSession = await sessionPool.getSession();- newSession.setCookies(cookies, url);+ newSession?.cookieJar.setCookieSync('cookie1=my-cookie', url);+ newSession?.cookieJar.setCookieSync('cookie2=your-cookie', url);const state = await sessionPool.getState();expect(state).toBeInstanceOf(Object);@@ -295,7 +293,7 @@ describe('SessionPool - testing session pool', () => {const recreatedSession = await loadedSessionPool.getSession();- expect(recreatedSession.maxUsageCount).toEqual(66);+ expect(recreatedSession?.maxUsageCount).toEqual(66);});test('should persist state on teardown', async () => {@@ -340,7 +338,7 @@ describe('SessionPool - testing session pool', () => {const session = await newSessionPool.getSession();expect(isCalled).toBe(true);expect(receivedOptions?.sessionOptions).toBeDefined();- expect(session.constructor.name).toBe('Session');+ expect(session?.constructor.name).toBe('Session');});it('should remove persist state event listener', async () => {@@ -387,7 +385,7 @@ describe('SessionPool - testing session pool', () => {await sessionPool.addSession({ id: 'another-test-session' });const session = await sessionPool.getSession('test-session');- expect(session.id).toBe('test-session');+ expect(session?.id).toBe('test-session');});test('should correctly populate session array and session map', async () => {@@ -433,10 +431,10 @@ describe('SessionPool - testing session pool', () => {const s2 = await sessionPool.getSession();const s3 = await sessionPool.getSession();- expect(new Set([s1.id, s2.id, s3.id]).size).toBe(3);+ expect(new Set([s1?.id, s2?.id, s3?.id]).size).toBe(3);const s4 = await sessionPool.getSession();- expect([s1.id, s2.id, s3.id]).toContain(s4.id);+ expect([s1?.id, s2?.id, s3?.id]).toContain(s4?.id);});test('round-robin should fill pool before cycling', async () => {@@ -446,24 +444,24 @@ describe('SessionPool - testing session pool', () => {const s2 = await sessionPool.getSession();const s3 = await sessionPool.getSession();- expect(new Set([s1.id, s2.id, s3.id]).size).toBe(3);+ expect(new Set([s1?.id, s2?.id, s3?.id]).size).toBe(3);const ids: string[] = [];for (let i = 0; i < 6; i++) {- ids.push((await sessionPool.getSession()).id);+ ids.push((await sessionPool.getSession())?.id!);}- expect(ids).toEqual([s1.id, s2.id, s3.id, s1.id, s2.id, s3.id]);+ expect(ids).toEqual([s1?.id, s2?.id, s3?.id, s1?.id, s2?.id, s3?.id]);});test('round-robin should create a new session when all existing ones are retired', async () => {sessionPool = new SessionPool({ sessionReuseStrategy: 'round-robin', maxPoolSize: 1 });const s1 = await sessionPool.getSession();- s1.retire();+ s1?.retire();const s2 = await sessionPool.getSession();- expect(s2.id).not.toBe(s1.id);+ expect(s2?.id).not.toBe(s1?.id);});test.each(['random', 'round-robin'] as const)(@@ -475,7 +473,7 @@ describe('SessionPool - testing session pool', () => {await sessionPool.getSession();await sessionPool.getSession();- s1.retire();+ s1?.retire();// @ts-expect-error private symbolexpect(sessionPool.sessions).toHaveLength(3);@@ -496,18 +494,18 @@ describe('SessionPool - testing session pool', () => {const s2 = await sessionPool.getSession();const s3 = await sessionPool.getSession();- expect(s1.id).toBe(s2.id);- expect(s2.id).toBe(s3.id);+ expect(s1?.id).toBe(s2?.id);+ expect(s2?.id).toBe(s3?.id);});test('use-until-failure should switch to a new session after the current one is retired', async () => {sessionPool = new SessionPool({ sessionReuseStrategy: 'use-until-failure' });const s1 = await sessionPool.getSession();- s1.retire();+ s1?.retire();const s2 = await sessionPool.getSession();- expect(s2.id).not.toBe(s1.id);+ expect(s2?.id).not.toBe(s1?.id);});});@@ -565,7 +563,7 @@ describe('SessionPool - testing session pool', () => {const session1 = await pool1.getSession();await pool2.getSession();- session1.retire();+ session1?.retire();expect(await pool1.retiredSessionsCount()).toBe(1);expect(await pool2.retiredSessionsCount()).toBe(0);