Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor: remove deprecated crawler options
Martin Adámek4ddca29005b9
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex e54230e50..fb17ba055 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -22,3 +22,12 @@ Support for older TypeScript versions was dropped. Older versions might work too## Cheerio v1Previously, we kept the dependency on cheerio locked to the latest RC version, since there were many breaking changes introduced in v1.0. This release bumps cheerio to the stable v1. Also, we now use the default `parse5` internally.++## Deprecated crawler options are removed++The crawler following options are removed:++- `handleRequestFunction` -> `requestHandler`+- `handlePageFunction` -> `requestHandler`+- `handleRequestTimeoutSecs` -> `requestHandlerTimeoutSecs`+- `handleFailedRequestFunction` -> `failedRequestHandler`diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 45e7b4616..d39b5c4b4 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -60,7 +60,7 @@ import type { Awaitable, BatchAddRequestsResult, Dictionary, SetStatusMessageOptimport { getObjectType, isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS, sleep } from '@crawlee/utils';import { stringify } from 'csv-stringify/sync';import { ensureDir, writeJSON } from 'fs-extra/esm';-import ow, { ArgumentError } from 'ow';+import ow from 'ow';import { getDomain } from 'tldts';import type { SetRequired } from 'type-fest';@@ -154,28 +154,6 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw*/requestHandler?: RequestHandler<LoadedContext<Context>>;- /**- * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.- *- * The function receives the {@apilink BasicCrawlingContext} as an argument,- * where the {@apilink BasicCrawlingContext.request|`request`} represents the URL to crawl.- *- * The function must return a promise, which is then awaited by the crawler.- *- * If the function throws an exception, the crawler will try to re-crawl the- * request later, up to the {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.- * If all the retries fail, the crawler calls the function- * provided to the {@apilink BasicCrawlerOptions.failedRequestHandler|`failedRequestHandler`} parameter.- * To make this work, we should **always**- * let our function throw exceptions rather than catch them.- * The exceptions are logged to the request using the- * {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function.- *- * @deprecated `handleRequestFunction` has been renamed to `requestHandler` and will be removed in a future version.- * @ignore- */- handleRequestFunction?: RequestHandler<Context>;-/*** Static list of URLs to be processed.* If not provided, the crawler will open the default request queue when the {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} function is called.@@ -206,14 +184,6 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw*/requestHandlerTimeoutSecs?: number;- /**- * Timeout in which the function passed as {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`} needs to finish, in seconds.- * @default 60- * @deprecated `handleRequestTimeoutSecs` has been renamed to `requestHandlerTimeoutSecs` and will be removed in a future version.- * @ignore- */- handleRequestTimeoutSecs?: number;-/*** User-provided function that allows modifying the request object before it gets retried by the crawler.* It's executed before each retry for the requests that failed less than {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.@@ -235,19 +205,6 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw*/failedRequestHandler?: ErrorHandler<Context>;- /**- * A function to handle requests that failed more than {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.- *- * The function receives the {@apilink BasicCrawlingContext} as the first argument,- * where the {@apilink BasicCrawlingContext.request|`request`} corresponds to the failed request.- * Second argument is the `Error` instance that- * represents the last error thrown during processing of the request.- *- * @deprecated `handleFailedRequestFunction` has been renamed to `failedRequestHandler` and will be removed in a future version.- * @ignore- */- handleFailedRequestFunction?: ErrorHandler<Context>;-/*** Specifies the maximum number of retries allowed for a request if its processing fails.* This includes retries due to navigation errors or errors thrown from user-supplied functions@@ -592,15 +549,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext// in constructor, so this validation needs to apply only// if the user creates an instance of BasicCrawler directly.requestHandler: ow.optional.function,- // TODO: remove in a future release- handleRequestFunction: ow.optional.function,requestHandlerTimeoutSecs: ow.optional.number,- // TODO: remove in a future release- handleRequestTimeoutSecs: ow.optional.number,errorHandler: ow.optional.function,failedRequestHandler: ow.optional.function,- // TODO: remove in a future release- handleFailedRequestFunction: ow.optional.function,maxRequestRetries: ow.optional.number,sameDomainDelaySecs: ow.optional.number,maxSessionRotations: ow.optional.number,@@ -662,28 +613,18 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextretryOnBlocked = false,respectRobotsTxtFile = false,onSkippedRequest,-- // internal- log = defaultLog.child({ prefix: this.constructor.name }),- experiments = {},-- // Old and new request handler methods- handleRequestFunction,requestHandler,-- handleRequestTimeoutSecs,requestHandlerTimeoutSecs,-errorHandler,-- handleFailedRequestFunction,failedRequestHandler,-statusMessageLoggingInterval = 10,statusMessageCallback,-statisticsOptions,httpClient,++ // internal+ log = defaultLog.child({ prefix: this.constructor.name }),+ experiments = {},} = options;if (requestManager !== undefined) {@@ -709,54 +650,21 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextthis.robotsTxtFileCache = new LruCache({ maxLength: 1000 });this.handleSkippedRequest = this.handleSkippedRequest.bind(this);- this._handlePropertyNameChange({- newName: 'requestHandler',- oldName: 'handleRequestFunction',- propertyKey: 'requestHandler',- newProperty: requestHandler,- oldProperty: handleRequestFunction,- allowUndefined: true, // fallback to the default router- });-- if (!this.requestHandler) {- this.requestHandler = this.router;- }-+ // FIXME any+ this.requestHandler = (requestHandler as any) ?? this.router;+ this.failedRequestHandler = failedRequestHandler;this.errorHandler = errorHandler;- this._handlePropertyNameChange({- newName: 'failedRequestHandler',- oldName: 'handleFailedRequestFunction',- propertyKey: 'failedRequestHandler',- newProperty: failedRequestHandler,- oldProperty: handleFailedRequestFunction,- allowUndefined: true,- });-- let newRequestHandlerTimeout: number | undefined;-- if (!handleRequestTimeoutSecs) {- if (!requestHandlerTimeoutSecs) {- newRequestHandlerTimeout = 60_000;- } else {- newRequestHandlerTimeout = requestHandlerTimeoutSecs * 1000;- }- } else if (requestHandlerTimeoutSecs) {- newRequestHandlerTimeout = requestHandlerTimeoutSecs * 1000;+ if (requestHandlerTimeoutSecs) {+ this.requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;+ } else {+ this.requestHandlerTimeoutMillis = 60_000;}this.retryOnBlocked = retryOnBlocked;this.respectRobotsTxtFile = respectRobotsTxtFile;this.onSkippedRequest = onSkippedRequest;- this._handlePropertyNameChange({- newName: 'requestHandlerTimeoutSecs',- oldName: 'handleRequestTimeoutSecs',- propertyKey: 'requestHandlerTimeoutMillis',- newProperty: newRequestHandlerTimeout,- oldProperty: handleRequestTimeoutSecs ? handleRequestTimeoutSecs * 1000 : undefined,- });-const tryEnv = (val?: string) => (val == null ? null : +val);// allow at least 5min for internal timeoutsthis.internalTimeoutMillis =@@ -2060,43 +1968,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextawait this.autoscaledPool?.abort();}- protected _handlePropertyNameChange<New, Old>({- newProperty,- newName,- oldProperty,- oldName,- propertyKey,- allowUndefined = false,- }: HandlePropertyNameChangeData<New, Old>) {- if (newProperty && oldProperty) {- this.log.warning(- [- `Both "${newName}" and "${oldName}" were provided in the crawler options.`,- `"${oldName}" has been renamed to "${newName}", and will be removed in a future version.`,- `As such, "${newName}" will be used instead.`,- ].join('\n'),- );-- // @ts-expect-error Assigning to possibly readonly properties- this[propertyKey] = newProperty;- } else if (oldProperty) {- this.log.warning(- [- `"${oldName}" has been renamed to "${newName}", and will be removed in a future version.`,- `The provided value will be used, but you should rename "${oldName}" to "${newName}" in your crawler options.`,- ].join('\n'),- );-- // @ts-expect-error Assigning to possibly readonly properties- this[propertyKey] = oldProperty;- } else if (newProperty) {- // @ts-expect-error Assigning to possibly readonly properties- this[propertyKey] = newProperty;- } else if (!allowUndefined) {- throw new ArgumentError(`"${newName}" must be provided in the crawler options`, this.constructor);- }- }-protected _getCookieHeaderFromRequest(request: Request) {if (request.headers?.Cookie && request.headers?.cookie) {this.log.warning(@@ -2188,15 +2059,6 @@ export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {purgeRequestQueue?: boolean;}-interface HandlePropertyNameChangeData<New, Old> {- oldProperty?: Old;- newProperty?: New;- oldName: string;- newName: string;- propertyKey: string;- allowUndefined?: boolean;-}-/*** Creates new {@apilink Router} instance that works based on request labels.* This instance can then serve as a {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`} of our {@apilink BasicCrawler}.diff --git a/packages/basic-crawler/test/migration.test.ts b/packages/basic-crawler/test/migration.test.tsdeleted file mode 100644index bb548162c..000000000--- a/packages/basic-crawler/test/migration.test.ts+++ /dev/null@@ -1,227 +0,0 @@-import type { Log } from '@apify/log';-import log from '@apify/log';--import { MemoryStorageEmulator } from '../../../test/shared/MemoryStorageEmulator.js';-import { BasicCrawler, RequestList } from '../src/index.js';--const localStorageEmulator = new MemoryStorageEmulator();--beforeEach(async () => {- await localStorageEmulator.init();-});--afterAll(async () => {- await localStorageEmulator.destroy();-});--describe('Moving from handleRequest* to requestHandler*', () => {- let requestList: RequestList;- let testLogger: Log;-- beforeEach(async () => {- requestList = await RequestList.open(null, []);- testLogger = log.child({ prefix: 'BasicCrawler' });- });-- describe('handleRequestFunction -> requestHandler', () => {- it('should log when providing both handleRequestFunction and requestHandler', () => {- const oldHandler = () => {};- const newHandler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- const crawler = new BasicCrawler({- requestList,- log: testLogger,- requestHandler: newHandler,- handleRequestFunction: oldHandler,- });-- expect(warningSpy).toHaveBeenCalledWith<[string]>(- [- `Both "requestHandler" and "handleRequestFunction" were provided in the crawler options.`,- `"handleRequestFunction" has been renamed to "requestHandler", and will be removed in a future version.`,- `As such, "requestHandler" will be used instead.`,- ].join('\n'),- );-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['requestHandler']).toBe(newHandler);- });-- it('should log when providing only the deprecated handleRequestFunction', () => {- const oldHandler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- const crawler = new BasicCrawler({- requestList,- log: testLogger,- handleRequestFunction: oldHandler,- });-- expect(warningSpy).toHaveBeenCalledWith<[string]>(- [- `"handleRequestFunction" has been renamed to "requestHandler", and will be removed in a future version.`,- `The provided value will be used, but you should rename "handleRequestFunction" to "requestHandler" in your crawler options.`,- ].join('\n'),- );-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['requestHandler']).toBe(oldHandler);- });-- it('should not log when providing only requestHandler', () => {- const handler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- const crawler = new BasicCrawler({- requestList,- log: testLogger,- requestHandler: handler,- });-- expect(warningSpy).not.toHaveBeenCalled();-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['requestHandler']).toBe(handler);- });- });-- describe('handleFailedRequestFunction -> failedRequestHandler', () => {- it('should log when providing both handleFailedRequestFunction and failedRequestHandler', () => {- const oldHandler = () => {};- const newHandler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- const crawler = new BasicCrawler({- requestList,- log: testLogger,- requestHandler: () => {},- failedRequestHandler: newHandler,- handleFailedRequestFunction: oldHandler,- });-- expect(warningSpy).toHaveBeenCalledWith<[string]>(- [- `Both "failedRequestHandler" and "handleFailedRequestFunction" were provided in the crawler options.`,- `"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,- `As such, "failedRequestHandler" will be used instead.`,- ].join('\n'),- );-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['failedRequestHandler']).toBe(newHandler);- });-- it('should log when providing only the deprecated handleFailedRequestFunction', () => {- const oldHandler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- const crawler = new BasicCrawler({- requestList,- log: testLogger,- requestHandler: () => {},- handleFailedRequestFunction: oldHandler,- });-- expect(warningSpy).toHaveBeenCalledWith<[string]>(- [- `"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,- `The provided value will be used, but you should rename "handleFailedRequestFunction" to "failedRequestHandler" in your crawler options.`,- ].join('\n'),- );-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['failedRequestHandler']).toBe(oldHandler);- });-- it('should not log when providing only failedRequestHandler', () => {- const handler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- const crawler = new BasicCrawler({- requestList,- log: testLogger,- requestHandler: () => {},- failedRequestHandler: handler,- });-- expect(warningSpy).not.toHaveBeenCalled();-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['failedRequestHandler']).toBe(handler);- });- });-- describe('handleRequestTimeoutSecs -> requestHandlerTimeoutSecs', () => {- it('should log when providing both handleRequestTimeoutSecs and requestHandlerTimeoutSecs', () => {- const warningSpy = vitest.spyOn(testLogger, 'warning');-- const crawler = new BasicCrawler({- requestList,- log: testLogger,- requestHandler: () => {},- requestHandlerTimeoutSecs: 420,- handleRequestTimeoutSecs: 69,- });-- expect(warningSpy).toHaveBeenCalledWith<[string]>(- [- `Both "requestHandlerTimeoutSecs" and "handleRequestTimeoutSecs" were provided in the crawler options.`,- `"handleRequestTimeoutSecs" has been renamed to "requestHandlerTimeoutSecs", and will be removed in a future version.`,- `As such, "requestHandlerTimeoutSecs" will be used instead.`,- ].join('\n'),- );-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['requestHandlerTimeoutMillis']).toEqual(420_000);- });-- it('should log when providing only the deprecated handleRequestTimeoutSecs', () => {- const warningSpy = vitest.spyOn(testLogger, 'warning');-- const crawler = new BasicCrawler({- requestList,- log: testLogger,- requestHandler: () => {},- handleRequestTimeoutSecs: 69,- });-- expect(warningSpy).toHaveBeenCalledWith<[string]>(- [- `"handleRequestTimeoutSecs" has been renamed to "requestHandlerTimeoutSecs", and will be removed in a future version.`,- `The provided value will be used, but you should rename "handleRequestTimeoutSecs" to "requestHandlerTimeoutSecs" in your crawler options.`,- ].join('\n'),- );-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['requestHandlerTimeoutMillis']).toEqual(69_000);- });-- it('should not log when providing some or no number to requestHandlerTimeoutSecs', () => {- const warningSpy = vitest.spyOn(testLogger, 'warning');-- const crawler = new BasicCrawler({- requestList,- log: testLogger,- requestHandler: () => {},- });-- expect(warningSpy).not.toHaveBeenCalled();-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['requestHandlerTimeoutMillis']).toBe(60_000);-- const crawler2 = new BasicCrawler({- requestList,- log: testLogger,- requestHandler: () => {},- requestHandlerTimeoutSecs: 420,- });-- expect(warningSpy).not.toHaveBeenCalled();-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler2['requestHandlerTimeoutMillis']).toBe(420_000);- });- });-});diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 34ce1d311..c6ddda820 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -83,11 +83,7 @@ export interface BrowserCrawlerOptions<> extends Omit<BasicCrawlerOptions,// Overridden with browser context- | 'requestHandler'- | 'handleRequestFunction'- | 'failedRequestHandler'- | 'handleFailedRequestFunction'- | 'errorHandler'+ 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {launchContext?: BrowserLaunchContext<any, any>;@@ -120,38 +116,6 @@ export interface BrowserCrawlerOptions<*/requestHandler?: BrowserRequestHandler<LoadedContext<Context>>;- /**- * Function that is called to process each request.- *- * The function receives the {@apilink BrowserCrawlingContext}- * (actual context will be enhanced with the crawler specific properties) as an argument, where:- * - {@apilink BrowserCrawlingContext.request|`request`} is an instance of the {@apilink Request} object- * with details about the URL to open, HTTP method etc;- * - {@apilink BrowserCrawlingContext.page|`page`} is an instance of the- * Puppeteer [Page](https://pptr.dev/api/puppeteer.page) or- * Playwright [Page](https://playwright.dev/docs/api/class-page);- * - {@apilink BrowserCrawlingContext.browserController|`browserController`} is an instance of the {@apilink BrowserController};- * - {@apilink BrowserCrawlingContext.response|`response`} is an instance of the- * Puppeteer [Response](https://pptr.dev/api/puppeteer.httpresponse) or- * Playwright [Response](https://playwright.dev/docs/api/class-response),- * which is the main resource response as returned by the respective `page.goto()` function.- *- * The function must return a promise, which is then awaited by the crawler.- *- * If the function throws an exception, the crawler will try to re-crawl the- * request later, up to the {@apilink BrowserCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.- * If all the retries fail, the crawler calls the function- * provided to the {@apilink BrowserCrawlerOptions.failedRequestHandler|`failedRequestHandler`} parameter.- * To make this work, we should **always**- * let our function throw exceptions rather than catch them.- * The exceptions are logged to the request using the- * {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function.- *- * @deprecated `handlePageFunction` has been renamed to `requestHandler` and will be removed in a future version.- * @ignore- */- handlePageFunction?: BrowserRequestHandler<LoadedContext<Context>>;-/*** User-provided function that allows modifying the request object before it gets retried by the crawler.* It's executed before each retry for the requests that failed less than {@apilink BrowserCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.@@ -175,20 +139,6 @@ export interface BrowserCrawlerOptions<*/failedRequestHandler?: BrowserErrorHandler<Context>;- /**- * A function to handle requests that failed more than `option.maxRequestRetries` times.- *- * The function receives the {@apilink BrowserCrawlingContext}- * (actual context will be enhanced with the crawler specific properties) as the first argument,- * where the {@apilink BrowserCrawlingContext.request|`request`} corresponds to the failed request.- * Second argument is the `Error` instance that- * represents the last error thrown during processing of the request.- *- * @deprecated `handleFailedRequestFunction` has been renamed to `failedRequestHandler` and will be removed in a future version.- * @ignore- */- handleFailedRequestFunction?: BrowserErrorHandler<Context>;-/*** Custom options passed to the underlying {@apilink BrowserPool} constructor.* We can tweak those to fine-tune browser management.@@ -341,7 +291,6 @@ export abstract class BrowserCrawler<protected static override optionsShape = {...BasicCrawler.optionsShape,- handlePageFunction: ow.optional.function,navigationTimeoutSecs: ow.optional.number.greaterThan(0),preNavigationHooks: ow.optional.array,@@ -375,14 +324,8 @@ export abstract class BrowserCrawler<browserPoolOptions,preNavigationHooks = [],postNavigationHooks = [],- // Ignored- handleRequestFunction,-- requestHandler: userProvidedRequestHandler,- handlePageFunction,-+ requestHandler,failedRequestHandler,- handleFailedRequestFunction,headless,ignoreShadowRoots,ignoreIframes,@@ -399,27 +342,9 @@ export abstract class BrowserCrawler<config,);- this._handlePropertyNameChange({- newName: 'requestHandler',- oldName: 'handlePageFunction',- propertyKey: 'userProvidedRequestHandler',- newProperty: userProvidedRequestHandler,- oldProperty: handlePageFunction,- allowUndefined: true, // fallback to the default router- });-- if (!this.userProvidedRequestHandler) {- this.userProvidedRequestHandler = this.router;- }-- this._handlePropertyNameChange({- newName: 'failedRequestHandler',- oldName: 'handleFailedRequestFunction',- propertyKey: 'failedRequestHandler',- newProperty: failedRequestHandler,- oldProperty: handleFailedRequestFunction,- allowUndefined: true,- });+ // FIXME any+ this.userProvidedRequestHandler = (requestHandler as any) ?? this.router;+ this.failedRequestHandler = failedRequestHandler; // FIXME is this even needed?// Cookies should be persisted per session only if session pool is usedif (!this.useSessionPool && persistCookiesPerSession) {diff --git a/packages/browser-crawler/test/migration.test.ts b/packages/browser-crawler/test/migration.test.tsdeleted file mode 100644index 47766c3ca..000000000--- a/packages/browser-crawler/test/migration.test.ts+++ /dev/null@@ -1,195 +0,0 @@-import { PuppeteerPlugin } from '@crawlee/browser-pool';-import puppeteer from 'puppeteer';--import type { Log } from '@apify/log';-import log from '@apify/log';--import { MemoryStorageEmulator } from '../../../test/shared/MemoryStorageEmulator.js';-import { BrowserCrawler, RequestList } from '../src/index.js';--const localStorageEmulator = new MemoryStorageEmulator();--beforeEach(async () => {- await localStorageEmulator.init();-});--afterAll(async () => {- await localStorageEmulator.destroy();-});--const plugin = new PuppeteerPlugin(puppeteer);--describe('Moving from handleRequest* to requestHandler*', () => {- let requestList: RequestList;- let testLogger: Log;-- beforeEach(async () => {- requestList = await RequestList.open(null, []);- testLogger = log.child({ prefix: 'BrowserCrawler' });- });-- describe('handlePageFunction -> requestHandler', () => {- it('should log when providing both handlePageFunction and requestHandler', async () => {- const oldHandler = () => {};- const newHandler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- // @ts-expect-error -- Protected constructor- const crawler = new BrowserCrawler({- requestList,- log: testLogger,- browserPoolOptions: {- browserPlugins: [plugin],- },- requestHandler: newHandler,- handlePageFunction: oldHandler,- });-- expect(warningSpy).toHaveBeenCalledWith<[string]>(- [- `Both "requestHandler" and "handlePageFunction" were provided in the crawler options.`,- `"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`,- `As such, "requestHandler" will be used instead.`,- ].join('\n'),- );-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['userProvidedRequestHandler']).toBe(newHandler);-- await crawler.browserPool.destroy();- });-- it('should log when providing only the deprecated handlePageFunction', async () => {- const oldHandler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- // @ts-expect-error -- We are verifying the deprecation warning- const crawler = new BrowserCrawler({- requestList,- log: testLogger,- browserPoolOptions: {- browserPlugins: [plugin],- },- handlePageFunction: oldHandler,- });-- expect(warningSpy).toHaveBeenCalledWith<[string]>(- [- `"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`,- `The provided value will be used, but you should rename "handlePageFunction" to "requestHandler" in your crawler options.`,- ].join('\n'),- );-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['userProvidedRequestHandler']).toBe(oldHandler);-- await crawler.browserPool.destroy();- });-- it('should not log when providing only requestHandler', async () => {- const handler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- // @ts-expect-error -- Protected constructor- const crawler = new BrowserCrawler({- requestList,- log: testLogger,- browserPoolOptions: {- browserPlugins: [plugin],- },- requestHandler: handler,- });-- expect(warningSpy).not.toHaveBeenCalled();-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['userProvidedRequestHandler']).toBe(handler);-- await crawler.browserPool.destroy();- });- });-- describe('handleFailedRequestFunction -> failedRequestHandler', () => {- it('should log when providing both handleFailedRequestFunction and failedRequestHandler', async () => {- const oldHandler = () => {};- const newHandler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- // @ts-expect-error -- Protected constructor- const crawler = new BrowserCrawler({- requestList,- log: testLogger,- browserPoolOptions: {- browserPlugins: [plugin],- },- requestHandler: () => {},- failedRequestHandler: newHandler,- handleFailedRequestFunction: oldHandler,- });-- expect(warningSpy).toHaveBeenCalledWith<[string]>(- [- `Both "failedRequestHandler" and "handleFailedRequestFunction" were provided in the crawler options.`,- `"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,- `As such, "failedRequestHandler" will be used instead.`,- ].join('\n'),- );-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['failedRequestHandler']).toBe(newHandler);-- await crawler.browserPool.destroy();- });-- it('should log when providing only the deprecated handleFailedRequestFunction', async () => {- const oldHandler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- // @ts-expect-error -- Protected constructor- const crawler = new BrowserCrawler({- requestList,- log: testLogger,- browserPoolOptions: {- browserPlugins: [plugin],- },- requestHandler: () => {},- handleFailedRequestFunction: oldHandler,- });-- expect(warningSpy).toHaveBeenCalledWith<[string]>(- [- `"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,- `The provided value will be used, but you should rename "handleFailedRequestFunction" to "failedRequestHandler" in your crawler options.`,- ].join('\n'),- );-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['failedRequestHandler']).toBe(oldHandler);-- await crawler.browserPool.destroy();- });-- it('should not log when providing only failedRequestHandler', async () => {- const handler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- // @ts-expect-error -- Protected constructor- const crawler = new BrowserCrawler({- requestList,- log: testLogger,- browserPoolOptions: {- browserPlugins: [plugin],- },- requestHandler: () => {},- failedRequestHandler: handler,- });-- expect(warningSpy).not.toHaveBeenCalled();-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['failedRequestHandler']).toBe(handler);-- await crawler.browserPool.destroy();- });- });-});diff --git a/packages/browser-crawler/test/tsconfig.json b/packages/browser-crawler/test/tsconfig.jsondeleted file mode 100644index bf55f9516..000000000--- a/packages/browser-crawler/test/tsconfig.json+++ /dev/null@@ -1,7 +0,0 @@-{- "extends": "../../../tsconfig.json",- "include": ["**/*", "../../**/*"],- "compilerOptions": {- "types": ["vitest/globals"]- }-}diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.tsindex 0acabe411..5c91c1163 100644--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts@@ -190,7 +190,13 @@ export abstract class BrowserPlugin<});}- abstract createController(): BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;+ abstract createController(): BrowserController<+ Library,+ LibraryOptions,+ LaunchResult,+ NewPageOptions,+ NewPageResult+ >;/*** Launches the browser using provided launch context.diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.tsindex 47ff66a08..8e345f74a 100644--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts@@ -99,7 +99,7 @@ export class PuppeteerPlugin extends BrowserPlugin<error,launchContext.launchOptions?.executablePath,'`apify/actor-node-puppeteer-chrome`',- 'Try installing a browser, if it\'s missing, by running `npx @puppeteer/browsers install chromium --path [path]` and pointing `executablePath` to the downloaded executable (https://pptr.dev/browsers-api)',+ "Try installing a browser, if it's missing, by running `npx @puppeteer/browsers install chromium --path [path]` and pointing `executablePath` to the downloaded executable (https://pptr.dev/browsers-api)",);}}diff --git a/packages/cheerio-crawler/test/migration.test.ts b/packages/cheerio-crawler/test/migration.test.tsdeleted file mode 100644index c9d381572..000000000--- a/packages/cheerio-crawler/test/migration.test.ts+++ /dev/null@@ -1,154 +0,0 @@-import type { Log } from '@apify/log';-import log from '@apify/log';--import { MemoryStorageEmulator } from '../../../test/shared/MemoryStorageEmulator.js';-import { CheerioCrawler, RequestList } from '../src/index.js';--const localStorageEmulator = new MemoryStorageEmulator();--beforeEach(async () => {- await localStorageEmulator.init();-});--afterAll(async () => {- await localStorageEmulator.destroy();-});--describe('Moving from handleRequest* to requestHandler*', () => {- let requestList: RequestList;- let testLogger: Log;-- beforeEach(async () => {- requestList = await RequestList.open(null, []);- testLogger = log.child({ prefix: 'CheerioCrawler' });- });-- describe('handlePageFunction -> requestHandler', () => {- it('should log when providing both handlePageFunction and requestHandler', () => {- const oldHandler = () => {};- const newHandler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- const crawler = new CheerioCrawler({- requestList,- log: testLogger,- requestHandler: newHandler,- handlePageFunction: oldHandler,- });-- expect(warningSpy).toHaveBeenCalledWith<[string]>(- [- `Both "requestHandler" and "handlePageFunction" were provided in the crawler options.`,- `"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`,- `As such, "requestHandler" will be used instead.`,- ].join('\n'),- );-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['requestHandler']).toBe(newHandler);- });-- it('should log when providing only the deprecated handlePageFunction', () => {- const oldHandler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- const crawler = new CheerioCrawler({- requestList,- log: testLogger,- handlePageFunction: oldHandler,- });-- expect(warningSpy).toHaveBeenCalledWith<[string]>(- [- `"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`,- `The provided value will be used, but you should rename "handlePageFunction" to "requestHandler" in your crawler options.`,- ].join('\n'),- );-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['requestHandler']).toBe(oldHandler);- });-- it('should not log when providing only requestHandler', () => {- const handler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- const crawler = new CheerioCrawler({- requestList,- log: testLogger,- requestHandler: handler,- });-- expect(warningSpy).not.toHaveBeenCalled();-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['requestHandler']).toBe(handler);- });- });-- describe('handleFailedRequestFunction -> failedRequestHandler', () => {- it('should log when providing both handleFailedRequestFunction and failedRequestHandler', () => {- const oldHandler = () => {};- const newHandler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- const crawler = new CheerioCrawler({- requestList,- log: testLogger,- requestHandler: () => {},- failedRequestHandler: newHandler,- handleFailedRequestFunction: oldHandler,- });-- expect(warningSpy).toHaveBeenCalledWith<[string]>(- [- `Both "failedRequestHandler" and "handleFailedRequestFunction" were provided in the crawler options.`,- `"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,- `As such, "failedRequestHandler" will be used instead.`,- ].join('\n'),- );-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['failedRequestHandler']).toBe(newHandler);- });-- it('should log when providing only the deprecated handleFailedRequestFunction', () => {- const oldHandler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- const crawler = new CheerioCrawler({- requestList,- log: testLogger,- requestHandler: () => {},- handleFailedRequestFunction: oldHandler,- });-- expect(warningSpy).toHaveBeenCalledWith<[string]>(- [- `"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,- `The provided value will be used, but you should rename "handleFailedRequestFunction" to "failedRequestHandler" in your crawler options.`,- ].join('\n'),- );-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['failedRequestHandler']).toBe(oldHandler);- });-- it('should not log when providing only failedRequestHandler', () => {- const handler = () => {};- const warningSpy = vitest.spyOn(testLogger, 'warning');-- const crawler = new CheerioCrawler({- requestList,- log: testLogger,- requestHandler: () => {},- failedRequestHandler: handler,- });-- expect(warningSpy).not.toHaveBeenCalled();-- // eslint-disable-next-line dot-notation -- accessing private property- expect(crawler['failedRequestHandler']).toBe(handler);- });- });-});diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 57cb2564c..42fccebd2 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -81,13 +81,6 @@ export type HttpErrorHandler<export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext>extends BasicCrawlerOptions<Context> {- /**- * An alias for {@apilink HttpCrawlerOptions.requestHandler}- * Soon to be removed, use `requestHandler` instead.- * @deprecated- */- handlePageFunction?: HttpCrawlerOptions<Context>['requestHandler'];-/*** Timeout in which the HTTP request to the resource needs to finish, given in seconds.*/@@ -353,7 +346,6 @@ export class HttpCrawler<protected static override optionsShape = {...BasicCrawler.optionsShape,- handlePageFunction: ow.optional.function,navigationTimeoutSecs: ow.optional.number,ignoreSslErrors: ow.optional.boolean,@@ -381,8 +373,6 @@ export class HttpCrawler<const {requestHandler,- handlePageFunction,-requestHandlerTimeoutSecs = 60,navigationTimeoutSecs = 30,ignoreSslErrors = true,@@ -396,9 +386,6 @@ export class HttpCrawler<additionalHttpErrorStatusCodes = [],ignoreHttpErrorStatusCodes = [],- // Ignored- handleRequestFunction,-// BasicCrawlerautoscaledPoolOptions = HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS,...basicCrawlerOptions@@ -417,18 +404,8 @@ export class HttpCrawler<config,);- this._handlePropertyNameChange({- newName: 'requestHandler',- oldName: 'handlePageFunction',- propertyKey: 'requestHandler',- newProperty: requestHandler,- oldProperty: handlePageFunction,- allowUndefined: true,- });-- if (!this.requestHandler) {- this.requestHandler = this.router;- }+ // FIXME any+ this.requestHandler = (requestHandler as any) ?? this.router;// Cookies should be persisted per session only if session pool is usedif (!this.useSessionPool && persistCookiesPerSession) {diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex 6b73211db..d2830dc06 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -164,10 +164,7 @@ interface AdaptiveHook> {}export interface AdaptivePlaywrightCrawlerOptions- extends Omit<- PlaywrightCrawlerOptions,- 'requestHandler' | 'handlePageFunction' | 'preNavigationHooks' | 'postNavigationHooks'- > {+ extends Omit<PlaywrightCrawlerOptions, 'requestHandler' | 'preNavigationHooks' | 'postNavigationHooks'> {/*** Function that is called to process each request.*diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 034af2109..9c30342b0 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -1151,14 +1151,14 @@ describe('BasicCrawler', () => {vitest.restoreAllMocks();});- test('should timeout after handleRequestTimeoutSecs', async () => {+ test('should timeout after requestHandlerTimeoutSecs', async () => {const url = 'https://example.com';const requestList = await RequestList.open({ sources: [{ url }] });const results: Request[] = [];const crawler = new BasicCrawler({requestList,- handleRequestTimeoutSecs: 0.01,+ requestHandlerTimeoutSecs: 0.01,maxRequestRetries: 1,requestHandler: async () => sleep(1000),failedRequestHandler: async ({ request }) => {@@ -1172,7 +1172,7 @@ describe('BasicCrawler', () => {results[0].errorMessages.forEach((msg) => expect(msg).toMatch('requestHandler timed out'));});- test('limits handleRequestTimeoutSecs and derived vars to a valid value', async () => {+ test('limits requestHandlerTimeoutSecs and derived vars to a valid value', async () => {const url = 'https://example.com';const requestList = await RequestList.open({ sources: [{ url }] });@@ -1408,7 +1408,7 @@ describe('BasicCrawler', () => {const crawler = new BasicCrawler({requestList,- handleRequestTimeoutSecs: 0.01,+ requestHandlerTimeoutSecs: 0.01,maxRequestRetries: 1,useSessionPool: true,sessionPoolOptions: {@@ -1435,7 +1435,7 @@ describe('BasicCrawler', () => {const crawler = new BasicCrawler({requestList,- handleRequestTimeoutSecs: 0.01,+ requestHandlerTimeoutSecs: 0.01,maxRequestRetries: 1,useSessionPool: true,sessionPoolOptions: {@@ -1458,7 +1458,7 @@ describe('BasicCrawler', () => {const crawler = new BasicCrawler({requestList,- handleRequestTimeoutSecs: 0.01,+ requestHandlerTimeoutSecs: 0.01,maxRequestRetries: 1,useSessionPool: true,sessionPoolOptions: {diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.tsindex 6915b8f3c..76ad560f4 100644--- a/test/core/crawlers/http_crawler.test.ts+++ b/test/core/crawlers/http_crawler.test.ts@@ -2,8 +2,7 @@ import http from 'node:http';import type { AddressInfo } from 'node:net';import { Readable } from 'node:stream';-import { GotScrapingHttpClient, HttpCrawler } from '@crawlee/http';-import { ImpitHttpClient } from '@crawlee/impit-client';+import { HttpCrawler } from '@crawlee/http';import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js';const router = new Map<string, http.RequestListener>();@@ -25,6 +24,12 @@ router.set('/invalidContentType', (req, res) => {res.end(`<html><head><title>Example Domain</title></head></html>`);});+router.set('/setCookie', (req, res) => {+ res.setHeader('content-type', 'text/html');+ res.setHeader('set-cookie', 'first=1');+ res.end();+});+router.set('/redirectAndCookies', (req, res) => {res.setHeader('content-type', 'text/html');res.setHeader('set-cookie', 'foo=bar');@@ -38,12 +43,6 @@ router.set('/cookies', (req, res) => {res.end(JSON.stringify(req.headers.cookie));});-router.set('/setCookie', (req, res) => {- res.setHeader('content-type', 'text/html');- res.setHeader('set-cookie', 'first=1');- res.end();-});-router.set('/redirectWithoutCookies', (req, res) => {res.setHeader('location', '/cookies');res.statusCode = 302;@@ -101,406 +100,384 @@ afterAll(async () => {await localStorageEmulator.destroy();});-describe.each(- process.version.startsWith('v16')- ? [new GotScrapingHttpClient()]- : [new GotScrapingHttpClient(), new ImpitHttpClient()],-)('HttpCrawler with %s', (httpClient) => {- test('works', async () => {- const results: string[] = [];-- const crawler = new HttpCrawler({- httpClient,- maxRequestRetries: 0,- requestHandler: ({ body }) => {- results.push(body as string);- },- });-- await crawler.run([url]);+test('works', async () => {+ const results: string[] = [];- expect(results[0].includes('Example Domain')).toBeTruthy();+ const crawler = new HttpCrawler({+ maxRequestRetries: 0,+ requestHandler: ({ body }) => {+ results.push(body as string);+ },});- test('parseWithCheerio works', async () => {- const results: string[] = [];+ await crawler.run([url]);- const crawler = new HttpCrawler({- httpClient,- maxRequestRetries: 0,- requestHandler: async ({ parseWithCheerio }) => {- const $ = await parseWithCheerio('title');- results.push($('title').text());- },- });+ expect(results[0].includes('Example Domain')).toBeTruthy();+});- await crawler.run([`${url}/hello.html`]);+test('parseWithCheerio works', async () => {+ const results: string[] = [];- expect(results).toStrictEqual(['Example Domain']);+ const crawler = new HttpCrawler({+ maxRequestRetries: 0,+ requestHandler: async ({ parseWithCheerio }) => {+ const $ = await parseWithCheerio('title');+ results.push($('title').text());+ },});- test('should parse content type from header', async () => {- const results: { type: string; encoding: BufferEncoding }[] = [];+ await crawler.run([`${url}/hello.html`]);- const crawler = new HttpCrawler({- httpClient,- maxRequestRetries: 0,- requestHandler: ({ contentType }) => {- results.push(contentType);- },- });+ expect(results).toStrictEqual(['Example Domain']);+});- await crawler.run([url]);+test('should parse content type from header', async () => {+ const results: { type: string; encoding: BufferEncoding }[] = [];- expect(results).toStrictEqual([- {- type: 'text/html',- encoding: 'utf-8',- },- ]);+ const crawler = new HttpCrawler({+ maxRequestRetries: 0,+ requestHandler: ({ contentType }) => {+ results.push(contentType);+ },});- test('should parse content type from file extension', async () => {- const results: { type: string; encoding: BufferEncoding }[] = [];+ await crawler.run([url]);- const crawler = new HttpCrawler({- httpClient,- maxRequestRetries: 0,- requestHandler: ({ contentType }) => {- results.push(contentType);- },- });+ expect(results).toStrictEqual([+ {+ type: 'text/html',+ encoding: 'utf-8',+ },+ ]);+});- await crawler.run([`${url}/hello.html`]);+test('should parse content type from file extension', async () => {+ const results: { type: string; encoding: BufferEncoding }[] = [];- expect(results).toStrictEqual([- {- type: 'text/html',- encoding: 'utf-8',- },- ]);+ const crawler = new HttpCrawler({+ maxRequestRetries: 0,+ requestHandler: ({ contentType }) => {+ results.push(contentType);+ },});- test('no content type defaults to octet-stream', async () => {- const results: { type: string; encoding: BufferEncoding }[] = [];+ await crawler.run([`${url}/hello.html`]);- const crawler = new HttpCrawler({- httpClient,- maxRequestRetries: 0,- additionalMimeTypes: ['*/*'],- requestHandler: ({ contentType }) => {- results.push(contentType);- },- });+ expect(results).toStrictEqual([+ {+ type: 'text/html',+ encoding: 'utf-8',+ },+ ]);+});- await crawler.run([`${url}/noext`]);+test('no content type defaults to octet-stream', async () => {+ const results: { type: string; encoding: BufferEncoding }[] = [];- expect(results).toStrictEqual([- {- type: 'application/octet-stream',- encoding: 'utf-8',- },- ]);+ const crawler = new HttpCrawler({+ maxRequestRetries: 0,+ additionalMimeTypes: ['*/*'],+ requestHandler: ({ contentType }) => {+ results.push(contentType);+ },});- test('invalid content type defaults to octet-stream', async () => {- const results: { type: string; encoding: BufferEncoding }[] = [];+ await crawler.run([`${url}/noext`]);- const crawler = new HttpCrawler({- httpClient,- maxRequestRetries: 0,- additionalMimeTypes: ['*/*'],- requestHandler: ({ contentType }) => {- results.push(contentType);- },- });+ expect(results).toStrictEqual([+ {+ type: 'application/octet-stream',+ encoding: 'utf-8',+ },+ ]);+});- await crawler.run([`${url}/invalidContentType`]);+test('invalid content type defaults to octet-stream', async () => {+ const results: { type: string; encoding: BufferEncoding }[] = [];- expect(results).toStrictEqual([- {- type: 'application/octet-stream',- encoding: 'utf-8',- },- ]);+ const crawler = new HttpCrawler({+ maxRequestRetries: 0,+ additionalMimeTypes: ['*/*'],+ requestHandler: ({ contentType }) => {+ results.push(contentType);+ },});- test('handles cookies from redirects', async () => {- const results: string[] = [];+ await crawler.run([`${url}/invalidContentType`]);- const crawler = new HttpCrawler({- httpClient,- sessionPoolOptions: {- maxPoolSize: 1,- },- handlePageFunction: async ({ body }) => {- results.push(JSON.parse(body.toString()));- },- });+ expect(results).toStrictEqual([+ {+ type: 'application/octet-stream',+ encoding: 'utf-8',+ },+ ]);+});- await crawler.run([`${url}/redirectAndCookies`]);+test('handles cookies from redirects', async () => {+ const results: string[] = [];- expect(results).toStrictEqual(['foo=bar']);+ const crawler = new HttpCrawler({+ sessionPoolOptions: {+ maxPoolSize: 1,+ },+ requestHandler: async ({ body }) => {+ results.push(JSON.parse(body.toString()));+ },});- test('handles cookies from redirects when the session already has cookies', async () => {- const results: string[] = [];-- const crawler = new HttpCrawler({- httpClient,- sessionPoolOptions: {- maxPoolSize: 1,- // isolated so that cookies stored by the other tests / clients don't leak in- persistStateKey: `SDK_SESSION_POOL_STATE_${httpClient.constructor.name}`,- },- maxConcurrency: 1,- requestHandler: async ({ body }) => {- results.push(body.toString());- },- });+ await crawler.run([`${url}/redirectAndCookies`]);- await crawler.run([`${url}/setCookie`, `${url}/redirectAndCookies`]);+ expect(results).toStrictEqual(['foo=bar']);+});- expect(results[1]).toBe('"first=1; foo=bar"');+test('handles cookies from redirects when the session already has cookies', async () => {+ const results: string[] = [];++ const crawler = new HttpCrawler({+ sessionPoolOptions: {+ maxPoolSize: 1,+ // isolated so that cookies stored by the other tests don't leak in+ persistStateKey: 'SDK_SESSION_POOL_STATE_setCookie',+ },+ maxConcurrency: 1,+ requestHandler: async ({ body }) => {+ results.push(body.toString());+ },});- test('handles cookies from redirects - no empty cookie header', async () => {- const results: string[] = [];+ await crawler.run([`${url}/setCookie`, `${url}/redirectAndCookies`]);- const crawler = new HttpCrawler({- httpClient,- sessionPoolOptions: {- maxPoolSize: 1,- },- handlePageFunction: async ({ body }) => {- const str = body.toString();+ expect(results[1]).toBe('"first=1; foo=bar"');+});- if (str !== '') {- results.push(JSON.parse(str));- }- },- });+test('handles cookies from redirects - no empty cookie header', async () => {+ const results: string[] = [];- await crawler.run([`${url}/redirectWithoutCookies`]);+ const crawler = new HttpCrawler({+ sessionPoolOptions: {+ maxPoolSize: 1,+ },+ requestHandler: async ({ body }) => {+ const str = body.toString();- expect(results).toStrictEqual([]);+ if (str !== '') {+ results.push(JSON.parse(str));+ }+ },});- test('no empty cookie header', async () => {- const results: string[] = [];+ await crawler.run([`${url}/redirectWithoutCookies`]);- const crawler = new HttpCrawler({- httpClient,- sessionPoolOptions: {- maxPoolSize: 1,- },- handlePageFunction: async ({ body }) => {- const str = body.toString();+ expect(results).toStrictEqual([]);+});- if (str !== '') {- results.push(JSON.parse(str));- }- },- });+test('no empty cookie header', async () => {+ const results: string[] = [];- await crawler.run([`${url}/cookies`]);+ const crawler = new HttpCrawler({+ sessionPoolOptions: {+ maxPoolSize: 1,+ },+ requestHandler: async ({ body }) => {+ const str = body.toString();- expect(results).toStrictEqual([]);+ if (str !== '') {+ results.push(JSON.parse(str));+ }+ },});- test('POST with undefined (empty) payload', async () => {- const results: string[] = [];+ await crawler.run([`${url}/cookies`]);- const crawler = new HttpCrawler({- httpClient,- handlePageFunction: async ({ body }) => {- results.push(body.toString());- },- });+ expect(results).toStrictEqual([]);+});- await crawler.run([- {- url: `${url}/echo`,- payload: undefined,- method: 'POST',- },- ]);+test('POST with undefined (empty) payload', async () => {+ const results: string[] = [];- expect(results).toStrictEqual(['']);+ const crawler = new HttpCrawler({+ requestHandler: async ({ body }) => {+ results.push(body.toString());+ },});- test('should ignore http error status codes set by user', async () => {- const failed: any[] = [];-- const crawler = new HttpCrawler({- httpClient,- minConcurrency: 2,- maxConcurrency: 2,- ignoreHttpErrorStatusCodes: [500],- requestHandler: () => {},- failedRequestHandler: ({ request }) => {- failed.push(request);- },- });+ await crawler.run([+ {+ url: `${url}/echo`,+ payload: undefined,+ method: 'POST',+ },+ ]);- await crawler.run([`${url}/500Error`]);+ expect(results).toStrictEqual(['']);+});- expect(crawler.autoscaledPool!.minConcurrency).toBe(2);- expect(failed).toHaveLength(0);+test('should ignore http error status codes set by user', async () => {+ const failed: any[] = [];++ const crawler = new HttpCrawler({+ minConcurrency: 2,+ maxConcurrency: 2,+ ignoreHttpErrorStatusCodes: [500],+ requestHandler: () => {},+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },});- test('should throw an error on http error status codes set by user', async () => {- const failed: any[] = [];-- const crawler = new HttpCrawler({- httpClient,- minConcurrency: 2,- maxConcurrency: 2,- additionalHttpErrorStatusCodes: [200],- requestHandler: () => {},- failedRequestHandler: ({ request }) => {- failed.push(request);- },- });+ await crawler.run([`${url}/500Error`]);- await crawler.run([`${url}/hello.html`]);+ expect(crawler.autoscaledPool!.minConcurrency).toBe(2);+ expect(failed).toHaveLength(0);+});- expect(crawler.autoscaledPool!.minConcurrency).toBe(2);- expect(failed).toHaveLength(1);+test('should throw an error on http error status codes set by user', async () => {+ const failed: any[] = [];++ const crawler = new HttpCrawler({+ minConcurrency: 2,+ maxConcurrency: 2,+ additionalHttpErrorStatusCodes: [200],+ requestHandler: () => {},+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },});- test('should work with delete requests', async () => {- const failed: any[] = [];-- const cheerioCrawler = new HttpCrawler({- httpClient,- maxConcurrency: 1,- maxRequestRetries: 0,- navigationTimeoutSecs: 5,- requestHandlerTimeoutSecs: 5,- requestHandler: async () => {},- failedRequestHandler: async ({ request }) => {- failed.push(request);- },- });+ await crawler.run([`${url}/hello.html`]);- await cheerioCrawler.run([- {- url: `${url}`,- method: 'DELETE',- },- ]);+ expect(crawler.autoscaledPool!.minConcurrency).toBe(2);+ expect(failed).toHaveLength(1);+});- expect(failed).toHaveLength(0);+test('should work with delete requests', async () => {+ const failed: any[] = [];++ const cheerioCrawler = new HttpCrawler({+ maxConcurrency: 1,+ maxRequestRetries: 0,+ navigationTimeoutSecs: 5,+ requestHandlerTimeoutSecs: 5,+ requestHandler: async () => {},+ failedRequestHandler: async ({ request }) => {+ failed.push(request);+ },});- test('should retry on 403 even with disallowed content-type', async () => {- const succeeded: any[] = [];-- const crawler = new HttpCrawler({- httpClient,- maxConcurrency: 1,- maxRequestRetries: 1,- preNavigationHooks: [- async ({ request }) => {- // mock 403 response with octet stream on first request attempt, but not on- // subsequent retries, so the request should eventually succeed- if (request.retryCount === 0) {- request.url = `${url}/403-with-octet-stream`;- } else {- request.url = url;- }- },- ],- requestHandler: async ({ request }) => {- succeeded.push(request);- },- });+ await cheerioCrawler.run([+ {+ url: `${url}`,+ method: 'DELETE',+ },+ ]);- await crawler.run([url]);+ expect(failed).toHaveLength(0);+});- expect(succeeded).toHaveLength(1);- expect(succeeded[0].retryCount).toBe(1);+test('should retry on 403 even with disallowed content-type', async () => {+ const succeeded: any[] = [];++ const crawler = new HttpCrawler({+ maxConcurrency: 1,+ maxRequestRetries: 1,+ preNavigationHooks: [+ async ({ request }) => {+ // mock 403 response with octet stream on first request attempt, but not on+ // subsequent retries, so the request should eventually succeed+ if (request.retryCount === 0) {+ request.url = `${url}/403-with-octet-stream`;+ } else {+ request.url = url;+ }+ },+ ],+ requestHandler: async ({ request }) => {+ succeeded.push(request);+ },});- test.skipIf(httpClient instanceof ImpitHttpClient)('should work with cacheable-request', async () => {- const isFromCache: Record<string, boolean> = {};- const cache = new Map();- const crawler = new HttpCrawler({- httpClient,- maxConcurrency: 1,- preNavigationHooks: [- async (_, gotOptions) => {- gotOptions.cache = cache;- gotOptions.headers = {- ...gotOptions.headers,- // to force cache- 'cache-control': 'max-stale',- };- },- ],- requestHandler: async ({ request, response }) => {- isFromCache[request.uniqueKey] = response.isFromCache;+ await crawler.run([url]);++ expect(succeeded).toHaveLength(1);+ expect(succeeded[0].retryCount).toBe(1);+});++test('should work with cacheable-request', async () => {+ const isFromCache: Record<string, boolean> = {};+ const cache = new Map();+ const crawler = new HttpCrawler({+ maxConcurrency: 1,+ preNavigationHooks: [+ async (_, gotOptions) => {+ gotOptions.cache = cache;+ gotOptions.headers = {+ ...gotOptions.headers,+ // to force cache+ 'cache-control': 'max-stale',+ };},- });- await crawler.run([- { url, uniqueKey: 'first' },- { url, uniqueKey: 'second' },- ]);- expect(isFromCache).toEqual({ first: false, second: true });+ ],+ requestHandler: async ({ request, response }) => {+ isFromCache[request.uniqueKey] = response.isFromCache;+ },});+ await crawler.run([+ { url, uniqueKey: 'first' },+ { url, uniqueKey: 'second' },+ ]);+ expect(isFromCache).toEqual({ first: false, second: true });+});- test('works with a custom HttpClient', async () => {- const results: string[] = [];+test('works with a custom HttpClient', async () => {+ const results: string[] = [];- const crawler = new HttpCrawler({- maxRequestRetries: 0,- requestHandler: async ({ body, sendRequest }) => {- results.push(body as string);+ const crawler = new HttpCrawler({+ maxRequestRetries: 0,+ requestHandler: async ({ body, sendRequest }) => {+ results.push(body as string);- results.push((await sendRequest()).body);+ results.push((await sendRequest()).body);+ },+ httpClient: {+ async sendRequest(request) {+ if (request.responseType !== 'text') {+ throw new Error('Not implemented');+ }++ return {+ body: 'Hello from sendRequest()' as any,+ request,+ url,+ redirectUrls: [],+ statusCode: 200,+ headers: {},+ trailers: {},+ complete: true,+ };},- httpClient: {- async sendRequest(request) {- if (request.responseType !== 'text') {- throw new Error('Not implemented');- }-- return {- body: 'Hello from sendRequest()' as any,- request,- url,- redirectUrls: [],- statusCode: 200,- headers: {},- trailers: {},- complete: true,- };- },- async stream(request) {- const stream = new Readable();- stream.push('<html><head><title>Schmexample Domain</title></head></html>');- stream.push(null);-- return {- stream,- downloadProgress: { percent: 100, transferred: 0 },- uploadProgress: { percent: 100, transferred: 0 },- request,- url,- redirectUrls: [],- statusCode: 200,- headers: { 'content-type': 'text/html; charset=utf-8' },- trailers: {},- complete: true,- };- },+ async stream(request) {+ const stream = new Readable();+ stream.push('<html><head><title>Schmexample Domain</title></head></html>');+ stream.push(null);++ return {+ stream,+ downloadProgress: { percent: 100, transferred: 0 },+ uploadProgress: { percent: 100, transferred: 0 },+ request,+ url,+ redirectUrls: [],+ statusCode: 200,+ headers: { 'content-type': 'text/html; charset=utf-8' },+ trailers: {},+ complete: true,+ };},- });+ },+ });- await crawler.run([url]);+ await crawler.run([url]);- expect(results[0].includes('Schmexample Domain')).toBeTruthy();- expect(results[1].includes('Hello')).toBeTruthy();- });+ expect(results[0].includes('Schmexample Domain')).toBeTruthy();+ expect(results[1].includes('Hello')).toBeTruthy();});diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.tsindex 49e5d2441..6bda30384 100644--- a/test/core/playwright_utils.test.ts+++ b/test/core/playwright_utils.test.ts@@ -50,9 +50,13 @@ describe('playwrightUtils', () => {// @ts-expect-errorlet result = await page.evaluate(() => window.injectedVariable === 42);expect(result).toBe(false);- await playwrightUtils.injectFile(page, path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'), {- surviveNavigations: true,- });+ await playwrightUtils.injectFile(+ page,+ path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'),+ {+ surviveNavigations: true,+ },+ );// @ts-expect-errorresult = await page.evaluate(() => window.injectedVariable);expect(result).toBe(42);@@ -75,7 +79,10 @@ describe('playwrightUtils', () => {// @ts-expect-errorresult = await page.evaluate(() => window.injectedVariable === 42);expect(result).toBe(false);- await playwrightUtils.injectFile(page, path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'));+ await playwrightUtils.injectFile(+ page,+ path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'),+ );// @ts-expect-errorresult = await page.evaluate(() => window.injectedVariable);expect(result).toBe(42);diff --git a/test/core/puppeteer_utils.test.ts b/test/core/puppeteer_utils.test.tsindex fc3399893..b43b6a83f 100644--- a/test/core/puppeteer_utils.test.ts+++ b/test/core/puppeteer_utils.test.ts@@ -52,9 +52,13 @@ describe('puppeteerUtils', () => {// @ts-expect-errorlet result = await page.evaluate(() => window.injectedVariable === 42);expect(result).toBe(false);- await puppeteerUtils.injectFile(page, path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'), {- surviveNavigations: true,- });+ await puppeteerUtils.injectFile(+ page,+ path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'),+ {+ surviveNavigations: true,+ },+ );// @ts-expect-errorresult = await page.evaluate(() => window.injectedVariable);expect(result).toBe(42);@@ -77,7 +81,10 @@ describe('puppeteerUtils', () => {// @ts-expect-errorresult = await page.evaluate(() => window.injectedVariable === 42);expect(result).toBe(false);- await puppeteerUtils.injectFile(page, path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'));+ await puppeteerUtils.injectFile(+ page,+ path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'),+ );// @ts-expect-errorresult = await page.evaluate(() => window.injectedVariable);expect(result).toBe(42);diff --git a/test/e2e/adaptive-playwright-robots-file/test.mjs b/test/e2e/adaptive-playwright-robots-file/test.mjsindex b58d0ff34..5e7a2c4df 100644--- a/test/e2e/adaptive-playwright-robots-file/test.mjs+++ b/test/e2e/adaptive-playwright-robots-file/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/cheerio-curl-impersonate-ts/test.mjs b/test/e2e/cheerio-curl-impersonate-ts/test.mjsindex 7ff4fce14..48aea4fe7 100644--- a/test/e2e/cheerio-curl-impersonate-ts/test.mjs+++ b/test/e2e/cheerio-curl-impersonate-ts/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/cheerio-enqueue-links/test.mjs b/test/e2e/cheerio-enqueue-links/test.mjsindex c105fab0d..2d0009abc 100644--- a/test/e2e/cheerio-enqueue-links/test.mjs+++ b/test/e2e/cheerio-enqueue-links/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/cheerio-error-snapshot/test.mjs b/test/e2e/cheerio-error-snapshot/test.mjsindex a41eef937..0b857750a 100644--- a/test/e2e/cheerio-error-snapshot/test.mjs+++ b/test/e2e/cheerio-error-snapshot/test.mjs@@ -1,4 +1,4 @@-import { expect, getActorTestDir, hasNestedKey,initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, hasNestedKey, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/cheerio-impit-ts/test.mjs b/test/e2e/cheerio-impit-ts/test.mjsindex f86d9e72d..218055485 100644--- a/test/e2e/cheerio-impit-ts/test.mjs+++ b/test/e2e/cheerio-impit-ts/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/cheerio-initial-cookies/test.mjs b/test/e2e/cheerio-initial-cookies/test.mjsindex fa3edf7e7..136a7d032 100644--- a/test/e2e/cheerio-initial-cookies/test.mjs+++ b/test/e2e/cheerio-initial-cookies/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/cheerio-max-requests/test.mjs b/test/e2e/cheerio-max-requests/test.mjsindex 098af799d..f3b80998f 100644--- a/test/e2e/cheerio-max-requests/test.mjs+++ b/test/e2e/cheerio-max-requests/test.mjs@@ -1,4 +1,4 @@-import { expect, getActorTestDir, initialize, runActor,validateDataset } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/cheerio-page-info/test.mjs b/test/e2e/cheerio-page-info/test.mjsindex b81c91abb..db70e11af 100644--- a/test/e2e/cheerio-page-info/test.mjs+++ b/test/e2e/cheerio-page-info/test.mjs@@ -1,4 +1,4 @@-import { expect, getActorTestDir, initialize, runActor,validateDataset } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/cheerio-robots-file/test.mjs b/test/e2e/cheerio-robots-file/test.mjsindex ef503d9c4..df7072351 100644--- a/test/e2e/cheerio-robots-file/test.mjs+++ b/test/e2e/cheerio-robots-file/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/cheerio-stop-resume-ts/test.mjs b/test/e2e/cheerio-stop-resume-ts/test.mjsindex f72492718..8beaf8681 100644--- a/test/e2e/cheerio-stop-resume-ts/test.mjs+++ b/test/e2e/cheerio-stop-resume-ts/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/jsdom-react-ts/test.mjs b/test/e2e/jsdom-react-ts/test.mjsindex 9f050695e..69c265224 100644--- a/test/e2e/jsdom-react-ts/test.mjs+++ b/test/e2e/jsdom-react-ts/test.mjs@@ -1,4 +1,4 @@-import { expect, getActorTestDir, initialize, runActor, skipTest,validateDataset } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor, skipTest, validateDataset } from '../tools.mjs';await skipTest('target site no longer exists');diff --git a/test/e2e/playwright-enqueue-links/test.mjs b/test/e2e/playwright-enqueue-links/test.mjsindex 5adabeae9..7dea0d946 100644--- a/test/e2e/playwright-enqueue-links/test.mjs+++ b/test/e2e/playwright-enqueue-links/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/playwright-initial-cookies/test.mjs b/test/e2e/playwright-initial-cookies/test.mjsindex 620a12f47..012966452 100644--- a/test/e2e/playwright-initial-cookies/test.mjs+++ b/test/e2e/playwright-initial-cookies/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/playwright-introduction-guide/test.mjs b/test/e2e/playwright-introduction-guide/test.mjsindex ee8780c4c..93a2a1609 100644--- a/test/e2e/playwright-introduction-guide/test.mjs+++ b/test/e2e/playwright-introduction-guide/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/playwright-multi-run/test.mjs b/test/e2e/playwright-multi-run/test.mjsindex fc8643704..55e1c47b0 100644--- a/test/e2e/playwright-multi-run/test.mjs+++ b/test/e2e/playwright-multi-run/test.mjs@@ -1,4 +1,4 @@-import { expect, getActorTestDir, initialize, runActor, skipTest,validateDataset } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor, skipTest, validateDataset } from '../tools.mjs';if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') {await skipTest('not supported on platform');diff --git a/test/e2e/playwright-robots-file/test.mjs b/test/e2e/playwright-robots-file/test.mjsindex d425adeec..b8f384d4e 100644--- a/test/e2e/playwright-robots-file/test.mjs+++ b/test/e2e/playwright-robots-file/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/puppeteer-enqueue-links/test.mjs b/test/e2e/puppeteer-enqueue-links/test.mjsindex 5adabeae9..7dea0d946 100644--- a/test/e2e/puppeteer-enqueue-links/test.mjs+++ b/test/e2e/puppeteer-enqueue-links/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/puppeteer-error-snapshot/test.mjs b/test/e2e/puppeteer-error-snapshot/test.mjsindex 87617e699..7306e295d 100644--- a/test/e2e/puppeteer-error-snapshot/test.mjs+++ b/test/e2e/puppeteer-error-snapshot/test.mjs@@ -1,4 +1,4 @@-import { expect, getActorTestDir, hasNestedKey,initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, hasNestedKey, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/puppeteer-initial-cookies/test.mjs b/test/e2e/puppeteer-initial-cookies/test.mjsindex 620a12f47..012966452 100644--- a/test/e2e/puppeteer-initial-cookies/test.mjs+++ b/test/e2e/puppeteer-initial-cookies/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/request-queue-with-concurrency/test.mjs b/test/e2e/request-queue-with-concurrency/test.mjsindex 633be23e6..5558860c2 100644--- a/test/e2e/request-queue-with-concurrency/test.mjs+++ b/test/e2e/request-queue-with-concurrency/test.mjs@@ -3,7 +3,7 @@ import { setTimeout } from 'node:timers/promises';import { Actor } from 'apify';import { log } from 'crawlee';-import { expect,getActorTestDir, initialize, pushActor, startActorOnPlatform } from '../tools.mjs';+import { expect, getActorTestDir, initialize, pushActor, startActorOnPlatform } from '../tools.mjs';if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') {const testActorDirname = getActorTestDir(import.meta.url);diff --git a/test/e2e/request-queue-zero-concurrency/test.mjs b/test/e2e/request-queue-zero-concurrency/test.mjsindex ff8bf40d6..4c3e1eee0 100644--- a/test/e2e/request-queue-zero-concurrency/test.mjs+++ b/test/e2e/request-queue-zero-concurrency/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/request-skip-navigation/test.mjs b/test/e2e/request-skip-navigation/test.mjsindex 5cc25546d..a83abf7cf 100644--- a/test/e2e/request-skip-navigation/test.mjs+++ b/test/e2e/request-skip-navigation/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/session-rotation/test.mjs b/test/e2e/session-rotation/test.mjsindex 2e99e2856..d6d72e9ff 100644--- a/test/e2e/session-rotation/test.mjs+++ b/test/e2e/session-rotation/test.mjs@@ -1,4 +1,4 @@-import { expect,getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjsindex fb548c9b2..515bb5fd0 100644--- a/test/e2e/tools.mjs+++ b/test/e2e/tools.mjs@@ -6,7 +6,7 @@ import { dirname, join } from 'node:path';import { setTimeout } from 'node:timers/promises';import { fileURLToPath } from 'node:url';-import { URL_NO_COMMAS_REGEX } from "@crawlee/utils";+import { URL_NO_COMMAS_REGEX } from '@crawlee/utils';import { Actor, ApifyClient } from 'apify';import fs from 'fs-extra';import { got } from 'got';diff --git a/test/tsconfig.json b/test/tsconfig.jsonindex d50c76d76..1202ae509 100644--- a/test/tsconfig.json+++ b/test/tsconfig.json@@ -2,9 +2,9 @@"extends": "../tsconfig.json","include": ["**/*", "../packages/*/src/**/*"],"exclude": ["e2e", "**/fixtures/*"],- "compilerOptions": {- "module": "NodeNext",- "moduleResolution": "NodeNext",+ "compilerOptions": {+ "module": "NodeNext",+ "moduleResolution": "NodeNext","sourceMap": true,"noUnusedLocals": false,"noUnusedParameters": false,diff --git a/tsconfig.build.json b/tsconfig.build.jsonindex 9b733418e..cdf9a2713 100644--- a/tsconfig.build.json+++ b/tsconfig.build.json@@ -1,8 +1,8 @@{"extends": "@apify/tsconfig","compilerOptions": {- "module": "NodeNext",- "moduleResolution": "NodeNext",+ "module": "NodeNext",+ "moduleResolution": "NodeNext","target": "ESNext","lib": ["DOM", "ES2023"],"types": ["node"],diff --git a/tsconfig.json b/tsconfig.jsonindex 6136fd01b..b3ad1bd0b 100644--- a/tsconfig.json+++ b/tsconfig.json@@ -2,9 +2,9 @@"extends": "./tsconfig.build.json","compilerOptions": {"baseUrl": ".",- "noErrorTruncation": true,- "sourceMap": true,- "declaration": true,+ "noErrorTruncation": true,+ "sourceMap": true,+ "declaration": true,"paths": {"crawlee": ["packages/crawlee/src/index.ts"],"@crawlee/basic": ["packages/basic-crawler/src/index.ts"],