Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor!: Introduce the `ContextPipeline` abstraction (#3119)
Jan Buchar70aa8c9df19e
diff --git a/docs/guides/request_storage_queue_crawler.ts b/docs/guides/request_storage_queue_crawler.tsindex 07af11ffa..d9c37f57f 100644--- a/docs/guides/request_storage_queue_crawler.ts+++ b/docs/guides/request_storage_queue_crawler.ts@@ -4,7 +4,7 @@ import { CheerioCrawler } from 'crawlee';// It's used the same way for Puppeteer/Playwright crawlers.const crawler = new CheerioCrawler({// Note that we're not specifying the requestQueue here- async requestHandler({ crawler, enqueueLinks }) {+ async requestHandler({ enqueueLinks }) {// Add new request to the queueawait crawler.addRequests([{ url: 'https://example.com/new-page' }]);// Add links found on page to the queuediff --git a/docs/guides/request_storage_queue_list.ts b/docs/guides/request_storage_queue_list.tsindex 456caf1c0..37de17eec 100644--- a/docs/guides/request_storage_queue_list.ts+++ b/docs/guides/request_storage_queue_list.ts@@ -25,7 +25,7 @@ const crawler = new PuppeteerCrawler({requestQueue,// Each request from the request list is enqueued to the request queue one by one.// At this point request with the same URL would exist in the list and the queue- async requestHandler({ crawler, enqueueLinks }) {+ async requestHandler({ enqueueLinks }) {// Add new request to the queueawait crawler.addRequests(['http://www.example.com/new-page']);diff --git a/docs/guides/request_storage_queue_only.ts b/docs/guides/request_storage_queue_only.tsindex 5d9a31379..305413550 100644--- a/docs/guides/request_storage_queue_only.ts+++ b/docs/guides/request_storage_queue_only.ts@@ -15,7 +15,7 @@ const sources = [// The crawler will automatically process requests from the queue.// It's used the same way for Cheerio/Playwright crawlersconst crawler = new PuppeteerCrawler({- async requestHandler({ crawler, enqueueLinks }) {+ async requestHandler({ enqueueLinks }) {// Add new request to the queueawait crawler.addRequests(['http://www.example.com/new-page']);diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex 56f527752..22a3ff28d 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -32,10 +32,53 @@ The crawler following options are removed:- `handleRequestTimeoutSecs` -> `requestHandlerTimeoutSecs`- `handleFailedRequestFunction` -> `failedRequestHandler`-## Crawling context no longer includes Error for failed requests+## Underscore prefix is removed from many protected and private methods++- `BasicCrawler._runRequestHandler` -> `BasicCrawler.runRequestHandler`++## Removed symbols++- `BasicCrawler._cleanupContext` (protected) - this is now handled by the `ContextPipeline`+- `BasicCrawler.isRequestBlocked` (protected)+- `BrowserRequestHandler` and `BrowserErrorHandler` types in `@crawlee/browser`+- `BrowserCrawler.userProvidedRequestHandler` (protected)+- `BrowserCrawler.requestHandlerTimeoutInnerMillis` (protected)+- `BrowserCrawler._enhanceCrawlingContextWithPageInfo` (protected)+- `BrowserCrawler._handleNavigation` (protected)+- `HttpCrawler.userRequestHandlerTimeoutMillis` (protected)+- `HttpCrawler._handleNavigation` (protected)+- `HttpCrawler._parseHTML` (protected)+- `HttpCrawler._parseResponse` (protected) - made private+- `HttpCrawler.use` and the `CrawlerExtension` class (experimental) - the `ContextPipeline` should be used for extending the crawler+- `FileDownloadOptions.streamHandler` - streaming should now be handled directly in the `requestHandler` instead+- `playwrightUtils.registerUtilsToContext` and `puppeteerUtils.registerUtilsToContext` - this is now added to the context via `ContextPipeline` composition+- `puppeteerUtils.blockResources` and `puppeteerUtils.cacheResponses` (deprecated)++### The protected `BasicCrawler.crawlingContexts` map is removed++The property was not used by the library itself and re-implementing the functionality in user code is fairly straightforward.++## Removed crawling context properties++### Crawling context no longer includes Error for failed requestsThe crawling context no longer includes the `Error` object for failed requests. Use the second parameter of the `errorHandler` or `failedRequestHandler` callbacks to access the error.+### Crawling context no longer includes a reference to the crawler itself++This was previously accessible via `context.crawler`. If you want to restore the functionality, you may use the `extendContext` option of the crawler:++```ts+const crawler = new CheerioCrawler({+ extendContext: () => ({ crawler }),+ requestHandler: async (context) => {+ if (Math.random() < 0.01) {+ context.crawler.stop()+ }+ }+})+```+## Crawling context is strictly typedPreviously, the crawling context extended a `Record` type, allowing to access any property. This was changed to a strict type, which means that you can only access properties that are defined in the context.diff --git a/packages/basic-crawler/src/index.ts b/packages/basic-crawler/src/index.tsindex 3aee898c6..df955c3f9 100644--- a/packages/basic-crawler/src/index.ts+++ b/packages/basic-crawler/src/index.ts@@ -1,4 +1,3 @@export * from '@crawlee/core';export * from './internals/basic-crawler.js';-export * from './internals/constants.js';-export { CheerioRoot, CheerioAPI, Cheerio, Element } from '@crawlee/utils';+export { CheerioRoot, CheerioAPI, Cheerio } from '@crawlee/utils';diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 0b3a6d014..60301f8df 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -14,12 +14,10 @@ import type {GetUserDataFromRequest,IRequestList,IRequestManager,- LoadedContext,ProxyInfo,Request,RequestsLike,RequestTransform,- RestrictedCrawlingContext,RouterHandler,RouterRoutes,Session,@@ -32,6 +30,10 @@ import type {import {AutoscaledPool,Configuration,+ ContextPipeline,+ ContextPipelineCleanupError,+ ContextPipelineInitializationError,+ ContextPipelineInterruptedError,CriticalError,Dataset,enqueueLinks,@@ -42,6 +44,7 @@ import {mergeCookies,NonRetryableError,purgeDefaultStorages,+ RequestHandlerError,RequestListAdapter,RequestManagerTandem,RequestProvider,@@ -62,7 +65,7 @@ import { stringify } from 'csv-stringify/sync';import { ensureDir, writeJSON } from 'fs-extra/esm';import ow from 'ow';import { getDomain } from 'tldts';-import type { SetRequired } from 'type-fest';+import type { ReadonlyDeep, SetRequired } from 'type-fest';import { LruCache } from '@apify/datastructures';import type { Log } from '@apify/log';@@ -72,33 +75,7 @@ import { cryptoRandomObjectId } from '@apify/utilities';import { createSendRequest } from './send-request.js';-export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary>- extends CrawlingContext<BasicCrawler, UserData> {- /**- * This function automatically finds and enqueues links from the current page, adding them to the {@apilink RequestQueue}- * currently used by the crawler.- *- * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions- * and override settings of the enqueued {@apilink Request} objects.- *- * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example- * for more details regarding its usage.- *- * **Example usage**- *- * ```ts- * async requestHandler({ enqueueLinks }) {- * await enqueueLinks({- * urls: [...],- * });- * },- * ```- *- * @param [options] All `enqueueLinks()` parameters are passed via an options object.- * @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.- */- enqueueLinks(options?: SetRequired<EnqueueLinksOptions, 'urls'>): Promise<BatchAddRequestsResult>;-}+export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {}/*** Since there's no set number of seconds before the container is terminated after@@ -111,13 +88,12 @@ export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary>*/const SAFE_MIGRATION_WAIT_MILLIS = 20000;-export type RequestHandler<- Context extends CrawlingContext = LoadedContext<BasicCrawlingContext & RestrictedCrawlingContext>,-> = (inputs: LoadedContext<Context>) => Awaitable<void>;+export type RequestHandler<Context extends CrawlingContext = CrawlingContext> = (inputs: Context) => Awaitable<void>;export type ErrorHandler<- Context extends CrawlingContext = LoadedContext<BasicCrawlingContext & RestrictedCrawlingContext>,-> = (inputs: LoadedContext<Context>, error: Error) => Awaitable<void>;+ Context extends CrawlingContext = CrawlingContext,+ ExtendedContext extends Context = Context,+> = (inputs: Context & Partial<ExtendedContext>, error: Error) => Awaitable<void>;export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext,@@ -134,7 +110,18 @@ export type StatusMessageCallback<Crawler extends BasicCrawler<any> = BasicCrawler<Context>,> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;-export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCrawlingContext> {+export type RequireContextPipeline<+ DefaultContextType extends CrawlingContext,+ FinalContextType extends DefaultContextType,+> = DefaultContextType extends FinalContextType+ ? {}+ : { contextPipelineBuilder: () => ContextPipeline<CrawlingContext, FinalContextType> };++export interface BasicCrawlerOptions<+ Context extends CrawlingContext = CrawlingContext,+ ContextExtension = {},+ ExtendedContext extends Context = Context & ContextExtension,+> {/*** User-provided function that performs the logic of the crawler. It is called for each URL to crawl.*@@ -152,7 +139,37 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw* The exceptions are logged to the request using the* {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function.*/- requestHandler?: RequestHandler<Context>;+ requestHandler?: RequestHandler<ExtendedContext>;++ /**+ * Allows the user to extend the crawling context passed to the request handler with custom functionality.+ *+ * **Example usage:**+ *+ * ```javascript+ * import { BasicCrawler } from 'crawlee';+ *+ * // Create a crawler instance+ * const crawler = new BasicCrawler({+ * extendContext(context) => ({+ * async customHelper() {+ * await context.pushData({ url: context.request.url })+ * }+ * }),+ * async requestHandler(context) {+ * await context.customHelper();+ * },+ * });+ * ```+ */+ extendContext?: (context: Context) => Awaitable<ContextExtension>;++ /**+ * *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.+ *+ * The option is not required if your crawler subclass does not extend the crawling context with custom information or helpers.+ */+ contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;/*** Static list of URLs to be processed.@@ -193,7 +210,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw* Second argument is the `Error` instance that* represents the last error thrown during processing of the request.*/- errorHandler?: ErrorHandler<Context>;+ errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;/*** A function to handle requests that failed more than {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.@@ -203,7 +220,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw* Second argument is the `Error` instance that* represents the last error thrown during processing of the request.*/- failedRequestHandler?: ErrorHandler<Context>;+ failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;/*** Specifies the maximum number of retries allowed for a request if its processing fails.@@ -449,7 +466,11 @@ export interface CrawlerExperiments {* ```* @category Crawlers*/-export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext> {+export class BasicCrawler<+ Context extends CrawlingContext = CrawlingContext,+ ContextExtension = {},+ ExtendedContext extends Context = Context & ContextExtension,+> {protected static readonly CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';/**@@ -494,16 +515,27 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext* Default {@apilink Router} instance that will be used if we don't specify any {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}.* See {@apilink Router.addHandler|`router.addHandler()`} and {@apilink Router.addDefaultHandler|`router.addDefaultHandler()`}.*/- readonly router: RouterHandler<LoadedContext<Context>> = Router.create<LoadedContext<Context>>();+ readonly router: RouterHandler<Context> = Router.create<Context>();++ private contextPipelineBuilder: () => ContextPipeline<CrawlingContext, ExtendedContext>;+ private _contextPipeline?: ContextPipeline<CrawlingContext, ExtendedContext>;++ get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext> {+ if (this._contextPipeline === undefined) {+ this._contextPipeline = this.contextPipelineBuilder();+ }++ return this._contextPipeline;+ }running = false;hasFinishedBefore = false;protected unexpectedStop = false;readonly log: Log;- protected requestHandler!: RequestHandler<Context>;- protected errorHandler?: ErrorHandler<Context>;- protected failedRequestHandler?: ErrorHandler<Context>;+ protected requestHandler!: RequestHandler<ExtendedContext>;+ protected errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;+ protected failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;protected requestHandlerTimeoutMillis!: number;protected internalTimeoutMillis: number;protected maxRequestRetries: number;@@ -529,7 +561,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextprotected statusMessageCallback?: StatusMessageCallback;protected sessionPoolOptions: SessionPoolOptions;protected useSessionPool: boolean;- protected crawlingContexts = new Map<string, Context>();protected autoscaledPoolOptions: AutoscaledPoolOptions;protected events: EventManager;protected httpClient: BaseHttpClient;@@ -543,6 +574,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextprivate _experimentWarnings: Partial<Record<keyof CrawlerExperiments, boolean>> = {};protected static optionsShape = {+ contextPipelineBuilder: ow.optional.object,+ extendContext: ow.optional.function,+requestList: ow.optional.object.validate(validators.requestList),requestQueue: ow.optional.object.validate(validators.requestQueue),// Subclasses override this function instead of passing it@@ -586,7 +620,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext* All `BasicCrawler` parameters are passed via an options object.*/constructor(- options: BasicCrawlerOptions<Context> = {},+ options: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> &+ RequireContextPipeline<CrawlingContext, Context> = {} as any, // cast because the constructor logic handles missing `contextPipelineBuilder` - the type is just for DXreadonly config = Configuration.getGlobalConfig(),) {ow(options, 'BasicCrawlerOptions', ow.object.exactShape(BasicCrawler.optionsShape));@@ -627,6 +662,40 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextexperiments = {},} = options;+ // Store the builder so that it can be run when the contextPipeline is needed.+ // Invoking it immediately would cause problems with parent constructor call order.+ this.contextPipelineBuilder = () => {+ let contextPipeline = (options.contextPipelineBuilder?.() ??+ ContextPipeline.create<CrawlingContext>()) as ContextPipeline<CrawlingContext, Context>; // Thanks to the RequireContextPipeline, contextPipeline will only be undefined if InitialContextType is CrawlingContext++ if (options.extendContext !== undefined) {+ contextPipeline = contextPipeline.compose({+ action: async (context) => await options.extendContext(context),+ });+ }++ contextPipeline = contextPipeline.compose({+ action: async (context) => {+ const { request } = context;+ if (!this.requestMatchesEnqueueStrategy(request)) {+ // eslint-disable-next-line dot-notation+ const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;+ this.log.debug(message);++ request.noRetry = true;+ request.state = RequestState.SKIPPED;++ await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });++ throw new ContextPipelineInterruptedError(message);+ }+ return context;+ },+ });++ return contextPipeline as ContextPipeline<CrawlingContext, ExtendedContext>;+ };+if (requestManager !== undefined) {if (requestList !== undefined || requestQueue !== undefined) {throw new Error(@@ -700,7 +769,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext}}this.useSessionPool = useSessionPool;- this.crawlingContexts = new Map();const maxSignedInteger = 2 ** 31 - 1;if (this.requestHandlerTimeoutMillis > maxSignedInteger) {@@ -799,15 +867,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextreturn ROTATE_PROXY_ERRORS.some((x: string) => (this._getMessageFromError(error) as any)?.includes(x));}- /**- * Checks whether the given crawling context is getting blocked by anti-bot protection using several heuristics.- * Returns `false` if the request is not blocked, otherwise returns a string with a description of the block reason.- * @param _crawlingContext The crawling context to check.- */- protected async isRequestBlocked(_crawlingContext: Context): Promise<string | false> {- throw new Error('the "isRequestBlocked" method is not implemented in this crawler.');- }-/*** This method is periodically called by the crawler, every `statusMessageLoggingInterval` seconds.*/@@ -1155,7 +1214,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext* @param options Options for the request queue*/async addRequests(- requests: RequestsLike,+ requests: ReadonlyDeep<RequestsLike>,options: CrawlerAddRequestsOptions = {},): Promise<CrawlerAddRequestsResult> {await this.getRequestQueue();@@ -1331,8 +1390,14 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextawait this.initializeRequestManager();}- protected async _runRequestHandler(crawlingContext: Context): Promise<void> {- await this.requestHandler(crawlingContext as LoadedContext<Context>);+ protected async runRequestHandler(crawlingContext: CrawlingContext): Promise<void> {+ await this.contextPipeline.call(crawlingContext, async (finalContext) => {+ await addTimeoutToPromise(+ async () => this.requestHandler(finalContext),+ this.requestHandlerTimeoutMillis,+ `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds (${finalContext.request.id}).`,+ );+ });}/**@@ -1453,12 +1518,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextreturn this.requestManager.fetchNextRequest();}- /**- * Executed when `errorHandler` finishes or the request is successful.- * Can be used to clean up orphaned browser pages.- */- protected async _cleanupContext(_crawlingContext: Context) {}-/*** Delays processing of the request based on the `sameDomainDelaySecs` option,* adding it back to the queue after the timeout passes. Returns `true` if the request@@ -1552,43 +1611,38 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextconst statisticsId = request.id || request.uniqueKey;this.stats.startJob(statisticsId);- // Shared crawling context- // @ts-expect-error- // All missing properties (that extend CrawlingContext) are set dynamically,- // but TS does not know that, so otherwise it would throw when compiling.- const crawlingContext: Context = {+ const deferredCleanup: (() => Promise<unknown>)[] = [];++ const crawlingContext: CrawlingContext = {id: cryptoRandomObjectId(10),- crawler: this,log: this.log,request,session,- enqueueLinks: async (options: SetRequired<EnqueueLinksOptions, 'urls'>) => {+ enqueueLinks: async (options) => {const requestQueue = await this.getRequestQueue();- return this.enqueueLinksWithCrawlDepth(options, request, requestQueue);+ return await this.enqueueLinksWithCrawlDepth(options, request, requestQueue);},- addRequests: async (requests: RequestsLike, options: CrawlerAddRequestsOptions = {}) => {+ addRequests: async (requests, options = {}) => {const newCrawlDepth = request.crawlDepth + 1;const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);- return this.addRequests(requestsGenerator, options);+ await this.addRequests(requestsGenerator, options);},pushData: this.pushData.bind(this),useState: this.useState.bind(this),sendRequest: createSendRequest(this.httpClient, request, session, () => crawlingContext.proxyInfo?.url),getKeyValueStore: async (idOrName?: string) => KeyValueStore.open(idOrName, { config: this.config }),+ registerDeferredCleanup: (cleanup) => {+ deferredCleanup.push(cleanup);+ },};- this.crawlingContexts.set(crawlingContext.id, crawlingContext);let isRequestLocked = true;try {request.state = RequestState.REQUEST_HANDLER;- await addTimeoutToPromise(- async () => this._runRequestHandler(crawlingContext),- this.requestHandlerTimeoutMillis,- `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds (${request.id}).`,- );+ await this.runRequestHandler(crawlingContext);await this._timeoutAndRetry(async () => source.markRequestHandled(request!),@@ -1604,11 +1658,13 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext// reclaim session if request finishes successfullyrequest.state = RequestState.DONE;crawlingContext.session?.markGood();- } catch (err) {+ } catch (rawError) {+ const err = this.unwrapError(rawError);+try {request.state = RequestState.ERROR_HANDLER;await addTimeoutToPromise(- async () => this._requestFunctionErrorHandler(err as Error, crawlingContext, source),+ async () => this._requestFunctionErrorHandler(err, crawlingContext, source),this.internalTimeoutMillis,`Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3@@ -1619,29 +1675,29 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext}request.state = RequestState.DONE;} catch (secondaryError: any) {+ const unwrappedSecondaryError = this.unwrapError(secondaryError) as any;+if (- !secondaryError.triggeredFromUserHandler &&+ !unwrappedSecondaryError.triggeredFromUserHandler &&// avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway- !(secondaryError instanceof CriticalError)+ !(unwrappedSecondaryError instanceof CriticalError)) {const apifySpecific = process.env.APIFY_IS_AT_HOME? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`: '';this.log.exception(- secondaryError as Error,+ unwrappedSecondaryError as Error,'An exception occurred during handling of failed request. ' +`This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`,);}request.state = RequestState.ERROR;- throw secondaryError;+ throw unwrappedSecondaryError;}// decrease the session score if the request fails (but the error handler did not throw)crawlingContext.session?.markBad();} finally {- await this._cleanupContext(crawlingContext);-- this.crawlingContexts.delete(crawlingContext.id);+ await Promise.all(deferredCleanup.map((cleanup) => cleanup()));// Safety net - release the lock if nobody managed to do it beforeif (isRequestLocked && source instanceof RequestProvider) {@@ -1705,7 +1761,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext// validation against the label's schema covers them all (a no-op unless the router declares a schema).await this.validateRequestUserData({ label: options.label, userData: options.userData });- return enqueueLinks({+ return await enqueueLinks({...options,// The options below are merged with the user options, so an explicitly `undefined` value@@ -1775,7 +1831,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextreturn !this.requestManager || (await this.requestManager.isFinished());}- private async _rotateSession(crawlingContext: Context) {+ private async _rotateSession(crawlingContext: CrawlingContext) {const { request } = crawlingContext;request.sessionRotationCount ??= 0;@@ -1783,12 +1839,27 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextcrawlingContext.session?.retire();}+ /**+ * Unwraps errors thrown by the context pipeline to get the actual user error.+ * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.+ */+ private unwrapError(error: unknown): Error {+ if (+ error instanceof RequestHandlerError ||+ error instanceof ContextPipelineInitializationError ||+ error instanceof ContextPipelineCleanupError+ ) {+ return this.unwrapError(error.cause);+ }+ return error as Error;+ }+/*** Handles errors thrown by user provided requestHandler()*/protected async _requestFunctionErrorHandler(error: Error,- crawlingContext: Context,+ crawlingContext: CrawlingContext,source: IRequestList | IRequestManager,): Promise<void> {const { request } = crawlingContext;@@ -1802,7 +1873,10 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextif (shouldRetryRequest) {await this.stats.errorTrackerRetry.addAsync(error, crawlingContext);- await this.errorHandler?.(crawlingContext as LoadedContext<Context>, error);+ await this.errorHandler?.(+ crawlingContext as CrawlingContext & Partial<ExtendedContext>, // valid cast - ExtendedContext transitively extends CrawlingContext+ error,+ );if (error instanceof SessionError) {await this._rotateSession(crawlingContext);@@ -1855,7 +1929,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext}}- protected async _handleFailedRequestHandler(crawlingContext: Context, error: Error): Promise<void> {+ protected async _handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise<void> {// Always log the last error regardless if the user provided a failedRequestHandlerconst { id, url, method, uniqueKey } = crawlingContext.request;const message = this._getMessageFromError(error, true);@@ -1863,7 +1937,10 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextthis.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });if (this.failedRequestHandler) {- await this.failedRequestHandler?.(crawlingContext as LoadedContext<Context>, error);+ await this.failedRequestHandler?.(+ crawlingContext as CrawlingContext & Partial<ExtendedContext>, // valid cast - ExtendedContext transitively extends CrawlingContext+ error,+ );}}@@ -1982,7 +2059,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextreturn RequestQueue.open(null, { config: this.config });}- protected requestMatchesEnqueueStrategy(request: Request) {+ private requestMatchesEnqueueStrategy(request: Request) {const { url, loadedUrl } = request;// eslint-disable-next-line dot-notation -- private accessdiff --git a/packages/basic-crawler/src/internals/constants.ts b/packages/basic-crawler/src/internals/constants.tsdeleted file mode 100644index 56c4aa46e..000000000--- a/packages/basic-crawler/src/internals/constants.ts+++ /dev/null@@ -1,6 +0,0 @@-/**- * Additional number of seconds used in {@apilink CheerioCrawler} and {@apilink BrowserCrawler} to set a reasonable- * {@apilink BasicCrawlerOptions.requestHandlerTimeoutSecs|`requestHandlerTimeoutSecs`} for {@apilink BasicCrawler}- * that would not impare functionality (not timeout before crawlers).- */-export const BASIC_CRAWLER_TIMEOUT_BUFFER_SECS = 10;diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 6b7878d3e..23a39653b 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -6,19 +6,20 @@ import type {Dictionary,EnqueueLinksOptions,ErrorHandler,- LoadedContext,+ LoadedRequest,ProxyConfiguration,ProxyInfo,+ Request,RequestHandler,RequestProvider,Session,SkippedRequestCallback,} from '@crawlee/basic';import {- BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,BasicCrawler,BLOCKED_STATUS_CODES as DEFAULT_BLOCKED_STATUS_CODES,Configuration,+ ContextPipeline,cookieStringToToughCookie,enqueueLinks,EVENT_SESSION_RETIRED,@@ -39,35 +40,55 @@ import type {LaunchContext,} from '@crawlee/browser-pool';import { BROWSER_CONTROLLER_EVENTS, BrowserPool } from '@crawlee/browser-pool';-import type { Cookie as CookieObject } from '@crawlee/types';+import type { BatchAddRequestsResult, Cookie as CookieObject } from '@crawlee/types';import type { RobotsTxtFile } from '@crawlee/utils';import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';import ow from 'ow';import type { ReadonlyDeep } from 'type-fest';-import { addTimeoutToPromise, tryCancel } from '@apify/timeout';+import { tryCancel } from '@apify/timeout';import type { BrowserLaunchContext } from './browser-launcher.js';const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;+interface BaseResponse {+ status(): number;+}++type ContextDifference<T, U> = Omit<U, keyof T> & Partial<U>;+export interface BrowserCrawlingContext<- Crawler = unknown,Page extends CommonPage = CommonPage,- Response = Dictionary,+ Response extends BaseResponse = BaseResponse,ProvidedController = BrowserController,UserData extends Dictionary = Dictionary,-> extends CrawlingContext<Crawler, UserData> {+> extends CrawlingContext<UserData> {+ /**+ * An instance of the {@apilink BrowserController} that manages the browser instance and provides access to its API.+ */browserController: ProvidedController;++ /**+ * The browser page object where the web page is loaded and rendered.+ */page: Page;- response?: Response;-}-export type BrowserRequestHandler<Context extends BrowserCrawlingContext = BrowserCrawlingContext> =- RequestHandler<Context>;+ /**+ * The request object that was successfully loaded and navigated to, including the {@apilink Request.loadedUrl|`loadedUrl`} property.+ */+ request: LoadedRequest<Request<UserData>>;-export type BrowserErrorHandler<Context extends BrowserCrawlingContext = BrowserCrawlingContext> =- ErrorHandler<Context>;+ /**+ * The HTTP response object returned by the browser's navigation.+ */+ response: Response;++ /**+ * Helper function for extracting URLs from the current page and adding them to the request queue.+ */+ enqueueLinks: (options?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;+}export type BrowserHook<Context = BrowserCrawlingContext, GoToOptions extends Dictionary | undefined = Dictionary> = (crawlingContext: Context,@@ -75,13 +96,23 @@ export type BrowserHook<Context = BrowserCrawlingContext, GoToOptions extends Di) => Awaitable<void>;export interface BrowserCrawlerOptions<- Context extends BrowserCrawlingContext = BrowserCrawlingContext,+ Page extends CommonPage = CommonPage,+ Response extends BaseResponse = BaseResponse,+ ProvidedController extends BrowserController = BrowserController,+ Context extends BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary> = BrowserCrawlingContext<+ Page,+ Response,+ ProvidedController,+ Dictionary+ >,+ ContextExtension = {},+ ExtendedContext extends Context = Context & ContextExtension,InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions,__BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>,__BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>,__LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>,> extends Omit<- BasicCrawlerOptions,+ BasicCrawlerOptions<Context, ExtendedContext>,// Overridden with browser context'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {@@ -114,7 +145,7 @@ export interface BrowserCrawlerOptions<* The exceptions are logged to the request using the* {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function.*/- requestHandler?: BrowserRequestHandler<Context>;+ requestHandler?: RequestHandler<ExtendedContext>;/*** User-provided function that allows modifying the request object before it gets retried by the crawler.@@ -126,7 +157,7 @@ export interface BrowserCrawlerOptions<* Second argument is the `Error` instance that* represents the last error thrown during processing of the request.*/- errorHandler?: BrowserErrorHandler<Context>;+ errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;/*** A function to handle requests that failed more than `option.maxRequestRetries` times.@@ -137,7 +168,7 @@ export interface BrowserCrawlerOptions<* Second argument is the `Error` instance that* represents the last error thrown during processing of the request.*/- failedRequestHandler?: BrowserErrorHandler<Context>;+ failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;/*** Custom options passed to the underlying {@apilink BrowserPool} constructor.@@ -264,11 +295,21 @@ export interface BrowserCrawlerOptions<* @category Crawlers*/export abstract class BrowserCrawler<+ Page extends CommonPage = CommonPage,+ Response extends BaseResponse = BaseResponse,+ ProvidedController extends BrowserController = BrowserController,InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions,LaunchOptions extends Dictionary | undefined = Dictionary,- Context extends BrowserCrawlingContext = BrowserCrawlingContext,+ Context extends BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary> = BrowserCrawlingContext<+ Page,+ Response,+ ProvidedController,+ Dictionary+ >,+ ContextExtension = {},+ ExtendedContext extends Context = Context & ContextExtension,GoToOptions extends Dictionary = Dictionary,-> extends BasicCrawler<Context> {+> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {/*** A reference to the underlying {@apilink ProxyConfiguration} class that manages the crawler's proxies.* Only available if used by the crawler.@@ -282,9 +323,10 @@ export abstract class BrowserCrawler<launchContext: BrowserLaunchContext<LaunchOptions, unknown>;- protected userProvidedRequestHandler!: BrowserRequestHandler<Context>;+ protected readonly ignoreShadowRoots: boolean;+ protected readonly ignoreIframes: boolean;+protected navigationTimeoutMillis: number;- protected requestHandlerTimeoutInnerMillis: number;protected preNavigationHooks: BrowserHook<Context>[];protected postNavigationHooks: BrowserHook<Context>[];protected persistCookiesPerSession: boolean;@@ -303,46 +345,54 @@ export abstract class BrowserCrawler<persistCookiesPerSession: ow.optional.boolean,useSessionPool: ow.optional.boolean,proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),- ignoreShadowRoots: ow.optional.boolean,- ignoreIframes: ow.optional.boolean,};/*** All `BrowserCrawler` parameters are passed via an options object.*/protected constructor(- options: BrowserCrawlerOptions<Context> = {},+ options: BrowserCrawlerOptions<+ Page,+ Response,+ ProvidedController,+ Context,+ ContextExtension,+ ExtendedContext+ > & {+ contextPipelineBuilder: () => ContextPipeline<CrawlingContext, Context>;+ },override readonly config = Configuration.getGlobalConfig(),) {ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape));const {navigationTimeoutSecs = 60,- requestHandlerTimeoutSecs = 60,persistCookiesPerSession,proxyConfiguration,launchContext = {},browserPoolOptions,preNavigationHooks = [],postNavigationHooks = [],- requestHandler,headless,- ignoreShadowRoots,- ignoreIframes,+ ignoreIframes = false,+ ignoreShadowRoots = false,+ contextPipelineBuilder,+ extendContext,...basicCrawlerOptions} = options;super({...basicCrawlerOptions,- requestHandler: async (...args) => this._runRequestHandler(...(args as [Context])),- requestHandlerTimeoutSecs:- navigationTimeoutSecs + requestHandlerTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,+ contextPipelineBuilder: () =>+ contextPipelineBuilder()+ .compose({ action: this.performNavigation.bind(this) })+ .compose({ action: this.handleBlockedRequestByContent.bind(this) })+ .compose({ action: this.restoreRequestState.bind(this) }),+ extendContext: extendContext as (context: Context) => Awaitable<ContextExtension>,},config,);- this.userProvidedRequestHandler = requestHandler ?? this.router;-// Cookies should be persisted per session only if session pool is usedif (!this.useSessionPool && persistCookiesPerSession) {throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');@@ -350,10 +400,11 @@ export abstract class BrowserCrawler<this.launchContext = launchContext;this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;- this.requestHandlerTimeoutInnerMillis = requestHandlerTimeoutSecs * 1000;this.proxyConfiguration = proxyConfiguration;this.preNavigationHooks = preNavigationHooks;this.postNavigationHooks = postNavigationHooks;+ this.ignoreIframes = ignoreIframes;+ this.ignoreShadowRoots = ignoreShadowRoots;if (headless != null) {this.launchContext.launchOptions ??= {} as LaunchOptions;@@ -381,18 +432,26 @@ export abstract class BrowserCrawler<});}- protected override async _cleanupContext(crawlingContext: Context): Promise<void> {- const { page } = crawlingContext;-- // Page creation may be aborted- if (page) {- // Puppeteer 25+ can hang `page.close()` indefinitely when the page's navigation was aborted, don't let it block the crawler.- await addTimeoutToPromise(- async () => page.close(),- PAGE_CLOSE_TIMEOUT_MILLIS,- `page.close() timed out after ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds`,- ).catch((error: Error) => this.log.debug('Error while closing page', { error }));- }+ protected buildContextPipeline(): ContextPipeline<+ CrawlingContext,+ BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary>+ > {+ return ContextPipeline.create<CrawlingContext>().compose({+ action: this.preparePage.bind(this),+ cleanup: async (context: {+ page: Page;+ registerDeferredCleanup: BasicCrawlingContext['registerDeferredCleanup'];+ }) => {+ context.registerDeferredCleanup(async () => {+ // Puppeteer 25+ can hang `page.close()` indefinitely when the page's navigation was aborted, don't let it block the crawler.+ await addTimeoutToPromise(+ async () => context.page.close(),+ PAGE_CLOSE_TIMEOUT_MILLIS,+ `page.close() timed out after ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds`,+ ).catch((error: Error) => this.log.debug('Error while closing page', { error }));+ });+ },+ });}private async containsSelectors(page: CommonPage, selectors: string[]): Promise<string[] | null> {@@ -404,7 +463,9 @@ export abstract class BrowserCrawler<return foundSelectors.length > 0 ? foundSelectors : null;}- protected override async isRequestBlocked(crawlingContext: Context): Promise<string | false> {+ protected async isRequestBlocked(+ crawlingContext: BrowserCrawlingContext<Page, Response, ProvidedController>,+ ): Promise<string | false> {const { page, response } = crawlingContext;const blockedStatusCodes =@@ -442,10 +503,11 @@ export abstract class BrowserCrawler<return this.userProvidedRequestHandler as RequestHandler<Context>;}- /**- * Wrapper around requestHandler that opens and closes pages etc.- */- protected override async _runRequestHandler(crawlingContext: Context) {+ private async preparePage(+ crawlingContext: CrawlingContext,+ ): Promise<+ ContextDifference<CrawlingContext, BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary>>+ > {const newPageOptions: Dictionary = {id: crawlingContext.id,};@@ -475,109 +537,66 @@ export abstract class BrowserCrawler<}}- const page = (await this.browserPool.newPage(newPageOptions)) as CommonPage;+ const page = (await this.browserPool.newPage(newPageOptions)) as Page;tryCancel();- this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages);-- // DO NOT MOVE THIS LINE ABOVE!- // `enhanceCrawlingContextWithPageInfo` gives us a valid session.- // For example, `sessionPoolOptions.sessionOptions.maxUsageCount` can be `1`.- // So we must not save the session prior to making sure it was used only once, otherwise we would use it twice.- const { request, session } = crawlingContext;-- if (!request.skipNavigation) {- await this._handleNavigation(crawlingContext);- tryCancel();-- await this._responseHandler(crawlingContext);- tryCancel();-- // save cookies- // TODO: Should we save the cookies also after/only the handle page?- if (this.persistCookiesPerSession) {- const cookies = await crawlingContext.browserController.getCookies(page);- tryCancel();- session?.setCookies(cookies, request.loadedUrl!);- }- }-- if (!this.requestMatchesEnqueueStrategy(request)) {- this.log.debug(- // eslint-disable-next-line dot-notation- `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`,- );- request.noRetry = true;- request.state = RequestState.SKIPPED;-- await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });-- return;- }-- if (this.retryOnBlocked) {- const error = await this.isRequestBlocked(crawlingContext);- if (error) throw new SessionError(error);- }-- request.state = RequestState.REQUEST_HANDLER;- try {- await addTimeoutToPromise(- async () => Promise.resolve(this.userProvidedRequestHandler(crawlingContext as LoadedContext<Context>)),- this.requestHandlerTimeoutInnerMillis,- `requestHandler timed out after ${this.requestHandlerTimeoutInnerMillis / 1000} seconds.`,- );-- request.state = RequestState.DONE;- } catch (e: any) {- request.state = RequestState.ERROR;- throw e;- }- tryCancel();- }-- protected _enhanceCrawlingContextWithPageInfo(- crawlingContext: Context,- page: CommonPage,- createNewSession?: boolean,- ): void {- crawlingContext.page = page;-- // This switch is because the crawlingContexts are created on per request basis.- // However, we need to add the proxy info and session from browser, which is created based on the browser-pool configuration.- // We would not have to do this switch if the proxy and configuration worked as in CheerioCrawler,- // which configures proxy and session for every new requestconst browserControllerInstance = this.browserPool.getBrowserControllerByPage(page as any,- ) as Context['browserController'];- crawlingContext.browserController = browserControllerInstance;-- if (!createNewSession) {- crawlingContext.session = browserControllerInstance.launchContext.session as Session;- }-- if (!crawlingContext.proxyInfo) {- crawlingContext.proxyInfo = browserControllerInstance.launchContext.proxyInfo as ProxyInfo;- }+ ) as ProvidedController;const contextEnqueueLinks = crawlingContext.enqueueLinks;- crawlingContext.enqueueLinks = async (enqueueOptions) => {- return browserCrawlerEnqueueLinks({- // `contextEnqueueLinks` clamps `limit` by the remaining `maxRequestsPerCrawl` budget itself;- // pre-clamping it here would make the crawler log the internal limit as a user-provided one- options: enqueueOptions,- page,- requestQueue: await this.getRequestQueue(),- robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),- onSkippedRequest: this.handleSkippedRequest,- originalRequestUrl: crawlingContext.request.url,- finalRequestUrl: crawlingContext.request.loadedUrl,- enqueueLinks: contextEnqueueLinks,- });++ return {+ page,+ get response(): Response {+ throw new Error(+ "The `response` property is not available. This might mean that you're trying to access it before navigation or that navigation resulted in `null` (this should only happen with `about:` URLs)",+ );+ },+ browserController: browserControllerInstance,+ session: useIncognitoPages+ ? crawlingContext.session+ : (browserControllerInstance.launchContext.session as Session),+ proxyInfo: crawlingContext.proxyInfo ?? (browserControllerInstance.launchContext.proxyInfo as ProxyInfo),+ enqueueLinks: async (enqueueOptions: EnqueueLinksOptions = {}) => {+ return (await browserCrawlerEnqueueLinks({+ // `contextEnqueueLinks` clamps `limit` by the remaining `maxRequestsPerCrawl` budget itself;+ // pre-clamping it here would make the crawler log the internal limit as a user-provided one+ options: enqueueOptions,+ page,+ requestQueue: await this.getRequestQueue(),+ robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),+ onSkippedRequest: this.handleSkippedRequest,+ originalRequestUrl: crawlingContext.request.url,+ finalRequestUrl: crawlingContext.request.loadedUrl,+ enqueueLinks: contextEnqueueLinks,+ })) as BatchAddRequestsResult; // TODO make this type safe+ },};}- protected async _handleNavigation(crawlingContext: Context) {+ private async performNavigation(crawlingContext: Context): Promise<{+ request: LoadedRequest<Request>;+ response?: Response;+ }> {+ if (crawlingContext.request.skipNavigation) {+ return {+ request: new Proxy(crawlingContext.request, {+ get(target, propertyName, receiver) {+ if (propertyName === 'loadedUrl') {+ throw new Error(+ 'The `request.loadedUrl` property is not available - `skipNavigation` was used',+ );+ }+ return Reflect.get(target, propertyName, receiver);+ },+ }) as LoadedRequest<Request>,+ get response(): Response {+ throw new Error('The `response` property is not available - `skipNavigation` was used');+ },+ };+ }+const gotoOptions = { timeout: this.navigationTimeoutMillis } as unknown as GoToOptions;const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);@@ -590,8 +609,10 @@ export abstract class BrowserCrawler<await this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);+ let response: Response | undefined;+try {- crawlingContext.response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;+ response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;} catch (error) {await this._handleNavigationTimeout(crawlingContext, error as Error);@@ -604,10 +625,48 @@ export abstract class BrowserCrawler<crawlingContext.request.state = RequestState.AFTER_NAV;await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);++ await this.processResponse(response, crawlingContext);+ tryCancel();++ // save cookies+ // TODO: Should we save the cookies also after/only the handle page?+ if (this.persistCookiesPerSession) {+ const cookies = await crawlingContext.browserController.getCookies(crawlingContext.page);+ tryCancel();+ crawlingContext.session?.setCookies(cookies, crawlingContext.request.loadedUrl!);+ }++ if (response !== undefined) {+ return {+ request: crawlingContext.request as LoadedRequest<Request>,+ response,+ };+ }++ return {+ request: crawlingContext.request as LoadedRequest<Request>,+ };+ }++ private async handleBlockedRequestByContent(+ crawlingContext: BrowserCrawlingContext<Page, Response, ProvidedController>,+ ) {+ if (this.retryOnBlocked) {+ const error = await this.isRequestBlocked(crawlingContext);+ if (error) throw new SessionError(error);+ }++ return {};+ }++ private async restoreRequestState(crawlingContext: CrawlingContext) {+ crawlingContext.request.state = RequestState.REQUEST_HANDLER;+ return {};}protected async _applyCookies(- { session, request, page, browserController }: Context,+ { session, request, page, browserController }: BrowserCrawlingContext,preHooksCookies: string,postHooksCookies: string,) {@@ -626,7 +685,7 @@ export abstract class BrowserCrawler</*** Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.*/- protected async _handleNavigationTimeout(crawlingContext: Context, error: Error): Promise<void> {+ protected async _handleNavigationTimeout(crawlingContext: BrowserCrawlingContext, error: Error): Promise<void> {const { session, page } = crawlingContext;if (error && error.constructor.name === 'TimeoutError') {@@ -648,15 +707,15 @@ export abstract class BrowserCrawler<}protected abstract _navigationHandler(- crawlingContext: Context,+ crawlingContext: BrowserCrawlingContext<Page, Response, ProvidedController>,gotoOptions: GoToOptions,): Promise<Context['response'] | null | undefined>;- /**- * Should be overridden in case of different automation library that does not support this response API.- */- protected async _responseHandler(crawlingContext: Context): Promise<void> {- const { response, session, request, page } = crawlingContext;+ private async processResponse(+ response: Response | undefined,+ crawlingContext: BrowserCrawlingContext,+ ): Promise<void> {+ const { session, request, page } = crawlingContext;if (typeof response === 'object' && typeof response.status === 'function') {const status: number = response.status();@@ -786,6 +845,7 @@ export async function browserCrawlerEnqueueLinks(baseUrl,});}+return enqueueLinks({requestQueue: options.requestQueue,robotsTxtFile: options.robotsTxtFile,diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.tsindex 722243672..0bf69f5a0 100644--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts@@ -1,6 +1,3 @@-import type { IncomingMessage } from 'node:http';-import { text as readStreamToString } from 'node:stream/consumers';-import type {BasicCrawlingContext,Configuration,@@ -19,9 +16,9 @@ import type {SkippedRequestCallback,} from '@crawlee/http';import { enqueueLinks, HttpCrawler, resolveBaseUrlForEnqueueLinksFiltering, Router } from '@crawlee/http';-import type { Dictionary } from '@crawlee/types';+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';import { type CheerioRoot, extractUrlsFromCheerio, type RobotsTxtFile } from '@crawlee/utils';-import type { CheerioOptions } from 'cheerio';+import type { CheerioAPI, CheerioOptions } from 'cheerio';import * as cheerio from 'cheerio';import { parseDocument } from 'htmlparser2';@@ -31,9 +28,10 @@ export type CheerioErrorHandler<> = ErrorHandler<CheerioCrawlingContext<UserData, JSONData>>;export interface CheerioCrawlerOptions<+ ExtendedContext extends CheerioCrawlingContext,UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>> {}+> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ExtendedContext> {}export type CheerioHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler@@ -43,7 +41,12 @@ export type CheerioHook<export interface CheerioCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> extends InternalHttpCrawlingContext<UserData, JSONData, CheerioCrawler> {+> extends InternalHttpCrawlingContext<UserData, JSONData> {+ /**+ * The raw HTML content of the web page as a string.+ */+ body: string;+/*** The [Cheerio](https://cheerio.js.org/) object with parsed HTML.* Cheerio is available only for HTML and XML content types.@@ -79,6 +82,11 @@ export interface CheerioCrawlingContext<* ```*/parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;++ /**+ * Helper function for extracting URLs from the parsed HTML and adding them to the request queue.+ */+ enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;}export type CheerioRequestHandler<@@ -163,64 +171,74 @@ export type CheerioRequestHandler<* ```* @category Crawlers*/-export class CheerioCrawler extends HttpCrawler<CheerioCrawlingContext> {+export class CheerioCrawler<+ ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext,+> extends HttpCrawler<CheerioCrawlingContext, ExtendedContext> {/*** All `CheerioCrawler` parameters are passed via an options object.*/- // eslint-disable-next-line @typescript-eslint/no-useless-constructor- constructor(options?: CheerioCrawlerOptions, config?: Configuration) {- super(options, config);+ constructor(options?: CheerioCrawlerOptions<ExtendedContext>, config?: Configuration) {+ super(+ {+ ...options,+ contextPipelineBuilder: () =>+ this.buildContextPipeline()+ .compose({+ action: async (context) => await this.parseContent(context),+ })+ .compose({ action: async (context) => await this.addHelpers(context) }),+ },+ config,+ );}- protected override async _parseHTML(- response: IncomingMessage,- isXml: boolean,- crawlingContext: CheerioCrawlingContext,- ) {- const body = await readStreamToString(response);+ private async parseContent(crawlingContext: InternalHttpCrawlingContext) {+ const isXml = crawlingContext.contentType.type.includes('xml');+ const body = Buffer.isBuffer(crawlingContext.body)+ ? crawlingContext.body.toString(crawlingContext.contentType.encoding)+ : crawlingContext.body;const dom = parseDocument(body, { decodeEntities: true, xmlMode: isXml });const $ = cheerio.load(dom, {xml: { decodeEntities: true, xmlMode: isXml },} as CheerioOptions);- const originalEnqueueLinks = crawlingContext.enqueueLinks;-return {- dom,$,body,+ };+ }++ private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { $: CheerioAPI }) {+ const originalEnqueueLinks = crawlingContext.enqueueLinks;++ return {enqueueLinks: async (enqueueOptions?: EnqueueLinksOptions) => {- return cheerioCrawlerEnqueueLinks({+ return (await cheerioCrawlerEnqueueLinks({// `originalEnqueueLinks` clamps `limit` by the remaining `maxRequestsPerCrawl` budget itself;// pre-clamping it here would make the crawler log the internal limit as a user-provided oneoptions: enqueueOptions,- $,+ $: crawlingContext.$,requestQueue: await this.getRequestQueue(),robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),onSkippedRequest: this.handleSkippedRequest,originalRequestUrl: crawlingContext.request.url,finalRequestUrl: crawlingContext.request.loadedUrl,enqueueLinks: originalEnqueueLinks,- });+ })) as BatchAddRequestsResult; // TODO make this type safe},- };- }-- protected override async _runRequestHandler(context: CheerioCrawlingContext) {- context.waitForSelector = async (selector?: string, _timeoutMs?: number) => {- if (context.$(selector).get().length === 0) {- throw new Error(`Selector '${selector}' not found.`);- }- };- context.parseWithCheerio = async (selector?: string, timeoutMs?: number) => {- if (selector) {- await context.waitForSelector(selector, timeoutMs);- }+ waitForSelector: async (selector: string, _timeoutMs?: number) => {+ if (crawlingContext.$(selector).get().length === 0) {+ throw new Error(`Selector '${selector}' not found.`);+ }+ },+ parseWithCheerio: async (selector?: string, timeoutMs?: number) => {+ if (selector) {+ await crawlingContext.waitForSelector(selector, timeoutMs);+ }- return context.$;+ return crawlingContext.$;+ },};-- await super._runRequestHandler(context);}}diff --git a/packages/core/src/crawlers/context_pipeline.ts b/packages/core/src/crawlers/context_pipeline.tsnew file mode 100644index 000000000..e84361a25--- /dev/null+++ b/packages/core/src/crawlers/context_pipeline.ts@@ -0,0 +1,170 @@+import type { Awaitable } from '@crawlee/types';++import {+ ContextPipelineCleanupError,+ ContextPipelineInitializationError,+ ContextPipelineInterruptedError,+ RequestHandlerError,+ SessionError,+} from '../errors.js';++/**+ * Represents a middleware step in the context pipeline.+ *+ * @template TCrawlingContext - The input context type for this middleware+ * @template TCrawlingContextExtension - The enhanced output context type+ */+export interface ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> {+ /** The main middleware function that enhances the context */+ action: (context: TCrawlingContext) => Awaitable<TCrawlingContextExtension>;+ /** Optional cleanup function called after the consumer finishes or fails */+ cleanup?: (context: TCrawlingContext & TCrawlingContextExtension, error?: unknown) => Awaitable<void>;+}++/**+ * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.+ *+ * The enhancement is done by a chain of middlewares that are added to the pipeline after its creation.+ * This class provides a type-safe way to build a pipeline of context transformations where each step+ * can enhance the context with additional properties or utilities.+ *+ * @template TContextBase - The base context type that serves as the starting point+ * @template TCrawlingContext - The final context type after all middleware transformations+ */+export abstract class ContextPipeline<TContextBase, TCrawlingContext extends TContextBase> {+ /**+ * Creates a new empty context pipeline.+ *+ * @template TContextBase - The base context type for the pipeline+ * @returns A new ContextPipeline instance with no transformations+ */+ static create<TContextBase>(): ContextPipeline<TContextBase, TContextBase> {+ return new ContextPipelineImpl<TContextBase, TContextBase>({ action: async (context) => context });+ }++ /**+ * Adds a middleware to the pipeline, creating a new pipeline instance.+ *+ * This method provides a fluent interface for building context transformation pipelines.+ * Each middleware can enhance the context with additional properties or utilities.+ *+ * @template TCrawlingContextExtension - The enhanced context type produced by this middleware+ * @param middleware - The middleware to add to the pipeline+ * @returns A new ContextPipeline instance with the added middleware+ */+ abstract compose<TCrawlingContextExtension>(+ middleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>,+ ): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension>;++ /**+ * Executes the middleware pipeline and passes the final context to a consumer function.+ *+ * This method runs the crawling context through the entire middleware chain, enhancing it+ * at each step, and then passes the final enhanced context to the provided consumer function.+ * Proper cleanup is performed even if exceptions occur during processing.+ *+ * @param crawlingContext - The initial context to process through the pipeline+ * @param finalContextConsumer - The function that will receive the final enhanced context+ *+ * @throws {ContextPipelineInitializationError} When a middleware fails during initialization+ * @throws {ContextPipelineInterruptedError} When the pipeline is intentionally interrupted during initialization+ * @throws {RequestHandlerError} When the final context consumer throws an exception+ * @throws {ContextPipelineCleanupError} When cleanup operations fail+ * @throws {SessionError} Session errors are re-thrown as-is for special handling+ */+ abstract call(+ crawlingContext: TContextBase,+ finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>,+ ): Promise<void>;+}++/**+ * Implementation of the `ContextPipeline` logic. This hides implementation details such as the `middleware` and `parent`+ * properties from the `ContextPipeline` interface, making type checking more reliable.+ */+class ContextPipelineImpl<TContextBase, TCrawlingContext extends TContextBase> extends ContextPipeline<+ TContextBase,+ TCrawlingContext+> {+ constructor(+ private middleware: ContextMiddleware<TContextBase, TCrawlingContext>,+ private parent?: ContextPipelineImpl<TContextBase, TContextBase>,+ ) {+ super();+ }++ /**+ * @inheritdoc+ */+ compose<TCrawlingContextExtension>(+ middleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>,+ ): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension> {+ return new ContextPipelineImpl<TContextBase, TCrawlingContext & TCrawlingContextExtension>(+ middleware as any,+ this as any,+ );+ }++ private *middlewareChain() {+ let step: ContextPipelineImpl<TContextBase, TContextBase> | undefined = this as any;++ while (step !== undefined) {+ yield step.middleware;+ step = step.parent;+ }+ }++ /**+ * @inheritdoc+ */+ async call(+ crawlingContext: TContextBase,+ finalContextConsumer: (finalContext: TCrawlingContext) => Promise<unknown>,+ ): Promise<void> {+ const middlewares = Array.from(this.middlewareChain()).reverse();+ const cleanupStack = [];+ let consumerException: unknown | undefined;++ try {+ for (const { action, cleanup } of middlewares) {+ try {+ const contextExtension = await action(crawlingContext);+ Object.defineProperties(crawlingContext, Object.getOwnPropertyDescriptors(contextExtension));++ if (cleanup) {+ cleanupStack.push(cleanup);+ }+ } catch (exception: unknown) {+ if (exception instanceof SessionError) {+ throw exception; // Session errors are re-thrown as-is+ }+ if (exception instanceof ContextPipelineInterruptedError) {+ throw exception;+ }++ throw new ContextPipelineInitializationError(exception);+ }+ }++ try {+ await finalContextConsumer(crawlingContext as TCrawlingContext);+ } catch (exception: unknown) {+ if (exception instanceof SessionError) {+ consumerException = exception;+ throw exception; // Session errors are re-thrown as-is+ }+ consumerException = exception;+ throw new RequestHandlerError(exception);+ }+ } finally {+ try {+ for (const cleanup of cleanupStack.reverse()) {+ await cleanup(crawlingContext, consumerException);+ }+ } catch (exception: unknown) {+ // eslint-disable-next-line no-unsafe-finally+ throw new ContextPipelineCleanupError(exception);+ }+ }+ }+}diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.tsindex 868195a23..ba9c90888 100644--- a/packages/core/src/crawlers/crawler_commons.ts+++ b/packages/core/src/crawlers/crawler_commons.ts@@ -1,6 +1,6 @@-import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';+import type { Dictionary } from '@crawlee/types';import type { OptionsInit, Response as GotResponse } from 'got-scraping';-import type { ReadonlyDeep } from 'type-fest';+import type { ReadonlyDeep, SetRequired } from 'type-fest';import type { Configuration } from '../configuration.js';import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';@@ -28,9 +28,7 @@ export type LoadedContext<Context extends RestrictedCrawlingContext> =request: LoadedRequest<Context['request']>;} & Omit<Context, 'request'>;-export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary>- // we need `Record<string & {}, unknown>` here, otherwise `Omit<Context>` is resolved badly- extends Record<string & {}, unknown> {+export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> {id: string;session?: Session;@@ -78,7 +76,9 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction** @param [options] All `enqueueLinks()` parameters are passed via an options object.*/- enqueueLinks: (options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>>) => Promise<unknown>;+ enqueueLinks: (+ options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestQueue' | 'robotsTxtFile'>>,+ ) => Promise<unknown>;/*** Add requests directly to the request queue.@@ -109,10 +109,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionlog: Log;}-export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary>- extends RestrictedCrawlingContext<UserData> {- crawler: Crawler;-+export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {/*** This function automatically finds and enqueues links from the current page, adding them to the {@apilink RequestQueue}* currently used by the crawler.@@ -139,13 +136,9 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary* @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.*/enqueueLinks(- options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>,- ): Promise<BatchAddRequestsResult>;-- /**- * Get a key-value store with given name or id, or the default one for the crawler.- */- getKeyValueStore: (idOrName?: string) => Promise<KeyValueStore>;+ options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestQueue' | 'robotsTxtFile'>> &+ Pick<EnqueueLinksOptions, 'requestQueue' | 'robotsTxtFile'>,+ ): Promise<unknown>;/*** Fires HTTP request via [`got-scraping`](https://crawlee.dev/js/docs/guides/got-scraping), allowing to override the request@@ -165,6 +158,11 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary* ```*/sendRequest<Response = string>(overrideOptions?: Partial<OptionsInit>): Promise<GotResponse<Response>>;++ /**+ * Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.+ */+ registerDeferredCleanup(cleanup: () => Promise<unknown>): void;}/**diff --git a/packages/core/src/crawlers/crawler_extension.ts b/packages/core/src/crawlers/crawler_extension.tsdeleted file mode 100644index b0d406389..000000000--- a/packages/core/src/crawlers/crawler_extension.ts+++ /dev/null@@ -1,15 +0,0 @@-import { type Log, log as defaultLog } from '../log.js';--/**- * Abstract class with pre-defined method to connect to the Crawlers class by the "use" crawler method.- * @category Crawlers- * @ignore- */-export abstract class CrawlerExtension {- name = this.constructor.name;- log: Log = defaultLog.child({ prefix: this.name });-- getCrawlerOptions(): Record<string, unknown> {- throw new Error(`${this.name} has not implemented "getCrawlerOptions" method.`);- }-}diff --git a/packages/core/src/crawlers/error_snapshotter.ts b/packages/core/src/crawlers/error_snapshotter.tsindex 298562b9b..427a53043 100644--- a/packages/core/src/crawlers/error_snapshotter.ts+++ b/packages/core/src/crawlers/error_snapshotter.ts@@ -3,16 +3,13 @@ import crypto from 'node:crypto';import type { CrawlingContext } from '../crawlers/crawler_commons.js';import type { KeyValueStore } from '../storages/key_value_store.js';import type { ErrnoException } from './error_tracker.js';+import type { SnapshottableProperties } from './internals/types.js';// Define the following types as we cannot import the complete types from the respective packagesinterface BrowserCrawlingContext {saveSnapshot: (options: { key: string }) => Promise<void>;}-interface BrowserPage {- content: () => Promise<string>;-}-export interface SnapshotResult {screenshotFileName?: string;htmlFileName?: string;@@ -49,9 +46,12 @@ export class ErrorSnapshotter {/*** Capture a snapshot of the error context.*/- async captureSnapshot(error: ErrnoException, context: CrawlingContext): Promise<ErrorSnapshot> {+ async captureSnapshot(+ error: ErrnoException,+ context: CrawlingContext & SnapshottableProperties,+ ): Promise<ErrorSnapshot> {try {- const page = context?.page as BrowserPage | undefined;+ const page = context?.page;const body = context?.body;const keyValueStore = await context?.getKeyValueStore();@@ -120,7 +120,11 @@ export class ErrorSnapshotter {/*** Save the HTML snapshot of the page, and return the fileName with the extension.*/- async saveHTMLSnapshot(html: string, keyValueStore: KeyValueStore, fileName: string): Promise<string | undefined> {+ async saveHTMLSnapshot(+ html: string,+ keyValueStore: Pick<KeyValueStore, 'setValue'>,+ fileName: string,+ ): Promise<string | undefined> {try {await keyValueStore.setValue(fileName, html, { contentType: 'text/html' });return `${fileName}.html`;diff --git a/packages/core/src/crawlers/error_tracker.ts b/packages/core/src/crawlers/error_tracker.tsindex 38ee8f4a4..fa085a188 100644--- a/packages/core/src/crawlers/error_tracker.ts+++ b/packages/core/src/crawlers/error_tracker.ts@@ -2,6 +2,7 @@ import { inspect } from 'node:util';import type { CrawlingContext } from '../crawlers/crawler_commons.js';import { ErrorSnapshotter } from './error_snapshotter.js';+import type { SnapshottableProperties } from './internals/types.js';/*** Node.js Error interface@@ -405,7 +406,11 @@ export class ErrorTracker {return result.sort((a, b) => b[0] - a[0]).slice(0, count);}- async captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext) {+ async captureSnapshot(+ storage: Record<string, unknown>,+ error: ErrnoException,+ context: CrawlingContext & SnapshottableProperties,+ ) {if (!this.errorSnapshotter) {return;}diff --git a/packages/core/src/crawlers/index.ts b/packages/core/src/crawlers/index.tsindex f9b2e116c..5fa44b458 100644--- a/packages/core/src/crawlers/index.ts+++ b/packages/core/src/crawlers/index.ts@@ -1,5 +1,5 @@+export * from './context_pipeline.js';export * from './crawler_commons.js';-export * from './crawler_extension.js';export * from './crawler_utils.js';export * from './statistics.js';export * from './error_tracker.js';diff --git a/packages/core/src/crawlers/internals/types.ts b/packages/core/src/crawlers/internals/types.tsnew file mode 100644index 000000000..f631f17ac--- /dev/null+++ b/packages/core/src/crawlers/internals/types.ts@@ -0,0 +1,8 @@+export interface BrowserPage {+ content: () => Promise<string>;+}++export interface SnapshottableProperties {+ body?: unknown;+ page?: BrowserPage;+}diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.tsindex b7124d472..8a6b796f2 100644--- a/packages/core/src/enqueue_links/enqueue_links.ts+++ b/packages/core/src/enqueue_links/enqueue_links.ts@@ -7,13 +7,13 @@ import type { SetRequired } from 'type-fest';import log from '@apify/log';import type { Request, RequestOptions } from '../request.js';-import type {+import type {AddRequestsBatchedOptions,AddRequestsBatchedResult,- RequestProvider,- RequestQueueOperationOptions+ RequestProvider,+ RequestQueueOperationOptions,} from '../storages/request_provider.js';-import type {+import type {GlobInput,PseudoUrlInput,RegExpInput,@@ -22,7 +22,6 @@ import type {SkippedRequestReason,UrlPatternObject,} from './shared.js';-import {constructGlobObjectsFromGlobs,constructRegExpObjectsFromPseudoUrls,diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.tsindex 4c9cdc704..0dd551944 100644--- a/packages/core/src/errors.ts+++ b/packages/core/src/errors.ts@@ -61,3 +61,27 @@ export class SessionError extends RetryRequestError {super(`Detected a session error, rotating session... ${message ? `\n${message}` : ''}`);}}++export class ContextPipelineInterruptedError extends Error {+ constructor(message?: string) {+ super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`);+ }+}++export class ContextPipelineInitializationError extends Error {+ constructor(error: unknown, options?: ErrorOptions) {+ super(undefined, { cause: error, ...options });+ }+}++export class ContextPipelineCleanupError extends CriticalError {+ constructor(error: unknown, options?: ErrorOptions) {+ super(undefined, { cause: error, ...options });+ }+}++export class RequestHandlerError extends Error {+ constructor(error: unknown, options?: ErrorOptions) {+ super(undefined, { cause: error, ...options });+ }+}diff --git a/packages/core/src/storages/request_provider.ts b/packages/core/src/storages/request_provider.tsindex bc13e52c9..ff915aa2a 100644--- a/packages/core/src/storages/request_provider.ts+++ b/packages/core/src/storages/request_provider.ts@@ -19,6 +19,7 @@ import {sleep,} from '@crawlee/utils';import ow from 'ow';+import type { ReadonlyDeep } from 'type-fest';import { ListDictionary, LruCache } from '@apify/datastructures';import type { Log } from '@apify/log';@@ -413,7 +414,7 @@ export abstract class RequestProvider implements IStorage, IRequestManager {* @param options Options for the request queue*/async addRequestsBatched(- requests: RequestsLike,+ requests: ReadonlyDeep<RequestsLike>,options: AddRequestsBatchedOptions = {},): Promise<AddRequestsBatchedResult> {checkStorageAccess();diff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.tsindex 8a9498845..cf031e89f 100644--- a/packages/http-crawler/src/internals/file-download.ts+++ b/packages/http-crawler/src/internals/file-download.ts@@ -1,66 +1,42 @@+import type { Readable } from 'node:stream';import { Transform } from 'node:stream';+import { buffer } from 'node:stream/consumers';import { finished } from 'node:stream/promises';-import { isPromise } from 'node:util/types';+import type { BasicCrawlerOptions } from '@crawlee/basic';+import { BasicCrawler, ContextPipeline } from '@crawlee/basic';+import type { CrawlingContext, HttpResponse, LoadedRequest, Request, StreamingHttpResponse } from '@crawlee/core';import type { Dictionary } from '@crawlee/types';-import type { Request } from 'got-scraping';-import type {- ErrorHandler,- GetUserDataFromRequest,- HttpCrawlerOptions,- InternalHttpCrawlingContext,- InternalHttpHook,- RequestHandler,- RouterHandler,- RouterRoutes,- RouteSchemas,- RoutesFromSchemas,-} from '../index.js';-import { HttpCrawler, Router } from '../index.js';+import type { ErrorHandler, GetUserDataFromRequest, InternalHttpHook, RequestHandler, RouterRoutes } from '../index.js';+import { Router } from '../index.js';+import { parseContentTypeFromResponse } from './utils.js';export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> = ErrorHandler<FileDownloadCrawlingContext<UserData, JSONData>>;--export type StreamHandlerContext = Omit<- FileDownloadCrawlingContext,- 'body' | 'parseWithCheerio' | 'json' | 'addRequests' | 'contentType'-> & {- stream: Request; // TODO BC - remove in v4- abortDownload: () => void;-};--type StreamHandler = (context: StreamHandlerContext) => void | Promise<void>;--export type FileDownloadOptions<- UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> =- | (Omit<HttpCrawlerOptions<FileDownloadCrawlingContext<UserData, JSONData>>, 'requestHandler'> & {- requestHandler?: never;- streamHandler?: StreamHandler;- })- | (Omit<HttpCrawlerOptions<FileDownloadCrawlingContext<UserData, JSONData>>, 'requestHandler'> & {- requestHandler: FileDownloadRequestHandler;- streamHandler?: never;- });+> = ErrorHandler<FileDownloadCrawlingContext<UserData>>;export type FileDownloadHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> = InternalHttpHook<FileDownloadCrawlingContext<UserData, JSONData>>;+> = InternalHttpHook<FileDownloadCrawlingContext<UserData>>;export interface FileDownloadCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> extends InternalHttpCrawlingContext<UserData, JSONData, FileDownload> {}+> extends CrawlingContext<UserData> {+ request: LoadedRequest<Request<UserData>>;+ response: HttpResponse<'buffer'> | StreamingHttpResponse;+ body: Promise<Buffer>;+ stream: Readable;+ contentType: { type: string; encoding: BufferEncoding };+}export type FileDownloadRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> = RequestHandler<FileDownloadCrawlingContext<UserData, JSONData>>;+> = RequestHandler<FileDownloadCrawlingContext<UserData>>;++interface ContextInternals {+ pollingInterval?: NodeJS.Timeout;+}/*** Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.@@ -187,123 +163,74 @@ export function ByteCounterStream({* ]);* ```*/-export class FileDownload extends HttpCrawler<FileDownloadCrawlingContext> {- private streamHandler?: StreamHandler;-- constructor(options: FileDownloadOptions = {}) {- const { streamHandler } = options;- delete options.streamHandler;-- if (streamHandler) {- // For streams, the navigation is done in the request handler.- (options as any).requestHandlerTimeoutSecs = options.navigationTimeoutSecs ?? 120;- }-- super(options);-- this.streamHandler = streamHandler;- if (this.streamHandler) {- this.requestHandler = this.streamRequestHandler as any;- }-- // The base HttpCrawler class only supports a handful of text based mime types.- // With the FileDownload crawler, we want to download any file type.- (this as any).supportedMimeTypes = new Set(['*/*']);- }-- protected override async _runRequestHandler(context: FileDownloadCrawlingContext) {- if (this.streamHandler) {- context.request.skipNavigation = true;- }-- await super._runRequestHandler(context);+export class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {+ #contextInternals = Symbol('contextInternals');++ // TODO hooks+ constructor(options: BasicCrawlerOptions<FileDownloadCrawlingContext> = {}) {+ super({+ ...options,+ contextPipelineBuilder: () =>+ ContextPipeline.create<CrawlingContext>().compose({+ action: this.initiateDownload.bind(this),+ cleanup: this.cleanupDownload.bind(this),+ }),+ });}- private async streamRequestHandler(context: FileDownloadCrawlingContext) {- const {- log,- request: { url },- } = context;-+ private async initiateDownload(context: CrawlingContext) {const response = await this.httpClient.stream({- url,+ url: context.request.url,timeout: { request: undefined },proxyUrl: context.proxyInfo?.url,});- let pollingInterval: NodeJS.Timeout | undefined;- let abortedByHandler = false;+ const { type, charset: encoding } = parseContentTypeFromResponse(response);- const abortDownload = () => {- if (abortedByHandler || !response.stream.readable) {- return;- }+ context.request.url = response.url;- abortedByHandler = true;- response.stream.destroy();- };+ const pollingInterval = setInterval(() => {+ const { total, transferred } = response.downloadProgress;- const cleanUp = () => {- clearInterval(pollingInterval!);- response.stream.destroy();+ if (transferred > 0) {+ context.log.debug(+ `Downloaded ${transferred} bytes of ${total ?? 0} bytes from ${context.request.url}.`,+ );+ }+ }, 5000);++ const contextExtension = {+ [this.#contextInternals]: { pollingInterval } as ContextInternals,+ request: context.request as LoadedRequest<Request>,+ response,+ contentType: { type, encoding },+ stream: response.stream,+ get body() {+ return buffer(response.stream);+ },};- const downloadPromise = new Promise<void>((resolve, reject) => {- pollingInterval = setInterval(() => {- const { total, transferred } = response.downloadProgress;-- if (transferred > 0) {- log.debug(`Downloaded ${transferred} bytes of ${total ?? 0} bytes from ${url}.`);- }- }, 5000);+ return contextExtension;+ }- response.stream.on('error', async (error: Error) => {- if (abortedByHandler) {- resolve();- return;+ private async cleanupDownload(+ context: FileDownloadCrawlingContext & { [k: symbol]: ContextInternals },+ error?: unknown,+ ) {+ clearInterval(context[this.#contextInternals].pollingInterval);++ // If there was no error and the stream is still readable, wait for it to be consumed before proceeding+ if (error === undefined) {+ if (!context.stream.destroyed && context.stream.readable) {+ try {+ await finished(context.stream);+ } catch {+ // Stream might have encountered an error or been closed, which is fine}-- cleanUp();- reject(error);- });-- let streamHandlerResult;-- try {- context.stream = response.stream;- context.response = response as any;- (context as any).abortDownload = abortDownload;- streamHandlerResult = this.streamHandler!(context as any);- } catch (e) {- cleanUp();- reject(e);- }-- if (isPromise(streamHandlerResult)) {- streamHandlerResult- .then(() => {- resolve();- })- .catch((e: Error) => {- cleanUp();- reject(e);- });- } else {- resolve();}- });-- const streamFinishedPromise = finished(response.stream).catch((error) => {- if (abortedByHandler) {- return;- }-- throw error;- });-- await Promise.all([downloadPromise, streamFinishedPromise]);-- cleanUp();+ } else {+ context.stream.destroy();+ }}}@@ -331,18 +258,9 @@ export class FileDownload extends HttpCrawler<FileDownloadCrawlingContext> {* await crawler.run();* ```*/-export function createFileRouter<- Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext,- Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,->(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;export function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext,UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,->(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;-export function createFileRouter<- Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext,- const Schemas extends RouteSchemas = RouteSchemas,->(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;-export function createFileRouter(routesOrSchemas?: any): any {- return Router.create(routesOrSchemas);+>(routes?: RouterRoutes<Context, UserData>) {+ return Router.create<Context>(routes);}diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 04b88440a..1b00549f1 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -1,5 +1,4 @@import type { IncomingHttpHeaders, IncomingMessage } from 'node:http';-import { extname } from 'node:path';import type { Readable } from 'node:stream';import util from 'node:util';@@ -9,10 +8,10 @@ import type {CrawlingContext,ErrorHandler,GetUserDataFromRequest,- LoadedContext,ProxyConfiguration,Request,RequestHandler,+ RequireContextPipeline,RouterHandler,RouterRoutes,RouteSchemas,@@ -20,11 +19,10 @@ import type {Session,} from '@crawlee/basic';import {- BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,BasicCrawler,BLOCKED_STATUS_CODES,Configuration,- CrawlerExtension,+ ContextPipeline,mergeCookies,processHttpRequestOptions,RequestState,@@ -32,7 +30,7 @@ import {SessionError,validators,} from '@crawlee/basic';-import type { HttpResponse, StreamingHttpResponse } from '@crawlee/core';+import type { HttpResponse, LoadedRequest, ProxyInfo, StreamingHttpResponse } from '@crawlee/core';import type { Awaitable, Dictionary } from '@crawlee/types';import { type CheerioRoot, RETRY_CSS_SELECTORS } from '@crawlee/utils';import * as cheerio from 'cheerio';@@ -40,13 +38,14 @@ import type { RequestLike, ResponseLike } from 'content-type';import contentTypeParser from 'content-type';import type { Method, OptionsInit, TimeoutError as TimeoutErrorClass } from 'got-scraping';import iconv from 'iconv-lite';-import mime from 'mime-types';-import ow, { ObjectPredicate } from 'ow';+import ow from 'ow';import type { JsonValue } from 'type-fest';import { addTimeoutToPromise, tryCancel } from '@apify/timeout';import { concatStreamToBuffer, readStreamToString } from '@apify/utilities';+import { parseContentTypeFromResponse } from './utils.js';+let TimeoutError: typeof TimeoutErrorClass;/**@@ -79,8 +78,10 @@ export type HttpErrorHandler<JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler> = ErrorHandler<HttpCrawlingContext<UserData, JSONData>>;-export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext>- extends BasicCrawlerOptions<Context> {+export interface HttpCrawlerOptions<+ Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext,+ ExtendedContext extends Context = Context,+> extends BasicCrawlerOptions<Context, ExtendedContext> {/*** Timeout in which the HTTP request to the resource needs to finish, given in seconds.*/@@ -114,7 +115,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext* Modyfing `pageOptions` is supported only in Playwright incognito.* See {@apilink PrePageCreateHook}*/- preNavigationHooks?: InternalHttpHook<Context>[];+ preNavigationHooks?: InternalHttpHook<CrawlingContext>[];/*** Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.@@ -128,7 +129,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext* ]* ```*/- postNavigationHooks?: InternalHttpHook<Context>[];+ postNavigationHooks?: ((crawlingContext: CrawlingContextWithReponse) => Awaitable<void>)[];/*** An array of [MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types)@@ -192,14 +193,27 @@ export type HttpHook<JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;+interface CrawlingContextWithReponse<+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler+> extends CrawlingContext<UserData> {+ /**+ * The request object that was successfully loaded and navigated to, including the {@apilink Request.loadedUrl|`loadedUrl`} property.+ */+ request: LoadedRequest<Request<UserData>>;++ /**+ * The HTTP response object containing status code, headers, and other response metadata.+ */+ response: PlainResponse;+}+/*** @internal*/export interface InternalHttpCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler- Crawler = HttpCrawler<any>,-> extends CrawlingContext<Crawler, UserData> {+> extends CrawlingContextWithReponse<UserData> {/*** The request body of the web page.* The type depends on the `Content-Type` header of the web page:@@ -217,7 +231,6 @@ export interface InternalHttpCrawlingContext<* Parsed `Content-Type header: { type, encoding }`.*/contentType: { type: string; encoding: BufferEncoding };- response: PlainResponse;/*** Wait for an element matching the selector to appear. Timeout is ignored.@@ -249,7 +262,7 @@ export interface InternalHttpCrawlingContext<}export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any>- extends InternalHttpCrawlingContext<UserData, JSONData, HttpCrawler<HttpCrawlingContext<UserData, JSONData>>> {}+ extends InternalHttpCrawlingContext<UserData, JSONData> {}export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler@@ -324,17 +337,18 @@ export type HttpRequestHandler<* @category Crawlers*/export class HttpCrawler<- Context extends InternalHttpCrawlingContext<any, any, HttpCrawler<Context>>,-> extends BasicCrawler<Context> {+ Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext,+ ContextExtension = {},+ ExtendedContext extends Context = Context & ContextExtension,+> extends BasicCrawler<Context, ExtendedContext> {/*** A reference to the underlying {@apilink ProxyConfiguration} class that manages the crawler's proxies.* Only available if used by the crawler.*/proxyConfiguration?: ProxyConfiguration;- protected userRequestHandlerTimeoutMillis: number;- protected preNavigationHooks: InternalHttpHook<Context>[];- protected postNavigationHooks: InternalHttpHook<Context>[];+ protected preNavigationHooks: InternalHttpHook<CrawlingContext>[];+ protected postNavigationHooks: ((crawlingContext: CrawlingContextWithReponse) => Awaitable<void>)[];protected persistCookiesPerSession: boolean;protected navigationTimeoutMillis: number;protected ignoreSslErrors: boolean;@@ -366,14 +380,13 @@ export class HttpCrawler<* All `HttpCrawlerOptions` parameters are passed via an options object.*/constructor(- options: HttpCrawlerOptions<Context> = {},+ options: HttpCrawlerOptions<Context, ExtendedContext> &+ RequireContextPipeline<InternalHttpCrawlingContext, Context> = {} as any,override readonly config = Configuration.getGlobalConfig(),) {ow(options, 'HttpCrawlerOptions', ow.object.exactShape(HttpCrawler.optionsShape));const {- requestHandler,- requestHandlerTimeoutSecs = 60,navigationTimeoutSecs = 30,ignoreSslErrors = true,additionalMimeTypes = [],@@ -388,24 +401,21 @@ export class HttpCrawler<// BasicCrawlerautoscaledPoolOptions = HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS,+ contextPipelineBuilder,...basicCrawlerOptions} = options;super({...basicCrawlerOptions,- requestHandler,autoscaledPoolOptions,- // We need to add some time for internal functions to finish,- // but not too much so that we would stall the crawler.- requestHandlerTimeoutSecs:- navigationTimeoutSecs + requestHandlerTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,+ contextPipelineBuilder:+ contextPipelineBuilder ??+ (() => this.buildContextPipeline() as ContextPipeline<CrawlingContext, Context>),},config,);- this.requestHandler = requestHandler ?? this.router;-// Cookies should be persisted per session only if session pool is usedif (!this.useSessionPool && persistCookiesPerSession) {throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');@@ -420,7 +430,6 @@ export class HttpCrawler<);}- this.userRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;this.ignoreSslErrors = ignoreSslErrors;this.suggestResponseEncoding = suggestResponseEncoding;@@ -441,138 +450,159 @@ export class HttpCrawler<}}- /**- * **EXPERIMENTAL**- * Function for attaching CrawlerExtensions such as the Unblockers.- * @param extension Crawler extension that overrides the crawler configuration.- */- use(extension: CrawlerExtension) {- ow(extension, ow.object.instanceOf(CrawlerExtension));-- const className = this.constructor.name;-- const extensionOptions = extension.getCrawlerOptions();-- for (const [key, value] of Object.entries(extensionOptions)) {- const isConfigurable = Object.hasOwn(this, key);- const originalType = typeof this[key as keyof this];- const extensionType = typeof value; // What if we want to null something? It is really needed?- const isSameType = originalType === extensionType || value == null; // fast track for deleting keys- const exists = this[key as keyof this] != null;-- if (!isConfigurable) {- // Test if the property can be configured on the crawler- throw new Error(- `${extension.name} tries to set property "${key}" that is not configurable on ${className} instance.`,- );- }-- if (!isSameType && exists) {- // Assuming that extensions will only add up configuration- throw new Error(- `${extension.name} tries to set property of different type "${extensionType}". "${className}.${key}: ${originalType}".`,- );- }+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext> {+ return ContextPipeline.create<CrawlingContext>()+ .compose({ action: this.prepareProxyInfo.bind(this) })+ .compose({+ action: this.makeHttpRequest.bind(this),+ })+ .compose({ action: this.processHttpResponse.bind(this) })+ .compose({ action: this.handleBlockedRequestByContent.bind(this) });+ }- this.log.warning(`${extension.name} is overriding "${className}.${key}: ${originalType}" with ${value}.`);+ private async prepareProxyInfo(crawlingContext: CrawlingContext) {+ const { request, session } = crawlingContext;+ let proxyInfo: ProxyInfo | undefined;- this[key as keyof this] = value as this[keyof this];+ if (this.proxyConfiguration) {+ const sessionId = session ? session.id : undefined;+ proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId, { request });}++ return { proxyInfo };}- /**- * Wrapper around requestHandler that opens and closes pages etc.- */- protected override async _runRequestHandler(crawlingContext: Context) {+ private async makeHttpRequest(+ crawlingContext: CrawlingContext,+ ): Promise<Omit<CrawlingContextWithReponse, keyof CrawlingContext> & Partial<CrawlingContextWithReponse>> {const { request, session } = crawlingContext;- if (this.proxyConfiguration) {- const sessionId = session ? session.id : undefined;- crawlingContext.proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId, { request });+ if (request.skipNavigation) {+ return {+ request: new Proxy(request, {+ get(target, propertyName, receiver) {+ if (propertyName === 'loadedUrl') {+ throw new Error(+ 'The `request.loadedUrl` property is not available - `skipNavigation` was used',+ );+ }+ return Reflect.get(target, propertyName, receiver);+ },+ }) as LoadedRequest<Request>,+ get response(): InternalHttpCrawlingContext['response'] {+ throw new Error('The `response` property is not available - `skipNavigation` was used');+ },+ };}- if (!request.skipNavigation) {- await this._handleNavigation(crawlingContext);- tryCancel();+ const gotOptions = {} as OptionsInit;+ const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request);- const parsed = await this._parseResponse(request, crawlingContext.response!, crawlingContext);- const response = parsed.response!;- const contentType = parsed.contentType!;- tryCancel();+ request.state = RequestState.BEFORE_NAV;+ // Execute pre navigation hooks before applying session pool cookies,+ // as they may also set cookies in the session+ await this._executeHooks(this.preNavigationHooks, crawlingContext, gotOptions);+ tryCancel();- // `??=` because descendant classes may already set optimized version- crawlingContext.waitForSelector ??= async (selector?: string, _timeoutMs?: number) => {- const $ = cheerio.load(parsed.body!.toString());+ const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request);- if ($(selector).get().length === 0) {- throw new Error(`Selector '${selector}' not found.`);- }- };- crawlingContext.parseWithCheerio ??= async (selector?: string, timeoutMs?: number) => {- const $ = cheerio.load(parsed.body!.toString());+ this._applyCookies(crawlingContext, gotOptions, preNavigationHooksCookies, postNavigationHooksCookies);- if (selector) {- await crawlingContext.waitForSelector(selector, timeoutMs);- }+ const proxyUrl = crawlingContext.proxyInfo?.url;- return $;- };+ const httpResponse = await addTimeoutToPromise(+ async () => this._requestFunction({ request, session, proxyUrl, gotOptions }),+ this.navigationTimeoutMillis,+ `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`,+ );+ tryCancel();- if (this.useSessionPool) {- this._throwOnBlockedRequest(crawlingContext.session!, response.statusCode!);- }+ request.loadedUrl = httpResponse.url;+ request.state = RequestState.AFTER_NAV;- if (this.persistCookiesPerSession) {- crawlingContext.session!.setCookiesFromResponse(response);- }+ return { request: request as LoadedRequest<Request>, response: httpResponse };+ }++ private async processHttpResponse(+ crawlingContext: CrawlingContextWithReponse,+ ): Promise<+ Omit<InternalHttpCrawlingContext, keyof CrawlingContextWithReponse> & Partial<InternalHttpCrawlingContext>+ > {+ if (crawlingContext.request.skipNavigation) {+ return {+ get contentType(): InternalHttpCrawlingContext['contentType'] {+ throw new Error('The `contentType` property is not available - `skipNavigation` was used');+ },+ get body(): InternalHttpCrawlingContext['body'] {+ throw new Error('The `body` property is not available - `skipNavigation` was used');+ },+ get json(): InternalHttpCrawlingContext['json'] {+ throw new Error('The `json` property is not available - `skipNavigation` was used');+ },+ get waitForSelector(): InternalHttpCrawlingContext['waitForSelector'] {+ throw new Error('The `waitForSelector` method is not available - `skipNavigation` was used');+ },+ get parseWithCheerio(): InternalHttpCrawlingContext['parseWithCheerio'] {+ throw new Error('The `parseWithCheerio` method is not available - `skipNavigation` was used');+ },+ };+ }- request.loadedUrl = response.url;+ await this._executeHooks(this.postNavigationHooks, crawlingContext);+ tryCancel();- if (!this.requestMatchesEnqueueStrategy(request)) {- this.log.debug(- // eslint-disable-next-line dot-notation- `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`,- );+ const parsed = await this._parseResponse(crawlingContext.request, crawlingContext.response);+ tryCancel();+ const response = parsed.response!;+ const contentType = parsed.contentType!;- request.noRetry = true;- request.state = RequestState.SKIPPED;+ const waitForSelector = async (selector: string, _timeoutMs?: number) => {+ const $ = cheerio.load(parsed.body!.toString());- await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });+ if ($(selector).get().length === 0) {+ throw new Error(`Selector '${selector}' not found.`);+ }+ };+ const parseWithCheerio = async (selector?: string, timeoutMs?: number) => {+ const $ = cheerio.load(parsed.body!.toString());- return;+ if (selector) {+ await (crawlingContext as InternalHttpCrawlingContext).waitForSelector(selector, timeoutMs);}- Object.assign(crawlingContext, parsed);+ return $;+ };++ if (this.useSessionPool) {+ this._throwOnBlockedRequest(crawlingContext.session!, response.statusCode!);+ }- Object.defineProperty(crawlingContext, 'json', {- get() {- if (contentType.type !== APPLICATION_JSON_MIME_TYPE) return null;- const jsonString = parsed.body!.toString(contentType.encoding);- return JSON.parse(jsonString);- },- });+ if (this.persistCookiesPerSession) {+ crawlingContext.session!.setCookiesFromResponse(response);}+ return {+ get json() {+ if (contentType.type !== APPLICATION_JSON_MIME_TYPE) return null;+ const jsonString = parsed.body!.toString(contentType.encoding);+ return JSON.parse(jsonString);+ },+ waitForSelector,+ parseWithCheerio,+ contentType,+ body: parsed.body!,+ };+ }++ private async handleBlockedRequestByContent(crawlingContext: InternalHttpCrawlingContext): Promise<{}> {if (this.retryOnBlocked) {const error = await this.isRequestBlocked(crawlingContext);if (error) throw new SessionError(error);}-- request.state = RequestState.REQUEST_HANDLER;- try {- await addTimeoutToPromise(- async () => Promise.resolve(this.requestHandler(crawlingContext as LoadedContext<Context>)),- this.userRequestHandlerTimeoutMillis,- `requestHandler timed out after ${this.userRequestHandlerTimeoutMillis / 1000} seconds.`,- );- request.state = RequestState.DONE;- } catch (e: any) {- request.state = RequestState.ERROR;- throw e;- }+ return {};}- protected override async isRequestBlocked(crawlingContext: Context): Promise<string | false> {+ protected async isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise<string | false> {if (HTML_AND_XML_MIME_TYPES.includes(crawlingContext.contentType.type)) {const $ = await crawlingContext.parseWithCheerio();@@ -597,35 +627,6 @@ export class HttpCrawler<return false;}- protected async _handleNavigation(crawlingContext: Context) {- const gotOptions = {} as OptionsInit;- const { request, session } = crawlingContext;- const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request);-- request.state = RequestState.BEFORE_NAV;- // Execute pre navigation hooks before applying session pool cookies,- // as they may also set cookies in the session- await this._executeHooks(this.preNavigationHooks, crawlingContext, gotOptions);- tryCancel();-- const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request);-- this._applyCookies(crawlingContext, gotOptions, preNavigationHooksCookies, postNavigationHooksCookies);-- const proxyUrl = crawlingContext.proxyInfo?.url;-- crawlingContext.response = await addTimeoutToPromise(- async () => this._requestFunction({ request, session, proxyUrl, gotOptions }),- this.navigationTimeoutMillis,- `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`,- );- tryCancel();-- request.state = RequestState.AFTER_NAV;- await this._executeHooks(this.postNavigationHooks, crawlingContext, gotOptions);- tryCancel();- }-/*** Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks.*/@@ -720,7 +721,7 @@ export class HttpCrawler</*** Encodes and parses response according to the provided content type*/- protected async _parseResponse(request: Request, responseStream: IncomingMessage, crawlingContext: Context) {+ private async _parseResponse(request: Request, responseStream: IncomingMessage) {const { statusCode } = responseStream;const { type, charset } = parseContentTypeFromResponse(responseStream);const { response, encoding } = this._encodeResponse(request, responseStream, charset);@@ -752,30 +753,17 @@ export class HttpCrawler<// It's not a JSON, so it's probably some text. Get the first 100 chars of it.throw new Error(`${statusCode} - Internal Server Error: ${body.slice(0, 100)}`);} else if (HTML_AND_XML_MIME_TYPES.includes(type)) {- const isXml = type.includes('xml');- const parsed = await this._parseHTML(response, isXml, crawlingContext);- return { ...parsed, isXml, response, contentType };+ return { response, contentType, body: await readStreamToString(response) };} else {const body = await concatStreamToBuffer(response);return {body,response,contentType,- enqueueLinks: async () => Promise.resolve({ processedRequests: [], unprocessedRequests: [] }),};}}- protected async _parseHTML(- response: IncomingMessage,- _isXml: boolean,- _crawlingContext: Context,- ): Promise<Partial<Context>> {- return {- body: await concatStreamToBuffer(response),- } as Partial<Context>;- }-/*** Combines the provided `requestOptions` with mandatory (non-overridable) values.*/@@ -883,7 +871,7 @@ export class HttpCrawler<*/protected _handleRequestTimeout(session?: Session) {session?.markBad();- throw new Error(`request timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds.`);+ throw new Error(`request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);}private _abortDownloadOfBody(request: Request, response: IncomingMessage) {@@ -982,44 +970,6 @@ function addResponsePropertiesToStream(stream: Readable, response: StreamingHttpreturn stream as unknown as PlainResponse;}-/**- * Gets parsed content type from response object- * @param response HTTP response object- */-function parseContentTypeFromResponse(response: unknown): { type: string; charset: BufferEncoding } {- ow(- response,- ow.object.partialShape({- url: ow.string.url,- headers: new ObjectPredicate<Record<string, unknown>>(),- }),- );-- const { url, headers } = response;- let parsedContentType;-- if (headers['content-type']) {- try {- parsedContentType = contentTypeParser.parse(headers['content-type'] as string);- } catch {- // Can not parse content type from Content-Type header. Try to parse it from file extension.- }- }-- // Parse content type from file extension as fallback- if (!parsedContentType) {- const parsedUrl = new URL(url);- const contentTypeFromExtname =- mime.contentType(extname(parsedUrl.pathname)) || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5- parsedContentType = contentTypeParser.parse(contentTypeFromExtname);- }-- return {- type: parsedContentType.type,- charset: parsedContentType.parameters.charset as BufferEncoding,- };-}-/*** Creates new {@apilink Router} instance that works based on request labels.* This instance can then serve as a `requestHandler` of your {@apilink HttpCrawler}.diff --git a/packages/http-crawler/src/internals/utils.ts b/packages/http-crawler/src/internals/utils.tsnew file mode 100644index 000000000..0dcfe707d--- /dev/null+++ b/packages/http-crawler/src/internals/utils.ts@@ -0,0 +1,43 @@+import { extname } from 'node:path';++import contentTypeParser from 'content-type';+import mime from 'mime-types';+import ow, { ObjectPredicate } from 'ow';++/**+ * Gets parsed content type from response object+ * @param response HTTP response object+ */+export function parseContentTypeFromResponse(response: unknown): { type: string; charset: BufferEncoding } {+ ow(+ response,+ ow.object.partialShape({+ url: ow.string.url,+ headers: new ObjectPredicate<Record<string, unknown>>(),+ }),+ );++ const { url, headers } = response;+ let parsedContentType;++ if (headers['content-type']) {+ try {+ parsedContentType = contentTypeParser.parse(headers['content-type'] as string);+ } catch {+ // Can not parse content type from Content-Type header. Try to parse it from file extension.+ }+ }++ // Parse content type from file extension as fallback+ if (!parsedContentType) {+ const parsedUrl = new URL(url);+ const contentTypeFromExtname =+ mime.contentType(extname(parsedUrl.pathname)) || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5+ parsedContentType = contentTypeParser.parse(contentTypeFromExtname);+ }++ return {+ type: parsedContentType.type,+ charset: parsedContentType.parameters.charset as BufferEncoding,+ };+}diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsindex e17aaa331..eb625110c 100644--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts@@ -1,5 +1,3 @@-import type { IncomingMessage } from 'node:http';-import type {BasicCrawlingContext,Configuration,@@ -32,7 +30,6 @@ import { JSDOM, ResourceLoader, VirtualConsole } from 'jsdom';import ow from 'ow';import { addTimeoutToPromise } from '@apify/timeout';-import { concatStreamToBuffer } from '@apify/utilities';export type JSDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler@@ -40,9 +37,10 @@ export type JSDOMErrorHandler<> = ErrorHandler<JSDOMCrawlingContext<UserData, JSONData>>;export interface JSDOMCrawlerOptions<+ ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext,UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>> {+> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ExtendedContext> {/*** Download and run scripts.*/@@ -61,10 +59,12 @@ export type JSDOMHook<export interface JSDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> extends InternalHttpCrawlingContext<UserData, JSONData, JSDOMCrawler> {+> extends InternalHttpCrawlingContext<UserData, JSONData> {window: DOMWindow;document: Document;+ body: string;+/*** Wait for an element matching the selector to appear.* Timeout defaults to 5s.@@ -180,7 +180,10 @@ const resources = new ResourceLoader({'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',});-export class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {+export class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext> extends HttpCrawler<+ JSDOMCrawlingContext,+ ExtendedContext+> {protected static override optionsShape = {...HttpCrawler.optionsShape,runScripts: ow.optional.boolean,@@ -191,10 +194,25 @@ export class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {protected hideInternalConsole: boolean;protected virtualConsole: VirtualConsole | null = null;- constructor(options: JSDOMCrawlerOptions = {}, config?: Configuration) {+ constructor(options: JSDOMCrawlerOptions<ExtendedContext> = {}, config?: Configuration) {const { runScripts = false, hideInternalConsole = false, ...httpOptions } = options;- super(httpOptions, config);+ super(+ {+ ...httpOptions,+ contextPipelineBuilder: () =>+ this.buildContextPipeline()+ .compose({+ action: async (context) => await this.parseContent(context),+ cleanup: async (context) => {+ this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler);+ context.window?.close();+ },+ })+ .compose({ action: async (context) => await this.addHelpers(context) }),+ },+ config,+ );this.runScripts = runScripts;this.hideInternalConsole = hideInternalConsole;@@ -232,20 +250,12 @@ export class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {private readonly jsdomErrorHandler = (error: Error) => this.log.debug('JSDOM error from console', error);- protected override async _cleanupContext(context: JSDOMCrawlingContext) {- this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler);- context.window?.close();- }-- protected override async _parseHTML(- response: IncomingMessage,- isXml: boolean,- crawlingContext: JSDOMCrawlingContext,- ) {- const body = await concatStreamToBuffer(response);+ private async parseContent(crawlingContext: InternalHttpCrawlingContext) {+ const isXml = crawlingContext.contentType.type.includes('xml');- const { window } = new JSDOM(body, {- url: response.url,+ // TODO handle non-string+ const { window } = new JSDOM(crawlingContext.body.toString(), {+ url: crawlingContext.response.url,contentType: isXml ? 'text/xml' : 'text/html',runScripts: this.runScripts ? 'dangerously' : undefined,resources,@@ -306,12 +316,17 @@ export class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {get document() {return window.document;},+ };+ }++ private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { body: string; window: DOMWindow }) {+ return {enqueueLinks: async (enqueueOptions?: EnqueueLinksOptions) => {return domCrawlerEnqueueLinks({// `originalEnqueueLinks` clamps `limit` by the remaining `maxRequestsPerCrawl` budget itself;// pre-clamping it here would make the crawler log the internal limit as a user-provided oneoptions: enqueueOptions,- window,+ window: crawlingContext.window,requestQueue: await this.getRequestQueue(),robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),onSkippedRequest: this.handleSkippedRequest,@@ -320,34 +335,29 @@ export class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {enqueueLinks: originalEnqueueLinks,});},- };- }+ async waitForSelector(selector: string, timeoutMs = 5_000) {+ const $ = cheerio.load(crawlingContext.body);- override async _runRequestHandler(context: JSDOMCrawlingContext) {- context.waitForSelector = async (selector: string, timeoutMs = 5_000) => {- const $ = cheerio.load(context.body);+ if ($(selector).get().length === 0) {+ if (timeoutMs) {+ await sleep(50);+ await this.waitForSelector(selector, Math.max(timeoutMs - 50, 0));+ return;+ }- if ($(selector).get().length === 0) {- if (timeoutMs) {- await sleep(50);- await context.waitForSelector(selector, Math.max(timeoutMs - 50, 0));- return;+ throw new Error(`Selector '${selector}' not found.`);}+ },+ async parseWithCheerio(selector?: string, _timeoutMs = 5_000) {+ const $ = cheerio.load(crawlingContext.body);- throw new Error(`Selector '${selector}' not found.`);- }- };- context.parseWithCheerio = async (selector?: string, _timeoutMs = 5_000) => {- const $ = cheerio.load(context.body);-- if (selector && $(selector).get().length === 0) {- throw new Error(`Selector '${selector}' not found.`);- }+ if (selector && $(selector).get().length === 0) {+ throw new Error(`Selector '${selector}' not found.`);+ }- return $;+ return $;+ },};-- await super._runRequestHandler(context);}}diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.tsindex 9ed57d0fe..d77aecf02 100644--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts@@ -1,5 +1,3 @@-import type { IncomingMessage } from 'node:http';-import type {BasicCrawlingContext,EnqueueLinksOptions,@@ -28,17 +26,16 @@ import { type CheerioRoot, type RobotsTxtFile, sleep } from '@crawlee/utils';import * as cheerio from 'cheerio';import { DOMParser } from 'linkedom/cached';-import { concatStreamToBuffer } from '@apify/utilities';-export type LinkeDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler> = ErrorHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;export interface LinkeDOMCrawlerOptions<+ ExtendedContext extends LinkeDOMCrawlingContext,UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>> {}+> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ExtendedContext> {}export interface LinkeDOMCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestQueue'> {}@@ -50,7 +47,7 @@ export type LinkeDOMHook<export interface LinkeDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> extends InternalHttpCrawlingContext<UserData, JSONData, LinkeDOMCrawler> {+> extends InternalHttpCrawlingContext<UserData, JSONData> {window: Window;// Technically the document is not of type Document but of type either HTMLDocument or XMLDocument// from linkedom/types/{html/xml}/document, depending on the content type of the response@@ -165,17 +162,29 @@ export type LinkeDOMRequestHandler<* @category Crawlers*/-export class LinkeDOMCrawler extends HttpCrawler<LinkeDOMCrawlingContext> {+export class LinkeDOMCrawler<+ ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext,+> extends HttpCrawler<LinkeDOMCrawlingContext, ExtendedContext> {private static parser = new DOMParser();- protected override async _parseHTML(- response: IncomingMessage,- isXml: boolean,- crawlingContext: LinkeDOMCrawlingContext,- ) {- const body = await concatStreamToBuffer(response);+ constructor(options: LinkeDOMCrawlerOptions<ExtendedContext>) {+ super({+ ...options,+ contextPipelineBuilder: () =>+ this.buildContextPipeline()+ .compose({+ action: async (context) => this.parseContent(context),+ })+ .compose({ action: async (context) => this.addHelpers(context) }),+ });+ }- const document = LinkeDOMCrawler.parser.parseFromString(body.toString(), isXml ? 'text/xml' : 'text/html');+ private async parseContent(crawlingContext: InternalHttpCrawlingContext) {+ const isXml = crawlingContext.contentType.type.includes('xml');+ const document = LinkeDOMCrawler.parser.parseFromString(+ crawlingContext.body.toString(),+ isXml ? 'text/xml' : 'text/html',+ );const originalEnqueueLinks = crawlingContext.enqueueLinks;@@ -188,6 +197,11 @@ export class LinkeDOMCrawler extends HttpCrawler<LinkeDOMCrawlingContext> {// See comment about typing in LinkeDOMCrawlingContext definitionreturn document as unknown as Document;},+ };+ }++ private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { body: string }) {+ return {enqueueLinks: async (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => {return linkedomCrawlerEnqueueLinks({// `originalEnqueueLinks` clamps `limit` by the remaining `maxRequestsPerCrawl` budget itself;@@ -202,34 +216,29 @@ export class LinkeDOMCrawler extends HttpCrawler<LinkeDOMCrawlingContext> {enqueueLinks: originalEnqueueLinks,});},- };- }+ async waitForSelector(selector: string, timeoutMs = 5_000) {+ const $ = cheerio.load(crawlingContext.body);- override async _runRequestHandler(context: LinkeDOMCrawlingContext) {- context.waitForSelector = async (selector: string, timeoutMs = 5_000) => {- const $ = cheerio.load(context.body);+ if ($(selector).get().length === 0) {+ if (timeoutMs) {+ await sleep(50);+ await this.waitForSelector(selector, Math.max(timeoutMs - 50, 0));+ return;+ }- if ($(selector).get().length === 0) {- if (timeoutMs) {- await sleep(50);- await context.waitForSelector(selector, Math.max(timeoutMs - 50, 0));- return;+ throw new Error(`Selector '${selector}' not found.`);}+ },+ async parseWithCheerio(selector?: string, _timeoutMs = 5_000) {+ const $ = cheerio.load(crawlingContext.body);- throw new Error(`Selector '${selector}' not found.`);- }- };- context.parseWithCheerio = async (selector?: string, _timeoutMs = 5_000) => {- const $ = cheerio.load(context.body);-- if (selector && $(selector).get().length === 0) {- throw new Error(`Selector '${selector}' not found.`);- }+ if (selector && $(selector).get().length === 0) {+ throw new Error(`Selector '${selector}' not found.`);+ }- return $;+ return $;+ },};-- await super._runRequestHandler(context);}}diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.jsonindex 5f9802078..5649a6d9e 100644--- a/packages/playwright-crawler/package.json+++ b/packages/playwright-crawler/package.json@@ -52,6 +52,7 @@"@apify/timeout": "^0.3.2","@crawlee/browser": "3.17.0","@crawlee/browser-pool": "3.17.0",+ "@crawlee/cheerio": "3.17.0","@crawlee/core": "3.17.0","@crawlee/types": "3.17.0","@crawlee/utils": "3.17.0",diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex 6f9022513..6016a289c 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -9,9 +9,16 @@ import type {RoutesFromSchemas,} from '@crawlee/browser';import { isDeepStrictEqual } from 'node:util';++import { BasicCrawler } from '@crawlee/basic';+import type { BasicCrawlerOptions, BrowserHook, LoadedRequest, Request } from '@crawlee/browser';import { extractUrlsFromPage } from '@crawlee/browser';+import type { CheerioCrawlingContext } from '@crawlee/cheerio';+import { CheerioCrawler } from '@crawlee/cheerio';import type {BaseHttpResponseData,+ ContextPipeline,+ CrawlingContext,EnqueueLinksOptions,GetUserDataFromRequest,RequestQueue,@@ -23,27 +30,24 @@ import type {} from '@crawlee/core';import {Configuration,+ RequestHandlerError,RequestHandlerResult,- RequestState,resolveBaseUrlForEnqueueLinksFiltering,Router,Statistics,withCheckedStorageAccess,} from '@crawlee/core';-import type { Awaitable, BatchAddRequestsResult, Dictionary } from '@crawlee/types';+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';import { type CheerioRoot, extractUrlsFromCheerio } from '@crawlee/utils';-import { type Cheerio, load } from 'cheerio';+import { type Cheerio } from 'cheerio';+import type { AnyNode } from 'domhandler';import type { Page } from 'playwright';import type { SetRequired } from 'type-fest';import type { Log } from '@apify/log';import { addTimeoutToPromise } from '@apify/timeout';-import type {- PlaywrightCrawlerOptions,- PlaywrightCrawlingContext,- PlaywrightGotoOptions,-} from './playwright-crawler.js';+import type { PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';import { PlaywrightCrawler } from './playwright-crawler.js';import { type RenderingType, RenderingTypePredictor } from './utils/rendering-type-prediction.js';@@ -110,7 +114,8 @@ class AdaptivePlaywrightCrawlerStatistics extends Statistics {}export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = Dictionary>- extends RestrictedCrawlingContext<UserData> {+ extends CrawlingContext<UserData> {+ request: LoadedRequest<Request<UserData>>;/*** The HTTP response, either from the HTTP client or from the initial request from playwright's navigation.*/@@ -125,7 +130,7 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =* Wait for an element matching the selector to appear and return a Cheerio object of matched elements.* Timeout defaults to 5s.*/- querySelector<T = any>(selector: string, timeoutMs?: number): Promise<Cheerio<T>>;+ querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;/*** Wait for an element matching the selector to appear.@@ -155,29 +160,25 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =* ```*/parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;++ enqueueLinks(options?: EnqueueLinksOptions): Promise<void>;}interface AdaptiveHookextends BrowserHook<- Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'request' | 'session' | 'proxyInfo' | 'log'> & { page?: Page },+ Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log'> & {+ page?: Page;+ request: Request;+ },PlaywrightGotoOptions> {}-export interface AdaptivePlaywrightCrawlerOptions- extends Omit<PlaywrightCrawlerOptions, 'requestHandler' | 'preNavigationHooks' | 'postNavigationHooks'> {- /**- * Function that is called to process each request.- *- * The function receives the {@apilink AdaptivePlaywrightCrawlingContext} as an argument, and it must refrain from calling code with side effects,- * other than the methods of the crawling context. Any other side effects may be invoked repeatedly by the crawler, which can lead to inconsistent results.- *- * The function must return a promise, which is then awaited by the crawler.- *- * If the function throws an exception, the crawler will try to re-crawl the- * request later, up to `option.maxRequestRetries` times.- */- requestHandler?: (crawlingContext: LoadedContext<AdaptivePlaywrightCrawlerContext>) => Awaitable<void>;-+export interface AdaptivePlaywrightCrawlerOptions<+ ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext,+> extends Omit<+ BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ExtendedContext>,+ 'preNavigationHooks' | 'postNavigationHooks'+ > {/*** Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.* The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,@@ -190,7 +191,7 @@ export interface AdaptivePlaywrightCrawlerOptions* The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,* an exception will be thrown. If it's not caught, the request will be transparently retried in a browser.*/- postNavigationHooks?: AdaptiveHook[];+ postNavigationHooks?: AdaptiveHook[]; // TODO should contain a LoadedRequest - reflect that/*** Specifies the frequency of rendering type detection checks - 0.1 means roughly 10% of requests.@@ -285,23 +286,22 @@ type LogProxyCall = [log: Log, method: (typeof proxyLogMethods)[number], ...args** @experimental*/-export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {- private adaptiveRequestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] & {};+export class AdaptivePlaywrightCrawler<+ ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext,+> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ExtendedContext> {private renderingTypePredictor: NonNullable<AdaptivePlaywrightCrawlerOptions['renderingTypePredictor']>;private resultChecker: NonNullable<AdaptivePlaywrightCrawlerOptions['resultChecker']>;private shouldPropagateError: NonNullable<AdaptivePlaywrightCrawlerOptions['shouldPropagateError']>;private resultComparator: NonNullable<AdaptivePlaywrightCrawlerOptions['resultComparator']>;private preventDirectStorageAccess: boolean;+ private staticContextPipeline: ContextPipeline<CrawlingContext, ExtendedContext>;+ private browserContextPipeline: ContextPipeline<CrawlingContext, ExtendedContext>;+ private individualRequestHandlerTimeoutMillis: number;declare readonly stats: AdaptivePlaywrightCrawlerStatistics;+ private resultObjects = new WeakMap<CrawlingContext, RequestHandlerResult>();private inFlightRenderingTypeDetections = 0;- /**- * Default {@apilink Router} instance that will be used if we don't specify any {@apilink AdaptivePlaywrightCrawlerOptions.requestHandler|`requestHandler`}.- * See {@apilink Router.addHandler|`router.addHandler()`} and {@apilink Router.addDefaultHandler|`router.addDefaultHandler()`}.- */- // @ts-ignore- override readonly router: RouterHandler<AdaptivePlaywrightCrawlerContext> =- Router.create<AdaptivePlaywrightCrawlerContext>();+ private teardownHooks: (() => Promise<unknown>)[] = [];/*** The `requestHandler` never reaches `BrowserCrawler` here — it is kept as `adaptiveRequestHandler` and@@ -312,7 +312,7 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {}constructor(- options: AdaptivePlaywrightCrawlerOptions = {},+ options: AdaptivePlaywrightCrawlerOptions<ExtendedContext> = {},override readonly config = Configuration.getGlobalConfig(),) {const {@@ -324,11 +324,33 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {resultComparator,statisticsOptions,preventDirectStorageAccess = true,+ requestHandlerTimeoutSecs = 60,+ errorHandler,+ failedRequestHandler,+ preNavigationHooks,+ postNavigationHooks,+ extendContext,+ contextPipelineBuilder,...rest} = options;- super(rest, config);- this.adaptiveRequestHandler = requestHandler ?? this.router;+ super(+ {+ ...rest,+ // Pass error handlers to the "main" crawler - we only pluck them from `rest` so that they don't go to the sub crawlers+ errorHandler,+ failedRequestHandler,+ // Same for request handler+ requestHandler,+ // The builder intentionally returns null so that it crashes the crawler when it tries to use this instead of one of two the specialized context pipelines+ // (that would be a logical error in this class)+ contextPipelineBuilder: () =>+ null as unknown as ContextPipeline<CrawlingContext, AdaptivePlaywrightCrawlerContext>,+ },+ config,+ );+ this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;+this.renderingTypePredictor =renderingTypePredictor ?? new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio });this.resultChecker = resultChecker ?? (() => true);@@ -349,6 +371,75 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {);};}+ const staticCrawler = new CheerioCrawler(+ {+ ...rest,+ useSessionPool: false,+ statisticsOptions: {+ persistenceOptions: { enable: false },+ },+ preNavigationHooks: [+ async (context) => {+ for (const hook of preNavigationHooks ?? []) {+ await hook(context, undefined);+ }+ },+ ],+ postNavigationHooks: [+ async (context) => {+ for (const hook of postNavigationHooks ?? []) {+ await hook(context, undefined);+ }+ },+ ],+ },+ config,+ );++ const browserCrawler = new PlaywrightCrawler(+ {+ ...rest,+ useSessionPool: false,+ statisticsOptions: {+ persistenceOptions: { enable: false },+ },+ preNavigationHooks: [+ async (context, gotoOptions) => {+ for (const hook of preNavigationHooks ?? []) {+ await hook(context, gotoOptions);+ }+ },+ ],+ postNavigationHooks: [+ async (context, gotoOptions) => {+ for (const hook of postNavigationHooks ?? []) {+ await hook(context, gotoOptions);+ }+ },+ ],+ },+ config,+ );++ this.teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));++ this.staticContextPipeline = staticCrawler.contextPipeline+ .compose({+ action: this.adaptCheerioContext.bind(this),+ })+ .compose({+ action: async (context) =>+ extendContext ? await extendContext(context) : (context as unknown as ExtendedContext),+ });++ this.browserContextPipeline = browserCrawler.contextPipeline+ .compose({+ action: this.adaptPlaywrightContext.bind(this),+ })+ .compose({+ action: async (context) =>+ extendContext ? await extendContext(context) : (context as unknown as ExtendedContext),+ });this.stats = new AdaptivePlaywrightCrawlerStatistics({logMessage: `${this.log.getOptions().prefix} request statistics:`,@@ -359,38 +450,183 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {this.preventDirectStorageAccess = preventDirectStorageAccess;}- /**- * Returns the number of rendering type detections currently in progress.- */- get inFlightRenderingTypeDetectionCount(): number {- return this.inFlightRenderingTypeDetections;- }-protected override async _init(): Promise<void> {await this.renderingTypePredictor.initialize();return await super._init();}- protected override async _runRequestHandler(crawlingContext: PlaywrightCrawlingContext): Promise<void> {+ private async adaptCheerioContext(cheerioContext: CheerioCrawlingContext) {+ // Capture the original response to avoid infinite recursion when the getter is copied to the context+ const originalResponse = cheerioContext.response;++ const result = this.resultObjects.get(cheerioContext);+ if (result === undefined) {+ throw new Error('Logical error - `this.resultObjects` does not contain the result object');+ }++ return {+ get page(): Page {+ throw new Error('Page object was used in HTTP-only request handler');+ },+ get response(): BaseHttpResponseData {+ return {+ // TODO remove this once cheerioContext.response is just a Response+ complete: true,+ headers: originalResponse.headers,+ trailers: {},+ url: originalResponse.url!,+ statusCode: originalResponse.statusCode!,+ redirectUrls: (originalResponse as unknown as BaseHttpResponseData).redirectUrls ?? [],+ };+ },+ async querySelector(selector: string) {+ return cheerioContext.$(selector);+ },+ enqueueLinks: async (options: EnqueueLinksOptions = {}) => {+ const urls =+ options.urls ??+ extractUrlsFromCheerio(+ cheerioContext.$,+ options.selector,+ options.baseUrl ?? cheerioContext.request.loadedUrl,+ );+ return await this.enqueueLinks({ ...options, urls }, cheerioContext.request, result);+ },+ };+ }++ private async adaptPlaywrightContext(playwrightContext: PlaywrightCrawlingContext) {+ // Capture the original response to avoid infinite recursion when the getter is copied to the context+ const originalResponse = playwrightContext.response;++ const result = this.resultObjects.get(playwrightContext);+ if (result === undefined) {+ throw new Error('Logical error - `this.resultObjects` does not contain the result object');+ }++ return {+ get response(): BaseHttpResponseData {+ return {+ url: originalResponse!.url(),+ statusCode: originalResponse!.status(),+ headers: originalResponse!.headers(),+ trailers: {},+ complete: true,+ redirectUrls: [],+ };+ },+ async querySelector(selector: string, timeoutMs = 5000) {+ const locator = playwrightContext.page.locator(selector).first();+ await locator.waitFor({ timeout: timeoutMs, state: 'attached' });+ const $ = await playwrightContext.parseWithCheerio();++ return $(selector) as Cheerio<any>;+ },+ enqueueLinks: async (options: EnqueueLinksOptions = {}, timeoutMs = 5000) => {+ // TODO consider using `context.parseWithCheerio` to make this universal and avoid code duplication+ let urls: readonly string[];++ if (options.urls === undefined) {+ const selector = options.selector ?? 'a';+ const locator = playwrightContext.page.locator(selector).first();+ await locator.waitFor({ timeout: timeoutMs, state: 'attached' });+ urls =+ options.urls ??+ (await extractUrlsFromPage(+ playwrightContext.page,+ selector,+ options.baseUrl ?? playwrightContext.request.loadedUrl,+ ));+ } else {+ urls = options.urls;+ }++ return await this.enqueueLinks({ ...options, urls }, playwrightContext.request, result);+ },+ };+ }++ private async crawlOne(+ renderingType: RenderingType,+ context: CrawlingContext,+ useStateFunction: (defaultValue?: Dictionary) => Promise<Dictionary>,+ ): Promise<Result<RequestHandlerResult>> {+ const result = new RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);+ const logs: LogProxyCall[] = [];++ const deferredCleanup: (() => Promise<unknown>)[] = [];++ const resultBoundContextHelpers = {+ addRequests: result.addRequests,+ pushData: result.pushData,+ useState: this.allowStorageAccess(useStateFunction),+ getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),+ enqueueLinks: async (options: SetRequired<EnqueueLinksOptions, 'urls'>) => {+ return await this.enqueueLinks(options, context.request, result);+ },+ log: this.createLogProxy(context.log, logs),+ registerDeferredCleanup: (cleanup: () => Promise<unknown>) => deferredCleanup.push(cleanup),+ };++ const subCrawlerContext = { ...context, ...resultBoundContextHelpers };+ this.resultObjects.set(subCrawlerContext, result);++ try {+ const callAdaptiveRequestHandler = async () => {+ if (renderingType === 'static') {+ await this.staticContextPipeline.call(+ subCrawlerContext,+ async (finalContext) => await this.requestHandler(finalContext),+ );+ } else if (renderingType === 'clientOnly') {+ await this.browserContextPipeline.call(+ subCrawlerContext,+ async (finalContext) => await this.requestHandler(finalContext),+ );+ }+ };++ await addTimeoutToPromise(+ async () =>+ withCheckedStorageAccess(() => {+ if (this.preventDirectStorageAccess) {+ throw new Error(+ 'Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler',+ );+ }+ }, callAdaptiveRequestHandler),+ this.individualRequestHandlerTimeoutMillis,+ 'Request handler timed out',+ );++ return { result, ok: true, logs };+ } catch (error) {+ return { error, ok: false, logs };+ } finally {+ await Promise.all(deferredCleanup.map((cleanup) => cleanup()));+ }+ }++ protected override async runRequestHandler(crawlingContext: CrawlingContext): Promise<void> {const renderingTypePrediction = this.renderingTypePredictor.predict(crawlingContext.request);const shouldDetectRenderingType = Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;+ if (!shouldDetectRenderingType) {+ crawlingContext.log.debug(+ `Predicted rendering type ${renderingTypePrediction.renderingType} for ${crawlingContext.request.url}`,+ );+ }+if (shouldDetectRenderingType) {- this.inFlightRenderingTypeDetections++;+ this.inFlightRenderingTypeDetections += 1;}try {- if (!shouldDetectRenderingType) {- crawlingContext.log.debug(- `Predicted rendering type ${renderingTypePrediction.renderingType} for ${crawlingContext.request.url}`,- );- }-if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);this.stats.trackHttpOnlyRequestHandlerRun();- const plainHTTPRun = await this.runRequestHandlerWithPlainHTTP(crawlingContext);+ const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState);if (plainHTTPRun.ok && this.resultChecker(plainHTTPRun.result)) {crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);@@ -398,15 +634,20 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {await this.commitResult(crawlingContext, plainHTTPRun.result);return;}++ // Execution will "fall through" and try running the request handler in a browserif (!plainHTTPRun.ok) {- const error = plainHTTPRun.error as Error;+ const actualError =+ plainHTTPRun.error instanceof RequestHandlerError+ ? (plainHTTPRun.error.cause as Error)+ : (plainHTTPRun.error as Error);- if (await this.shouldPropagateError(error, crawlingContext)) {- throw error;+ if (await this.shouldPropagateError(actualError, crawlingContext)) {+ throw actualError;}crawlingContext.log.exception(- error,+ actualError,`HTTP-only request handler failed for ${crawlingContext.request.url}`,);} else {@@ -424,7 +665,30 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {// a rendering type detection if necessary. Without this measure, the HTTP request handler would run// under different conditions, which could change its behavior. Changes done to the crawler state by// the HTTP request handler will not be committed to the actual storage.- const { result: browserRun, initialStateCopy } = await this.runRequestHandlerInBrowser(crawlingContext);+ const stateTracker = {+ stateCopy: null,+ async getLiveState(defaultValue: Dictionary = {}) {+ const state = await crawlingContext.useState(defaultValue);++ if (this.stateCopy === null) {+ this.stateCopy = JSON.parse(JSON.stringify(state));+ }++ return state;+ },+ async getStateCopy(defaultValue: Dictionary = {}) {+ if (this.stateCopy === null) {+ return defaultValue;+ }+ return this.stateCopy;+ },+ };++ const browserRun = await this.crawlOne(+ 'clientOnly',+ crawlingContext,+ stateTracker.getLiveState.bind(stateTracker),+ );if (!browserRun.ok) {throw browserRun.error;@@ -434,7 +698,11 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {if (shouldDetectRenderingType) {crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);- const plainHTTPRun = await this.runRequestHandlerWithPlainHTTP(crawlingContext, initialStateCopy);+ const plainHTTPRun = await this.crawlOne(+ 'static',+ crawlingContext,+ stateTracker.getStateCopy.bind(stateTracker),+ );const detectionResult: RenderingType | undefined = (() => {if (!plainHTTPRun.ok) {@@ -463,13 +731,13 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {}} finally {if (shouldDetectRenderingType) {- this.inFlightRenderingTypeDetections--;+ this.inFlightRenderingTypeDetections -= 1;}}}protected async commitResult(- crawlingContext: PlaywrightCrawlingContext,+ crawlingContext: CrawlingContext,{ calls, keyValueStoreChanges }: RequestHandlerResult,): Promise<void> {await Promise.all([@@ -496,234 +764,6 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {);}- protected async runRequestHandlerInBrowser(- crawlingContext: PlaywrightCrawlingContext,- ): Promise<{ result: Result<RequestHandlerResult>; initialStateCopy?: Record<string, unknown> }> {- const result = new RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);- let initialStateCopy: Record<string, unknown> | undefined;-- try {- await super._runRequestHandler.call(- new Proxy(this, {- get: (target, propertyName, receiver) => {- if (propertyName === 'userProvidedRequestHandler') {- return async (playwrightContext: PlaywrightCrawlingContext) =>- withCheckedStorageAccess(- () => {- if (this.preventDirectStorageAccess) {- throw new Error(- 'Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler',- );- }- },- () =>- this.adaptiveRequestHandler({- id: crawlingContext.id,- session: crawlingContext.session,- proxyInfo: crawlingContext.proxyInfo,- request: crawlingContext.request as LoadedRequest<Request>,- response: {- url: crawlingContext.response!.url(),- statusCode: crawlingContext.response!.status(),- headers: crawlingContext.response!.headers(),- trailers: {},- complete: true,- redirectUrls: [],- },- log: crawlingContext.log,- page: crawlingContext.page,- querySelector: async (selector, timeoutMs = 5_000) => {- const locator = playwrightContext.page.locator(selector).first();- await locator.waitFor({ timeout: timeoutMs, state: 'attached' });- const $ = await playwrightContext.parseWithCheerio();-- return $(selector) as Cheerio<any>;- },- async waitForSelector(selector, timeoutMs = 5_000) {- const locator = playwrightContext.page.locator(selector).first();- await locator.waitFor({ timeout: timeoutMs, state: 'attached' });- },- async parseWithCheerio(- selector?: string,- timeoutMs = 5_000,- ): Promise<CheerioRoot> {- if (selector) {- const locator = playwrightContext.page.locator(selector).first();- await locator.waitFor({ timeout: timeoutMs, state: 'attached' });- }-- return playwrightContext.parseWithCheerio();- },- enqueueLinks: async (options = {}, timeoutMs = 5_000) => {- let urls;-- if (options.urls === undefined) {- const selector = options.selector ?? 'a';- const locator = playwrightContext.page.locator(selector).first();- await locator.waitFor({ timeout: timeoutMs, state: 'attached' });-- urls = await extractUrlsFromPage(- playwrightContext.page,- selector,- options.baseUrl ??- playwrightContext.request.loadedUrl ??- playwrightContext.request.url,- );- } else {- urls = options.urls;- }-- return await this.enqueueLinks(- { ...options, urls },- crawlingContext.request,- result,- );- },- addRequests: result.addRequests,- pushData: result.pushData,- useState: this.allowStorageAccess(async (defaultValue) => {- const state = await result.useState(defaultValue);- if (initialStateCopy === undefined) {- initialStateCopy = JSON.parse(JSON.stringify(state));- }- return state;- }),- getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),- }),- );- }- return Reflect.get(target, propertyName, receiver);- },- }),- crawlingContext,- );- return { result: { result, ok: true }, initialStateCopy };- } catch (error) {- return { result: { error, ok: false }, initialStateCopy };- }- }-- protected async runRequestHandlerWithPlainHTTP(- crawlingContext: PlaywrightCrawlingContext,- oldStateCopy?: Dictionary,- ): Promise<Result<RequestHandlerResult>> {- const result = new RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);- const logs: LogProxyCall[] = [];-- const pageGotoOptions = { timeout: this.navigationTimeoutMillis }; // Irrelevant, but required by BrowserCrawler-- try {- await withCheckedStorageAccess(- () => {- if (this.preventDirectStorageAccess) {- throw new Error(- 'Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler',- );- }- },- async () =>- addTimeoutToPromise(- async () => {- const hookContext: Parameters<AdaptiveHook>[0] = {- id: crawlingContext.id,- session: crawlingContext.session,- proxyInfo: crawlingContext.proxyInfo,- request: crawlingContext.request,- log: this.createLogProxy(crawlingContext.log, logs),- };-- await this._executeHooks(- this.preNavigationHooks,- {- ...hookContext,- get page(): Page {- throw new Error('Page object was used in HTTP-only pre-navigation hook');- },- } as PlaywrightCrawlingContext, // This is safe because `executeHooks` just passes the context to the hooks which accept the partial context- pageGotoOptions,- );-- const response = await crawlingContext.sendRequest({});-- const loadedUrl = response.url;- crawlingContext.request.loadedUrl = loadedUrl;-- if (!this.requestMatchesEnqueueStrategy(crawlingContext.request)) {- const request = crawlingContext.request;-- this.log.debug(- // eslint-disable-next-line dot-notation- `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`,- );-- request.noRetry = true;- request.state = RequestState.SKIPPED;-- await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });-- return;- }-- const $ = load(response.body);-- await this.adaptiveRequestHandler({- ...hookContext,- request: crawlingContext.request as LoadedRequest<Request>,- response,- get page(): Page {- throw new Error('Page object was used in HTTP-only request handler');- },- async querySelector(selector, _timeoutMs?: number) {- return $(selector) as Cheerio<any>;- },- async waitForSelector(selector, _timeoutMs?: number) {- if ($(selector).get().length === 0) {- throw new Error(`Selector '${selector}' not found.`);- }- },- async parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<CheerioRoot> {- if (selector && $(selector).get().length === 0) {- throw new Error(`Selector '${selector}' not found.`);- }-- return $;- },- enqueueLinks: async (- options: Parameters<RestrictedCrawlingContext['enqueueLinks']>[0] = {},- ) => {- const urls =- options.urls ??- extractUrlsFromCheerio($, options.selector, options.baseUrl ?? loadedUrl);-- return this.enqueueLinks({ ...options, urls }, crawlingContext.request, result);- },- addRequests: result.addRequests,- pushData: result.pushData,- useState: async (defaultValue) => {- // return the old state before the browser handler was executed- // when rerunning the handler via HTTP for detection- if (oldStateCopy !== undefined) {- return oldStateCopy ?? defaultValue; // fallback to the default for `null`- }-- return this.allowStorageAccess(result.useState)(defaultValue);- },- getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),- });-- await this._executeHooks(this.postNavigationHooks, crawlingContext, pageGotoOptions);- },- this.requestHandlerTimeoutInnerMillis,- 'Request handler timed out',- ),- );-- return { result, logs, ok: true };- } catch (error) {- return { error, logs, ok: false };- }- }-protected async enqueueLinks(options: SetRequired<EnqueueLinksOptions, 'urls'>,request: RestrictedCrawlingContext['request'],@@ -768,6 +808,13 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {},});}++ override async teardown() {+ await super.teardown();+ for (const hook of this.teardownHooks) {+ await hook();+ }+ }}export function createAdaptivePlaywrightRouter<diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex 62144d57e..77f0a508a 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -2,33 +2,51 @@ import type {BrowserCrawlerOptions,BrowserCrawlingContext,BrowserHook,- BrowserRequestHandler,GetUserDataFromRequest,+ RequestHandler,RouterHandler,RouterRoutes,RouteSchemas,RoutesFromSchemas,} from '@crawlee/browser';-import { BrowserCrawler, Configuration, Router } from '@crawlee/browser';-import type { BrowserPoolOptions, CommonPage, PlaywrightController, PlaywrightPlugin } from '@crawlee/browser-pool';+import { BrowserCrawler, Configuration, RequestState, Router } from '@crawlee/browser';+import type { BrowserPoolOptions, PlaywrightController, PlaywrightPlugin } from '@crawlee/browser-pool';import type { Dictionary } from '@crawlee/types';import ow from 'ow';import type { Download, LaunchOptions, Page, Response } from 'playwright';+import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js';import type { PlaywrightLaunchContext } from './playwright-launcher.js';import { PlaywrightLauncher } from './playwright-launcher.js';-import type { DirectNavigationOptions, PlaywrightContextUtils } from './utils/playwright-utils.js';-import { gotoExtended, registerUtilsToContext } from './utils/playwright-utils.js';+import type {+ BlockRequestsOptions,+ DirectNavigationOptions,+ HandleCloudflareChallengeOptions,+ InfiniteScrollOptions,+ InjectFileOptions,+ PlaywrightContextUtils,+ SaveSnapshotOptions,+} from './utils/playwright-utils.js';+import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';export interface PlaywrightCrawlingContext<UserData extends Dictionary = Dictionary>- extends BrowserCrawlingContext<PlaywrightCrawler, Page, Response, PlaywrightController, UserData>,+ extends BrowserCrawlingContext<Page, Response, PlaywrightController, UserData>,PlaywrightContextUtils {}export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext, PlaywrightGotoOptions> {}-export interface PlaywrightRequestHandler extends BrowserRequestHandler<PlaywrightCrawlingContext> {}export type PlaywrightGotoOptions = Parameters<Page['goto']>[1];-export interface PlaywrightCrawlerOptions- extends BrowserCrawlerOptions<PlaywrightCrawlingContext, { browserPlugins: [PlaywrightPlugin] }> {+export interface PlaywrightCrawlerOptions<+ ContextExtension = {},+ ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension,+> extends BrowserCrawlerOptions<+ Page,+ Response,+ PlaywrightController,+ PlaywrightCrawlingContext,+ ContextExtension,+ ExtendedContext,+ { browserPlugins: [PlaywrightPlugin] }+ > {/*** The same options as used by {@apilink launchPlaywright}.*/@@ -58,36 +76,7 @@ export interface PlaywrightCrawlerOptions* The exceptions are logged to the request using the* {@apilink Request.pushErrorMessage} function.*/- requestHandler?: PlaywrightRequestHandler;-- /**- * Function that is called to process each request.- *- * The function receives the {@apilink PlaywrightCrawlingContext} as an argument, where:- * - `request` is an instance of the {@apilink Request} object with details about the URL to open, HTTP method etc.- * - `page` is an instance of the `Playwright`- * [`Page`](https://playwright.dev/docs/api/class-page)- * - `browserController` is an instance of the- * [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),- * - `response` is an instance of the `Playwright`- * [`Response`](https://playwright.dev/docs/api/class-response),- * which is the main resource response as returned by `page.goto(request.url)`.- *- * The function must return a promise, which is then awaited by the crawler.- *- * If the function throws an exception, the crawler will try to re-crawl the- * request later, up to `option.maxRequestRetries` times.- * If all the retries fail, the crawler calls the function- * provided to the `failedRequestHandler` parameter.- * To make this work, you should **always**- * let your function throw exceptions rather than catch them.- * The exceptions are logged to the request using the- * {@apilink Request.pushErrorMessage} function.- *- * @deprecated `handlePageFunction` has been renamed to `requestHandler` and will be removed in a future version.- * @ignore- */- handlePageFunction?: PlaywrightRequestHandler;+ requestHandler?: RequestHandler<ExtendedContext>;/*** Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies@@ -189,22 +178,32 @@ export interface PlaywrightCrawlerOptions* ```* @category Crawlers*/-export class PlaywrightCrawler extends BrowserCrawler<+export class PlaywrightCrawler<+ ContextExtension = {},+ ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension,+> extends BrowserCrawler<+ Page,+ Response,+ PlaywrightController,{ browserPlugins: [PlaywrightPlugin] },LaunchOptions,- PlaywrightCrawlingContext+ PlaywrightCrawlingContext,+ ContextExtension,+ ExtendedContext> {protected static override optionsShape = {...BrowserCrawler.optionsShape,browserPoolOptions: ow.optional.object,launcher: ow.optional.object,+ ignoreIframes: ow.optional.boolean,+ ignoreShadowRoots: ow.optional.boolean,};/*** All `PlaywrightCrawler` parameters are passed via an options object.*/constructor(- private readonly options: PlaywrightCrawlerOptions = {},+ options: PlaywrightCrawlerOptions<ExtendedContext> = {},override readonly config = Configuration.getGlobalConfig(),) {ow(options, 'PlaywrightCrawlerOptions', ow.object.exactShape(PlaywrightCrawler.optionsShape));@@ -237,23 +236,16 @@ export class PlaywrightCrawler extends BrowserCrawler<browserPoolOptions.browserPlugins = [playwrightLauncher.createBrowserPlugin()];- super({ ...browserCrawlerOptions, launchContext, browserPoolOptions }, config);- }-- protected override _enhanceCrawlingContextWithPageInfo(- crawlingContext: PlaywrightCrawlingContext,- page: CommonPage,- createNewSession?: boolean,- ): void {- super._enhanceCrawlingContextWithPageInfo(crawlingContext, page, createNewSession);- const downloads: Download[] = [];- (page as Page).on('download', (download) => downloads.push(download));- crawlingContext.listDownloads = async () => downloads;- }-- protected override async _runRequestHandler(context: PlaywrightCrawlingContext) {- registerUtilsToContext(context, this.options);- await super._runRequestHandler(context);+ super(+ {+ ...(browserCrawlerOptions as PlaywrightCrawlerOptions<ExtendedContext>),+ launchContext,+ browserPoolOptions,+ contextPipelineBuilder: () =>+ this.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) }),+ },+ config,+ );}protected override async _navigationHandler(@@ -262,6 +254,64 @@ export class PlaywrightCrawler extends BrowserCrawler<) {return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);}++ private async enhanceContext(context: BrowserCrawlingContext<Page, Response, PlaywrightController, Dictionary>) {+ const waitForSelector = async (selector: string, timeoutMs = 5_000) => {+ const locator = context.page.locator(selector).first();+ await locator.waitFor({ timeout: timeoutMs, state: 'attached' });+ };++ const downloads: Download[] = [];+ context.page.on('download', (download) => downloads.push(download));++ return {+ injectFile: async (filePath: string, options?: InjectFileOptions) =>+ playwrightUtils.injectFile(context.page, filePath, options),+ injectJQuery: async () => {+ if (context.request.state === RequestState.BEFORE_NAV) {+ context.log.warning(+ 'Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.',+ );+ await playwrightUtils.injectJQuery(context.page);+ return;+ }+ await playwrightUtils.injectJQuery(context.page, { surviveNavigations: false });+ },+ blockRequests: async (options?: BlockRequestsOptions) =>+ playwrightUtils.blockRequests(context.page, options),+ waitForSelector,+ parseWithCheerio: async (selector?: string, timeoutMs = 5_000) => {+ if (selector) {+ await waitForSelector(selector, timeoutMs);+ }++ return playwrightUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);+ },+ infiniteScroll: async (options?: InfiniteScrollOptions) =>+ playwrightUtils.infiniteScroll(context.page, options),+ listDownloads: async () => downloads,+ saveSnapshot: async (options?: SaveSnapshotOptions) =>+ playwrightUtils.saveSnapshot(context.page, { ...options, config: this.config }),+ enqueueLinksByClickingElements: async (+ options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>,+ ) =>+ playwrightUtils.enqueueLinksByClickingElements({+ ...options,+ page: context.page,+ requestQueue: this.requestQueue!,+ }),+ compileScript: (scriptString: string, ctx?: Dictionary) => playwrightUtils.compileScript(scriptString, ctx),+ closeCookieModals: async () => playwrightUtils.closeCookieModals(context.page),+ handleCloudflareChallenge: async (options?: HandleCloudflareChallengeOptions) => {+ return playwrightUtils.handleCloudflareChallenge(+ context.page,+ context.request.url,+ context.session,+ options,+ );+ },+ };+ }}/**diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.tsindex 19495bb52..5987a0c37 100644--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts@@ -22,15 +22,7 @@ import { readFile } from 'node:fs/promises';import { createRequire } from 'node:module';import vm from 'node:vm';-import {- Configuration,- KeyValueStore,- type Request,- RequestState,- type Session,- SessionError,- validators,-} from '@crawlee/browser';+import { Configuration, KeyValueStore, type Request, type Session, SessionError, validators } from '@crawlee/browser';import type { BatchAddRequestsResult } from '@crawlee/types';import { type CheerioRoot, type Dictionary, expandShadowRoots, sleep } from '@crawlee/utils';import * as cheerio from 'cheerio';@@ -42,7 +34,6 @@ import log_ from '@apify/log';import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';-import type { PlaywrightCrawlerOptions, PlaywrightCrawlingContext } from '../playwright-crawler.js';import { RenderingTypePredictor } from './rendering-type-prediction.js';const log = log_.child({ prefix: 'Playwright Utils' });@@ -693,7 +684,7 @@ export async function closeCookieModals(page: Page): Promise<void> {}}-interface HandleCloudflareChallengeOptions {+export interface HandleCloudflareChallengeOptions {/** Logging defaults to the `debug` level, use this flag to log to `info` level instead. */verbose?: boolean;/** How long should we wait after the challenge is completed for the final page to load. */@@ -1101,52 +1092,6 @@ export interface PlaywrightContextUtils {listDownloads(): Promise<Download[]>;}-export function registerUtilsToContext(- context: PlaywrightCrawlingContext,- crawlerOptions: PlaywrightCrawlerOptions,-): void {- context.injectFile = async (filePath: string, options?: InjectFileOptions) =>- injectFile(context.page, filePath, options);- context.injectJQuery = async () => {- if (context.request.state === RequestState.BEFORE_NAV) {- log.warning(- 'Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.',- );- await injectJQuery(context.page);- return;- }- await injectJQuery(context.page, { surviveNavigations: false });- };- context.blockRequests = async (options?: BlockRequestsOptions) => blockRequests(context.page, options);- context.waitForSelector = async (selector: string, timeoutMs = 5_000) => {- const locator = context.page.locator(selector).first();- await locator.waitFor({ timeout: timeoutMs, state: 'attached' });- };- context.parseWithCheerio = async (selector?: string, timeoutMs = 5_000) => {- if (selector) {- await context.waitForSelector(selector, timeoutMs);- }-- return parseWithCheerio(context.page, crawlerOptions.ignoreShadowRoots, crawlerOptions.ignoreIframes);- };- context.infiniteScroll = async (options?: InfiniteScrollOptions) => infiniteScroll(context.page, options);- context.saveSnapshot = async (options?: SaveSnapshotOptions) =>- saveSnapshot(context.page, { ...options, config: context.crawler.config });- context.enqueueLinksByClickingElements = async (- options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>,- ) =>- enqueueLinksByClickingElements({- ...options,- page: context.page,- requestQueue: context.crawler.requestQueue!,- });- context.compileScript = (scriptString: string, ctx?: Dictionary) => compileScript(scriptString, ctx);- context.closeCookieModals = async () => closeCookieModals(context.page);- context.handleCloudflareChallenge = async (options?: HandleCloudflareChallengeOptions) => {- return handleCloudflareChallenge(context.page, context.request.url, context.session, options);- };-}-export { enqueueLinksByClickingElements };/** @internal */diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex 4491ca397..d47ad2a56 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -2,34 +2,51 @@ import type {BrowserCrawlerOptions,BrowserCrawlingContext,BrowserHook,- BrowserRequestHandler,GetUserDataFromRequest,RouterHandler,RouterRoutes,RouteSchemas,RoutesFromSchemas,} from '@crawlee/browser';-import { BrowserCrawler, Configuration, Router } from '@crawlee/browser';+import { BrowserCrawler, Configuration, RequestState, Router } from '@crawlee/browser';import type { BrowserPoolOptions, PuppeteerController, PuppeteerPlugin } from '@crawlee/browser-pool';import type { Dictionary } from '@crawlee/types';import ow from 'ow';// @ts-ignore This only throws when compiled against puppeteer 25+ (ESM only), we only import types, so its alllll goooooodimport type { HTTPResponse, LaunchOptions, Page } from 'puppeteer';+import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js';import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';import { PuppeteerLauncher } from './puppeteer-launcher.js';-import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer_utils.js';-import { gotoExtended, registerUtilsToContext } from './utils/puppeteer_utils.js';+import type { InterceptHandler } from './utils/puppeteer_request_interception.js';+import type {+ BlockRequestsOptions,+ DirectNavigationOptions,+ InfiniteScrollOptions,+ InjectFileOptions,+ PuppeteerContextUtils,+ SaveSnapshotOptions,+} from './utils/puppeteer_utils.js';+import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';export interface PuppeteerCrawlingContext<UserData extends Dictionary = Dictionary>- extends BrowserCrawlingContext<PuppeteerCrawler, Page, HTTPResponse, PuppeteerController, UserData>,+ extends BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController, UserData>,PuppeteerContextUtils {}export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext, PuppeteerGoToOptions> {}-export interface PuppeteerRequestHandler extends BrowserRequestHandler<PuppeteerCrawlingContext> {}export type PuppeteerGoToOptions = Parameters<Page['goto']>[1];-export interface PuppeteerCrawlerOptions- extends BrowserCrawlerOptions<PuppeteerCrawlingContext, { browserPlugins: [PuppeteerPlugin] }> {+export interface PuppeteerCrawlerOptions<+ ContextExtension = {},+ ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension,+> extends BrowserCrawlerOptions<+ Page,+ HTTPResponse,+ PuppeteerController,+ PuppeteerCrawlingContext,+ ContextExtension,+ ExtendedContext,+ { browserPlugins: [PuppeteerPlugin] }+ > {/*** Options used by {@apilink launchPuppeteer} to start new Puppeteer instances.*/@@ -135,10 +152,18 @@ export interface PuppeteerCrawlerOptions* ```* @category Crawlers*/-export class PuppeteerCrawler extends BrowserCrawler<+export class PuppeteerCrawler<+ ContextExtension = {},+ ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension,+> extends BrowserCrawler<+ Page,+ HTTPResponse,+ PuppeteerController,{ browserPlugins: [PuppeteerPlugin] },LaunchOptions,- PuppeteerCrawlingContext+ PuppeteerCrawlingContext,+ ContextExtension,+ ExtendedContext> {protected static override optionsShape = {...BrowserCrawler.optionsShape,@@ -149,7 +174,7 @@ export class PuppeteerCrawler extends BrowserCrawler<* All `PuppeteerCrawler` parameters are passed via an options object.*/constructor(- private readonly options: PuppeteerCrawlerOptions = {},+ options: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext> = {},override readonly config = Configuration.getGlobalConfig(),) {ow(options, 'PuppeteerCrawlerOptions', ow.object.exactShape(PuppeteerCrawler.optionsShape));@@ -182,12 +207,73 @@ export class PuppeteerCrawler extends BrowserCrawler<browserPoolOptions.browserPlugins = [puppeteerLauncher.createBrowserPlugin()];- super({ ...browserCrawlerOptions, launchContext, proxyConfiguration, browserPoolOptions }, config);+ super(+ {+ ...(browserCrawlerOptions as BrowserCrawlerOptions<+ Page,+ HTTPResponse,+ PuppeteerController,+ PuppeteerCrawlingContext,+ ContextExtension,+ ExtendedContext+ >),+ launchContext,+ proxyConfiguration,+ browserPoolOptions,+ contextPipelineBuilder: () =>+ this.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) }),+ },+ config,+ );}- protected override async _runRequestHandler(context: PuppeteerCrawlingContext) {- registerUtilsToContext(context, this.options);- await super._runRequestHandler(context);+ private async enhanceContext(context: BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController>) {+ const waitForSelector = async (selector: string, timeoutMs = 5_000) => {+ await context.page.waitForSelector(selector, { timeout: timeoutMs });+ };++ return {+ injectFile: async (filePath: string, options?: InjectFileOptions) =>+ puppeteerUtils.injectFile(context.page, filePath, options),+ injectJQuery: async () => {+ if (context.request.state === RequestState.BEFORE_NAV) {+ context.log.warning(+ 'Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.',+ );+ await puppeteerUtils.injectJQuery(context.page);+ return;+ }+ await puppeteerUtils.injectJQuery(context.page, { surviveNavigations: false });+ },+ waitForSelector,+ parseWithCheerio: async (selector?: string, timeoutMs = 5_000) => {+ if (selector) {+ await waitForSelector(selector, timeoutMs);+ }++ return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);+ },+ enqueueLinksByClickingElements: async (+ options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>,+ ) =>+ puppeteerUtils.enqueueLinksByClickingElements({+ page: context.page,+ requestQueue: this.requestQueue!,+ ...options,+ }),+ blockRequests: async (options?: BlockRequestsOptions) =>+ puppeteerUtils.blockRequests(context.page, options),+ compileScript: (scriptString: string, ctx?: Dictionary) => puppeteerUtils.compileScript(scriptString, ctx),+ addInterceptRequestHandler: async (handler: InterceptHandler) =>+ puppeteerUtils.addInterceptRequestHandler(context.page, handler),+ removeInterceptRequestHandler: async (handler: InterceptHandler) =>+ puppeteerUtils.removeInterceptRequestHandler(context.page, handler),+ infiniteScroll: async (options?: InfiniteScrollOptions) =>+ puppeteerUtils.infiniteScroll(context.page, options),+ saveSnapshot: async (options?: SaveSnapshotOptions) =>+ puppeteerUtils.saveSnapshot(context.page, { ...options, config: this.config }),+ closeCookieModals: async () => puppeteerUtils.closeCookieModals(context.page),+ };}protected override async _navigationHandler(diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.tsindex be627ad1d..a4205e42c 100644--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts@@ -23,7 +23,7 @@ import { createRequire } from 'node:module';import vm from 'node:vm';import type { Request } from '@crawlee/browser';-import { Configuration, KeyValueStore, RequestState, validators } from '@crawlee/browser';+import { Configuration, KeyValueStore, validators } from '@crawlee/browser';import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';import { type CheerioRoot, expandShadowRoots, sleep } from '@crawlee/utils';import * as cheerio from 'cheerio';@@ -37,7 +37,6 @@ import log_ from '@apify/log';import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';-import type { PuppeteerCrawlerOptions, PuppeteerCrawlingContext } from '../puppeteer-crawler.js';import type { InterceptHandler } from './puppeteer_request_interception.js';import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer_request_interception.js';@@ -976,32 +975,6 @@ export interface PuppeteerContextUtils {*/blockRequests(options?: BlockRequestsOptions): Promise<void>;- /**- * `blockResources()` has a high impact on performance in recent versions of Puppeteer.- * Until this resolves, please use `utils.puppeteer.blockRequests()`.- * @deprecated- */- blockResources(resourceTypes?: string[]): Promise<void>;-- /**- * *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal- * performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary.- *- * Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.- * *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.- * This issue should be resolved or atleast mitigated in future iterations of this feature.- * @param cache- * Object in which responses are stored- * @param responseUrlRules- * List of rules that are used to check if the response should be cached.- * String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).- * @deprecated- */- cacheResponses(- cache: Dictionary<Partial<ResponseForRequest>>,- responseUrlRules: (string | RegExp)[],- ): Promise<void>;-/*** Compiles a Puppeteer script into an async function that may be executed at any time* by providing it with the following object:@@ -1114,60 +1087,6 @@ export interface PuppeteerContextUtils {closeCookieModals(): Promise<void>;}-/** @internal */-export function registerUtilsToContext(- context: PuppeteerCrawlingContext,- crawlerOptions: PuppeteerCrawlerOptions,-): void {- context.injectFile = async (filePath: string, options?: InjectFileOptions) =>- injectFile(context.page, filePath, options);- context.injectJQuery = async () => {- if (context.request.state === RequestState.BEFORE_NAV) {- log.warning(- 'Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.',- );- await injectJQuery(context.page);- return;- }- await injectJQuery(context.page, { surviveNavigations: false });- };- context.waitForSelector = async (selector: string, timeoutMs = 5_000) => {- await context.page.waitForSelector(selector, { timeout: timeoutMs });- };- context.parseWithCheerio = async (selector?: string, timeoutMs = 5_000) => {- if (selector) {- await context.waitForSelector(selector, timeoutMs);- }-- return parseWithCheerio(context.page, crawlerOptions.ignoreShadowRoots, crawlerOptions.ignoreIframes);- };- context.enqueueLinksByClickingElements = async (- options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>,- ) =>- enqueueLinksByClickingElements({- page: context.page,- requestQueue: context.crawler.requestQueue!,- ...options,- });- context.blockRequests = async (options?: BlockRequestsOptions) => blockRequests(context.page, options);- context.blockResources = async (resourceTypes?: string[]) => blockResources(context.page, resourceTypes);- context.cacheResponses = async (- cache: Dictionary<Partial<ResponseForRequest>>,- responseUrlRules: (string | RegExp)[],- ) => {- return cacheResponses(context.page, cache, responseUrlRules);- };- context.compileScript = (scriptString: string, ctx?: Dictionary) => compileScript(scriptString, ctx);- context.addInterceptRequestHandler = async (handler: InterceptHandler) =>- addInterceptRequestHandler(context.page, handler);- context.removeInterceptRequestHandler = async (handler: InterceptHandler) =>- removeInterceptRequestHandler(context.page, handler);- context.infiniteScroll = async (options?: InfiniteScrollOptions) => infiniteScroll(context.page, options);- context.saveSnapshot = async (options?: SaveSnapshotOptions) =>- saveSnapshot(context.page, { ...options, config: context.crawler.config });- context.closeCookieModals = async () => closeCookieModals(context.page);-}-export { enqueueLinksByClickingElements, addInterceptRequestHandler, removeInterceptRequestHandler };/** @internal */@@ -1176,8 +1095,6 @@ export const puppeteerUtils = {injectJQuery,enqueueLinksByClickingElements,blockRequests,- blockResources,- cacheResponses,compileScript,gotoExtended,addInterceptRequestHandler,diff --git a/test/core/crawlers/basic_browser_crawler.ts b/test/core/crawlers/basic_browser_crawler.tsindex 5ce577a1a..83c5f80a3 100644--- a/test/core/crawlers/basic_browser_crawler.ts+++ b/test/core/crawlers/basic_browser_crawler.ts@@ -1,16 +1,31 @@-import type { PuppeteerPlugin } from '@crawlee/browser-pool';-import type { PuppeteerCrawlerOptions, PuppeteerCrawlingContext, PuppeteerGoToOptions } from '@crawlee/puppeteer';+import type { PuppeteerController, PuppeteerPlugin } from '@crawlee/browser-pool';+import type {+ BrowserCrawlerOptions,+ BrowserCrawlingContext,+ PuppeteerCrawlingContext,+ PuppeteerGoToOptions,+} from '@crawlee/puppeteer';import { BrowserCrawler } from '@crawlee/puppeteer';// @ts-ignore This only throws when compiled against puppeteer 25+ (ESM only), we only import types, so its alllll gooooood-import type { HTTPResponse, LaunchOptions } from 'puppeteer';+import type { HTTPResponse, LaunchOptions, Page } from 'puppeteer';++export type TestCrawlingContext = BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController>;export class BrowserCrawlerTest extends BrowserCrawler<+ Page,+ HTTPResponse,+ PuppeteerController,{ browserPlugins: [PuppeteerPlugin] },LaunchOptions,- PuppeteerCrawlingContext+ TestCrawlingContext> {- constructor(options: Partial<PuppeteerCrawlerOptions> = {}) {- super(options as any);+ constructor(+ options: Partial<BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerController, TestCrawlingContext>> = {},+ ) {+ super({+ ...options,+ contextPipelineBuilder: () => this.buildContextPipeline(),+ });}protected async _navigationHandler(diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 9c30342b0..898868074 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -30,6 +30,7 @@ import { RequestState, SessionPool, Statistics } from '@crawlee/core';import type { Dictionary } from '@crawlee/utils';import { RobotsTxtFile, sleep } from '@crawlee/utils';import express from 'express';+import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js';import type { SetRequired } from 'type-fest';import type { Mock } from 'vitest';import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'vitest';@@ -418,9 +419,9 @@ describe('BasicCrawler', () => {const processed: { url: string }[] = [];const requestList = await RequestList.open(null, sources);- const requestHandler: RequestHandler = async ({ request, crawler }) => {+ const requestHandler: RequestHandler = async ({ request, useState }) => {await sleep(10);- const state = await crawler.useState({ processed });+ const state = await useState({ processed });state.processed.push({ url: request.url });};@@ -1214,9 +1215,9 @@ describe('BasicCrawler', () => {for (const args of warningSpy.mock.calls) {expect(args.length).toBe(2);expect(typeof args[0]).toBe('string');- expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);- expect(/requestHandler timed out after/.test(args[0])).toBe(true);- expect(/at Timeout\._onTimeout/.test(args[0])).toBe(false);+ expect(args[0]).toMatch(/Reclaiming failed request back to the list or queue/);+ expect(args[0]).toMatch(/requestHandler timed out after/);+ expect(args[0]).not.toMatch(/at Timeout\._onTimeout/);expect(args[1]).toBeDefined();}@@ -1224,9 +1225,9 @@ describe('BasicCrawler', () => {for (const args of errorSpy.mock.calls) {expect(args.length).toBe(2);expect(typeof args[0]).toBe('string');- expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);- expect(/requestHandler timed out after/.test(args[0])).toBe(true);- expect(/at Timeout\._onTimeout/.test(args[0])).toBe(false);+ expect(args[0]).toMatch(/Request failed and reached maximum retries/);+ expect(args[0]).toMatch(/requestHandler timed out after/);+ expect(args[0]).not.toMatch(/at Timeout\._onTimeout/);expect(args[1]).toBeDefined();}});@@ -1252,8 +1253,8 @@ describe('BasicCrawler', () => {for (const args of warningSpy.mock.calls) {expect(args.length).toBe(2);expect(typeof args[0]).toBe('string');- expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);- expect(/Other non-timeout error/.test(args[0])).toBe(true);+ expect(args[0]).toMatch(/Reclaiming failed request back to the list or queue/);+ expect(args[0]).toMatch(/Other non-timeout error/);expect(args[0].split('\n').length).toBeLessThanOrEqual(2);expect(args[1]).toBeDefined();}@@ -1262,9 +1263,9 @@ describe('BasicCrawler', () => {for (const args of errorSpy.mock.calls) {expect(args.length).toBe(2);expect(typeof args[0]).toBe('string');- expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);- expect(/Other non-timeout error/.test(args[0])).toBe(true);- expect(/at _?BasicCrawler\.requestHandler/.test(args[0])).toBe(true);+ expect(args[0]).toMatch(/Request failed and reached maximum retries/);+ expect(args[0]).toMatch(/Other non-timeout error/);+ expect(args[0]).toMatch(/at _?BasicCrawler\.requestHandler/);expect(args[1]).toBeDefined();}});@@ -1291,9 +1292,9 @@ describe('BasicCrawler', () => {for (const args of warningSpy.mock.calls) {expect(args.length).toBe(2);expect(typeof args[0]).toBe('string');- expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);- expect(/requestHandler timed out after/.test(args[0])).toBe(true);- expect(/at Timeout\._onTimeout/.test(args[0])).toBe(true);+ expect(args[0]).toMatch(/Reclaiming failed request back to the list or queue/);+ expect(args[0]).toMatch(/requestHandler timed out after/);+ expect(args[0]).toMatch(/at Timeout\._onTimeout/);expect(args[1]).toBeDefined();}@@ -1301,9 +1302,9 @@ describe('BasicCrawler', () => {for (const args of errorSpy.mock.calls) {expect(args.length).toBe(2);expect(typeof args[0]).toBe('string');- expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);- expect(/requestHandler timed out after/.test(args[0])).toBe(true);- expect(/at Timeout\._onTimeout/.test(args[0])).toBe(true);+ expect(args[0]).toMatch(/Request failed and reached maximum retries/);+ expect(args[0]).toMatch(/requestHandler timed out after/);+ expect(args[0]).toMatch(/at Timeout\._onTimeout/);expect(args[1]).toBeDefined();}@@ -1334,9 +1335,9 @@ describe('BasicCrawler', () => {for (const args of warningSpy.mock.calls) {expect(args.length).toBe(2);expect(typeof args[0]).toBe('string');- expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);- expect(/Other non-timeout error/.test(args[0])).toBe(true);- expect(/at _?BasicCrawler\.requestHandler/.test(args[0])).toBe(true);+ expect(args[0]).toMatch(/Reclaiming failed request back to the list or queue/);+ expect(args[0]).toMatch(/Other non-timeout error/);+ expect(args[0]).toMatch(/at _?BasicCrawler\.requestHandler/);expect(args[1]).toBeDefined();}@@ -1344,9 +1345,9 @@ describe('BasicCrawler', () => {for (const args of errorSpy.mock.calls) {expect(args.length).toBe(2);expect(typeof args[0]).toBe('string');- expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);- expect(/Other non-timeout error/.test(args[0])).toBe(true);- expect(/at _?BasicCrawler\.requestHandler/.test(args[0])).toBe(true);+ expect(args[0]).toMatch(/Request failed and reached maximum retries/);+ expect(args[0]).toMatch(/Other non-timeout error/);+ expect(args[0]).toMatch(/at _?BasicCrawler\.requestHandler/);expect(args[1]).toBeDefined();}@@ -1475,50 +1476,20 @@ describe('BasicCrawler', () => {});});- describe('CrawlingContext', () => {- test('should be kept and later deleted', async () => {- const urls = [- 'https://example.com/0',- 'https://example.com/1',- 'https://example.com/2',- 'https://example.com/3',- ];- const requestList = await RequestList.open(null, urls);- let counter = 0;- let finish: (value?: unknown) => void;- const allFinishedPromise = new Promise((resolve) => {- finish = resolve;- });- const mainContexts: CrawlingContext[] = [];- const otherContexts: CrawlingContext[][] = [];- const crawler = new BasicCrawler({- requestList,- minConcurrency: 4,- async requestHandler(crawlingContext) {- // @ts-expect-error Accessing private prop- mainContexts[counter] = crawler.crawlingContexts.get(crawlingContext.id);- // @ts-expect-error Accessing private prop- otherContexts[counter] = Array.from(crawler.crawlingContexts).map(([, v]) => v);- counter++;- if (counter === 4) finish();- await allFinishedPromise;- },- });- await crawler.run();+ test('extendContext', async () => {+ const url = 'https://example.com';+ const requestHandlerImplementation = vi.fn();- expect(counter).toBe(4);- expect(mainContexts).toHaveLength(4);- expect(otherContexts).toHaveLength(4);- // @ts-expect-error Accessing private prop- expect(crawler.crawlingContexts.size).toBe(0);- mainContexts.forEach((ctx, idx) => {- expect(typeof ctx.id).toBe('string');- expect(otherContexts[idx]).toContain(ctx);- });- otherContexts.forEach((list, idx) => {- expect(list).toHaveLength(idx + 1);- });+ const crawler = new BasicCrawler({+ extendContext: () => ({ hello: 'world' }),+ requestHandler: async ({ hello }) => {+ requestHandlerImplementation({ hello });+ },});++ await crawler.run([url]);+ expect(requestHandlerImplementation).toHaveBeenCalledOnce();+ expect(requestHandlerImplementation.mock.calls[0][0]).toMatchObject({ hello: 'world' });});describe('sendRequest', () => {diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.tsindex 76145c853..0e6057e36 100644--- a/test/core/crawlers/browser_crawler.test.ts+++ b/test/core/crawlers/browser_crawler.test.ts@@ -1,18 +1,11 @@import type { Server } from 'node:http';+import { ENV_VARS } from '@apify/consts';import log from '@apify/log';-import { BROWSER_POOL_EVENTS, BrowserPool, OperatingSystemsName, PuppeteerPlugin } from '@crawlee/browser-pool';+import { BROWSER_POOL_EVENTS, OperatingSystemsName, PuppeteerPlugin } from '@crawlee/browser-pool';import { BLOCKED_STATUS_CODES } from '@crawlee/core';-import type { PuppeteerCrawlingContext, PuppeteerGoToOptions, PuppeteerRequestHandler } from '@crawlee/puppeteer';-import {- AutoscaledPool,- EnqueueStrategy,- ProxyConfiguration,- Request,- RequestList,- RequestState,- Session,-} from '@crawlee/puppeteer';+import type { PuppeteerGoToOptions } from '@crawlee/puppeteer';+import { EnqueueStrategy, ProxyConfiguration, Request, RequestList, RequestState, Session } from '@crawlee/puppeteer';import { sleep } from '@crawlee/utils';// @ts-ignore This only throws when compiled against puppeteer 25+ (ESM only), we only import types, so its alllll goooooodimport type { HTTPResponse } from 'puppeteer';@@ -21,6 +14,7 @@ import puppeteer from 'puppeteer';import { runExampleComServer } from 'test/shared/_helper.js';import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js';+import type { TestCrawlingContext } from './basic_browser_crawler.js';import { BrowserCrawlerTest } from './basic_browser_crawler.js';describe('BrowserCrawler', () => {@@ -65,7 +59,7 @@ describe('BrowserCrawler', () => {const processed: Request[] = [];const failed: Request[] = [];const requestList = await RequestList.open(null, sources);- const requestHandler: PuppeteerRequestHandler = async ({ page, request, response }) => {+ const requestHandler = async ({ page, request, response }: TestCrawlingContext) => {await page.waitForSelector('title');expect(response!.status()).toBe(200);@@ -149,7 +143,7 @@ describe('BrowserCrawler', () => {let sessionGoto!: Session;const browserCrawler = new (class extends BrowserCrawlerTest {protected override async _navigationHandler(- ctx: PuppeteerCrawlingContext,+ ctx: TestCrawlingContext,): Promise<HTTPResponse | null | undefined> {sessionGoto = ctx.session!;const originalMarkBad = sessionGoto.markBad.bind(sessionGoto);@@ -185,17 +179,12 @@ describe('BrowserCrawler', () => {const requestList = await RequestList.open({sources: [{ url: 'http://example.com/?q=1' }],});- let isEvaluated = false;- const browserCrawler = new (class extends BrowserCrawlerTest {- protected override async _navigationHandler(- ctx: PuppeteerCrawlingContext,- gotoOptions: PuppeteerGoToOptions,- ): Promise<HTTPResponse | null | undefined> {- isEvaluated = ctx.hookFinished as boolean;- return ctx.page.goto(ctx.request.url, gotoOptions);- }- })({+ const hook = vi.fn(async () => {+ await sleep(10);+ });++ const browserCrawler = new BrowserCrawlerTest({browserPoolOptions: {browserPlugins: [puppeteerPlugin],},@@ -203,17 +192,12 @@ describe('BrowserCrawler', () => {useSessionPool: true,requestHandler: async () => {},maxRequestRetries: 0,- preNavigationHooks: [- async (crawlingContext) => {- await sleep(10);- crawlingContext.hookFinished = true;- },- ],+ preNavigationHooks: [hook],});await browserCrawler.run();- expect(isEvaluated).toBeTruthy();+ expect(hook).toHaveBeenCalled();} finally {await localStorageEmulator.destroy();}@@ -228,7 +212,10 @@ describe('BrowserCrawler', () => {const requestList = await RequestList.open({sources: [{ url: `${serverAddress}/?q=1` }],});- let isEvaluated = false;++ const hook = vi.fn(async () => {+ await sleep(10);+ });const browserCrawler = new BrowserCrawlerTest({browserPoolOptions: {@@ -236,21 +223,14 @@ describe('BrowserCrawler', () => {},requestList,useSessionPool: true,- requestHandler: async ({ hookFinished }) => {- isEvaluated = hookFinished as boolean;- },+ requestHandler: async () => {},maxRequestRetries: 0,- postNavigationHooks: [- async (crawlingContext) => {- await sleep(10);- crawlingContext.hookFinished = true;- },- ],+ postNavigationHooks: [hook],});await browserCrawler.run();- expect(isEvaluated).toBeTruthy();+ expect(hook).toHaveBeenCalled();} finally {await localStorageEmulator.destroy();}@@ -278,7 +258,7 @@ describe('BrowserCrawler', () => {},maxRequestRetries: 1,errorHandler: async (ctx, error) => {- result.push(await ctx.page.evaluate(() => window.location.origin));+ result.push(await ctx.page!.evaluate(() => window.location.origin));},});@@ -391,7 +371,7 @@ describe('BrowserCrawler', () => {let optionsGoto: PuppeteerGoToOptions;const browserCrawler = new (class extends BrowserCrawlerTest {protected override async _navigationHandler(- ctx: PuppeteerCrawlingContext,+ ctx: TestCrawlingContext,gotoOptions: PuppeteerGoToOptions,): Promise<HTTPResponse | null | undefined> {optionsGoto = gotoOptions;@@ -815,9 +795,9 @@ describe('BrowserCrawler', () => {let called = false;const browserCrawler = new (class extends BrowserCrawlerTest {protected override async _navigationHandler(- ctx: PuppeteerCrawlingContext,+ ctx: TestCrawlingContext,): Promise<HTTPResponse | null | undefined> {- ctx.crawler.browserPool.on(BROWSER_POOL_EVENTS.BROWSER_RETIRED, () => {+ browserCrawler.browserPool.on(BROWSER_POOL_EVENTS.BROWSER_RETIRED, () => {resolve();called = true;});@@ -1073,7 +1053,7 @@ describe('BrowserCrawler', () => {const browserCrawler = new (class extends BrowserCrawlerTest {protected override async _navigationHandler(- ctx: PuppeteerCrawlingContext,+ ctx: TestCrawlingContext,): Promise<HTTPResponse | null | undefined> {const { session } = ctx;const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id);@@ -1129,7 +1109,7 @@ describe('BrowserCrawler', () => {let numberOfRotations = -requestList!.length();const browserCrawler = new (class extends BrowserCrawlerTest {protected override async _navigationHandler(- ctx: PuppeteerCrawlingContext,+ ctx: TestCrawlingContext,): Promise<HTTPResponse | null | undefined> {const { session } = ctx;const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id);@@ -1184,7 +1164,7 @@ describe('BrowserCrawler', () => {const crawler = new (class extends BrowserCrawlerTest {protected override async _navigationHandler(- ctx: PuppeteerCrawlingContext,+ ctx: TestCrawlingContext,): Promise<HTTPResponse | null | undefined> {const { session } = ctx;const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id);@@ -1236,36 +1216,30 @@ describe('BrowserCrawler', () => {const sources = ['http://example.com/'];const requestList = await RequestList.open(null, sources.slice());- let prepareCrawlingContext: PuppeteerCrawlingContext;+ let prepareCrawlingContext: TestCrawlingContext;- const gotoFunction = async (crawlingContext: PuppeteerCrawlingContext) => {+ const gotoFunction = async (crawlingContext: TestCrawlingContext) => {prepareCrawlingContext = crawlingContext;expect(crawlingContext.request).toBeInstanceOf(Request);- expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);expect(crawlingContext.session).toBeInstanceOf(Session);expect(typeof crawlingContext.page).toBe('object');};- const requestHandler = async (crawlingContext: PuppeteerCrawlingContext) => {+ const requestHandler = async (crawlingContext: TestCrawlingContext) => {expect(crawlingContext === prepareCrawlingContext).toEqual(true);expect(crawlingContext.request).toBeInstanceOf(Request);- expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);expect(crawlingContext.session).toBeInstanceOf(Session);expect(typeof crawlingContext.page).toBe('object');- expect(crawlingContext.crawler).toBeInstanceOf(BrowserCrawlerTest);expect(Object.hasOwn(crawlingContext, 'response')).toBe(true);throw new Error('some error');};- const failedRequestHandler = async (crawlingContext: PuppeteerCrawlingContext, error: Error) => {+ const failedRequestHandler = async (crawlingContext: Partial<TestCrawlingContext>, error: Error) => {expect(crawlingContext).toBe(prepareCrawlingContext);expect(crawlingContext.request).toBeInstanceOf(Request);- expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);expect(crawlingContext.session).toBeInstanceOf(Session);expect(typeof crawlingContext.page).toBe('object');- expect(crawlingContext.crawler).toBeInstanceOf(BrowserCrawlerTest);- expect(crawlingContext.crawler.browserPool).toBeInstanceOf(BrowserPool);expect(Object.hasOwn(crawlingContext, 'response')).toBe(true);expect(error).toBeInstanceOf(Error);diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex 16dabca49..721915a51 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -2,19 +2,15 @@ import type { IncomingHttpHeaders, Server } from 'node:http';import { Readable } from 'node:stream';import type {- Cheerio,- CheerioAPI,+ BasicCrawlingContext,CheerioCrawlingContext,CheerioRequestHandler,- CheerioRoot,- Element,+ CrawlingContext,ProxyInfo,Source,} from '@crawlee/cheerio';import {- AutoscaledPool,CheerioCrawler,- CrawlerExtension,createCheerioRouter,EnqueueStrategy,mergeCookies,@@ -23,6 +19,7 @@ import {RequestList,Session,} from '@crawlee/cheerio';+import { ImpitHttpClient } from '@crawlee/impit-client';import type { Dictionary } from '@crawlee/utils';import { sleep } from '@crawlee/utils';import type { OptionsInit } from 'got-scraping';@@ -65,6 +62,10 @@ beforeAll(async () => {serverAddress += port;});+afterEach(() => {+ vi.useRealTimers();+});+afterAll(() => {server.close();});@@ -341,9 +342,9 @@ describe('CheerioCrawler', () => {test('after requestHandlerTimeoutSecs', async () => {const failed: Request[] = [];const requestList = await getRequestListForMirror();- const requestHandler = async () => {+ const requestHandler = vi.fn(async () => {await sleep(2000);- };+ });const cheerioCrawler = new CheerioCrawler({requestList,@@ -357,18 +358,20 @@ describe('CheerioCrawler', () => {},});- // Override low value to prevent seeing timeouts from BasicCrawler- // @ts-expect-error Overriding private property- cheerioCrawler.handleRequestTimeoutMillis = 10000;-await cheerioCrawler.run();+ expect(requestHandler).toHaveBeenCalledTimes(8);expect(failed).toHaveLength(4);failed.forEach((request) => {- expect(request.errorMessages).toHaveLength(2);- expect(request.errorMessages[0]).toMatch('requestHandler timed out');- expect(request.errorMessages[1]).toMatch('requestHandler timed out');+ expect(request).toEqual(+ expect.objectContaining({+ errorMessages: [+ expect.stringContaining('requestHandler timed out'),+ expect.stringContaining('requestHandler timed out'),+ ],+ }),+ );});});});@@ -607,7 +610,7 @@ describe('CheerioCrawler', () => {const url = `${serverAddress}/special/json-type`;await runCrawler(url);expect(handlePageInvocationParams.json).toBeInstanceOf(Object);- expect(handlePageInvocationParams.body).toEqual(Buffer.from(JSON.stringify(responseSamples.json)));+ expect(handlePageInvocationParams.body).toEqual(JSON.stringify(responseSamples.json));expect(handlePageInvocationParams.contentType.type).toBe('application/json');expect(handleFailedInvoked).toBe(false);});@@ -622,8 +625,8 @@ describe('CheerioCrawler', () => {test('when response is image/png', async () => {const url = `${serverAddress}/special/image-type`;await runCrawler(url);- expect(handlePageInvocationParams.body).toBeInstanceOf(Buffer);- expect(handlePageInvocationParams.body).toEqual(responseSamples.image);+ expect(typeof handlePageInvocationParams.body).toBe('string');+ expect(handlePageInvocationParams.body).toEqual(responseSamples.image.toString());expect(handlePageInvocationParams.contentType.type).toBe('image/png');});});@@ -805,20 +808,24 @@ describe('CheerioCrawler', () => {*/let numberOfRotations = -1;const failedRequestHandler = vitest.fn();+ const impit = new ImpitHttpClient();const crawler = new CheerioCrawler({proxyConfiguration,maxSessionRotations: 5,requestHandler: async () => {},failedRequestHandler,- });-- vitest.spyOn(crawler, '_requestAsBrowser' as any).mockImplementation(async ({ proxyUrl }: any) => {- if (proxyUrl.includes('localhost')) {- numberOfRotations++;- throw new Error('Proxy responded with 400 - Bad request');- }-- return null;+ httpClient: {+ sendRequest: async () => {+ throw new Error("Don't");+ },+ stream: async (request, onRedirect) => {+ if (request.proxyUrl!.includes('localhost')) {+ numberOfRotations++;+ throw new Error('Proxy responded with 400 - Bad request');+ }+ return await impit.stream(request);+ },+ },});await crawler.run([serverAddress]);@@ -832,26 +839,30 @@ describe('CheerioCrawler', () => {const proxyError ='Proxy responded with 400 - Bad request. Also, this error message contains some useful payload.';+ const impit = new ImpitHttpClient();+const crawler = new CheerioCrawler({proxyConfiguration,maxSessionRotations: 1,requestHandler: async () => {},- });-- vitest.spyOn(crawler, '_requestAsBrowser' as any).mockImplementation(async ({ proxyUrl }: any) => {- if (proxyUrl.includes('localhost')) {- throw new Error(proxyError);- }-- return null;+ httpClient: {+ sendRequest: async () => {+ throw new Error("Don't");+ },+ stream: async (request, onRedirect) => {+ if (request.proxyUrl!.includes('localhost')) {+ throw new Error(proxyError);+ }+ return await impit.stream(request);+ },+ },});const spy = vitest.spyOn((crawler as any).log, 'warning' as any).mockImplementation(() => {});await crawler.run([serverAddress]);- expect(spy).toBeCalled();- expect(spy.mock.calls[0][0]).toEqual(expect.stringContaining(proxyError));+ expect(spy).toHaveBeenCalledWith(expect.stringContaining(proxyError), expect.any(Object));});});@@ -1098,6 +1109,8 @@ describe('CheerioCrawler', () => {test('should work with `context.request.headers` being undefined', async () => {const requests: Request[] = [];const responses: unknown[] = [];+ const errorHandler = vi.fn(async () => {});+const crawler = new CheerioCrawler({requestList: await RequestList.open(null, [{@@ -1109,6 +1122,7 @@ describe('CheerioCrawler', () => {responses.push(json);requests.push(request);},+ errorHandler,preNavigationHooks: [({ request }) => {request.headers!.Cookie = 'foo=override; coo=kie';@@ -1117,6 +1131,9 @@ describe('CheerioCrawler', () => {});await crawler.run();++ expect(errorHandler).not.toHaveBeenCalled();+expect(requests).toHaveLength(1);expect(requests[0].retryCount).toBe(0);expect(responses).toHaveLength(1);@@ -1173,34 +1190,37 @@ describe('CheerioCrawler', () => {test('should use sessionId in proxyUrl when the session pool is enabled', async () => {const sourcesNew = [{ url: 'http://example.com/?q=1' }];const requestListNew = await RequestList.open({ sources: sourcesNew });- let usedSession: Session;const proxyConfiguration = new ProxyConfiguration({ proxyUrls: ['http://localhost:8080'] });const newUrlSpy = vitest.spyOn(proxyConfiguration, 'newUrl');++ const requestHandler = vi.fn(async () => {});+const cheerioCrawler = new CheerioCrawler({requestList: requestListNew,maxRequestRetries: 0,maxSessionRotations: 0,- requestHandler: () => {},+ requestHandler,failedRequestHandler: () => {},useSessionPool: true,proxyConfiguration,+ preNavigationHooks: [+ async (context) => {+ context.proxyInfo = undefined;+ },+ ],});- // @ts-expect-error Accessing private method- const oldHandleRequestF = cheerioCrawler._runRequestHandler;- // @ts-expect-error Overriding private method- cheerioCrawler._runRequestHandler = async (opts) => {- usedSession = opts.session!;- return oldHandleRequestF.call(cheerioCrawler, opts);- };-try {await cheerioCrawler.run();} catch (e) {// localhost proxy causes proxy errors, session rotations and finally throws, but we don't care}+ expect(requestHandler).toHaveBeenCalledOnce();++ const usedSession: Session = ((requestHandler.mock.calls?.[0] as any)[0] as CrawlingContext).session!;+expect(newUrlSpy).toBeCalledWith(usedSession!.id,expect.objectContaining({ request: expect.any(Request) }),@@ -1222,19 +1242,17 @@ describe('CheerioCrawler', () => {});test('uses correct crawling context', async () => {- let prepareCrawlingContext: CheerioCrawlingContext;+ let prepareCrawlingContext: unknown;- const prepareRequestFunction = (crawlingContext: CheerioCrawlingContext) => {+ const preNavigationHook = (crawlingContext: BasicCrawlingContext) => {prepareCrawlingContext = crawlingContext;expect(crawlingContext.request).toBeInstanceOf(Request);- expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);expect(crawlingContext.session).toBeInstanceOf(Session);};const requestHandler = (crawlingContext: CheerioCrawlingContext) => {expect(crawlingContext === prepareCrawlingContext).toEqual(true);expect(crawlingContext.request).toBeInstanceOf(Request);- expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);expect(crawlingContext.session).toBeInstanceOf(Session);expect(typeof crawlingContext.$).toBe('function');expect(typeof crawlingContext.response).toBe('object');@@ -1243,10 +1261,9 @@ describe('CheerioCrawler', () => {throw new Error('some error');};- const failedRequestHandler = (crawlingContext: CheerioCrawlingContext, error: Error) => {+ const failedRequestHandler = (crawlingContext: Partial<CheerioCrawlingContext>, error: Error) => {expect(crawlingContext === prepareCrawlingContext).toEqual(true);expect(crawlingContext.request).toBeInstanceOf(Request);- expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);expect(crawlingContext.session).toBeInstanceOf(Session);expect(typeof crawlingContext.$).toBe('function');expect(typeof crawlingContext.response).toBe('object');@@ -1261,99 +1278,12 @@ describe('CheerioCrawler', () => {maxRequestRetries: 0,maxConcurrency: 1,useSessionPool: true,- preNavigationHooks: [prepareRequestFunction],+ preNavigationHooks: [preNavigationHook],requestHandler,failedRequestHandler,});await cheerioCrawler.run();});-- test('should have correct types in crawling context', async () => {- const requestHandler = (crawlingContext: CheerioCrawlingContext) => {- // Checking that types are correct- const _cheerioRootType: CheerioRoot = crawlingContext.$;- const _apiType: CheerioAPI = crawlingContext.$;- const _cheerioElementType: Cheerio<Element> = crawlingContext.$('div');- };-- const cheerioCrawler = new CheerioCrawler({- requestList,- maxRequestRetries: 0,- maxConcurrency: 1,- requestHandler,- });- await cheerioCrawler.run();- });- });-- describe('use', () => {- const sources = ['http://example.com/'];- let requestList: RequestList;-- class DummyExtension extends CrawlerExtension {- constructor(readonly options: Dictionary) {- super();- }-- override getCrawlerOptions() {- return this.options;- }- }-- beforeEach(async () => {- requestList = await RequestList.open(null, sources.slice());- });-- test('should throw if "CrawlerExtension" class is not used', () => {- const cheerioCrawler = new CheerioCrawler({- requestList,- maxRequestRetries: 0,- requestHandler: () => {},- failedRequestHandler: () => {},- });- expect(- // @ts-expect-error Validating JS side checks- () => cheerioCrawler.use({}),- ).toThrow('Expected object `{}` to be of type `CrawlerExtension`');- });-- test('Should throw if "CrawlerExtension" is trying to override non existing property', () => {- const extension = new DummyExtension({- doesNotExist: true,- });- const cheerioCrawler = new CheerioCrawler({- requestList,- maxRequestRetries: 0,- requestHandler: () => {},- failedRequestHandler: () => {},- });- expect(() => cheerioCrawler.use(extension)).toThrow(- 'DummyExtension tries to set property "doesNotExist" that is not configurable on CheerioCrawler instance.',- );- });-- test('should override crawler properties', () => {- const extension = new DummyExtension({- useSessionPool: true,- requestHandler: undefined,- });- const cheerioCrawler = new CheerioCrawler({- requestList,- useSessionPool: false,- maxRequestRetries: 0,- requestHandler: () => {},- failedRequestHandler: () => {},- });- // @ts-expect-error Accessing private prop- expect(cheerioCrawler.useSessionPool).toEqual(false);- cheerioCrawler.use(extension);- // @ts-expect-error Accessing private prop- expect(cheerioCrawler.useSessionPool).toEqual(true);- // @ts-expect-error Accessing private prop- expect(cheerioCrawler.requestHandler).toBeUndefined();- // @ts-expect-error Accessing private prop- expect(cheerioCrawler.requestHandler).toBeUndefined();- });});test('should work with delete requests', async () => {diff --git a/test/core/crawlers/context_pipeline.test.ts b/test/core/crawlers/context_pipeline.test.tsnew file mode 100644index 000000000..a02d2416d--- /dev/null+++ b/test/core/crawlers/context_pipeline.test.ts@@ -0,0 +1,167 @@+import {+ ContextPipeline,+ ContextPipelineCleanupError,+ ContextPipelineInitializationError,+ ContextPipelineInterruptedError,+ RequestHandlerError,+} from '@crawlee/core';+import { describe, expect, it, vi } from 'vitest';++describe('ContextPipeline', () => {+ it('should call middlewares in a sequence', async () => {+ const pipeline = ContextPipeline.create()+ .compose({+ action: async () => ({ a: 2, b: 1, c: [1] }),+ })+ .compose({+ action: async (context) => ({ a: context.a * 2, c: [...context.c, 2] }),+ });++ const consumer = vi.fn();+ await pipeline.call({}, consumer);++ expect(consumer).toHaveBeenCalledWith({ a: 4, b: 1, c: [1, 2] });+ });++ it('should call cleanup routines', async () => {+ const pipeline = ContextPipeline.create()+ .compose({+ action: async () => ({ c: [] as number[] }),+ cleanup: async (context) => {+ context.c.push(1);+ },+ })+ .compose({+ action: async () => ({}),+ cleanup: async (context) => {+ context.c.push(2);+ },+ });++ const consumer = vi.fn();+ await pipeline.call({}, consumer);++ expect(consumer).toHaveBeenCalledWith({ c: [2, 1] });+ });++ it('should allow interrupting the pipeline in middlewares', async () => {+ const context = { a: 3 };++ const firstAction = vi.fn().mockResolvedValue({});+ const firstCleanup = vi.fn();+ const secondAction = vi.fn().mockRejectedValue(new ContextPipelineInterruptedError());+ const secondCleanup = vi.fn();+ const thirdAction = vi.fn().mockResolvedValue({});+ const thirdCleanup = vi.fn();++ const pipeline = ContextPipeline.create()+ .compose({ action: firstAction, cleanup: firstCleanup })+ .compose({+ action: secondAction,+ cleanup: secondCleanup,+ })+ .compose({ action: thirdAction, cleanup: thirdCleanup });++ const consumer = vi.fn();++ await expect(pipeline.call(context, consumer)).rejects.toThrow(ContextPipelineInterruptedError);++ expect(firstAction).toHaveBeenCalled();+ expect(firstCleanup).toHaveBeenCalled();+ expect(secondAction).toHaveBeenCalled();+ expect(secondCleanup).not.toHaveBeenCalled();+ expect(thirdAction).not.toHaveBeenCalled();+ expect(thirdCleanup).not.toHaveBeenCalled();+ expect(consumer).not.toHaveBeenCalled();+ });++ it('should wrap pipeline initialization errors', async () => {+ const initializationError = new Error('Pipeline initialization failed');+ const context = { a: 3 };+ const secondMiddleware = vi.fn();++ const pipeline = ContextPipeline.create()+ .compose({+ action: async () => {+ throw initializationError;+ },+ })+ .compose({ action: secondMiddleware });++ const consumer = vi.fn();++ await expect(pipeline.call(context, consumer)).rejects.toThrow(+ expect.objectContaining({+ cause: initializationError,+ constructor: ContextPipelineInitializationError,+ }),+ );++ expect(consumer).not.toHaveBeenCalled();+ expect(secondMiddleware).not.toHaveBeenCalled();+ });++ it('should wrap errors in the final consumer', async () => {+ const consumerError = new Error('Request handler failed');+ const context = { a: 3 };++ const pipeline = ContextPipeline.create().compose({+ action: async () => ({+ b: 4,+ }),+ });++ const consumer = vi.fn().mockRejectedValue(consumerError);++ await expect(pipeline.call(context, consumer)).rejects.toThrow(+ expect.objectContaining({+ cause: consumerError,+ constructor: RequestHandlerError,+ }),+ );++ expect(consumer).toHaveBeenCalledWith({ a: 3, b: 4 });+ });++ it('should call cleanup routines even if the final consumer fails', async () => {+ const consumerError = new Error('Request handler failed');+ const context = { a: 3 };+ const cleanup = vi.fn();++ const pipeline = ContextPipeline.create().compose({+ action: async () => ({+ b: 4,+ }),+ cleanup,+ });++ await expect(pipeline.call(context, vi.fn().mockRejectedValue(consumerError))).rejects.toThrow();++ expect(cleanup).toHaveBeenCalledWith({ a: 3, b: 4 }, consumerError);+ });++ it('should wrap cleanup errors', async () => {+ const cleanupError = new Error('Pipeline cleanup failed');+ const context = { a: 3 };++ const pipeline = ContextPipeline.create().compose({+ action: async () => ({+ b: 4,+ }),+ cleanup: async () => {+ throw cleanupError;+ },+ });++ const consumer = vi.fn();++ await expect(pipeline.call(context, consumer)).rejects.toThrow(+ expect.objectContaining({+ cause: cleanupError,+ constructor: ContextPipelineCleanupError,+ }),+ );++ expect(consumer).toHaveBeenCalledWith({ a: 3, b: 4 });+ });+});diff --git a/test/core/crawlers/crawler_extension.test.ts b/test/core/crawlers/crawler_extension.test.tsdeleted file mode 100644index 6949953d1..000000000--- a/test/core/crawlers/crawler_extension.test.ts+++ /dev/null@@ -1,15 +0,0 @@-import { CrawlerExtension } from '@crawlee/core';--describe('CrawlerExtension', () => {- test('should work', () => {- class MyExtension extends CrawlerExtension {}- const myExtension = new MyExtension();- expect(myExtension.name).toEqual('MyExtension');- expect(() => myExtension.getCrawlerOptions()).toThrow(- `${myExtension.name} has not implemented "getCrawlerOptions" method.`,- );- expect(myExtension.log.info).toBeDefined();- // @ts-expect-error Accessing private prop- expect(myExtension.log.options.prefix).toEqual('MyExtension');- });-});diff --git a/test/core/crawlers/file_download.test.ts b/test/core/crawlers/file_download.test.tsindex 01a7ab5d9..8525b2b3f 100644--- a/test/core/crawlers/file_download.test.ts+++ b/test/core/crawlers/file_download.test.ts@@ -1,13 +1,14 @@import type { Server } from 'node:http';import type { AddressInfo } from 'node:net';-import { Duplex } from 'node:stream';+import { Duplex, pipeline as pipelineWithCallbacks } from 'node:stream';import { pipeline } from 'node:stream/promises';import { ReadableStream } from 'node:stream/web';import { setTimeout } from 'node:timers/promises';-import { Configuration, FileDownload } from '@crawlee/http';+import { FileDownload } from '@crawlee/http';import express from 'express';import { startExpressAppPromise } from 'test/shared/_helper.js';+import { afterAll, beforeAll, expect, test } from 'vitest';class ReadableStreamGenerator {private static async generateRandomData(size: number, seed: number) {@@ -80,13 +81,13 @@ afterAll(async () => {server.close();});-test('requestHandler works', async () => {+test('requestHandler - `body` property works', async () => {const results: Buffer[] = [];const crawler = new FileDownload({maxRequestRetries: 0,- requestHandler: ({ body }) => {- results.push(body as Buffer);+ requestHandler: async ({ body }) => {+ results.push(await body);},});@@ -99,13 +100,13 @@ test('requestHandler works', async () => {expect(results[0]).toEqual(await ReadableStreamGenerator.getBuffer(1024, 123));});-test('streamHandler works', async () => {+test('requestHandler - `stream` property works', async () => {let result: Buffer = Buffer.alloc(0);const crawler = new FileDownload({maxRequestRetries: 0,- streamHandler: async ({ stream }) => {- for await (const chunk of stream as unknown as ReadableStream<any>) {+ requestHandler: async ({ stream }) => {+ for await (const chunk of stream) {result = Buffer.concat([result, chunk]);}},@@ -119,13 +120,11 @@ test('streamHandler works', async () => {expect(result).toEqual(await ReadableStreamGenerator.getBuffer(1024, 456));});-test('streamHandler receives response', async () => {+test('requestHandler receives response', async () => {const crawler = new FileDownload({maxRequestRetries: 0,- streamHandler: async ({ response }) => {+ requestHandler: async ({ response }) => {expect(response.headers['content-type']).toBe('application/octet-stream');- expect(response.rawHeaders[0]).toBe('content-type');- expect(response.rawHeaders[1]).toBe('application/octet-stream');expect(response.statusCode).toBe(200);expect(response.statusMessage).toBe('OK');},@@ -136,10 +135,10 @@ test('streamHandler receives response', async () => {await crawler.run([fileUrl]);});-test('crawler with streamHandler waits for the stream to finish', async () => {+test('crawler waits for the stream to be consumed', async () => {const bufferingStream = new Duplex({read() {},- write(chunk, encoding, callback) {+ write(chunk, _encoding, callback) {this.push(chunk);callback();},@@ -147,15 +146,15 @@ test('crawler with streamHandler waits for the stream to finish', async () => {const crawler = new FileDownload({maxRequestRetries: 0,- streamHandler: ({ stream }) => {- pipeline(stream as any, bufferingStream)- .then(() => {+ requestHandler: ({ stream }) => {+ pipelineWithCallbacks(stream, bufferingStream, (err) => {+ if (!err) {bufferingStream.push(null);bufferingStream.end();- })- .catch((e) => {- bufferingStream.destroy(e);- });+ } else {+ bufferingStream.destroy(err);+ }+ });},});diff --git a/test/core/crawlers/playwright_crawler.test.ts b/test/core/crawlers/playwright_crawler.test.tsindex 232bf1bab..0ff8a9ce5 100644--- a/test/core/crawlers/playwright_crawler.test.ts+++ b/test/core/crawlers/playwright_crawler.test.ts@@ -38,7 +38,7 @@ describe('PlaywrightCrawler', () => {const app = express();server = await startExpressAppPromise(app, 0);port = (server.address() as AddressInfo).port;- app.get('/', (req, res) => {+ app.get('/', (_req, res) => {res.send(`<html><head><title>Example Domain</title></head></html>`);res.status(200);});@@ -96,13 +96,8 @@ describe('PlaywrightCrawler', () => {const processed: Request[] = [];const failed: Request[] = [];const requestListLarge = await RequestList.open({ sources: sourcesLarge });- const requestHandler = async ({- page,- request,- response,- useState,- }: Parameters<PlaywrightRequestHandler>[0]) => {- const state = await useState([]);+ const requestHandler = async ({ page, request, response, useState }: PlaywrightCrawlingContext) => {+ await useState([]);expect(response!.status()).toBe(200);request.userData.title = await page.title();processed.push(request);diff --git a/test/e2e/adaptive-playwright-default/test.mjs b/test/e2e/adaptive-playwright-default/test.mjsindex 5e6f662e2..bb3185ebd 100644--- a/test/e2e/adaptive-playwright-default/test.mjs+++ b/test/e2e/adaptive-playwright-default/test.mjs@@ -1,4 +1,4 @@-import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/run.mjs b/test/e2e/run.mjsindex 4c6d2ed7b..bae63aebd 100644--- a/test/e2e/run.mjs+++ b/test/e2e/run.mjs@@ -1,6 +1,5 @@/* eslint-disable no-loop-func */import { execSync } from 'node:child_process';-import { once } from 'node:events';import { readdir } from 'node:fs/promises';import { dirname } from 'node:path';import { fileURLToPath } from 'node:url';diff --git a/test/e2e/storage-open-return-storage-object/test.mjs b/test/e2e/storage-open-return-storage-object/test.mjsindex ed808f241..a186160db 100644--- a/test/e2e/storage-open-return-storage-object/test.mjs+++ b/test/e2e/storage-open-return-storage-object/test.mjs@@ -1,4 +1,4 @@-import { initialize, expect, getActorTestDir, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';/* This test verifies that the storageObject is correctly returned when the KeyValueStore or Dataset is opened.* The storageObject is the result of the KeyValueStoreClient.get() or Dataset.get() methods,