Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor: use `ContextPipeline` to initialize `BasicCrawler`'s context idiomatically (#3388)
Jindřich Bärd33ab5fe3023
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 7808381ee..0fd68a0f6 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -79,7 +79,7 @@ import { getDomain } from 'tldts';import type { ReadonlyDeep, SetRequired } from 'type-fest';import { LruCache } from '@apify/datastructures';-import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';+import { addTimeoutToPromise, TimeoutError } from '@apify/timeout';import { cryptoRandomObjectId } from '@apify/utilities';import { createSendRequest } from './send-request.js';@@ -97,6 +97,8 @@ export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary>*/const SAFE_MIGRATION_WAIT_MILLIS = 20000;+const deferredCleanupKey = Symbol('deferredCleanup');+export type RequestHandler<Context extends CrawlingContext = CrawlingContext> = (inputs: Context) => Awaitable<void>;export type ErrorHandler<@@ -578,12 +580,29 @@ export class BasicCrawler<*/readonly router: RouterHandler<Context> = Router.create<Context>();- private contextPipelineBuilder: () => ContextPipeline<CrawlingContext, ExtendedContext>;+ private _basicContextPipeline?: ContextPipeline<{ request: Request }, CrawlingContext>;++ /**+ * The basic part of the context pipeline. Unlike the subclass pipeline, this+ * part has no major side effects (e.g. launching a browser). It also makes typing more explicit, as subclass+ * pipelines expect the basic crawler fields to already be present in the context at runtime.+ *+ * Context built with this pipeline can be passed into multiple crawler pipelines at once.+ * This is used e.g. in the {@apilink AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.+ */+ get basicContextPipeline(): ContextPipeline<{ request: Request }, CrawlingContext> {+ if (this._basicContextPipeline === undefined) {+ this._basicContextPipeline = this.buildBasicContextPipeline();+ }++ return this._basicContextPipeline;+ }+private _contextPipeline?: ContextPipeline<CrawlingContext, ExtendedContext>;get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext> {if (this._contextPipeline === undefined) {- this._contextPipeline = this.contextPipelineBuilder();+ this._contextPipeline = this.buildFinalContextPipeline();}return this._contextPipeline;@@ -639,6 +658,10 @@ export class BasicCrawler<private _experimentWarnings: Partial<Record<keyof CrawlerExperiments, boolean>> = {};private readonly crawlerId: string;private readonly hasExplicitId: boolean;+ private readonly contextPipelineOptions: {+ contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;+ extendContext?: (context: Context) => Awaitable<ContextExtension>;+ };protected static optionsShape = {contextPipelineBuilder: ow.optional.object,@@ -762,6 +785,10 @@ export class BasicCrawler<try {serviceLocatorScope.enterScope();+ this.contextPipelineOptions = {+ contextPipelineBuilder: options.contextPipelineBuilder,+ extendContext: options.extendContext,+ };this.#log = serviceLocator.getLogger().child({ prefix: this.constructor.name });@@ -770,40 +797,6 @@ export class BasicCrawler<// Store the user-provided ID, or generate a unique one for tracking purposes (not for state key)this.crawlerId = id ?? cryptoRandomObjectId();- // Store the builder so that it can be run when the contextPipeline is needed.- // Invoking it immediately would cause problems with parent constructor call order.- this.contextPipelineBuilder = () => {- let contextPipeline = (options.contextPipelineBuilder?.() ??- ContextPipeline.create<CrawlingContext>()) as ContextPipeline<CrawlingContext, Context>; // Thanks to the RequireContextPipeline, contextPipeline will only be undefined if InitialContextType is CrawlingContext-- if (options.extendContext !== undefined) {- contextPipeline = contextPipeline.compose({- action: async (context) => await options.extendContext(context),- });- }-- contextPipeline = contextPipeline.compose({- action: async (context) => {- const { request } = context;- if (!this.requestMatchesEnqueueStrategy(request)) {- // eslint-disable-next-line dot-notation- const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;- this.log.debug(message);-- request.noRetry = true;- request.state = RequestState.SKIPPED;-- await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });-- throw new ContextPipelineInterruptedError(message);- }- return context;- },- });-- return contextPipeline as ContextPipeline<CrawlingContext, ExtendedContext>;- };-if (requestManager !== undefined) {if (requestList !== undefined || requestQueue !== undefined) {throw new Error(@@ -906,7 +899,52 @@ export class BasicCrawler<minConcurrency: minConcurrency ?? autoscaledPoolOptions?.minConcurrency,maxConcurrency: maxConcurrency ?? autoscaledPoolOptions?.maxConcurrency,maxTasksPerMinute: maxRequestsPerMinute ?? autoscaledPoolOptions?.maxTasksPerMinute,- runTaskFunction: this._runTaskFunction.bind(this),+ runTaskFunction: async () => {+ const source = this.requestManager;+ if (!source) throw new Error('Request provider is not initialized!');++ const request = await this.resolveRequest();+ if (!request || this.delayRequest(request, source)) {+ return;+ }++ const crawlingContext = { request } as { request: Request } & Partial<CrawlingContext>;+ try {+ await this.basicContextPipeline+ .chain(this.contextPipeline)+ .call(crawlingContext, (ctx) => this.handleRequest(ctx, source));+ } catch (error) {+ // ContextPipelineInterruptedError means the request was intentionally skipped+ // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.+ if (error instanceof ContextPipelineInterruptedError) {+ await this._timeoutAndRetry(+ async () => this.requestManager?.markRequestHandled(crawlingContext.request!),+ this.internalTimeoutMillis,+ `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${+ this.internalTimeoutMillis / 1e3+ } seconds.`,+ );+ return;+ }++ // If the error happened during pipeline initialization (e.g., navigation timeout, session/proxy error,+ // i.e. not in user's requestHandler), handle it through the normal error flow.+ const isPipelineError =+ error instanceof ContextPipelineInitializationError || error instanceof SessionError;+ if (isPipelineError) {+ const unwrappedError = this.unwrapError(error);++ await this._requestFunctionErrorHandler(+ unwrappedError,+ crawlingContext as CrawlingContext,+ this.requestManager!,+ );+ crawlingContext.session?.markBad();+ return;+ }+ throw this.unwrapError(error);+ }+ },isTaskReadyFunction: async () => {if (isMaxPagesExceeded()) {this.logOncePerRun(@@ -967,6 +1005,150 @@ export class BasicCrawler<}}+ /**+ * Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.+ * This handles base context creation, session resolution, and context helpers.+ */+ protected buildBasicContextPipeline(): ContextPipeline<{ request: Request }, CrawlingContext> {+ return ContextPipeline.create<{ request: Request }>()+ .compose({ action: this.checkRobotsTxt.bind(this) })+ .compose({+ action: () => this.createBaseContext(),+ cleanup: async (context) => {+ await Promise.all(context[deferredCleanupKey].map((fn) => fn()));+ },+ })+ .compose({ action: this.resolveSession.bind(this) })+ .compose({ action: this.createContextHelpers.bind(this) });+ }++ private async checkRobotsTxt({ request }: { request: Request }) {+ if (!(await this.isAllowedBasedOnRobotsTxtFile(request.url))) {+ this.log.warning(+ `Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`,+ );+ request.state = RequestState.SKIPPED;+ request.noRetry = true;+ await this.handleSkippedRequest({+ url: request.url,+ reason: 'robotsTxt',+ });++ throw new ContextPipelineInterruptedError(`Skipping request ${request.url} as disallowed by robots.txt`);+ }++ return {};+ }++ /**+ * Builds the subclass-specific context pipeline that transforms a `CrawlingContext` into the crawler's target context type.+ * Subclasses should override this to add their own pipeline stages.+ */+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, CrawlingContext> {+ return ContextPipeline.create<CrawlingContext>();+ }++ private createBaseContext() {+ const deferredCleanup: (() => Promise<unknown>)[] = [];++ return {+ id: cryptoRandomObjectId(10),+ log: this.log,+ pushData: this.pushData.bind(this),+ useState: this.useState.bind(this),+ getKeyValueStore: async (idOrName?: string) => KeyValueStore.open(idOrName),+ registerDeferredCleanup: (cleanup: () => Promise<unknown>) => {+ deferredCleanup.push(cleanup);+ },+ [deferredCleanupKey]: deferredCleanup,+ };+ }++ private async resolveRequest(): Promise<Request | null> {+ const request = await this._timeoutAndRetry(+ this._fetchNextRequest.bind(this),+ this.internalTimeoutMillis,+ `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,+ );++ // Reset loadedUrl so an old one is not carried over to retries.+ if (request) {+ request.loadedUrl = undefined;+ }++ return request;+ }++ private async resolveSession({ request }: { request: Request }) {+ const session = this.useSessionPool+ ? await this._timeoutAndRetry(+ async () => {+ return await this.sessionPool!.newSession({+ proxyInfo: await this.proxyConfiguration?.newProxyInfo({+ request: request ?? undefined,+ }),+ maxUsageCount: 1,+ });+ },+ this.internalTimeoutMillis,+ `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,+ )+ : undefined;++ return { session, proxyInfo: session?.proxyInfo };+ }++ private async createContextHelpers({ request, session }: { request: Request; session?: Session }) {+ const enqueueLinksWrapper: CrawlingContext['enqueueLinks'] = async (options) => {+ const requestQueue = await this.getRequestQueue();++ return await this.enqueueLinksWithCrawlDepth(options, request!, requestQueue);+ };+ const addRequests: CrawlingContext['addRequests'] = async (requests, options = {}) => {+ const newCrawlDepth = request!.crawlDepth + 1;+ const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);++ await this.addRequests(requestsGenerator, options);+ };++ const sendRequest = createSendRequest(this.httpClient, request!, session);++ return { enqueueLinks: enqueueLinksWrapper, addRequests, sendRequest };+ }++ private buildFinalContextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext> {+ let contextPipeline = (this.contextPipelineOptions.contextPipelineBuilder?.() ??+ this.buildContextPipeline()) as ContextPipeline<CrawlingContext, Context>;++ const { extendContext } = this.contextPipelineOptions;+ if (extendContext !== undefined) {+ contextPipeline = contextPipeline.compose({+ action: async (context) => await extendContext(context),+ });+ }++ contextPipeline = contextPipeline.compose({+ action: async (context) => {+ const { request } = context;+ if (request && !this.requestMatchesEnqueueStrategy(request)) {+ // eslint-disable-next-line dot-notation+ const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;+ this.log.debug(message);++ request.noRetry = true;+ request.state = RequestState.SKIPPED;++ await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });++ throw new ContextPipelineInterruptedError(message);+ }+ return context;+ },+ });++ return contextPipeline as ContextPipeline<CrawlingContext, ExtendedContext>;+ }+/*** Checks if the given error is a proxy error by comparing its message to a list of known proxy error messages.* Used for retrying requests that failed due to proxy errors.@@ -1525,14 +1707,12 @@ export class BasicCrawler<await this.initializeRequestManager();}- protected async runRequestHandler(crawlingContext: CrawlingContext): Promise<void> {- await this.contextPipeline.call(crawlingContext, async (finalContext) => {- await addTimeoutToPromise(- async () => this.requestHandler(finalContext),- this.requestHandlerTimeoutMillis,- `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds (${finalContext.request.id}).`,- );- });+ protected async runRequestHandler(crawlingContext: ExtendedContext): Promise<void> {+ await addTimeoutToPromise(+ async () => this.requestHandler(crawlingContext),+ this.requestHandlerTimeoutMillis,+ `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds (${crawlingContext.request.id}).`,+ );}/**@@ -1696,92 +1876,13 @@ export class BasicCrawler<return true;}- /**- * Wrapper around requestHandler that fetches requests from RequestList/RequestQueue- * then retries them in a case of an error, etc.- */- protected async _runTaskFunction() {- const source = this.requestManager;- if (!source) throw new Error('Request provider is not initialized!');-- const request = await this._timeoutAndRetry(- this._fetchNextRequest.bind(this),- this.internalTimeoutMillis,- `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,- );-- tryCancel();-- const session = this.useSessionPool- ? await this._timeoutAndRetry(- async () => {- return await this.sessionPool!.newSession({- proxyInfo: await this.proxyConfiguration?.newProxyInfo({- request: request ?? undefined,- }),- maxUsageCount: 1,- });- },- this.internalTimeoutMillis,- `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,- )- : undefined;-- tryCancel();-- if (!request || this.delayRequest(request, source)) {- return;- }-- if (!(await this.isAllowedBasedOnRobotsTxtFile(request.url))) {- this.log.warning(- `Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`,- );- request.state = RequestState.SKIPPED;- request.noRetry = true;- await source.markRequestHandled(request);- await this.handleSkippedRequest({- url: request.url,- reason: 'robotsTxt',- });- return;- }-- // Reset loadedUrl so an old one is not carried over to retries.- request.loadedUrl = undefined;+ /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */+ protected async handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager) {+ const { request } = crawlingContext;const statisticsId = request.id || request.uniqueKey;this.stats.startJob(statisticsId);- const deferredCleanup: (() => Promise<unknown>)[] = [];-- const crawlingContext: CrawlingContext = {- id: cryptoRandomObjectId(10),- log: this.log,- request,- session,- proxyInfo: session?.proxyInfo,- enqueueLinks: async (options) => {- const requestQueue = await this.getRequestQueue();-- return await this.enqueueLinksWithCrawlDepth(options, request, requestQueue);- },- addRequests: async (requests, options = {}) => {- const newCrawlDepth = request.crawlDepth + 1;- const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);-- await this.addRequests(requestsGenerator, options);- },- pushData: this.pushData.bind(this),- useState: this.useState.bind(this),- sendRequest: createSendRequest(this.httpClient, request!, session),- getKeyValueStore: async (idOrName?: string) =>- KeyValueStore.open(idOrName, { config: serviceLocator.getConfiguration() }),- registerDeferredCleanup: (cleanup) => {- deferredCleanup.push(cleanup);- },- };-let isRequestLocked = true;try {@@ -1789,7 +1890,7 @@ export class BasicCrawler<await this.runRequestHandler(crawlingContext);await this._timeoutAndRetry(- async () => source.markRequestHandled(request!),+ async () => requestSource.markRequestHandled(request!),this.internalTimeoutMillis,`Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3@@ -1808,7 +1909,7 @@ export class BasicCrawler<try {request.state = RequestState.ERROR_HANDLER;await addTimeoutToPromise(- async () => this._requestFunctionErrorHandler(err, crawlingContext, source),+ async () => this._requestFunctionErrorHandler(err, crawlingContext, requestSource),this.internalTimeoutMillis,`Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3@@ -1841,12 +1942,10 @@ export class BasicCrawler<// decrease the session score if the request fails (but the error handler did not throw)crawlingContext.session?.markBad();} finally {- await Promise.all(deferredCleanup.map((cleanup) => cleanup()));-// Safety net - release the lock if nobody managed to do it before- if (isRequestLocked && source instanceof RequestProvider) {+ if (isRequestLocked && requestSource instanceof RequestProvider) {try {- await source.client.deleteRequestLock(request.id!);+ await requestSource.client.deleteRequestLock(request.id!);} catch {// We don't have the lock, or the request was never locked. Either way it's fine}diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex a8871583f..04dc6909b 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -413,7 +413,7 @@ export abstract class BrowserCrawler<});}- protected buildContextPipeline(): ContextPipeline<+ protected override buildContextPipeline(): ContextPipeline<CrawlingContext,BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary>> {diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.tsindex 177360f40..522a13306 100644--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts@@ -179,17 +179,23 @@ export class CheerioCrawler<* All `CheerioCrawler` parameters are passed via an options object.*/constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext>) {+ const { contextPipelineBuilder, ...rest } = options ?? {};+super({- ...options,- contextPipelineBuilder: () =>- this.buildContextPipeline()- .compose({- action: async (context) => await this.parseContent(context),- })- .compose({ action: async (context) => await this.addHelpers(context) }),+ ...rest,+ contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),});}+ protected override buildContextPipeline() {+ return super+ .buildContextPipeline()+ .compose({+ action: async (context) => await this.parseContent(context),+ })+ .compose({ action: async (context) => await this.addHelpers(context) });+ }+private async parseContent(crawlingContext: InternalHttpCrawlingContext) {const isXml = crawlingContext.contentType.type.includes('xml');const body = Buffer.isBuffer(crawlingContext.body)diff --git a/packages/core/src/crawlers/context_pipeline.ts b/packages/core/src/crawlers/context_pipeline.tsindex e84361a25..8f7fe8483 100644--- a/packages/core/src/crawlers/context_pipeline.ts+++ b/packages/core/src/crawlers/context_pipeline.ts@@ -1,5 +1,7 @@import type { Awaitable } from '@crawlee/types';+import log from '@apify/log';+import {ContextPipelineCleanupError,ContextPipelineInitializationError,@@ -56,6 +58,19 @@ export abstract class ContextPipeline<TContextBase, TCrawlingContext extends TComiddleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>,): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension>;+ /**+ * Chains another pipeline onto this one. The other pipeline's base context must match+ * this pipeline's output context. Returns a new pipeline that runs this pipeline's+ * middlewares first, then the other pipeline's middlewares.+ *+ * @template TFinalContext - The final context type after the chained pipeline's transformations+ * @param other - The pipeline to append after this one+ * @returns A new ContextPipeline combining both pipelines' middlewares+ */+ abstract chain<TFinalContext extends TCrawlingContext>(+ other: ContextPipeline<TCrawlingContext, TFinalContext>,+ ): ContextPipeline<TContextBase, TFinalContext>;+/*** Executes the middleware pipeline and passes the final context to a consumer function.*@@ -105,6 +120,21 @@ class ContextPipelineImpl<TContextBase, TCrawlingContext extends TContextBase> e);}+ chain<TFinalContext extends TCrawlingContext>(+ other: ContextPipeline<TCrawlingContext, TFinalContext>,+ ): ContextPipeline<TContextBase, TFinalContext> {+ const otherMiddlewares = Array.from(+ (other as any).middlewareChain() as Iterable<ContextMiddleware<any, any>>,+ ).reverse();++ let result: ContextPipeline<TContextBase, any> = this as any;+ for (const middleware of otherMiddlewares) {+ result = result.compose(middleware as any);+ }++ return result as ContextPipeline<TContextBase, TFinalContext>;+ }+private *middlewareChain() {let step: ContextPipelineImpl<TContextBase, TContextBase> | undefined = this as any;@@ -129,7 +159,25 @@ class ContextPipelineImpl<TContextBase, TCrawlingContext extends TContextBase> efor (const { action, cleanup } of middlewares) {try {const contextExtension = await action(crawlingContext);- Object.defineProperties(crawlingContext, Object.getOwnPropertyDescriptors(contextExtension));++ const extensionNames = [+ ...Object.getOwnPropertyNames(contextExtension),+ ...Object.getOwnPropertySymbols(contextExtension),+ ];++ for (const key of extensionNames) {+ try {+ if (Object.getOwnPropertyDescriptor(crawlingContext, key)?.configurable !== false) {+ Object.defineProperty(+ crawlingContext,+ key,+ Object.getOwnPropertyDescriptor(contextExtension, key)!,+ );+ }+ } catch (error: any) {+ log.debug(`Context pipeline failed to define property ${key.toString()}:`, error);+ }+ }if (cleanup) {cleanupStack.push(cleanup);diff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.tsindex 099944bdc..fbb09d994 100644--- a/packages/http-crawler/src/internals/file-download.ts+++ b/packages/http-crawler/src/internals/file-download.ts@@ -1,7 +1,7 @@import { Transform } from 'node:stream';import type { BasicCrawlerOptions } from '@crawlee/basic';-import { BasicCrawler, ContextPipeline } from '@crawlee/basic';+import { BasicCrawler } from '@crawlee/basic';import type { CrawlingContext, LoadedRequest, Request } from '@crawlee/core';import { ResponseWithUrl } from '@crawlee/http-client';import type { Dictionary } from '@crawlee/types';@@ -162,19 +162,22 @@ export class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {constructor(options: BasicCrawlerOptions<FileDownloadCrawlingContext> = {}) {super({...options,- contextPipelineBuilder: () =>- ContextPipeline.create<CrawlingContext>().compose({- action: async (context) => this.initiateDownload(context),- cleanup: async (context) => {- if (!context.response.bodyUsed) {- // Nobody consumed the body — cancel it so the- // underlying connection can be released.- await context.response.body?.cancel();- }+ contextPipelineBuilder: () => this.buildContextPipeline(),+ });+ }++ protected override buildContextPipeline() {+ return super.buildContextPipeline().compose({+ action: async (context) => this.initiateDownload(context),+ cleanup: async (context) => {+ if (!context.response.bodyUsed) {+ // Nobody consumed the body — cancel it so the+ // underlying connection can be released.+ await context.response.body?.cancel();+ }- await (context as { [kBodyDrained]: Promise<void> })[kBodyDrained];- },- }),+ await (context as { [kBodyDrained]: Promise<void> })[kBodyDrained];+ },});}diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex e6cc3be49..cf2d18825 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -414,7 +414,7 @@ export class HttpCrawler<}}- protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext> {+ protected override buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext> {return ContextPipeline.create<CrawlingContext>().compose({action: this.makeHttpRequest.bind(this),diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsindex 6e8c4cdb8..405c7dad1 100644--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts@@ -195,26 +195,30 @@ export class JSDOMCrawler<protected virtualConsole: VirtualConsole | null = null;constructor(options: JSDOMCrawlerOptions<ContextExtension, ExtendedContext> = {}) {- const { runScripts = false, hideInternalConsole = false, ...httpOptions } = options;+ const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = options;super({...httpOptions,- contextPipelineBuilder: () =>- this.buildContextPipeline()- .compose({- action: async (context) => await this.parseContent(context),- cleanup: async (context) => {- this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler);- context.window?.close();- },- })- .compose({ action: async (context) => await this.addHelpers(context) }),+ contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),});this.runScripts = runScripts;this.hideInternalConsole = hideInternalConsole;}+ protected override buildContextPipeline() {+ return super+ .buildContextPipeline()+ .compose({+ action: async (context) => await this.parseContent(context),+ cleanup: async (context) => {+ this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler);+ context.window?.close();+ },+ })+ .compose({ action: async (context) => await this.addHelpers(context) });+ }+/*** Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.*diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.tsindex 86e2b0e68..66d33bac3 100644--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts@@ -170,17 +170,23 @@ export class LinkeDOMCrawler<private static parser = new DOMParser();constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext>) {+ const { contextPipelineBuilder, ...rest } = options;+super({- ...options,- contextPipelineBuilder: () =>- this.buildContextPipeline()- .compose({- action: async (context) => this.parseContent(context),- })- .compose({ action: async (context) => this.addHelpers(context) }),+ ...rest,+ contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),});}+ protected override buildContextPipeline() {+ return super+ .buildContextPipeline()+ .compose({+ action: async (context) => this.parseContent(context),+ })+ .compose({ action: async (context) => this.addHelpers(context) });+ }+private async parseContent(crawlingContext: InternalHttpCrawlingContext) {const isXml = crawlingContext.contentType.type.includes('xml');const document = LinkeDOMCrawler.parser.parseFromString(diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex 21e7c37e2..0b390b2f6 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -160,7 +160,7 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =*/parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;- enqueueLinks(options?: EnqueueLinksOptions): Promise<void>;+ enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;}interface AdaptiveHook@@ -332,17 +332,11 @@ export class AdaptivePlaywrightCrawler<super({...rest,- // Pass error handlers to the "main" crawler - we only pluck them from `rest` so that they don't go to the sub crawlerserrorHandler,failedRequestHandler,- // Same for request handlerrequestHandler,- // The builder intentionally returns null so that it crashes the crawler when it tries to use this instead of one of two the specialized context pipelines- // (that would be a logical error in this class)- contextPipelineBuilder: () =>- null as unknown as ContextPipeline<CrawlingContext, AdaptivePlaywrightCrawlerContext>,+ contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),});-this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;this.renderingTypePredictor =@@ -442,6 +436,34 @@ export class AdaptivePlaywrightCrawler<return await super._init();}+ protected override buildContextPipeline() {+ const errorMessage = (prop: string) =>+ `The \`${prop}\` property is not available on the outer context pipeline of AdaptivePlaywrightCrawler - it is provided by the inner (static/browser) pipelines`;++ return super.buildContextPipeline().compose({+ action: async ({ request }) => ({+ get request(): LoadedRequest<Request<Dictionary>> {+ return request as LoadedRequest<Request<Dictionary>>;+ },+ get response(): Response {+ throw new Error(errorMessage('response'));+ },+ get page(): Page {+ throw new Error(errorMessage('page'));+ },+ get querySelector(): AdaptivePlaywrightCrawlerContext['querySelector'] {+ throw new Error(errorMessage('querySelector'));+ },+ get waitForSelector(): AdaptivePlaywrightCrawlerContext['waitForSelector'] {+ throw new Error(errorMessage('waitForSelector'));+ },+ get parseWithCheerio(): AdaptivePlaywrightCrawlerContext['parseWithCheerio'] {+ throw new Error(errorMessage('parseWithCheerio'));+ },+ }),+ });+ }+private async adaptCheerioContext(cheerioContext: CheerioCrawlingContext) {// Capture the original response to avoid infinite recursion when the getter is copied to the contextconst result = this.resultObjects.get(cheerioContext);@@ -541,28 +563,29 @@ export class AdaptivePlaywrightCrawler<pushData: result.pushData,useState: this.allowStorageAccess(useStateFunction),getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),- enqueueLinks: async (options: SetRequired<EnqueueLinksOptions, 'urls'>) => {- return await this.enqueueLinks(options, context.request, result);- },log: this.createLogProxy(context.log, logs),registerDeferredCleanup: (cleanup: () => Promise<unknown>) => deferredCleanup.push(cleanup),};- const subCrawlerContext = { ...context, ...resultBoundContextHelpers };+ const subCrawlerContext = Object.defineProperties(+ {},+ Object.getOwnPropertyDescriptors(context),+ ) as typeof context;++ // Mark result-bound helpers as non-configurable so they survive the sub-crawler context pipeline+ // (which would otherwise override them with the sub-crawler's own versions, losing the result binding).+ for (const [key, descriptor] of Object.entries(Object.getOwnPropertyDescriptors(resultBoundContextHelpers))) {+ Object.defineProperty(subCrawlerContext, key, { ...descriptor, configurable: false });+ }+this.resultObjects.set(subCrawlerContext, result);try {const callAdaptiveRequestHandler = async () => {if (renderingType === 'static') {- await this.staticContextPipeline.call(- subCrawlerContext,- async (finalContext) => await this.requestHandler(finalContext),- );+ await this.staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));} else if (renderingType === 'clientOnly') {- await this.browserContextPipeline.call(- subCrawlerContext,- async (finalContext) => await this.requestHandler(finalContext),- );+ await this.browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));}};diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex b51abec7d..b58e581f9 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -205,7 +205,7 @@ export class PlaywrightCrawler<constructor(options: PlaywrightCrawlerOptions<ExtendedContext> = {}) {ow(options, 'PlaywrightCrawlerOptions', ow.object.exactShape(PlaywrightCrawler.optionsShape));- const { launchContext = {}, headless, ...browserCrawlerOptions } = options;+ const { launchContext = {}, headless, contextPipelineBuilder, ...browserCrawlerOptions } = options;const browserPoolOptions = {...options.browserPoolOptions,@@ -237,11 +237,14 @@ export class PlaywrightCrawler<...(browserCrawlerOptions as PlaywrightCrawlerOptions<ExtendedContext>),launchContext,browserPoolOptions,- contextPipelineBuilder: () =>- this.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) }),+ contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),});}+ protected override buildContextPipeline() {+ return super.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) });+ }+protected override async _navigationHandler(crawlingContext: PlaywrightCrawlingContext,gotoOptions: DirectNavigationOptions,diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex a2cee00b4..349c4571f 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -177,7 +177,13 @@ export class PuppeteerCrawler<constructor(options: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext> = {}) {ow(options, 'PuppeteerCrawlerOptions', ow.object.exactShape(PuppeteerCrawler.optionsShape));- const { launchContext = {}, headless, proxyConfiguration, ...browserCrawlerOptions } = options;+ const {+ launchContext = {},+ headless,+ proxyConfiguration,+ contextPipelineBuilder,+ ...browserCrawlerOptions+ } = options;const browserPoolOptions = {...options.browserPoolOptions,@@ -217,11 +223,14 @@ export class PuppeteerCrawler<launchContext,proxyConfiguration,browserPoolOptions,- contextPipelineBuilder: () =>- this.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) }),+ contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),});}+ protected override buildContextPipeline() {+ return super.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) });+ }+private async enhanceContext(context: BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController>) {const waitForSelector = async (selector: string, timeoutMs = 5_000) => {await context.page.waitForSelector(selector, { timeout: timeoutMs });diff --git a/test/core/crawlers/context_pipeline.test.ts b/test/core/crawlers/context_pipeline.test.tsindex a02d2416d..daf7e3109 100644--- a/test/core/crawlers/context_pipeline.test.ts+++ b/test/core/crawlers/context_pipeline.test.ts@@ -164,4 +164,55 @@ describe('ContextPipeline', () => {expect(consumer).toHaveBeenCalledWith({ a: 3, b: 4 });});++ it('should not override non-configurable properties on the context', async () => {+ const context = {} as Record<string, unknown>;+ Object.defineProperty(context, 'frozen', { value: 'original', configurable: false });++ const pipeline = ContextPipeline.create<typeof context>().compose({+ action: async () => ({ frozen: 'overridden', other: 'new' }),+ });++ const consumer = vi.fn();+ await pipeline.call(context, consumer);++ expect(consumer).toHaveBeenCalledWith(expect.objectContaining({ frozen: 'original', other: 'new' }));+ });++ describe('chain', () => {+ it('should run middlewares from both pipelines in order', async () => {+ const first = ContextPipeline.create<{ a: number }>().compose({+ action: async (ctx) => ({ b: ctx.a + 1 }),+ });+ const second = ContextPipeline.create<{ a: number; b: number }>().compose({+ action: async (ctx) => ({ c: ctx.b * 2 }),+ });++ const consumer = vi.fn();+ await first.chain(second).call({ a: 1 }, consumer);++ expect(consumer).toHaveBeenCalledWith({ a: 1, b: 2, c: 4 });+ });++ it('should call cleanup routines from both pipelines', async () => {+ const order: string[] = [];++ const first = ContextPipeline.create<object>().compose({+ action: async () => ({}),+ cleanup: async () => {+ order.push('first');+ },+ });+ const second = ContextPipeline.create<object>().compose({+ action: async () => ({}),+ cleanup: async () => {+ order.push('second');+ },+ });++ await first.chain(second).call({}, vi.fn());++ expect(order).toEqual(['second', 'first']);+ });+ });});