Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor: Finalize the ContextPipeline compose API (#4149)
Jan Buchar2bca0a25fb60
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.mdindex 0a351957e..235618f5a 100644--- a/docs/public-api/crawlee-basic.api.md+++ b/docs/public-api/crawlee-basic.api.md@@ -199,6 +199,9 @@ export interface CalculatedStatistics {requestTotalDurationMillis: number;}+// @public+export type CleanupRegistrar = (cleanup: (error?: unknown) => Awaitable<void>) => void;+// @publicexport interface ConcurrencyConsumer {readonly id: string;@@ -245,14 +248,11 @@ export interface ConcurrencySystemOptions {}// @public-export interface ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> {- action: (context: TCrawlingContext) => Awaitable<TCrawlingContextExtension>;- cleanup?: (context: TCrawlingContext & TCrawlingContextExtension, error?: unknown) => Awaitable<void>;-}+export type ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> = (context: TCrawlingContext, onCleanup: CleanupRegistrar) => Awaitable<TCrawlingContextExtension>;// @publicexport abstract class ContextPipeline<TContextBase, TCrawlingContext extends TContextBase> {- abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>): Promise<void>;+ abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>, onInitializationError: (error: unknown) => Awaitable<void>): Promise<void>;abstract chain<TFinalContext extends TCrawlingContext>(other: ContextPipeline<TCrawlingContext, TFinalContext>): ContextPipeline<TContextBase, TFinalContext>;abstract compose<TCrawlingContextExtension>(middleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension>;static create<TContextBase>(): ContextPipeline<TContextBase, TContextBase>;@@ -263,11 +263,6 @@ export class ContextPipelineCleanupError extends CriticalError {constructor(error: unknown, options?: ErrorOptions);}-// @public (undocumented)-export class ContextPipelineInitializationError extends Error {- constructor(error: unknown, options?: ErrorOptions);-}-// @public (undocumented)export class ContextPipelineInterruptedError extends Error {constructor(message?: string);diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex f2c5c9be2..883eaefed 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -66,13 +66,13 @@ import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscalingimport { ConcurrencySystem } from './autoscaling/concurrency_system.js';import type { FinalStatistics } from './autoscaling/system_status.js';import { mergeCookies } from './cookie_utils.js';+import type { CleanupRegistrar } from './crawlers/context_pipeline.js';import { ContextPipeline } from './crawlers/context_pipeline.js';import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler_commons.js';import type { CrawlingRequestData, SkippedRequestReason } from './crawling_request.js';import { CrawlingRequest, RequestState } from './crawling_request.js';import {ContextPipelineCleanupError,- ContextPipelineInitializationError,ContextPipelineInterruptedError,MissingSessionError,NavigationSkippedError,@@ -147,8 +147,6 @@ export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary>*/const SAFE_MIGRATION_WAIT_MILLIS = 20000;-const deferredCleanupKey = Symbol('deferredCleanup');-// The request timeout plumbing (the window helper, the context symbols, and the race) lives in its own module.export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';@@ -1238,6 +1236,43 @@ export class BasicCrawler<this.statistics.recordRequestStart(request.id || request.uniqueKey);const crawlingContext = { request } as { request: CrawlingRequest } & Partial<CrawlingContext>;++ // Runs inside the pipeline call, so the deferred cleanups registered by earlier middlewares+ // (e.g. closing the page) only fire after the error handler has seen the partial context.+ const handleInitializationError = async (rawError: unknown) => {+ // Roll back *before* the error handler runs, for the same reason `handleRequest` does.+ currentStorageTransaction()?.rollback();++ // ContextPipelineInterruptedError means the request was intentionally skipped+ // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.+ if (rawError instanceof ContextPipelineInterruptedError) {+ this.statistics.discardRequestRecord(request.id || request.uniqueKey);+ await this.timeoutAndRetry(+ async () => this.requestManager?.markRequestAsHandled(request),+ this.internalTimeoutMillis,+ `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${+ this.internalTimeoutMillis / 1e3+ } seconds.`,+ );+ return;+ }++ // An error during pipeline initialization (e.g., navigation timeout, session/proxy error,+ // i.e. not in user's requestHandler) goes through the normal error flow.+ const error = this.unwrapError(rawError);+ await this.requestFunctionErrorHandler(+ error,+ crawlingContext as CrawlingContext,+ request,+ this.requestManager!,+ );+ // SessionError already retired the session in `requestFunctionErrorHandler`;+ // skip `markBad` to avoid double-counting usage/error score.+ if (!this.errorAbsolvesSession(error)) {+ crawlingContext.session?.markBad();+ }+ };+try {// The transaction spans the whole pipeline call, covering the navigation hooks// and `extendContext` too; `handleRequest` drives its outcome explicitly.@@ -1250,32 +1285,18 @@ export class BasicCrawler<crawlingContext,this.basicContextPipeline.chain(this.contextPipeline)- .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request)),+ .call(+ crawlingContext,+ (ctx) => this.handleRequest(ctx, source, request),+ handleInitializationError,+ ),),);} catch (error) {- // ContextPipelineInterruptedError means the request was intentionally skipped- // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.- if (error instanceof ContextPipelineInterruptedError) {- this.statistics.discardRequestRecord(request.id || request.uniqueKey);- await this.timeoutAndRetry(- async () => this.requestManager?.markRequestAsHandled(request),- this.internalTimeoutMillis,- `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${- this.internalTimeoutMillis / 1e3- } seconds.`,- );- return;- }-- // If the error happened during pipeline initialization (e.g., navigation timeout, session/proxy error,- // i.e. not in user's requestHandler), handle it through the normal error flow. A bare `TimeoutError`- // here is the internal timeout above firing - anything else thrown inside the pipeline arrives wrapped.- const isPipelineError =- error instanceof ContextPipelineInitializationError ||- error instanceof SessionError ||- error instanceof TimeoutError;- if (isPipelineError) {+ // A bare `TimeoutError` is the internal timeout above firing; a `SessionError` can only be a+ // secondary error from `handleRequest` - both take the normal error flow. Anything else thrown+ // inside the pipeline arrives wrapped.+ if (error instanceof SessionError || error instanceof TimeoutError) {const unwrappedError = this.unwrapError(error);await this.requestFunctionErrorHandler(@@ -1284,27 +1305,12 @@ export class BasicCrawler<request,this.requestManager!,);- // SessionError already retired the session in `requestFunctionErrorHandler`;- // skip `markBad` to avoid double-counting usage/error score.if (!this.errorAbsolvesSession(unwrappedError)) {crawlingContext.session?.markBad();}return;}throw this.unwrapError(error);- } finally {- // Run request-scoped deferred cleanups only after the whole request lifecycle - including the user's error handler - has finished.- const deferredCleanup =- (crawlingContext as Partial<Record<typeof deferredCleanupKey, (() => Promise<unknown>)[]>>)[- deferredCleanupKey- ] ?? [];- await Promise.all(- deferredCleanup.map((fn) =>- fn().catch((cleanupError) =>- this.log.debug('Error in deferred cleanup', { error: cleanupError }),- ),- ),- );}},isTaskReadyFunction: async () => {@@ -1421,10 +1427,10 @@ export class BasicCrawler<*/private buildBasicContextPipeline(): ContextPipeline<{ request: CrawlingRequest }, CrawlingContext> {return ContextPipeline.create<{ request: CrawlingRequest }>()- .compose({ action: this.checkRobotsTxt.bind(this) })- .compose({ action: (context) => this.createBaseContext(context) })- .compose({ action: this.resolveSession.bind(this) })- .compose({ action: this.createContextHelpers.bind(this) });+ .compose(this.checkRobotsTxt.bind(this))+ .compose(this.createBaseContext.bind(this))+ .compose(this.resolveSession.bind(this))+ .compose(this.createContextHelpers.bind(this));}private async checkRobotsTxt({ request }: { request: CrawlingRequest }) {@@ -1453,9 +1459,7 @@ export class BasicCrawler<return ContextPipeline.create<CrawlingContext>();}- private createBaseContext(context: PendingCrawlingContext) {- const deferredCleanup: (() => Promise<unknown>)[] = [];-+ private createBaseContext(context: PendingCrawlingContext, onCleanup: CleanupRegistrar) {return {id: cryptoRandomObjectId(10),log: this.log,@@ -1463,7 +1467,13 @@ export class BasicCrawler<useState: this.useState.bind(this),getKeyValueStore: async (identifier?: string | StorageIdentifier) => KeyValueStore.open(identifier),registerDeferredCleanup: (cleanup: () => Promise<unknown>) => {- deferredCleanup.push(cleanup);+ onCleanup(async () => {+ try {+ await cleanup();+ } catch (error) {+ this.log.debug('Error in deferred cleanup', { error });+ }+ });},afterStorageCommit: (callback: (error?: Error) => Awaitable<void>) => {const transaction = currentStorageTransaction();@@ -1491,7 +1501,6 @@ export class BasicCrawler<context[navigationDeadlineKey] += extraMillis;}},- [deferredCleanupKey]: deferredCleanup,};}@@ -1524,7 +1533,7 @@ export class BasicCrawler<const existingSession = await this.sessionPool.getSession(request.sessionId);if (!existingSession) {- throw new ContextPipelineInitializationError(new MissingSessionError(request.sessionId));+ throw new MissingSessionError(request.sessionId);}return existingSession;@@ -1567,7 +1576,7 @@ export class BasicCrawler<let contextPipeline: ContextPipeline<CrawlingContext, Context>;if (extendContext !== undefined) {contextPipeline = ContextPipeline.create<CrawlingContext>()- .compose({ action: async (context) => await extendContext(context) })+ .compose(async (context) => await extendContext(context)).chain(subclassPipeline as unknown as ContextPipeline<CrawlingContext & ContextExtension,@@ -1578,23 +1587,21 @@ export class BasicCrawler<contextPipeline = subclassPipeline;}- contextPipeline = contextPipeline.compose({- action: async (context) => {- const { request } = context;- if (request && !this.requestMatchesEnqueueStrategy(request)) {- // eslint-disable-next-line dot-notation- const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;- this.log.debug(message);+ contextPipeline = contextPipeline.compose(async (context) => {+ const { request } = context;+ if (request && !this.requestMatchesEnqueueStrategy(request)) {+ // eslint-disable-next-line dot-notation+ const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;+ this.log.debug(message);- request.noRetry = true;- request.state = RequestState.SKIPPED;+ request.noRetry = true;+ request.state = RequestState.SKIPPED;- await this.#handleSkippedRequest({ request, reason: 'redirect' });+ await this.#handleSkippedRequest({ request, reason: 'redirect' });- throw new ContextPipelineInterruptedError(message);- }- return context;- },+ throw new ContextPipelineInterruptedError(message);+ }+ return context;});return contextPipeline as ContextPipeline<CrawlingContext, ExtendedContext>;@@ -2796,14 +2803,10 @@ export class BasicCrawler</*** Unwraps errors thrown by the context pipeline to get the actual user error.- * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.+ * RequestHandlerError and ContextPipelineCleanupError wrap the actual error.*/private unwrapError(error: unknown): Error {- if (- error instanceof RequestHandlerError ||- error instanceof ContextPipelineInitializationError ||- error instanceof ContextPipelineCleanupError- ) {+ if (error instanceof RequestHandlerError || error instanceof ContextPipelineCleanupError) {return this.unwrapError(error.cause);}return error as Error;diff --git a/packages/basic-crawler/src/internals/crawlers/context_pipeline.ts b/packages/basic-crawler/src/internals/crawlers/context_pipeline.tsindex d88dac8fd..7d383525c 100644--- a/packages/basic-crawler/src/internals/crawlers/context_pipeline.ts+++ b/packages/basic-crawler/src/internals/crawlers/context_pipeline.ts@@ -1,25 +1,25 @@import { serviceLocator, SessionError } from '@crawlee/core';import type { Awaitable } from '@crawlee/types';-import {- ContextPipelineCleanupError,- ContextPipelineInitializationError,- ContextPipelineInterruptedError,- RequestHandlerError,-} from '../errors.js';+import { ContextPipelineCleanupError, RequestHandlerError } from '../errors.js';/**- * Represents a middleware step in the context pipeline.+ * Registers a callback that runs after the final context consumer finishes. `error` is the consumer's failure, if any;+ * failures of downstream middlewares are not reported. Cleanups run in reverse registration order.+ */+export type CleanupRegistrar = (cleanup: (error?: unknown) => Awaitable<void>) => void;++/**+ * A middleware step in the context pipeline: receives the context built so far and returns an extension that gets+ * merged into it. Anything that needs tearing down is registered via `onCleanup`.** @template TCrawlingContext - The input context type for this middleware* @template TCrawlingContextExtension - The enhanced output context type*/-export interface ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> {- /** The main middleware function that enhances the context */- action: (context: TCrawlingContext) => Awaitable<TCrawlingContextExtension>;- /** Optional cleanup function called after the consumer finishes or fails */- cleanup?: (context: TCrawlingContext & TCrawlingContextExtension, error?: unknown) => Awaitable<void>;-}+export type ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> = (+ context: TCrawlingContext,+ onCleanup: CleanupRegistrar,+) => Awaitable<TCrawlingContextExtension>;/*** Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.@@ -39,7 +39,7 @@ export abstract class ContextPipeline<TContextBase, TCrawlingContext extends TCo* @returns A new ContextPipeline instance with no transformations*/static create<TContextBase>(): ContextPipeline<TContextBase, TContextBase> {- return new ContextPipelineImpl<TContextBase, TContextBase>({ action: async (context) => context });+ return new ContextPipelineImpl<TContextBase, TContextBase>(async (context) => context);}/**@@ -78,9 +78,9 @@ export abstract class ContextPipeline<TContextBase, TCrawlingContext extends TCo** @param crawlingContext - The initial context to process through the pipeline* @param finalContextConsumer - The function that will receive the final enhanced context+ * @param onInitializationError - Receives a middleware failure as-is; the consumer is then skipped. Runs before+ * the cleanups, so it still sees whatever the failed middleware's predecessors set up.*- * @throws {ContextPipelineInitializationError} When a middleware fails during initialization- * @throws {ContextPipelineInterruptedError} When the pipeline is intentionally interrupted during initialization* @throws {RequestHandlerError} When the final context consumer throws an exception* @throws {ContextPipelineCleanupError} When cleanup operations fail* @throws {SessionError} Session errors are re-thrown as-is for special handling@@ -88,6 +88,7 @@ export abstract class ContextPipeline<TContextBase, TCrawlingContext extends TCoabstract call(crawlingContext: TContextBase,finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>,+ onInitializationError: (error: unknown) => Awaitable<void>,): Promise<void>;}@@ -153,15 +154,19 @@ class ContextPipelineImpl<TContextBase, TCrawlingContext extends TContextBase> easync call(crawlingContext: TContextBase,finalContextConsumer: (finalContext: TCrawlingContext) => Promise<unknown>,+ onInitializationError: (error: unknown) => Awaitable<void>,): Promise<void> {const middlewares = Array.from(this.middlewareChain()).reverse();- const cleanupStack = [];+ const cleanupStack: Parameters<CleanupRegistrar>[0][] = [];+ const onCleanup: CleanupRegistrar = (cleanup) => {+ cleanupStack.push(cleanup);+ };let consumerException: unknown | undefined;try {- for (const { action, cleanup } of middlewares) {+ for (const middleware of middlewares) {try {- const contextExtension = await action(crawlingContext);+ const contextExtension = await middleware(crawlingContext, onCleanup);const extensionNames = [...Object.getOwnPropertyNames(contextExtension),@@ -183,19 +188,9 @@ class ContextPipelineImpl<TContextBase, TCrawlingContext extends TContextBase> e.debug(`Context pipeline failed to define property ${key.toString()}:`, error);}}-- if (cleanup) {- cleanupStack.push(cleanup);- }} catch (exception: unknown) {- if (exception instanceof SessionError) {- throw exception; // Session errors are re-thrown as-is- }- if (exception instanceof ContextPipelineInterruptedError) {- throw exception;- }-- throw new ContextPipelineInitializationError(exception);+ await onInitializationError(exception);+ return;}}@@ -212,7 +207,7 @@ class ContextPipelineImpl<TContextBase, TCrawlingContext extends TContextBase> e} finally {try {for (const cleanup of cleanupStack.reverse()) {- await cleanup(crawlingContext, consumerException);+ await cleanup(consumerException);}} catch (exception: unknown) {// eslint-disable-next-line no-unsafe-finallydiff --git a/packages/basic-crawler/src/internals/errors.ts b/packages/basic-crawler/src/internals/errors.tsindex 8cdc1334e..27e943be9 100644--- a/packages/basic-crawler/src/internals/errors.ts+++ b/packages/basic-crawler/src/internals/errors.ts@@ -56,12 +56,6 @@ export class ContextPipelineInterruptedError extends Error {}}-export class ContextPipelineInitializationError extends Error {- constructor(error: unknown, options?: ErrorOptions) {- super(undefined, { cause: error, ...options });- }-}-export class ContextPipelineCleanupError extends CriticalError {constructor(error: unknown, options?: ErrorOptions) {super(undefined, { cause: error, ...options });diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 3d093a99d..60196ee81 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -1,7 +1,7 @@import type {AddRequestsBatchedResult,BasicCrawlerOptions,- BasicCrawlingContext,+ CleanupRegistrar,ContextMiddleware,CrawlingContext,EnqueueLinksOptions,@@ -30,7 +30,7 @@ import {validators,} from '@crawlee/basic';import type { CommonPage, CrawlerRemoteBrowserOptions } from '@crawlee/browser-pool';-import type { Awaitable, Cookie as CookieObject, Dictionary, IBrowserPool, ISession } from '@crawlee/types';+import type { Awaitable, Cookie as CookieObject, Dictionary, IBrowserPool } from '@crawlee/types';import {CLOUDFLARE_RETRY_CSS_SELECTORS,parseArgument,@@ -458,11 +458,12 @@ export abstract class BrowserCrawler<assertBrowserPoolNotConfigured(new.target.name, { remoteBrowser });}- const skipGuard = <Ctx extends Context>(- action: (ctx: Ctx) => Awaitable<void | Partial<Ctx>>,- ): ContextMiddleware<Ctx, Partial<Ctx>> => ({- action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),- });+ const skipGuard =+ <Ctx extends Context>(+ action: (ctx: Ctx) => Awaitable<void | Partial<Ctx>>,+ ): ContextMiddleware<Ctx, Partial<Ctx>> =>+ async (ctx) =>+ ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {});super({...basicCrawlerOptions,@@ -488,7 +489,7 @@ export abstract class BrowserCrawler<);});- let pipeline = contextPipelineBuilder().compose({ action: this.prepareNavigation.bind(this) });+ let pipeline = contextPipelineBuilder().compose(this.prepareNavigation.bind(this));for (const hook of this.#preNavigationHooks) {pipeline = pipeline.compose(windowGuard(hook));@@ -502,8 +503,8 @@ export abstract class BrowserCrawler<return pipeline.compose(skipGuard(this.finalizeNavigation.bind(this)))- .compose({ action: this.handleBlockedRequestByContent.bind(this) })- .compose({ action: this.restoreRequestState.bind(this) });+ .compose(this.handleBlockedRequestByContent.bind(this))+ .compose(this.restoreRequestState.bind(this));},extendContext,});@@ -531,26 +532,7 @@ export abstract class BrowserCrawler<CrawlingContext,BrowserCrawlingContext<Page, Response, Dictionary>> {- return ContextPipeline.create<CrawlingContext>().compose({- action: this.preparePage.bind(this),- cleanup: async (context: {- page: Page;- session: ISession;- registerDeferredCleanup: BasicCrawlingContext['registerDeferredCleanup'];- }) => {- context.registerDeferredCleanup(async () => {- const error = !context.session.isUsable()- ? new SessionError('Session is no longer usable')- : undefined;-- await this.browserPool- .closePage(context.page, { error })- .catch((closeError: Error) =>- this.log.debug('Error while closing page', { error: closeError }),- );- });- },- });+ return ContextPipeline.create<CrawlingContext>().compose(this.preparePage.bind(this));}private async containsSelectors(page: CommonPage, selectors: string[]): Promise<string[] | null> {@@ -587,6 +569,7 @@ export abstract class BrowserCrawler<private async preparePage(crawlingContext: CrawlingContext,+ onCleanup: CleanupRegistrar,): Promise<ContextDifference<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>> {const page = await this.browserPool.newPage({id: crawlingContext.id,@@ -594,6 +577,18 @@ export abstract class BrowserCrawler<});tryCancel();+ onCleanup(() => {+ crawlingContext.registerDeferredCleanup(async () => {+ const error = !crawlingContext.session.isUsable()+ ? new SessionError('Session is no longer usable')+ : undefined;++ await this.browserPool+ .closePage(page, { error })+ .catch((closeError: Error) => this.log.debug('Error while closing page', { error: closeError }));+ });+ });+const addRequests = crawlingContext.addRequests;const extractLinks = async (options?: ExtractLinksOptions): Promise<string[]> => {diff --git a/packages/http-crawler/src/internals/dom-crawler.ts b/packages/http-crawler/src/internals/dom-crawler.tsindex 6e0f4b243..ef5346bca 100644--- a/packages/http-crawler/src/internals/dom-crawler.ts+++ b/packages/http-crawler/src/internals/dom-crawler.ts@@ -1,5 +1,6 @@import type {AddRequestsBatchedResult,+ CleanupRegistrar,ContextPipeline,CrawlingContext,EnqueueLinksOptions,@@ -192,22 +193,17 @@ export class DOMCrawler<protected override buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed>> {return super.buildContextPipeline()- .compose({- action: async (context) => this.#parseContent(context),- cleanup: async (context) => {- // The `skipNavigation` placeholders below throw on access, so there is nothing to clean up.- if (!context.request.skipNavigation) {- await this.#parser.cleanup?.(context as unknown as Parsed);- }- },- })- .compose({ action: async (context) => this.#addHelpers(context) });+ .compose(this.#parseContent.bind(this))+ .compose(async (context) => this.#addHelpers(context));}- async #parseContent(context: InternalHttpCrawlingContext): Promise<Parsed> {+ async #parseContent(context: InternalHttpCrawlingContext, onCleanup: CleanupRegistrar): Promise<Parsed> {try {- return await this.#parser.parse(context);+ const parsed = await this.#parser.parse(context);+ onCleanup(() => this.#parser.cleanup?.(parsed));+ return parsed;} catch (err) {+ // The `skipNavigation` placeholders below throw on access, so there is nothing to clean up.if (err instanceof NavigationSkippedError) {return Object.defineProperties({},diff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.tsindex ca7966d69..00c101b01 100644--- a/packages/http-crawler/src/internals/file-download.ts+++ b/packages/http-crawler/src/internals/file-download.ts@@ -2,6 +2,7 @@ import { Transform } from 'node:stream';import type {BasicCrawlerOptions,+ CleanupRegistrar,ContextPipeline,CrawlingContext,CrawlingRequest,@@ -24,8 +25,6 @@ import type {import { Router } from '../index.js';import { parseContentTypeFromResponse } from './utils.js';-const kBodyDrained = Symbol('bodyDrained');-export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerContextExtension = Dictionary<never>,@@ -182,21 +181,10 @@ export class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {}protected override buildContextPipeline(): ContextPipeline<CrawlingContext, FileDownloadCrawlingContext> {- return super.buildContextPipeline().compose({- action: async (context) => this.initiateDownload(context),- cleanup: async (context) => {- if (!context.response.bodyUsed) {- // Nobody consumed the body — cancel it so the- // underlying connection can be released.- await context.response.body?.cancel();- }-- await (context as { [kBodyDrained]: Promise<void> })[kBodyDrained];- },- });+ return super.buildContextPipeline().compose(this.initiateDownload.bind(this));}- private async initiateDownload(context: CrawlingContext) {+ private async initiateDownload(context: CrawlingContext, onCleanup: CleanupRegistrar) {const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {session: context.session,});@@ -207,14 +195,21 @@ export class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {const { response: trackedResponse, bodyDrained } = trackBodyConsumption(response);- const contextExtension = {+ onCleanup(async () => {+ if (!trackedResponse.bodyUsed) {+ // Nobody consumed the body — cancel it so the+ // underlying connection can be released.+ await trackedResponse.body?.cancel();+ }++ await bodyDrained;+ });++ return {request: context.request as LoadedRequest<CrawlingRequest>,response: trackedResponse,contentType: { type, encoding },- [kBodyDrained]: bodyDrained,};-- return contextExtension;}}diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex f3049ed46..9667de241 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -457,11 +457,12 @@ export class HttpCrawler<protected override buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext> {// When navigation is skipped, `prepareHttpRequest` has already installed throwing getters for// the response-derived members, so the guarded action is bypassed and the context left untouched.- const skipGuard = <Ctx extends CrawlingContext, Ext>(- action: (ctx: Ctx) => Awaitable<void | Ext>,- ): ContextMiddleware<Ctx, Ext> => ({- action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})) as Ext,- });+ const skipGuard =+ <Ctx extends CrawlingContext, Ext>(+ action: (ctx: Ctx) => Awaitable<void | Ext>,+ ): ContextMiddleware<Ctx, Ext> =>+ async (ctx) =>+ (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})) as Ext;// A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation// hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same@@ -478,9 +479,7 @@ export class HttpCrawler<return addTimeoutToPromise(async () => step(ctx), remaining, navigationTimedOut);});- let pipeline = ContextPipeline.create<CrawlingContext>().compose({- action: this.prepareHttpRequest.bind(this),- });+ let pipeline = ContextPipeline.create<CrawlingContext>().compose(this.prepareHttpRequest.bind(this));for (const hook of this.#preNavigationHooks) {pipeline = pipeline.compose(windowGuard(hook));@@ -493,8 +492,8 @@ export class HttpCrawler<}return pipelineWithNavigation- .compose({ action: this.processHttpResponse.bind(this) })- .compose({ action: this.handleBlockedRequestByContent.bind(this) });+ .compose(this.processHttpResponse.bind(this))+ .compose(this.handleBlockedRequestByContent.bind(this));}private async prepareHttpRequest(crawlingContext: CrawlingContext): Promise<Partial<CrawlingContextWithResponse>> {diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex b1e9df28e..a346c723d 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -494,13 +494,13 @@ export class AdaptivePlaywrightCrawler<this.#staticCrawler = staticCrawler;this.#browserCrawler = browserCrawler;- this.#staticContextPipeline = staticCrawler.contextPipeline.compose({- action: this.adaptCheerioContext.bind(this),- }) as unknown as ContextPipeline<CrawlingContext, ExtendedContext>;+ this.#staticContextPipeline = staticCrawler.contextPipeline.compose(+ this.adaptCheerioContext.bind(this),+ ) as unknown as ContextPipeline<CrawlingContext, ExtendedContext>;- this.#browserContextPipeline = browserCrawler.contextPipeline.compose({- action: this.adaptPlaywrightContext.bind(this),- }) as unknown as ContextPipeline<CrawlingContext, ExtendedContext>;+ this.#browserContextPipeline = browserCrawler.contextPipeline.compose(+ this.adaptPlaywrightContext.bind(this),+ ) as unknown as ContextPipeline<CrawlingContext, ExtendedContext>;}protected override async init(): Promise<void> {@@ -516,34 +516,32 @@ export class AdaptivePlaywrightCrawler<const errorMessage = (prop: string) =>`The \`${prop}\` property is not available on the outer context pipeline of AdaptivePlaywrightCrawler - it is provided by the inner (static/browser) pipelines`;- return super.buildContextPipeline().compose({- action: async ({ request }) => ({- get request(): LoadedRequest<CrawlingRequest<Dictionary>> {- return request as LoadedRequest<CrawlingRequest<Dictionary>>;- },- get response(): Response {- throw new Error(errorMessage('response'));- },- get page(): Page {- throw new Error(errorMessage('page'));- },- get querySelector(): AdaptivePlaywrightCrawlerContext['querySelector'] {- throw new Error(errorMessage('querySelector'));- },- get querySelectorAll(): AdaptivePlaywrightCrawlerContext['querySelectorAll'] {- throw new Error(errorMessage('querySelectorAll'));- },- get waitForSelector(): AdaptivePlaywrightCrawlerContext['waitForSelector'] {- throw new Error(errorMessage('waitForSelector'));- },- get parseWithCheerio(): AdaptivePlaywrightCrawlerContext['parseWithCheerio'] {- throw new Error(errorMessage('parseWithCheerio'));- },- get enqueueLinks(): AdaptivePlaywrightCrawlerContext['enqueueLinks'] {- throw new Error(errorMessage('enqueueLinks'));- },- }),- });+ return super.buildContextPipeline().compose(async ({ request }) => ({+ get request(): LoadedRequest<CrawlingRequest<Dictionary>> {+ return request as LoadedRequest<CrawlingRequest<Dictionary>>;+ },+ get response(): Response {+ throw new Error(errorMessage('response'));+ },+ get page(): Page {+ throw new Error(errorMessage('page'));+ },+ get querySelector(): AdaptivePlaywrightCrawlerContext['querySelector'] {+ throw new Error(errorMessage('querySelector'));+ },+ get querySelectorAll(): AdaptivePlaywrightCrawlerContext['querySelectorAll'] {+ throw new Error(errorMessage('querySelectorAll'));+ },+ get waitForSelector(): AdaptivePlaywrightCrawlerContext['waitForSelector'] {+ throw new Error(errorMessage('waitForSelector'));+ },+ get parseWithCheerio(): AdaptivePlaywrightCrawlerContext['parseWithCheerio'] {+ throw new Error(errorMessage('parseWithCheerio'));+ },+ get enqueueLinks(): AdaptivePlaywrightCrawlerContext['enqueueLinks'] {+ throw new Error(errorMessage('enqueueLinks'));+ },+ }));}private async adaptCheerioContext(cheerioContext: CheerioCrawlingContext) {@@ -649,11 +647,15 @@ export class AdaptivePlaywrightCrawler<}try {+ // Any failure - middleware or handler - ends up in the `ok: false` branch below.+ const rethrow = (error: unknown) => {+ throw error;+ };const callAdaptiveRequestHandler = async () => {if (renderingType === 'static') {- await this.#staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));+ await this.#staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this), rethrow);} else if (renderingType === 'clientOnly') {- await this.#browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));+ await this.#browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this), rethrow);}};diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex 1f35c39d4..b89dbc08b 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -275,7 +275,7 @@ export class PlaywrightCrawler<}protected override buildContextPipeline(): ContextPipeline<CrawlingContext, PlaywrightCrawlingContext> {- return super.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) });+ return super.buildContextPipeline().compose(this.enhanceContext.bind(this));}protected override async navigationHandler(diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex 7d6292823..a46bcfa97 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -263,7 +263,7 @@ export class PuppeteerCrawler<}protected override buildContextPipeline(): ContextPipeline<CrawlingContext, PuppeteerCrawlingContext> {- return super.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) });+ return super.buildContextPipeline().compose(this.enhanceContext.bind(this));}private async enhanceContext(context: BrowserCrawlingContext<Page, HTTPResponse>) {diff --git a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts b/packages/stagehand-crawler/src/internals/stagehand-crawler.tsindex 150d67ddc..4b50c035c 100644--- a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts+++ b/packages/stagehand-crawler/src/internals/stagehand-crawler.ts@@ -484,7 +484,7 @@ export class StagehandCrawler<}protected override buildContextPipeline(): ContextPipeline<CrawlingContext, StagehandCrawlingContext> {- return super.buildContextPipeline().compose({ action: this.setUpStagehand.bind(this) });+ return super.buildContextPipeline().compose(this.setUpStagehand.bind(this));}/**diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 3c73f8e9b..59dd56c3a 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -2592,6 +2592,29 @@ describe('BasicCrawler', () => {expect(requestHandlerImplementation.mock.calls[0][0]).toMatchObject({ hello: 'world' });});+ test('registerDeferredCleanup runs after the error handler, even when the context pipeline itself fails', async () => {+ const order: string[] = [];++ const crawler = new BasicCrawler({+ maxRequestRetries: 0,+ extendContext: ({ registerDeferredCleanup }) => {+ registerDeferredCleanup(async () => {+ order.push('cleanup');+ });+ throw new Error('extendContext failed');+ },+ requestHandler: async () => {+ order.push('requestHandler');+ },+ failedRequestHandler: async () => {+ order.push('failedRequestHandler');+ },+ });++ await crawler.run(['https://example.com']);+ expect(order).toEqual(['failedRequestHandler', 'cleanup']);+ });+describe('sendRequest', () => {const html = `<!DOCTYPE html><html><head><title>foobar</title></head><body><p>Hello, world!</p></body></html>`;diff --git a/test/core/crawlers/context_pipeline.test.ts b/test/core/crawlers/context_pipeline.test.tsindex cd86619e9..8d96e9feb 100644--- a/test/core/crawlers/context_pipeline.test.ts+++ b/test/core/crawlers/context_pipeline.test.ts@@ -1,119 +1,84 @@-import {- ContextPipeline,- ContextPipelineCleanupError,- ContextPipelineInitializationError,- ContextPipelineInterruptedError,- RequestHandlerError,-} from '@crawlee/basic';+import { ContextPipeline, ContextPipelineCleanupError, RequestHandlerError } from '@crawlee/basic';import { describe, expect, it, vi } from 'vitest';+const rethrow = (error: unknown) => {+ throw error;+};+describe('ContextPipeline', () => {it('should call middlewares in a sequence', async () => {const pipeline = ContextPipeline.create()- .compose({- action: async () => ({ a: 2, b: 1, c: [1] }),- })- .compose({- action: async (context) => ({ a: context.a * 2, c: [...context.c, 2] }),- });+ .compose(async () => ({ a: 2, b: 1, c: [1] }))+ .compose(async (context) => ({ a: context.a * 2, c: [...context.c, 2] }));const consumer = vi.fn();- await pipeline.call({}, consumer);+ await pipeline.call({}, consumer, rethrow);expect(consumer).toHaveBeenCalledWith({ a: 4, b: 1, c: [1, 2] });});- it('should call cleanup routines', async () => {+ it('should call cleanup routines in reverse order, with access to the middleware closure', async () => {const pipeline = ContextPipeline.create()- .compose({- action: async () => ({ c: [] as number[] }),- cleanup: async (context) => {- context.c.push(1);- },+ .compose(async (_, onCleanup) => {+ const c: number[] = [];+ onCleanup(() => {+ c.push(1);+ });+ return { c };})- .compose({- action: async () => ({}),- cleanup: async (context) => {+ .compose(async (context, onCleanup) => {+ onCleanup(() => {context.c.push(2);- },+ });+ return {};});const consumer = vi.fn();- await pipeline.call({}, consumer);+ await pipeline.call({}, consumer, rethrow);expect(consumer).toHaveBeenCalledWith({ c: [2, 1] });});- it('should allow interrupting the pipeline in middlewares', async () => {- const context = { a: 3 };-- const firstAction = vi.fn().mockResolvedValue({});- const firstCleanup = vi.fn();- const secondAction = vi.fn().mockRejectedValue(new ContextPipelineInterruptedError());- const secondCleanup = vi.fn();- const thirdAction = vi.fn().mockResolvedValue({});- const thirdCleanup = vi.fn();-- const pipeline = ContextPipeline.create()- .compose({ action: firstAction, cleanup: firstCleanup })- .compose({- action: secondAction,- cleanup: secondCleanup,- })- .compose({ action: thirdAction, cleanup: thirdCleanup });-- const consumer = vi.fn();-- await expect(pipeline.call(context, consumer)).rejects.toThrow(ContextPipelineInterruptedError);-- expect(firstAction).toHaveBeenCalled();- expect(firstCleanup).toHaveBeenCalled();- expect(secondAction).toHaveBeenCalled();- expect(secondCleanup).not.toHaveBeenCalled();- expect(thirdAction).not.toHaveBeenCalled();- expect(thirdCleanup).not.toHaveBeenCalled();- expect(consumer).not.toHaveBeenCalled();- });-- it('should wrap pipeline initialization errors', async () => {+ it('should hand initialization errors to onInitializationError before running cleanups', async () => {const initializationError = new Error('Pipeline initialization failed');- const context = { a: 3 };- const secondMiddleware = vi.fn();+ const order: string[] = [];+ const thirdMiddleware = vi.fn();const pipeline = ContextPipeline.create()- .compose({- action: async () => {- throw initializationError;- },+ .compose(async (_, onCleanup) => {+ onCleanup(() => {+ order.push('cleanup');+ });+ return {};})- .compose({ action: secondMiddleware });+ .compose(async () => {+ throw initializationError;+ })+ .compose(thirdMiddleware);const consumer = vi.fn();+ const onInitializationError = vi.fn(async (error: unknown) => {+ order.push('onInitializationError');+ expect(error).toBe(initializationError);+ });- await expect(pipeline.call(context, consumer)).rejects.toThrow(- expect.objectContaining({- cause: initializationError,- constructor: ContextPipelineInitializationError,- }),- );+ await pipeline.call({}, consumer, onInitializationError);+ expect(onInitializationError).toHaveBeenCalledOnce();+ expect(thirdMiddleware).not.toHaveBeenCalled();expect(consumer).not.toHaveBeenCalled();- expect(secondMiddleware).not.toHaveBeenCalled();+ expect(order).toEqual(['onInitializationError', 'cleanup']);});it('should wrap errors in the final consumer', async () => {const consumerError = new Error('Request handler failed');const context = { a: 3 };- const pipeline = ContextPipeline.create().compose({- action: async () => ({- b: 4,- }),- });+ const pipeline = ContextPipeline.create().compose(async () => ({ b: 4 }));const consumer = vi.fn().mockRejectedValue(consumerError);- await expect(pipeline.call(context, consumer)).rejects.toThrow(+ await expect(pipeline.call(context, consumer, rethrow)).rejects.toThrow(expect.objectContaining({cause: consumerError,constructor: RequestHandlerError,@@ -123,39 +88,48 @@ describe('ContextPipeline', () => {expect(consumer).toHaveBeenCalledWith({ a: 3, b: 4 });});- it('should call cleanup routines even if the final consumer fails', async () => {+ it('should pass the consumer error to cleanup routines', async () => {const consumerError = new Error('Request handler failed');const context = { a: 3 };const cleanup = vi.fn();- const pipeline = ContextPipeline.create().compose({- action: async () => ({- b: 4,- }),- cleanup,+ const pipeline = ContextPipeline.create().compose(async (_, onCleanup) => {+ onCleanup(cleanup);+ return { b: 4 };+ });++ await expect(pipeline.call(context, vi.fn().mockRejectedValue(consumerError), rethrow)).rejects.toThrow();++ expect(cleanup).toHaveBeenCalledWith(consumerError);+ });++ it('should call cleanup routines without an error if the consumer succeeds', async () => {+ const cleanup = vi.fn();++ const pipeline = ContextPipeline.create().compose(async (_, onCleanup) => {+ onCleanup(cleanup);+ return {};});- await expect(pipeline.call(context, vi.fn().mockRejectedValue(consumerError))).rejects.toThrow();+ await pipeline.call({}, vi.fn(), rethrow);- expect(cleanup).toHaveBeenCalledWith({ a: 3, b: 4 }, consumerError);+ expect(cleanup).toHaveBeenCalledWith(undefined);});it('should wrap cleanup errors', async () => {const cleanupError = new Error('Pipeline cleanup failed');const context = { a: 3 };- const pipeline = ContextPipeline.create().compose({- action: async () => ({- b: 4,- }),- cleanup: async () => {+ const pipeline = ContextPipeline.create().compose(async (_, onCleanup) => {+ onCleanup(async () => {throw cleanupError;- },+ });+ return { b: 4 };});const consumer = vi.fn();- await expect(pipeline.call(context, consumer)).rejects.toThrow(+ await expect(pipeline.call(context, consumer, rethrow)).rejects.toThrow(expect.objectContaining({cause: cleanupError,constructor: ContextPipelineCleanupError,@@ -169,27 +143,26 @@ describe('ContextPipeline', () => {const context = {} as Record<string, unknown>;Object.defineProperty(context, 'frozen', { value: 'original', configurable: false });- const pipeline = ContextPipeline.create<typeof context>().compose({- action: async () => ({ frozen: 'overridden', other: 'new' }),- });+ const pipeline = ContextPipeline.create<typeof context>().compose(async () => ({+ frozen: 'overridden',+ other: 'new',+ }));const consumer = vi.fn();- await pipeline.call(context, consumer);+ await pipeline.call(context, consumer, rethrow);expect(consumer).toHaveBeenCalledWith(expect.objectContaining({ frozen: 'original', other: 'new' }));});describe('chain', () => {it('should run middlewares from both pipelines in order', async () => {- const first = ContextPipeline.create<{ a: number }>().compose({- action: async (ctx) => ({ b: ctx.a + 1 }),- });- const second = ContextPipeline.create<{ a: number; b: number }>().compose({- action: async (ctx) => ({ c: ctx.b * 2 }),- });+ const first = ContextPipeline.create<{ a: number }>().compose(async (ctx) => ({ b: ctx.a + 1 }));+ const second = ContextPipeline.create<{ a: number; b: number }>().compose(async (ctx) => ({+ c: ctx.b * 2,+ }));const consumer = vi.fn();- await first.chain(second).call({ a: 1 }, consumer);+ await first.chain(second).call({ a: 1 }, consumer, rethrow);expect(consumer).toHaveBeenCalledWith({ a: 1, b: 2, c: 4 });});@@ -197,20 +170,20 @@ describe('ContextPipeline', () => {it('should call cleanup routines from both pipelines', async () => {const order: string[] = [];- const first = ContextPipeline.create<object>().compose({- action: async () => ({}),- cleanup: async () => {+ const first = ContextPipeline.create<object>().compose(async (_, onCleanup) => {+ onCleanup(() => {order.push('first');- },+ });+ return {};});- const second = ContextPipeline.create<object>().compose({- action: async () => ({}),- cleanup: async () => {+ const second = ContextPipeline.create<object>().compose(async (_, onCleanup) => {+ onCleanup(() => {order.push('second');- },+ });+ return {};});- await first.chain(second).call({}, vi.fn());+ await first.chain(second).call({}, vi.fn(), rethrow);expect(order).toEqual(['second', 'first']);});