Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

refactor: Improve the way IBrowserPool.closePage contract receives session errors (#4161)

Jan Buchar438e3419626b
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md
index c0ab78338..e4e2a8024 100644
--- a/docs/public-api/crawlee-basic.api.md
+++ b/docs/public-api/crawlee-basic.api.md
@@ -225,7 +225,7 @@ export type ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> = (co
// @public
export abstract class ContextPipeline<TContextBase, TCrawlingContext extends TContextBase> {
- abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>, onInitializationError: (error: unknown) => Awaitable<void>): Promise<void>;
+ abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>, onError: (error: unknown) => Awaitable<void>): Promise<void>;
abstract chain<TFinalContext extends TCrawlingContext>(other: ContextPipeline<TCrawlingContext, TFinalContext>): ContextPipeline<TContextBase, TFinalContext>;
abstract compose<TCrawlingContextExtension>(middleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension>;
static create<TContextBase>(): ContextPipeline<TContextBase, TContextBase>;
@@ -236,6 +236,11 @@ export class ContextPipelineCleanupError extends CriticalError {
constructor(error: unknown, options?: ErrorOptions);
}
+// @public
+export class ContextPipelineInitializationError extends Error {
+ constructor(error: unknown, options?: ErrorOptions);
+}
+
// @public (undocumented)
export class ContextPipelineInterruptedError extends Error {
constructor(message?: string);
@@ -586,7 +591,7 @@ export interface RegExpObject {
// @public (undocumented)
export type RequestHandler<Context extends CrawlingContext = CrawlingContext> = (inputs: Context) => Awaitable<void>;
-// @public (undocumented)
+// @public
export class RequestHandlerError extends Error {
constructor(error: unknown, options?: ErrorOptions);
}
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md
index d60880e67..e09ce53fe 100644
--- a/docs/public-api/crawlee-core.api.md
+++ b/docs/public-api/crawlee-core.api.md
@@ -900,6 +900,10 @@ export class SessionError extends Error {
constructor(message?: string);
}
+// @public
+export class SessionRetiredError extends SessionError {
+}
+
// @public (undocumented)
export type Source = (Partial<RequestOptions> & {
requestsFromUrl?: string;
diff --git a/docs/public-api/crawlee-types.api.md b/docs/public-api/crawlee-types.api.md
index b70612959..6cacc61df 100644
--- a/docs/public-api/crawlee-types.api.md
+++ b/docs/public-api/crawlee-types.api.md
@@ -202,6 +202,7 @@ export interface ISession {
fingerprint?: SessionFingerprint;
// (undocumented)
readonly id: string;
+ isBlocked(): boolean;
isUsable(): boolean;
markBad(): void;
markGood(): void;
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md
index a6e871a57..6c338619e 100644
--- a/docs/upgrading/upgrading_v4.md
+++ b/docs/upgrading/upgrading_v4.md
@@ -1104,7 +1104,7 @@ const crawler = new BasicCrawler({
});
```
-The crawler depends only on the `ISession` interface — `id`, `cookieJar`, `proxyInfo`, `fingerprint`, and the `isUsable()` / `markGood()` / `markBad()` / `retire()` methods — so a custom pool may hand out its own session implementation instead of instances of the built-in `Session` class. `Session` implements `ISession`, so returning `Session` instances (as above) is the shortest path; `crawlingContext.session` is typed as `ISession` either way.
+The crawler depends only on the `ISession` interface — `id`, `cookieJar`, `proxyInfo`, `fingerprint`, and the `isUsable()` / `isBlocked()` / `markGood()` / `markBad()` / `retire()` methods — so a custom pool may hand out its own session implementation instead of instances of the built-in `Session` class. `Session` implements `ISession`, so returning `Session` instances (as above) is the shortest path; `crawlingContext.session` is typed as `ISession` either way.
Returning `undefined` means the pool has no usable session for the request. The crawler turns that into a `MissingSessionError` and retries the request like any other failure.
@@ -1229,7 +1229,7 @@ Applies when you construct a `BrowserPool` yourself, reach for `browserControlle
Browser crawlers now accept any object implementing the new `IBrowserPool` interface as their `browserPool` option, not just instances of the built-in `BrowserPool`. The interface follows the classic acquire/release pattern, plus a pair of helpers for moving state between the crawling session and the page:
- **`newPage(options?)`** — opens a new page. An optional `session` can be passed as a best-effort hint — the pool may use it for proxy configuration, fingerprinting, etc., but nothing is guaranteed.
-- **`closePage(page, options?)`** — signals the pool that the caller is done with the page. If the optional `error` is a `SessionError`, the pool should purge all state associated with the session (e.g. retire the underlying browser).
+- **`closePage(page, options?)`** — signals the pool that the caller is done with the page. If the optional `error` is a `SessionError`, the session that served the page is finished. A plain `SessionError` means it was blocked, so the pool should purge all state associated with it (e.g. retire the underlying browser). The `SessionRetiredError` subclass means the session merely reached its `maxUsageCount` or `maxAgeSecs`, and a pool may keep a warm page instead.
- **`extractPageState(page)`** — reads the relevant state (currently cookies) out of a page so the crawler can persist it back into the session.
- **`injectPageState(page, state)`** — the counterpart to `extractPageState`; seeds a page with state (currently cookies) before navigation. Isolation between pages is best-effort and depends on the pool implementation.
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 6fcdede4a..c47e11980 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -76,6 +76,7 @@ import {
MissingSessionError,
NavigationSkippedError,
PersistentRateLimitError,
+ ContextPipelineInitializationError,
RequestHandlerError,
RequestThrottledError,
RetryRequestError,
@@ -98,6 +99,7 @@ import {
navigationDeadlineKey,
raceWithTimeout,
type RequestTimeoutContext,
+ suspendTimeoutKey,
timeoutExpiredKey,
} from './request-timeout.js';
import { createSendRequest } from './send-request.js';
@@ -1244,47 +1246,11 @@ export class BasicCrawler<
// matching `recordRequestStart`.
this.statistics.recordRequestStart(request.id || request.uniqueKey);
- const crawlingContext = { request } as { request: CrawlingRequest } & Partial<CrawlingContext>;
-
- // Runs inside the pipeline call, so the deferred cleanups registered by earlier middlewares
- // (e.g. closing the page) only fire after the error handler has seen the partial context.
- const handleInitializationError = async (rawError: unknown) => {
- // Roll back *before* the error handler runs, for the same reason `handleRequest` does.
- currentStorageTransaction()?.rollback();
-
- // ContextPipelineInterruptedError means the request was intentionally skipped
- // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
- if (rawError instanceof ContextPipelineInterruptedError) {
- this.statistics.discardRequestRecord(request.id || request.uniqueKey);
- await this.timeoutAndRetry(
- async () => this.requestManager?.markRequestAsHandled(request),
- this.internalTimeoutMillis,
- `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${
- this.internalTimeoutMillis / 1e3
- } seconds.`,
- );
- return;
- }
-
- // An error during pipeline initialization (e.g., navigation timeout, session/proxy error,
- // i.e. not in user's requestHandler) goes through the normal error flow.
- const error = this.unwrapError(rawError);
- await this.requestFunctionErrorHandler(
- error,
- crawlingContext as CrawlingContext,
- request,
- this.requestManager!,
- );
- // SessionError already retired the session in `requestFunctionErrorHandler`;
- // skip `markBad` to avoid double-counting usage/error score.
- if (!this.errorAbsolvesSession(error)) {
- crawlingContext.session?.markBad();
- }
- };
+ const crawlingContext: PendingCrawlingContext = { request };
try {
// The transaction spans the whole pipeline call, covering the navigation hooks
- // and `extendContext` too; `handleRequest` drives its outcome explicitly.
+ // and `extendContext` too; `handleRequest` commits it and `handleRequestFailure` rolls it back.
await this.runInStorageTransaction(
async () =>
// Navigation, the navigation hooks and the request handler are timed individually, but the
@@ -1292,20 +1258,17 @@ export class BasicCrawler<
// catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
await this.withRequestTimeout(
crawlingContext,
- this.#basicPipeline
- .chain(this.contextPipeline)
- .call(
- crawlingContext,
- (ctx) => this.handleRequest(ctx, source, request),
- handleInitializationError,
- ),
+ this.#basicPipeline.chain(this.contextPipeline).call(
+ crawlingContext,
+ (ctx) => this.handleRequest(ctx, source, request),
+ (error) => this.#handleRequestFailure(error, crawlingContext, source, request),
+ ),
),
);
} catch (error) {
- // A bare `TimeoutError` is the internal timeout above firing; a `SessionError` can only be a
- // secondary error from `handleRequest` - both take the normal error flow. Anything else thrown
- // inside the pipeline arrives wrapped.
- if (error instanceof SessionError || error instanceof TimeoutError) {
+ // The internal timeout above fired, or `handleRequestFailure` threw a `SessionError` - both
+ // take the normal error flow. Anything else is fatal.
+ if (error instanceof SessionError || crawlingContext[timeoutExpiredKey]?.()) {
const unwrappedError = this.unwrapError(error);
await this.requestFunctionErrorHandler(
@@ -2460,8 +2423,8 @@ export class BasicCrawler<
/**
* Runs `callback` inside a {@apilink StorageTransaction}, unless transactional storage is disabled.
- * Deliberately does **not** commit on return - `handleRequest` swallows request handler failures, so
- * a normal return says nothing about success. `handleRequest` owns the outcome.
+ * Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
+ * normal return says nothing about success. `handleRequest` commits on success.
*/
private async runInStorageTransaction<T>(callback: () => Promise<T>): Promise<T> {
if (!this.#transactionalStorageEnabled) {
@@ -2480,7 +2443,7 @@ export class BasicCrawler<
return result;
} finally {
if (transaction.state === 'open') {
- // `handleRequest` commits or rolls back on every normal path, so an open transaction on
+ // `handleRequest` commits or `handleRequestFailure` rolls back on every normal path, so an open transaction on
// a normal return is a wiring bug; on a propagating throw (a pipeline-level failure) it is
// expected. Either way, discard the unvalidated writes; only the former is worth flagging.
if (!threw) {
@@ -2636,7 +2599,7 @@ export class BasicCrawler<
return this.requestManager.fetchNextRequest();
}
- /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
+ /** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
private async handleRequest(
crawlingContext: ExtendedContext,
@@ -2650,91 +2613,123 @@ export class BasicCrawler<
return;
}
- const statisticsId = request.id || request.uniqueKey;
+ request.state = RequestState.REQUEST_HANDLER;
+ await this.runRequestHandler(crawlingContext);
+ // Commit *before* marking the request as handled, so a commit failure fails the request and
+ // it is retried. This also closes the transaction, so everything below passes through.
// Opened by `runInStorageTransaction`; absent when disabled or when the subclass opens its own.
- const transaction = currentStorageTransaction();
+ await currentStorageTransaction()?.commit();
+
+ await this.timeoutAndRetry(
+ async () => requestSource.markRequestAsHandled(request!),
+ this.internalTimeoutMillis,
+ `Marking request ${request.url} (${request.id}) as handled timed out after ${
+ this.internalTimeoutMillis / 1e3
+ } seconds.`,
+ );
- let isRequestLocked = true;
+ this.statistics.recordRequestSuccess(request.id || request.uniqueKey, request.retryCount);
- try {
- request.state = RequestState.REQUEST_HANDLER;
- await this.runRequestHandler(crawlingContext);
+ // reclaim session if request finishes successfully
+ request.state = RequestState.DONE;
+ crawlingContext.session.markGood();
+ }
- // Commit *before* marking the request as handled, so a commit failure fails the request and
- // it is retried. This also closes the transaction, so everything below passes through.
- await transaction?.commit();
+ /**
+ * Handles a request that failed anywhere in the context pipeline - a middleware (e.g. navigation or its hooks)
+ * or `handleRequest`. Runs inside the pipeline call, so the cleanups (e.g. closing the page) only fire after the
+ * error handler has seen the context.
+ */
+ async #handleRequestFailure(
+ rawError: unknown,
+ crawlingContext: PendingCrawlingContext,
+ requestSource: IRequestManager,
+ request: CrawlingRequest,
+ ) {
+ // Roll back *before* any error handler runs - error handlers write to real storage precisely
+ // because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.
+ currentStorageTransaction()?.rollback();
+
+ const skipped =
+ rawError instanceof ContextPipelineInitializationError &&
+ rawError.cause instanceof ContextPipelineInterruptedError;
+ const err = this.unwrapError(rawError);
+ const context = crawlingContext as CrawlingContext;
+ const retryCountBefore = request.retryCount;
+ const resumeRequestTimeout = crawlingContext[suspendTimeoutKey]?.();
+
+ try {
+ // ContextPipelineInterruptedError means the request was intentionally skipped
+ // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
+ if (skipped) {
+ this.statistics.discardRequestRecord(request.id || request.uniqueKey);
+ await this.timeoutAndRetry(
+ async () => requestSource.markRequestAsHandled(request),
+ this.internalTimeoutMillis,
+ `Marking request ${request.url} (${request.id}) as handled timed out after ${
+ this.internalTimeoutMillis / 1e3
+ } seconds.`,
+ );
+ return;
+ }
- await this.timeoutAndRetry(
- async () => requestSource.markRequestAsHandled(request!),
+ request.state = RequestState.ERROR_HANDLER;
+ await addTimeoutToPromise(
+ async () => this.requestFunctionErrorHandler(err, context, request, requestSource),
this.internalTimeoutMillis,
- `Marking request ${request.url} (${request.id}) as handled timed out after ${
+ `Handling request failure of ${request.url} (${request.id}) timed out after ${
this.internalTimeoutMillis / 1e3
} seconds.`,
);
- isRequestLocked = false; // markRequestAsHandled succeeded and unlocked the request
-
- this.statistics.recordRequestSuccess(statisticsId, request.retryCount);
-
- // reclaim session if request finishes successfully
request.state = RequestState.DONE;
- crawlingContext.session.markGood();
- } catch (rawError) {
- // Roll back *before* any error handler runs - error handlers write to real storage precisely
- // because the transaction is already closed. A no-op when the commit above succeeded.
- transaction?.rollback();
-
- const err = this.unwrapError(rawError);
+ } catch (secondaryError) {
+ const unwrappedSecondaryError = this.unwrapError(secondaryError);
+ const timedOut = unwrappedSecondaryError instanceof TimeoutError;
+ request.state = RequestState.ERROR;
+
+ if (timedOut) {
+ // Not worth crashing the crawler over. The request's state is unknown, so it gets retried - and the
+ // retry is counted, so that an error handler that always times out does not retry it forever.
+ this.log.warning(unwrappedSecondaryError.message);
+ if (!skipped && request.retryCount === retryCountBefore) {
+ request.retryCount++;
+ }
+ } else if (!(unwrappedSecondaryError instanceof CriticalError)) {
+ // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
+ const apifySpecific = process.env.APIFY_IS_AT_HOME
+ ? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
+ : '';
+ this.log.exception(
+ unwrappedSecondaryError,
+ 'An exception occurred during handling of failed request. ' +
+ `This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`,
+ );
+ }
+ // Safety net - the error handler neither marked the request as handled nor reclaimed it (e.g. after a
+ // CriticalError). Reclaiming a request that is no longer in progress is a harmless no-op on the storage backend.
try {
- request.state = RequestState.ERROR_HANDLER;
await addTimeoutToPromise(
- async () => this.requestFunctionErrorHandler(err, crawlingContext, request, requestSource),
+ async () => requestSource.reclaimRequest(request),
this.internalTimeoutMillis,
- `Handling request failure of ${request.url} (${request.id}) timed out after ${
- this.internalTimeoutMillis / 1e3
- } seconds.`,
+ `Reclaiming request ${request.url} (${request.id}) timed out.`,
);
- if (!(err instanceof CriticalError)) {
- isRequestLocked = false; // requestFunctionErrorHandler calls either markRequestAsHandled or reclaimRequest
- }
- request.state = RequestState.DONE;
- } catch (secondaryError: any) {
- const unwrappedSecondaryError = this.unwrapError(secondaryError) as any;
-
- if (
- !unwrappedSecondaryError.triggeredFromUserHandler &&
- // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
- !(unwrappedSecondaryError instanceof CriticalError)
- ) {
- const apifySpecific = process.env.APIFY_IS_AT_HOME
- ? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
- : '';
- this.log.exception(
- unwrappedSecondaryError as Error,
- 'An exception occurred during handling of failed request. ' +
- `This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`,
- );
- }
- request.state = RequestState.ERROR;
- throw unwrappedSecondaryError;
+ } catch {
+ // The request was never in progress, or could not be reclaimed. Either way it's fine.
}
- // decrease the session score if the request fails (but the error handler did not throw);
- // skip when the error is a SessionError, which already retired the session
- if (!this.errorAbsolvesSession(err)) {
- crawlingContext.session.markBad();
+
+ if (!timedOut) {
+ throw unwrappedSecondaryError;
}
} finally {
- // Safety net - return the request to the queue if nobody managed to mark it as handled
- // or reclaim it before (e.g. after a CriticalError). Reclaiming a request that is no longer
- // in progress is a harmless no-op on the storage backend.
- if (isRequestLocked && requestSource instanceof RequestQueue) {
- try {
- await requestSource.reclaimRequest(request);
- } catch {
- // The request was never in progress, or could not be reclaimed. Either way it's fine.
- }
- }
+ resumeRequestTimeout?.();
+ }
+
+ // decrease the session score if the request fails (but the error handler did not throw);
+ // skip when the error is a SessionError, which already retired the session
+ if (!skipped && !this.errorAbsolvesSession(err)) {
+ context.session?.markBad();
}
}
@@ -2804,10 +2799,14 @@ export class BasicCrawler<
/**
* Unwraps errors thrown by the context pipeline to get the actual user error.
- * RequestHandlerError and ContextPipelineCleanupError wrap the actual error.
+ * RequestHandlerError, ContextPipelineInitializationError and ContextPipelineCleanupError wrap the actual error.
*/
private unwrapError(error: unknown): Error {
- if (error instanceof RequestHandlerError || error instanceof ContextPipelineCleanupError) {
+ if (
+ error instanceof RequestHandlerError ||
+ error instanceof ContextPipelineInitializationError ||
+ error instanceof ContextPipelineCleanupError
+ ) {
return this.unwrapError(error.cause);
}
return error as Error;
diff --git a/packages/basic-crawler/src/internals/crawlers/context_pipeline.ts b/packages/basic-crawler/src/internals/crawlers/context_pipeline.ts
index 7d383525c..e6e24c604 100644
--- a/packages/basic-crawler/src/internals/crawlers/context_pipeline.ts
+++ b/packages/basic-crawler/src/internals/crawlers/context_pipeline.ts
@@ -1,11 +1,14 @@
-import { serviceLocator, SessionError } from '@crawlee/core';
+import { serviceLocator } from '@crawlee/core';
import type { Awaitable } from '@crawlee/types';
-import { ContextPipelineCleanupError, RequestHandlerError } from '../errors.js';
+import { ContextPipelineCleanupError, ContextPipelineInitializationError, RequestHandlerError } from '../errors.js';
/**
- * Registers a callback that runs after the final context consumer finishes. `error` is the consumer's failure, if any;
- * failures of downstream middlewares are not reported. Cleanups run in reverse registration order.
+ * Registers a callback that runs after the final context consumer finishes. Cleanups run in reverse registration
+ * order and receive the same `error` as the `onError` parameter of {@apilink ContextPipeline.call}: `undefined` on
+ * success, otherwise a {@apilink ContextPipelineInitializationError} (a middleware failed) or a
+ * {@apilink RequestHandlerError} (the consumer failed), with the original in `cause`. A cleanup that throws fails the
+ * whole call with a {@apilink ContextPipelineCleanupError}, which is critical.
*/
export type CleanupRegistrar = (cleanup: (error?: unknown) => Awaitable<void>) => void;
@@ -78,17 +81,18 @@ export abstract class ContextPipeline<TContextBase, TCrawlingContext extends TCo
*
* @param crawlingContext - The initial context to process through the pipeline
* @param finalContextConsumer - The function that will receive the final enhanced context
- * @param onInitializationError - Receives a middleware failure as-is; the consumer is then skipped. Runs before
- * the cleanups, so it still sees whatever the failed middleware's predecessors set up.
+ * @param onError - Receives a middleware failure wrapped in a {@apilink ContextPipelineInitializationError}, or a
+ * consumer failure wrapped in a {@apilink RequestHandlerError}, with the original in `cause`. After a middleware
+ * failure, the rest of the chain and the consumer are skipped. Runs before the cleanups, so it still sees
+ * whatever the middlewares set up.
*
- * @throws {RequestHandlerError} When the final context consumer throws an exception
* @throws {ContextPipelineCleanupError} When cleanup operations fail
- * @throws {SessionError} Session errors are re-thrown as-is for special handling
+ * @throws Whatever `onError` throws
*/
abstract call(
crawlingContext: TContextBase,
finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>,
- onInitializationError: (error: unknown) => Awaitable<void>,
+ onError: (error: unknown) => Awaitable<void>,
): Promise<void>;
}
@@ -154,60 +158,53 @@ class ContextPipelineImpl<TContextBase, TCrawlingContext extends TContextBase> e
async call(
crawlingContext: TContextBase,
finalContextConsumer: (finalContext: TCrawlingContext) => Promise<unknown>,
- onInitializationError: (error: unknown) => Awaitable<void>,
+ onError: (error: unknown) => Awaitable<void>,
): Promise<void> {
const middlewares = Array.from(this.middlewareChain()).reverse();
const cleanupStack: Parameters<CleanupRegistrar>[0][] = [];
const onCleanup: CleanupRegistrar = (cleanup) => {
cleanupStack.push(cleanup);
};
- let consumerException: unknown | undefined;
+ let failure: unknown;
+ let consumerStarted = false;
try {
for (const middleware of middlewares) {
- try {
- const contextExtension = await middleware(crawlingContext, onCleanup);
-
- const extensionNames = [
- ...Object.getOwnPropertyNames(contextExtension),
- ...Object.getOwnPropertySymbols(contextExtension),
- ];
-
- for (const key of extensionNames) {
- try {
- if (Object.getOwnPropertyDescriptor(crawlingContext, key)?.configurable !== false) {
- Object.defineProperty(
- crawlingContext,
- key,
- Object.getOwnPropertyDescriptor(contextExtension, key)!,
- );
- }
- } catch (error: any) {
- serviceLocator
- .getLogger()
- .debug(`Context pipeline failed to define property ${key.toString()}:`, error);
+ const contextExtension = await middleware(crawlingContext, onCleanup);
+
+ const extensionNames = [
+ ...Object.getOwnPropertyNames(contextExtension),
+ ...Object.getOwnPropertySymbols(contextExtension),
+ ];
+
+ for (const key of extensionNames) {
+ try {
+ if (Object.getOwnPropertyDescriptor(crawlingContext, key)?.configurable !== false) {
+ Object.defineProperty(
+ crawlingContext,
+ key,
+ Object.getOwnPropertyDescriptor(contextExtension, key)!,
+ );
}
+ } catch (error: any) {
+ serviceLocator
+ .getLogger()
+ .debug(`Context pipeline failed to define property ${key.toString()}:`, error);
}
- } catch (exception: unknown) {
- await onInitializationError(exception);
- return;
}
}
- try {
- await finalContextConsumer(crawlingContext as TCrawlingContext);
- } catch (exception: unknown) {
- if (exception instanceof SessionError) {
- consumerException = exception;
- throw exception; // Session errors are re-thrown as-is
- }
- consumerException = exception;
- throw new RequestHandlerError(exception);
- }
+ consumerStarted = true;
+ await finalContextConsumer(crawlingContext as TCrawlingContext);
+ } catch (exception: unknown) {
+ failure = consumerStarted
+ ? new RequestHandlerError(exception)
+ : new ContextPipelineInitializationError(exception);
+ await onError(failure);
} finally {
try {
for (const cleanup of cleanupStack.reverse()) {
- await cleanup(consumerException);
+ await cleanup(failure);
}
} catch (exception: unknown) {
// eslint-disable-next-line no-unsafe-finally
diff --git a/packages/basic-crawler/src/internals/errors.ts b/packages/basic-crawler/src/internals/errors.ts
index 27e943be9..6d3cc26c0 100644
--- a/packages/basic-crawler/src/internals/errors.ts
+++ b/packages/basic-crawler/src/internals/errors.ts
@@ -62,6 +62,14 @@ export class ContextPipelineCleanupError extends CriticalError {
}
}
+/** Wraps a failure of a context pipeline middleware, telling it apart from a {@apilink RequestHandlerError}. */
+export class ContextPipelineInitializationError extends Error {
+ constructor(error: unknown, options?: ErrorOptions) {
+ super(undefined, { cause: error, ...options });
+ }
+}
+
+/** Wraps a failure of the context pipeline's final consumer (the request handler), telling it apart from a middleware failure. */
export class RequestHandlerError extends Error {
constructor(error: unknown, options?: ErrorOptions) {
super(undefined, { cause: error, ...options });
diff --git a/packages/basic-crawler/src/internals/request-timeout.ts b/packages/basic-crawler/src/internals/request-timeout.ts
index d624db69c..8df133625 100644
--- a/packages/basic-crawler/src/internals/request-timeout.ts
+++ b/packages/basic-crawler/src/internals/request-timeout.ts
@@ -21,10 +21,17 @@ export const extendTimeoutKey = Symbol('extendTimeout');
*/
export const timeoutExpiredKey = Symbol('timeoutExpired');
+/**
+ * Stops the internal request timeout's clock until the returned callback is called. Error handling has a timeout
+ * of its own, and must not be cut short (and handled again) by the request timeout.
+ */
+export const suspendTimeoutKey = Symbol('suspendTimeout');
+
/** The slots the internal request timeout hangs on the in-flight crawling context. */
export interface RequestTimeoutContext {
[extendTimeoutKey]?: (extraMillis: number) => void;
[timeoutExpiredKey]?: () => boolean;
+ [suspendTimeoutKey]?: () => () => void;
[navigationDeadlineKey]?: number;
}
@@ -62,6 +69,7 @@ export async function raceWithTimeout(
let deadline = Date.now() + timeoutMillis;
let settled = false;
let firedByTimeout = false;
+ let suspendedAt: number | undefined;
const timeout = new Promise<never>((_, reject) => {
const fire = () => {
@@ -70,6 +78,11 @@ export async function raceWithTimeout(
reject(new TimeoutError(`Request timed out after ${timeoutMillis / 1e3} seconds (${requestId}).`));
};
+ const schedule = () => {
+ clearTimeout(timer);
+ timer = setTimeout(fire, Math.max(deadline - Date.now(), 0));
+ };
+
timer = setTimeout(fire, timeoutMillis);
context[timeoutExpiredKey] = () => firedByTimeout;
@@ -79,9 +92,27 @@ export async function raceWithTimeout(
return;
}
- clearTimeout(timer);
deadline += extraMillis;
- timer = setTimeout(fire, Math.max(deadline - Date.now(), 0));
+ if (suspendedAt === undefined) {
+ schedule();
+ }
+ };
+
+ context[suspendTimeoutKey] = () => {
+ if (settled || suspendedAt !== undefined) {
+ return () => {};
+ }
+
+ clearTimeout(timer);
+ suspendedAt = Date.now();
+
+ return () => {
+ deadline += Date.now() - suspendedAt!;
+ suspendedAt = undefined;
+ if (!settled) {
+ schedule();
+ }
+ };
};
});
diff --git a/packages/basic-crawler/src/internals/session_pool/session.ts b/packages/basic-crawler/src/internals/session_pool/session.ts
index a87d6ecb3..e87278fd8 100644
--- a/packages/basic-crawler/src/internals/session_pool/session.ts
+++ b/packages/basic-crawler/src/internals/session_pool/session.ts
@@ -254,6 +254,7 @@ export class Session implements ISession {
* It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
*/
markGood() {
+ if (this.#retired) return;
this.#usageCount += 1;
if (this.#errorScore > 0) {
@@ -294,7 +295,7 @@ export class Session implements ISession {
* For transient external failures (such as `5XX` responses), use `markBad()` instead.
*/
retire() {
- if (this.#retired) return;
+ if (this.#retired && this.isBlocked()) return;
this.#errorScore += this.#maxErrorScore;
this.#usageCount += 1;
this.#retired = true;
@@ -333,11 +334,12 @@ export class Session implements ISession {
}
/**
- * Checks if session is not usable. if it is not retires the session.
+ * Makes an unusable session's state terminal. Unlike `retire()`, this leaves the error score alone so that
+ * `isBlocked()` keeps telling a burned identity from one that merely ran out of uses.
*/
private maybeSelfRetire(): void {
if (!this.isUsable()) {
- this.retire();
+ this.#retired = true;
}
}
}
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index e13ac47a0..4dc748e23 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -23,9 +23,12 @@ import {
OwnedOrInjected,
remainingNavigationWindowMillis,
RequestState,
+ ContextPipelineInitializationError,
+ RequestHandlerError,
RequestThrottledError,
resolveBaseUrlForEnqueueLinksFiltering,
SessionError,
+ SessionRetiredError,
toughCookieToBrowserPoolCookie,
validators,
} from '@crawlee/basic';
@@ -551,14 +554,29 @@ export abstract class BrowserCrawler<
});
tryCancel();
- onCleanup(() => {
+ onCleanup((failure) => {
crawlingContext.registerDeferredCleanup(async () => {
- const error = !crawlingContext.session.isUsable()
- ? new SessionError('Session is no longer usable')
- : undefined;
+ const cause =
+ failure instanceof RequestHandlerError || failure instanceof ContextPipelineInitializationError
+ ? failure.cause
+ : failure;
+
+ // Only tells the pool what to do with the page's browser state - the request's failure, if any, has
+ // already been handled. A thrown `SessionError` or a blocked session counts as a block; a session
+ // unusable for any other reason is reported as retired. `retire()`/`markBad()` also bump the usage
+ // count, so the block check must come first.
+ const { session } = crawlingContext;
+ const closeReason =
+ cause instanceof SessionError
+ ? cause
+ : session.isBlocked()
+ ? new SessionError()
+ : session.isUsable()
+ ? undefined
+ : new SessionRetiredError();
await this.browserPool
- .closePage(page, { error })
+ .closePage(page, { error: closeReason })
.catch((closeError: Error) => this.log.debug('Error while closing page', { error: closeError }));
});
});
diff --git a/packages/browser-pool/src/browser-pool.ts b/packages/browser-pool/src/browser-pool.ts
index 1c2445004..d6d90d66b 100644
--- a/packages/browser-pool/src/browser-pool.ts
+++ b/packages/browser-pool/src/browser-pool.ts
@@ -705,8 +705,9 @@ export class BrowserPool<
/**
* Releases a page back to the pool. The page is closed and, if the
- * optional `error` is a {@apilink SessionError}, the browser controller
- * that served the page is retired so that its tainted state (cookies,
+ * optional `error` is a {@apilink SessionError} (including
+ * {@apilink SessionRetiredError}), the browser controller that served
+ * the page is retired so that the finished session's state (cookies,
* storage, etc.) cannot leak into future sessions.
*
* This is the primary way the crawler should return pages to the pool.
diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.ts
index 847072f6a..2d2be3a05 100644
--- a/packages/core/src/errors.ts
+++ b/packages/core/src/errors.ts
@@ -71,6 +71,12 @@ export class SessionError extends Error {
}
}
+/**
+ * A {@apilink SessionError} for a session that finished without being blocked, e.g. by reaching its
+ * `maxUsageCount` or `maxAgeSecs`. A plain `SessionError` means the session was blocked.
+ */
+export class SessionRetiredError extends SessionError {}
+
/**
* Wraps the failure of a callback registered with {@apilink StorageTransaction.afterCommit|`afterCommit`}
* that ran after the request's writes had already been committed.
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index 664655119..04f205436 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -21,6 +21,7 @@ import type {
import {
BasicCrawler,
ContextPipeline,
+ ContextPipelineInitializationError,
RequestHandlerError,
resolveBaseUrlForEnqueueLinksFiltering,
Router,
@@ -718,7 +719,8 @@ export class AdaptivePlaywrightCrawler<
// Execution will "fall through" and try running the request handler in a browser
if (!plainHTTPRun.ok) {
const actualError =
- plainHTTPRun.error instanceof RequestHandlerError
+ plainHTTPRun.error instanceof RequestHandlerError ||
+ plainHTTPRun.error instanceof ContextPipelineInitializationError
? (plainHTTPRun.error.cause as Error)
: (plainHTTPRun.error as Error);
diff --git a/packages/types/src/browser.ts b/packages/types/src/browser.ts
index df8fd1149..d8544e2ca 100644
--- a/packages/types/src/browser.ts
+++ b/packages/types/src/browser.ts
@@ -125,14 +125,14 @@ export interface IBrowserPool<Page = unknown> {
/**
* Signals the pool that the caller is done with the page. The pool is
* responsible for closing the page and performing any necessary cleanup
- * (e.g. retiring the underlying browser when a session has gone bad).
+ * (e.g. retiring the underlying browser once the page's session is finished).
*
* @param page The page to release back to the pool.
- * @param options.error If the page is being released because of an error,
- * pass the error here. In particular, if the error is a
- * {@apilink SessionError}, implementations should treat it as a signal
- * to purge all state associated with the session (e.g. discard any
- * browser that served the page).
+ * @param options.error If the error is a {@apilink SessionError}, the
+ * session that served the page is finished. A plain `SessionError`
+ * means it was blocked: discard any browser state tied to it. A
+ * {@apilink SessionRetiredError} means it merely reached its usage or
+ * age limit, so a pool may keep a warm page instead.
*/
closePage(page: Page, options?: { error?: Error }): Promise<void>;
diff --git a/packages/types/src/session.ts b/packages/types/src/session.ts
index 628de404e..83f284d78 100644
--- a/packages/types/src/session.ts
+++ b/packages/types/src/session.ts
@@ -121,6 +121,12 @@ export interface ISession {
*/
isUsable(): boolean;
+ /**
+ * Indicates whether the session is blocked, i.e. its error score reached the maximum or it was retired via `retire()`.
+ * Unlike other reasons for not being usable, this means the identity itself is burned.
+ */
+ isBlocked(): boolean;
+
/**
* This method should be called after a successful session usage.
*/
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index e43cbae08..67610e299 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -19,6 +19,7 @@ import {
AfterCommitError,
BasicCrawler,
CrawlingRequest,
+ ContextPipelineInterruptedError,
ConcurrencySystem,
Configuration,
CriticalError,
@@ -2075,6 +2076,115 @@ describe('BasicCrawler', () => {
expect(processed).toHaveLength(1);
});
+ test('an error handler outliving the internal timeout does not crash the crawler', async () => {
+ serviceLocator.reset();
+ serviceLocator.setStorageBackend(new MemoryStorageBackend());
+ serviceLocator.setConfiguration(new Configuration({ internalTimeoutMillis: 100 }));
+
+ let errorHandlerCalls = 0;
+
+ const crawler = new BasicCrawler({
+ maxRequestRetries: 1,
+ // keeps the request timeout well clear, so only the error handler's own timer can fire
+ requestHandlerTimeoutSecs: 1,
+ requestHandler: async () => {
+ throw new Error('boom');
+ },
+ errorHandler: async () => {
+ if (errorHandlerCalls++ === 0) await sleep(300);
+ },
+ });
+ const warningSpy = vitest.spyOn(crawler.log, 'warning');
+
+ await expect(crawler.run(['https://example.com'])).resolves.toMatchObject({ requestsFailed: 1 });
+
+ // without this, the test would also pass if the error handler simply weren't timed out at all
+ expect(warningSpy).toHaveBeenCalledWith(expect.stringContaining('Handling request failure'));
+ });
+
+ test('a timed out error handler still uses up a retry', async () => {
+ serviceLocator.reset();
+ serviceLocator.setStorageBackend(new MemoryStorageBackend());
+ serviceLocator.setConfiguration(new Configuration({ internalTimeoutMillis: 100 }));
+
+ // never settles, so every retry times out - if that did not use up a retry, the request would go round forever
+ const errorHandler = vitest.fn(() => new Promise<never>(() => {}));
+
+ const crawler = new BasicCrawler({
+ maxRequestRetries: 2,
+ requestHandlerTimeoutSecs: 1,
+ requestHandler: async () => {
+ throw new Error('boom');
+ },
+ errorHandler,
+ });
+
+ await expect(crawler.run(['https://example.com'])).resolves.toMatchObject({ requestsFailed: 1 });
+ expect(errorHandler).toHaveBeenCalledTimes(2);
+ });
+
+ test('the request timeout does not fire while the error handler is running', async () => {
+ serviceLocator.reset();
+ serviceLocator.setStorageBackend(new MemoryStorageBackend());
+ // the request timeout is max(internal, handler) = 200ms; the handler times out at 100ms and the
+ // failure handler runs past 200ms, while staying within its own 200ms budget
+ serviceLocator.setConfiguration(new Configuration({ internalTimeoutMillis: 200 }));
+
+ let failedRequestHandlerFinished = false;
+ const failedRequestHandler = vitest.fn(async () => {
+ await sleep(150);
+ failedRequestHandlerFinished = true;
+ });
+
+ const crawler = new BasicCrawler({
+ maxRequestRetries: 0,
+ requestHandlerTimeoutSecs: 0.1,
+ requestHandler: async () => sleep(150),
+ failedRequestHandler,
+ });
+
+ await Promise.all([
+ crawler.run(['https://example.com']).then(() => expect(failedRequestHandlerFinished).toBe(true)),
+ // a second, concurrent failure handling may outlive `run()`; this covers it with room to spare
+ sleep(500),
+ ]);
+
+ // the concurrent second run would get the request timeout instead of the handler's own
+ expect(failedRequestHandler).toHaveBeenCalledExactlyOnceWith(
+ expect.anything(),
+ expect.objectContaining({ message: expect.stringContaining('requestHandler timed out') }),
+ );
+ });
+
+ test('a skipped request whose markRequestAsHandled times out does not crash the crawler', async () => {
+ serviceLocator.reset();
+ serviceLocator.setStorageBackend(new MemoryStorageBackend());
+ serviceLocator.setConfiguration(new Configuration({ internalTimeoutMillis: 100 }));
+
+ const requestQueue = await RequestQueue.open();
+
+ // the whole backend is down, so the safety-net reclaim cannot rescue the request either
+ vitest.spyOn(requestQueue, 'markRequestAsHandled').mockImplementation(() => new Promise<never>(() => {}));
+ vitest.spyOn(requestQueue, 'reclaimRequest').mockImplementation(() => new Promise<never>(() => {}));
+
+ const crawler = new BasicCrawler({
+ requestQueue,
+ // the request is never handled, so the queue never finishes; `stop()` lets the in-flight request wind down
+ extendContext: () => {
+ crawler.stop();
+ throw new ContextPipelineInterruptedError('skipped by the test');
+ },
+ requestHandler: async () => {},
+ });
+ const warningSpy = vitest.spyOn(crawler.log, 'warning');
+
+ await expect(crawler.run(['https://example.com'])).resolves.toBeDefined();
+ // anchored, so that the `timeoutAndRetry` retry warnings ("... (retrying 1/3)") do not count
+ expect(warningSpy).toHaveBeenCalledWith(
+ expect.stringMatching(/^Marking request .* as handled timed out after 0\.1 seconds\.$/),
+ );
+ });
+
test('warns when the internal timeout is shorter than the phase timeouts', async () => {
serviceLocator.reset();
serviceLocator.setStorageBackend(new MemoryStorageBackend());
diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts
index 14c830eb1..89f2e5899 100644
--- a/test/core/crawlers/browser_crawler.test.ts
+++ b/test/core/crawlers/browser_crawler.test.ts
@@ -8,7 +8,13 @@ import {
PuppeteerPlugin,
RemoteBrowserPool,
} from '@crawlee/browser-pool';
-import { BLOCKED_STATUS_CODES, type ConcurrencySystem, SessionPool } from '@crawlee/basic';
+import {
+ BLOCKED_STATUS_CODES,
+ type ConcurrencySystem,
+ SessionError,
+ SessionPool,
+ SessionRetiredError,
+} from '@crawlee/basic';
import { bindMethodsToServiceLocator, MemoryStorageBackend, serviceLocator, ServiceLocator } from '@crawlee/core';
import type { PuppeteerGoToOptions } from '@crawlee/puppeteer';
import { EnqueueStrategy, ProxyConfiguration, Request, RequestList, RequestState, Session } from '@crawlee/puppeteer';
@@ -645,6 +651,52 @@ describe('BrowserCrawler', () => {
expect(state.sessions[0].maxUsageCount).toBe(1);
});
+ test('tells closePage whether the session was blocked or just used up', async () => {
+ const externalPool = new BrowserPoolClass({ browserPlugins: [new PuppeteerPlugin(puppeteer)] });
+ const closeErrors = new Map<string | null, Error | undefined>();
+ const originalClosePage = externalPool.closePage.bind(externalPool);
+ externalPool.closePage = async (page, options) => {
+ closeErrors.set(new URL(page.url()).searchParams.get('q'), options?.error);
+ return originalClosePage(page, options);
+ };
+ const blockError = new SessionError('blocked');
+
+ try {
+ const crawler = new BrowserCrawlerTest({
+ browserPool: externalPool,
+ // `maxUsageCount: 1` means `markBad()`/`retire()` also exhaust usage, so a block must win over that.
+ sessionPool: new SessionPool({ sessionOptions: { maxUsageCount: 1, maxErrorScore: 3 } }),
+ maxConcurrency: 1,
+ maxRequestRetries: 0,
+ requestHandler: async ({ request, session }) => {
+ if (request.url.endsWith('blocked')) throw blockError;
+ if (request.url.endsWith('markbad')) for (let i = 0; i < 3; i++) session.markBad();
+ if (request.url.endsWith('retire')) {
+ session.retire();
+ throw new Error('captcha');
+ }
+ },
+ });
+
+ await crawler.run([
+ `${serverAddress}/?q=blocked`,
+ `${serverAddress}/?q=ok`,
+ `${serverAddress}/?q=markbad`,
+ `${serverAddress}/?q=retire`,
+ ]);
+
+ expect(closeErrors.size).toBe(4);
+ expect(closeErrors.get('blocked')).toBe(blockError);
+ expect(closeErrors.get('ok')).toBeInstanceOf(SessionRetiredError);
+ for (const q of ['markbad', 'retire']) {
+ expect(closeErrors.get(q)).toBeInstanceOf(SessionError);
+ expect(closeErrors.get(q)).not.toBeInstanceOf(SessionRetiredError);
+ }
+ } finally {
+ await externalPool.destroy();
+ }
+ });
+
test.skip('should persist cookies per session', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
const name = `list-${Math.random()}`;
diff --git a/test/core/crawlers/context_pipeline.test.ts b/test/core/crawlers/context_pipeline.test.ts
index 8d96e9feb..faf1cb5a7 100644
--- a/test/core/crawlers/context_pipeline.test.ts
+++ b/test/core/crawlers/context_pipeline.test.ts
@@ -1,4 +1,9 @@
-import { ContextPipeline, ContextPipelineCleanupError, RequestHandlerError } from '@crawlee/basic';
+import {
+ ContextPipeline,
+ ContextPipelineCleanupError,
+ ContextPipelineInitializationError,
+ RequestHandlerError,
+} from '@crawlee/basic';
import { describe, expect, it, vi } from 'vitest';
const rethrow = (error: unknown) => {
@@ -39,15 +44,15 @@ describe('ContextPipeline', () => {
expect(consumer).toHaveBeenCalledWith({ c: [2, 1] });
});
- it('should hand initialization errors to onInitializationError before running cleanups', async () => {
+ it('should hand middleware errors to onError wrapped in ContextPipelineInitializationError, then to the cleanups', async () => {
const initializationError = new Error('Pipeline initialization failed');
- const order: string[] = [];
+ const order: [string, unknown][] = [];
const thirdMiddleware = vi.fn();
const pipeline = ContextPipeline.create()
.compose(async (_, onCleanup) => {
- onCleanup(() => {
- order.push('cleanup');
+ onCleanup((error) => {
+ order.push(['cleanup', error]);
});
return {};
})
@@ -57,50 +62,45 @@ describe('ContextPipeline', () => {
.compose(thirdMiddleware);
const consumer = vi.fn();
- const onInitializationError = vi.fn(async (error: unknown) => {
- order.push('onInitializationError');
- expect(error).toBe(initializationError);
- });
- await pipeline.call({}, consumer, onInitializationError);
+ await pipeline.call({}, consumer, (error) => {
+ order.push(['onError', error]);
+ });
- expect(onInitializationError).toHaveBeenCalledOnce();
expect(thirdMiddleware).not.toHaveBeenCalled();
expect(consumer).not.toHaveBeenCalled();
- expect(order).toEqual(['onInitializationError', 'cleanup']);
- });
-
- it('should wrap errors in the final consumer', async () => {
- const consumerError = new Error('Request handler failed');
- const context = { a: 3 };
-
- const pipeline = ContextPipeline.create().compose(async () => ({ b: 4 }));
-
- const consumer = vi.fn().mockRejectedValue(consumerError);
+ expect(order.map(([step]) => step)).toEqual(['onError', 'cleanup']);
- await expect(pipeline.call(context, consumer, rethrow)).rejects.toThrow(
- expect.objectContaining({
- cause: consumerError,
- constructor: RequestHandlerError,
- }),
+ const [[, reported], [, cleanedUpWith]] = order;
+ expect(reported).toEqual(
+ expect.objectContaining({ cause: initializationError, constructor: ContextPipelineInitializationError }),
);
-
- expect(consumer).toHaveBeenCalledWith({ a: 3, b: 4 });
+ expect(cleanedUpWith).toBe(reported);
});
- it('should pass the consumer error to cleanup routines', async () => {
+ it('should hand consumer errors to onError wrapped in RequestHandlerError, then to the cleanups', async () => {
const consumerError = new Error('Request handler failed');
- const context = { a: 3 };
- const cleanup = vi.fn();
+ const order: [string, unknown][] = [];
const pipeline = ContextPipeline.create().compose(async (_, onCleanup) => {
- onCleanup(cleanup);
+ onCleanup((error) => {
+ order.push(['cleanup', error]);
+ });
return { b: 4 };
});
- await expect(pipeline.call(context, vi.fn().mockRejectedValue(consumerError), rethrow)).rejects.toThrow();
+ const consumer = vi.fn().mockRejectedValue(consumerError);
+
+ await pipeline.call({ a: 3 }, consumer, (error) => {
+ order.push(['onError', error]);
+ });
+
+ expect(consumer).toHaveBeenCalledWith({ a: 3, b: 4 });
+ expect(order.map(([step]) => step)).toEqual(['onError', 'cleanup']);
- expect(cleanup).toHaveBeenCalledWith(consumerError);
+ const [[, reported], [, cleanedUpWith]] = order;
+ expect(reported).toEqual(expect.objectContaining({ cause: consumerError, constructor: RequestHandlerError }));
+ expect(cleanedUpWith).toBe(reported);
});
it('should call cleanup routines without an error if the consumer succeeds', async () => {
diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.ts
index 995ff9a56..5937f5813 100644
--- a/test/core/session_pool/session.test.ts
+++ b/test/core/session_pool/session.test.ts
@@ -79,39 +79,40 @@ describe('Session - testing session behaviour', () => {
expect(session.usageCount).toBe(usageCount);
});
- test('should retire session after marking bad', () => {
- vitest.spyOn(session, 'retire');
+ test('running out of uses is terminal but not a block', () => {
+ session = new Session({ maxUsageCount: 1 });
+ session.markGood();
+
+ expect(session.isUsable()).toBe(false);
+ expect(session.isBlocked()).toBe(false);
+ expect(session.getState().retired).toBe(true);
+ });
+
+ test('marking bad up to maxErrorScore blocks the session', () => {
+ session = new Session({ maxErrorScore: 2 });
session.markBad();
- expect(session.retire).toBeCalledTimes(0);
- session.isUsable = () => false;
+ expect(session.isBlocked()).toBe(false);
session.markBad();
- expect(session.retire).toBeCalledTimes(1);
- });
- test('should retire session after marking good', () => {
- vitest.spyOn(session, 'retire');
+ expect(session.isBlocked()).toBe(true);
+ expect(session.getState().retired).toBe(true);
+ });
- session.markGood();
- expect(session.retire).toBeCalledTimes(0);
+ test('retire() blocks a session that already ran out of uses', () => {
+ session = new Session({ maxUsageCount: 1 });
+ session.markBad();
+ session.retire();
- session.isUsable = () => false;
- session.markGood();
- expect(session.retire).toBeCalledTimes(1);
+ expect(session.isBlocked()).toBe(true);
});
- test('should reevaluate usability of session after marking the session', () => {
- vitest.spyOn(session, 'retire');
-
- // A usable session is not retired when marked.
+ test('markGood() does not unblock a blocked session', () => {
+ session = new Session({ maxErrorScore: 2 });
+ session.markBad();
+ session.markBad();
session.markGood();
- expect(session.retire).toBeCalledTimes(0);
- // Once the session becomes unusable, marking it (good or bad) retires it.
- session.isUsable = () => false;
- session.markGood();
- expect(session.retire).toBeCalledTimes(1);
- session.markBad();
- expect(session.retire).toBeCalledTimes(2);
+ expect(session.isBlocked()).toBe(true);
});
test('should get state', () => {