Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: redesign request timeouts and allow per-route overrides (#3860)
Martin Adámekf01b62035255
diff --git a/docs/introduction/08-refactoring.mdx b/docs/introduction/08-refactoring.mdxindex 450048013..235fe88f0 100644--- a/docs/introduction/08-refactoring.mdx+++ b/docs/introduction/08-refactoring.mdx@@ -154,6 +154,30 @@ Initially, using a simple `if/else` statement for selecting different logic baseIt's good practice in any programming language to split your logic into bite-sized chunks that are easy to read and reason about. Scrolling through a thousand line long `requestHandler()` where everything interacts with everything and variables can be used everywhere is not a beautiful thing to do and a pain to debug. That's why we prefer the separation of routes into their own files.+### Giving a route its own timeout++The crawler-wide `requestHandlerTimeoutSecs` applies to every request alike. Once the routes are separated, one of them can be given its own timeout — longer or shorter — without changing the limit for the rest, by passing a per-route timeout as the last argument of `addHandler`. This is handy when one page type does noticeably more work than the others, like a `CATEGORY` page that paginates through a long list:++```js+router.addHandler('CATEGORY', async ({ page, enqueueLinks }) => {+ // ...+}, { requestHandlerTimeoutSecs: 120 });++router.addHandler('DETAIL', async ({ page }) => {+ // ...+}); // keeps the crawler's default+```++If how much longer a route needs only becomes clear once it is already running, call `extendTimeout()` from inside the handler to buy more time on the spot:++```js+router.addHandler('CATEGORY', async ({ page, enqueueLinks, extendTimeout }) => {+ const pageCount = await countPages(page);+ extendTimeout(pageCount * 10); // ask for 10 more seconds per page+ // ...+});+```+## Next stepsIn the next and final step, you'll see how to deploy your Crawlee project to the cloud. If you used the CLI to bootstrap your project, you already have a **Dockerfile** ready, and the next section will show you how to deploy it to the [Apify Platform](../deployment/apify-platform) with ease.diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.mdindex 4993b0cc4..a561c8aff 100644--- a/docs/public-api/crawlee-basic.api.md+++ b/docs/public-api/crawlee-basic.api.md@@ -91,6 +91,7 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, CongetData(...args: Parameters<Dataset['getData']>): ReturnType<Dataset['getData']>;getDataset(identifier?: string | StorageIdentifier): Promise<Dataset>;protected _getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;+ protected getNavigationTimeoutMillis(): number;// (undocumented)protected getPendingRequestCountApproximation(): Promise<number>;getRequestManager(): Promise<IRequestManager>;diff --git a/docs/public-api/crawlee-browser.api.md b/docs/public-api/crawlee-browser.api.mdindex 4b8d6c76c..4e7dd1873 100644--- a/docs/public-api/crawlee-browser.api.md+++ b/docs/public-api/crawlee-browser.api.md@@ -9,7 +9,7 @@ import { ArrayPredicate } from 'ow';import type { Awaitable } from '@crawlee/types';import { BasePredicate } from 'ow';import { BasicCrawler } from '@crawlee/basic';-import type { BasicCrawlerOptions } from '@crawlee/basic';+import { BasicCrawlerOptions } from '@crawlee/basic';import type { BasicCrawlingContext } from '@crawlee/basic';import type { BatchAddRequestsResult } from '@crawlee/types';import { BooleanPredicate } from 'ow';@@ -53,6 +53,8 @@ export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Respo// (undocumented)protected buildContextPipeline(): ContextPipeline<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>;// (undocumented)+ protected getNavigationTimeoutMillis(): number;+ // (undocumented)protected readonly ignoreIframes: boolean;// (undocumented)protected readonly ignoreShadowRoots: boolean;diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.mdindex 279f55f55..d56979ca9 100644--- a/docs/public-api/crawlee-core.api.md+++ b/docs/public-api/crawlee-core.api.md@@ -281,6 +281,7 @@ export const crawleeConfigFields: {availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;memoryMbytes: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;+ internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;inputKey: ConfigField<z.ZodDefault<z.ZodString>>;headless: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;@@ -301,6 +302,7 @@ export { CrawleeLoggerOptions }// @public (undocumented)export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {enqueueLinks(options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestManager' | 'robotsTxtFile'>> & Pick<EnqueueLinksOptions, 'requestManager' | 'robotsTxtFile'>): Promise<unknown>;+ extendTimeout(secs: number): void;registerDeferredCleanup(cleanup: () => Promise<unknown>): void;sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;}@@ -1325,17 +1327,24 @@ export class RetryRequestError extends Error {constructor(message?: string);}+// @public+export interface RouteOptions {+ requestHandlerTimeoutSecs?: number;+}+// @publicexport class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {- addDefaultHandler<UserData extends Dictionary = DefaultRouteUserData<Routes, GetUserDataFromRequest<Context['request']>>>(handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>): void;- addHandler<Label extends keyof Routes & string>(label: Label, handler: (ctx: RouterHandlerContext<Context, Routes[Label], Routes>) => Awaitable<void>): void;- addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: RouterLabel<Routes>, handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>): void;+ addDefaultHandler<UserData extends Dictionary = DefaultRouteUserData<Routes, GetUserDataFromRequest<Context['request']>>>(handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>, options?: RouteOptions): void;+ addHandler<Label extends keyof Routes & string>(label: Label, handler: (ctx: RouterHandlerContext<Context, Routes[Label], Routes>) => Awaitable<void>, options?: RouteOptions): void;+ addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: RouterLabel<Routes>, handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>, options?: RouteOptions): void;static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;// (undocumented)static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;// (undocumented)static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;getHandler(label?: string | symbol): (ctx: Context) => Awaitable<void>;+ getMaxTimeoutSecs(): number | undefined;+ getTimeoutSecs(label?: string | symbol): number | undefined;use(middleware: (ctx: Context) => Awaitable<void>): void;}diff --git a/docs/public-api/crawlee-http.api.md b/docs/public-api/crawlee-http.api.mdindex 204dffa49..fc757f9f7 100644--- a/docs/public-api/crawlee-http.api.md+++ b/docs/public-api/crawlee-http.api.md@@ -103,6 +103,8 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> =protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;// (undocumented)+ protected getNavigationTimeoutMillis(): number;+ // (undocumented)protected isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise<string | false>;// (undocumented)protected static optionsShape: {diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex 2b9d132ac..b80339cb5 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -556,6 +556,43 @@ The `KeyValueStore.getPublicUrl` method is now asynchronous and reads the publicThe `preNavigationHooks` option in `HttpCrawler` subclasses no longer accepts the `gotOptions` object as a second parameter. Modify the `crawlingContext` fields (e.g. `.request`) directly instead.+## Navigation and the request handler are timed separately++In v3, navigation ran inside the request handler's time window, and the two options were summed (plus an undocumented 10 second buffer) to form the actual limit. Setting `requestHandlerTimeoutSecs: 60` on a `PlaywrightCrawler` therefore produced errors complaining about 130 seconds.++Navigation and the request handler are now timed independently, and each reports itself:++| Option | Covers | Default |+| --- | --- | --- |+| `requestHandlerTimeoutSecs` | your `requestHandler` only | 60 |+| `navigationTimeoutSecs` | the `preNavigationHooks`, the navigation, and the `postNavigationHooks` together | 30 (HTTP), 60 (browser) |++`navigationTimeoutSecs` is a single budget shared by the whole navigation phase (matching Crawlee for Python), so a slow hook eats into the same window the navigation uses. The separate `navigationHooksTimeoutSecs` option has been removed.++Two things to watch for when upgrading:++- **Navigation hooks are now bounded.** They had no timeout of their own before, so a `preNavigationHooks` / `postNavigationHooks` function that pushes the whole phase past `navigationTimeoutSecs` will now fail the request. Raise `navigationTimeoutSecs`, or call `context.extendTimeout()` from inside the hook when the extra time is only needed occasionally.+- **A request can no longer hang forever.** An internal timeout now bounds the whole request, covering the phases that have no timeout of their own (`extendContext`, the robots.txt check, response processing). By default it is deliberately generous (twice the request handler timeout, and never less than 5 minutes), so it only fires when a request is genuinely stuck. Set `CRAWLEE_INTERNAL_TIMEOUT` (in milliseconds) to override it. Because it wraps the whole request, it is raised per request to outlast the navigation and request handler timeouts, so a value below them cannot cut those phases short; if you configure one that low, the crawler warns at startup that it is being ignored.++## Per-route and per-request handler timeouts++`requestHandlerTimeoutSecs` still applies to every request alike, but a single route can now opt out of it — useful when one page type needs markedly more time than the rest, and you do not want to raise the timeout for everything else to accommodate it:++```ts+router.addHandler('LIST', async ({ enqueueLinks }) => { ... }, { requestHandlerTimeoutSecs: 120 });+router.addHandler('DETAIL', async ({ pushData }) => { ... }); // keeps the crawler's default+```++When the time needed is only apparent once the handler is already running, `context.extendTimeout()` buys it more:++```ts+router.addHandler('LIST', async ({ page, extendTimeout }) => {+ const pageCount = await countPages(page);+ extendTimeout(pageCount * 10);+ await scrapeAllPages(page);+});+```+## Configuration class redesignThe `Configuration` class has been redesigned for v4. The main changes are:diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.jsonindex 337359dfd..db7b16f84 100644--- a/packages/basic-crawler/package.json+++ b/packages/basic-crawler/package.json@@ -40,7 +40,7 @@},"dependencies": {"@apify/datastructures": "^2.0.0",- "@apify/timeout": "^0.3.2",+ "@apify/timeout": "^0.4.4","@apify/utilities": "^2.15.5","@crawlee/core": "workspace:*","@crawlee/http-client": "workspace:^",diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex d0027ff9e..799738d66 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -91,9 +91,16 @@ import { getDomain } from 'tldts';import type { ReadonlyDeep, SetRequired } from 'type-fest';import { LruCache } from '@apify/datastructures';-import { addTimeoutToPromise, TimeoutError } from '@apify/timeout';+import { addTimeoutToPromise, extendTimeout, TimeoutError } from '@apify/timeout';import { cryptoRandomObjectId } from '@apify/utilities';+import {+ extendTimeoutKey,+ navigationDeadlineKey,+ raceWithTimeout,+ type RequestTimeoutContext,+ timeoutExpiredKey,+} from './request-timeout.js';import { createSendRequest } from './send-request.js';class LazyDefaultHttpClient implements BaseHttpClient {@@ -131,6 +138,12 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000;const deferredCleanupKey = Symbol('deferredCleanup');+// The request timeout plumbing (the window helper, the context symbols, and the race) lives in its own module.+export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';++/** The in-flight context, carrying the timeout slots ({@apilink raceWithTimeout} hangs its extender on them). */+type PendingCrawlingContext = { request: Request } & Partial<CrawlingContext> & RequestTimeoutContext;+export type RequestHandler<Context extends CrawlingContext = CrawlingContext> = (inputs: Context) => Awaitable<void>;/**@@ -976,10 +989,10 @@ export class BasicCrawler<this.respectRobotsTxtFile = respectRobotsTxtFile;this.onSkippedRequest = onSkippedRequest;- const tryEnv = (val?: string) => (val == null ? null : +val);// allow at least 5min for internal timeoutsthis.internalTimeoutMillis =- tryEnv(process.env.CRAWLEE_INTERNAL_TIMEOUT) ?? Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);+ serviceLocator.getConfiguration().internalTimeoutMillis ??+ Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);this.maxRequestRetries = maxRequestRetries;this.maxCrawlDepth = maxCrawlDepth;@@ -1057,9 +1070,15 @@ export class BasicCrawler<const crawlingContext = { request } as { request: Request } & Partial<CrawlingContext>;try {- await this.basicContextPipeline- .chain(this.contextPipeline)- .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request));+ // Navigation, the navigation hooks and the request handler are timed individually, but the+ // phases between them are not, so a request could still get stuck indefinitely. This is the+ // catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.+ await this.withRequestTimeout(+ crawlingContext,+ this.basicContextPipeline+ .chain(this.contextPipeline)+ .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request)),+ );} catch (error) {// ContextPipelineInterruptedError means the request was intentionally skipped// (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.@@ -1076,9 +1095,12 @@ export class BasicCrawler<}// If the error happened during pipeline initialization (e.g., navigation timeout, session/proxy error,- // i.e. not in user's requestHandler), handle it through the normal error flow.+ // i.e. not in user's requestHandler), handle it through the normal error flow. A bare `TimeoutError`+ // here is the internal timeout above firing - anything else thrown inside the pipeline arrives wrapped.const isPipelineError =- error instanceof ContextPipelineInitializationError || error instanceof SessionError;+ error instanceof ContextPipelineInitializationError ||+ error instanceof SessionError ||+ error instanceof TimeoutError;if (isPipelineError) {const unwrappedError = this.unwrapError(error);@@ -1213,7 +1235,7 @@ export class BasicCrawler<private buildBasicContextPipeline(): ContextPipeline<{ request: Request }, CrawlingContext> {return ContextPipeline.create<{ request: Request }>().compose({ action: this.checkRobotsTxt.bind(this) })- .compose({ action: () => this.createBaseContext() })+ .compose({ action: (context) => this.createBaseContext(context) }).compose({ action: this.resolveSession.bind(this) }).compose({ action: this.createContextHelpers.bind(this) });}@@ -1244,7 +1266,7 @@ export class BasicCrawler<return ContextPipeline.create<CrawlingContext>();}- private createBaseContext() {+ private createBaseContext(context: PendingCrawlingContext) {const deferredCleanup: (() => Promise<unknown>)[] = [];return {@@ -1256,6 +1278,19 @@ export class BasicCrawler<registerDeferredCleanup: (cleanup: () => Promise<unknown>) => {deferredCleanup.push(cleanup);},+ extendTimeout: (secs: number) => {+ const extraMillis = secs * 1000;++ // the current `addTimeoutToPromise` window (the request handler, or a navigation hook)...+ extendTimeout(extraMillis);+ // ...the internal timeout around the whole request, which is not an `addTimeoutToPromise` frame...+ context[extendTimeoutKey]?.(extraMillis);+ // ...and, when called from within the navigation phase, its shared window, so extending a hook+ // extends the whole navigation budget rather than just that hook's step.+ if (context[navigationDeadlineKey] !== undefined) {+ context[navigationDeadlineKey] += extraMillis;+ }+ },[deferredCleanupKey]: deferredCleanup,};}@@ -1688,9 +1723,13 @@ export class BasicCrawler<* regardless of whether the manager is a plain {@apilink RequestQueue} or a `RequestManagerTandem`.*/private async applyRequestManagerTimeouts(requestManager: IRequestManager): Promise<void> {- await requestManager.setExpectedRequestProcessingTimeSecs?.(- Math.max(this.requestHandlerTimeoutMillis / 1000 + 5, 60),- );+ // A router route may hold a request for longer than the crawler's own timeout, and we cannot know+ // which routes a run will hit, so reserve for the longest one any route asked for. The hint is+ // raise-only, so erring high here is safe.+ const maxRouteTimeoutSecs = (this.requestHandler as Partial<RouterHandler>).getMaxTimeoutSecs?.() ?? 0;+ const handlerTimeoutSecs = Math.max(this.requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);++ await requestManager.setExpectedRequestProcessingTimeSecs?.(Math.max(handlerTimeoutSecs + 5, 60));}/**@@ -1977,6 +2016,19 @@ export class BasicCrawler<this._ownsEventManager = true;}+ // Warn once at startup if the internal timeout is shorter than the phases it is meant to outlast. It is+ // floored per request so it will not actually cut them short, but the configured value is then effectively+ // ignored, which is worth flagging. Checked here (not in the constructor) because a subclass sets its+ // navigation timeout only after `super()`.+ const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(undefined);+ if (this.internalTimeoutMillis < phasesMillis) {+ this.log.warning(+ `CRAWLEE_INTERNAL_TIMEOUT (${this.internalTimeoutMillis / 1000}s) is shorter than the navigation ` ++ `and request handler timeouts combined (${phasesMillis / 1000}s); it will be raised per request ` ++ `so it does not cut them short.`,+ );+ }+// An owned governor is rebuilt (and started) for every run, so it always starts from a clean slate — stale// resource snapshots or a previous run's scaled desired concurrency would otherwise distort this run's// scaling. An injected one is long-lived and its lifecycle belongs to the caller.@@ -1992,11 +2044,66 @@ export class BasicCrawler<await this.getRequestManager();}+ /**+ * The navigation timeout (pre-navigation hooks, navigation, and post-navigation hooks) in milliseconds, used+ * to size the internal request timeout. `BasicCrawler` has no navigation phase, so this is 0; the HTTP and+ * browser crawlers override it with their `navigationTimeoutSecs`.+ */+ protected getNavigationTimeoutMillis(): number {+ return 0;+ }++ /**+ * Races the request against the internal timeout (see {@apilink raceWithTimeout}), sized to outlast the phases+ * that have their own timeout - the navigation, its hooks, and the request handler - so a legitimately slow+ * request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes+ * whichever is larger: the configured internal timeout, or this request's combined phase budget.+ */+ private async withRequestTimeout(crawlingContext: PendingCrawlingContext, work: Promise<void>): Promise<void> {+ const { request } = crawlingContext;+ const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(request.label);+ const timeoutMillis = Math.max(this.internalTimeoutMillis, phasesMillis);++ await raceWithTimeout(crawlingContext, work, { timeoutMillis, requestId: request.id });+ }++ /**+ * The request handler timeout for a request with the given route label. A router route may override the+ * crawler's own `requestHandlerTimeoutSecs`; anything else falls back to `fallbackMillis`.+ *+ * @param label The request's route label, or `undefined` for the default route / no specific request.+ * @param fallbackMillis Timeout to use when no route overrides it.+ */+ private resolveRequestHandlerTimeoutMillis(+ label: string | undefined,+ fallbackMillis = this.requestHandlerTimeoutMillis,+ ): number {+ return this.getRouteTimeoutMillis(label) ?? fallbackMillis;+ }++ /**+ * The timeout the router route with the given label asked for, or `undefined` when it did not override one+ * (or the request handler is not a router at all).+ */+ private getRouteTimeoutMillis(label: string | undefined): number | undefined {+ const getTimeoutSecs = (this.requestHandler as Partial<RouterHandler>).getTimeoutSecs;++ if (typeof getTimeoutSecs !== 'function') {+ return undefined;+ }++ const timeoutSecs = getTimeoutSecs(label);++ return timeoutSecs === undefined ? undefined : timeoutSecs * 1000;+ }+protected async runRequestHandler(crawlingContext: ExtendedContext): Promise<void> {+ const timeoutMillis = this.resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);+await addTimeoutToPromise(async () => this.requestHandler(crawlingContext),- this.requestHandlerTimeoutMillis,- `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds (${crawlingContext.request.id}).`,+ timeoutMillis,+ `requestHandler timed out after ${timeoutMillis / 1000} seconds (${crawlingContext.request.id}).`,);}@@ -2132,6 +2239,13 @@ export class BasicCrawler</** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */private async handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager, request: Request) {+ // An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,+ // which already failed the request in `runTaskFunction`. Bail before running the handler so it does not+ // execute (and re-report) on top of a request the crawler has already moved past.+ if ((crawlingContext as PendingCrawlingContext)[timeoutExpiredKey]?.()) {+ return;+ }+const statisticsId = request.id || request.uniqueKey;let isRequestLocked = true;diff --git a/packages/basic-crawler/src/internals/request-timeout.ts b/packages/basic-crawler/src/internals/request-timeout.tsnew file mode 100644index 000000000..d624db69c--- /dev/null+++ b/packages/basic-crawler/src/internals/request-timeout.ts@@ -0,0 +1,94 @@+import { TimeoutError } from '@apify/timeout';++/**+ * The shared navigation-window deadline (epoch millis), stored on the in-flight context so the pre- and+ * post-navigation hooks and the navigation itself all draw from one budget - and so `context.extendTimeout`+ * can push the whole window, not just the current step.+ * @internal+ */+export const navigationDeadlineKey = Symbol('navigationDeadline');++/**+ * Lets `context.extendTimeout` push back the internal request timeout as well. That timeout is a bare timer+ * rather than an `addTimeoutToPromise` frame, so `extendTimeout` from `@apify/timeout` cannot reach it on its own.+ */+export const extendTimeoutKey = Symbol('extendTimeout');++/**+ * Returns `true` once the internal request timeout has fired for this request. It cannot cancel work stuck+ * somewhere we do not control, but the phases we do run (e.g. the request handler) check this at their start and+ * bail, so a request whose timeout already elapsed does not carry on after the crawler moved past it.+ */+export const timeoutExpiredKey = Symbol('timeoutExpired');++/** The slots the internal request timeout hangs on the in-flight crawling context. */+export interface RequestTimeoutContext {+ [extendTimeoutKey]?: (extraMillis: number) => void;+ [timeoutExpiredKey]?: () => boolean;+ [navigationDeadlineKey]?: number;+}++/**+ * Milliseconds left in the shared navigation window for `ctx`, lazily starting the window on first use.+ * @internal+ */+export function remainingNavigationWindowMillis(ctx: object, windowMillis: number): number {+ const store = ctx as Record<symbol, number>;+ store[navigationDeadlineKey] ??= Date.now() + windowMillis;+ return store[navigationDeadlineKey] - Date.now();+}++/**+ * Races `work` against a `timeoutMillis` timeout, so a request stuck in a phase that has no timeout of its own+ * (anything that is not the navigation, a navigation hook or the request handler) still fails instead of stalling+ * the crawler.+ *+ * This is a bare timer, not an `addTimeoutToPromise` frame: nested `addTimeoutToPromise` calls share one+ * `AbortController`, so wrapping the request in one would let the request handler timing out abort this outer+ * context too, cancelling the error handling that follows it.+ *+ * `Promise.race` attaches a handler to both sides, so a late rejection from `work` cannot go unhandled. The losing+ * side is not cancelled - by definition it is stuck somewhere we do not control - but the phases we do run check+ * {@apilink timeoutExpiredKey} at their start and bail, so a request whose timeout already fired does not carry+ * on (e.g. run the handler) after the crawler has moved past it. `context.extendTimeout` pushes the deadline back+ * via {@apilink extendTimeoutKey}.+ */+export async function raceWithTimeout(+ context: RequestTimeoutContext,+ work: Promise<void>,+ { timeoutMillis, requestId }: { timeoutMillis: number; requestId?: string },+): Promise<void> {+ let timer: NodeJS.Timeout | undefined;+ let deadline = Date.now() + timeoutMillis;+ let settled = false;+ let firedByTimeout = false;++ const timeout = new Promise<never>((_, reject) => {+ const fire = () => {+ settled = true;+ firedByTimeout = true;+ reject(new TimeoutError(`Request timed out after ${timeoutMillis / 1e3} seconds (${requestId}).`));+ };++ timer = setTimeout(fire, timeoutMillis);++ context[timeoutExpiredKey] = () => firedByTimeout;++ context[extendTimeoutKey] = (extraMillis: number) => {+ if (settled) {+ return;+ }++ clearTimeout(timer);+ deadline += extraMillis;+ timer = setTimeout(fire, Math.max(deadline - Date.now(), 0));+ };+ });++ try {+ await Promise.race([work, timeout]);+ } finally {+ settled = true;+ clearTimeout(timer);+ }+}diff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.jsonindex 579533fba..14b2743b2 100644--- a/packages/browser-crawler/package.json+++ b/packages/browser-crawler/package.json@@ -47,7 +47,7 @@"access": "public"},"dependencies": {- "@apify/timeout": "^0.3.2",+ "@apify/timeout": "^0.4.4","@crawlee/basic": "workspace:*","@crawlee/browser-pool": "workspace:*","@crawlee/types": "workspace:*",diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 530f42b3d..c954349ce 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -19,9 +19,9 @@ import {ContextPipeline,cookieStringToToughCookie,enqueueLinks,- handleRequestTimeout,NavigationSkippedError,OwnedOrInjected,+ remainingNavigationWindowMillis,RequestState,resolveBaseUrlForEnqueueLinksFiltering,SessionError,@@ -53,7 +53,7 @@ import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@craimport ow from 'ow';import type { ReadonlyDeep } from 'type-fest';-import { tryCancel } from '@apify/timeout';+import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';import type { BrowserLaunchContext } from './browser-launcher.js';@@ -266,7 +266,11 @@ export interface BrowserCrawlerOptions<postNavigationHooks?: BrowserHook<Context, ContextExtension>[];/**- * Timeout in which page navigation needs to finish, in seconds.+ * Timeout for the whole navigation phase, in seconds. A single window shared by the `preNavigationHooks`,+ * the page navigation, and the `postNavigationHooks` - so a slow hook eats into the same budget the+ * navigation uses. Separate from the+ * {@apilink BasicCrawlerOptions.requestHandlerTimeoutSecs|`requestHandlerTimeoutSecs`}, which times only the+ * request handler.*/navigationTimeoutSecs?: number;@@ -294,6 +298,20 @@ export interface BrowserCrawlerOptions<ignoreIframes?: boolean;}+/**+ * Whether an error thrown by `page.goto()` is a navigation timeout - either our own {@apilink TimeoutError}+ * or the driver's, which Playwright/Puppeteer report with their own class and a `Timeout ... exceeded` message+ * naming the raw millisecond value rather than the configured window.+ */+function isNavigationTimeoutError(error: Error): boolean {+ return (+ error instanceof TimeoutError ||+ error?.name === 'TimeoutError' ||+ error?.constructor?.name === 'TimeoutError' ||+ /timeout.*exceeded/i.test(error?.message ?? '')+ );+}+/*** Provides a simple framework for parallel crawling of web pages* using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)@@ -421,16 +439,37 @@ export abstract class BrowserCrawler<super({...basicCrawlerOptions,contextPipelineBuilder: () => {+ // A single navigation window covers the pre-navigation hooks, the navigation, and the+ // post-navigation hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow+ // hook eats into the same window the navigation uses. The navigation itself is bounded by+ // capping its `gotoOptions.timeout` to the remaining budget.+ const windowGuard = <Ctx extends Context>(+ step: (ctx: Ctx) => Awaitable<void | Partial<Ctx>>,+ ): ContextMiddleware<Ctx, Partial<Ctx>> =>+ skipGuard(async (ctx: Ctx) => {+ const remaining = remainingNavigationWindowMillis(ctx, this.navigationTimeoutMillis);+ if (remaining <= 0) {+ throw new TimeoutError(+ `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`,+ );+ }+ return addTimeoutToPromise(+ async () => step(ctx),+ remaining,+ `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`,+ );+ });+let pipeline = contextPipelineBuilder().compose({ action: this.prepareNavigation.bind(this) });for (const hook of this.preNavigationHooks) {- pipeline = pipeline.compose(skipGuard(hook));+ pipeline = pipeline.compose(windowGuard(hook));}pipeline = pipeline.compose(skipGuard(this.navigate.bind(this)));for (const hook of this.postNavigationHooks) {- pipeline = pipeline.compose(skipGuard(hook));+ pipeline = pipeline.compose(windowGuard(hook));}return pipeline@@ -491,6 +530,10 @@ export abstract class BrowserCrawler<});}+ protected override getNavigationTimeoutMillis(): number {+ return this.navigationTimeoutMillis;+ }+protected override buildContextPipeline(): ContextPipeline<CrawlingContext,BrowserCrawlingContext<Page, Response, Dictionary>@@ -611,6 +654,8 @@ export abstract class BrowserCrawler<crawlingContext.request.state = RequestState.BEFORE_NAV;return {+ // Default to the full navigation timeout so a pre-navigation hook can read it; `navigate` narrows it+ // to the remaining shared window unless a hook overrode it (see there).gotoOptions: { timeout: this.navigationTimeoutMillis } as unknown as GoToOptions,[COOKIES_BEFORE_HOOKS]: this._getCookieHeaderFromRequest(crawlingContext.request),} as unknown as Partial<Context>;@@ -620,6 +665,18 @@ export abstract class BrowserCrawler<tryCancel();const gotoOptions = crawlingContext.gotoOptions as GoToOptions;+ const remaining = remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis);+ if (remaining <= 0) {+ throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);+ }+ // If a hook left the default `navigationTimeoutMillis` in place, bound the goto to whatever is left of the+ // shared navigation window. If it overrode the value - including `0`, Playwright's "no timeout" - honour+ // that verbatim as the goto's own timeout. The driver enforces this natively (so a timed-out goto is+ // aborted, not left lingering) and `handleNavigationTimeout` turns its error into our own message.+ const gotoTimeout = gotoOptions as { timeout?: number };+ if (gotoTimeout.timeout === this.navigationTimeoutMillis) {+ gotoTimeout.timeout = remaining;+ }const cookiesBeforeHooks = readContextField<string>(crawlingContext, COOKIES_BEFORE_HOOKS);const cookiesAfterHooks = this._getCookieHeaderFromRequest(crawlingContext.request);@@ -742,13 +799,16 @@ export abstract class BrowserCrawler<private async handleNavigationTimeout(crawlingContext: BrowserCrawlingContext, error: Error): Promise<void> {const { session, page } = crawlingContext;- if (error?.constructor.name === 'TimeoutError') {- handleRequestTimeout({ session, errorMessage: error.message });- }-// Fire-and-forget: no user code will run on this page after a failed navigation.// Swallow rejections: the page may already be detached.void page.evaluate(() => window.stop()).catch(() => {});++ if (isNavigationTimeoutError(error)) {+ session?.markBad();+ // The driver was handed the remaining window (usually shorter than `navigationTimeoutSecs` once the+ // hooks have run), so it names that value in its own error; report the configured window instead.+ throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);+ }}/**diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.jsonindex 45f8e93c6..3804982a5 100644--- a/packages/browser-pool/package.json+++ b/packages/browser-pool/package.json@@ -30,7 +30,7 @@"copy": "tsx ../../scripts/copy.ts"},"dependencies": {- "@apify/timeout": "^0.3.2",+ "@apify/timeout": "^0.4.4","@crawlee/core": "workspace:*","@crawlee/types": "workspace:*","fingerprint-generator": "^2.1.68",diff --git a/packages/core/package.json b/packages/core/package.jsonindex e8f2b35ef..637be7d55 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -51,7 +51,7 @@"@apify/datastructures": "^2.0.3","@apify/log": "^2.5.18","@apify/pseudo_url": "^2.0.59",- "@apify/timeout": "^0.3.2",+ "@apify/timeout": "^0.4.4","@apify/utilities": "^2.15.5","@crawlee/fs-storage": "workspace:*","@crawlee/types": "workspace:*",diff --git a/packages/core/src/configuration.ts b/packages/core/src/configuration.tsindex b01c13642..4a8402945 100644--- a/packages/core/src/configuration.ts+++ b/packages/core/src/configuration.ts@@ -67,6 +67,11 @@ export const crawleeConfigFields = {memoryMbytes: field(coerceNumber.optional(), 'CRAWLEE_MEMORY_MBYTES'),/** @default 60_000 */persistStateIntervalMillis: field(coerceNumber.default(60_000), 'CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS'),+ /**+ * Internal safety-net timeout for a single request, in milliseconds. When unset the crawler derives it from+ * the request handler timeout (twice it, and never below 5 minutes).+ */+ internalTimeoutMillis: field(coerceNumber.optional(), 'CRAWLEE_INTERNAL_TIMEOUT'),/** @default 1_000 */systemInfoIntervalMillis: field(coerceNumber.default(1_000)),/** @default 'INPUT' */@@ -155,6 +160,7 @@ export interface Configuration extends ResolvedConfigValues {}* `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`* `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`* `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`+ * `internalTimeoutMillis` | `CRAWLEE_INTERNAL_TIMEOUT` | -* `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`* `persistStorage` | `CRAWLEE_PERSIST_STORAGE` | `true`* `storageDir` | `CRAWLEE_STORAGE_DIR` | `'./storage'`diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.tsindex 70fa454a7..ab30cf840 100644--- a/packages/core/src/crawlers/crawler_commons.ts+++ b/packages/core/src/crawlers/crawler_commons.ts@@ -237,6 +237,26 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten* Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.*/registerDeferredCleanup(cleanup: () => Promise<unknown>): void;++ /**+ * Gives the current request `secs` more seconds to finish, for when how long it needs is only apparent+ * once it is already running - a listing page that turns out to have far more to scroll through than+ * usual, say. Prefer `requestHandlerTimeoutSecs`, or a per-route override via+ * {@apilink Router.addHandler|`router.addHandler`}, whenever the time needed is known up front.+ *+ * ```ts+ * router.addHandler('LIST', async ({ extendTimeout, page }) => {+ * const pageCount = await countPages(page);+ * extendTimeout(pageCount * 10);+ * await scrapeAllPages(page);+ * });+ * ```+ *+ * Extends the request handler's own timeout and the crawler's internal one together, so the extension+ * is not immediately undone by the latter. Calling it from a handler that has already timed out does+ * nothing.+ */+ extendTimeout(secs: number): void;}/**diff --git a/packages/core/src/crawlers/crawler_utils.ts b/packages/core/src/crawlers/crawler_utils.tsdeleted file mode 100644index 992ef595e..000000000--- a/packages/core/src/crawlers/crawler_utils.ts+++ /dev/null@@ -1,14 +0,0 @@-import type { ISession } from '@crawlee/types';--import { TimeoutError } from '@apify/timeout';--/**- * Handles timeout request- * @internal- */-export function handleRequestTimeout({ session, errorMessage }: { session?: ISession; errorMessage: string }) {- session?.markBad();- const timeoutMillis = /(\d+)\s?ms/.exec(errorMessage)?.[1]; // first capturing group- const timeoutSecs = Number(timeoutMillis) / 1000;- throw new TimeoutError(`Navigation timed out after ${timeoutSecs} seconds.`);-}diff --git a/packages/core/src/crawlers/index.ts b/packages/core/src/crawlers/index.tsindex 5fa44b458..fc8ce2593 100644--- a/packages/core/src/crawlers/index.ts+++ b/packages/core/src/crawlers/index.ts@@ -1,6 +1,5 @@export * from './context_pipeline.js';export * from './crawler_commons.js';-export * from './crawler_utils.js';export * from './statistics.js';export * from './error_tracker.js';export * from './error_snapshotter.js';diff --git a/packages/core/src/router.ts b/packages/core/src/router.tsindex 23d0ab399..6a402bd56 100644--- a/packages/core/src/router.ts+++ b/packages/core/src/router.ts@@ -131,6 +131,21 @@ export interface RouterHandler<export type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;+/**+ * Per-route overrides, passed as the last argument of {@apilink Router.addHandler|`addHandler`} and+ * {@apilink Router.addDefaultHandler|`addDefaultHandler`}.+ */+export interface RouteOptions {+ /**+ * Overrides the crawler's `requestHandlerTimeoutSecs` for this route only. Useful when one kind of page+ * needs markedly more time than the rest - a listing page behind an infinite scroll, say - and you do not+ * want to raise the timeout for every other page to accommodate it.+ *+ * Applies only to this route's handler. The navigation and the navigation hooks keep their own timeouts.+ */+ requestHandlerTimeoutSecs?: number;+}+export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary>> = {[Label in keyof Routes]: (ctx: Omit<Context, 'request'> & { request: Request<Routes[Label]> }) => Awaitable<void>;};@@ -240,6 +255,26 @@ export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary* request.userData.price; // number, inferred from the schema and validated at runtime* });* ```+ *+ * A single route can take longer than the rest without raising the crawler-wide+ * `requestHandlerTimeoutSecs` for everything - pass a per-route timeout as the last argument:+ *+ * ```ts+ * // LIST pages scroll through a lot of content, DETAIL pages are quick+ * router.addHandler('LIST', async (ctx) => { ... }, { requestHandlerTimeoutSecs: 120 });+ * router.addHandler('DETAIL', async (ctx) => { ... }); // keeps the crawler's default+ * ```+ *+ * When the time a route needs is only apparent once it is already running, call+ * {@apilink CrawlingContext.extendTimeout|`context.extendTimeout`} from inside the handler:+ *+ * ```ts+ * router.addHandler('LIST', async ({ page, extendTimeout }) => {+ * const pageCount = await countPages(page);+ * extendTimeout(pageCount * 10); // ask for 10 more seconds per page+ * await scrapeAllPages(page);+ * });+ * ```*/export class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>,@@ -247,6 +282,7 @@ export class Router<> {private readonly routes: Map<string | symbol, (ctx: any) => Awaitable<void>> = new Map();private readonly schemas: Map<string | symbol, StandardSchemaV1> = new Map();+ private readonly timeouts: Map<string | symbol, number> = new Map();private readonly middlewares: ((ctx: Context) => Awaitable<void>)[] = [];/**@@ -257,11 +293,14 @@ export class Router</*** Registers new route handler for given label. When the router declares a route map, the- * `label` is restricted to the declared labels and `request.userData` is typed accordingly.+ * `label` is restricted to the declared labels and `request.userData` is typed accordingly. Pass+ * {@apilink RouteOptions|`options`} to give this route its own `requestHandlerTimeoutSecs`,+ * overriding the crawler's default for requests with this label.*/addHandler<Label extends keyof Routes & string>(label: Label,handler: (ctx: RouterHandlerContext<Context, Routes[Label], Routes>) => Awaitable<void>,+ options?: RouteOptions,): void;/**@@ -272,24 +311,35 @@ export class Router<addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: RouterLabel<Routes>,handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>,+ options?: RouteOptions,): void;- addHandler(label: string | symbol, handler: (ctx: any) => Awaitable<void>): void {+ addHandler(label: string | symbol, handler: (ctx: any) => Awaitable<void>, options: RouteOptions = {}): void {this.validate(label);this.routes.set(label, handler);++ if (options.requestHandlerTimeoutSecs !== undefined) {+ this.timeouts.set(label, options.requestHandlerTimeoutSecs);+ }}/*** Registers default route handler. As a fallback it can receive any request (including labels not* declared in the route map). When the router was created with a {@apilink defaultRoute} schema,* `request.userData` is typed from it; otherwise it defaults to the context's (loosely typed) `userData`.- * Pass an explicit `UserData` type argument to narrow it.+ * Pass an explicit `UserData` type argument to narrow it. Pass {@apilink RouteOptions|`options`} to give the+ * default route its own `requestHandlerTimeoutSecs`, overriding the crawler's default for requests that fall+ * through to it.*/addDefaultHandler<UserData extends Dictionary = DefaultRouteUserData<Routes, GetUserDataFromRequest<Context['request']>>,- >(handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>) {+ >(handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>, options: RouteOptions = {}) {this.validate(defaultRoute);this.routes.set(defaultRoute, handler);++ if (options.requestHandlerTimeoutSecs !== undefined) {+ this.timeouts.set(defaultRoute, options.requestHandlerTimeoutSecs);+ }}/**@@ -324,6 +374,28 @@ export class Router<this.middlewares.push(middleware);}+ /**+ * Returns the `requestHandlerTimeoutSecs` registered for a label, or `undefined` when the route did not+ * override it and the crawler's own timeout should apply. Falls back to the default route the same way+ * {@apilink Router.getHandler|`getHandler`} does, so a label with no route of its own inherits whatever+ * the default route asked for. Used by the crawler; not meant to be called directly.+ */+ getTimeoutSecs(label?: string | symbol): number | undefined {+ if (label && this.routes.has(label)) {+ return this.timeouts.get(label);+ }++ return this.timeouts.get(defaultRoute);+ }++ /**+ * The longest `requestHandlerTimeoutSecs` any route asked for, or `undefined` when no route overrides it.+ * The crawler needs an upper bound up front, before it knows which routes a run will actually hit.+ */+ getMaxTimeoutSecs(): number | undefined {+ return this.timeouts.size > 0 ? Math.max(...this.timeouts.values()) : undefined;+ }+/*** Returns route handler for given label. If no label is provided, the default request handler will be returned.*/@@ -423,6 +495,8 @@ export class Router<obj.addDefaultHandler = router.addDefaultHandler.bind(router);obj.getSchema = router.getSchema.bind(router);obj.getHandler = router.getHandler.bind(router);+ obj.getTimeoutSecs = router.getTimeoutSecs.bind(router);+ obj.getMaxTimeoutSecs = router.getMaxTimeoutSecs.bind(router);obj.use = router.use.bind(router);// `Reflect.ownKeys` (unlike `Object.entries`) also yields the `defaultRoute` symbol key.diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.jsonindex 61877839b..0f82063d8 100644--- a/packages/http-crawler/package.json+++ b/packages/http-crawler/package.json@@ -47,7 +47,7 @@"access": "public"},"dependencies": {- "@apify/timeout": "^0.3.2",+ "@apify/timeout": "^0.4.4","@apify/utilities": "^2.15.5","@crawlee/basic": "workspace:*","@crawlee/core": "workspace:*",diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex a56a27d33..f303d8f63 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -21,6 +21,7 @@ import {BasicCrawler,ContextPipeline,NavigationSkippedError,+ remainingNavigationWindowMillis,RequestState,Router,SessionError,@@ -35,7 +36,7 @@ import iconv from 'iconv-lite';import ow from 'ow';import type { JsonValue } from 'type-fest';-import { addTimeoutToPromise, tryCancel } from '@apify/timeout';+import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';@@ -79,7 +80,11 @@ export interface HttpCrawlerOptions<Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes> {/**- * Timeout in which the HTTP request to the resource needs to finish, given in seconds.+ * Timeout for the whole navigation phase, given in seconds. A single window shared by the+ * `preNavigationHooks`, the navigation (the HTTP request to the resource), and the `postNavigationHooks` -+ * so a slow hook eats into the same budget the navigation uses. Separate from the+ * {@apilink BasicCrawlerOptions.requestHandlerTimeoutSecs|`requestHandlerTimeoutSecs`}, which times only the+ * request handler.*/navigationTimeoutSecs?: number;@@ -422,6 +427,10 @@ export class HttpCrawler<this.saveResponseCookies = saveResponseCookies;}+ protected override getNavigationTimeoutMillis(): number {+ return this.navigationTimeoutMillis;+ }+/*** Folds {@apilink HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's* concurrency shortcuts on top. Not called for a supplied@@ -444,18 +453,33 @@ export class HttpCrawler<action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})) as Ext,});+ // A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation+ // hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same+ // window the navigation uses instead of each step being timed on its own.+ const navigationTimedOut = `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`;+ const windowGuard = <Ctx extends CrawlingContext, Ext>(+ step: (ctx: Ctx) => Awaitable<void | Ext>,+ ): ContextMiddleware<Ctx, Ext> =>+ skipGuard(async (ctx: Ctx) => {+ const remaining = remainingNavigationWindowMillis(ctx, this.navigationTimeoutMillis);+ if (remaining <= 0) {+ throw new TimeoutError(navigationTimedOut);+ }+ return addTimeoutToPromise(async () => step(ctx), remaining, navigationTimedOut);+ });+let pipeline = ContextPipeline.create<CrawlingContext>().compose({action: this.prepareHttpRequest.bind(this),});for (const hook of this.preNavigationHooks) {- pipeline = pipeline.compose(skipGuard(hook));+ pipeline = pipeline.compose(windowGuard(hook));}let pipelineWithNavigation = pipeline.compose(skipGuard(this.makeHttpRequest.bind(this)));for (const hook of this.postNavigationHooks) {- pipelineWithNavigation = pipelineWithNavigation.compose(skipGuard(hook));+ pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));}return pipelineWithNavigation@@ -498,10 +522,13 @@ export class HttpCrawler<const { request, session } = crawlingContext;const proxyUrl = crawlingContext.proxyInfo?.url;+ // Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may+ // have already spent part of it), so it produces a clean navigation-timeout error rather than the raw+ // client abort.const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }),- this.navigationTimeoutMillis,- `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`,+ Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis)),+ `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`,);tryCancel();@@ -548,7 +575,18 @@ export class HttpCrawler<tryCancel();- const parsed = await this.parseResponse(crawlingContext.request, crawlingContext.response);+ // Reading the body is still part of the navigation, so it draws from the same shared window: on a server+ // that streams the body slowly the request completes (headers arrive) but the body read would otherwise+ // run unbounded. `extendTimeout` from a post-navigation hook has already pushed this deadline out if asked.+ const remaining = remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis);+ if (remaining <= 0) {+ throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);+ }+ const parsed = await addTimeoutToPromise(+ async () => this.parseResponse(crawlingContext.request, crawlingContext.response),+ remaining,+ `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`,+ );tryCancel();const response = parsed.response!;const contentType = parsed.contentType!;@@ -810,7 +848,7 @@ export class HttpCrawler<*/private handleRequestTimeout(session: ISession) {session.markBad();- throw new Error(`request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);+ throw new Error(`Request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);}private _abortDownloadOfBody(request: CrawleeRequest, response: Response) {@@ -842,6 +880,13 @@ export class HttpCrawler<// hooks keeps working) but a per-request clone is passed in so writes are discarded.const cookieJar = this.saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();+ // Bind the request to the shared navigation window instead of a fixed per-request timeout, so+ // `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and+ // kill a lazily-read body mid-extension. This aborts the socket only during the header phase; the body+ // read is bounded separately at the promise level (see `processHttpResponse`), so a slow-streaming body+ // still fails cleanly with a navigation timeout, though the socket is left to close on its own.+ const cancelSignal = storage.getStore()?.cancelTask.signal;+const response = await this.httpClient.sendRequest(new Request(opts.url, {body: opts.body ? (Readable.toWeb(opts.body) as any) : undefined,@@ -853,7 +898,8 @@ export class HttpCrawler<{session,cookieJar,- timeoutMillis: opts.timeout,+ signal: cancelSignal,+ timeoutMillis: cancelSignal ? undefined : opts.timeout,},);diff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.jsonindex 147704170..98294bac4 100644--- a/packages/jsdom-crawler/package.json+++ b/packages/jsdom-crawler/package.json@@ -47,7 +47,7 @@"access": "public"},"dependencies": {- "@apify/timeout": "^0.3.0",+ "@apify/timeout": "^0.4.4","@apify/utilities": "^2.7.10","@crawlee/http": "workspace:*","@crawlee/types": "workspace:*",diff --git a/packages/linkedom-crawler/package.json b/packages/linkedom-crawler/package.jsonindex d508ebb48..427fcfe5f 100644--- a/packages/linkedom-crawler/package.json+++ b/packages/linkedom-crawler/package.json@@ -47,7 +47,7 @@"access": "public"},"dependencies": {- "@apify/timeout": "^0.3.2",+ "@apify/timeout": "^0.4.4","@apify/utilities": "^2.15.5","@crawlee/http": "workspace:*","@crawlee/types": "workspace:*",diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.jsonindex 9240ed765..0d9399856 100644--- a/packages/playwright-crawler/package.json+++ b/packages/playwright-crawler/package.json@@ -48,7 +48,7 @@},"dependencies": {"@apify/datastructures": "^2.0.3",- "@apify/timeout": "^0.3.2",+ "@apify/timeout": "^0.4.4","@crawlee/basic": "workspace:*","@crawlee/browser": "workspace:*","@crawlee/browser-pool": "workspace:*",diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex ca8c5d9be..36ae9e14d 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -361,6 +361,7 @@ export class AdaptivePlaywrightCrawler<errorHandler,failedRequestHandler,requestHandler,+ requestHandlerTimeoutSecs,contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),});this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;@@ -608,6 +609,14 @@ export class AdaptivePlaywrightCrawler<}};+ // this crawler overrides `runRequestHandler` and times each rendering-type run itself, so it has+ // to resolve any per-route override too - otherwise routes would be silently ignored here+ const routeTimeoutSecs = (this.requestHandler as Partial<RouterHandler>).getTimeoutSecs?.(+ context.request.label,+ );+ const timeoutMillis =+ routeTimeoutSecs === undefined ? this.individualRequestHandlerTimeoutMillis : routeTimeoutSecs * 1000;+await addTimeoutToPromise(async () =>withCheckedStorageAccess(() => {@@ -617,7 +626,7 @@ export class AdaptivePlaywrightCrawler<);}}, callAdaptiveRequestHandler),- this.individualRequestHandlerTimeoutMillis,+ timeoutMillis,'Request handler timed out',);diff --git a/packages/stagehand-crawler/package.json b/packages/stagehand-crawler/package.jsonindex a49e78d85..f8d18bbed 100644--- a/packages/stagehand-crawler/package.json+++ b/packages/stagehand-crawler/package.json@@ -56,7 +56,7 @@"access": "public"},"dependencies": {- "@apify/timeout": "^0.3.2",+ "@apify/timeout": "^0.4.4","@crawlee/browser": "workspace:*","@crawlee/browser-pool": "workspace:*","@crawlee/core": "workspace:*",diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex bdb34a185..84bdbce3e 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -300,8 +300,8 @@ importers:specifier: ^2.0.0version: 2.0.4'@apify/timeout':- specifier: ^0.3.2- version: 0.3.3+ specifier: ^0.4.4+ version: 0.4.4'@apify/utilities':specifier: ^2.15.5version: 2.27.0@@ -343,8 +343,8 @@ importers:packages/browser-crawler:dependencies:'@apify/timeout':- specifier: ^0.3.2- version: 0.3.3+ specifier: ^0.4.4+ version: 0.4.4'@crawlee/basic':specifier: workspace:*version: link:../basic-crawler@@ -376,8 +376,8 @@ importers:packages/browser-pool:dependencies:'@apify/timeout':- specifier: ^0.3.2- version: 0.3.3+ specifier: ^0.4.4+ version: 0.4.4'@crawlee/core':specifier: workspace:*version: link:../core@@ -478,8 +478,8 @@ importers:specifier: ^2.0.59version: 2.0.76'@apify/timeout':- specifier: ^0.3.2- version: 0.3.3+ specifier: ^0.4.4+ version: 0.4.4'@apify/utilities':specifier: ^2.15.5version: 2.27.0@@ -631,8 +631,8 @@ importers:packages/http-crawler:dependencies:'@apify/timeout':- specifier: ^0.3.2- version: 0.3.3+ specifier: ^0.4.4+ version: 0.4.4'@apify/utilities':specifier: ^2.15.5version: 2.27.0@@ -697,8 +697,8 @@ importers:packages/jsdom-crawler:dependencies:'@apify/timeout':- specifier: ^0.3.0- version: 0.3.3+ specifier: ^0.4.4+ version: 0.4.4'@apify/utilities':specifier: ^2.7.10version: 2.27.0@@ -730,8 +730,8 @@ importers:packages/linkedom-crawler:dependencies:'@apify/timeout':- specifier: ^0.3.2- version: 0.3.3+ specifier: ^0.4.4+ version: 0.4.4'@apify/utilities':specifier: ^2.15.5version: 2.27.0@@ -763,8 +763,8 @@ importers:specifier: ^2.0.3version: 2.0.4'@apify/timeout':- specifier: ^0.3.2- version: 0.3.3+ specifier: ^0.4.4+ version: 0.4.4'@crawlee/basic':specifier: workspace:*version: link:../basic-crawler@@ -862,8 +862,8 @@ importers:packages/stagehand-crawler:dependencies:'@apify/timeout':- specifier: ^0.3.2- version: 0.3.3+ specifier: ^0.4.4+ version: 0.4.4'@crawlee/browser':specifier: workspace:*version: link:../browser-crawler@@ -1322,6 +1322,9 @@ packages:'@apify/[email protected]':resolution: {integrity: sha512-lyvwMXee8SJNjNyxhr+nSTNyvjyoxbxol51xikq9VytFOPNSEMz8N02mUAuLVJNqrnqCBFRybjeqZdg4Y5AZlA==}+ '@apify/[email protected]':+ resolution: {integrity: sha512-WOjdkuNRSo09ikse47t6jIW4R55FEmZdOaoUTbSmpwfmNNHVht+iTgoqeHIVr7vvI9s+mtTmvzPoesoZa0cjVA==}+'@apify/[email protected]':resolution: {integrity: sha512-0i6rF+hgXw6mQXhENYNTnP+Yo4WY38d698k/pk4rIpdCx/sSuEprR2N5WdtMaN628vCB+aZjff0jPN2EnQYtjQ==}@@ -12965,6 +12968,8 @@ snapshots:'@apify/[email protected]': {}+ '@apify/[email protected]': {}+'@apify/[email protected]': {}diff --git a/test/core/crawlers/adaptive_playwright_crawler.test.ts b/test/core/crawlers/adaptive_playwright_crawler.test.tsindex b7ea362a9..119c9731a 100644--- a/test/core/crawlers/adaptive_playwright_crawler.test.ts+++ b/test/core/crawlers/adaptive_playwright_crawler.test.ts@@ -24,6 +24,7 @@ import {fullResultComparator,RenderingTypePredictor,RequestList,+ RequestQueue,RequestValidationError,} from '@crawlee/playwright';import type { Dictionary } from '@crawlee/types';@@ -861,4 +862,18 @@ describe('AdaptivePlaywrightCrawler', () => {crawler.addRequests([{ url: 'https://example.com/a', label: 'DETAIL', userData: { id: 123 } }] as never),).rejects.toThrow(RequestValidationError);});++ test('reserves a request for the longest route timeout override', async () => {+ const requestQueue = await RequestQueue.open(`rq-adaptive-${Math.random() * 10000}`);+ const hintSpy = vitest.spyOn(requestQueue, 'setExpectedRequestProcessingTimeSecs');++ const router = createAdaptivePlaywrightRouter();+ router.addHandler('LIST', async () => {}, { requestHandlerTimeoutSecs: 300 });++ const crawler = new AdaptivePlaywrightCrawler({ requestQueue, requestHandler: router });+ await crawler.getRequestManager();++ const maxHint = Math.max(...hintSpy.mock.calls.map((call) => call[0]));+ expect(maxHint).toBeGreaterThanOrEqual(300);+ });});diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 1c30154d4..3cc1103d3 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -7,6 +7,7 @@ import type { EnqueueLinksOptions, ErrorHandler, RequestHandler, RequestOptions,import type { Session } from '@crawlee/basic';import {BasicCrawler,+ Configuration,CriticalError,defaultRoute,EventType,@@ -1516,6 +1517,203 @@ describe('BasicCrawler', () => {results[0].errorMessages.forEach((msg) => expect(msg).toMatch('requestHandler timed out'));});+ test('context.extendTimeout buys a running handler more time', async () => {+ const requestList = await RequestList.open({ sources: [{ url: 'https://example.com' }] });++ const failed: Request[] = [];++ const crawler = new BasicCrawler({+ requestList,+ requestHandlerTimeoutSecs: 0.2,+ maxRequestRetries: 0,+ requestHandler: async ({ extendTimeout }) => {+ await sleep(100);+ // only now do we know we need longer than the 0.2s we were given+ extendTimeout(5);+ await sleep(300);+ },+ failedRequestHandler: async ({ request }) => {+ failed.push(request);+ },+ });++ await crawler.run();++ // the handler needs 400ms in total, so without the extension it would have timed out+ expect(failed).toHaveLength(0);+ });++ test('context.extendTimeout also holds off the internal timeout', async () => {+ // shorter than the 400ms the handler needs, so an unextended backstop would definitely cut it+ serviceLocator.reset();+ serviceLocator.setStorageBackend(new MemoryStorageBackend());+ serviceLocator.setConfiguration(new Configuration({ internalTimeoutMillis: 250 }));++ const requestList = await RequestList.open({ sources: [{ url: 'https://example.com' }] });++ const failed: Request[] = [];++ const crawler = new BasicCrawler({+ requestList,+ requestHandlerTimeoutSecs: 0.2,+ maxRequestRetries: 0,+ requestHandler: async ({ extendTimeout }) => {+ await sleep(100);+ extendTimeout(5);+ await sleep(300);+ },+ failedRequestHandler: async ({ request }) => {+ failed.push(request);+ },+ });++ await crawler.run();++ // extending only the handler would be pointless if the backstop cut it down anyway+ expect(failed).toHaveLength(0);+ });++ test('a route can override requestHandlerTimeoutSecs, other routes keep the default', async () => {+ const requestList = await RequestList.open({+ sources: [+ { url: 'https://example.com/list', userData: { label: 'LIST' } },+ { url: 'https://example.com/detail', userData: { label: 'DETAIL' } },+ ],+ });++ const processed: Request[] = [];+ const failed: Request[] = [];++ const router = Router.create();+ // LIST is allowed to take its time, DETAIL is left on the crawler's default and must not be+ router.addHandler(+ 'LIST',+ async ({ request }) => {+ await sleep(300);+ processed.push(request as Request);+ },+ { requestHandlerTimeoutSecs: 5 },+ );+ router.addHandler('DETAIL', async ({ request }) => {+ await sleep(300);+ processed.push(request as Request);+ });++ const crawler = new BasicCrawler({+ requestList,+ requestHandlerTimeoutSecs: 0.1,+ maxRequestRetries: 0,+ requestHandler: router,+ failedRequestHandler: async ({ request }) => {+ failed.push(request);+ },+ });++ await crawler.run();++ // only DETAIL is held to the crawler's 0.1s default; LIST got its own 5s and finished+ // (`processed` alone would not prove it - cancellation is cooperative, so a timed out handler+ // keeps running to completion and still pushes)+ expect(failed.map((request) => request.label)).toEqual(['DETAIL']);+ expect(processed.map((request) => request.label)).toContain('LIST');+ failed[0].errorMessages.forEach((msg) => expect(msg).toMatch('requestHandler timed out after 0.1 seconds'));+ });++ test('internal timeout catches a request stuck outside the timed phases', async () => {+ // reset so we can install a configuration with a custom internal timeout before anything resolves it+ serviceLocator.reset();+ serviceLocator.setStorageBackend(new MemoryStorageBackend());+ serviceLocator.setConfiguration(new Configuration({ internalTimeoutMillis: 100 }));++ const url = 'https://example.com';+ const requestList = await RequestList.open({ sources: [{ url }] });++ const results: Request[] = [];+ const requestHandler = vitest.fn();++ const crawler = new BasicCrawler({+ requestList,+ maxRequestRetries: 0,+ // keep the handler timeout small so the backstop is not floored above the 100ms set above+ requestHandlerTimeoutSecs: 0.05,+ // `extendContext` is not the navigation, the hooks or the request handler, so none of their+ // timeouts apply to it - only the internal one stands between this and a stuck crawler. It also+ // outlives the backstop, so it is the case where the losing side of the race keeps running.+ extendContext: async () => {+ await sleep(500);+ return {};+ },+ requestHandler,+ failedRequestHandler: async ({ request }) => {+ results.push(request);+ },+ });++ await crawler.run();++ // Wait past the stuck `extendContext`: `run()` resolves when the backstop fails the request, but the+ // pipeline keeps running underneath - the handler must still not fire once it finally gets there.+ await sleep(600);++ expect(requestHandler).not.toHaveBeenCalled();+ expect(results).toHaveLength(1);++ results[0].errorMessages.forEach((msg) => {+ expect(msg).toMatch('Request timed out');+ // the request handler never even started, so blaming it would be a lie+ expect(msg).not.toMatch('requestHandler timed out');+ });+ });++ test('the internal timeout does not cut a request handler short of its own timeout', async () => {+ // deliberately below the request handler timeout - the backstop must still not fire before it+ serviceLocator.reset();+ serviceLocator.setStorageBackend(new MemoryStorageBackend());+ serviceLocator.setConfiguration(new Configuration({ internalTimeoutMillis: 100 }));++ const url = 'https://example.com';+ const requestList = await RequestList.open({ sources: [{ url }] });++ const processed: Request[] = [];++ const crawler = new BasicCrawler({+ requestList,+ maxRequestRetries: 0,+ requestHandlerTimeoutSecs: 1,+ requestHandler: async ({ request }) => {+ // comfortably within the 1s handler timeout, but well past the 100ms internal one+ await sleep(500);+ processed.push(request as Request);+ },+ });++ await crawler.run();++ expect(processed).toHaveLength(1);+ });++ test('warns when the internal timeout is shorter than the phase timeouts', async () => {+ serviceLocator.reset();+ serviceLocator.setStorageBackend(new MemoryStorageBackend());+ serviceLocator.setConfiguration(new Configuration({ internalTimeoutMillis: 100 }));++ const requestList = await RequestList.open({ sources: [{ url: 'https://example.com' }] });+ const crawler = new BasicCrawler({+ requestList,+ requestHandlerTimeoutSecs: 1, // 1000ms, well above the 100ms internal timeout+ maxRequestRetries: 0,+ requestHandler: async () => {},+ });+ const warnSpy = vitest.spyOn(crawler.log, 'warning');++ await crawler.run();++ const warned = warnSpy.mock.calls.some((call) =>+ String(call[0]).includes('shorter than the navigation and request handler'),+ );+ expect(warned).toBe(true);+ });+test('timeouted request should not access storages', async () => {const url = 'https://example.com';const requestList = await RequestList.open({ sources: [{ url }] });diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.tsindex 45b7930d3..11619d717 100644--- a/test/core/crawlers/browser_crawler.test.ts+++ b/test/core/crawlers/browser_crawler.test.ts@@ -269,6 +269,38 @@ describe('BrowserCrawler', () => {expect(hook).toHaveBeenCalled();});+ test.concurrent('should time out a hanging preNavigationHook after navigationTimeoutSecs', async () => {+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);++ const requestList = await RequestList.open({+ sources: [{ url: `${serverAddress}/?q=1` }],+ });++ const failed: Request[] = [];+ const requestHandler = vi.fn();++ const browserCrawler = new BrowserCrawlerTest({+ browserPoolOptions: {+ browserPlugins: [puppeteerPlugin],+ },+ requestList,+ navigationTimeoutSecs: 0.1,+ maxRequestRetries: 0,+ preNavigationHooks: [async () => sleep(5000)],+ requestHandler,+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ await browserCrawler.run();++ expect(requestHandler).not.toHaveBeenCalled();+ expect(failed).toHaveLength(1);+ expect(failed[0].errorMessages[0]).toMatch('Navigation timed out');+ expect(failed[0].errorMessages[0]).not.toMatch('requestHandler timed out');+ });+test.concurrent('should evaluate postNavigationHooks', async () => {const puppeteerPlugin = new PuppeteerPlugin(puppeteer);diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex 65aa76044..13c21976e 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -1,6 +1,7 @@-import type { Server } from 'node:http';+import { createServer, type Server } from 'node:http';import type { BasicCrawlingContext, CheerioCrawlingContext, CheerioRequestHandler, Source } from '@crawlee/cheerio';+import { FetchHttpClient } from '@crawlee/http-client';import {CheerioCrawler,createCheerioRouter,@@ -352,8 +353,8 @@ describe('CheerioCrawler', () => {failed.forEach((request) => {expect(request.errorMessages).toHaveLength(2);- expect(request.errorMessages[0]).toMatch('request timed out');- expect(request.errorMessages[1]).toMatch('request timed out');+ expect(request.errorMessages[0]).toMatch('Navigation timed out');+ expect(request.errorMessages[1]).toMatch('Navigation timed out');});});@@ -392,6 +393,208 @@ describe('CheerioCrawler', () => {);});});++ test('when a pre-navigation hook exceeds the navigation window', async () => {+ const failed: Request[] = [];+ const requestList = await getExampleRequestList();+ const requestHandler = vi.fn();++ const cheerioCrawler = new CheerioCrawler({+ requestList,+ navigationTimeoutSecs: 0.1,+ maxRequestRetries: 1,+ minConcurrency: 2,+ maxConcurrency: 2,+ preNavigationHooks: [async () => sleep(2000)],+ requestHandler,+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ await cheerioCrawler.run();++ expect(requestHandler).not.toHaveBeenCalled();+ expect(failed).toHaveLength(4);++ failed.forEach((request) => {+ expect(request.errorMessages).toHaveLength(2);++ request.errorMessages.forEach((message) => {+ // a hook overrunning shares the navigation window, so it reads as a navigation timeout+ expect(message).toMatch('Navigation timed out');+ // ...not the request handler, which is timed separately+ expect(message).not.toMatch('requestHandler timed out');+ });+ });+ });++ test('after navigationTimeoutSecs in a post-navigation hook', async () => {+ const failed: Request[] = [];+ const requestList = await getExampleRequestList();+ const requestHandler = vi.fn();++ const cheerioCrawler = new CheerioCrawler({+ requestList,+ navigationTimeoutSecs: 0.1,+ maxRequestRetries: 1,+ minConcurrency: 2,+ maxConcurrency: 2,+ postNavigationHooks: [async () => sleep(2000)],+ requestHandler,+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ await cheerioCrawler.run();++ expect(requestHandler).not.toHaveBeenCalled();+ expect(failed).toHaveLength(4);++ failed.forEach((request) => {+ request.errorMessages.forEach((message) => {+ expect(message).toMatch('Navigation timed out');+ });+ });+ });++ test('navigation hooks that finish in time do not affect the handler timeout', async () => {+ const processed: Request[] = [];+ const requestList = await getExampleRequestList();++ const cheerioCrawler = new CheerioCrawler({+ requestList,+ // the hooks eat 300ms in total, which must not be charged to the 1s handler window+ navigationTimeoutSecs: 1,+ requestHandlerTimeoutSecs: 1,+ maxRequestRetries: 0,+ minConcurrency: 2,+ maxConcurrency: 2,+ preNavigationHooks: [async () => sleep(150)],+ postNavigationHooks: [async () => sleep(150)],+ requestHandler: async ({ request }) => {+ await sleep(700);+ processed.push(request);+ },+ });++ await cheerioCrawler.run();++ expect(processed).toHaveLength(4);+ });++ test('extendTimeout from a pre-navigation hook keeps it from timing out', async () => {+ const processed: Request[] = [];+ const failed: Request[] = [];+ const requestList = await getExampleRequestList();++ const cheerioCrawler = new CheerioCrawler({+ requestList,+ navigationTimeoutSecs: 0.2,+ maxRequestRetries: 0,+ minConcurrency: 2,+ maxConcurrency: 2,+ // 400ms total, past the 0.2s window - only survives because the hook asks for more time+ preNavigationHooks: [+ async ({ extendTimeout }) => {+ await sleep(100);+ extendTimeout(5);+ await sleep(300);+ },+ ],+ requestHandler: async ({ request }) => {+ processed.push(request);+ },+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ await cheerioCrawler.run();++ expect(failed).toHaveLength(0);+ expect(processed).toHaveLength(4);+ });++ test('extendTimeout from a post-navigation hook keeps it from timing out', async () => {+ const processed: Request[] = [];+ const failed: Request[] = [];+ const requestList = await getExampleRequestList();++ const cheerioCrawler = new CheerioCrawler({+ requestList,+ // enough for the (fast, local) navigation, but far short of the hook below+ navigationTimeoutSecs: 1,+ maxRequestRetries: 0,+ minConcurrency: 2,+ maxConcurrency: 2,+ postNavigationHooks: [+ async ({ extendTimeout }) => {+ // the navigation already spent part of the shared window, so ask for more up front,+ // then take far longer than the window would otherwise have allowed+ extendTimeout(5);+ await sleep(2000);+ },+ ],+ requestHandler: async ({ request }) => {+ processed.push(request);+ },+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ await cheerioCrawler.run();++ expect(failed).toHaveLength(0);+ expect(processed).toHaveLength(4);+ });++ test('navigationTimeoutSecs bounds a slowly-streamed response body', async () => {+ // the headers arrive at once, but the body dribbles out over ~5s - reading it is still part of the+ // navigation, so it must be bound by `navigationTimeoutSecs` rather than run unbounded+ const slowServer = createServer(async (_req, res) => {+ res.writeHead(200, { 'content-type': 'text/html', 'content-length': '20' });+ for (let i = 0; i < 20; i++) {+ res.write('x');+ await sleep(250);+ }+ res.end();+ });+ await new Promise<void>((resolve) => slowServer.listen(0, resolve));+ const { port: slowPort } = slowServer.address() as import('node:net').AddressInfo;++ try {+ const processed: Request[] = [];+ const failed: Request[] = [];+ const requestList = await RequestList.open(null, [{ url: `http://localhost:${slowPort}/` }]);++ const cheerioCrawler = new CheerioCrawler({+ requestList,+ navigationTimeoutSecs: 1,+ maxRequestRetries: 0,+ httpClient: new FetchHttpClient(),+ requestHandler: ({ request }) => {+ processed.push(request);+ },+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ const startedAt = Date.now();+ await cheerioCrawler.run();++ expect(processed).toHaveLength(0);+ expect(failed).toHaveLength(1);+ expect(failed[0].errorMessages[0]).toMatch('Navigation timed out');+ // it must give up around the 1s window, not wait out the whole ~5s body+ expect(Date.now() - startedAt).toBeLessThan(3000);+ } finally {+ slowServer.close();+ }+ });});describe('should not timeout by the default httpRequest timeoutSecs', () => {diff --git a/test/core/crawlers/playwright_crawler.test.ts b/test/core/crawlers/playwright_crawler.test.tsindex 67813b3e5..64a6a6a29 100644--- a/test/core/crawlers/playwright_crawler.test.ts+++ b/test/core/crawlers/playwright_crawler.test.ts@@ -6,6 +6,7 @@ import type { PlaywrightCrawlingContext, PlaywrightGotoOptions, Request } from 'import { type ConcurrencySystem, MemoryStorageBackend, serviceLocator } from '@crawlee/core';import { createPlaywrightRouter, PlaywrightCrawler, RequestList, RequestValidationError } from '@crawlee/playwright';import type { Cheerio, CheerioAPI, CheerioRoot, Element } from '@crawlee/utils';+import { sleep } from '@crawlee/utils';import express from 'express';import playwright from 'playwright';import { z } from 'zod';@@ -33,6 +34,8 @@ describe('PlaywrightCrawler', () => {res.send(`<html><head><title>Example Domain</title></head></html>`);res.status(200);});+ // never responds, so a navigation to it runs until the navigation timeout+ app.get('/hang', () => {});app.get('/page-with-download', (_req, res) => {res.status(200).send(`<html><body><a id="download-link" href="/download-file" download="hello.txt">download</a></body></html>`,@@ -296,4 +299,213 @@ describe('PlaywrightCrawler', () => {] as never),).rejects.toThrow(RequestValidationError);});++ describe('timeouts', () => {+ test('a hanging preNavigationHook times out after navigationTimeoutSecs', async () => {+ const failed: Request[] = [];+ const requestHandler = vitest.fn();++ const crawler = new PlaywrightCrawler({+ requestList,+ navigationTimeoutSecs: 0.1,+ maxRequestRetries: 0,+ preNavigationHooks: [async () => sleep(3000)],+ requestHandler,+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ await crawler.run();++ expect(requestHandler).not.toHaveBeenCalled();+ expect(failed).toHaveLength(1);+ expect(failed[0].errorMessages[0]).toMatch('Navigation timed out');+ // the hook is neither the navigation nor the request handler+ expect(failed[0].errorMessages[0]).not.toMatch('requestHandler timed out');+ });++ test('a slow navigation reports the configured window, not the driver value', async () => {+ const failed: Request[] = [];+ const requestList = await RequestList.open(`sources-${Math.random() * 10000}`, [+ `http://${HOSTNAME}:${port}/hang`,+ ]);++ const crawler = new PlaywrightCrawler({+ requestList,+ navigationTimeoutSecs: 2,+ maxRequestRetries: 0,+ // eat most of the window, so the goto is handed a small remaining budget - the driver would+ // otherwise report that raw value ("Timeout NNNms exceeded") instead of the configured window+ preNavigationHooks: [async () => sleep(1500)],+ requestHandler: () => {},+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ await crawler.run();++ expect(failed).toHaveLength(1);+ expect(failed[0].errorMessages[0]).toMatch('Navigation timed out after 2 seconds');+ expect(failed[0].errorMessages[0]).not.toMatch(/Timeout \d+ms exceeded/);+ });++ test('a hook can disable the navigation timeout with gotoOptions.timeout = 0', async () => {+ const processed: Request[] = [];+ const failed: Request[] = [];++ const crawler = new PlaywrightCrawler({+ requestList,+ navigationTimeoutSecs: 0.5,+ maxRequestRetries: 0,+ // `0` is Playwright's "no timeout"; it must be honoured verbatim, not clamped to 1ms (which would+ // make the navigation time out immediately)+ preNavigationHooks: [+ ({ gotoOptions }) => {+ gotoOptions.timeout = 0;+ },+ ],+ requestHandler: ({ request }) => {+ processed.push(request);+ },+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ await crawler.run();++ expect(failed).toHaveLength(0);+ expect(processed).toHaveLength(1);+ });++ test('extendTimeout from a preNavigationHook keeps it from timing out', async () => {+ const processed: Request[] = [];+ const failed: Request[] = [];++ const crawler = new PlaywrightCrawler({+ requestList,+ navigationTimeoutSecs: 0.3,+ maxRequestRetries: 0,+ // 600ms total, past the 0.3s window - only survives because the hook asks for more time+ preNavigationHooks: [+ async ({ extendTimeout }) => {+ await sleep(150);+ extendTimeout(5);+ await sleep(450);+ },+ ],+ requestHandler: ({ request }) => {+ processed.push(request);+ },+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ await crawler.run();++ expect(failed).toHaveLength(0);+ expect(processed).toHaveLength(1);+ });++ test('extendTimeout from a postNavigationHook keeps it from timing out', async () => {+ const processed: Request[] = [];+ const failed: Request[] = [];++ const crawler = new PlaywrightCrawler({+ requestList,+ // enough for the navigation, but far short of the hook below+ navigationTimeoutSecs: 1,+ maxRequestRetries: 0,+ postNavigationHooks: [+ async ({ extendTimeout }) => {+ // the navigation already spent part of the shared window, so ask for more up front,+ // then take far longer than the window would otherwise have allowed+ extendTimeout(5);+ await sleep(2000);+ },+ ],+ requestHandler: ({ request }) => {+ processed.push(request);+ },+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ await crawler.run();++ expect(failed).toHaveLength(0);+ expect(processed).toHaveLength(1);+ });++ test('a route can override requestHandlerTimeoutSecs, other routes keep the default', async () => {+ const requestList = await RequestList.open(`sources-${Math.random() * 10000}`, [+ { url: `http://${HOSTNAME}:${port}/?type=list`, label: 'LIST' },+ { url: `http://${HOSTNAME}:${port}/?type=detail`, label: 'DETAIL' },+ ]);++ const processed: (string | undefined)[] = [];+ const failed: (string | undefined)[] = [];++ const router = createPlaywrightRouter();+ // LIST is allowed to take its time, DETAIL is left on the crawler's short default+ router.addHandler(+ 'LIST',+ async ({ request }) => {+ await sleep(600);+ processed.push(request.label);+ },+ { requestHandlerTimeoutSecs: 5 },+ );+ router.addHandler('DETAIL', async ({ request }) => {+ await sleep(600);+ processed.push(request.label);+ });++ const crawler = new PlaywrightCrawler({+ requestList,+ requestHandlerTimeoutSecs: 0.3,+ maxRequestRetries: 0,+ maxConcurrency: 2,+ requestHandler: router,+ failedRequestHandler: ({ request }) => {+ failed.push(request.label);+ },+ });++ await crawler.run();++ // only DETAIL is held to the 0.3s default; LIST got its own 5s and finished+ expect(failed).toEqual(['DETAIL']);+ expect(processed).toContain('LIST');+ });++ test('extendTimeout from the request handler keeps it from timing out', async () => {+ const processed: Request[] = [];+ const failed: Request[] = [];++ const crawler = new PlaywrightCrawler({+ requestList,+ requestHandlerTimeoutSecs: 0.3,+ maxRequestRetries: 0,+ requestHandler: async ({ request, extendTimeout }) => {+ await sleep(150);+ extendTimeout(5);+ await sleep(450);+ processed.push(request);+ },+ failedRequestHandler: ({ request }) => {+ failed.push(request);+ },+ });++ await crawler.run();++ expect(failed).toHaveLength(0);+ expect(processed).toHaveLength(1);+ });+ });});diff --git a/test/core/router.test.ts b/test/core/router.test.tsindex 073841f3c..8d1b4b49f 100644--- a/test/core/router.test.ts+++ b/test/core/router.test.ts@@ -101,6 +101,33 @@ describe('Router', () => {]);});+ test('should expose the per-route timeouts, falling back the same way as getHandler', async () => {+ const router = Router.create();++ router.addHandler('SLOW', async () => {}, { requestHandlerTimeoutSecs: 120 });+ router.addHandler('PLAIN', async () => {});+ router.addDefaultHandler(async () => {}, { requestHandlerTimeoutSecs: 30 });++ expect(router.getTimeoutSecs('SLOW')).toBe(120);+ // a route of its own with no override means the crawler's own timeout, not the default route's+ expect(router.getTimeoutSecs('PLAIN')).toBeUndefined();+ // no route of its own, so it inherits the default route - handler and timeout alike+ expect(router.getTimeoutSecs('UNKNOWN')).toBe(30);+ expect(router.getTimeoutSecs()).toBe(30);++ expect(router.getMaxTimeoutSecs()).toBe(120);+ });++ test('getMaxTimeoutSecs is undefined when no route overrides the timeout', async () => {+ const router = Router.create();++ router.addHandler('A', async () => {});+ router.addDefaultHandler(async () => {});++ expect(router.getMaxTimeoutSecs()).toBeUndefined();+ expect(router.getTimeoutSecs('A')).toBeUndefined();+ });+test('should be possible to define routes when creating router', async () => {const logs: string[] = [];// it should be possible to define router inline when creating router