Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: Add skipRequest context helper (#4158)

Jan Buchar69a2c07cdf2d
diff --git a/docs/examples/skip-request.mdx b/docs/examples/skip-request.mdx
new file mode 100644
index 000000000..42f0c6630
--- /dev/null
+++ b/docs/examples/skip-request.mdx
@@ -0,0 +1,14 @@
+---
+id: skip-request
+title: Skipping requests
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import SkipRequestSource from '!!raw-loader!roa-loader!./skip-request.ts';
+
+Calling <ApiLink to="basic-crawler/interface/RestrictedCrawlingContext#skipRequest">`skipRequest()`</ApiLink> stops processing the current request without counting it as a failure: it is marked as handled, is not retried, does not reach `failedRequestHandler`, and its storage writes are rolled back. It works in `extendContext`, navigation hooks, the request handler and the `errorHandler`, and each skip is reported to <ApiLink to="basic-crawler/interface/BasicCrawlerOptions#onSkippedRequest">`onSkippedRequest`</ApiLink>.
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+ {SkipRequestSource}
+</RunnableCodeBlock>
diff --git a/docs/examples/skip-request.ts b/docs/examples/skip-request.ts
new file mode 100644
index 000000000..a6dd8bec6
--- /dev/null
+++ b/docs/examples/skip-request.ts
@@ -0,0 +1,28 @@
+import { CheerioCrawler } from 'crawlee';
+
+const crawler = new CheerioCrawler({
+ preNavigationHooks: [
+ async ({ request, skipRequest }) => {
+ // Decide before the page is even downloaded
+ if (request.url.endsWith('.pdf')) skipRequest('not a web page');
+ },
+ ],
+ async requestHandler({ request, $, enqueueLinks, pushData, skipRequest }) {
+ // Or after looking at the content
+ if ($('meta[name="robots"][content*="noindex"]').length) skipRequest('noindex');
+
+ await pushData({ url: request.url, title: $('title').text() });
+ await enqueueLinks();
+ },
+ async errorHandler({ skipRequest }, error) {
+ // Or instead of retrying a request that can never succeed
+ if (error.message.includes('404')) skipRequest('page is gone');
+ },
+ onSkippedRequest({ request, reason, message }) {
+ // `reason` is 'manual' for skipRequest(), `message` is what was passed to it
+ console.log(`Skipped ${request.url} (${reason}): ${message}`);
+ },
+ maxRequestsPerCrawl: 20,
+});
+
+await crawler.run(['https://crawlee.dev']);
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md
index e4e2a8024..6052c6d8c 100644
--- a/docs/public-api/crawlee-basic.api.md
+++ b/docs/public-api/crawlee-basic.api.md
@@ -241,11 +241,6 @@ export class ContextPipelineInitializationError extends Error {
constructor(error: unknown, options?: ErrorOptions);
}
-// @public (undocumented)
-export class ContextPipelineInterruptedError extends Error {
- constructor(message?: string);
-}
-
// @public
export class CpuLoadSignal implements LoadSignal {
constructor(options?: CpuLoadSignalOptions);
@@ -642,6 +637,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
request: CrawlingRequest<UserData>;
// (undocumented)
session: ISession;
+ skipRequest(message?: string): never;
useState: <State extends Dictionary = Dictionary>(defaultValue?: State) => Promise<State>;
}
@@ -851,10 +847,11 @@ export interface SitemapRequestLoaderOptions extends UrlConstraints {
export type SkippedRequestCallback = (args: {
request: Request_2;
reason: SkippedRequestReason;
+ message?: string;
}) => Awaitable<void>;
// @public (undocumented)
-export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
+export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth' | 'manual';
// @public
export class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md
index a18a968a4..4a3139464 100644
--- a/docs/upgrading/upgrading_v4.md
+++ b/docs/upgrading/upgrading_v4.md
@@ -904,6 +904,12 @@ The `robotsTxtFile` / `respectRobotsTxtFile` per-call options are removed from `
The callback now gets `{ request, reason }` instead of `{ url, reason }` — use `request.url` for the URL.
+### Skipping a request with `context.skipRequest()`
+
+In v3, skipping a request without it counting as a failure took a hack: set `request.noRetry`, throw, then decrement `requestsFailed` and silence the error log. Call `skipRequest(message?)` from the crawling context instead — in `extendContext`, a navigation hook, the request handler or the `errorHandler`. The request is marked as handled with `state` set to `RequestState.SKIPPED`, is neither retried nor passed to `failedRequestHandler`, and `onSkippedRequest` fires with the new `'manual'` reason. Storage writes made for the request before the skip are rolled back.
+
+See the [Skipping requests](../examples/skip-request) example.
+
### robots.txt error responses follow RFC 9309
v3 parsed the body of a robots.txt response regardless of its status code. v4 follows [RFC 9309](https://www.rfc-editor.org/rfc/rfc9309#section-2.3.1.3) instead: a `4xx` response allows everything and a `5xx` response disallows everything, whatever the body says.
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 27b02683f..70532c505 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -510,7 +510,8 @@ export interface BasicCrawlerOptions<
* 1. based on robots.txt file,
* 2. because they don't match enqueueLinks filters,
* 3. because they are redirected to a URL that doesn't match the enqueueLinks strategy,
- * 4. or because the {@apilink BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached
+ * 4. because the {@apilink BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached,
+ * 5. or by {@apilink RestrictedCrawlingContext.skipRequest|`context.skipRequest()`}.
*/
onSkippedRequest?: SkippedRequestCallback;
@@ -1411,14 +1412,7 @@ export class BasicCrawler<
this.log.warning(
`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`,
);
- request.state = RequestState.SKIPPED;
- request.noRetry = true;
- await this.#handleSkippedRequest({
- request,
- reason: 'robotsTxt',
- });
-
- throw new ContextPipelineInterruptedError(`Skipping request ${request.url} as disallowed by robots.txt`);
+ throw new ContextPipelineInterruptedError('robotsTxt');
}
return {};
@@ -1471,6 +1465,12 @@ export class BasicCrawler<
this.log.debug('Extending the request processing time failed', { url: context.request.url, error });
});
},
+ skipRequest: (message?: string): never => {
+ this.log.debug(
+ `Skipping request ${context.request.url} (${context.request.id})${message ? `: ${message}` : ''}`,
+ );
+ throw new ContextPipelineInterruptedError('manual', message);
+ },
};
}
@@ -1563,13 +1563,7 @@ export class BasicCrawler<
// eslint-disable-next-line dot-notation
const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
this.log.debug(message);
-
- request.noRetry = true;
- request.state = RequestState.SKIPPED;
-
- await this.#handleSkippedRequest({ request, reason: 'redirect' });
-
- throw new ContextPipelineInterruptedError(message);
+ throw new ContextPipelineInterruptedError('redirect');
}
return context;
});
@@ -2656,29 +2650,33 @@ export class BasicCrawler<
// because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.
currentStorageTransaction()?.rollback();
- const skipped =
- rawError instanceof ContextPipelineInitializationError &&
- rawError.cause instanceof ContextPipelineInterruptedError;
- const err = this.#unwrapError(rawError);
- const context = crawlingContext as CrawlingContext;
- const retryCountBefore = request.retryCount;
const resumeRequestTimeout = crawlingContext[suspendTimeoutKey]?.();
try {
- // ContextPipelineInterruptedError means the request was intentionally skipped
- // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
- if (skipped) {
- this.statistics.discardRequestRecord(request.id || request.uniqueKey);
- await this.#timeoutAndRetry(
- async () => requestSource.markRequestAsHandled(request),
- this.internalTimeoutMillis,
- `Marking request ${request.url} (${request.id}) as handled timed out after ${
- this.internalTimeoutMillis / 1e3
- } seconds.`,
- );
- return;
+ const unwrappedError = this.#unwrapError(rawError);
+ // A failing or timed out `onSkippedRequest` makes this an ordinary failure, so the request gets retried.
+ const err =
+ unwrappedError instanceof ContextPipelineInterruptedError
+ ? await this.#trySkipRequest(unwrappedError, request, requestSource)
+ : unwrappedError;
+
+ if (err) {
+ await this.#runErrorHandler(err, crawlingContext as CrawlingContext, requestSource, request);
}
+ } finally {
+ resumeRequestTimeout?.();
+ }
+ }
+
+ async #runErrorHandler(
+ err: Error,
+ context: CrawlingContext,
+ requestSource: IRequestManager,
+ request: CrawlingRequest,
+ ) {
+ const retryCountBefore = request.retryCount;
+ try {
request.state = RequestState.ERROR_HANDLER;
await addTimeoutToPromise(
async () => this.requestFunctionErrorHandler(err, context, request, requestSource),
@@ -2687,7 +2685,10 @@ export class BasicCrawler<
this.internalTimeoutMillis / 1e3
} seconds.`,
);
- request.state = RequestState.DONE;
+ // The error handler may have set `SKIPPED`; TS keeps the narrowing from the assignment above.
+ if ((request.state as RequestState) !== RequestState.SKIPPED) {
+ request.state = RequestState.DONE;
+ }
} catch (secondaryError) {
const unwrappedSecondaryError = this.#unwrapError(secondaryError);
const timedOut = unwrappedSecondaryError instanceof TimeoutError;
@@ -2697,7 +2698,7 @@ export class BasicCrawler<
// Not worth crashing the crawler over. The request's state is unknown, so it gets retried - and the
// retry is counted, so that an error handler that always times out does not retry it forever.
this.log.warning(unwrappedSecondaryError.message);
- if (!skipped && request.retryCount === retryCountBefore) {
+ if (request.retryCount === retryCountBefore) {
request.retryCount++;
}
} else if (!(unwrappedSecondaryError instanceof CriticalError)) {
@@ -2727,13 +2728,11 @@ export class BasicCrawler<
if (!timedOut) {
throw unwrappedSecondaryError;
}
- } finally {
- resumeRequestTimeout?.();
}
// decrease the session score if the request fails (but the error handler did not throw);
// skip when the error is a SessionError, which already retired the session
- if (!skipped && !this.#errorAbsolvesSession(err)) {
+ if (!this.#errorAbsolvesSession(err)) {
context.session?.markBad();
}
}
@@ -2850,10 +2849,26 @@ export class BasicCrawler<
if (shouldRetryRequest) {
await this.statistics.errorTrackerRetry.addAsync(error, crawlingContext);
- await this.#errorHandler?.(
- crawlingContext as CrawlingContext & Partial<ExtendedContext>, // valid cast - ExtendedContext transitively extends CrawlingContext
- error,
- );
+ try {
+ await this.#errorHandler?.(
+ crawlingContext as CrawlingContext & Partial<ExtendedContext>, // valid cast - ExtendedContext transitively extends CrawlingContext
+ error,
+ );
+ } catch (handlerError) {
+ if (handlerError instanceof ContextPipelineInterruptedError) {
+ const skipError = await this.#trySkipRequest(handlerError, request, source);
+ if (!skipError) {
+ return;
+ }
+ // Retry for the original error. Routing `skipError` through `errorHandler` again
+ // would loop forever with a handler that always skips.
+ this.log.warning(`Skipping request ${request.url} failed, retrying it instead.`, {
+ error: skipError.message,
+ });
+ } else {
+ throw handlerError;
+ }
+ }
if (error instanceof SessionError) {
crawlingContext.session?.retire();
@@ -2901,6 +2916,34 @@ export class BasicCrawler<
await this.handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
}
+ /** Settles a skipped request. Returns the error that prevented that, leaving the request untouched if `onSkippedRequest` failed. */
+ async #trySkipRequest(
+ error: ContextPipelineInterruptedError,
+ request: CrawlingRequest,
+ source: IRequestManager,
+ ): Promise<Error | undefined> {
+ try {
+ await addTimeoutToPromise(
+ async () => this.#handleSkippedRequest({ request, reason: error.reason, message: error.skipMessage }),
+ this.internalTimeoutMillis,
+ `onSkippedRequest for ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,
+ );
+ request.state = RequestState.SKIPPED;
+ request.noRetry = true;
+ await this.#timeoutAndRetry(
+ async () => source.markRequestAsHandled(request),
+ this.internalTimeoutMillis,
+ `Marking request ${request.url} (${request.id}) as handled timed out after ${
+ this.internalTimeoutMillis / 1e3
+ } seconds.`,
+ );
+ this.statistics.discardRequestRecord(request.id || request.uniqueKey);
+ return undefined;
+ } catch (settleError) {
+ return this.#unwrapError(settleError);
+ }
+ }
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
private async handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise<void> {
// Always log the last error regardless if the user provided a failedRequestHandler
@@ -2909,11 +2952,16 @@ export class BasicCrawler<
this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
- if (this.#failedRequestHandler) {
+ try {
await this.#failedRequestHandler?.(
crawlingContext as CrawlingContext & Partial<ExtendedContext>, // valid cast - ExtendedContext transitively extends CrawlingContext
error,
);
+ } catch (handlerError) {
+ if (!(handlerError instanceof ContextPipelineInterruptedError)) throw handlerError;
+ this.log.warning(
+ `skipRequest() has no effect in failedRequestHandler, request ${url} (${id}) has already failed.`,
+ );
}
}
@@ -2947,7 +2995,11 @@ export class BasicCrawler<
* failure says nothing about the session (a rate limit is a property of the domain).
*/
#errorAbsolvesSession(error: Error): boolean {
- return error instanceof SessionError || error instanceof RequestThrottledError;
+ return (
+ error instanceof SessionError ||
+ error instanceof RequestThrottledError ||
+ error instanceof ContextPipelineInterruptedError
+ );
}
#canRequestBeRetried(request: CrawlingRequest, error: Error) {
diff --git a/packages/basic-crawler/src/internals/crawlers/crawler_commons.ts b/packages/basic-crawler/src/internals/crawlers/crawler_commons.ts
index 00b25ff89..6614b5458 100644
--- a/packages/basic-crawler/src/internals/crawlers/crawler_commons.ts
+++ b/packages/basic-crawler/src/internals/crawlers/crawler_commons.ts
@@ -162,6 +162,29 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
* A preconfigured logger for the request handler.
*/
log: CrawleeLogger;
+
+ /**
+ * Stops processing the current request and marks it as handled without counting it as a failure: no retry, no
+ * `failedRequestHandler`, and no error log. The request's `state` becomes {@apilink RequestState.SKIPPED|`SKIPPED`}
+ * and {@apilink BasicCrawlerOptions.onSkippedRequest|`onSkippedRequest`} fires with the `'manual'` reason.
+ *
+ * Storage writes made for the request so far are rolled back, as with a failure, unless
+ * {@apilink BasicCrawlerOptions.transactionalStorage|`transactionalStorage`} is disabled.
+ *
+ * Works in `errorHandler` too, skipping a request that would otherwise be retried. In `failedRequestHandler` it
+ * has no effect, since the request has already failed.
+ *
+ * ```ts
+ * preNavigationHooks: [
+ * async ({ request, skipRequest }) => {
+ * if (await alreadyScraped(request.url)) skipRequest('already scraped');
+ * },
+ * ],
+ * ```
+ *
+ * @param message Passed to `onSkippedRequest`.
+ */
+ skipRequest(message?: string): never;
}
export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
diff --git a/packages/basic-crawler/src/internals/crawling_request.ts b/packages/basic-crawler/src/internals/crawling_request.ts
index 1a7232587..1bdcd3d0c 100644
--- a/packages/basic-crawler/src/internals/crawling_request.ts
+++ b/packages/basic-crawler/src/internals/crawling_request.ts
@@ -11,7 +11,8 @@ export type SkippedRequestReason =
| 'filters'
| 'transform'
| 'redirect'
- | 'depth';
+ | 'depth'
+ | 'manual';
export enum RequestState {
UNPROCESSED,
diff --git a/packages/basic-crawler/src/internals/enqueue_links/shared.ts b/packages/basic-crawler/src/internals/enqueue_links/shared.ts
index 05a62d5c4..8f8c12dbd 100644
--- a/packages/basic-crawler/src/internals/enqueue_links/shared.ts
+++ b/packages/basic-crawler/src/internals/enqueue_links/shared.ts
@@ -45,7 +45,12 @@ export const urlPatternSchema = z.union([
schemas.objectWithKeys(['regexp']),
]) as z.ZodType<UrlPatternInput>;
-export type SkippedRequestCallback = (args: { request: Request; reason: SkippedRequestReason }) => Awaitable<void>;
+export type SkippedRequestCallback = (args: {
+ request: Request;
+ reason: SkippedRequestReason;
+ /** The `message` passed to {@apilink RestrictedCrawlingContext.skipRequest|`context.skipRequest()`}, if any. */
+ message?: string;
+}) => Awaitable<void>;
/**
* Builds the `{ request, reason }` argument passed to a {@apilink SkippedRequestCallback}, constructing the
diff --git a/packages/basic-crawler/src/internals/errors.ts b/packages/basic-crawler/src/internals/errors.ts
index 6d3cc26c0..493d5793c 100644
--- a/packages/basic-crawler/src/internals/errors.ts
+++ b/packages/basic-crawler/src/internals/errors.ts
@@ -1,5 +1,7 @@
import { CriticalError, NonRetryableError } from '@crawlee/core';
+import type { SkippedRequestReason } from './crawling_request.js';
+
/**
* @ignore
*/
@@ -50,9 +52,17 @@ export class MissingSessionError extends Error {
}
}
+/**
+ * Settles the request as skipped rather than failed: no retry, no failure, and its storage writes are rolled back.
+ * Use {@apilink RestrictedCrawlingContext.skipRequest|`context.skipRequest()`} rather than throwing this directly.
+ * @internal
+ */
export class ContextPipelineInterruptedError extends Error {
- constructor(message?: string) {
- super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`);
+ constructor(
+ readonly reason: SkippedRequestReason,
+ readonly skipMessage?: string,
+ ) {
+ super(skipMessage ?? 'Request was skipped');
}
}
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index ea15be6a4..0e94a3f02 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -22,6 +22,7 @@ import {
BasicCrawler,
ContextPipeline,
ContextPipelineInitializationError,
+ ContextPipelineInterruptedError,
RequestHandlerError,
resolveBaseUrlForEnqueueLinksFiltering,
Router,
@@ -724,7 +725,10 @@ export class AdaptivePlaywrightCrawler<
? (plainHTTPRun.error.cause as Error)
: (plainHTTPRun.error as Error);
- if (await this.#shouldPropagateError(actualError, crawlingContext as any)) {
+ if (
+ actualError instanceof ContextPipelineInterruptedError ||
+ (await this.#shouldPropagateError(actualError, crawlingContext as any))
+ ) {
throw actualError;
}
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index a744740f6..d8683847a 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -2172,16 +2172,15 @@ describe('BasicCrawler', () => {
// the request is never handled, so the queue never finishes; `stop()` lets the in-flight request wind down
extendContext: () => {
crawler.stop();
- throw new ContextPipelineInterruptedError('skipped by the test');
+ throw new ContextPipelineInterruptedError('manual', 'skipped by the test');
},
requestHandler: async () => {},
});
const warningSpy = vitest.spyOn(crawler.log, 'warning');
await expect(crawler.run(['https://example.com'])).resolves.toBeDefined();
- // anchored, so that the `timeoutAndRetry` retry warnings ("... (retrying 1/3)") do not count
expect(warningSpy).toHaveBeenCalledWith(
- expect.stringMatching(/^Marking request .* as handled timed out after 0\.1 seconds\.$/),
+ expect.stringMatching(/^Handling request failure of .* timed out after 0\.1 seconds\.$/),
);
});
@@ -3782,6 +3781,198 @@ describe('BasicCrawler', () => {
});
});
+ describe('skipRequest()', () => {
+ test.each(['extendContext', 'requestHandler'] as const)(
+ 'called from %s settles the request as skipped, not failed',
+ async (caller) => {
+ const url = 'https://example.com/skipped';
+ const onSkippedRequest = vitest.fn();
+ const failedRequestHandler = vitest.fn();
+ let attempts = 0;
+ const requestManager = await RequestQueue.open();
+
+ const crawler = new BasicCrawler({
+ requestManager,
+ maxRequestRetries: 3,
+ extendContext: ({ skipRequest }) => {
+ if (caller === 'extendContext') {
+ attempts++;
+ skipRequest('not interesting');
+ }
+ return {};
+ },
+ requestHandler: async ({ skipRequest }) => {
+ attempts++;
+ skipRequest('not interesting');
+ },
+ failedRequestHandler,
+ onSkippedRequest,
+ });
+
+ const stats = await crawler.run([url]);
+
+ expect(attempts).toBe(1);
+ expect(failedRequestHandler).not.toHaveBeenCalled();
+ expect(stats).toMatchObject({ requestsFailed: 0, requestsSucceeded: 0 });
+ expect(onSkippedRequest).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({ reason: 'manual', message: 'not interesting' }),
+ );
+
+ const stored = await requestManager.getRequest(url);
+ expect(stored?.handledAt).toBeDefined();
+ expect(CrawlingRequest.fromSchema(stored!).state).toBe(RequestState.SKIPPED);
+ },
+ );
+
+ test('called from errorHandler settles the request as skipped', async () => {
+ const url = 'https://example.com/skipped';
+ const onSkippedRequest = vitest.fn();
+ const failedRequestHandler = vitest.fn();
+ const requestHandler = vitest.fn(async () => {
+ throw new Error('404');
+ });
+ const requestManager = await RequestQueue.open();
+ let inMemoryRequest: CrawlingRequest | undefined;
+
+ const crawler = new BasicCrawler({
+ requestManager,
+ maxRequestRetries: 3,
+ requestHandler,
+ errorHandler: ({ request, skipRequest }) => {
+ inMemoryRequest = request;
+ skipRequest('gone');
+ },
+ failedRequestHandler,
+ onSkippedRequest,
+ });
+
+ const stats = await crawler.run([url]);
+
+ expect(requestHandler).toHaveBeenCalledOnce();
+ expect(failedRequestHandler).not.toHaveBeenCalled();
+ expect(stats).toMatchObject({ requestsFailed: 0, requestsSucceeded: 0 });
+ expect(onSkippedRequest).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({ reason: 'manual', message: 'gone' }),
+ );
+
+ const stored = await requestManager.getRequest(url);
+ expect(stored?.handledAt).toBeDefined();
+ expect(CrawlingRequest.fromSchema(stored!).state).toBe(RequestState.SKIPPED);
+ expect(inMemoryRequest?.state).toBe(RequestState.SKIPPED);
+ });
+
+ test('called from failedRequestHandler leaves the request failed and does not crash the crawler', async () => {
+ const onSkippedRequest = vitest.fn();
+
+ const crawler = new BasicCrawler({
+ maxRequestRetries: 0,
+ requestHandler: async () => {
+ throw new Error('404');
+ },
+ failedRequestHandler: ({ skipRequest }) => skipRequest('gone'),
+ onSkippedRequest,
+ });
+ const warningSpy = vitest.spyOn(crawler.log, 'warning');
+
+ const stats = await crawler.run(['https://example.com/failed']);
+
+ expect(stats).toMatchObject({ requestsFailed: 1 });
+ expect(onSkippedRequest).not.toHaveBeenCalled();
+ expect(warningSpy).toHaveBeenCalledWith(expect.stringContaining('skipRequest()'));
+ });
+
+ test('a throwing onSkippedRequest retries the request instead of crashing the crawler', async () => {
+ const url = 'https://example.com/skipped';
+ const requestHandler = vitest.fn(async ({ skipRequest }) => skipRequest());
+ const onSkippedRequest = vitest
+ .fn()
+ .mockRejectedValueOnce(new Error('webhook down'))
+ .mockResolvedValue(undefined);
+ const requestManager = await RequestQueue.open();
+
+ const crawler = new BasicCrawler({ requestManager, requestHandler, onSkippedRequest });
+ const stats = await crawler.run([url]);
+
+ expect(requestHandler).toHaveBeenCalledTimes(2);
+ expect(onSkippedRequest).toHaveBeenCalledTimes(2);
+ expect(stats).toMatchObject({ requestsFailed: 0 });
+ const stored = await requestManager.getRequest(url);
+ expect(CrawlingRequest.fromSchema(stored!).state).toBe(RequestState.SKIPPED);
+ });
+
+ test('a throwing onSkippedRequest does not loop with an errorHandler that always skips', async () => {
+ const requestHandler = vitest.fn(async () => {
+ throw new Error('404');
+ });
+ const failedRequestHandler = vitest.fn();
+
+ const crawler = new BasicCrawler({
+ maxRequestRetries: 2,
+ requestHandler,
+ errorHandler: ({ skipRequest }) => skipRequest(),
+ failedRequestHandler,
+ onSkippedRequest: async () => {
+ throw new Error('webhook down');
+ },
+ });
+ const stats = await crawler.run(['https://example.com/skipped']);
+
+ expect(requestHandler).toHaveBeenCalledTimes(3);
+ expect(failedRequestHandler).toHaveBeenCalledOnce();
+ expect(stats).toMatchObject({ requestsFailed: 1 });
+ });
+
+ test(
+ 'a timed out onSkippedRequest uses up the retry budget instead of looping',
+ { timeout: 5_000 },
+ async () => {
+ serviceLocator.reset();
+ serviceLocator.setStorageBackend(new MemoryStorageBackend());
+ serviceLocator.setConfiguration(new Configuration({ internalTimeoutMillis: 100 }));
+
+ const requestHandler = vitest.fn(async ({ skipRequest }) => skipRequest());
+ const failedRequestHandler = vitest.fn();
+
+ // never settles, so every attempt times out
+ const onSkippedRequest = vitest.fn(() => new Promise<never>(() => {}));
+
+ const crawler = new BasicCrawler({
+ maxRequestRetries: 2,
+ requestHandlerTimeoutSecs: 1,
+ requestHandler,
+ failedRequestHandler,
+ onSkippedRequest,
+ });
+
+ const stats = await crawler.run(['https://example.com/skipped']);
+
+ expect(requestHandler).toHaveBeenCalledTimes(3);
+ expect(onSkippedRequest).toHaveBeenCalledTimes(3);
+ expect(failedRequestHandler).toHaveBeenCalledOnce();
+ expect(stats).toMatchObject({ requestsFailed: 1 });
+ },
+ );
+
+ test('marking a skipped request as handled is retried', async () => {
+ const url = 'https://example.com/skipped';
+ const requestManager = await RequestQueue.open();
+ const markRequestAsHandled = requestManager.markRequestAsHandled.bind(requestManager);
+ vitest
+ .spyOn(requestManager, 'markRequestAsHandled')
+ .mockRejectedValueOnce(new Error('storage hiccup'))
+ .mockImplementation(markRequestAsHandled);
+
+ const crawler = new BasicCrawler({
+ requestManager,
+ requestHandler: async ({ skipRequest }) => skipRequest(),
+ });
+ await crawler.run([url]);
+
+ const stored = await requestManager.getRequest(url);
+ expect(stored?.handledAt).toBeDefined();
+ });
+ });
+
describe('transactional storage', () => {
test('a failing request handler leaves no dataset/KVS writes, but write-through enqueues survive', async () => {
const crawler = new BasicCrawler({
diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts
index a1a85b0d0..f6bce1189 100644
--- a/test/core/crawlers/http_crawler.test.ts
+++ b/test/core/crawlers/http_crawler.test.ts
@@ -623,6 +623,32 @@ test('extendContext is visible to pre/post-navigation hooks and the request hand
expect(seenIn.requestHandler).toBe('from-extend-context');
});
+test('skipRequest() in a navigation hook skips the request and rolls back its storage writes', async () => {
+ const onSkippedRequest = vitest.fn();
+ const requestHandler = vitest.fn();
+
+ const crawler = new HttpCrawler({
+ maxRequestRetries: 3,
+ preNavigationHooks: [
+ async ({ pushData, skipRequest }) => {
+ await pushData({ written: 'before the skip' });
+ skipRequest('already scraped');
+ },
+ ],
+ requestHandler,
+ onSkippedRequest,
+ });
+
+ const stats = await crawler.run([url]);
+
+ expect(requestHandler).not.toHaveBeenCalled();
+ expect(stats).toMatchObject({ requestsFailed: 0, requestsSucceeded: 0 });
+ expect(onSkippedRequest).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({ reason: 'manual', message: 'already scraped' }),
+ );
+ await expect(crawler.getData()).resolves.toMatchObject({ items: [] });
+});
+
test('works with a custom HttpClient', async () => {
const results: string[] = [];