Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: Fix `skipNavigation` behavior in `HttpCrawler` subclasses (#3381)

Jan Buchard9b5fe08038a
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index c53a3defa..511b3a27f 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -44,6 +44,7 @@ import {
KeyValueStore,
LogLevel,
mergeCookies,
+ NavigationSkippedError,
NonRetryableError,
purgeDefaultStorages,
RequestHandlerError,
@@ -931,13 +932,13 @@ export class BasicCrawler<
try {
await this.basicContextPipeline
.chain(this.contextPipeline)
- .call(crawlingContext, (ctx) => this.handleRequest(ctx, source));
+ .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request));
} catch (error) {
// ContextPipelineInterruptedError means the request was intentionally skipped
// (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
if (error instanceof ContextPipelineInterruptedError) {
await this._timeoutAndRetry(
- async () => this.requestManager?.markRequestHandled(crawlingContext.request!),
+ async () => this.requestManager?.markRequestHandled(request),
this.internalTimeoutMillis,
`Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${
this.internalTimeoutMillis / 1e3
@@ -956,6 +957,7 @@ export class BasicCrawler<
await this._requestFunctionErrorHandler(
unwrappedError,
crawlingContext as CrawlingContext,
+ request,
this.requestManager!,
);
crawlingContext.session?.markBad();
@@ -1902,9 +1904,7 @@ export class BasicCrawler<
}
/** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
- protected async handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager) {
- const { request } = crawlingContext;
-
+ protected async handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager, request: Request) {
const statisticsId = request.id || request.uniqueKey;
this.stats.startJob(statisticsId);
@@ -1934,7 +1934,7 @@ export class BasicCrawler<
try {
request.state = RequestState.ERROR_HANDLER;
await addTimeoutToPromise(
- async () => this._requestFunctionErrorHandler(err, crawlingContext, requestSource),
+ async () => this._requestFunctionErrorHandler(err, crawlingContext, request, requestSource),
this.internalTimeoutMillis,
`Handling request failure of ${request.url} (${request.id}) timed out after ${
this.internalTimeoutMillis / 1e3
@@ -2129,13 +2129,15 @@ export class BasicCrawler<
/**
* Handles errors thrown by user provided requestHandler()
+ *
+ * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
*/
protected async _requestFunctionErrorHandler(
error: Error,
crawlingContext: CrawlingContext,
+ request: Request,
source: IRequestList | IRequestManager,
): Promise<void> {
- const { request } = crawlingContext;
request.pushErrorMessage(error);
if (error instanceof CriticalError) {
@@ -2338,6 +2340,18 @@ export class BasicCrawler<
}
private requestMatchesEnqueueStrategy(request: Request) {
+ // If `skipNavigation` was used, just return `true`
+ try {
+ // eslint-disable-next-line @typescript-eslint/no-unused-expressions
+ request.loadedUrl;
+ } catch (err) {
+ if (err instanceof NavigationSkippedError) {
+ return true;
+ }
+
+ throw err;
+ }
+
const { url, loadedUrl } = request;
// eslint-disable-next-line dot-notation -- private access
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 6cd7ce06d..0cc511a5e 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -20,6 +20,7 @@ import {
enqueueLinks,
EVENT_SESSION_RETIRED,
handleRequestTimeout,
+ NavigationSkippedError,
RequestState,
resolveBaseUrlForEnqueueLinksFiltering,
SessionError,
@@ -523,7 +524,7 @@ export abstract class BrowserCrawler<
request: new Proxy(crawlingContext.request, {
get(target, propertyName, receiver) {
if (propertyName === 'loadedUrl') {
- throw new Error(
+ throw new NavigationSkippedError(
'The `request.loadedUrl` property is not available - `skipNavigation` was used',
);
}
@@ -531,7 +532,9 @@ export abstract class BrowserCrawler<
},
}) as LoadedRequest<Request>,
get response(): Response {
- throw new Error('The `response` property is not available - `skipNavigation` was used');
+ throw new NavigationSkippedError(
+ 'The `response` property is not available - `skipNavigation` was used',
+ );
},
};
}
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index 522a13306..49bf74a89 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -14,7 +14,13 @@ import type {
RoutesFromSchemas,
SkippedRequestCallback,
} from '@crawlee/http';
-import { enqueueLinks, HttpCrawler, resolveBaseUrlForEnqueueLinksFiltering, Router } from '@crawlee/http';
+import {
+ enqueueLinks,
+ HttpCrawler,
+ NavigationSkippedError,
+ resolveBaseUrlForEnqueueLinksFiltering,
+ Router,
+} from '@crawlee/http';
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
import { type CheerioRoot, extractUrlsFromCheerio, type RobotsTxtFile } from '@crawlee/utils';
import type { CheerioAPI, CheerioOptions } from 'cheerio';
@@ -197,19 +203,40 @@ export class CheerioCrawler<
}
private async parseContent(crawlingContext: InternalHttpCrawlingContext) {
- const isXml = crawlingContext.contentType.type.includes('xml');
- const body = Buffer.isBuffer(crawlingContext.body)
- ? crawlingContext.body.toString(crawlingContext.contentType.encoding)
- : crawlingContext.body;
- const dom = parseDocument(body, { decodeEntities: true, xmlMode: isXml });
- const $ = cheerio.load(dom, {
- xml: { decodeEntities: true, xmlMode: isXml },
- } as CheerioOptions);
+ try {
+ const isXml = crawlingContext.contentType.type.includes('xml');
+ const body = Buffer.isBuffer(crawlingContext.body)
+ ? crawlingContext.body.toString(crawlingContext.contentType.encoding)
+ : crawlingContext.body;
+ const dom = parseDocument(body, { decodeEntities: true, xmlMode: isXml });
+ const $ = cheerio.load(dom, {
+ xml: { decodeEntities: true, xmlMode: isXml },
+ } as CheerioOptions);
- return {
- $,
- body,
- };
+ return {
+ $,
+ body,
+ };
+ } catch (err) {
+ if (err instanceof NavigationSkippedError) {
+ return {
+ get body(): string {
+ throw new NavigationSkippedError(
+ 'The `body` property is not available - `skipNavigation` was used',
+ { cause: err },
+ );
+ },
+ get $(): CheerioAPI {
+ throw new NavigationSkippedError(
+ 'The `$` property is not available - `skipNavigation` was used',
+ { cause: err },
+ );
+ },
+ };
+ }
+
+ throw err;
+ }
}
private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { $: CheerioAPI }) {
diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.ts
index 36b4b6549..40c8e3028 100644
--- a/packages/core/src/errors.ts
+++ b/packages/core/src/errors.ts
@@ -104,3 +104,9 @@ export class ServiceConflictError extends Error {
);
}
}
+
+/**
+ * Thrown by crawlers when `skipNavigation` is used on a request.
+ * Subclasses can catch this error to skip their own navigation-dependent logic.
+ */
+export class NavigationSkippedError extends NonRetryableError {}
diff --git a/packages/core/src/request.ts b/packages/core/src/request.ts
index 5f26ffa27..8dcac31fa 100644
--- a/packages/core/src/request.ts
+++ b/packages/core/src/request.ts
@@ -275,12 +275,24 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
});
}
- /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+ /**
+ * Tells the crawler processing this request to skip the navigation and process the request directly.
+ *
+ * When this is set to `true`, the crawling context will not contain the results of the navigation
+ * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+ * Accessing these properties will throw a {@apilink NavigationSkippedError} at runtime.
+ */
get skipNavigation(): boolean {
return this.userData.__crawlee?.skipNavigation ?? false;
}
- /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+ /**
+ * Tells the crawler processing this request to skip the navigation and process the request directly.
+ *
+ * When this is set to `true`, the crawling context will not contain the results of the navigation
+ * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+ * Accessing these properties will throw a {@apilink NavigationSkippedError} at runtime.
+ */
set skipNavigation(value: boolean) {
if (!this.userData.__crawlee) {
(this.userData as Dictionary).__crawlee = { skipNavigation: value };
@@ -548,6 +560,10 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
/**
* If set to `true` then the crawler processing this request evaluates
* the `requestHandler` immediately without prior browser navigation.
+ *
+ * When enabled, the crawling context will not contain the results of the navigation
+ * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+ * Accessing these properties will throw a {@apilink NavigationSkippedError} at runtime.
* @default false
*/
skipNavigation?: boolean;
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 131b999d1..9c6d9fcda 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -16,7 +16,15 @@ import type {
RoutesFromSchemas,
Session,
} from '@crawlee/basic';
-import { BasicCrawler, ContextPipeline, mergeCookies, RequestState, Router, SessionError } from '@crawlee/basic';
+import {
+ BasicCrawler,
+ ContextPipeline,
+ mergeCookies,
+ NavigationSkippedError,
+ RequestState,
+ Router,
+ SessionError,
+} from '@crawlee/basic';
import type { LoadedRequest } from '@crawlee/core';
import { ResponseWithUrl } from '@crawlee/http-client';
import type { Awaitable, Dictionary } from '@crawlee/types';
@@ -395,7 +403,7 @@ export class HttpCrawler<
request: new Proxy(request, {
get(target, propertyName, receiver) {
if (propertyName === 'loadedUrl') {
- throw new Error(
+ throw new NavigationSkippedError(
'The `request.loadedUrl` property is not available - `skipNavigation` was used',
);
}
@@ -403,7 +411,9 @@ export class HttpCrawler<
},
}) as LoadedRequest<CrawleeRequest>,
get response(): InternalHttpCrawlingContext['response'] {
- throw new Error('The `response` property is not available - `skipNavigation` was used');
+ throw new NavigationSkippedError(
+ 'The `response` property is not available - `skipNavigation` was used',
+ );
},
};
}
@@ -443,19 +453,29 @@ export class HttpCrawler<
if (crawlingContext.request.skipNavigation) {
return {
get contentType(): InternalHttpCrawlingContext['contentType'] {
- throw new Error('The `contentType` property is not available - `skipNavigation` was used');
+ throw new NavigationSkippedError(
+ 'The `contentType` property is not available - `skipNavigation` was used',
+ );
},
get body(): InternalHttpCrawlingContext['body'] {
- throw new Error('The `body` property is not available - `skipNavigation` was used');
+ throw new NavigationSkippedError(
+ 'The `body` property is not available - `skipNavigation` was used',
+ );
},
get json(): InternalHttpCrawlingContext['json'] {
- throw new Error('The `json` property is not available - `skipNavigation` was used');
+ throw new NavigationSkippedError(
+ 'The `json` property is not available - `skipNavigation` was used',
+ );
},
get waitForSelector(): InternalHttpCrawlingContext['waitForSelector'] {
- throw new Error('The `waitForSelector` method is not available - `skipNavigation` was used');
+ throw new NavigationSkippedError(
+ 'The `waitForSelector` method is not available - `skipNavigation` was used',
+ );
},
get parseWithCheerio(): InternalHttpCrawlingContext['parseWithCheerio'] {
- throw new Error('The `parseWithCheerio` method is not available - `skipNavigation` was used');
+ throw new NavigationSkippedError(
+ 'The `parseWithCheerio` method is not available - `skipNavigation` was used',
+ );
},
};
}
diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
index 405c7dad1..7323af64a 100644
--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
@@ -17,6 +17,7 @@ import type {
import {
enqueueLinks,
HttpCrawler,
+ NavigationSkippedError,
resolveBaseUrlForEnqueueLinksFiltering,
Router,
tryAbsoluteURL,
@@ -252,75 +253,102 @@ export class JSDOMCrawler<
private readonly jsdomErrorHandler = (error: Error) => this.log.debug('JSDOM error from console', { error });
private async parseContent(crawlingContext: InternalHttpCrawlingContext) {
- const isXml = crawlingContext.contentType.type.includes('xml');
-
- // TODO handle non-string
- const { window } = new JSDOM(crawlingContext.body.toString(), {
- url: crawlingContext.response.url,
- contentType: isXml ? 'text/xml' : 'text/html',
- runScripts: this.runScripts ? 'dangerously' : undefined,
- resources,
- virtualConsole: this.getVirtualConsole(),
- pretendToBeVisual: true,
- });
-
- // add some stubs in place of missing API so processing won't fail
- Object.defineProperty(window, 'matchMedia', {
- writable: true,
- value: (query: unknown): any => ({
- matches: false,
- media: query,
- onchange: null,
- addListener: () => {},
- removeListener: () => {},
- addEventListener: () => {},
- removeEventListener: () => {},
- dispatchEvent: () => {},
- }),
- });
- window.document.createRange = () => {
- const range = new window.Range();
- range.getBoundingClientRect = () => ({}) as any;
- range.getClientRects = () => ({ item: () => null as any, length: 0 }) as any;
- return range;
- };
+ try {
+ const isXml = crawlingContext.contentType.type.includes('xml');
+
+ // TODO handle non-string
+ const { window } = new JSDOM(crawlingContext.body.toString(), {
+ url: crawlingContext.response.url,
+ contentType: isXml ? 'text/xml' : 'text/html',
+ runScripts: this.runScripts ? 'dangerously' : undefined,
+ resources,
+ virtualConsole: this.getVirtualConsole(),
+ pretendToBeVisual: true,
+ });
+
+ // add some stubs in place of missing API so processing won't fail
+ Object.defineProperty(window, 'matchMedia', {
+ writable: true,
+ value: (query: unknown): any => ({
+ matches: false,
+ media: query,
+ onchange: null,
+ addListener: () => {},
+ removeListener: () => {},
+ addEventListener: () => {},
+ removeEventListener: () => {},
+ dispatchEvent: () => {},
+ }),
+ });
+ window.document.createRange = () => {
+ const range = new window.Range();
+ range.getBoundingClientRect = () => ({}) as any;
+ range.getClientRects = () => ({ item: () => null as any, length: 0 }) as any;
+ return range;
+ };
+
+ if (this.runScripts) {
+ try {
+ await addTimeoutToPromise(
+ async () => {
+ return new Promise<void>((resolve) => {
+ window.addEventListener(
+ 'load',
+ () => {
+ resolve();
+ },
+ false,
+ );
+ }).catch();
+ },
+ 10_000,
+ 'Window.load event not fired after 10 seconds.',
+ ).catch();
+ } catch (e) {
+ this.log.debug((e as Error).message);
+ }
+ }
- if (this.runScripts) {
- try {
- await addTimeoutToPromise(
- async () => {
- return new Promise<void>((resolve) => {
- window.addEventListener(
- 'load',
- () => {
- resolve();
- },
- false,
- );
- }).catch();
+ return {
+ window,
+ get body() {
+ return window.document.documentElement.outerHTML;
+ },
+ get document() {
+ return window.document;
+ },
+ };
+ } catch (err) {
+ if (err instanceof NavigationSkippedError) {
+ return {
+ get window(): DOMWindow {
+ throw new NavigationSkippedError(
+ 'The `window` property is not available - `skipNavigation` was used',
+ { cause: err },
+ );
+ },
+ get body(): string {
+ throw new NavigationSkippedError(
+ 'The `body` property is not available - `skipNavigation` was used',
+ { cause: err },
+ );
+ },
+ get document(): Document {
+ throw new NavigationSkippedError(
+ 'The `document` property is not available - `skipNavigation` was used',
+ { cause: err },
+ );
},
- 10_000,
- 'Window.load event not fired after 10 seconds.',
- ).catch();
- } catch (e) {
- this.log.debug((e as Error).message);
+ };
}
- }
-
- const originalEnqueueLinks = crawlingContext.enqueueLinks;
- return {
- window,
- get body() {
- return window.document.documentElement.outerHTML;
- },
- get document() {
- return window.document;
- },
- };
+ throw err;
+ }
}
private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { body: string; window: DOMWindow }) {
+ const originalEnqueueLinks = crawlingContext.enqueueLinks;
+
return {
enqueueLinks: async (enqueueOptions?: EnqueueLinksOptions) => {
return domCrawlerEnqueueLinks({
diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
index 66d33bac3..77ead2cd7 100644
--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
@@ -17,6 +17,7 @@ import type {
import {
enqueueLinks,
HttpCrawler,
+ NavigationSkippedError,
resolveBaseUrlForEnqueueLinksFiltering,
Router,
tryAbsoluteURL,
@@ -188,27 +189,54 @@ export class LinkeDOMCrawler<
}
private async parseContent(crawlingContext: InternalHttpCrawlingContext) {
- const isXml = crawlingContext.contentType.type.includes('xml');
- const document = LinkeDOMCrawler.parser.parseFromString(
- crawlingContext.body.toString(),
- isXml ? 'text/xml' : 'text/html',
- );
-
- const originalEnqueueLinks = crawlingContext.enqueueLinks;
+ try {
+ const isXml = crawlingContext.contentType.type.includes('xml');
+ const document = LinkeDOMCrawler.parser.parseFromString(
+ crawlingContext.body.toString(),
+ isXml ? 'text/xml' : 'text/html',
+ );
+
+ return {
+ window: document.defaultView,
+ get body() {
+ return document.documentElement.outerHTML;
+ },
+ get document() {
+ // See comment about typing in LinkeDOMCrawlingContext definition
+ return document as unknown as Document;
+ },
+ };
+ } catch (err) {
+ if (err instanceof NavigationSkippedError) {
+ return {
+ get window(): Window {
+ throw new NavigationSkippedError(
+ 'The `window` property is not available - `skipNavigation` was used',
+ { cause: err },
+ );
+ },
+ get body(): string {
+ throw new NavigationSkippedError(
+ 'The `body` property is not available - `skipNavigation` was used',
+ { cause: err },
+ );
+ },
+ get document(): Document {
+ throw new NavigationSkippedError(
+ 'The `document` property is not available - `skipNavigation` was used',
+ { cause: err },
+ );
+ },
+ };
+ }
- return {
- window: document.defaultView,
- get body() {
- return document.documentElement.outerHTML;
- },
- get document() {
- // See comment about typing in LinkeDOMCrawlingContext definition
- return document as unknown as Document;
- },
- };
+ throw err;
+ }
}
private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { body: string }) {
+ const originalEnqueueLinks = crawlingContext.enqueueLinks;
+
return {
enqueueLinks: async (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => {
return linkedomCrawlerEnqueueLinks({
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index f70d59eb9..097ceb89b 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -16,7 +16,6 @@ import { ImpitHttpClient } from '@crawlee/impit-client';
import type { ProxyInfo } from '@crawlee/types';
import type { Dictionary } from '@crawlee/utils';
import { sleep } from '@crawlee/utils';
-import type { OptionsInit } from 'got-scraping';
import iconv from 'iconv-lite';
import { responseSamples, runExampleComServer } from 'test/shared/_helper.js';
import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js';
@@ -218,6 +217,31 @@ describe('CheerioCrawler', () => {
expect(cheerioCrawler.ignoreSslErrors).toBeTruthy();
});
+ test('should work with skipNavigation', async () => {
+ const processed: Request[] = [];
+ const failed: Request[] = [];
+
+ const cheerioCrawler = new CheerioCrawler({
+ maxConcurrency: 1,
+ requestHandler: ({ request }) => {
+ processed.push(request);
+ },
+ failedRequestHandler: ({ request }) => {
+ failed.push(request);
+ },
+ });
+
+ await cheerioCrawler.run([
+ {
+ url: 'http://example.com/',
+ skipNavigation: true,
+ },
+ ]);
+
+ expect(processed).toHaveLength(1);
+ expect(failed).toHaveLength(0);
+ });
+
test('should work with not encoded urls', async () => {
const sources = [
{ url: `${serverAddress}/mirror?q=abc` },