Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(core): make skipNavigation request bookkeeping proxy-safe (#4099)
Noah73ee99530e79
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex ca4e8aadb..ca3a4c440 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -1262,8 +1262,8 @@ export class BasicCrawler<crawlingContext,this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext,- (ctx) => this.handleRequest(ctx, source, request),- (error) => this.#handleRequestFailure(error, crawlingContext, source, request),+ (ctx) => this.handleRequest(ctx, source),+ (error) => this.#handleRequestFailure(error, crawlingContext, source),),),);@@ -1276,7 +1276,6 @@ export class BasicCrawler<await this.requestFunctionErrorHandler(unwrappedError,crawlingContext as CrawlingContext,- request,this.requestManager!,);if (!this.#errorAbsolvesSession(unwrappedError)) {@@ -2607,11 +2606,7 @@ export class BasicCrawler</** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel- private async handleRequest(- crawlingContext: ExtendedContext,- requestSource: IRequestManager,- request: CrawlingRequest,- ) {+ private async handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager) {// An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,// which already failed the request in `runTaskFunction`. Bail before running the handler so it does not// execute (and re-report) on top of a request the crawler has already moved past.@@ -2619,6 +2614,8 @@ export class BasicCrawler<return;}+ const { request } = crawlingContext;+request.state = RequestState.REQUEST_HANDLER;await this.runRequestHandler(crawlingContext);@@ -2628,7 +2625,7 @@ export class BasicCrawler<await currentStorageTransaction()?.commit();await this.#timeoutAndRetry(- async () => requestSource.markRequestAsHandled(request!),+ async () => requestSource.markRequestAsHandled(request),this.internalTimeoutMillis,`Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3@@ -2651,8 +2648,9 @@ export class BasicCrawler<rawError: unknown,crawlingContext: PendingCrawlingContext,requestSource: IRequestManager,- request: CrawlingRequest,) {+ const { request } = crawlingContext;+// Roll back *before* any error handler runs - error handlers write to real storage precisely// because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.currentStorageTransaction()?.rollback();@@ -2668,25 +2666,21 @@ export class BasicCrawler<: unwrappedError;if (err) {- await this.#runErrorHandler(err, crawlingContext as CrawlingContext, requestSource, request);+ await this.#runErrorHandler(err, crawlingContext as CrawlingContext, requestSource);}} finally {resumeRequestTimeout?.();}}- async #runErrorHandler(- err: Error,- context: CrawlingContext,- requestSource: IRequestManager,- request: CrawlingRequest,- ) {+ async #runErrorHandler(err: Error, context: CrawlingContext, requestSource: IRequestManager) {+ const { request } = context;const retryCountBefore = request.retryCount;try {request.state = RequestState.ERROR_HANDLER;await addTimeoutToPromise(- async () => this.requestFunctionErrorHandler(err, context, request, requestSource),+ async () => this.requestFunctionErrorHandler(err, context, requestSource),this.internalTimeoutMillis,`Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3@@ -2825,16 +2819,15 @@ export class BasicCrawler</*** Handles errors thrown by user provided requestHandler()- *- * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request*/// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otelprivate async requestFunctionErrorHandler(error: Error,crawlingContext: CrawlingContext,- request: CrawlingRequest,source: IRequestManager,): Promise<void> {+ const { request } = crawlingContext;+if (error instanceof RequestThrottledError) {// The domain told us to come back later, so the request was never really attempted. Put it back where// it was, without recording a failure - it costs neither a retry nor session reputation.diff --git a/packages/basic-crawler/src/internals/errors.ts b/packages/basic-crawler/src/internals/errors.tsindex c0f92faf6..a1a279270 100644--- a/packages/basic-crawler/src/internals/errors.ts+++ b/packages/basic-crawler/src/internals/errors.ts@@ -91,3 +91,31 @@ export class RequestHandlerError extends Error {* Subclasses can catch this error to skip their own navigation-dependent logic.*/export class NavigationSkippedError extends NonRetryableError {}++/**+ * Wraps a `skipNavigation` request so that reading `loadedUrl` throws a {@apilink NavigationSkippedError}.+ * @internal+ */+export function skippedNavigationRequest<R extends object>(request: R): R {+ return new Proxy(request, {+ get(target, propertyName) {+ if (propertyName === 'loadedUrl') {+ throw new NavigationSkippedError(+ 'The `request.loadedUrl` property is not available - `skipNavigation` was used',+ );+ }+ // `target` as receiver and bound methods, or `#` members throw on the proxy+ const value = Reflect.get(target, propertyName, target);+ return typeof value === 'function' ? value.bind(target) : value;+ },+ // Hide `loadedUrl` from enumeration so bookkeeping (spread/JSON/zod) skips it; direct reads still throw above.+ getOwnPropertyDescriptor(target, propertyName) {+ const descriptor = Reflect.getOwnPropertyDescriptor(target, propertyName);+ if (propertyName === 'loadedUrl' && descriptor) {+ return { ...descriptor, enumerable: false };+ }+ return descriptor;+ },+ set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),+ });+}diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 8faf3e8de..0dcd4d57e 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -20,6 +20,7 @@ import {cookieStringToToughCookie,EnqueueStrategy,NavigationSkippedError,+ skippedNavigationRequest,OwnedOrInjected,remainingNavigationWindowMillis,RequestState,@@ -625,19 +626,7 @@ export abstract class BrowserCrawler<async #prepareNavigation(crawlingContext: Context): Promise<Partial<Context>> {if (crawlingContext.request.skipNavigation) {return {- request: new Proxy(crawlingContext.request, {- get(target, propertyName) {- if (propertyName === 'loadedUrl') {- throw new NavigationSkippedError(- 'The `request.loadedUrl` property is not available - `skipNavigation` was used',- );- }- // `target` as receiver and bound methods, or `#` members throw on the proxy- const value = Reflect.get(target, propertyName, target);- return typeof value === 'function' ? value.bind(target) : value;- },- set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),- }) as LoadedRequest<CrawlingRequest>,+ request: skippedNavigationRequest(crawlingContext.request) as LoadedRequest<CrawlingRequest>,get response(): Response {throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used',diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 3ecdf2e3c..12189e33e 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -23,6 +23,7 @@ import {ContextPipeline,getCookiesFromResponse,NavigationSkippedError,+ skippedNavigationRequest,remainingNavigationWindowMillis,RequestState,RequestThrottledError,@@ -513,19 +514,7 @@ export class HttpCrawler<if (request.skipNavigation) {return {- request: new Proxy(request, {- get(target, propertyName) {- if (propertyName === 'loadedUrl') {- throw new NavigationSkippedError(- 'The `request.loadedUrl` property is not available - `skipNavigation` was used',- );- }- // `target` as receiver and bound methods, or `#` members throw on the proxy- const value = Reflect.get(target, propertyName, target);- return typeof value === 'function' ? value.bind(target) : value;- },- set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),- }) as LoadedRequest<CrawlingRequest>,+ request: skippedNavigationRequest(request) as LoadedRequest<CrawlingRequest>,get response(): InternalHttpCrawlingContext['response'] {throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used',diff --git a/packages/otel/src/constants.ts b/packages/otel/src/constants.tsindex fa2dca65a..86247f145 100644--- a/packages/otel/src/constants.ts+++ b/packages/otel/src/constants.ts@@ -72,7 +72,7 @@ export const requestHandlingInstrumentationMethods: ClassMethodToInstrument[] =className: 'BasicCrawler',methodName: 'handleRequest',spanName: 'crawlee.crawler.handleRequest',- // `handleRequest(crawlingContext, requestSource, request)`+ // `handleRequest(crawlingContext, requestSource)`spanOptions(crawlingContext: CrawlingContextLike) {return { attributes: requestAttributes(crawlingContext) };},@@ -103,7 +103,7 @@ export const requestHandlingInstrumentationMethods: ClassMethodToInstrument[] =className: 'BasicCrawler',methodName: 'requestFunctionErrorHandler',spanName: 'crawlee.crawler.requestFunctionErrorHandler',- // `requestFunctionErrorHandler(error, crawlingContext, request, source)`+ // `requestFunctionErrorHandler(error, crawlingContext, source)`spanOptions(_error: Error, crawlingContext: CrawlingContextLike) {return { attributes: requestAttributes(crawlingContext) };},diff --git a/test/core/crawlers/browser-crawler.test.ts b/test/core/crawlers/browser-crawler.test.tsindex e8581d662..9ddd7836a 100644--- a/test/core/crawlers/browser-crawler.test.ts+++ b/test/core/crawlers/browser-crawler.test.ts@@ -17,7 +17,16 @@ import {} from '@crawlee/basic';import { bindMethodsToServiceLocator, MemoryStorageBackend, serviceLocator, ServiceLocator } from '@crawlee/core';import type { PuppeteerGoToOptions } from '@crawlee/puppeteer';-import { EnqueueStrategy, ProxyConfiguration, Request, RequestList, RequestState, Session } from '@crawlee/puppeteer';+import {+ EnqueueStrategy,+ NavigationSkippedError,+ ProxyConfiguration,+ Request,+ RequestList,+ RequestQueue,+ RequestState,+ Session,+} from '@crawlee/puppeteer';import { sleep } from '@crawlee/utils';import type { HTTPResponse } from 'puppeteer';import puppeteer from 'puppeteer';@@ -118,6 +127,32 @@ describe('BrowserCrawler', () => {});});+ test.concurrent('marks a skipNavigation request as handled via the proxied context', async () => {+ // A RequestQueue (unlike RequestList) serializes the whole request during bookkeeping.+ const requestQueue = await RequestQueue.open();+ const { uniqueKey } = await requestQueue.addRequest({+ url: `${serverAddress}/`,+ userData: { foo: 'bar' },+ skipNavigation: true,+ });+ const failedRequestHandler = vitest.fn();++ await new BrowserCrawlerTest({+ browserPoolOptions: { browserPlugins: [new PuppeteerPlugin(puppeteer)] },+ requestQueue,+ requestHandler: async ({ request }) => {+ expect(() => request.loadedUrl).toThrow(NavigationSkippedError);+ },+ failedRequestHandler,+ }).run();++ expect(failedRequestHandler).not.toHaveBeenCalled();+ expect(await requestQueue.getRequest(uniqueKey)).toMatchObject({+ handledAt: expect.any(String),+ userData: { foo: 'bar' },+ });+ });+test.concurrent('a run releases the browsers of an owned pool instead of destroying it', async () => {const puppeteerPlugin = new PuppeteerPlugin(puppeteer);diff --git a/test/core/crawlers/http-crawler.test.ts b/test/core/crawlers/http-crawler.test.tsindex f6bce1189..3864e0a0f 100644--- a/test/core/crawlers/http-crawler.test.ts+++ b/test/core/crawlers/http-crawler.test.ts@@ -7,6 +7,7 @@ import { MemoryStorageBackend, serviceLocator } from '@crawlee/core';import {ConcurrencySystem,HttpCrawler,+ NavigationSkippedError,PersistentRateLimitError,RequestList,RequestQueue,@@ -649,6 +650,28 @@ test('skipRequest() in a navigation hook skips the request and rolls back its stawait expect(crawler.getData()).resolves.toMatchObject({ items: [] });});+test('marks a skipNavigation request as handled via the proxied context', async () => {+ // A RequestQueue (unlike RequestList) serializes the whole request during bookkeeping.+ const requestQueue = await RequestQueue.open();+ const { uniqueKey } = await requestQueue.addRequest({ url, userData: { foo: 'bar' }, skipNavigation: true });+ const failedRequestHandler = vitest.fn();++ await new HttpCrawler({+ requestQueue,+ maxRequestRetries: 0,+ requestHandler: async ({ request }) => {+ expect(() => request.loadedUrl).toThrow(NavigationSkippedError);+ },+ failedRequestHandler,+ }).run();++ expect(failedRequestHandler).not.toHaveBeenCalled();+ expect(await requestQueue.getRequest(uniqueKey)).toMatchObject({+ handledAt: expect.any(String),+ userData: { foo: 'bar' },+ });+});+test('works with a custom HttpClient', async () => {const results: string[] = [];diff --git a/test/core/storages/request-queue.test.ts b/test/core/storages/request-queue.test.tsindex cd9f3fb78..c8bb84f5d 100644--- a/test/core/storages/request-queue.test.ts+++ b/test/core/storages/request-queue.test.ts@@ -2,7 +2,8 @@ import { rm } from 'node:fs/promises';import { resolve } from 'node:path';/* eslint-disable dot-notation */-import { CrawlingRequest } from '@crawlee/basic';+import type { LoadedRequest } from '@crawlee/basic';+import { CrawlingRequest, NavigationSkippedError, skippedNavigationRequest } from '@crawlee/basic';import { MemoryStorageBackend, ProxyConfiguration, Request, RequestQueue, serviceLocator } from '@crawlee/core';import { BaseHttpClient } from '@crawlee/http-client';import { FileSystemStorageBackend } from '@crawlee/fs-storage';@@ -758,6 +759,32 @@ describe('RequestQueue background batches', () => {}, 10_000);});+describe('RequestQueue bookkeeping with a skipNavigation request', () => {+ beforeEach(async () => {+ serviceLocator.setStorageBackend(new MemoryStorageBackend());+ vitest.clearAllMocks();+ });++ test.each([{ method: 'markRequestAsHandled' }, { method: 'reclaimRequest' }] as const)(+ '$method accepts the proxied request',+ async ({ method }) => {+ const queue = await RequestQueue.open();+ await queue.addRequest({ url: 'http://example.com/', userData: { foo: 'bar' }, skipNavigation: true });+ const fetched = (await queue.fetchNextRequest())!;+ const proxied = skippedNavigationRequest(fetched) as LoadedRequest<Request>;++ await expect(queue[method](proxied)).resolves.not.toBeNull();++ // Hiding `loadedUrl` from enumeration must not disarm the guard for user code.+ expect(() => proxied.loadedUrl).toThrow(NavigationSkippedError);+ expect(await queue.getRequest(fetched.uniqueKey)).toMatchObject({+ url: 'http://example.com/',+ userData: { foo: 'bar' },+ });+ },+ );+});+describe('MemoryStorageBackend request queue', () => {test('head operations do not scan already-handled requests', async () => {const backend = new MemoryStorageBackend();diff --git a/test/otel/instrumentation-config.test.ts b/test/otel/instrumentation-config.test.tsindex c013515c0..0d085c61c 100644--- a/test/otel/instrumentation-config.test.ts+++ b/test/otel/instrumentation-config.test.ts@@ -244,7 +244,7 @@ describe('requestHandlingInstrumentationMethods', () => {['makeHttpRequest', 0],['navigate', 0],['handleFailedRequestHandler', 0],- // `requestFunctionErrorHandler(error, crawlingContext, request, source)`+ // `requestFunctionErrorHandler(error, crawlingContext, source)`['requestFunctionErrorHandler', 1],])('%s reads request attributes from argument %i', (methodName, contextArgIndex) => {const methods = requestHandlingInstrumentationMethods.filter((m) => m.methodName === methodName);