Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: rename interface `FailedRequestHandler` to `ErrorHandler` (#1410)
Martin Adámekc5b74c6a741f
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex ba744364f..96e6734d3 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -61,7 +61,7 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000;export type RequestHandler<Context extends CrawlingContext = BasicCrawlingContext> = (inputs: Context) => Awaitable<void>;-export type FailedRequestHandler<Context extends CrawlingContext = BasicCrawlingContext> = (inputs: Context, error: Error) => Awaitable<void>;+export type ErrorHandler<Context extends CrawlingContext = BasicCrawlingContext> = (inputs: Context, error: Error) => Awaitable<void>;export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCrawlingContext> {/**@@ -144,7 +144,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw* Second argument is the `Error` instance that* represents the last error thrown during processing of the request.*/- errorHandler?: FailedRequestHandler<Context>;+ errorHandler?: ErrorHandler<Context>;/*** A function to handle requests that failed more than {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.@@ -154,7 +154,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw* Second argument is the `Error` instance that* represents the last error thrown during processing of the request.*/- failedRequestHandler?: FailedRequestHandler<Context>;+ failedRequestHandler?: ErrorHandler<Context>;/*** A function to handle requests that failed more than {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.@@ -167,7 +167,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw* @deprecated `handleFailedRequestFunction` has been renamed to `failedRequestHandler` and will be removed in a future version.* @ignore*/- handleFailedRequestFunction?: FailedRequestHandler<Context>;+ handleFailedRequestFunction?: ErrorHandler<Context>;/*** Indicates how many times the request is retried if {@link BasicCrawlerOptions.requestHandler|`requestHandler`} fails.@@ -335,8 +335,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextprotected log: Log;protected requestHandler!: RequestHandler<Context>;- protected errorHandler?: FailedRequestHandler<Context>;- protected failedRequestHandler?: FailedRequestHandler<Context>;+ protected errorHandler?: ErrorHandler<Context>;+ protected failedRequestHandler?: ErrorHandler<Context>;protected requestHandlerTimeoutMillis!: number;protected internalTimeoutMillis: number;protected maxRequestRetries: number;diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.tsindex 981380dc5..1186cbcb5 100644--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts@@ -1,6 +1,6 @@import { addTimeoutToPromise, tryCancel } from '@apify/timeout';import { concatStreamToBuffer, readStreamToString } from '@apify/utilities';-import type { BasicCrawlerOptions } from '@crawlee/basic';+import type { BasicCrawlerOptions, ErrorHandler } from '@crawlee/basic';import { BasicCrawler, BASIC_CRAWLER_TIMEOUT_BUFFER_SECS } from '@crawlee/basic';import type { CrawlingContext, EnqueueLinksOptions, ProxyConfiguration, Request, RequestQueue, Session } from '@crawlee/core';import { CrawlerExtension, enqueueLinks, mergeCookies, Router, resolveBaseUrl, validators } from '@crawlee/core';@@ -35,7 +35,7 @@ const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {},};-export type CheerioFailedRequestHandler<JSONData = Dictionary> = (inputs: CheerioCrawlingContext<JSONData>, error: Error) => Awaitable<void>;+export type CheerioErrorHandler<JSONData = Dictionary> = ErrorHandler<CheerioCrawlingContext<JSONData>>;export interface CheerioCrawlerOptions<JSONData = Dictionary> extends Omit<BasicCrawlerOptions<CheerioCrawlingContext<JSONData>>,// Overridden with cheerio context@@ -137,7 +137,7 @@ export interface CheerioCrawlerOptions<JSONData = Dictionary> extends Omit<Basic* Second argument is the `Error` instance that* represents the last error thrown during processing of the request.*/- errorHandler?: CheerioFailedRequestHandler<JSONData>;+ errorHandler?: CheerioErrorHandler<JSONData>;/*** A function to handle requests that failed more than `option.maxRequestRetries` times.@@ -150,7 +150,7 @@ export interface CheerioCrawlerOptions<JSONData = Dictionary> extends Omit<Basic* See [source code](https://github.com/apify/crawlee/blob/master/src/crawlers/cheerio_crawler.js#L13)* for the default implementation of this function.*/- failedRequestHandler?: CheerioFailedRequestHandler<JSONData>;+ failedRequestHandler?: CheerioErrorHandler<JSONData>;/*** A function to handle requests that failed more than `option.maxRequestRetries` times.@@ -166,7 +166,7 @@ export interface CheerioCrawlerOptions<JSONData = Dictionary> extends Omit<Basic* @deprecated `handleFailedRequestFunction` has been renamed to `failedRequestHandler` and will be removed in a future version.* @ignore*/- handleFailedRequestFunction?: CheerioFailedRequestHandler<JSONData>;+ handleFailedRequestFunction?: CheerioErrorHandler<JSONData>;/*** Async functions that are sequentially evaluated before the navigation. Good for setting additional cookiesdiff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 1d0d4d096..8b82ef52a 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -4,7 +4,7 @@ import type { AddressInfo } from 'net';import log from '@apify/log';import type {CrawlingContext,- FailedRequestHandler,+ ErrorHandler,RequestHandler } from '@crawlee/basic';import {Request,@@ -278,13 +278,13 @@ describe('BasicCrawler', () => {throw new Error(`This is an error ${errorHandlerCalls}`);};- const errorHandler: FailedRequestHandler = async ({ request }, error) => {+ const errorHandler: ErrorHandler = async ({ request }, error) => {expect(error.message).toBe(`This is an error ${errorHandlerCalls}`);errorHandlerCalls++;request.label = `error_${errorHandlerCalls}`;};- const failedRequestHandler: FailedRequestHandler = async ({ request }, error) => {+ const failedRequestHandler: ErrorHandler = async ({ request }, error) => {failed[request.url] = { request, error };failedRequestHandlerCalls++;};@@ -322,7 +322,7 @@ describe('BasicCrawler', () => {processed[request.url] = request;};- const failedRequestHandler: FailedRequestHandler = async ({ request }, error) => {+ const failedRequestHandler: ErrorHandler = async ({ request }, error) => {failed[request.url] = request;errors.push(error);};@@ -362,7 +362,7 @@ describe('BasicCrawler', () => {throw new NonRetryableError('some-error');};- const failedRequestHandler: FailedRequestHandler = async ({ request }, error) => {+ const failedRequestHandler: ErrorHandler = async ({ request }, error) => {failed[request.url] = request;errors.push(error);};@@ -399,7 +399,7 @@ describe('BasicCrawler', () => {throw new CriticalError('some-error');};- const failedRequestHandler = jest.fn() as FailedRequestHandler;+ const failedRequestHandler = jest.fn() as ErrorHandler;const basicCrawler = new BasicCrawler({requestList,@@ -421,7 +421,7 @@ describe('BasicCrawler', () => {];const requestList = await RequestList.open(null, sources);- const failedRequestHandler = jest.fn() as FailedRequestHandler;+ const failedRequestHandler = jest.fn() as ErrorHandler;const basicCrawler = new BasicCrawler({requestList,