Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(core): trigger `errorHandler` for session errors (#2683)

Martin Adámek7d72bcb36f32
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index f10a498dd..19635417f 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -26,6 +26,7 @@ import type {
StatisticState,
StatisticsOptions,
LoadedContext,
+ RestrictedCrawlingContext,
} from '@crawlee/core';
import {
AutoscaledPool,
@@ -99,14 +100,13 @@ export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary>
*/
const SAFE_MIGRATION_WAIT_MILLIS = 20000;
-export type RequestHandler<Context extends CrawlingContext = BasicCrawlingContext> = (
- inputs: LoadedContext<Context>,
-) => Awaitable<void>;
+export type RequestHandler<
+ Context extends CrawlingContext = LoadedContext<BasicCrawlingContext & RestrictedCrawlingContext>,
+> = (inputs: LoadedContext<Context>) => Awaitable<void>;
-export type ErrorHandler<Context extends CrawlingContext = BasicCrawlingContext> = (
- inputs: LoadedContext<Context>,
- error: Error,
-) => Awaitable<void>;
+export type ErrorHandler<
+ Context extends CrawlingContext = LoadedContext<BasicCrawlingContext & RestrictedCrawlingContext>,
+> = (inputs: LoadedContext<Context>, error: Error) => Awaitable<void>;
export interface StatusMessageCallbackParams<
Context extends CrawlingContext = BasicCrawlingContext,
@@ -1444,13 +1444,12 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
if (shouldRetryRequest) {
await this.stats.errorTrackerRetry.addAsync(error, crawlingContext);
+ await this._tagUserHandlerError(() =>
+ this.errorHandler?.(this._augmentContextWithDeprecatedError(crawlingContext, error), error),
+ );
if (error instanceof SessionError) {
await this._rotateSession(crawlingContext);
- } else {
- await this._tagUserHandlerError(() =>
- this.errorHandler?.(this._augmentContextWithDeprecatedError(crawlingContext, error), error),
- );
}
if (!request.noRetry) {
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 53af604e8..3a9879cd4 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -1019,9 +1019,7 @@ describe('BasicCrawler', () => {
requestHandler: async () => sleep(1e3),
});
- // @ts-expect-error Overriding protected method
const warningSpy = vitest.spyOn(crawler.log, 'warning');
- // @ts-expect-error Overriding protected method
const errorSpy = vitest.spyOn(crawler.log, 'error');
await crawler.run();
@@ -1059,9 +1057,7 @@ describe('BasicCrawler', () => {
},
});
- // @ts-expect-error Overriding protected method
const warningSpy = vitest.spyOn(crawler.log, 'warning');
- // @ts-expect-error Overriding protected method
const errorSpy = vitest.spyOn(crawler.log, 'error');
await crawler.run();
@@ -1100,9 +1096,7 @@ describe('BasicCrawler', () => {
requestHandler: async () => sleep(1e3),
});
- // @ts-expect-error Overriding protected method
const warningSpy = vitest.spyOn(crawler.log, 'warning');
- // @ts-expect-error Overriding protected method
const errorSpy = vitest.spyOn(crawler.log, 'error');
await crawler.run();
@@ -1145,9 +1139,7 @@ describe('BasicCrawler', () => {
},
});
- // @ts-expect-error Overriding protected method
const warningSpy = vitest.spyOn(crawler.log, 'warning');
- // @ts-expect-error Overriding protected method
const errorSpy = vitest.spyOn(crawler.log, 'error');
await crawler.run();