Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(core): trigger `errorHandler` for session errors (#2683)
Martin Adámek7d72bcb36f32
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex f10a498dd..19635417f 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -26,6 +26,7 @@ import type {StatisticState,StatisticsOptions,LoadedContext,+ RestrictedCrawlingContext,} from '@crawlee/core';import {AutoscaledPool,@@ -99,14 +100,13 @@ export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary>*/const SAFE_MIGRATION_WAIT_MILLIS = 20000;-export type RequestHandler<Context extends CrawlingContext = BasicCrawlingContext> = (- inputs: LoadedContext<Context>,-) => Awaitable<void>;+export type RequestHandler<+ Context extends CrawlingContext = LoadedContext<BasicCrawlingContext & RestrictedCrawlingContext>,+> = (inputs: LoadedContext<Context>) => Awaitable<void>;-export type ErrorHandler<Context extends CrawlingContext = BasicCrawlingContext> = (- inputs: LoadedContext<Context>,- error: Error,-) => Awaitable<void>;+export type ErrorHandler<+ Context extends CrawlingContext = LoadedContext<BasicCrawlingContext & RestrictedCrawlingContext>,+> = (inputs: LoadedContext<Context>, error: Error) => Awaitable<void>;export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext,@@ -1444,13 +1444,12 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextif (shouldRetryRequest) {await this.stats.errorTrackerRetry.addAsync(error, crawlingContext);+ await this._tagUserHandlerError(() =>+ this.errorHandler?.(this._augmentContextWithDeprecatedError(crawlingContext, error), error),+ );if (error instanceof SessionError) {await this._rotateSession(crawlingContext);- } else {- await this._tagUserHandlerError(() =>- this.errorHandler?.(this._augmentContextWithDeprecatedError(crawlingContext, error), error),- );}if (!request.noRetry) {diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 53af604e8..3a9879cd4 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -1019,9 +1019,7 @@ describe('BasicCrawler', () => {requestHandler: async () => sleep(1e3),});- // @ts-expect-error Overriding protected methodconst warningSpy = vitest.spyOn(crawler.log, 'warning');- // @ts-expect-error Overriding protected methodconst errorSpy = vitest.spyOn(crawler.log, 'error');await crawler.run();@@ -1059,9 +1057,7 @@ describe('BasicCrawler', () => {},});- // @ts-expect-error Overriding protected methodconst warningSpy = vitest.spyOn(crawler.log, 'warning');- // @ts-expect-error Overriding protected methodconst errorSpy = vitest.spyOn(crawler.log, 'error');await crawler.run();@@ -1100,9 +1096,7 @@ describe('BasicCrawler', () => {requestHandler: async () => sleep(1e3),});- // @ts-expect-error Overriding protected methodconst warningSpy = vitest.spyOn(crawler.log, 'warning');- // @ts-expect-error Overriding protected methodconst errorSpy = vitest.spyOn(crawler.log, 'error');await crawler.run();@@ -1145,9 +1139,7 @@ describe('BasicCrawler', () => {},});- // @ts-expect-error Overriding protected methodconst warningSpy = vitest.spyOn(crawler.log, 'warning');- // @ts-expect-error Overriding protected methodconst errorSpy = vitest.spyOn(crawler.log, 'error');await crawler.run();