Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat(basic-crawler): update request retry statistics in real-time (#3773)
Harryb0237d1de4a8
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.mdindex 2cc4b0aac..23a105c3b 100644--- a/docs/public-api/crawlee-basic.api.md+++ b/docs/public-api/crawlee-basic.api.md@@ -474,6 +474,7 @@ export interface IStatistics<StateExtension extends object = {}> {recordRequestFailure(id: number | string, retryCount: number): void;recordRequestStart(id: number | string): void;recordRequestSuccess(id: number | string, retryCount: number): void;+ registerRetry?(retryCount: number): void;registerStatusCode(code: number): void;readonly requestRetryHistogram: number[];startCapturing(): Promise<void>;@@ -897,6 +898,7 @@ export class Statistics<StateExtension extends object = {}, PersistedStateExtensreadonly errorTrackerRetry: ErrorTracker;readonly id: string;persistState(): Promise<void>;+ registerRetry(retryCount: number): void;registerStatusCode(code: number): void;get requestRetryHistogram(): number[];reset(): void;diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 1fd56aea5..ca4e8aadb 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -2883,6 +2883,7 @@ export class BasicCrawler<if (!request.noRetry) {request.retryCount++;+ this.statistics.registerRetry?.(request.retryCount);const { url, retryCount, id } = request;diff --git a/packages/basic-crawler/src/internals/crawlers/statistics.ts b/packages/basic-crawler/src/internals/crawlers/statistics.tsindex 09a322e04..92cdce01a 100644--- a/packages/basic-crawler/src/internals/crawlers/statistics.ts+++ b/packages/basic-crawler/src/internals/crawlers/statistics.ts@@ -228,6 +228,12 @@ export interface IStatistics<StateExtension extends object = {}> {/** Ends the capture window: stops logging, unsubscribes, and persists the final state. */stopCapturing(): Promise<void>;+ /**+ * Registers a retry attempt in real-time when a request temporarily fails.+ * Optional - crawlers call this when a request is reclaimed back to the queue or list.+ */+ registerRetry?(retryCount: number): void;+/*** Persists the current state to the key-value store. Optional - the crawler calls it on migration, but a backend* with no persistence of its own can omit it.@@ -591,10 +597,48 @@ export class Statistics<await this.#recoverableState.teardown();}- #saveRetryCountForRequest(retryCount: number) {- if (retryCount > 0) this.state.requestsRetries++;- this.requestRetryHistogram[retryCount] ??= 0;+ /**+ * Registers a retry attempt in real-time when a request temporarily fails.+ * Includes safeguards against NaN, negative values, sparse arrays, and migration state mismatch.+ */+ registerRetry(retryCount: number) {+ if (!Number.isInteger(retryCount) || retryCount < 0) return;++ if (retryCount === 1) {+ this.state.requestsRetries++;+ }++ // Fill intermediate indices with 0 to prevent sparse arrays and JSON serialization null values+ while (this.requestRetryHistogram.length <= retryCount) {+ this.requestRetryHistogram.push(0);+ }this.requestRetryHistogram[retryCount]++;++ const previousRetryCount = retryCount - 1;+ if (previousRetryCount > 0) {+ while (this.requestRetryHistogram.length <= previousRetryCount) {+ this.requestRetryHistogram.push(0);+ }+ // Decrement previous count only if it is strictly greater than 0+ if (this.requestRetryHistogram[previousRetryCount] > 0) {+ this.requestRetryHistogram[previousRetryCount]--;+ }+ }+ }++ #saveRetryCountForRequest(retryCount: number) {+ if (!Number.isInteger(retryCount) || retryCount < 0) return;++ while (this.requestRetryHistogram.length <= retryCount) {+ this.requestRetryHistogram.push(0);+ }++ if (retryCount === 0) {+ this.requestRetryHistogram[0]++;+ } else if (this.requestRetryHistogram[retryCount] === 0) {+ this.requestRetryHistogram[retryCount]++;+ this.state.requestsRetries += retryCount;+ }}/**diff --git a/test/core/crawlers/basic-crawler.test.ts b/test/core/crawlers/basic-crawler.test.tsindex 40becff12..b7bc898df 100644--- a/test/core/crawlers/basic-crawler.test.ts+++ b/test/core/crawlers/basic-crawler.test.ts@@ -14,6 +14,7 @@ import type {RequestHandler,RequestOptions,Session,+ StatisticPersistedState,} from '@crawlee/basic';import {AfterCommitError,@@ -2776,6 +2777,94 @@ describe('BasicCrawler', () => {});});+ describe('retry statistics', () => {+ test('are updated as soon as a request is reclaimed for a retry, not once it finishes', async () => {+ const seenOnAttempt: { retryCount: number; requestsRetries: number; histogram: number[] }[] = [];++ const crawler = new BasicCrawler({+ maxRequestRetries: 3,+ requestHandler: async ({ request }) => {+ seenOnAttempt.push({+ retryCount: request.retryCount,+ requestsRetries: crawler.statistics.state.requestsRetries,+ histogram: [...crawler.statistics.requestRetryHistogram],+ });++ if (request.retryCount < 2) throw new Error(`attempt ${request.retryCount} failed`);+ },+ });++ await crawler.run(['https://example.com']);++ // The request is still in flight on each attempt, yet the stats already reflect the retries so far.+ expect(seenOnAttempt).toEqual([+ { retryCount: 0, requestsRetries: 0, histogram: [] },+ { retryCount: 1, requestsRetries: 1, histogram: [0, 1] },+ { retryCount: 2, requestsRetries: 1, histogram: [0, 0, 1] },+ ]);++ // Finishing the request does not count it a second time.+ expect(crawler.statistics.state.requestsRetries).toBe(1);+ expect(crawler.statistics.requestRetryHistogram).toEqual([0, 0, 1]);+ expect(crawler.statistics.state.requestsSucceeded).toBe(1);+ expect(crawler.statistics.state.requestsFailed).toBe(0);+ });++ test('count each request once and bucket it by its final retry count', async () => {+ const retriesBeforeSuccess: Record<string, number> = {+ 'https://example.com/clean': 0,+ 'https://example.com/flaky': 2,+ 'https://example.com/broken': Infinity,+ };++ const crawler = new BasicCrawler({+ maxRequestRetries: 3,+ requestHandler: async ({ request }) => {+ if (request.retryCount < retriesBeforeSuccess[request.url]) {+ throw new Error(`attempt ${request.retryCount} failed`);+ }+ },+ });++ await crawler.run(Object.keys(retriesBeforeSuccess));++ expect(crawler.statistics.state.requestsSucceeded).toBe(2);+ expect(crawler.statistics.state.requestsFailed).toBe(1);+ // One request was never retried, two were - regardless of how many times.+ expect(crawler.statistics.state.requestsRetries).toBe(2);+ // index = retries at the time the request finished: clean -> 0, flaky -> 2, broken -> 3 (gave up)+ expect(crawler.statistics.requestRetryHistogram).toEqual([1, 0, 1, 1]);+ });++ test('are visible in a snapshot persisted while the retried request is still in flight', async () => {+ const stats = new Statistics({ id: 'retry-stats-in-flight' });+ const store = await KeyValueStore.open();+ let persistedMidCrawl: StatisticPersistedState | null = null;++ const crawler = new BasicCrawler({+ statistics: stats,+ maxRequestRetries: 3,+ requestHandler: async ({ request }) => {+ if (request.retryCount === 0) throw new Error('first attempt failed');++ // Simulates a migration-triggered persist before the request finished.+ await stats.persistState();+ persistedMidCrawl = await store.getValue<StatisticPersistedState>(+ 'CRAWLEE_CRAWLER_STATISTICS_retry-stats-in-flight',+ );+ },+ });++ await crawler.run(['https://example.com']);++ expect(persistedMidCrawl).toMatchObject({+ requestsRetries: 1,+ requestRetryHistogram: [0, 1],+ requestsTotal: 0,+ });+ });+ });+describe('proxyConfiguration', () => {it('assigns a proxyInfo from the proxyConfiguration to each Session and exposes it on the context', async () => {const proxyUrls = [0, 1, 2].map((n) => `http://proxy.example.com:${1000 + n}`);diff --git a/test/core/crawlers/statistics.test.ts b/test/core/crawlers/statistics.test.tsindex 67fe40eeb..655cc8e56 100644--- a/test/core/crawlers/statistics.test.ts+++ b/test/core/crawlers/statistics.test.ts@@ -638,4 +638,43 @@ describe('Statistics', () => {await statsB.stopCapturing();});});++ test('should track retries in real-time and handle safeguards against negative/NaN indices', () => {+ // TC-01: First retry increment+ stats.registerRetry(1);+ expect(stats.state.requestsRetries).toEqual(1);+ expect(stats.requestRetryHistogram[1]).toEqual(1);++ // TC-02: Subsequent retries decrement previous indices+ stats.registerRetry(2);+ expect(stats.state.requestsRetries).toEqual(1); // Should not increment again+ expect(stats.requestRetryHistogram[2]).toEqual(1);+ expect(stats.requestRetryHistogram[1]).toEqual(0); // Successfully decremented++ // TC-04: Non-sequential retries (e.g., after migration / state restore)+ // Simulate a job reporting a 4th retry out of nowhere+ stats.registerRetry(4);+ expect(stats.requestRetryHistogram[4]).toEqual(1);+ expect(stats.requestRetryHistogram[3]).toEqual(0); // Filled with 0, not null/undefined+ expect(stats.requestRetryHistogram[1]).toEqual(0);++ // TC-03: Finish job with 0 retries (happy path without errors)+ stats.recordRequestStart(10);+ stats.recordRequestSuccess(10, 0);+ expect(stats.requestRetryHistogram[0]).toEqual(1);++ // Verify final array serialization safety (no sparse arrays or null values)+ const serialized = JSON.parse(JSON.stringify(stats.toJSON()));+ expect(serialized.requestRetryHistogram).not.toContain(null);+ expect(serialized.requestRetryHistogram).toEqual([1, 0, 1, 0, 1]);+ });++ test('should handle invalid or negative retryCount safely', () => {+ // Safeguard check against negative or invalid inputs to prevent infinite loops+ stats.registerRetry(-1);+ expect(stats.requestRetryHistogram.length).toEqual(0); // Should be ignored or not crash++ stats.registerRetry(NaN);+ expect(stats.requestRetryHistogram.length).toEqual(0); // Should be ignored or not crash+ });});