Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(statistics): track actual request.retryCount in Statistics (#2940)

Lukáš Křivkac9f7f5494ac4
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 54c77c173..538696b22 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -1438,7 +1438,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
} seconds.`,
);
- this.stats.finishJob(statisticsId);
+ this.stats.finishJob(statisticsId, request.retryCount);
this.handledRequestsCount++;
// reclaim session if request finishes successfully
@@ -1609,7 +1609,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
// Mark the request as failed and do not retry.
this.handledRequestsCount++;
await source.markRequestHandled(request);
- this.stats.failJob(request.id || request.uniqueKey);
+ this.stats.failJob(request.id || request.uniqueKey, request.retryCount);
await this._handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
}
diff --git a/packages/core/src/crawlers/statistics.ts b/packages/core/src/crawlers/statistics.ts
index 11c0e3894..1d81b2b88 100644
--- a/packages/core/src/crawlers/statistics.ts
+++ b/packages/core/src/crawlers/statistics.ts
@@ -14,22 +14,16 @@ import { ErrorTracker } from './error_tracker';
*/
class Job {
private lastRunAt: number | null = null;
- private runs = 0;
private durationMillis?: number;
run() {
this.lastRunAt = Date.now();
- return ++this.runs;
}
finish() {
this.durationMillis = Date.now() - this.lastRunAt!;
return this.durationMillis;
}
-
- retryCount() {
- return Math.max(0, this.runs - 1);
- }
}
const errorTrackerConfig = {
@@ -226,13 +220,13 @@ export class Statistics {
* Mark job as finished and sets the state
* @ignore
*/
- finishJob(id: number | string) {
+ finishJob(id: number | string, retryCount: number) {
const job = this.requestsInProgress.get(id);
if (!job) return;
const jobDurationMillis = job.finish();
this.state.requestsFinished++;
this.state.requestTotalFinishedDurationMillis += jobDurationMillis;
- this._saveRetryCountForJob(job);
+ this._saveRetryCountForJob(retryCount);
if (jobDurationMillis < this.state.requestMinDurationMillis)
this.state.requestMinDurationMillis = jobDurationMillis;
if (jobDurationMillis > this.state.requestMaxDurationMillis)
@@ -244,12 +238,12 @@ export class Statistics {
* Mark job as failed and sets the state
* @ignore
*/
- failJob(id: number | string) {
+ failJob(id: number | string, retryCount: number) {
const job = this.requestsInProgress.get(id);
if (!job) return;
this.state.requestTotalFailedDurationMillis += job.finish();
this.state.requestsFailed++;
- this._saveRetryCountForJob(job);
+ this._saveRetryCountForJob(retryCount);
this.requestsInProgress.delete(id);
}
@@ -313,12 +307,10 @@ export class Statistics {
await this.persistState();
}
- protected _saveRetryCountForJob(job: Job) {
- const retryCount = job.retryCount();
+ protected _saveRetryCountForJob(retryCount: number) {
if (retryCount > 0) this.state.requestsRetries++;
- this.requestRetryHistogram[retryCount] = this.requestRetryHistogram[retryCount]
- ? this.requestRetryHistogram[retryCount] + 1
- : 1;
+ this.requestRetryHistogram[retryCount] ??= 0;
+ this.requestRetryHistogram[retryCount]++;
}
/**
diff --git a/test/core/crawlers/statistics.test.ts b/test/core/crawlers/statistics.test.ts
index d8872fa6c..e2b2d7ca6 100644
--- a/test/core/crawlers/statistics.test.ts
+++ b/test/core/crawlers/statistics.test.ts
@@ -51,7 +51,7 @@ describe('Statistics', () => {
vitest.advanceTimersByTime(startedAt);
stats.startJob(0);
vitest.advanceTimersByTime(100);
- stats.finishJob(0);
+ stats.finishJob(0, 0);
await stats.startCapturing();
await stats.persistState();
@@ -172,7 +172,7 @@ describe('Statistics', () => {
test('on persistState event', async () => {
stats.startJob(0);
vitest.advanceTimersByTime(100);
- stats.finishJob(0);
+ stats.finishJob(0, 0);
await stats.startCapturing(); // keyValueStore is initialized here
@@ -194,7 +194,7 @@ describe('Statistics', () => {
test('should finish a job', () => {
stats.startJob(0);
vitest.advanceTimersByTime(1);
- stats.finishJob(0);
+ stats.finishJob(0, 0);
vitest.advanceTimersByTime(1);
const current = stats.calculate();
expect(current).toEqual({
@@ -211,7 +211,7 @@ describe('Statistics', () => {
test('should fail a job', () => {
stats.startJob(0);
vitest.advanceTimersByTime(0);
- stats.failJob(0);
+ stats.failJob(0, 0);
vitest.advanceTimersByTime(1);
const current = stats.calculate();
expect(current).toEqual({
@@ -230,12 +230,9 @@ describe('Statistics', () => {
stats.startJob(0);
stats.startJob(1);
stats.startJob(2);
- stats.finishJob(0);
- stats.startJob(1);
- stats.startJob(2);
- stats.finishJob(1);
- stats.startJob(2);
- stats.finishJob(2);
+ stats.finishJob(0, 0);
+ stats.finishJob(1, 1);
+ stats.finishJob(2, 2);
const current = stats.calculate();
expect(current).toEqual({
crawlerRuntimeMillis: 0,
@@ -256,11 +253,11 @@ describe('Statistics', () => {
vitest.advanceTimersByTime(1);
stats.startJob(2);
vitest.advanceTimersByTime(2);
- stats.finishJob(1); // runtime: 3ms
+ stats.finishJob(1, 0); // runtime: 3ms
vitest.advanceTimersByTime(1); // since startedAt: 5ms
- stats.failJob(0); // runtime: irrelevant
+ stats.failJob(0, 0); // runtime: irrelevant
vitest.advanceTimersByTime(10);
- stats.finishJob(2); // runtime: 13ms
+ stats.finishJob(2, 0); // runtime: 13ms
vitest.advanceTimersByTime(10); // since startedAt: 25ms
const current = stats.calculate();
@@ -290,7 +287,7 @@ describe('Statistics', () => {
stats.startJob(0);
vitest.advanceTimersByTime(1);
- stats.finishJob(0);
+ stats.finishJob(0, 0);
await stats.startCapturing();
vitest.advanceTimersByTime(50000);
expect(logged).toHaveLength(0);
@@ -327,7 +324,7 @@ describe('Statistics', () => {
await stats.startCapturing();
stats.startJob(1);
vitest.advanceTimersByTime(3);
- stats.finishJob(1);
+ stats.finishJob(1, 0);
expect(stats.state.requestsFinished).toEqual(1);
expect(stats.requestRetryHistogram).toEqual([1]);
stats.reset();