Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add Statistics tests

Ondra Urbanaa3ecf55c1e1
diff --git a/src/crawlers/statistics.js b/src/crawlers/statistics.js
index d4357671e..44225fa76 100644
--- a/src/crawlers/statistics.js
+++ b/src/crawlers/statistics.js
@@ -17,6 +17,10 @@ class Job {
this.durationMillis = new Date() - this.lastRunAt;
return this.durationMillis;
}
+
+ retryCount() {
+ return Math.max(0, this.runs - 1);
+ }
}
export default class Statistics {
@@ -47,8 +51,7 @@ export default class Statistics {
if (!this.startedAt) this.startedAt = new Date();
let job = this.jobsInProgress.get(id);
if (!job) job = new Job();
- const retryCount = job.run() - 1;
- this.jobRetryHistogram[retryCount] = this.jobRetryHistogram[retryCount]++ || 1;
+ job.run();
this.jobsInProgress.set(id, job);
}
@@ -58,6 +61,7 @@ export default class Statistics {
const jobDurationMillis = job.finish();
this.finishedJobs++;
this.totalJobDurationMillis += jobDurationMillis;
+ this._saveRetryCountForJob(job);
if (jobDurationMillis < this.minJobDurationMillis) this.minJobDurationMillis = jobDurationMillis;
if (jobDurationMillis > this.maxJobDurationMillis) this.maxJobDurationMillis = jobDurationMillis;
this.jobsInProgress.delete(id);
@@ -67,15 +71,16 @@ export default class Statistics {
const job = this.jobsInProgress.get(id);
if (!job) return;
this.failedJobs++;
+ this._saveRetryCountForJob(job);
this.jobsInProgress.delete(id);
}
- getStatistics() {
+ getCurrent() {
const totalMillis = new Date() - this.startedAt;
const totalMinutes = totalMillis / 1000 / 60;
return {
- avgDurationMillis: this.totalJobDurationMillis / this.finishedJobs,
+ avgDurationMillis: (this.totalJobDurationMillis / this.finishedJobs) || Infinity,
perMinute: Math.round(this.finishedJobs / totalMinutes),
finished: this.finishedJobs,
failed: this.failedJobs,
@@ -85,11 +90,18 @@ export default class Statistics {
startLogging() {
this.logInterval = betterSetInterval(() => {
- log.info(this.logMessage, this.getStatistics());
+ log.info(this.logMessage, this.getCurrent());
}, this.logIntervalMillis);
}
stopLogging() {
betterClearInterval(this.logInterval);
}
+
+ _saveRetryCountForJob(job) {
+ const retryCount = job.retryCount();
+ this.jobRetryHistogram[retryCount] = this.jobRetryHistogram[retryCount]
+ ? this.jobRetryHistogram[retryCount] + 1
+ : 1;
+ }
}
diff --git a/test/basic_crawler.js b/test/crawlers/basic_crawler.js
similarity index 99%
rename from test/basic_crawler.js
rename to test/crawlers/basic_crawler.js
index 3c6610af0..ab340eb3a 100644
--- a/test/basic_crawler.js
+++ b/test/crawlers/basic_crawler.js
@@ -4,10 +4,10 @@ import sinon from 'sinon';
import log from 'apify-shared/log';
import { ACTOR_EVENT_NAMES } from 'apify-shared/consts';
import { delayPromise } from 'apify-shared/utilities';
-import * as Apify from '../build/index';
-import * as keyValueStore from '../build/key_value_store';
-import { RequestQueue, RequestQueueLocal } from '../build/request_queue';
-import { LOCAL_STORAGE_DIR } from './_helper';
+import * as Apify from '../../build';
+import * as keyValueStore from '../../build/key_value_store';
+import { RequestQueue, RequestQueueLocal } from '../../build/request_queue';
+import { LOCAL_STORAGE_DIR } from '../_helper';
describe('BasicCrawler', () => {
let logLevel;
diff --git a/test/cheerio_crawler.js b/test/crawlers/cheerio_crawler.js
similarity index 99%
rename from test/cheerio_crawler.js
rename to test/crawlers/cheerio_crawler.js
index 4cbc237af..1610f38c7 100644
--- a/test/cheerio_crawler.js
+++ b/test/crawlers/cheerio_crawler.js
@@ -8,7 +8,7 @@ import { expect } from 'chai';
import log from 'apify-shared/log';
import { delayPromise } from 'apify-shared/utilities';
import { ENV_VARS } from 'apify-shared/consts';
-import Apify from '../build/index';
+import Apify from '../../build';
// Add common props to mocked request responses.
const responseMock = {
diff --git a/test/data/sample.html b/test/crawlers/data/sample.html
similarity index 100%
rename from test/data/sample.html
rename to test/crawlers/data/sample.html
diff --git a/test/puppeteer_crawler.js b/test/crawlers/puppeteer_crawler.js
similarity index 98%
rename from test/puppeteer_crawler.js
rename to test/crawlers/puppeteer_crawler.js
index 6052f24ae..259997a0e 100644
--- a/test/puppeteer_crawler.js
+++ b/test/crawlers/puppeteer_crawler.js
@@ -1,7 +1,7 @@
import { expect } from 'chai';
import log from 'apify-shared/log';
import { ENV_VARS } from 'apify-shared/consts';
-import * as Apify from '../build/index';
+import * as Apify from '../../build';
describe('PuppeteerCrawler', () => {
let prevEnvHeadless;
diff --git a/test/crawlers/statistics.js b/test/crawlers/statistics.js
new file mode 100644
index 000000000..c4a9b2c8e
--- /dev/null
+++ b/test/crawlers/statistics.js
@@ -0,0 +1,97 @@
+import { expect } from 'chai';
+import sinon from 'sinon';
+import Statistics from '../../build/crawlers/statistics';
+
+describe('Statistics', () => {
+ const getPerMinute = (jobCount, totalTickMillis) => {
+ return Math.round(jobCount / (totalTickMillis / 1000 / 60));
+ };
+
+ let clock;
+ let stats;
+
+ beforeEach(() => {
+ clock = sinon.useFakeTimers();
+ stats = new Statistics();
+ });
+
+ afterEach(() => {
+ clock.restore();
+ stats = null;
+ clock = null;
+ });
+
+ it('should finish a job', () => {
+ stats.startJob(0);
+ clock.tick(1);
+ stats.finishJob(0);
+ clock.tick(1);
+ const current = stats.getCurrent();
+ expect(current).to.be.eql({
+ avgDurationMillis: 1,
+ perMinute: getPerMinute(1, 2),
+ finished: 1,
+ failed: 0,
+ retryHistogram: [1],
+ });
+ });
+
+ it('should fail a job', () => {
+ stats.startJob(0);
+ clock.tick(0);
+ stats.failJob(0);
+ clock.tick(1);
+ const current = stats.getCurrent();
+ expect(current).to.be.eql({
+ avgDurationMillis: Infinity,
+ perMinute: 0,
+ finished: 0,
+ failed: 1,
+ retryHistogram: [1],
+ });
+ });
+
+ it('should collect retries', () => {
+ stats.startJob(0);
+ stats.startJob(1);
+ stats.startJob(2);
+ stats.finishJob(0);
+ stats.startJob(1);
+ stats.startJob(2);
+ stats.finishJob(1);
+ stats.startJob(2);
+ stats.finishJob(2);
+ const current = stats.getCurrent();
+ expect(current).to.be.eql({
+ avgDurationMillis: Infinity,
+ perMinute: Infinity,
+ finished: 3,
+ failed: 0,
+ retryHistogram: [1, 1, 1],
+ });
+ });
+
+ it('should return correct stats for multiple parallel jobs', () => {
+ stats.startJob(0);
+ clock.tick(1);
+ stats.startJob(1);
+ clock.tick(1);
+ stats.startJob(2);
+ clock.tick(2);
+ stats.finishJob(1); // runtime: 3ms
+ clock.tick(1); // since startedAt: 5ms
+ stats.failJob(0); // runtime: irrelevant
+ clock.tick(10);
+ stats.finishJob(2); // runtime: 13ms
+ clock.tick(10); // since startedAt: 25ms
+
+ const current = stats.getCurrent();
+ expect(current).to.be.eql({
+ avgDurationMillis: (13 + 3) / 2,
+ perMinute: getPerMinute(2, 25),
+ finished: 2,
+ failed: 1,
+ retryHistogram: [3],
+ });
+ });
+});