Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add Statistics tests
Ondra Urbanaa3ecf55c1e1
diff --git a/src/crawlers/statistics.js b/src/crawlers/statistics.jsindex d4357671e..44225fa76 100644--- a/src/crawlers/statistics.js+++ b/src/crawlers/statistics.js@@ -17,6 +17,10 @@ class Job {this.durationMillis = new Date() - this.lastRunAt;return this.durationMillis;}++ retryCount() {+ return Math.max(0, this.runs - 1);+ }}export default class Statistics {@@ -47,8 +51,7 @@ export default class Statistics {if (!this.startedAt) this.startedAt = new Date();let job = this.jobsInProgress.get(id);if (!job) job = new Job();- const retryCount = job.run() - 1;- this.jobRetryHistogram[retryCount] = this.jobRetryHistogram[retryCount]++ || 1;+ job.run();this.jobsInProgress.set(id, job);}@@ -58,6 +61,7 @@ export default class Statistics {const jobDurationMillis = job.finish();this.finishedJobs++;this.totalJobDurationMillis += jobDurationMillis;+ this._saveRetryCountForJob(job);if (jobDurationMillis < this.minJobDurationMillis) this.minJobDurationMillis = jobDurationMillis;if (jobDurationMillis > this.maxJobDurationMillis) this.maxJobDurationMillis = jobDurationMillis;this.jobsInProgress.delete(id);@@ -67,15 +71,16 @@ export default class Statistics {const job = this.jobsInProgress.get(id);if (!job) return;this.failedJobs++;+ this._saveRetryCountForJob(job);this.jobsInProgress.delete(id);}- getStatistics() {+ getCurrent() {const totalMillis = new Date() - this.startedAt;const totalMinutes = totalMillis / 1000 / 60;return {- avgDurationMillis: this.totalJobDurationMillis / this.finishedJobs,+ avgDurationMillis: (this.totalJobDurationMillis / this.finishedJobs) || Infinity,perMinute: Math.round(this.finishedJobs / totalMinutes),finished: this.finishedJobs,failed: this.failedJobs,@@ -85,11 +90,18 @@ export default class Statistics {startLogging() {this.logInterval = betterSetInterval(() => {- log.info(this.logMessage, this.getStatistics());+ log.info(this.logMessage, this.getCurrent());}, this.logIntervalMillis);}stopLogging() {betterClearInterval(this.logInterval);}++ _saveRetryCountForJob(job) {+ const retryCount = job.retryCount();+ this.jobRetryHistogram[retryCount] = this.jobRetryHistogram[retryCount]+ ? this.jobRetryHistogram[retryCount] + 1+ : 1;+ }}diff --git a/test/basic_crawler.js b/test/crawlers/basic_crawler.jssimilarity index 99%rename from test/basic_crawler.jsrename to test/crawlers/basic_crawler.jsindex 3c6610af0..ab340eb3a 100644--- a/test/basic_crawler.js+++ b/test/crawlers/basic_crawler.js@@ -4,10 +4,10 @@ import sinon from 'sinon';import log from 'apify-shared/log';import { ACTOR_EVENT_NAMES } from 'apify-shared/consts';import { delayPromise } from 'apify-shared/utilities';-import * as Apify from '../build/index';-import * as keyValueStore from '../build/key_value_store';-import { RequestQueue, RequestQueueLocal } from '../build/request_queue';-import { LOCAL_STORAGE_DIR } from './_helper';+import * as Apify from '../../build';+import * as keyValueStore from '../../build/key_value_store';+import { RequestQueue, RequestQueueLocal } from '../../build/request_queue';+import { LOCAL_STORAGE_DIR } from '../_helper';describe('BasicCrawler', () => {let logLevel;diff --git a/test/cheerio_crawler.js b/test/crawlers/cheerio_crawler.jssimilarity index 99%rename from test/cheerio_crawler.jsrename to test/crawlers/cheerio_crawler.jsindex 4cbc237af..1610f38c7 100644--- a/test/cheerio_crawler.js+++ b/test/crawlers/cheerio_crawler.js@@ -8,7 +8,7 @@ import { expect } from 'chai';import log from 'apify-shared/log';import { delayPromise } from 'apify-shared/utilities';import { ENV_VARS } from 'apify-shared/consts';-import Apify from '../build/index';+import Apify from '../../build';// Add common props to mocked request responses.const responseMock = {diff --git a/test/data/sample.html b/test/crawlers/data/sample.htmlsimilarity index 100%rename from test/data/sample.htmlrename to test/crawlers/data/sample.htmldiff --git a/test/puppeteer_crawler.js b/test/crawlers/puppeteer_crawler.jssimilarity index 98%rename from test/puppeteer_crawler.jsrename to test/crawlers/puppeteer_crawler.jsindex 6052f24ae..259997a0e 100644--- a/test/puppeteer_crawler.js+++ b/test/crawlers/puppeteer_crawler.js@@ -1,7 +1,7 @@import { expect } from 'chai';import log from 'apify-shared/log';import { ENV_VARS } from 'apify-shared/consts';-import * as Apify from '../build/index';+import * as Apify from '../../build';describe('PuppeteerCrawler', () => {let prevEnvHeadless;diff --git a/test/crawlers/statistics.js b/test/crawlers/statistics.jsnew file mode 100644index 000000000..c4a9b2c8e--- /dev/null+++ b/test/crawlers/statistics.js@@ -0,0 +1,97 @@+import { expect } from 'chai';+import sinon from 'sinon';+import Statistics from '../../build/crawlers/statistics';++describe('Statistics', () => {+ const getPerMinute = (jobCount, totalTickMillis) => {+ return Math.round(jobCount / (totalTickMillis / 1000 / 60));+ };++ let clock;+ let stats;++ beforeEach(() => {+ clock = sinon.useFakeTimers();+ stats = new Statistics();+ });++ afterEach(() => {+ clock.restore();+ stats = null;+ clock = null;+ });++ it('should finish a job', () => {+ stats.startJob(0);+ clock.tick(1);+ stats.finishJob(0);+ clock.tick(1);+ const current = stats.getCurrent();+ expect(current).to.be.eql({+ avgDurationMillis: 1,+ perMinute: getPerMinute(1, 2),+ finished: 1,+ failed: 0,+ retryHistogram: [1],+ });+ });++ it('should fail a job', () => {+ stats.startJob(0);+ clock.tick(0);+ stats.failJob(0);+ clock.tick(1);+ const current = stats.getCurrent();+ expect(current).to.be.eql({+ avgDurationMillis: Infinity,+ perMinute: 0,+ finished: 0,+ failed: 1,+ retryHistogram: [1],+ });+ });++ it('should collect retries', () => {+ stats.startJob(0);+ stats.startJob(1);+ stats.startJob(2);+ stats.finishJob(0);+ stats.startJob(1);+ stats.startJob(2);+ stats.finishJob(1);+ stats.startJob(2);+ stats.finishJob(2);+ const current = stats.getCurrent();+ expect(current).to.be.eql({+ avgDurationMillis: Infinity,+ perMinute: Infinity,+ finished: 3,+ failed: 0,+ retryHistogram: [1, 1, 1],+ });+ });++ it('should return correct stats for multiple parallel jobs', () => {+ stats.startJob(0);+ clock.tick(1);+ stats.startJob(1);+ clock.tick(1);+ stats.startJob(2);+ clock.tick(2);+ stats.finishJob(1); // runtime: 3ms+ clock.tick(1); // since startedAt: 5ms+ stats.failJob(0); // runtime: irrelevant+ clock.tick(10);+ stats.finishJob(2); // runtime: 13ms+ clock.tick(10); // since startedAt: 25ms++ const current = stats.getCurrent();+ expect(current).to.be.eql({+ avgDurationMillis: (13 + 3) / 2,+ perMinute: getPerMinute(2, 25),+ finished: 2,+ failed: 1,+ retryHistogram: [3],+ });+ });+});