Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add statistics logging test
Ondra Urbane121aca41134
diff --git a/src/crawlers/statistics.js b/src/crawlers/statistics.jsindex 6d9ad35ff..52ffa1946 100644--- a/src/crawlers/statistics.js+++ b/src/crawlers/statistics.js@@ -1,5 +1,4 @@import log from 'apify-shared/log';-import { betterSetInterval, betterClearInterval } from 'apify-shared/utilities';import { checkParamOrThrow } from 'apify-client/build/utils';class Job {@@ -101,13 +100,13 @@ export default class Statistics {}startLogging() {- this.logInterval = betterSetInterval(() => {+ this.logInterval = setInterval(() => {log.info(this.logMessage, this.getCurrent());}, this.logIntervalMillis);}stopLogging() {- betterClearInterval(this.logInterval);+ clearInterval(this.logInterval);}_saveRetryCountForJob(job) {diff --git a/test/crawlers/statistics.js b/test/crawlers/statistics.jsindex c4a9b2c8e..0af610c0b 100644--- a/test/crawlers/statistics.js+++ b/test/crawlers/statistics.js@@ -1,5 +1,6 @@import { expect } from 'chai';import sinon from 'sinon';+import log from 'apify-shared/log';import Statistics from '../../build/crawlers/statistics';describe('Statistics', () => {@@ -94,4 +95,39 @@ describe('Statistics', () => {retryHistogram: [3],});});++ it('should regularly log stats', () => {+ const logged = [];+ sinon.stub(log, 'info').callsFake((...args) => {+ logged.push(args);+ });++ stats.startJob(0);+ clock.tick(1);+ stats.finishJob(0);+ stats.startLogging();+ clock.tick(50000);+ expect(logged).to.have.lengthOf(0);+ clock.tick(10001);+ expect(logged).to.have.lengthOf(1);+ expect(logged[0][0]).to.eql('Statistics');+ expect(logged[0][1]).to.eql({+ avgDurationMillis: 1,+ perMinute: 1,+ finished: 1,+ failed: 0,+ retryHistogram: [1],+ });+ stats.stopLogging();+ clock.tick(60001);+ expect(logged).to.have.lengthOf(1);+ expect(logged[0][0]).to.eql('Statistics');+ expect(logged[0][1]).to.eql({+ avgDurationMillis: 1,+ perMinute: 1,+ finished: 1,+ failed: 0,+ retryHistogram: [1],+ });+ });});