Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add statistics logging test

Ondra Urbane121aca41134
diff --git a/src/crawlers/statistics.js b/src/crawlers/statistics.js
index 6d9ad35ff..52ffa1946 100644
--- a/src/crawlers/statistics.js
+++ b/src/crawlers/statistics.js
@@ -1,5 +1,4 @@
import log from 'apify-shared/log';
-import { betterSetInterval, betterClearInterval } from 'apify-shared/utilities';
import { checkParamOrThrow } from 'apify-client/build/utils';
class Job {
@@ -101,13 +100,13 @@ export default class Statistics {
}
startLogging() {
- this.logInterval = betterSetInterval(() => {
+ this.logInterval = setInterval(() => {
log.info(this.logMessage, this.getCurrent());
}, this.logIntervalMillis);
}
stopLogging() {
- betterClearInterval(this.logInterval);
+ clearInterval(this.logInterval);
}
_saveRetryCountForJob(job) {
diff --git a/test/crawlers/statistics.js b/test/crawlers/statistics.js
index c4a9b2c8e..0af610c0b 100644
--- a/test/crawlers/statistics.js
+++ b/test/crawlers/statistics.js
@@ -1,5 +1,6 @@
import { expect } from 'chai';
import sinon from 'sinon';
+import log from 'apify-shared/log';
import Statistics from '../../build/crawlers/statistics';
describe('Statistics', () => {
@@ -94,4 +95,39 @@ describe('Statistics', () => {
retryHistogram: [3],
});
});
+
+ it('should regularly log stats', () => {
+ const logged = [];
+ sinon.stub(log, 'info').callsFake((...args) => {
+ logged.push(args);
+ });
+
+ stats.startJob(0);
+ clock.tick(1);
+ stats.finishJob(0);
+ stats.startLogging();
+ clock.tick(50000);
+ expect(logged).to.have.lengthOf(0);
+ clock.tick(10001);
+ expect(logged).to.have.lengthOf(1);
+ expect(logged[0][0]).to.eql('Statistics');
+ expect(logged[0][1]).to.eql({
+ avgDurationMillis: 1,
+ perMinute: 1,
+ finished: 1,
+ failed: 0,
+ retryHistogram: [1],
+ });
+ stats.stopLogging();
+ clock.tick(60001);
+ expect(logged).to.have.lengthOf(1);
+ expect(logged[0][0]).to.eql('Statistics');
+ expect(logged[0][1]).to.eql({
+ avgDurationMillis: 1,
+ perMinute: 1,
+ finished: 1,
+ failed: 0,
+ retryHistogram: [1],
+ });
+ });
});