Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add Statistics class and integrate it into BasicCrawler

Ondra Urban0f6ffb6e2145
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.js
index 6ae6335b0..4da49881c 100644
--- a/src/crawlers/basic_crawler.js
+++ b/src/crawlers/basic_crawler.js
@@ -8,6 +8,7 @@ import { RequestList } from '../request_list';
import { RequestQueue, RequestQueueLocal } from '../request_queue';
import events from '../events';
import { addTimeoutToPromise } from '../utils';
+import Statistics from './statistics';
/**
* Since there's no set number of seconds before the container is terminated after
@@ -192,6 +193,7 @@ class BasicCrawler {
this.handleFailedRequestFunction = handleFailedRequestFunction;
this.maxRequestRetries = maxRequestRetries;
this.handledRequestsCount = 0;
+ this.stats = new Statistics();
let shouldLogMaxPagesExceeded = true;
const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;
@@ -256,7 +258,12 @@ class BasicCrawler {
await this._loadHandledRequestCount();
this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);
this.isRunningPromise = this.autoscaledPool.run();
- await this.isRunningPromise;
+ this.stats.startLogging();
+ try {
+ await this.isRunningPromise;
+ } finally {
+ this.stats.stopLogging();
+ }
}
async _pauseOnMigration() {
@@ -328,6 +335,8 @@ class BasicCrawler {
// Reset loadedUrl so an old one is not carried over to retries.
request.loadedUrl = null;
+ const statisticsId = request.id || request.url;
+ this.stats.startJob(statisticsId);
try {
await addTimeoutToPromise(
this.handleRequestFunction({ request, autoscaledPool: this.autoscaledPool }),
@@ -335,6 +344,7 @@ class BasicCrawler {
'BasicCrawler: handleRequestFunction timed out.',
);
await source.markRequestHandled(request);
+ this.stats.finishJob(statisticsId);
this.handledRequestsCount++;
} catch (err) {
try {
@@ -408,6 +418,7 @@ class BasicCrawler {
// Mark the request as failed and do not retry.
this.handledRequestsCount++;
await source.markRequestHandled(request);
+ this.stats.failJob(request.id || request.url);
return this.handleFailedRequestFunction({ request, error }); // This function prints an error message.
}
diff --git a/src/crawlers/run_statistics.js b/src/crawlers/run_statistics.js
deleted file mode 100644
index a3eda3723..000000000
--- a/src/crawlers/run_statistics.js
+++ /dev/null
@@ -1,3 +0,0 @@
-class RunStatistics {
-
-}
diff --git a/src/crawlers/statistics.js b/src/crawlers/statistics.js
new file mode 100644
index 000000000..9d1a00de5
--- /dev/null
+++ b/src/crawlers/statistics.js
@@ -0,0 +1,96 @@
+import log from 'apify-shared/log';
+import { betterSetInterval, betterClearInterval } from 'apify-shared/utilities';
+import { checkParamOrThrow } from 'apify-client/build/utils';
+
+class Job {
+ constructor() {
+ this.lastRunAt = null;
+ this.runs = 0;
+ }
+
+ run() {
+ this.lastRunAt = new Date();
+ this.runs++;
+ return this.runs;
+ }
+
+ finish() {
+ this.durationMillis = new Date() - this.lastRunAt;
+ return this.durationMillis;
+ }
+}
+
+export default class Statistics {
+ constructor(options = {}) {
+ const {
+ logIntervalSecs = 60,
+ logMessage = 'Statistics',
+ } = options;
+
+ checkParamOrThrow(logIntervalSecs, 'options.logIntervalSecs', 'Number');
+ checkParamOrThrow(logIntervalSecs, 'options.logMessage', 'String');
+
+ this.logIntervalMillis = logIntervalSecs * 1000;
+ this.logMessage = logMessage;
+
+ this.jobRetryHistogram = [];
+ this.finishedJobs = 0;
+ this.failedJobs = 0;
+ this.jobsInProgress = new Map();
+ this.minJobDurationMillis = Infinity;
+ this.maxJobDurationMillis = 0;
+ this.totalJobDurationMillis = 0;
+ this.startedAt = null;
+ this.logInterval = null;
+ }
+
+ startJob(id) {
+ if (!this.startedAt) this.startedAt = new Date();
+ let job = this.jobsInProgress.get(id);
+ if (!job) job = new Job();
+ const retryCount = job.run() - 1;
+ this.jobRetryHistogram[retryCount] = this.jobRetryHistogram[retryCount]++ || 1;
+ this.jobsInProgress.set(id, job);
+ }
+
+ finishJob(id) {
+ const job = this.jobsInProgress.get(id);
+ if (!job) return;
+ const jobDurationMillis = job.finish();
+ this.finishedJobs++;
+ this.totalJobDurationMillis += jobDurationMillis;
+ if (jobDurationMillis < this.minJobDurationMillis) this.minJobDurationMillis = jobDurationMillis;
+ if (jobDurationMillis > this.maxJobDurationMillis) this.maxJobDurationMillis = jobDurationMillis;
+ this.jobsInProgress.delete(id);
+ }
+
+ failJob(id) {
+ const job = this.jobsInProgress.get(id);
+ if (!job) return;
+ this.failedJobs++;
+ this.jobsInProgress.delete(id);
+ }
+
+ getStatistics() {
+ const totalMillis = new Date() - this.startedAt;
+ const totalMinutes = totalMillis / 1000 / 60;
+
+ return {
+ avgDurationMillis: this.totalJobDurationMillis / this.finishedJobs,
+ perMinute: Math.round(this.finishedJobs / totalMinutes),
+ finished: this.finishedJobs,
+ failed: this.failedJobs,
+ retryHistogram: [...this.jobRetryHistogram],
+ };
+ }
+
+ startLogging() {
+ this.logInterval = betterSetInterval(() => {
+ log.info(this.logMessage, this.getStatistics());
+ }, this.logIntervalMillis);
+ }
+
+ stopLogging() {
+ betterClearInterval(this.logInterval);
+ }
+}