Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add Statistics class and integrate it into BasicCrawler
Ondra Urban0f6ffb6e2145
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex 6ae6335b0..4da49881c 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -8,6 +8,7 @@ import { RequestList } from '../request_list';import { RequestQueue, RequestQueueLocal } from '../request_queue';import events from '../events';import { addTimeoutToPromise } from '../utils';+import Statistics from './statistics';/*** Since there's no set number of seconds before the container is terminated after@@ -192,6 +193,7 @@ class BasicCrawler {this.handleFailedRequestFunction = handleFailedRequestFunction;this.maxRequestRetries = maxRequestRetries;this.handledRequestsCount = 0;+ this.stats = new Statistics();let shouldLogMaxPagesExceeded = true;const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;@@ -256,7 +258,12 @@ class BasicCrawler {await this._loadHandledRequestCount();this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);this.isRunningPromise = this.autoscaledPool.run();- await this.isRunningPromise;+ this.stats.startLogging();+ try {+ await this.isRunningPromise;+ } finally {+ this.stats.stopLogging();+ }}async _pauseOnMigration() {@@ -328,6 +335,8 @@ class BasicCrawler {// Reset loadedUrl so an old one is not carried over to retries.request.loadedUrl = null;+ const statisticsId = request.id || request.url;+ this.stats.startJob(statisticsId);try {await addTimeoutToPromise(this.handleRequestFunction({ request, autoscaledPool: this.autoscaledPool }),@@ -335,6 +344,7 @@ class BasicCrawler {'BasicCrawler: handleRequestFunction timed out.',);await source.markRequestHandled(request);+ this.stats.finishJob(statisticsId);this.handledRequestsCount++;} catch (err) {try {@@ -408,6 +418,7 @@ class BasicCrawler {// Mark the request as failed and do not retry.this.handledRequestsCount++;await source.markRequestHandled(request);+ this.stats.failJob(request.id || request.url);return this.handleFailedRequestFunction({ request, error }); // This function prints an error message.}diff --git a/src/crawlers/run_statistics.js b/src/crawlers/run_statistics.jsdeleted file mode 100644index a3eda3723..000000000--- a/src/crawlers/run_statistics.js+++ /dev/null@@ -1,3 +0,0 @@-class RunStatistics {--}diff --git a/src/crawlers/statistics.js b/src/crawlers/statistics.jsnew file mode 100644index 000000000..9d1a00de5--- /dev/null+++ b/src/crawlers/statistics.js@@ -0,0 +1,96 @@+import log from 'apify-shared/log';+import { betterSetInterval, betterClearInterval } from 'apify-shared/utilities';+import { checkParamOrThrow } from 'apify-client/build/utils';++class Job {+ constructor() {+ this.lastRunAt = null;+ this.runs = 0;+ }++ run() {+ this.lastRunAt = new Date();+ this.runs++;+ return this.runs;+ }++ finish() {+ this.durationMillis = new Date() - this.lastRunAt;+ return this.durationMillis;+ }+}++export default class Statistics {+ constructor(options = {}) {+ const {+ logIntervalSecs = 60,+ logMessage = 'Statistics',+ } = options;++ checkParamOrThrow(logIntervalSecs, 'options.logIntervalSecs', 'Number');+ checkParamOrThrow(logIntervalSecs, 'options.logMessage', 'String');++ this.logIntervalMillis = logIntervalSecs * 1000;+ this.logMessage = logMessage;++ this.jobRetryHistogram = [];+ this.finishedJobs = 0;+ this.failedJobs = 0;+ this.jobsInProgress = new Map();+ this.minJobDurationMillis = Infinity;+ this.maxJobDurationMillis = 0;+ this.totalJobDurationMillis = 0;+ this.startedAt = null;+ this.logInterval = null;+ }++ startJob(id) {+ if (!this.startedAt) this.startedAt = new Date();+ let job = this.jobsInProgress.get(id);+ if (!job) job = new Job();+ const retryCount = job.run() - 1;+ this.jobRetryHistogram[retryCount] = this.jobRetryHistogram[retryCount]++ || 1;+ this.jobsInProgress.set(id, job);+ }++ finishJob(id) {+ const job = this.jobsInProgress.get(id);+ if (!job) return;+ const jobDurationMillis = job.finish();+ this.finishedJobs++;+ this.totalJobDurationMillis += jobDurationMillis;+ if (jobDurationMillis < this.minJobDurationMillis) this.minJobDurationMillis = jobDurationMillis;+ if (jobDurationMillis > this.maxJobDurationMillis) this.maxJobDurationMillis = jobDurationMillis;+ this.jobsInProgress.delete(id);+ }++ failJob(id) {+ const job = this.jobsInProgress.get(id);+ if (!job) return;+ this.failedJobs++;+ this.jobsInProgress.delete(id);+ }++ getStatistics() {+ const totalMillis = new Date() - this.startedAt;+ const totalMinutes = totalMillis / 1000 / 60;++ return {+ avgDurationMillis: this.totalJobDurationMillis / this.finishedJobs,+ perMinute: Math.round(this.finishedJobs / totalMinutes),+ finished: this.finishedJobs,+ failed: this.failedJobs,+ retryHistogram: [...this.jobRetryHistogram],+ };+ }++ startLogging() {+ this.logInterval = betterSetInterval(() => {+ log.info(this.logMessage, this.getStatistics());+ }, this.logIntervalMillis);+ }++ stopLogging() {+ betterClearInterval(this.logInterval);+ }+}