Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat(basic-crawler): allow configuring the automatic status message (#2001)

Martin Adámek3eb4e4c558b4
diff --git a/.eslintrc.json b/.eslintrc.json
index a23542024..0a78e0638 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -52,6 +52,11 @@
"no-console": "error",
"no-underscore-dangle": 0,
"no-void": 0,
+ "max-len": ["error", {
+ "code": 160,
+ "ignoreUrls": true,
+ "ignoreComments": true
+ }],
"import/order": ["error", {
"groups": ["builtin", "external", ["parent", "sibling"], "index", "object"],
"alphabetize": { "order": "asc", "caseInsensitive": true },
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 8dcbf1b5f..de2b582b1 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -1,5 +1,5 @@
+import defaultLog, { LogLevel } from '@apify/log';
import type { Log } from '@apify/log';
-import defaultLog from '@apify/log';
import { addTimeoutToPromise, tryCancel, TimeoutError } from '@apify/timeout';
import { cryptoRandomObjectId } from '@apify/utilities';
import type {
@@ -20,6 +20,7 @@ import type {
Session,
SessionPoolOptions,
Source,
+ StatisticState,
} from '@crawlee/core';
import {
mergeCookies,
@@ -89,6 +90,21 @@ export type RequestHandler<Context extends CrawlingContext = BasicCrawlingContex
export type ErrorHandler<Context extends CrawlingContext = BasicCrawlingContext> = (inputs: Context, error: Error) => Awaitable<void>;
+export interface StatusMessageCallbackParams<
+ Context extends CrawlingContext = BasicCrawlingContext,
+ Crawler extends BasicCrawler<any> = BasicCrawler<Context>,
+> {
+ state: StatisticState;
+ crawler: Crawler;
+ previousState: StatisticState;
+ message: string;
+}
+
+export type StatusMessageCallback<
+ Context extends CrawlingContext = BasicCrawlingContext,
+ Crawler extends BasicCrawler<any> = BasicCrawler<Context>,
+> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
+
export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCrawlingContext> {
/**
* User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
@@ -261,6 +277,24 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
*/
statusMessageLoggingInterval?: number;
+ /**
+ * Allows overriding the default status message. The callback needs to call `crawler.setStatusMessage()` explicitly.
+ * The default status message is provided in the parameters.
+ *
+ * ```ts
+ * const crawler = new CheerioCrawler({
+ * statusMessageCallback: async (ctx) => {
+ * return ctx.crawler.setStatusMessage(`this is status message from ${new Date().toISOString()}`, { level: 'INFO' }); // log level defaults to 'DEBUG'
+ * },
+ * statusMessageLoggingInterval: 1, // defaults to 10s
+ * async requestHandler({ $, enqueueLinks, request, log }) {
+ * // ...
+ * },
+ * });
+ * ```
+ */
+ statusMessageCallback?: StatusMessageCallback;
+
/**
* If set to `true`, the crawler will automatically try to bypass any detected bot protection.
*
@@ -391,6 +425,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
protected maxRequestRetries: number;
protected handledRequestsCount: number;
protected statusMessageLoggingInterval: number;
+ protected statusMessageCallback?: StatusMessageCallback;
protected sessionPoolOptions: SessionPoolOptions;
protected useSessionPool: boolean;
protected crawlingContexts = new Map<string, Context>();
@@ -420,7 +455,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
autoscaledPoolOptions: ow.optional.object,
sessionPoolOptions: ow.optional.object,
useSessionPool: ow.optional.boolean,
+
statusMessageLoggingInterval: ow.optional.number,
+ statusMessageCallback: ow.optional.function,
retryOnBlocked: ow.optional.boolean,
@@ -473,12 +510,14 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
failedRequestHandler,
statusMessageLoggingInterval = 10,
+ statusMessageCallback,
} = options;
this.requestList = requestList;
this.requestQueue = requestQueue;
this.log = log;
this.statusMessageLoggingInterval = statusMessageLoggingInterval;
+ this.statusMessageCallback = statusMessageCallback as StatusMessageCallback;
this.events = config.getEventManager();
this._handlePropertyNameChange({
@@ -545,8 +584,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
if (this.retryOnBlocked) {
this.sessionPoolOptions.blockedStatusCodes = sessionPoolOptions.blockedStatusCodes ?? [];
if (this.sessionPoolOptions.blockedStatusCodes.length !== 0) {
- log.warning(`Both 'blockedStatusCodes' and 'retryOnBlocked' are set.
-Please note that the 'retryOnBlocked' feature might not work as expected.`);
+ // eslint-disable-next-line max-len
+ log.warning(`Both 'blockedStatusCodes' and 'retryOnBlocked' are set. Please note that the 'retryOnBlocked' feature might not work as expected.`);
}
}
this.useSessionPool = useSessionPool;
@@ -620,8 +659,12 @@ Please note that the 'retryOnBlocked' feature might not work as expected.`);
throw new Error('the "isRequestBlocked" method is not implemented in this crawler.');
}
- private setStatusMessage(message: string, options: SetStatusMessageOptions = {}) {
- this.log.debug(`${options.isStatusMessageTerminal ? 'Terminal status message' : 'Status message'}: ${message}`);
+ /**
+ * This method is periodically called by the crawler, every `statusMessageLoggingInterval` seconds.
+ */
+ setStatusMessage(message: string, options: SetStatusMessageOptions = {}) {
+ const data = options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined;
+ this.log.internal(LogLevel[options.level as 'DEBUG' ?? 'DEBUG'], message, data);
const client = this.config.getStorageClient();
return client.setStatusMessage?.(message, options);
@@ -645,14 +688,21 @@ Please note that the 'retryOnBlocked' feature might not work as expected.`);
const log = async () => {
const operationMode = getOperationMode();
+ let message: string;
+
if (operationMode === 'ERROR') {
// eslint-disable-next-line max-len
- await this.setStatusMessage(`Experiencing problems, ${this.stats.state.requestsFailed - previousState.requestsFailed || this.stats.state.requestsFailed} failed requests in the past ${this.statusMessageLoggingInterval} seconds.`);
+ message = `Experiencing problems, ${this.stats.state.requestsFailed - previousState.requestsFailed || this.stats.state.requestsFailed} failed requests in the past ${this.statusMessageLoggingInterval} seconds.`;
} else {
const total = this.requestQueue?.assumedTotalCount || this.requestList?.length();
- // eslint-disable-next-line max-len
- await this.setStatusMessage(`Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.stats.state.requestsFailed} failed requests.`);
+ message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.stats.state.requestsFailed} failed requests.`;
+ }
+
+ if (this.statusMessageCallback) {
+ return this.statusMessageCallback({ crawler: this as any, state: this.stats.state, previousState, message });
}
+
+ await this.setStatusMessage(message);
};
const interval = setInterval(log, this.statusMessageLoggingInterval * 1e3);
@@ -696,7 +746,7 @@ Please note that the 'retryOnBlocked' feature might not work as expected.`);
await this._init();
await this.stats.startCapturing();
const periodicLogger = this.getPeriodicLogger();
- await this.setStatusMessage(`Initializing the crawler.`);
+ await this.setStatusMessage(`Initializing the crawler.`, { level: 'INFO' });
const sigintHandler = async () => {
this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
@@ -711,7 +761,6 @@ Please note that the 'retryOnBlocked' feature might not work as expected.`);
this.events.on(EventType.ABORTING, boundPauseOnMigration);
try {
- this.log.info('Starting the crawl');
await this.autoscaledPool!.run();
} finally {
await this.teardown();
@@ -729,7 +778,7 @@ Please note that the 'retryOnBlocked' feature might not work as expected.`);
retryHistogram: this.stats.requestRetryHistogram,
...finalStats,
};
- this.log.info('Crawl finished. Final request statistics:', stats);
+ this.log.info('Final request statistics:', stats);
if (this.stats.errorTracker.total !== 0) {
const prettify = ([count, info]: [number, string[]]) => `${count}x: ${info.at(-1)!.trim()} (${info[0]})`;
@@ -756,7 +805,7 @@ Please note that the 'retryOnBlocked' feature might not work as expected.`);
periodicLogger.stop();
// eslint-disable-next-line max-len
- await this.setStatusMessage(`Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${this.stats.state.requestsFinished} succeeded, ${this.stats.state.requestsFailed} failed.`, { isStatusMessageTerminal: true });
+ await this.setStatusMessage(`Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${this.stats.state.requestsFinished} succeeded, ${this.stats.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
this.running = false;
return stats;
diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.ts
index 19c29b2ae..1bfad46ed 100644
--- a/packages/types/src/storages.ts
+++ b/packages/types/src/storages.ts
@@ -1,5 +1,3 @@
-import type { LogLevel } from '@apify/log';
-
import type { AllowedHttpMethods, Dictionary } from './utility-types';
/**
@@ -309,7 +307,7 @@ export interface RequestQueueOptions {
export interface SetStatusMessageOptions {
isStatusMessageTerminal?: boolean;
- level?: LogLevel.DEBUG | LogLevel.INFO | LogLevel.WARNING | LogLevel.ERROR;
+ level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
}
/**
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 3d3b63a1f..7d078d997 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -19,9 +19,7 @@ import {
CriticalError,
MissingRouteError,
} from '@crawlee/basic';
-import {
- AutoscaledPool, RequestState,
-} from '@crawlee/core';
+import { RequestState } from '@crawlee/core';
import type { Dictionary } from '@crawlee/utils';
import { sleep } from '@crawlee/utils';
import express from 'express';
diff --git a/test/e2e/cheerio-default/actor/main.js b/test/e2e/cheerio-default/actor/main.js
index 7819ee19e..4144f7419 100644
--- a/test/e2e/cheerio-default/actor/main.js
+++ b/test/e2e/cheerio-default/actor/main.js
@@ -8,6 +8,10 @@ const mainOptions = {
await Actor.main(async () => {
const crawler = new CheerioCrawler({
+ statusMessageCallback: async (ctx) => {
+ return ctx.crawler.setStatusMessage(`this is status message from ${new Date().toISOString()}`, { level: 'INFO' });
+ },
+ statusMessageLoggingInterval: 1,
async requestHandler({ $, enqueueLinks, request, log }) {
const { url } = request;
await enqueueLinks({