Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor!: Move status message handling from StorageClient to the event system (#3818)
Jan Buchard73291ee2bf1
diff --git a/REBASE-DEFERRED.md b/REBASE-DEFERRED.mdindex 78198a37d..8dacdc617 100644--- a/REBASE-DEFERRED.md+++ b/REBASE-DEFERRED.md@@ -469,3 +469,18 @@ checked directly (either by reading the native crate's source if vendored, orby a quick escape-attempt test against the new `FileSystemStorageClient`)before considering the `a04c29766` security fix's guarantees intactpost-rewrite.++**Commit `9f94f67e2` (Move status message handling from StorageClient to the+event system)**: `setStatusMessage()` becomes fully synchronous (emits+`EventType.STATUS_MESSAGE` directly instead of routing through an async+storage-client call). This makes master's `await Promise.race([this+.setStatusMessage(...), sleep(1)])` "flush the HTTP" workaround+(`7aed264f1 fix: make crawler terminal status message reliably delivered+(#3733)`) look pointless — there's no longer an in-flight HTTP call for a+tick to flush. Checked `v4-reverse`'s equivalent spot and it kept the+`Promise.race(...)` wrapper as-is rather than simplifying to a bare call, so+I matched that instead of "fixing" it myself. It's harmless either way (still+awaits one microtask via `Promise.resolve(undefined)` racing `sleep(1)`), but+worth a final look at whether it should be simplified now that the+underlying async-flush problem it solved no longer applies in this+architecture.diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.mdindex df4d2721e..6b0680b3b 100644--- a/docs/public-api/crawlee-basic.api.md+++ b/docs/public-api/crawlee-basic.api.md@@ -202,7 +202,7 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con// (undocumented)protected sameDomainDelayMillis: number;sessionPool: ISessionPool;- setStatusMessage(message: string, options?: SetStatusMessageOptions): Promise<void>;+ setStatusMessage(message: string, options?: SetStatusMessageOptions): void;readonly stats: Statistics;// (undocumented)protected statusMessageCallback?: StatusMessageCallback;diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.mdindex 843cbc2f2..903290b39 100644--- a/docs/public-api/crawlee-core.api.md+++ b/docs/public-api/crawlee-core.api.md@@ -620,6 +620,14 @@ export interface EventManagerOptions {persistStateIntervalMillis: number;}+// @public+export interface EventStatusMessageData {+ crawlerId: string;+ isStatusMessageTerminal?: boolean;+ level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';+ message: string;+}+// @public (undocumented)export const enum EventType {// (undocumented)@@ -631,11 +639,13 @@ export const enum EventType {// (undocumented)PERSIST_STATE = "persistState",// (undocumented)+ STATUS_MESSAGE = "statusMessage",+ // (undocumented)SYSTEM_INFO = "systemInfo"}// @public (undocumented)-export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit';+export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit' | 'statusMessage';// @publicexport type ExplicitStorageIdentifier = {@@ -905,8 +915,6 @@ export class MemoryStorageClient implements storage.StorageClient {// (undocumented)readonly requestQueueCache: RequestQueueClient_2[];// (undocumented)- setStatusMessage(message: string, options?: storage.SetStatusMessageOptions): Promise<void>;- // (undocumented)storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;teardown(): Promise<void>;}diff --git a/docs/public-api/crawlee-fs-storage.api.md b/docs/public-api/crawlee-fs-storage.api.mdindex 67745a529..321281eeb 100644--- a/docs/public-api/crawlee-fs-storage.api.md+++ b/docs/public-api/crawlee-fs-storage.api.md@@ -41,8 +41,6 @@ export class FileSystemStorageClient implements storage.StorageClient {// (undocumented)readonly requestQueuesDirectory: string;// (undocumented)- setStatusMessage(message: string, options?: storage.SetStatusMessageOptions): Promise<void>;- // (undocumented)storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;teardown(): Promise<void>;}diff --git a/docs/public-api/crawlee-types.api.md b/docs/public-api/crawlee-types.api.mdindex 45400a26f..0290166ab 100644--- a/docs/public-api/crawlee-types.api.md+++ b/docs/public-api/crawlee-types.api.md@@ -458,11 +458,9 @@ export interface SessionState {userData: object;}-// @public (undocumented)+// @publicexport interface SetStatusMessageOptions {- // (undocumented)isStatusMessageTerminal?: boolean;- // (undocumented)level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';}@@ -475,8 +473,6 @@ export interface StorageClient {// (undocumented)purge?(): Promise<void>;// (undocumented)- setStatusMessage?(message: string, options?: SetStatusMessageOptions): Promise<void>;- // (undocumented)stats?: {rateLimitErrors: number[];};diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 05a072305..408a105a2 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -11,6 +11,7 @@ import type {DatasetExportOptions,EnqueueLinksOptions,EventManager,+ EventStatusMessageData,FinalStatistics,GetUserDataFromRequest,IRequestLoader,@@ -1224,25 +1225,29 @@ export class BasicCrawler<}/**+ * Sets the status message for the current crawler run.+ ** This method is periodically called by the crawler, every `statusMessageLoggingInterval` seconds.+ *+ * The message is logged and broadcast via the {@apilink EventType.STATUS_MESSAGE|`statusMessage`}+ * event. Integrations such as the Apify SDK subscribe to that event and forward the message to+ * their status-reporting backend (e.g. the Apify platform).*/- async setStatusMessage(message: string, options: SetStatusMessageOptions = {}) {+ setStatusMessage(message: string, options: SetStatusMessageOptions = {}) {const data =options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined;this.log.logWithLevel(LogLevel[(options.level as 'DEBUG') ?? 'DEBUG'], message, data);- const client = serviceLocator.getStorageClient();-- if (!client.setStatusMessage) {- return;- }-- // just to be sure, this should be fast- await addTimeoutToPromise(- async () => client.setStatusMessage!(message, options),- 1000,- 'Setting status message timed out after 1s',- ).catch((e) => this.log.debug(e.message));+ // Broadcast the status message through the event system. Consumers (e.g. the Apify SDK) can+ // subscribe to `EventType.STATUS_MESSAGE` and propagate it to their status-reporting backend.+ // Setting the status message is not a storage concern, so we intentionally don't route it+ // through the storage client anymore.+ serviceLocator.getEventManager().emit(EventType.STATUS_MESSAGE, {+ crawlerId: this.crawlerId,+ message,+ isStatusMessageTerminal: options.isStatusMessageTerminal,+ level: options.level,+ } satisfies EventStatusMessageData);}private getPeriodicLogger() {@@ -1286,7 +1291,7 @@ export class BasicCrawler<return;}- await this.setStatusMessage(message);+ this.setStatusMessage(message);};const interval = setInterval(log, this.statusMessageLoggingInterval * 1e3);@@ -1350,8 +1355,7 @@ export class BasicCrawler<await this._init();await this.stats.startCapturing();const periodicLogger = this.getPeriodicLogger();- // Don't await, we don't want to block the execution- void this.setStatusMessage('Starting the crawler.', { level: 'INFO' });+ this.setStatusMessage('Starting the crawler.', { level: 'INFO' });const sigintHandler = async () => {this.log.warning(diff --git a/packages/core/src/events/event_manager.ts b/packages/core/src/events/event_manager.tsindex ffe6b14f5..b08814985 100644--- a/packages/core/src/events/event_manager.ts+++ b/packages/core/src/events/event_manager.ts@@ -16,9 +16,41 @@ export const enum EventType {MIGRATING = 'migrating',ABORTING = 'aborting',EXIT = 'exit',+ STATUS_MESSAGE = 'statusMessage',}-export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit';+export type EventTypeName =+ | EventType+ | 'systemInfo'+ | 'persistState'+ | 'migrating'+ | 'aborting'+ | 'exit'+ | 'statusMessage';++/**+ * Payload emitted with the {@apilink EventType.STATUS_MESSAGE|`statusMessage`} event.+ *+ * The crawler broadcasts these whenever it wants to report its progress (e.g. periodically, or on+ * start/finish). Consumers such as the Apify SDK can listen for the event and propagate the message+ * to the platform. This keeps the crawler decoupled from any specific status-reporting backend.+ */+export interface EventStatusMessageData {+ /**+ * Identifies the crawler that emitted the message.+ *+ * Either the user-provided `id` from the crawler options, or a randomly generated one.+ * Since a single event manager may be shared by multiple crawlers, consumers can use this+ * to attribute the message to a specific crawler instance.+ */+ crawlerId: string;+ /** The human-readable status message. */+ message: string;+ /** Whether this is the final status message of the run. */+ isStatusMessageTerminal?: boolean;+ /** The log level the message was logged with. */+ level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';+}interface Intervals {persistState?: BetterIntervalID;diff --git a/packages/core/src/memory-storage/memory-storage.ts b/packages/core/src/memory-storage/memory-storage.tsindex bd7694acf..4f39a0fb7 100644--- a/packages/core/src/memory-storage/memory-storage.ts+++ b/packages/core/src/memory-storage/memory-storage.ts@@ -2,7 +2,6 @@ import { randomUUID } from 'node:crypto';import type * as storage from '@crawlee/types';import type { CrawleeLogger } from '@crawlee/types';-import { s } from '@sapphire/shapeshift';import { DatasetClient } from './resource-clients/dataset.js';import { KeyValueStoreClient } from './resource-clients/key-value-store.js';@@ -149,15 +148,6 @@ export class MemoryStorageClient implements storage.StorageClient {return clients.some((store) => store.id === id);}- async setStatusMessage(message: string, options: storage.SetStatusMessageOptions = {}): Promise<void> {- s.string().parse(message);- s.object({- isStatusMessageTerminal: s.boolean().optional(),- }).parse(options);-- return Promise.resolve();- }-/*** Cleans up the default storages before the run starts. For the in-memory storage this simply* resets the in-memory state of the cached default dataset, key-value store and request queue.diff --git a/packages/fs-storage/src/file-system-storage.ts b/packages/fs-storage/src/file-system-storage.tsindex 217e9d918..88fadd719 100644--- a/packages/fs-storage/src/file-system-storage.ts+++ b/packages/fs-storage/src/file-system-storage.ts@@ -284,15 +284,6 @@ export class FileSystemStorageClient implements storage.StorageClient {}}- async setStatusMessage(message: string, options: storage.SetStatusMessageOptions = {}): Promise<void> {- s.string().parse(message);- s.object({- isStatusMessageTerminal: s.boolean().optional(),- }).parse(options);-- return Promise.resolve();- }-/*** Cleans up the default storages before the run starts:* - the default dataset;diff --git a/packages/fs-storage/src/resource-clients/request-queue.ts b/packages/fs-storage/src/resource-clients/request-queue.tsindex 494f7006a..fb8a86fe5 100644--- a/packages/fs-storage/src/resource-clients/request-queue.ts+++ b/packages/fs-storage/src/resource-clients/request-queue.ts@@ -6,6 +6,22 @@ import type { FileSystemRequestQueueClient as NativeFileSystemRequestQueueClientimport { CachedIdClient } from './cached-id-client.js';+/**+ * Convert a request (either a Crawlee `Request` instance or a plain schema object) into a plain object+ * whose properties are all enumerable.+ *+ * Crawlee's `Request` stores internal metadata (crawl depth, enqueue strategy, session id, ...) in a+ * *non-enumerable* `userData.__crawlee` bag. The native `@crawlee/fs-storage-native` client reads+ * request properties directly over the N-API boundary, which only exposes enumerable own properties+ * and does not honor `toJSON`. Passing a `Request` straight through would therefore silently drop the+ * `__crawlee` metadata, resetting `crawlDepth` to 0 on the next `fetchNextRequest` (breaking e.g.+ * `maxCrawlDepth` and enqueue-strategy handling). Round-tripping through JSON invokes the request's+ * `toJSON`, flattening everything into enumerable properties the native client can persist.+ */+function plainifyRequest(request: unknown): Record<string, unknown> {+ return JSON.parse(JSON.stringify(request)) as Record<string, unknown>;+}+const requestShape = s.object({id: s.string(),@@ -94,7 +110,7 @@ export class RequestQueueClient extends CachedIdClient implements storage.RequesrequestOptionsShape.parse(options);const response = await this.nativeClient.addBatchOfRequests(- requests as unknown as Record<string, unknown>[],+ requests.map((request) => plainifyRequest(request)),options.forefront ?? false,);@@ -121,9 +137,7 @@ export class RequestQueueClient extends CachedIdClient implements storage.Requesasync markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | undefined> {requestShape.parse(request);- return (- (await this.nativeClient.markRequestAsHandled(request as unknown as Record<string, unknown>)) ?? undefined- );+ return (await this.nativeClient.markRequestAsHandled(plainifyRequest(request))) ?? undefined;}async reclaimRequest(@@ -133,10 +147,7 @@ export class RequestQueueClient extends CachedIdClient implements storage.RequesrequestShape.parse(request);requestOptionsShape.parse(options);return (- (await this.nativeClient.reclaimRequest(- request as unknown as Record<string, unknown>,- options.forefront ?? false,- )) ?? undefined+ (await this.nativeClient.reclaimRequest(plainifyRequest(request), options.forefront ?? false)) ?? undefined);}diff --git a/packages/fs-storage/test/request-queue/adapter.test.ts b/packages/fs-storage/test/request-queue/adapter.test.tsindex d4e39b636..cea9641c0 100644--- a/packages/fs-storage/test/request-queue/adapter.test.ts+++ b/packages/fs-storage/test/request-queue/adapter.test.ts@@ -40,6 +40,37 @@ describe('RequestQueueClient adapter', () => {expect(typeof request!.id).toBe('string');});+ test('preserves non-enumerable `userData.__crawlee` metadata across the native round-trip', async () => {+ // Regression guard for a bug where `crawlDepth` (and other internal metadata living in the+ // non-enumerable `userData.__crawlee` bag) was silently dropped when a request was handed to the+ // native client, resetting `crawlDepth` to 0 on the next fetch and breaking `maxCrawlDepth` /+ // enqueue-strategy handling. The native client reads enumerable own properties over N-API and+ // does not honor `toJSON`, so the adapter must flatten the request before persisting it.+ const requestWithHiddenMetadata: Record<string, unknown> = {+ url: 'http://example.com/1',+ uniqueKey: '1',+ userData: {},+ };+ // Mirror how Crawlee's `Request` stores internal metadata: in a *non-enumerable* `__crawlee` bag+ // with a `toJSON` that surfaces it. A naive pass-through to the native client would lose this.+ Object.defineProperty(requestWithHiddenMetadata.userData, '__crawlee', {+ value: { crawlDepth: 3, enqueueStrategy: 'same-domain' },+ enumerable: false,+ });+ Object.defineProperty(requestWithHiddenMetadata.userData, 'toJSON', {+ value() {+ return { __crawlee: (this as any).__crawlee };+ },+ enumerable: false,+ });++ await requestQueue.addBatchOfRequests([requestWithHiddenMetadata as any]);++ const request = await requestQueue.fetchNextRequest();++ expect(request!.userData).toStrictEqual({ __crawlee: { crawlDepth: 3, enqueueStrategy: 'same-domain' } });+ });+test('getRequest looks up by uniqueKey', async () => {await requestQueue.addBatchOfRequests([{ url: 'http://example.com/1', uniqueKey: '1' }]);diff --git a/packages/types/src/index.ts b/packages/types/src/index.tsindex 65b2b5e70..e4af087a5 100644--- a/packages/types/src/index.ts+++ b/packages/types/src/index.ts@@ -1,3 +1,4 @@+export type * from './status-message.js';export type * from './storages.js';export type * from './utility-types.js';export type * from './browser.js';diff --git a/packages/types/src/status-message.ts b/packages/types/src/status-message.tsnew file mode 100644index 000000000..9a561f351--- /dev/null+++ b/packages/types/src/status-message.ts@@ -0,0 +1,13 @@+/**+ * Options for setting a crawler run's status message via {@apilink BasicCrawler.setStatusMessage}.+ *+ * Setting a status message is not a storage concern — the crawler broadcasts it through the event+ * system (`EventType.STATUS_MESSAGE`), and integrations such as the Apify SDK forward it to their+ * status-reporting backend.+ */+export interface SetStatusMessageOptions {+ /** Whether this is the final status message of the run. */+ isStatusMessageTerminal?: boolean;+ /** The log level to log the message with. Defaults to `'DEBUG'`. */+ level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';+}diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.tsindex ff98ffc1e..772504769 100644--- a/packages/types/src/storages.ts+++ b/packages/types/src/storages.ts@@ -388,11 +388,6 @@ export interface RequestQueueClient {setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;}-export interface SetStatusMessageOptions {- isStatusMessageTerminal?: boolean;- level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';-}-/*** Identifies a storage by its ID, name, or alias. At most one may be provided.*@@ -493,6 +488,5 @@ export interface StorageClient {getStorageClientCacheKey?(): string;purge?(): Promise<void>;teardown?(): Promise<void>;- setStatusMessage?(message: string, options?: SetStatusMessageOptions): Promise<void>;stats?: { rateLimitErrors: number[] };}diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 57c5b74ca..fc5212849 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -91,6 +91,48 @@ describe('BasicCrawler', () => {expect(process.listenerCount('SIGINT')).toBe(count - 1);});+ test('setStatusMessage emits a STATUS_MESSAGE event', async () => {+ const events = serviceLocator.getEventManager();+ const received: any[] = [];+ const listener = (data: any) => received.push(data);+ events.on(EventType.STATUS_MESSAGE, listener);++ try {+ const crawler = new BasicCrawler({+ id: 'my-crawler',+ requestHandler: () => {},+ });++ crawler.setStatusMessage('hello there', { level: 'INFO', isStatusMessageTerminal: true });++ expect(received).toEqual([+ { crawlerId: 'my-crawler', message: 'hello there', level: 'INFO', isStatusMessageTerminal: true },+ ]);+ } finally {+ events.off(EventType.STATUS_MESSAGE, listener);+ }+ });++ test('run() broadcasts start and terminal STATUS_MESSAGE events', async () => {+ const events = serviceLocator.getEventManager();+ const messages: string[] = [];+ const listener = (data: any) => messages.push(data.message);+ events.on(EventType.STATUS_MESSAGE, listener);++ try {+ const crawler = new BasicCrawler({+ requestHandler: () => {},+ });++ await crawler.run(['https://example.com']);++ expect(messages.some((m) => m === 'Starting the crawler.')).toBe(true);+ expect(messages.some((m) => m.startsWith('Finished!'))).toBe(true);+ } finally {+ events.off(EventType.STATUS_MESSAGE, listener);+ }+ });+test('should run in parallel thru all the requests', async () => {const sources = [...Array(500).keys()].map((index) => ({ url: `https://example.com/${index}` }));const sourcesCopy = JSON.parse(JSON.stringify(sources));diff --git a/test/core/enqueue_links/click_elements.test.ts b/test/core/enqueue_links/click_elements.test.tsindex 42c58e173..5ab39711b 100644--- a/test/core/enqueue_links/click_elements.test.ts+++ b/test/core/enqueue_links/click_elements.test.ts@@ -4,17 +4,18 @@ import type { RequestQueueOperationOptions, Source } from 'crawlee';import {launchPlaywright,launchPuppeteer,+ MemoryStorageClient,playwrightClickElements,playwrightUtils,puppeteerClickElements,puppeteerUtils,RequestQueue,+ serviceLocator,} from 'crawlee';import type { Browser as PWBrowser, Page as PWPage } from 'playwright';// @ts-ignore This only throws when compiled against puppeteer 25+ (ESM only), we only import types, so its alllll goooooodimport type { Browser as PPBrowser, Target } from 'puppeteer';import { runExampleComServer } from '../../shared/_helper.js';-import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator.js';function isPuppeteerBrowser(browser: PPBrowser | PWBrowser): browser is PPBrowser {return (browser as PPBrowser).targets !== undefined;@@ -58,7 +59,6 @@ const testCases = [testCases.forEach(({ caseName, launchBrowser, clickElements, utils }) => {describe(`${caseName}: enqueueLinksByClickingElements()`, () => {- const localStorageEmulator = new MemoryStorageEmulator();let browser: PPBrowser | PWBrowser;let server: Server;@@ -75,12 +75,11 @@ testCases.forEach(({ caseName, launchBrowser, clickElements, utils }) => {afterAll(async () => {await browser.close();server.close();- await localStorageEmulator.destroy();});beforeEach(async () => {- await localStorageEmulator.init();page = await browser.newPage();+ serviceLocator.setStorageClient(new MemoryStorageClient());});afterEach(async () => {diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.tsindex a90750eaf..5f233c0c1 100644--- a/test/core/enqueue_links/enqueue_links.test.ts+++ b/test/core/enqueue_links/enqueue_links.test.ts@@ -1,4 +1,9 @@-import { type AddRequestsBatchedOptions, cheerioCrawlerEnqueueLinks } from '@crawlee/cheerio';+import {+ type AddRequestsBatchedOptions,+ cheerioCrawlerEnqueueLinks,+ MemoryStorageClient,+ serviceLocator,+} from '@crawlee/cheerio';import { enqueueLinks } from '@crawlee/core';import { launchPlaywright } from '@crawlee/playwright';import type { RequestQueueOperationOptions, Source } from '@crawlee/puppeteer';@@ -17,8 +22,6 @@ import type { Browser as PuppeteerBrowser, Page as PuppeteerPage } from 'puppeteimport log from '@apify/log';-import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator.js';-const HTML = `<html><head>@@ -64,7 +67,6 @@ async function createRequestQueueMock() {}describe('enqueueLinks()', () => {- const localStorageEmulator = new MemoryStorageEmulator();let ll: number;beforeAll(() => {ll = log.getLevel();@@ -72,12 +74,11 @@ describe('enqueueLinks()', () => {});beforeEach(async () => {- await localStorageEmulator.init();+ serviceLocator.setStorageClient(new MemoryStorageClient());});afterAll(async () => {log.setLevel(ll);- await localStorageEmulator.destroy();});describe.each([[launchPuppeteer], [launchPlaywright]] as const)('using %s', (method) => {