Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: use crawler id for `Statistics` and non-default `RequestQueue`s (#3834)
Jindřich Bärbf20a152d7f4
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.mdindex 5b97c00e4..f8040f324 100644--- a/docs/public-api/crawlee-basic.api.md+++ b/docs/public-api/crawlee-basic.api.md@@ -58,7 +58,7 @@ import type { StorageIdentifier } from '@crawlee/core';import { StringPredicate } from 'ow';import { TimeoutError } from '@apify/timeout';-// @public+// @public (undocumented)export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<CrawlingContext, Context>);// (undocumented)@@ -119,6 +119,8 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con// (undocumented)protected httpClient: BaseHttpClient;// (undocumented)+ protected readonly identity: CrawlerIdentity;+ // (undocumented)protected ignoreHttpErrorStatusCodes: Set<number>;protected _init(): Promise<void>;// (undocumented)diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 67f47e292..d7fa481b0 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -542,6 +542,22 @@ export interface BasicCrawlerOptions<* ```* @category Crawlers*/++/**+ * Identifies a crawler instance for storage aliasing, `useState()` and status-message events.+ */+interface CrawlerIdentity {+ /**+ * 0-based instantiation order across all crawlers in the process.+ * Note that the value can be subject to race conditions between different script invocations.+ */+ readonly instanceIndex: number;+ /** The user-supplied `id` option, or a fallback derived from `instanceIndex`. */+ readonly id: string;+ /** Whether `id` came from the user (as opposed to being derived from `instanceIndex`). */+ readonly hasExplicitId: boolean;+}+export class BasicCrawler<Context extends CrawlingContext = CrawlingContext,ContextExtension = Dictionary<never>,@@ -549,12 +565,6 @@ export class BasicCrawler<> {protected static readonly CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';- /**- * Tracks crawler instances that accessed shared state without having an explicit id.- * Used to detect and warn about multiple crawlers sharing the same state.- */- private static useStateCrawlerIds = new Set<string>();-/*** Tracks the number of crawler instances created. The first crawler uses the default* request queue; subsequent ones get their own queue via a unique alias so they don't@@ -562,6 +572,12 @@ export class BasicCrawler<*/private static instanceCount = 0;+ /**+ * Tracks crawler instances that accessed shared state without having an explicit id.+ * Used to detect and warn about multiple crawlers sharing the same state.+ */+ private static useStateAnonymousIndices = new Set<number>();+/*** A reference to the underlying {@apilink Statistics} class that collects and logs run statistics for requests.*/@@ -697,9 +713,7 @@ export class BasicCrawler<private _ownsEventManager = false;private loggedPerRun = new Set<string>();private readonly robotsTxtFileCache: LruCache<RobotsTxtFile>;- private readonly crawlerId: string;- private readonly hasExplicitId: boolean;- private readonly crawlerInstanceIndex: number;+ protected readonly identity: CrawlerIdentity;private readonly contextPipelineOptions: {contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;extendContext?: (context: Context) => Awaitable<ContextExtension>;@@ -837,11 +851,8 @@ export class BasicCrawler<// Initialize the Configuration instance to avoid lazy loading in the componentsserviceLocator.getConfiguration();- // Store whether the user explicitly provided an ID- this.hasExplicitId = id !== undefined;- // Store the user-provided ID, or generate a unique one for tracking purposes (not for state key)- this.crawlerId = id ?? cryptoRandomObjectId();- this.crawlerInstanceIndex = BasicCrawler.instanceCount++;+ const instanceIndex = BasicCrawler.instanceCount++;+ this.identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };if (requestManager !== undefined) {if (requestList !== undefined || requestQueue !== undefined) {@@ -899,7 +910,7 @@ export class BasicCrawler<this.stats = new Statistics({logMessage: `${this.constructor.name} request statistics:`,log: this.log,- ...(this.hasExplicitId ? { id: this.crawlerId } : {}),+ id: this.identity.id,...statisticsOptions,});@@ -1267,7 +1278,7 @@ export class BasicCrawler<// Setting the status message is not a storage concern, so we intentionally don't route it// through the storage client anymore.serviceLocator.getEventManager().emit(EventType.STATUS_MESSAGE, {- crawlerId: this.crawlerId,+ crawlerId: this.identity.id,message,isStatusMessageTerminal: options.isStatusMessageTerminal,level: options.level,@@ -1507,8 +1518,7 @@ export class BasicCrawler<private async openOwnedRequestQueue(): Promise<IRequestManager> {// The first crawler instance uses the default queue (null identifier);// subsequent instances get their own queue via a unique alias so they don't collide.- const identifier =- this.crawlerInstanceIndex === 0 ? null : { alias: `__default_${this.crawlerInstanceIndex}__` };+ const identifier = this.identity.instanceIndex === 0 ? null : { alias: `__default_${this.identity.id}__` };const requestQueue = await RequestQueue.open(identifier, { config: serviceLocator.getConfiguration() });this.ownedRequestManager = requestQueue;@@ -1563,14 +1573,14 @@ export class BasicCrawler<async useState<State extends Dictionary = Dictionary>(defaultValue = {} as State): Promise<State> {const kvs = await KeyValueStore.open(null, { config: serviceLocator.getConfiguration() });- if (this.hasExplicitId) {- const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.crawlerId}`;+ if (this.identity.hasExplicitId) {+ const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.identity.id}`;return kvs.getAutoSavedValue<State>(stateKey, defaultValue);}- BasicCrawler.useStateCrawlerIds.add(this.crawlerId);+ BasicCrawler.useStateAnonymousIndices.add(this.identity.instanceIndex);- if (BasicCrawler.useStateCrawlerIds.size > 1) {+ if (BasicCrawler.useStateAnonymousIndices.size > 1) {serviceLocator.getLogger().warningOnce(