Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: log warning on sharing `useState()`, add `id` option to `BasicCrawler` (#3309)
Jindřich Bär58eb9fa29bbf
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 647bf09d6..66fa40477 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -396,6 +396,19 @@ export interface BasicCrawlerOptions<* the Proxy URLs provided and rotated according to the configuration.*/proxyConfiguration?: ProxyConfiguration;++ /**+ * A unique identifier for the crawler instance. This ID is used to isolate the state returned by+ * {@apilink BasicCrawler.useState|`crawler.useState()`} from other crawler instances.+ *+ * When multiple crawler instances use `useState()` without an explicit `id`, they will share the same+ * state object for backward compatibility. A warning will be logged in this case.+ *+ * To ensure each crawler has its own isolated state that also persists across script restarts+ * (e.g., during Apify migrations), provide a stable, unique ID for each crawler instance.+ *+ */+ id?: string;}/**@@ -485,6 +498,12 @@ export class BasicCrawler<> {protected static readonly CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';+ /**+ * Tracks crawler instances that accessed shared state without having an explicit id.+ * Used to detect and warn about multiple crawlers sharing the same state.+ */+ private static useStateCrawlerIds = new Set<string>();+/*** A reference to the underlying {@apilink Statistics} class that collects and logs run statistics for requests.*/@@ -590,6 +609,8 @@ export class BasicCrawler<private experiments: CrawlerExperiments;private readonly robotsTxtFileCache: LruCache<RobotsTxtFile>;private _experimentWarnings: Partial<Record<keyof CrawlerExperiments, boolean>> = {};+ private readonly crawlerId: string;+ private readonly hasExplicitId: boolean;protected static optionsShape = {contextPipelineBuilder: ow.optional.object,@@ -633,6 +654,8 @@ export class BasicCrawler<experiments: ow.optional.object,statisticsOptions: ow.optional.object,++ id: ow.optional.string,};/**@@ -680,8 +703,15 @@ export class BasicCrawler<// internallog = defaultLog.child({ prefix: this.constructor.name }),experiments = {},++ id,} = options;+ // Store whether the user explicitly provided an ID+ this.hasExplicitId = id !== undefined;+ // Store the user-provided ID, or generate a unique one for tracking purposes (not for state key)+ this.crawlerId = id ?? cryptoRandomObjectId();+// Store the builder so that it can be run when the contextPipeline is needed.// Invoking it immediately would cause problems with parent constructor call order.this.contextPipelineBuilder = () => {@@ -775,6 +805,7 @@ export class BasicCrawler<logMessage: `${log.getOptions().prefix} request statistics:`,log,config,+ ...(this.hasExplicitId ? { id: this.crawlerId } : {}),...statisticsOptions,});this.sessionPoolOptions = {@@ -1177,6 +1208,23 @@ export class BasicCrawler<async useState<State extends Dictionary = Dictionary>(defaultValue = {} as State): Promise<State> {const kvs = await KeyValueStore.open(null, { config: this.config });++ if (this.hasExplicitId) {+ const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.crawlerId}`;+ return kvs.getAutoSavedValue<State>(stateKey, defaultValue);+ }++ BasicCrawler.useStateCrawlerIds.add(this.crawlerId);++ if (BasicCrawler.useStateCrawlerIds.size > 1) {+ defaultLog.warningOnce(+ 'Multiple crawler instances are calling useState() without an explicit `id` option. \n' ++ 'This means they will share the same state object, which is likely unintended. \n' ++ 'To fix this, provide a unique `id` option to each crawler instance. \n' ++ 'Example: new BasicCrawler({ id: "my-crawler-1", ... })',+ );+ }+return kvs.getAutoSavedValue<State>(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue);}diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 6a922a1c3..6cc22096d 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -33,7 +33,7 @@ import express from 'express';import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js';import type { SetRequired } from 'type-fest';import type { Mock } from 'vitest';-import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'vitest';+import { afterAll, beforeAll, beforeEach, describe, expect, test, vitest } from 'vitest';import { z } from 'zod';import log from '@apify/log';@@ -441,6 +441,61 @@ describe('BasicCrawler', () => {expect(await requestList.isEmpty()).toBe(true);});+ test('print a warning on sharing state between two crawlers', async () => {+ function createCrawler() {+ return new BasicCrawler({+ requestHandler: async ({ request, useState }) => {+ const state = await useState<{ urls: string[] }>({ urls: [] });+ state.urls.push(request.url);+ },+ });+ }++ const loggerSpy = vitest.spyOn(log, 'warning');++ const [crawler1, crawler2] = [createCrawler(), createCrawler()];++ await crawler1.run([`http://${HOSTNAME}:${port}/`]);+ await crawler2.run([`http://${HOSTNAME}:${port}/?page=2`]);++ // Both crawlers should share the same state (backward compatibility)+ const state1 = await crawler1.useState<{ urls: string[] }>();+ const state2 = await crawler2.useState<{ urls: string[] }>();++ expect(state1).toBe(state2);+ expect(state1.urls).toHaveLength(2);+ expect(state1.urls).toContain(`http://${HOSTNAME}:${port}/`);+ expect(state1.urls).toContain(`http://${HOSTNAME}:${port}/?page=2`);+ expect(loggerSpy).toBeCalledWith(expect.stringContaining('Multiple crawler instances are calling useState()'));+ });++ test('crawlers with explicit id have isolated state', async () => {+ function createCrawler(id: string) {+ return new BasicCrawler({+ id,+ requestHandler: async ({ request, useState }) => {+ const state = await useState<{ urls: string[] }>({ urls: [] });+ state.urls.push(request.url);+ },+ });+ }++ const [crawler1, crawler2] = [createCrawler('crawler-1'), createCrawler('crawler-2')];++ await crawler1.run([`http://${HOSTNAME}:${port}/`]);+ await crawler2.run([`http://${HOSTNAME}:${port}/?page=2`]);++ // Each crawler should have its own isolated state+ const state1 = await crawler1.useState<{ urls: string[] }>();+ const state2 = await crawler2.useState<{ urls: string[] }>();++ expect(state1).not.toBe(state2);+ expect(state1.urls).toHaveLength(1);+ expect(state1.urls).toContain(`http://${HOSTNAME}:${port}/`);+ expect(state2.urls).toHaveLength(1);+ expect(state2.urls).toContain(`http://${HOSTNAME}:${port}/?page=2`);+ });+test.each([EventType.MIGRATING, EventType.ABORTING])('should pause on %s event and persist RequestList state',async (event) => {diff --git a/test/core/crawlers/statistics.test.ts b/test/core/crawlers/statistics.test.tsindex 5cbbb565b..9de4ed857 100644--- a/test/core/crawlers/statistics.test.ts+++ b/test/core/crawlers/statistics.test.ts@@ -356,4 +356,42 @@ describe('Statistics', () => {expect(stats.state.requestsFinished).toEqual(0);expect(stats.requestRetryHistogram).toEqual([]);});++ describe('explicit id option', () => {+ test('statistics with same explicit id should share persisted state', async () => {+ const stats1 = new Statistics({ id: 'shared-stats' });+ stats1.startJob(0);+ vitest.advanceTimersByTime(100);+ stats1.finishJob(0, 0);++ await stats1.startCapturing();+ await stats1.persistState();+ await stats1.stopCapturing();++ const stats2 = new Statistics({ id: 'shared-stats' });+ await stats2.startCapturing();++ expect(stats2.state.requestsFinished).toEqual(1);++ await stats2.stopCapturing();+ });++ test('statistics with different explicit ids should have isolated state', async () => {+ const statsA = new Statistics({ id: 'stats-a' });+ statsA.startJob(0);+ vitest.advanceTimersByTime(100);+ statsA.finishJob(0, 0);++ await statsA.startCapturing();+ await statsA.persistState();+ await statsA.stopCapturing();++ const statsB = new Statistics({ id: 'stats-b' });+ await statsB.startCapturing();++ expect(statsB.state.requestsFinished).toEqual(0);++ await statsB.stopCapturing();+ });+ });});