Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: allow explicit `id` for Statistics persistence (#3879)

nikitachapovskii-dev45fa7e509c93
diff --git a/packages/core/src/crawlers/statistics.ts b/packages/core/src/crawlers/statistics.ts
index 975a65379..ea85d8f81 100644
--- a/packages/core/src/crawlers/statistics.ts
+++ b/packages/core/src/crawlers/statistics.ts
@@ -72,7 +72,7 @@ export class Statistics {
/**
* Statistic instance id.
*/
- readonly id = Statistics.id++; // assign an id while incrementing so it can be saved/restored from KV
+ readonly id: string;
/**
* Current statistic state used for doing calculations on {@apilink Statistics.calculate} calls
@@ -90,7 +90,7 @@ export class Statistics {
private readonly config: Configuration;
protected keyValueStore?: KeyValueStore = undefined;
- protected persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`;
+ protected persistStateKey: string;
private logIntervalMillis: number;
private logMessage: string;
private listener: () => Promise<void>;
@@ -115,6 +115,7 @@ export class Statistics {
config: ow.optional.object,
persistenceOptions: ow.optional.object,
saveErrorSnapshots: ow.optional.boolean,
+ id: ow.optional.any(ow.number, ow.string),
}),
);
@@ -127,8 +128,12 @@ export class Statistics {
enable: true,
},
saveErrorSnapshots = false,
+ id,
} = options;
+ this.id = id ?? String(Statistics.id++);
+ this.persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`;
+
this.log = (options.log ?? defaultLog).child({ prefix: 'Statistics' });
this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
@@ -474,6 +479,16 @@ export interface StatisticsOptions {
* @default false
*/
saveErrorSnapshots?: boolean;
+
+ /**
+ * A unique identifier for this statistics instance. This ID is used for persistence
+ * to the key value store, ensuring the same statistics can be loaded after script restarts.
+ *
+ * If not provided, an auto-incremented ID will be used for backward compatibility.
+ * This means statistics may not persist correctly across script restarts
+ * if crawler creation order changes.
+ */
+ id?: string;
}
/**
@@ -481,7 +496,7 @@ export interface StatisticsOptions {
*/
export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {
requestRetryHistogram: number[];
- statsId: number;
+ statsId: string;
requestAvgFailedDurationMillis: number;
requestAvgFinishedDurationMillis: number;
requestTotalDurationMillis: number;
diff --git a/test/core/crawlers/statistics.test.ts b/test/core/crawlers/statistics.test.ts
index ef9c54267..708073287 100644
--- a/test/core/crawlers/statistics.test.ts
+++ b/test/core/crawlers/statistics.test.ts
@@ -35,14 +35,14 @@ describe('Statistics', () => {
describe('persist state', () => {
// needs to go first for predictability
test('should increment id by each new consecutive instance', () => {
- expect(stats.id).toEqual(0);
+ expect(stats.id).toEqual('0');
// @ts-expect-error Accessing private prop
expect(Statistics.id).toEqual(1);
// @ts-expect-error Accessing private prop
expect(stats.persistStateKey).toEqual('SDK_CRAWLER_STATISTICS_0');
const [n1, n2] = [new Statistics(), new Statistics()];
- expect(n1.id).toEqual(1);
- expect(n2.id).toEqual(2);
+ expect(n1.id).toEqual('1');
+ expect(n2.id).toEqual('2');
// @ts-expect-error Accessing private prop
expect(Statistics.id).toEqual(3);
});
@@ -197,6 +197,24 @@ describe('Statistics', () => {
},
);
}, 2000);
+
+ test('an explicit id restores state across instances under a stable key', async () => {
+ const stats1 = new Statistics({ id: 'shared-stats' });
+ stats1.startJob(0);
+ vitest.advanceTimersByTime(100);
+ stats1.finishJob(0, 0);
+
+ await stats1.startCapturing();
+ await stats1.persistState();
+ await stats1.stopCapturing();
+
+ const stats2 = new Statistics({ id: 'shared-stats' });
+ await stats2.startCapturing();
+
+ expect(stats2.state.requestsFinished).toEqual(1);
+
+ await stats2.stopCapturing();
+ });
});
test('should finish a job', () => {