Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: avoid duplicate final crawler persistence (#3866)

Akhil Trivedi79dd6c41eff1
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 2dbc47698..6663e784d 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -575,7 +575,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
protected retryOnBlocked: boolean;
protected respectRobotsTxtFile: boolean | { userAgent?: string };
protected onSkippedRequest?: SkippedRequestCallback;
- private _closeEvents?: boolean;
+ private _ownsEventManager = false;
private loggedPerRun = new Set<string>();
private experiments: CrawlerExperiments;
private readonly robotsTxtFileCache: LruCache<RobotsTxtFile>;
@@ -1042,8 +1042,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
try {
await this.autoscaledPool!.run();
} finally {
- await this.teardown();
await this.stats.stopCapturing();
+ await this.teardown();
process.off('SIGINT', sigintHandler);
this.events.off(EventType.MIGRATING, boundPauseOnMigration);
@@ -1406,7 +1406,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
protected async _init(): Promise<void> {
if (!this.events.isInitialized()) {
await this.events.init();
- this._closeEvents = true;
+ this._ownsEventManager = true;
}
this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions, this.config);
@@ -2031,13 +2031,18 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
* To stop the crawler gracefully (waiting for all running requests to finish), use {@apilink BasicCrawler.stop|`crawler.stop()`} instead.
*/
async teardown(): Promise<void> {
- this.events.emit(EventType.PERSIST_STATE, { isMigrating: false });
+ // When this crawler initialized the event manager, its close() call emits
+ // the final persistence event after the crawler-specific state has been
+ // saved. External event managers still need an explicit event here.
+ if (!this._ownsEventManager) {
+ this.events.emit(EventType.PERSIST_STATE, { isMigrating: false });
+ }
if (this.useSessionPool) {
- await this.sessionPool!.teardown();
+ await this.sessionPool!.teardown({ persistState: this._ownsEventManager });
}
- if (this._closeEvents) {
+ if (this._ownsEventManager) {
await this.events.close();
}
diff --git a/packages/core/src/session_pool/session_pool.ts b/packages/core/src/session_pool/session_pool.ts
index 22528a29e..1a57ed3c7 100644
--- a/packages/core/src/session_pool/session_pool.ts
+++ b/packages/core/src/session_pool/session_pool.ts
@@ -384,10 +384,13 @@ export class SessionPool extends EventEmitter {
/**
* Removes listener from `persistState` event.
* This function should be called after you are done with using the `SessionPool` instance.
+ * @param options - Set `persistState` to false when the final state was already persisted by the event manager.
*/
- async teardown(): Promise<void> {
+ async teardown({ persistState = true }: { persistState?: boolean } = {}): Promise<void> {
this.events.off(EventType.PERSIST_STATE, this._listener);
- await this.persistState();
+ if (persistState) {
+ await this.persistState();
+ }
}
/**
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 72c572560..9844470e2 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -26,7 +26,7 @@ import {
RequestValidationError,
Router,
} from '@crawlee/basic';
-import { RequestState } from '@crawlee/core';
+import { RequestState, SessionPool, Statistics } from '@crawlee/core';
import type { Dictionary } from '@crawlee/utils';
import { RobotsTxtFile, sleep } from '@crawlee/utils';
import express from 'express';
@@ -1337,6 +1337,52 @@ describe('BasicCrawler', () => {
});
describe('Uses SessionPool', () => {
+ it('persists statistics and the session pool once when finishing', async () => {
+ const persistStatistics = vitest.spyOn(Statistics.prototype, 'persistState');
+ const persistSessionPool = vitest.spyOn(SessionPool.prototype, 'persistState');
+ const config = new Configuration();
+ const crawler = new BasicCrawler(
+ {
+ useSessionPool: true,
+ requestHandler: async () => {
+ persistStatistics.mockClear();
+ persistSessionPool.mockClear();
+ },
+ },
+ config,
+ );
+
+ await crawler.run(['https://example.com']);
+
+ expect(persistStatistics).toHaveBeenCalledTimes(1);
+ expect(persistSessionPool).toHaveBeenCalledTimes(1);
+ });
+
+ it('persists the session pool once with a shared event manager', async () => {
+ const persistStatistics = vitest.spyOn(Statistics.prototype, 'persistState');
+ const persistSessionPool = vitest.spyOn(SessionPool.prototype, 'persistState');
+ const config = new Configuration();
+ await config.getEventManager().init();
+ const crawler = new BasicCrawler(
+ {
+ useSessionPool: true,
+ requestHandler: async () => {
+ persistStatistics.mockClear();
+ persistSessionPool.mockClear();
+ },
+ },
+ config,
+ );
+
+ await crawler.run(['https://example.com']);
+ expect(config.getEventManager().listenerCount(EventType.PERSIST_STATE)).toBe(0);
+ expect(persistSessionPool).toHaveBeenCalledTimes(1);
+ await config.getEventManager().close();
+
+ expect(persistStatistics).toHaveBeenCalledTimes(1);
+ expect(persistSessionPool).toHaveBeenCalledTimes(1);
+ });
+
it('should use SessionPool when useSessionPool is true ', async () => {
const url = 'https://example.com';
const requestList = await RequestList.open({ sources: [{ url }] });