Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(core): remove `RequestList` persist state listener on `teardown` (#3846)
Anas Khan63394ee4bca9
diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.tsindex 2c0736f9b..a7037737e 100644--- a/packages/core/src/storages/request_list.ts+++ b/packages/core/src/storages/request_list.ts@@ -403,6 +403,8 @@ export class RequestList implements IRequestList {// The proxy configuration used for `requestsFromUrl` requests.this.proxyConfiguration = proxyConfiguration;++ this.persistState = this.persistState.bind(this);}/**@@ -431,7 +433,7 @@ export class RequestList implements IRequestList {this.isInitialized = true;if (this.persistRequestsKey && !this.areRequestsPersisted) await this._persistRequests();if (this.persistStateKey) {- this.events.on(EventType.PERSIST_STATE, this.persistState.bind(this));+ this.events.on(EventType.PERSIST_STATE, this.persistState);}return this;@@ -519,6 +521,19 @@ export class RequestList implements IRequestList {}}+ /**+ * Removes the `PERSIST_STATE` event listener registered during initialization and persists+ * the current state one last time. Call this when you are done with the `RequestList` to avoid+ * leaking the listener (and the requests it retains) on the shared event manager.+ */+ async teardown(): Promise<void> {+ this.events.off(EventType.PERSIST_STATE, this.persistState);++ if (this.persistStateKey) {+ await this.persistState();+ }+ }+/*** Unlike persistState(), this is used only internally, since the sources* are automatically persisted at RequestList initialization (if the persistRequestsKey is set),diff --git a/test/core/request_list.test.ts b/test/core/request_list.test.tsindex ebafe3ffa..a2a89794c 100644--- a/test/core/request_list.test.ts+++ b/test/core/request_list.test.ts@@ -505,6 +505,21 @@ describe('RequestList', () => {expect(requestList2.getState()).toEqual(requestList.getState());});+ test('teardown removes the persist state listener when persistStateKey is set', async () => {+ const listenerCountBefore = events.listenerCount(EventType.PERSIST_STATE);++ const requestList = await RequestList.open({+ sources: [{ url: 'https://example.com/1' }],+ persistStateKey: 'teardown-key',+ });++ expect(events.listenerCount(EventType.PERSIST_STATE)).toBe(listenerCountBefore + 1);++ await requestList.teardown();++ expect(events.listenerCount(EventType.PERSIST_STATE)).toBe(listenerCountBefore);+ });+test('should correctly persist its sources when persistRequestsKey is set', async () => {const PERSIST_REQUESTS_KEY = 'some-key';const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue');