Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: use correct config for storage classes to avoid memory leaks (#3144)
Edgars Beigarts911a2eb45cdb
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex f5b28a95b..9bfa903e4 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -976,7 +976,11 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextthis.shouldLogMaxProcessedRequestsExceeded = true;this.shouldLogMaxEnqueuedRequestsExceeded = true;- await purgeDefaultStorages({ onlyPurgeOnce: true });+ await purgeDefaultStorages({+ onlyPurgeOnce: true,+ client: this.config.getStorageClient(),+ config: this.config,+ });if (requests) {await this.addRequests(requests, addRequestsOptions);diff --git a/packages/core/src/storages/storage_manager.ts b/packages/core/src/storages/storage_manager.tsindex 78c51cb80..aea40468c 100644--- a/packages/core/src/storages/storage_manager.ts+++ b/packages/core/src/storages/storage_manager.ts@@ -81,13 +81,15 @@ export class StorageManager<T extends IStorage = IStorage> {if (!storage) {client ??= this.config.getStorageClient();const storageObject = await this._getOrCreateStorage(idOrName, this.name, client);- storage = new this.StorageConstructor({- id: storageObject.id,- name: storageObject.name,- storageObject,- client,- });-+ storage = new this.StorageConstructor(+ {+ id: storageObject.id,+ name: storageObject.name,+ storageObject,+ client,+ },+ this.config,+ );this._addStorageToCache(storage);}diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 24c1fe9ba..9c63293d3 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -1796,5 +1796,21 @@ describe('BasicCrawler', () => {expect(crawlerA.stats.state.requestsFinished).toBe(1);expect(crawlerB.stats.state.requestsFinished).toBe(1);});++ test('Crawlers with different Configurations does not use global Configuration', async () => {+ const getGlobalConfigSpy = vitest.spyOn(Configuration, 'getGlobalConfig');++ const configA = new Configuration({ persistStorage: false });+ const crawlerA = new BasicCrawler({ requestHandler: () => {} }, configA);+ const configB = new Configuration({ persistStorage: false });+ const crawlerB = new BasicCrawler({ requestHandler: () => {} }, configB);++ await crawlerA.run([{ url: `http://${HOSTNAME}:${port}` }]);+ await crawlerB.run([{ url: `http://${HOSTNAME}:${port}` }]);++ expect(getGlobalConfigSpy.mock.calls.length).toBe(0);+ expect(crawlerA.requestQueue?.config).toBe(configA);+ expect(crawlerB.requestQueue?.config).toBe(configB);+ });});});