Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: use correct config for storage classes to avoid memory leaks (#3144)

Edgars Beigarts911a2eb45cdb
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index f5b28a95b..9bfa903e4 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -976,7 +976,11 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
this.shouldLogMaxProcessedRequestsExceeded = true;
this.shouldLogMaxEnqueuedRequestsExceeded = true;
- await purgeDefaultStorages({ onlyPurgeOnce: true });
+ await purgeDefaultStorages({
+ onlyPurgeOnce: true,
+ client: this.config.getStorageClient(),
+ config: this.config,
+ });
if (requests) {
await this.addRequests(requests, addRequestsOptions);
diff --git a/packages/core/src/storages/storage_manager.ts b/packages/core/src/storages/storage_manager.ts
index 78c51cb80..aea40468c 100644
--- a/packages/core/src/storages/storage_manager.ts
+++ b/packages/core/src/storages/storage_manager.ts
@@ -81,13 +81,15 @@ export class StorageManager<T extends IStorage = IStorage> {
if (!storage) {
client ??= this.config.getStorageClient();
const storageObject = await this._getOrCreateStorage(idOrName, this.name, client);
- storage = new this.StorageConstructor({
- id: storageObject.id,
- name: storageObject.name,
- storageObject,
- client,
- });
-
+ storage = new this.StorageConstructor(
+ {
+ id: storageObject.id,
+ name: storageObject.name,
+ storageObject,
+ client,
+ },
+ this.config,
+ );
this._addStorageToCache(storage);
}
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 24c1fe9ba..9c63293d3 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -1796,5 +1796,21 @@ describe('BasicCrawler', () => {
expect(crawlerA.stats.state.requestsFinished).toBe(1);
expect(crawlerB.stats.state.requestsFinished).toBe(1);
});
+
+ test('Crawlers with different Configurations does not use global Configuration', async () => {
+ const getGlobalConfigSpy = vitest.spyOn(Configuration, 'getGlobalConfig');
+
+ const configA = new Configuration({ persistStorage: false });
+ const crawlerA = new BasicCrawler({ requestHandler: () => {} }, configA);
+ const configB = new Configuration({ persistStorage: false });
+ const crawlerB = new BasicCrawler({ requestHandler: () => {} }, configB);
+
+ await crawlerA.run([{ url: `http://${HOSTNAME}:${port}` }]);
+ await crawlerB.run([{ url: `http://${HOSTNAME}:${port}` }]);
+
+ expect(getGlobalConfigSpy.mock.calls.length).toBe(0);
+ expect(crawlerA.requestQueue?.config).toBe(configA);
+ expect(crawlerB.requestQueue?.config).toBe(configB);
+ });
});
});