Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: crawler instances with different StorageClients do not affect each other (#2056)

Jindřich Bär3f4c86352bdb
diff --git a/packages/core/src/configuration.ts b/packages/core/src/configuration.ts
index 182483715..8493cbd14 100644
--- a/packages/core/src/configuration.ts
+++ b/packages/core/src/configuration.ts
@@ -8,9 +8,9 @@ import type { MemoryStorageOptions } from '@crawlee/memory-storage';
import type { Dictionary, StorageClient } from '@crawlee/types';
import { pathExistsSync, readFileSync } from 'fs-extra';
-import type { EventManager } from './events';
-import { LocalEventManager } from './events';
-import { entries } from './typedefs';
+import { LocalEventManager, type EventManager } from './events';
+import type { StorageManager } from './storages';
+import { entries, type Constructor } from './typedefs';
export interface ConfigurationOptions {
/**
@@ -280,6 +280,8 @@ export class Configuration {
/** @internal */
static globalConfig?: Configuration;
+ public readonly storageManagers = new Map<Constructor, StorageManager>();
+
/**
* Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
*/
diff --git a/packages/core/src/storages/storage_manager.ts b/packages/core/src/storages/storage_manager.ts
index 42fae1ca1..dd2cc533a 100644
--- a/packages/core/src/storages/storage_manager.ts
+++ b/packages/core/src/storages/storage_manager.ts
@@ -21,7 +21,6 @@ export interface IStorage {
* @ignore
*/
export class StorageManager<T extends IStorage = IStorage> {
- private static readonly storageManagers = new Map<Constructor, StorageManager>();
private readonly name: 'Dataset' | 'KeyValueStore' | 'RequestQueue';
private readonly StorageConstructor: Constructor<T> & { name: string };
private readonly cache = new Map<string, T>();
@@ -48,24 +47,24 @@ export class StorageManager<T extends IStorage = IStorage> {
storageClass: Constructor<T>,
config = Configuration.getGlobalConfig(),
): StorageManager<T> {
- if (!this.storageManagers.has(storageClass)) {
+ if (!config.storageManagers.has(storageClass)) {
const manager = new StorageManager(storageClass, config);
- this.storageManagers.set(storageClass, manager);
+ config.storageManagers.set(storageClass, manager);
}
- return this.storageManagers.get(storageClass) as StorageManager<T>;
+ return config.storageManagers.get(storageClass) as StorageManager<T>;
}
/** @internal */
- static clearCache(): void {
- this.storageManagers.forEach((manager) => {
+ static clearCache(config = Configuration.getGlobalConfig()): void {
+ config.storageManagers.forEach((manager) => {
if (manager.name === 'KeyValueStore') {
manager.cache.forEach((item) => {
(item as Dictionary).clearCache?.();
});
}
});
- this.storageManagers.clear();
+ config.storageManagers.clear();
}
async openStorage(idOrName?: string | null, client?: StorageClient): Promise<T> {
diff --git a/test/core/multiple_crawlers.test.ts b/test/core/multiple_crawlers.test.ts
new file mode 100644
index 000000000..cd40f7cb7
--- /dev/null
+++ b/test/core/multiple_crawlers.test.ts
@@ -0,0 +1,31 @@
+import { MemoryStorage } from '@crawlee/memory-storage';
+import { CheerioCrawler, Configuration } from 'crawlee';
+
+describe('multiple crawlers', () => {
+ test('Crawler instances with different StorageClients do not affect each other', async () => {
+ const getCrawler = () => {
+ return new CheerioCrawler({
+ requestHandler: async () => {},
+ }, new Configuration({
+ storageClient: new MemoryStorage({
+ persistStorage: false,
+ }),
+ }));
+ };
+
+ const a = getCrawler();
+
+ await a.run([
+ { url: 'https://example.org/' },
+ ]);
+
+ const b = getCrawler();
+
+ await b.run([
+ { url: 'https://example.org/' },
+ ]);
+
+ expect(a.stats.state.requestsFinished).toBe(1);
+ expect(b.stats.state.requestsFinished).toBe(1);
+ });
+});