Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(Storage): queue up opening storages to prevent issues in concurrent calls (#1865)

Vlad Frangu044c740101dd
diff --git a/packages/core/package.json b/packages/core/package.json
index 18478facb..78b3ae990 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -62,6 +62,7 @@
"@crawlee/memory-storage": "^3.3.0",
"@crawlee/types": "^3.3.0",
"@crawlee/utils": "^3.3.0",
+ "@sapphire/async-queue": "^1.5.0",
"@types/tough-cookie": "^4.0.2",
"@vladfrangu/async_event_emitter": "^2.0.0",
"csv-stringify": "^6.2.0",
diff --git a/packages/core/src/storages/storage_manager.ts b/packages/core/src/storages/storage_manager.ts
index 8b4a02c9d..d220a4111 100644
--- a/packages/core/src/storages/storage_manager.ts
+++ b/packages/core/src/storages/storage_manager.ts
@@ -1,4 +1,5 @@
import type { Dictionary, StorageClient } from '@crawlee/types';
+import { AsyncQueue } from '@sapphire/async-queue';
import { Configuration } from '../configuration';
import type { Constructor } from '../typedefs';
@@ -22,6 +23,7 @@ export class StorageManager<T extends IStorage = IStorage> {
private readonly name: 'Dataset' | 'KeyValueStore' | 'RequestQueue';
private readonly StorageConstructor: Constructor<T> & { name: string };
private readonly cache = new Map<string, T>();
+ private readonly storageOpenQueue = new AsyncQueue();
constructor(
StorageConstructor: Constructor<T>,
@@ -65,6 +67,8 @@ export class StorageManager<T extends IStorage = IStorage> {
}
async openStorage(idOrName?: string | null, client?: StorageClient): Promise<T> {
+ await this.storageOpenQueue.wait();
+
if (!idOrName) {
const defaultIdConfigKey = DEFAULT_ID_CONFIG_KEYS[this.name];
idOrName = this.config.get(defaultIdConfigKey) as string;
@@ -84,6 +88,8 @@ export class StorageManager<T extends IStorage = IStorage> {
this._addStorageToCache(storage);
}
+ this.storageOpenQueue.shift();
+
return storage;
}
diff --git a/yarn.lock b/yarn.lock
index 082675d19..5d3a25b05 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -832,6 +832,7 @@ __metadata:
"@crawlee/memory-storage": ^3.3.0
"@crawlee/types": ^3.3.0
"@crawlee/utils": ^3.3.0
+ "@sapphire/async-queue": ^1.5.0
"@types/tough-cookie": ^4.0.2
"@vladfrangu/async_event_emitter": ^2.0.0
csv-stringify: ^6.2.0