Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

refactor!: Rename StorageManager to StorageInstanceManager and use it to centralize storage instance caching (#3584)

Jan Bucharb6e5dd6f349c
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index ad169a07f..a0c1acd37 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -1304,7 +1304,8 @@ export class BasicCrawler<
// we need to purge the default RQ to allow processing the same requests again - this is important so users can
// pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
// ignored - as a failed requests is still handled.
- if (this.requestQueue?.name === 'default' && purgeRequestQueue) {
+ const isDefaultQueue = this.requestQueue?.name === 'default';
+ if (isDefaultQueue && purgeRequestQueue && this.requestQueue) {
await this.requestQueue.drop();
this.requestQueue = await this._getRequestQueue();
this.requestManager = undefined;
diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.ts
index 343a7265d..4fb4d2925 100644
--- a/packages/core/src/crawlers/crawler_commons.ts
+++ b/packages/core/src/crawlers/crawler_commons.ts
@@ -9,7 +9,7 @@ import type { Session } from '../session_pool/session.js';
import type { Dataset } from '../storages/dataset.js';
import { KeyValueStore, type RecordOptions } from '../storages/key_value_store.js';
import type { RequestQueueOperationOptions } from '../storages/request_provider.js';
-import type { StorageIdentifier } from '../storages/storage_manager.js';
+import type { StorageIdentifier } from '../storages/storage_instance_manager.js';
/** @internal */
export type IsAny<T> = 0 extends 1 & T ? true : false;
diff --git a/packages/core/src/service_locator.ts b/packages/core/src/service_locator.ts
index 4051c75fe..8ffb98cac 100644
--- a/packages/core/src/service_locator.ts
+++ b/packages/core/src/service_locator.ts
@@ -11,8 +11,7 @@ import type { EventManager } from './events/event_manager.js';
import { LocalEventManager } from './events/local_event_manager.js';
import type { CrawleeLogger } from './log.js';
import { ApifyLogAdapter } from './log.js';
-import type { IStorage, StorageManager } from './storages/storage_manager.js';
-import type { Constructor } from './typedefs.js';
+import { StorageInstanceManager } from './storages/storage_instance_manager.js';
interface ServiceLocatorInterface {
/**
@@ -77,15 +76,10 @@ interface ServiceLocatorInterface {
*/
getChildLog(prefix: string): CrawleeLogger;
- getStorageManager(constructor: Constructor<IStorage>): StorageManager | undefined;
-
- setStorageManager(constructor: Constructor<IStorage>, storageManager: StorageManager): void;
-
/**
- * Clears all storage manager caches.
- * @internal
+ * Get the storage instance manager (shared across all storage types).
*/
- clearStorageManagerCache(): void;
+ getStorageInstanceManager(): StorageInstanceManager;
/**
* Resets the service locator to its initial state.
@@ -134,10 +128,11 @@ export class ServiceLocator implements ServiceLocatorInterface {
private logger?: CrawleeLogger;
/**
- * Storage managers for Dataset, KeyValueStore, and RequestQueue.
- * Manages caching and lifecycle of storage instances.
+ * Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
+ * Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
+ * Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
*/
- private storageManagers = new Map<Constructor<IStorage>, StorageManager>();
+ private static storageInstanceManager?: StorageInstanceManager;
/**
* Creates a new ServiceLocator instance.
@@ -264,34 +259,11 @@ export class ServiceLocator implements ServiceLocatorInterface {
return this.getLogger().child({ prefix });
}
- getStorageManager(constructor: Constructor<IStorage>): StorageManager | undefined {
- return this.storageManagers.get(constructor);
- }
-
- setStorageManager(constructor: Constructor<IStorage>, storageManager: StorageManager): void {
- if (this.storageManagers.has(constructor)) {
- throw new ServiceConflictError(
- `StorageManager(${constructor.name})`,
- storageManager,
- this.storageManagers.get(constructor),
- );
+ getStorageInstanceManager(): StorageInstanceManager {
+ if (!ServiceLocator.storageInstanceManager) {
+ ServiceLocator.storageInstanceManager = new StorageInstanceManager();
}
-
- this.storageManagers.set(constructor, storageManager);
- }
-
- clearStorageManagerCache(): void {
- this.storageManagers.forEach((manager) => {
- // KeyValueStore has a clearCache method on its instances
- // TODO this uses fragile string matching and `any` casts into private fields - remove as part of
- // https://github.com/apify/crawlee/issues/3075 (Storage instance management will be reworked significantly)
- if ((manager as any).name === 'KeyValueStore') {
- (manager as any).cache?.forEach((item: any) => {
- item.clearCache?.();
- });
- }
- });
- this.storageManagers.clear();
+ return ServiceLocator.storageInstanceManager;
}
reset(): void {
@@ -299,7 +271,8 @@ export class ServiceLocator implements ServiceLocatorInterface {
this.eventManager = undefined;
this.storageClient = undefined;
this.logger = undefined;
- this.clearStorageManagerCache();
+ ServiceLocator.storageInstanceManager?.clearCache();
+ ServiceLocator.storageInstanceManager = undefined;
}
}
diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.ts
index f2a903912..9b371450d 100644
--- a/packages/core/src/storages/dataset.ts
+++ b/packages/core/src/storages/dataset.ts
@@ -10,8 +10,9 @@ import { serviceLocator } from '../service_locator.js';
import type { Awaitable } from '../typedefs.js';
import { checkStorageAccess } from './access_checking.js';
import { KeyValueStore } from './key_value_store.js';
-import type { StorageIdentifier, StorageManagerOptions } from './storage_manager.js';
-import { StorageManager } from './storage_manager.js';
+import type { StorageIdentifier } from './storage_instance_manager.js';
+import type { StorageOpenOptions } from './utils.js';
+import { resolveStorageIdentifier } from './storage_instance_manager.js';
import { purgeDefaultStorages } from './utils.js';
/** @internal */
@@ -681,8 +682,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {
checkStorageAccess();
await this.client.delete();
- const manager = StorageManager.getManager(Dataset);
- manager.closeStorage(this);
+ serviceLocator.getStorageInstanceManager().removeFromCache(this);
}
/**
@@ -702,7 +702,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {
*/
static async open<Data extends Dictionary = Dictionary>(
identifier?: string | StorageIdentifier | null,
- options: StorageManagerOptions = {},
+ options: StorageOpenOptions = {},
): Promise<Dataset<Data>> {
checkStorageAccess();
@@ -715,13 +715,18 @@ export class Dataset<Data extends Dictionary = Dictionary> {
);
options.config ??= Configuration.getGlobalConfig();
- options.storageClient ??= serviceLocator.getStorageClient();
- await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
+ const client = options.storageClient ?? serviceLocator.getStorageClient();
- const manager = StorageManager.getManager<Dataset<Data>>(this);
+ await purgeDefaultStorages({ onlyPurgeOnce: true, client, config: options.config });
- return manager.openStorage(identifier, options.storageClient);
+ const resolved = await resolveStorageIdentifier(identifier, client, 'Dataset');
+
+ return serviceLocator.getStorageInstanceManager().openStorage<Dataset<Data>>(this, {
+ ...resolved,
+ clientOpener: () => client.createDatasetClient(resolved),
+ clientCacheKey: client.getStorageClientCacheKey?.() ?? client.constructor.name,
+ });
}
/**
diff --git a/packages/core/src/storages/index.ts b/packages/core/src/storages/index.ts
index 46e381398..9bc71874d 100644
--- a/packages/core/src/storages/index.ts
+++ b/packages/core/src/storages/index.ts
@@ -6,7 +6,7 @@ export * from './request_provider.js';
export { RequestQueueV1 } from './request_queue.js';
export { RequestQueue } from './request_queue_v2.js';
export { RequestQueue as RequestQueueV2 } from './request_queue_v2.js';
-export * from './storage_manager.js';
+export * from './storage_instance_manager.js';
export * from './utils.js';
export * from './access_checking.js';
export * from './sitemap_request_list.js';
diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.ts
index c1c49f253..e04866a63 100644
--- a/packages/core/src/storages/key_value_store.ts
+++ b/packages/core/src/storages/key_value_store.ts
@@ -12,8 +12,9 @@ import { Configuration } from '../configuration.js';
import { serviceLocator } from '../service_locator.js';
import type { Awaitable } from '../typedefs.js';
import { checkStorageAccess } from './access_checking.js';
-import type { StorageIdentifier, StorageManagerOptions } from './storage_manager.js';
-import { StorageManager } from './storage_manager.js';
+import type { StorageIdentifier } from './storage_instance_manager.js';
+import type { StorageOpenOptions } from './utils.js';
+import { resolveStorageIdentifier } from './storage_instance_manager.js';
import { purgeDefaultStorages } from './utils.js';
/**
@@ -417,8 +418,7 @@ export class KeyValueStore {
checkStorageAccess();
await this.client.delete();
- const manager = StorageManager.getManager(KeyValueStore);
- manager.closeStorage(this);
+ serviceLocator.getStorageInstanceManager().removeFromCache(this);
}
/** @internal */
@@ -599,7 +599,7 @@ export class KeyValueStore {
*/
static async open(
identifier?: string | StorageIdentifier | null,
- options: StorageManagerOptions = {},
+ options: StorageOpenOptions = {},
): Promise<KeyValueStore> {
checkStorageAccess();
@@ -612,13 +612,17 @@ export class KeyValueStore {
);
options.config ??= Configuration.getGlobalConfig();
- options.storageClient ??= serviceLocator.getStorageClient();
+ const client = options.storageClient ?? serviceLocator.getStorageClient();
- await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
+ await purgeDefaultStorages({ onlyPurgeOnce: true, client, config: options.config });
- const manager = StorageManager.getManager(this);
+ const resolved = await resolveStorageIdentifier(identifier, client, 'KeyValueStore');
- return manager.openStorage(identifier, options.storageClient);
+ return serviceLocator.getStorageInstanceManager().openStorage<KeyValueStore>(this, {
+ ...resolved,
+ clientOpener: () => client.createKeyValueStoreClient(resolved),
+ clientCacheKey: client.getStorageClientCacheKey?.() ?? client.constructor.name,
+ });
}
/**
diff --git a/packages/core/src/storages/request_provider.ts b/packages/core/src/storages/request_provider.ts
index d2618bf6d..bf002fd1f 100644
--- a/packages/core/src/storages/request_provider.ts
+++ b/packages/core/src/storages/request_provider.ts
@@ -32,11 +32,11 @@ import type { ProxyConfiguration } from '../proxy_configuration.js';
import type { InternalSource, RequestOptions, Source } from '../request.js';
import { Request } from '../request.js';
import { serviceLocator } from '../service_locator.js';
-import type { Constructor } from '../typedefs.js';
import { checkStorageAccess } from './access_checking.js';
import { RequestDeduplicationCache } from './request_dedup_cache.js';
-import type { IStorage, StorageIdentifier, StorageManagerOptions } from './storage_manager.js';
-import { StorageManager } from './storage_manager.js';
+import type { IStorage, StorageIdentifier } from './storage_instance_manager.js';
+import { resolveStorageIdentifier } from './storage_instance_manager.js';
+import type { StorageOpenOptions } from './utils.js';
import { getRequestId, purgeDefaultStorages, QUERY_HEAD_MIN_LENGTH } from './utils.js';
export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
@@ -127,6 +127,7 @@ export abstract class RequestProvider implements IStorage, IRequestManager {
private initialCount = 0;
private initialHandledCount = 0; // We track this separately from `assumedHandledCount` which is used non-trivially by RequestQueueV1
+ private isInitialized = false;
protected queueHeadIds = new ListDictionary<string>();
protected requestCache: LruCache<RequestLruItem>;
@@ -154,7 +155,7 @@ export abstract class RequestProvider implements IStorage, IRequestManager {
constructor(
options: InternalRequestProviderOptions,
- protected readonly config: Configuration,
+ protected readonly config: Configuration = Configuration.getGlobalConfig(),
) {
this.id = options.id;
this.name = options.name;
@@ -804,8 +805,7 @@ export abstract class RequestProvider implements IStorage, IRequestManager {
checkStorageAccess();
await this.client.delete();
- const manager = StorageManager.getManager(this.constructor as Constructor<IStorage>);
- manager.closeStorage(this);
+ serviceLocator.getStorageInstanceManager().removeFromCache(this);
}
/**
@@ -943,7 +943,7 @@ export abstract class RequestProvider implements IStorage, IRequestManager {
*/
static async open(
identifier?: string | StorageIdentifier | null,
- options: StorageManagerOptions = {},
+ options: StorageOpenOptions = {},
): Promise<RequestProvider> {
checkStorageAccess();
@@ -957,30 +957,41 @@ export abstract class RequestProvider implements IStorage, IRequestManager {
}),
);
- options.storageClient ??= serviceLocator.getStorageClient();
+ const client = options.storageClient ?? serviceLocator.getStorageClient();
+ const config = options.config ?? serviceLocator.getConfiguration();
- await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
+ await purgeDefaultStorages({ onlyPurgeOnce: true, client, config });
- const manager = StorageManager.getManager(this as typeof BuiltRequestProvider);
- const queue = await manager.openStorage(identifier, options.storageClient);
+ const resolved = await resolveStorageIdentifier(identifier, client, 'RequestQueue');
+
+ const queue = await serviceLocator
+ .getStorageInstanceManager()
+ .openStorage<RequestProvider>(this as typeof BuiltRequestProvider, {
+ ...resolved,
+ clientOpener: () => client.createRequestQueueClient(resolved),
+ clientCacheKey: client.getStorageClientCacheKey?.() ?? client.constructor.name,
+ });
queue.proxyConfiguration = options.proxyConfiguration;
+ queue.httpClient = options.httpClient;
- // Re-create the request queue client with clientKey and timeoutSecs so that
- // request locking works correctly for API-backed implementations.
- // TODO: clientKey/timeoutSecs are Apify-platform concerns and should eventually be pushed
- // down into the Apify SDK's client implementation, aligning with crawlee-python's approach
- // where locking is handled internally by the client (see crawlee-python PR #1194).
- queue.client = await options.storageClient.createRequestQueueClient({
- id: queue.id,
- clientKey: queue.clientKey,
- timeoutSecs: queue.timeoutSecs,
- });
+ if (!queue.isInitialized) {
+ // Re-create the request queue client with clientKey and timeoutSecs so that
+ // request locking works correctly for API-backed implementations.
+ // TODO: clientKey/timeoutSecs are Apify-platform concerns and should eventually be pushed
+ // down into the Apify SDK's client implementation, aligning with crawlee-python's approach
+ // where locking is handled internally by the client (see crawlee-python PR #1194).
+ queue.client = await client.createRequestQueueClient({
+ id: queue.id,
+ clientKey: queue.clientKey,
+ timeoutSecs: queue.timeoutSecs,
+ });
- const queueInfo = await queue.client.getMetadata();
+ const queueInfo = await queue.client.getMetadata();
- queue.initialCount = queueInfo.totalRequestCount;
- queue.initialHandledCount = queueInfo.handledRequestCount;
- queue.httpClient = options.httpClient;
+ queue.initialCount = queueInfo.totalRequestCount;
+ queue.initialHandledCount = queueInfo.handledRequestCount;
+ queue.isInitialized = true;
+ }
return queue;
}
diff --git a/packages/core/src/storages/storage_instance_manager.ts b/packages/core/src/storages/storage_instance_manager.ts
new file mode 100644
index 000000000..ee8f73c3b
--- /dev/null
+++ b/packages/core/src/storages/storage_instance_manager.ts
@@ -0,0 +1,329 @@
+import type {
+ DatasetClient,
+ KeyValueStoreClient,
+ RequestQueueClient,
+ StorageClient,
+ StorageIdentifier,
+} from '@crawlee/types';
+import { AsyncQueue } from '@sapphire/async-queue';
+
+import type { Constructor } from '../typedefs.js';
+
+export type { StorageIdentifier } from '@crawlee/types';
+
+/**
+ * Matches an `IStorage` – a storage "frontend" (Dataset, KeyValueStore, RequestQueue).
+ */
+export interface IStorage {
+ id: string;
+ name?: string;
+}
+
+type Hashable = string;
+
+/** Reserved alias for the default (unnamed) storage. */
+const DEFAULT_STORAGE_ALIAS = '__default__';
+
+type CacheTier = Map<Constructor<IStorage>, Map<string, Map<Hashable, IStorage>>>;
+
+/**
+ * Three-tier cache for storage instances, modelled after crawlee-python's `_StorageCache`.
+ *
+ * Each tier maps `[storageClass][key][clientCacheKey] → instance`:
+ * - `byId` — keyed by the backend-assigned storage id
+ * - `byName` — keyed by the persistent storage name
+ * - `byAlias` — keyed by a run-scoped alias (e.g. `'__default__'` for unnamed storages)
+ */
+class StorageCache {
+ readonly byId: CacheTier = new Map();
+ readonly byName: CacheTier = new Map();
+ readonly byAlias: CacheTier = new Map();
+
+ get<T extends IStorage>(
+ cls: Constructor<T>,
+ {
+ id,
+ name,
+ alias,
+ clientCacheKey,
+ }: (
+ | { id: string; name?: string; alias?: undefined }
+ | { id?: string; name: string; alias?: undefined }
+ | { id?: undefined; name?: undefined; alias: string }
+ ) & { clientCacheKey: Hashable },
+ ): T | undefined {
+ for (const [tier, key] of [
+ [this.byId, id],
+ [this.byName, name],
+ [this.byAlias, alias],
+ ] as [CacheTier, string | undefined][]) {
+ if (key === undefined) continue;
+ const cached = tier.get(cls)?.get(key)?.get(clientCacheKey);
+ if (cached) {
+ if (cached instanceof (cls as unknown as abstract new (...args: any[]) => any)) {
+ return cached as T;
+ }
+ throw new Error('Cached storage instance type mismatch.');
+ }
+ }
+
+ return undefined;
+ }
+
+ /** Write a single entry into a given tier. */
+ private setInMap<T extends IStorage>(
+ tier: CacheTier,
+ cls: Constructor<T>,
+ key: string,
+ instance: T,
+ clientCacheKey: Hashable,
+ ): void {
+ if (!tier.has(cls)) tier.set(cls, new Map());
+ const keyMap = tier.get(cls)!;
+ if (!keyMap.has(key)) keyMap.set(key, new Map());
+ keyMap.get(key)!.set(clientCacheKey, instance);
+ }
+
+ /**
+ * Cache an instance under its actual id, name, and an optional alias.
+ */
+ set<T extends IStorage>(cls: Constructor<T>, instance: T, clientCacheKey: Hashable, alias?: string): void {
+ // Always cache by id.
+ this.setInMap(this.byId, cls, instance.id, instance, clientCacheKey);
+
+ // Cache by name — only for named storages.
+ if (instance.name) {
+ this.setInMap(this.byName, cls, instance.name, instance, clientCacheKey);
+ }
+
+ // Cache by alias — only for unnamed storages opened via alias.
+ if (alias !== undefined) {
+ this.setInMap(this.byAlias, cls, alias, instance, clientCacheKey);
+ }
+ }
+
+ removeFromCache(instance: IStorage): void {
+ const storageType = instance.constructor as Constructor<IStorage>;
+
+ for (const tier of [this.byId, this.byName, this.byAlias]) {
+ const classMap = tier.get(storageType);
+ if (!classMap) continue;
+
+ for (const keyMap of classMap.values()) {
+ for (const [cacheKey, cached] of keyMap) {
+ if (cached === instance) {
+ keyMap.delete(cacheKey);
+ }
+ }
+ }
+ }
+ }
+
+ /** Iterate all cached instances across all storage types. */
+ *allValues(): IterableIterator<IStorage> {
+ const seen = new Set<IStorage>();
+ for (const classMap of this.byId.values()) {
+ for (const keyMap of classMap.values()) {
+ for (const instance of keyMap.values()) {
+ if (!seen.has(instance)) {
+ seen.add(instance);
+ yield instance;
+ }
+ }
+ }
+ }
+ }
+
+ clear(): void {
+ this.byId.clear();
+ this.byName.clear();
+ this.byAlias.clear();
+ }
+}
+
+/**
+ * Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
+ *
+ * A single instance manages all storage types. Instances are cached by
+ * `(storageClass, id/name/alias, clientCacheKey)` so the same storage is never opened twice.
+ *
+ * The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
+ * and a pre-bound `clientOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
+ * assigns a reserved default alias.
+ *
+ * @ignore
+ */
+export class StorageInstanceManager {
+ private readonly cache = new StorageCache();
+ private readonly openerLocks = new Map<string, AsyncQueue>();
+
+ /**
+ * Open (or retrieve from cache) a storage instance.
+ *
+ * @param cls The storage class constructor (e.g. `Dataset`, `KeyValueStore`, `RequestQueue`).
+ * @param id Storage ID (mutually exclusive with `name` and `alias`).
+ * @param name Storage name (mutually exclusive with `id` and `alias`).
+ * @param alias Run-scoped alias (mutually exclusive with `id` and `name`).
+ * Automatically assigned when no identifier is provided.
+ * @param clientOpener A **lazy** factory that creates the sub-client.
+ * Only called on a cache miss.
+ * @param clientCacheKey Opaque key identifying the storage backend, so that the same logical
+ * storage opened through different clients is cached separately.
+ */
+ async openStorage<TStorage extends IStorage>(
+ cls: Constructor<TStorage>,
+ {
+ id,
+ name,
+ alias,
+ clientOpener,
+ clientCacheKey,
+ }: (ExplicitStorageIdentifier | DefaultStorageIdentifier) & {
+ clientOpener: () => Promise<DatasetClient | KeyValueStoreClient | RequestQueueClient>;
+ clientCacheKey: Hashable;
+ },
+ ): Promise<TStorage> {
+ // Auto-set alias='__default__' when no parameters are specified (mirrors crawlee-python).
+ if (!id && !name && !alias) {
+ alias = DEFAULT_STORAGE_ALIAS;
+ }
+
+ // Fast-path cache check (no lock).
+ if (alias !== undefined) {
+ const cached = this.cache.get(cls, { alias, clientCacheKey });
+ if (cached) return cached;
+ } else if (id) {
+ const cached = this.cache.get(cls, { id, clientCacheKey });
+ if (cached) return cached;
+ } else if (name) {
+ const cached = this.cache.get(cls, { name, clientCacheKey });
+ if (cached) return cached;
+ }
+
+ const identifierKey = id ?? name ?? alias ?? DEFAULT_STORAGE_ALIAS;
+ const lockKey = `${cls.name}:${identifierKey}:${clientCacheKey}`;
+
+ if (!this.openerLocks.has(lockKey)) {
+ this.openerLocks.set(lockKey, new AsyncQueue());
+ }
+ const queue = this.openerLocks.get(lockKey)!;
+
+ await queue.wait();
+ try {
+ // Double-check cache under lock (another caller may have filled it while we waited).
+ if (alias !== undefined) {
+ const cached = this.cache.get(cls, { alias, clientCacheKey });
+ if (cached) return cached;
+ } else if (id) {
+ const cached = this.cache.get(cls, { id, clientCacheKey });
+ if (cached) return cached;
+ } else if (name) {
+ const cached = this.cache.get(cls, { name, clientCacheKey });
+ if (cached) return cached;
+ }
+
+ // Cache miss — create the sub-client and storage instance.
+ const subClient = await clientOpener();
+ const storageInfo = await (
+ subClient as DatasetClient | KeyValueStoreClient | RequestQueueClient
+ ).getMetadata();
+
+ const instance = new cls({
+ id: storageInfo.id,
+ name: storageInfo.name,
+ client: subClient,
+ }) as TStorage;
+
+ // Atomic cache writes (no awaits between these).
+ this.cache.set(cls, instance, clientCacheKey, alias);
+
+ return instance;
+ } finally {
+ queue.shift();
+
+ // Clean up idle locks so the map doesn't grow unboundedly
+ // (mirrors crawlee-python's WeakValueDictionary behaviour).
+ if (queue.remaining === 0) {
+ this.openerLocks.delete(lockKey);
+ }
+ }
+ }
+
+ /**
+ * Remove a storage instance from the cache (called from `storage.drop()`).
+ */
+ removeFromCache(instance: IStorage): void {
+ this.cache.removeFromCache(instance);
+ }
+
+ /**
+ * Clear the entire cache. Also calls `clearCache()` on any cached KeyValueStore
+ * instances (duck-typed to avoid importing KeyValueStore and circular dependencies).
+ * Called during service locator reset.
+ */
+ clearCache(): void {
+ for (const instance of this.cache.allValues()) {
+ if ('clearCache' in instance && typeof (instance as any).clearCache === 'function') {
+ (instance as any).clearCache();
+ }
+ }
+
+ this.cache.clear();
+ }
+}
+
+/**
+ * A storage identifier where exactly one of `id`, `name`, or `alias` is specified.
+ * Produced by {@link resolveStorageIdentifier} from ambiguous user input.
+ */
+export type ExplicitStorageIdentifier =
+ | { id: string; name?: never; alias?: never }
+ | { id?: never; name: string; alias?: never }
+ | { id?: never; name?: never; alias: string };
+
+/**
+ * Represents the case where no identifier was provided — the caller wants the default storage.
+ */
+export interface DefaultStorageIdentifier {
+ id?: never;
+ name?: never;
+ alias?: never;
+}
+
+/**
+ * Decompose a user-provided `identifier` (the `Dataset.open()` / `KeyValueStore.open()` /
+ * `RequestQueue.open()` argument) into separate `id`, `name`, and `alias` fields that
+ * the `StorageInstanceManager` and `StorageClient.create*Client` expect.
+ *
+ * - `null` / `undefined` / `{}` → default storage alias
+ * - `string` → resolved via `storageExists` (ID-first, then name)
+ * - `{ id }` → `{ id }`
+ * - `{ name }` → `{ name }`
+ */
+export async function resolveStorageIdentifier(
+ identifier: string | StorageIdentifier | null | undefined,
+ client: StorageClient,
+ storageType: 'Dataset' | 'KeyValueStore' | 'RequestQueue',
+): Promise<ExplicitStorageIdentifier> {
+ if (identifier === null || identifier === undefined) {
+ return { alias: DEFAULT_STORAGE_ALIAS };
+ }
+
+ if (typeof identifier === 'string') {
+ if (client.storageExists && (await client.storageExists(identifier, storageType))) {
+ return { id: identifier };
+ }
+ return { name: identifier };
+ }
+
+ if (identifier.id) {
+ return { id: identifier.id };
+ }
+
+ if (identifier.name) {
+ return { name: identifier.name };
+ }
+
+ // Empty object — treated as default storage.
+ return { alias: DEFAULT_STORAGE_ALIAS };
+}
diff --git a/packages/core/src/storages/storage_manager.ts b/packages/core/src/storages/storage_manager.ts
deleted file mode 100644
index 46f486cf4..000000000
--- a/packages/core/src/storages/storage_manager.ts
+++ /dev/null
@@ -1,196 +0,0 @@
-import type {
- BaseHttpClient,
- DatasetClient,
- KeyValueStoreClient,
- RequestQueueClient,
- StorageClient,
- StorageIdentifier,
-} from '@crawlee/types';
-import { AsyncQueue } from '@sapphire/async-queue';
-
-import type { Configuration } from '../configuration.js';
-import type { ProxyConfiguration } from '../proxy_configuration.js';
-import { serviceLocator } from '../service_locator.js';
-import type { Constructor } from '../typedefs.js';
-
-export type { StorageIdentifier } from '@crawlee/types';
-
-const DEFAULT_ID_CONFIG_KEYS = {
- Dataset: 'defaultDatasetId',
- KeyValueStore: 'defaultKeyValueStoreId',
- RequestQueue: 'defaultRequestQueueId',
-} as const;
-
-export interface IStorage {
- id: string;
- name?: string;
-}
-
-/**
- * StorageManager takes care of opening remote or local storages.
- * @ignore
- */
-export class StorageManager<T extends IStorage = IStorage> {
- private readonly name: 'Dataset' | 'KeyValueStore' | 'RequestQueue';
- private readonly StorageConstructor: Constructor<T> & { name: string };
- private readonly cache = new Map<string, T>();
- private readonly storageOpenQueue = new AsyncQueue();
-
- constructor(
- StorageConstructor: Constructor<T>,
- private readonly config: Configuration,
- ) {
- this.StorageConstructor = StorageConstructor;
- this.name = this.StorageConstructor.name as 'Dataset' | 'KeyValueStore' | 'RequestQueue';
- }
-
- static async openStorage<T extends IStorage>(
- storageClass: Constructor<T>,
- identifier?: string | StorageIdentifier,
- client?: StorageClient,
- ): Promise<T> {
- return this.getManager(storageClass).openStorage(identifier, client);
- }
-
- static getManager<T extends IStorage>(storageClass: Constructor<T>): StorageManager<T> {
- let storageManager = serviceLocator.getStorageManager(storageClass);
- if (storageManager === undefined) {
- storageManager = new StorageManager(storageClass, serviceLocator.getConfiguration());
- serviceLocator.setStorageManager(storageClass, storageManager);
- }
-
- return serviceLocator.getStorageManager(storageClass) as StorageManager<T>;
- }
-
- async openStorage(identifier?: string | StorageIdentifier | null, client?: StorageClient): Promise<T> {
- await this.storageOpenQueue.wait();
-
- try {
- const resolvedIdentifier = await this._resolveIdentifier(identifier, client);
-
- const cacheKey = resolvedIdentifier.id ?? resolvedIdentifier.name!;
- let storage = this.cache.get(cacheKey);
-
- if (!storage) {
- client ??= serviceLocator.getStorageClient();
- const subClient = await this._createSubClient(resolvedIdentifier, this.name, client);
- const storageInfo = await (
- subClient as DatasetClient | KeyValueStoreClient | RequestQueueClient
- ).getMetadata();
- storage = new this.StorageConstructor(
- {
- id: storageInfo.id,
- name: storageInfo.name,
- client: subClient,
- },
- this.config,
- );
- this._addStorageToCache(storage);
- }
-
- return storage;
- } finally {
- this.storageOpenQueue.shift();
- }
- }
-
- /**
- * Resolves the user-provided identifier to an unambiguous `StorageIdentifier`.
- *
- * - `null`/`undefined` → uses the default storage ID from config
- * - `StorageIdentifier` object → passed through (with default ID fallback if empty)
- * - `string` → tries to find an existing storage with that ID first;
- * if none exists, treats the string as a name
- */
- private async _resolveIdentifier(
- identifier?: string | StorageIdentifier | null,
- client?: StorageClient,
- ): Promise<StorageIdentifier> {
- if (typeof identifier === 'string') {
- client ??= serviceLocator.getStorageClient();
-
- if (client.storageExists && (await client.storageExists(identifier, this.name))) {
- return { id: identifier };
- }
-
- return { name: identifier };
- }
-
- if (identifier?.id) {
- return { id: identifier.id };
- }
-
- if (identifier?.name) {
- return { name: identifier.name };
- }
-
- const defaultIdConfigKey = DEFAULT_ID_CONFIG_KEYS[this.name];
- const defaultId = this.config[defaultIdConfigKey] as string;
-
- return { id: defaultId };
- }
-
- closeStorage(storage: { id: string; name?: string }): void {
- const idKey = storage.id;
- this.cache.delete(idKey);
-
- if (storage.name) {
- const nameKey = storage.name;
- this.cache.delete(nameKey);
- }
- }
-
- /**
- * Creates a sub-client for the given storage type using the new factory-based StorageClient interface.
- */
- protected async _createSubClient(
- identifier: StorageIdentifier,
- storageConstructorName: string,
- apiClient: StorageClient,
- ): Promise<DatasetClient | KeyValueStoreClient | RequestQueueClient> {
- switch (storageConstructorName) {
- case 'Dataset':
- return apiClient.createDatasetClient(identifier);
- case 'KeyValueStore':
- return apiClient.createKeyValueStoreClient(identifier);
- case 'RequestQueue':
- return apiClient.createRequestQueueClient(identifier);
- default:
- throw new Error(`Unknown storage type: ${storageConstructorName}`);
- }
- }
-
- protected _addStorageToCache(storage: T): void {
- const idKey = storage.id;
- this.cache.set(idKey, storage);
-
- if (storage.name) {
- const nameKey = storage.name;
- this.cache.set(nameKey, storage);
- }
- }
-}
-
-export interface StorageManagerOptions {
- /**
- * SDK configuration instance, defaults to the static register.
- */
- config?: Configuration;
-
- /**
- * Optional storage client that should be used to open storages.
- */
- storageClient?: StorageClient;
-
- /**
- * Used to pass the proxy configuration for the `requestsFromUrl` objects.
- * Takes advantage of the internal address rotation and authentication process.
- * If undefined, the `requestsFromUrl` requests will be made without proxy.
- */
- proxyConfiguration?: ProxyConfiguration;
-
- /**
- * HTTP client to be used to download the list of URLs in `RequestQueue`.
- */
- httpClient?: BaseHttpClient;
-}
diff --git a/packages/core/src/storages/utils.ts b/packages/core/src/storages/utils.ts
index 53fa50abb..fbdb98132 100644
--- a/packages/core/src/storages/utils.ts
+++ b/packages/core/src/storages/utils.ts
@@ -1,8 +1,9 @@
import crypto from 'node:crypto';
-import type { Dictionary, StorageClient } from '@crawlee/types';
+import type { BaseHttpClient, Dictionary, StorageClient } from '@crawlee/types';
import { Configuration } from '../configuration.js';
+import type { ProxyConfiguration } from '../proxy_configuration.js';
import { serviceLocator } from '../service_locator.js';
import { KeyValueStore } from './key_value_store.js';
@@ -149,3 +150,30 @@ export const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10_000;
* @internal
*/
export const MAX_QUERIES_FOR_CONSISTENCY = 6;
+
+/**
+ * Options for the static `open()` method on storage classes ({@apilink Dataset}, {@apilink KeyValueStore}, {@apilink RequestQueue}).
+ */
+export interface StorageOpenOptions {
+ /**
+ * SDK configuration instance, defaults to the static register.
+ */
+ config?: Configuration;
+
+ /**
+ * Optional storage client that should be used to open storages.
+ */
+ storageClient?: StorageClient;
+
+ /**
+ * Used to pass the proxy configuration for the `requestsFromUrl` objects.
+ * Takes advantage of the internal address rotation and authentication process.
+ * If undefined, the `requestsFromUrl` requests will be made without proxy.
+ */
+ proxyConfiguration?: ProxyConfiguration;
+
+ /**
+ * HTTP client to be used to download the list of URLs in `RequestQueue`.
+ */
+ httpClient?: BaseHttpClient;
+}
diff --git a/packages/memory-storage/src/memory-storage.ts b/packages/memory-storage/src/memory-storage.ts
index a5b33fb63..0ccc6da2f 100644
--- a/packages/memory-storage/src/memory-storage.ts
+++ b/packages/memory-storage/src/memory-storage.ts
@@ -93,9 +93,24 @@ export class MemoryStorage implements storage.StorageClient {
: true);
}
+ /**
+ * Return a cache key that includes the resolved storage directory, so that
+ * two `MemoryStorage` instances pointing at different directories get separate
+ * cache partitions. Mirrors crawlee-python's `FileSystemStorageClient` which
+ * includes `configuration.storage_dir` in its cache key.
+ */
+ getStorageClientCacheKey(): string {
+ return `MemoryStorage:${resolve(this.localDataDirectory)}`;
+ }
+
+ private static resolveStorageName(options: { id?: string; name?: string; alias?: string }): string | undefined {
+ const name = 'alias' in options ? options.alias : (options.name ?? options.id);
+ // Normalize the internal __default__ alias to the user-facing 'default' name.
+ return name === '__default__' ? 'default' : name;
+ }
+
async createDatasetClient(options: storage.CreateDatasetClientOptions = {}): Promise<storage.DatasetClient> {
- // In MemoryStorage, both id and name resolve to the same directory name.
- const name = options.name ?? options.id;
+ const name = MemoryStorage.resolveStorageName(options);
if (name) {
const found = await findOrCacheDatasetByPossibleId(this, name);
@@ -129,8 +144,7 @@ export class MemoryStorage implements storage.StorageClient {
async createKeyValueStoreClient(
options: storage.CreateKeyValueStoreClientOptions = {},
): Promise<storage.KeyValueStoreClient> {
- // In MemoryStorage, both id and name resolve to the same directory name.
- const name = options.name ?? options.id;
+ const name = MemoryStorage.resolveStorageName(options);
if (name) {
const found = await findOrCacheKeyValueStoreByPossibleId(this, name);
@@ -168,8 +182,7 @@ export class MemoryStorage implements storage.StorageClient {
async createRequestQueueClient(
options: storage.CreateRequestQueueClientOptions = {},
): Promise<storage.RequestQueueClient> {
- // In MemoryStorage, both id and name resolve to the same directory name.
- const name = options.name ?? options.id;
+ const name = MemoryStorage.resolveStorageName(options);
if (name) {
const found = await findRequestQueueByPossibleId(this, name);
@@ -264,7 +277,7 @@ export class MemoryStorage implements storage.StorageClient {
keyValueStorePromises.push(
(await this.batchRemoveFiles(resolve(this.keyValueStoresDirectory, keyValueStoreFolder)))(),
);
- } else if (keyValueStoreFolder === 'default') {
+ } else if (keyValueStoreFolder === 'default' || keyValueStoreFolder === '__default__') {
keyValueStorePromises.push(
this.handleDefaultKeyValueStore(resolve(this.keyValueStoresDirectory, keyValueStoreFolder))(),
);
@@ -278,7 +291,11 @@ export class MemoryStorage implements storage.StorageClient {
const datasetPromises: Promise<void>[] = [];
for (const datasetFolder of datasets) {
- if (datasetFolder === 'default' || datasetFolder.startsWith('__CRAWLEE_TEMPORARY')) {
+ if (
+ datasetFolder === 'default' ||
+ datasetFolder === '__default__' ||
+ datasetFolder.startsWith('__CRAWLEE_TEMPORARY')
+ ) {
datasetPromises.push((await this.batchRemoveFiles(resolve(this.datasetsDirectory, datasetFolder)))());
}
}
@@ -290,7 +307,11 @@ export class MemoryStorage implements storage.StorageClient {
const requestQueuePromises: Promise<void>[] = [];
for (const requestQueueFolder of requestQueues) {
- if (requestQueueFolder === 'default' || requestQueueFolder.startsWith('__CRAWLEE_TEMPORARY')) {
+ if (
+ requestQueueFolder === 'default' ||
+ requestQueueFolder === '__default__' ||
+ requestQueueFolder.startsWith('__CRAWLEE_TEMPORARY')
+ ) {
requestQueuePromises.push(
(await this.batchRemoveFiles(resolve(this.requestQueuesDirectory, requestQueueFolder)))(),
);
diff --git a/packages/memory-storage/src/resource-clients/dataset.ts b/packages/memory-storage/src/resource-clients/dataset.ts
index bbb3bdbf8..37a389d0f 100644
--- a/packages/memory-storage/src/resource-clients/dataset.ts
+++ b/packages/memory-storage/src/resource-clients/dataset.ts
@@ -8,7 +8,6 @@ import { s } from '@sapphire/shapeshift';
import { move } from 'fs-extra';
import { scheduleBackgroundTask } from '../background-handler/index.js';
-import { findOrCacheDatasetByPossibleId } from '../cache-helpers.js';
import { StorageTypes } from '../consts.js';
import type { StorageImplementation } from '../fs/common.js';
import { createDatasetStorageImplementation } from '../fs/dataset/index.js';
@@ -57,13 +56,7 @@ export class DatasetClient<Data extends Dictionary = Dictionary>
}
async getMetadata(): Promise<storage.DatasetInfo> {
- const found = await findOrCacheDatasetByPossibleId(this.client, this.name ?? this.id);
-
- if (found) {
- found.updateTimestamps(false);
- return found.toDatasetInfo();
- }
-
+ this.updateTimestamps(false);
return this.toDatasetInfo();
}
@@ -74,16 +67,9 @@ export class DatasetClient<Data extends Dictionary = Dictionary>
})
.parse(newFields);
- // Check by id
- const existingStoreById = await findOrCacheDatasetByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingStoreById) {
- this.throwOnNonExisting(StorageTypes.Dataset);
- }
-
// Skip if no changes
if (!parsed.name) {
- return existingStoreById.toDatasetInfo();
+ return this.toDatasetInfo();
}
// Check that name is not in use already
@@ -95,21 +81,21 @@ export class DatasetClient<Data extends Dictionary = Dictionary>
this.throwOnDuplicateEntry(StorageTypes.Dataset, 'name', parsed.name);
}
- existingStoreById.name = parsed.name;
+ this.name = parsed.name;
- const previousDir = existingStoreById.datasetDirectory;
+ const previousDir = this.datasetDirectory;
- existingStoreById.datasetDirectory = resolveWithinDirectory(
+ this.datasetDirectory = resolveWithinDirectory(
this.client.datasetsDirectory,
- parsed.name ?? existingStoreById.name ?? existingStoreById.id,
+ parsed.name ?? this.name ?? this.id,
);
- await move(previousDir, existingStoreById.datasetDirectory, { overwrite: true });
+ await move(previousDir, this.datasetDirectory, { overwrite: true });
// Update timestamps
- existingStoreById.updateTimestamps(true);
+ this.updateTimestamps(true);
- return existingStoreById.toDatasetInfo();
+ return this.toDatasetInfo();
}
async delete(): Promise<void> {
@@ -175,15 +161,8 @@ export class DatasetClient<Data extends Dictionary = Dictionary>
private async listItemsPage(options: storage.DatasetClientListOptions = {}): Promise<storage.PaginatedList<Data>> {
const { limit = LIST_ITEMS_LIMIT, offset = 0, desc } = options;
- // Check by id
- const existingStoreById = await findOrCacheDatasetByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingStoreById) {
- this.throwOnNonExisting(StorageTypes.Dataset);
- }
-
- const [start, end] = existingStoreById.getStartAndEndIndexes(
- desc ? Math.max(existingStoreById.itemCount - offset - limit, 0) : offset,
+ const [start, end] = this.getStartAndEndIndexes(
+ desc ? Math.max(this.itemCount - offset - limit, 0) : offset,
limit,
);
@@ -191,10 +170,10 @@ export class DatasetClient<Data extends Dictionary = Dictionary>
for (let idx = start; idx < end; idx++) {
const entryNumber = this.generateLocalEntryName(idx);
- items.push(await existingStoreById.datasetEntries.get(entryNumber)!.get());
+ items.push(await this.datasetEntries.get(entryNumber)!.get());
}
- existingStoreById.updateTimestamps(false);
+ this.updateTimestamps(false);
return {
count: items.length,
@@ -202,7 +181,7 @@ export class DatasetClient<Data extends Dictionary = Dictionary>
items: desc ? items.reverse() : items,
limit,
offset,
- total: existingStoreById.itemCount,
+ total: this.itemCount,
};
}
@@ -215,32 +194,22 @@ export class DatasetClient<Data extends Dictionary = Dictionary>
])
.parse(items) as Data[];
- // Check by id
- const existingStoreById = await findOrCacheDatasetByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingStoreById) {
- this.throwOnNonExisting(StorageTypes.Dataset);
- }
-
const normalized = this.normalizeItems(rawItems);
- const addedIds: string[] = [];
-
for (const entry of normalized) {
- const idx = this.generateLocalEntryName(++existingStoreById.itemCount);
+ const idx = this.generateLocalEntryName(++this.itemCount);
const storageEntry = createDatasetStorageImplementation({
entityId: idx,
persistStorage: this.client.persistStorage,
- storeDirectory: existingStoreById.datasetDirectory,
+ storeDirectory: this.datasetDirectory,
});
await storageEntry.update(entry);
- existingStoreById.datasetEntries.set(idx, storageEntry);
- addedIds.push(idx);
+ this.datasetEntries.set(idx, storageEntry);
}
- existingStoreById.updateTimestamps(true);
+ this.updateTimestamps(true);
}
toDatasetInfo(): storage.DatasetInfo {
diff --git a/packages/memory-storage/src/resource-clients/key-value-store.ts b/packages/memory-storage/src/resource-clients/key-value-store.ts
index f9f5fb0fd..7f2406650 100644
--- a/packages/memory-storage/src/resource-clients/key-value-store.ts
+++ b/packages/memory-storage/src/resource-clients/key-value-store.ts
@@ -10,7 +10,6 @@ import pLimit from 'p-limit';
import { scheduleBackgroundTask } from '../background-handler/index.js';
import { maybeParseBody } from '../body-parser.js';
-import { findOrCacheKeyValueStoreByPossibleId } from '../cache-helpers.js';
import { DEFAULT_API_PARAM_LIMIT, StorageTypes } from '../consts.js';
import type { StorageImplementation } from '../fs/common.js';
import { createKeyValueStorageImplementation } from '../fs/key-value-store/index.js';
@@ -61,13 +60,7 @@ export class KeyValueStoreClient extends BaseClient {
}
async getMetadata(): Promise<storage.KeyValueStoreInfo> {
- const found = await findOrCacheKeyValueStoreByPossibleId(this.client, this.name ?? this.id);
-
- if (found) {
- found.updateTimestamps(false);
- return found.toKeyValueStoreInfo();
- }
-
+ this.updateTimestamps(false);
return this.toKeyValueStoreInfo();
}
@@ -78,16 +71,9 @@ export class KeyValueStoreClient extends BaseClient {
})
.parse(newFields);
- // Check by id
- const existingStoreById = await findOrCacheKeyValueStoreByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingStoreById) {
- this.throwOnNonExisting(StorageTypes.KeyValueStore);
- }
-
// Skip if no changes
if (!parsed.name) {
- return existingStoreById.toKeyValueStoreInfo();
+ return this.toKeyValueStoreInfo();
}
// Check that name is not in use already
@@ -99,21 +85,21 @@ export class KeyValueStoreClient extends BaseClient {
this.throwOnDuplicateEntry(StorageTypes.KeyValueStore, 'name', parsed.name);
}
- existingStoreById.name = parsed.name;
+ this.name = parsed.name;
- const previousDir = existingStoreById.keyValueStoreDirectory;
+ const previousDir = this.keyValueStoreDirectory;
- existingStoreById.keyValueStoreDirectory = resolveWithinDirectory(
+ this.keyValueStoreDirectory = resolveWithinDirectory(
this.client.keyValueStoresDirectory,
- parsed.name ?? existingStoreById.name ?? existingStoreById.id,
+ parsed.name ?? this.name ?? this.id,
);
- await move(previousDir, existingStoreById.keyValueStoreDirectory, { overwrite: true });
+ await move(previousDir, this.keyValueStoreDirectory, { overwrite: true });
// Update timestamps
- existingStoreById.updateTimestamps(true);
+ this.updateTimestamps(true);
- return existingStoreById.toKeyValueStoreInfo();
+ return this.toKeyValueStoreInfo();
}
async delete(): Promise<void> {
@@ -278,16 +264,9 @@ export class KeyValueStoreClient extends BaseClient {
): Promise<storage.KeyValueStoreClientListData> {
const { limit = DEFAULT_API_PARAM_LIMIT, exclusiveStartKey, prefix } = options;
- // Check by id
- const existingStoreById = await findOrCacheKeyValueStoreByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingStoreById) {
- this.throwOnNonExisting(StorageTypes.KeyValueStore);
- }
-
const items = [];
- for (const storageEntry of existingStoreById.keyValueEntries.values()) {
+ for (const storageEntry of this.keyValueEntries.values()) {
const record = await storageEntry.get();
const size = Buffer.byteLength(record.value);
@@ -318,7 +297,7 @@ export class KeyValueStoreClient extends BaseClient {
const isLastSelectedItemAbsolutelyLast = lastItemInStore === lastSelectedItem;
const nextExclusiveStartKey = isLastSelectedItemAbsolutelyLast ? undefined : lastSelectedItem?.key;
- existingStoreById.updateTimestamps(false);
+ this.updateTimestamps(false);
return {
count: limitedItems.length,
@@ -339,14 +318,7 @@ export class KeyValueStoreClient extends BaseClient {
async getRecordPublicUrl(key: string): Promise<string | undefined> {
s.string().parse(key);
- // Check by id
- const existingStoreById = await findOrCacheKeyValueStoreByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingStoreById) {
- this.throwOnNonExisting(StorageTypes.KeyValueStore);
- }
-
- const storageEntry = await existingStoreById.keyValueEntries.get(key)?.get();
+ const storageEntry = await this.keyValueEntries.get(key)?.get();
return storageEntry?.filePath;
}
@@ -360,14 +332,7 @@ export class KeyValueStoreClient extends BaseClient {
async recordExists(key: string): Promise<boolean> {
s.string().parse(key);
- // Check by id
- const existingStoreById = await findOrCacheKeyValueStoreByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingStoreById) {
- this.throwOnNonExisting(StorageTypes.KeyValueStore);
- }
-
- return existingStoreById.keyValueEntries.has(key);
+ return this.keyValueEntries.has(key);
}
async getRecord(
@@ -383,14 +348,7 @@ export class KeyValueStoreClient extends BaseClient {
disableRedirect: s.boolean().optional(),
}).parse(options);
- // Check by id
- const existingStoreById = await findOrCacheKeyValueStoreByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingStoreById) {
- this.throwOnNonExisting(StorageTypes.KeyValueStore);
- }
-
- const storageEntry = existingStoreById.keyValueEntries.get(key);
+ const storageEntry = this.keyValueEntries.get(key);
if (!storageEntry) {
return undefined;
@@ -412,7 +370,7 @@ export class KeyValueStoreClient extends BaseClient {
record.value = maybeParseBody(record.value, record.contentType!);
}
- existingStoreById.updateTimestamps(false);
+ this.updateTimestamps(false);
return record;
}
@@ -433,13 +391,6 @@ export class KeyValueStoreClient extends BaseClient {
contentType: s.string().lengthGreaterThan(0).optional(),
}).parse(record);
- // Check by id
- const existingStoreById = await findOrCacheKeyValueStoreByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingStoreById) {
- this.throwOnNonExisting(StorageTypes.KeyValueStore);
- }
-
const { key } = record;
let { value, contentType } = record;
@@ -483,33 +434,26 @@ export class KeyValueStoreClient extends BaseClient {
const entry = createKeyValueStorageImplementation({
persistStorage: this.client.persistStorage,
- storeDirectory: existingStoreById.keyValueStoreDirectory,
- writeMetadata: existingStoreById.client.writeMetadata,
+ storeDirectory: this.keyValueStoreDirectory,
+ writeMetadata: this.client.writeMetadata,
logger: this.client.logger,
});
await entry.update(_record);
- existingStoreById.keyValueEntries.set(key, entry);
+ this.keyValueEntries.set(key, entry);
- existingStoreById.updateTimestamps(true);
+ this.updateTimestamps(true);
}
async deleteRecord(key: string): Promise<void> {
s.string().parse(key);
- // Check by id
- const existingStoreById = await findOrCacheKeyValueStoreByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingStoreById) {
- this.throwOnNonExisting(StorageTypes.KeyValueStore);
- }
-
- const entry = existingStoreById.keyValueEntries.get(key);
+ const entry = this.keyValueEntries.get(key);
if (entry) {
- existingStoreById.keyValueEntries.delete(key);
- existingStoreById.updateTimestamps(true);
+ this.keyValueEntries.delete(key);
+ this.updateTimestamps(true);
await entry.delete();
}
}
diff --git a/packages/memory-storage/src/resource-clients/request-queue.ts b/packages/memory-storage/src/resource-clients/request-queue.ts
index e4772c492..f60f4e805 100644
--- a/packages/memory-storage/src/resource-clients/request-queue.ts
+++ b/packages/memory-storage/src/resource-clients/request-queue.ts
@@ -7,7 +7,6 @@ import { s } from '@sapphire/shapeshift';
import { move } from 'fs-extra/esm';
import { scheduleBackgroundTask } from '../background-handler/index.js';
-import { findRequestQueueByPossibleId } from '../cache-helpers.js';
import { StorageTypes } from '../consts.js';
import { createRequestQueueStorageImplementation } from '../fs/request-queue/index.js';
import type { RequestQueueFileSystemEntry } from '../fs/request-queue/fs.js';
@@ -73,26 +72,8 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
this.client = options.client;
}
- private async getQueue(): Promise<RequestQueueClient> {
- const existingQueueById = await findRequestQueueByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingQueueById) {
- this.throwOnNonExisting(StorageTypes.RequestQueue);
- }
-
- existingQueueById.updateTimestamps(false);
-
- return existingQueueById;
- }
-
async getMetadata(): Promise<storage.RequestQueueInfo> {
- const found = await findRequestQueueByPossibleId(this.client, this.name ?? this.id);
-
- if (found) {
- found.updateTimestamps(false);
- return found.toRequestQueueInfo();
- }
-
+ this.updateTimestamps(false);
return this.toRequestQueueInfo();
}
@@ -106,15 +87,9 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
.passthrough()
.parse(newFields);
- const existingQueueById = await findRequestQueueByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingQueueById) {
- this.throwOnNonExisting(StorageTypes.RequestQueue);
- }
-
// Skip if no changes
if (!parsed.name) {
- return existingQueueById.toRequestQueueInfo();
+ return this.toRequestQueueInfo();
}
// Check that name is not in use already
@@ -126,21 +101,21 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
this.throwOnDuplicateEntry(StorageTypes.RequestQueue, 'name', parsed.name);
}
- existingQueueById.name = parsed.name;
+ this.name = parsed.name;
- const previousDir = existingQueueById.requestQueueDirectory;
+ const previousDir = this.requestQueueDirectory;
- existingQueueById.requestQueueDirectory = resolveWithinDirectory(
+ this.requestQueueDirectory = resolveWithinDirectory(
this.client.requestQueuesDirectory,
- parsed.name ?? existingQueueById.name ?? existingQueueById.id,
+ parsed.name ?? this.name ?? this.id,
);
- await move(previousDir, existingQueueById.requestQueueDirectory, { overwrite: true });
+ await move(previousDir, this.requestQueueDirectory, { overwrite: true });
// Update timestamps
- existingQueueById.updateTimestamps(true);
+ this.updateTimestamps(true);
- return existingQueueById.toRequestQueueInfo();
+ return this.toRequestQueueInfo();
}
async delete(): Promise<void> {
@@ -172,13 +147,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
})
.parse(options);
- const existingQueueById = await findRequestQueueByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingQueueById) {
- this.throwOnNonExisting(StorageTypes.RequestQueue);
- }
-
- existingQueueById.updateTimestamps(false);
+ this.updateTimestamps(false);
const items = [];
@@ -187,7 +156,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
// Tracks handled request IDs from `forefrontRequestIds` to be removed.
const handledForefrontIds = new Set<string>();
- for (const requestId of this.requestKeyIterator(existingQueueById)) {
+ for (const requestId of this.requestKeyIterator(this)) {
if (items.length === limit) {
break;
}
@@ -198,7 +167,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
seenRequestIds.add(requestId);
- const storageEntry = existingQueueById.requests.get(requestId)!;
+ const storageEntry = this.requests.get(requestId)!;
let { orderNo } = storageEntry;
let loaded: InternalRequest;
@@ -223,7 +192,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
return {
limit,
hadMultipleClients: false,
- queueModifiedAt: existingQueueById.modifiedAt,
+ queueModifiedAt: this.modifiedAt,
items: items.sort((a, b) => a.orderNo! - b.orderNo!).map(({ json }) => this._jsonToRequest(json)!),
};
}
@@ -236,7 +205,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
})
.parse(options);
- const queue = await this.getQueue();
+ this.updateTimestamps(false);
const start = Date.now();
const isLocked = (request: InternalRequest) =>
@@ -244,7 +213,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
const items = [];
- await queue.mutex.wait();
+ await this.mutex.wait();
try {
// Tracks processed request IDs to avoid duplicates (when a request is in both `forefrontRequestIds` and `requests`).
@@ -252,7 +221,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
// Tracks handled request IDs from `forefrontRequestIds` (to be all removed at once).
const handledForefrontIds = new Set<string>();
- for (const requestId of this.requestKeyIterator(queue)) {
+ for (const requestId of this.requestKeyIterator(this)) {
if (items.length === limit) {
break;
}
@@ -263,7 +232,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
seenRequestIds.add(requestId);
- const storageEntry = queue.requests.get(requestId)!;
+ const storageEntry = this.requests.get(requestId)!;
// This is set to null when the request has been handled, so we don't need to re-fetch from fs
if (storageEntry.orderNo === null) {
@@ -292,11 +261,11 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
limit,
lockSecs,
hadMultipleClients: false,
- queueModifiedAt: queue.modifiedAt,
+ queueModifiedAt: this.modifiedAt,
items: items.map(({ json }) => this._jsonToRequest(json)!),
};
} finally {
- queue.mutex.shift();
+ this.mutex.shift();
}
}
@@ -312,19 +281,19 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
})
.parse(options);
- const queue = await this.getQueue();
- const request = queue.requests.get(id);
+ this.updateTimestamps(false);
+ const request = this.requests.get(id);
const internalRequest = await request?.get();
if (!internalRequest) {
- throw new Error(`Request with ID ${id} not found in queue ${queue.name ?? queue.id}`);
+ throw new Error(`Request with ID ${id} not found in queue ${this.name ?? this.id}`);
}
const canProlong = (r: InternalRequest) => !!r.orderNo;
if (!canProlong(internalRequest)) {
- throw new Error(`Request with ID ${id} has already been handled in queue ${queue.name ?? queue.id}`);
+ throw new Error(`Request with ID ${id} has already been handled in queue ${this.name ?? this.id}`);
}
const unlockTimestamp = Math.abs(internalRequest.orderNo!) + lockSecs * 1000;
@@ -346,13 +315,13 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
})
.parse(options);
- const queue = await this.getQueue();
- const request = queue.requests.get(id);
+ this.updateTimestamps(false);
+ const request = this.requests.get(id);
const internalRequest = await request?.get();
if (!internalRequest) {
- throw new Error(`Request with ID ${id} not found in queue ${queue.name ?? queue.id}`);
+ throw new Error(`Request with ID ${id} not found in queue ${this.name ?? this.id}`);
}
const start = Date.now();
@@ -360,7 +329,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
// If there is no `orderNo` -> request was marked as handled
const isLocked = (r: InternalRequest) => r.orderNo && (r.orderNo > start || r.orderNo < -start);
if (!isLocked(internalRequest)) {
- throw new Error(`Request with ID ${id} is not locked in queue ${queue.name ?? queue.id}`);
+ throw new Error(`Request with ID ${id} is not locked in queue ${this.name ?? this.id}`);
}
internalRequest.orderNo = forefront ? -start : start;
@@ -376,20 +345,14 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
requestShapeWithoutId.parse(request);
requestOptionsShape.parse(options);
- const existingQueueById = await findRequestQueueByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingQueueById) {
- this.throwOnNonExisting(StorageTypes.RequestQueue);
- }
-
const requestModel = this._createInternalRequest(request, options.forefront);
- const existingRequestWithIdEntry = existingQueueById.requests.get(requestModel.id);
+ const existingRequestWithIdEntry = this.requests.get(requestModel.id);
// We already have the request present, so we return information about it
if (existingRequestWithIdEntry) {
const existingRequestWithId = await existingRequestWithIdEntry.get();
- existingQueueById.updateTimestamps(false);
+ this.updateTimestamps(false);
return {
requestId: existingRequestWithId.id,
@@ -399,20 +362,20 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
}
const newEntry = createRequestQueueStorageImplementation({
- persistStorage: existingQueueById.client.persistStorage,
+ persistStorage: this.client.persistStorage,
requestId: requestModel.id,
- storeDirectory: existingQueueById.requestQueueDirectory,
+ storeDirectory: this.requestQueueDirectory,
});
await newEntry.update(requestModel);
- existingQueueById.requests.set(requestModel.id, newEntry);
- existingQueueById.updateTimestamps(true);
+ this.requests.set(requestModel.id, newEntry);
+ this.updateTimestamps(true);
if (requestModel.orderNo) {
- existingQueueById.pendingRequestCount += 1;
+ this.pendingRequestCount += 1;
} else {
- existingQueueById.handledRequestCount += 1;
+ this.handledRequestCount += 1;
}
if (options.forefront) {
@@ -435,12 +398,6 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
batchRequestShapeWithoutId.parse(requests);
requestOptionsShape.parse(options);
- const existingQueueById = await findRequestQueueByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingQueueById) {
- this.throwOnNonExisting(StorageTypes.RequestQueue);
- }
-
const result: storage.BatchAddRequestsResult = {
processedRequests: [],
unprocessedRequests: [],
@@ -449,7 +406,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
for (const model of requests) {
const requestModel = this._createInternalRequest(model, options.forefront);
- const existingRequestWithIdEntry = existingQueueById.requests.get(requestModel.id);
+ const existingRequestWithIdEntry = this.requests.get(requestModel.id);
if (existingRequestWithIdEntry) {
const existingRequestWithId = await existingRequestWithIdEntry.get();
@@ -465,19 +422,19 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
}
const newEntry = createRequestQueueStorageImplementation({
- persistStorage: existingQueueById.client.persistStorage,
+ persistStorage: this.client.persistStorage,
requestId: requestModel.id,
- storeDirectory: existingQueueById.requestQueueDirectory,
+ storeDirectory: this.requestQueueDirectory,
});
await newEntry.update(requestModel);
- existingQueueById.requests.set(requestModel.id, newEntry);
+ this.requests.set(requestModel.id, newEntry);
if (requestModel.orderNo) {
- existingQueueById.pendingRequestCount += 1;
+ this.pendingRequestCount += 1;
} else {
- existingQueueById.handledRequestCount += 1;
+ this.handledRequestCount += 1;
}
if (options.forefront) {
@@ -494,15 +451,15 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
});
}
- existingQueueById.updateTimestamps(true);
+ this.updateTimestamps(true);
return result;
}
async getRequest(id: string): Promise<storage.RequestOptions | undefined> {
s.string().parse(id);
- const queue = await this.getQueue();
- const json = (await queue.requests.get(id)?.get())?.json;
+ this.updateTimestamps(false);
+ const json = (await this.requests.get(id)?.get())?.json;
return this._jsonToRequest(json);
}
@@ -513,18 +470,12 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
requestShape.parse(request);
requestOptionsShape.parse(options);
- const existingQueueById = await findRequestQueueByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingQueueById) {
- this.throwOnNonExisting(StorageTypes.RequestQueue);
- }
-
const requestModel = this._createInternalRequest(request, options.forefront);
// First we need to check the existing request to be
// able to return information about its handled state.
- const existingRequestEntry = existingQueueById.requests.get(requestModel.id);
+ const existingRequestEntry = this.requests.get(requestModel.id);
// Undefined means that the request is not present in the queue.
// We need to insert it, to behave the same as API.
@@ -535,30 +486,30 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
const existingRequest = await existingRequestEntry.get();
const newEntry = createRequestQueueStorageImplementation({
- persistStorage: existingQueueById.client.persistStorage,
+ persistStorage: this.client.persistStorage,
requestId: requestModel.id,
- storeDirectory: existingQueueById.requestQueueDirectory,
+ storeDirectory: this.requestQueueDirectory,
});
await newEntry.update(requestModel);
// When updating the request, we need to make sure that
// the handled counts are updated correctly in all cases.
- existingQueueById.requests.set(requestModel.id, newEntry);
+ this.requests.set(requestModel.id, newEntry);
const isRequestHandledStateChanging = typeof existingRequest.orderNo !== typeof requestModel.orderNo;
const requestWasHandledBeforeUpdate = existingRequest.orderNo === null;
const requestIsHandledAfterUpdate = requestModel.orderNo === null;
if (isRequestHandledStateChanging) {
- existingQueueById.pendingRequestCount += requestWasHandledBeforeUpdate ? 1 : -1;
+ this.pendingRequestCount += requestWasHandledBeforeUpdate ? 1 : -1;
}
if (isRequestHandledStateChanging && requestIsHandledAfterUpdate) {
- existingQueueById.handledRequestCount += 1;
+ this.handledRequestCount += 1;
}
- existingQueueById.updateTimestamps(true);
+ this.updateTimestamps(true);
if (options.forefront && !requestIsHandledAfterUpdate) {
this.forefrontRequestIds.push(requestModel.id);
@@ -572,24 +523,18 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
}
async deleteRequest(id: string): Promise<void> {
- const existingQueueById = await findRequestQueueByPossibleId(this.client, this.name ?? this.id);
-
- if (!existingQueueById) {
- this.throwOnNonExisting(StorageTypes.RequestQueue);
- }
-
- const entry = existingQueueById.requests.get(id);
+ const entry = this.requests.get(id);
if (entry) {
const request = await entry.get();
- existingQueueById.requests.delete(id);
- existingQueueById.updateTimestamps(true);
+ this.requests.delete(id);
+ this.updateTimestamps(true);
if (request.orderNo) {
- existingQueueById.pendingRequestCount -= 1;
+ this.pendingRequestCount -= 1;
} else {
- existingQueueById.handledRequestCount -= 1;
+ this.handledRequestCount -= 1;
}
await entry.delete();
diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.ts
index 687e74e40..f0d11453c 100644
--- a/packages/types/src/storages.ts
+++ b/packages/types/src/storages.ts
@@ -311,30 +311,40 @@ export type StorageIdentifier =
| { id?: never; name: string }
| { id?: never; name?: never };
+/**
+ * Options for opening a storage client. At most one of `id`, `name`, or `alias` may be provided.
+ * `alias` identifies a run-scoped unnamed storage (e.g. `'__default__'` for the default storage).
+ *
+ * TODO merge this into `StorageIdentifier` in https://github.com/apify/crawlee/issues/3074
+ */
+export type StorageClientOpenOptions = StorageIdentifier | { id?: never; name?: never; alias: string };
+
/**
* Options for creating a dataset client via {@apilink StorageClient.createDatasetClient}.
*/
-export type CreateDatasetClientOptions = StorageIdentifier;
+export type CreateDatasetClientOptions = StorageClientOpenOptions;
/**
* Options for creating a key-value store client via {@apilink StorageClient.createKeyValueStoreClient}.
*/
-export type CreateKeyValueStoreClientOptions = StorageIdentifier;
+export type CreateKeyValueStoreClientOptions = StorageClientOpenOptions;
/**
* Options for creating a request queue client via {@apilink StorageClient.createRequestQueueClient}.
*/
-export type CreateRequestQueueClientOptions = StorageIdentifier & {
+export type CreateRequestQueueClientOptions = StorageClientOpenOptions & {
/**
* Client key for request locking.
* TODO: This is an Apify-platform concern and should eventually be pushed down
* into the Apify SDK's client implementation (aligning with crawlee-python).
+ * https://github.com/apify/crawlee/issues/3328
*/
clientKey?: string;
/**
* Timeout in seconds for request queue operations.
* TODO: This is an Apify-platform concern and should eventually be pushed down
* into the Apify SDK's client implementation (aligning with crawlee-python).
+ * https://github.com/apify/crawlee/issues/3328
*/
timeoutSecs?: number;
};
@@ -380,6 +390,17 @@ export interface StorageClient {
* `KeyValueStore.open()`, and `RequestQueue.open()`.
*/
storageExists?(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
+ /**
+ * Return an opaque key that uniquely identifies this storage backend instance.
+ *
+ * The key is used by `StorageInstanceManager` to partition the storage cache per-backend,
+ * so that two storages with the same name but backed by different clients
+ * (e.g. a local `MemoryStorage` and a cloud `ApifyClient`) are cached as separate instances.
+ *
+ * When not provided, the fallback uses the client's constructor name, so different
+ * `StorageClient` implementations automatically get separate cache partitions.
+ */
+ getStorageClientCacheKey?(): string;
purge?(): Promise<void>;
teardown?(): Promise<void>;
setStatusMessage?(message: string, options?: SetStatusMessageOptions): Promise<void>;
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index b74507ee3..5f00c015b 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -7,7 +7,6 @@ import type { EnqueueLinksOptions, ErrorHandler, RequestHandler, RequestOptions,
import type { Session } from '@crawlee/basic';
import {
BasicCrawler,
- Configuration,
CriticalError,
defaultRoute,
EventType,
@@ -24,6 +23,7 @@ import {
SessionPool,
} from '@crawlee/basic';
import { RequestState, SessionPool, Statistics } from '@crawlee/core';
+import { MemoryStorage } from '@crawlee/memory-storage';
import type { ProxyInfo } from '@crawlee/types';
import type { Dictionary } from '@crawlee/utils';
import { RobotsTxtFile, sleep } from '@crawlee/utils';
@@ -685,7 +685,7 @@ describe('BasicCrawler', () => {
expect(await requestList.isEmpty()).toBe(true);
});
- test('should not retry requests with noRetry set to true ', async () => {
+ test('should not retry requests with noRetry set to true', async () => {
const noRetryRequest = new Request({ url: 'http://example.com/3' });
noRetryRequest.noRetry = true;
@@ -2593,9 +2593,14 @@ describe('BasicCrawler', () => {
await rm(`${tmpDir}/result.csv`);
});
- test("Crawlers with different Configurations don't share Datasets", async () => {
- const crawlerA = new BasicCrawler({ configuration: new Configuration({ persistStorage: false }) });
- const crawlerB = new BasicCrawler({ configuration: new Configuration({ persistStorage: false }) });
+ test("Crawlers with different storage clients don't share Datasets", async () => {
+ // Each crawler gets its own MemoryStorage with a different localDataDirectory,
+ // producing different clientCacheKeys and thus separate cache partitions.
+ const storageA = new MemoryStorage({ persistStorage: false, localDataDirectory: `${tmpDir}/storageA` });
+ const storageB = new MemoryStorage({ persistStorage: false, localDataDirectory: `${tmpDir}/storageB` });
+
+ const crawlerA = new BasicCrawler({ storageClient: storageA });
+ const crawlerB = new BasicCrawler({ storageClient: storageB });
await crawlerA.pushData(getPayload('A'));
await crawlerB.pushData(getPayload('B'));
@@ -2605,14 +2610,17 @@ describe('BasicCrawler', () => {
expect((await crawlerB.getData()).items).toEqual(getPayload('B'));
});
- test('Crawlers with different Configurations run separately', async () => {
+ test('Crawlers with different storage clients run separately', async () => {
+ const storageA = new MemoryStorage({ persistStorage: false, localDataDirectory: `${tmpDir}/storageA` });
+ const storageB = new MemoryStorage({ persistStorage: false, localDataDirectory: `${tmpDir}/storageB` });
+
const crawlerA = new BasicCrawler({
requestHandler: () => {},
- configuration: new Configuration({ persistStorage: false }),
+ storageClient: storageA,
});
const crawlerB = new BasicCrawler({
requestHandler: () => {},
- configuration: new Configuration({ persistStorage: false }),
+ storageClient: storageB,
});
await crawlerA.run([{ url: `http://${HOSTNAME}:${port}` }]);
@@ -2621,24 +2629,6 @@ describe('BasicCrawler', () => {
expect(crawlerA.stats.state.requestsFinished).toBe(1);
expect(crawlerB.stats.state.requestsFinished).toBe(1);
});
-
- test('Crawlers with different Configurations does not use global Configuration', async () => {
- const getGlobalConfigSpy = vitest.spyOn(Configuration, 'getGlobalConfig');
-
- const configA = new Configuration({ persistStorage: false });
- const crawlerA = new BasicCrawler({ requestHandler: () => {}, configuration: configA });
- const configB = new Configuration({ persistStorage: false });
- const crawlerB = new BasicCrawler({ requestHandler: () => {}, configuration: configB });
-
- await crawlerA.run([{ url: `http://${HOSTNAME}:${port}` }]);
- await crawlerB.run([{ url: `http://${HOSTNAME}:${port}` }]);
-
- expect(getGlobalConfigSpy.mock.calls.length).toBe(0);
- // @ts-expect-error Accessing protected property for testing
- expect(crawlerA.requestQueue?.config).toBe(configA);
- // @ts-expect-error Accessing protected property for testing
- expect(crawlerB.requestQueue?.config).toBe(configB);
- });
});
describe('schema validation on add', () => {
diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.ts
index 99de81377..1854aaf5b 100644
--- a/test/core/session_pool/session.test.ts
+++ b/test/core/session_pool/session.test.ts
@@ -3,7 +3,7 @@ import { ResponseWithUrl } from '@crawlee/http-client';
import { entries, sleep } from '@crawlee/utils';
import { CookieJar } from 'tough-cookie';
-describe('Session - testing session behaviour ', () => {
+describe('Session - testing session behaviour', () => {
let sessionPool: SessionPool;
let session: Session;
diff --git a/test/core/session_pool/session_pool.test.ts b/test/core/session_pool/session_pool.test.ts
index fcb596f18..2cc1782e5 100644
--- a/test/core/session_pool/session_pool.test.ts
+++ b/test/core/session_pool/session_pool.test.ts
@@ -359,7 +359,7 @@ describe('SessionPool - testing session pool', () => {
expect(session.id).toBe('test-session');
});
- test('should be able to add session instance and create new session with provided sessionOptions with addSession() ', async () => {
+ test('should be able to add session instance and create new session with provided sessionOptions with addSession()', async () => {
const session = new Session({ sessionPool, id: 'test-session-instance' });
await sessionPool.addSession(session);
diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.ts
index 31881f15e..bb24fb142 100644
--- a/test/core/storages/request_queue.test.ts
+++ b/test/core/storages/request_queue.test.ts
@@ -1006,7 +1006,7 @@ describe('RequestQueue v2', () => {
vitest.useRealTimers();
});
- test('prolongRequestLock works as expected ', async () => {
+ test('prolongRequestLock works as expected', async () => {
vitest.useFakeTimers();
const queue = await getEmptyQueue('prolong-request-lock');
await queue.addRequests(getUniqueRequests(1));
diff --git a/test/shared/MemoryStorageEmulator.ts b/test/shared/MemoryStorageEmulator.ts
index 991914832..ba1537b64 100644
--- a/test/shared/MemoryStorageEmulator.ts
+++ b/test/shared/MemoryStorageEmulator.ts
@@ -31,8 +31,7 @@ export class MemoryStorageEmulator extends StorageEmulator {
}
getDataset(id?: string) {
- const resolvedId = id ?? Configuration.getGlobalConfig().defaultDatasetId;
- return this.storage.createDatasetClient(resolvedId ? { id: resolvedId } : {});
+ return this.storage.createDatasetClient(id ? { id } : { alias: '__default__' });
}
async getDatasetItems(id?: string) {
@@ -41,8 +40,7 @@ export class MemoryStorageEmulator extends StorageEmulator {
}
getRequestQueue(id?: string) {
- const resolvedId = id ?? Configuration.getGlobalConfig().defaultRequestQueueId;
- return this.storage.createRequestQueueClient(resolvedId ? { id: resolvedId } : {});
+ return this.storage.createRequestQueueClient(id ? { id } : { alias: '__default__' });
}
async getRequestQueueItems(id?: string) {
@@ -52,8 +50,7 @@ export class MemoryStorageEmulator extends StorageEmulator {
}
getKeyValueStore(id?: string) {
- const resolvedId = id ?? Configuration.getGlobalConfig().defaultKeyValueStoreId;
- return this.storage.createKeyValueStoreClient(resolvedId ? { id: resolvedId } : {});
+ return this.storage.createKeyValueStoreClient(id ? { id } : { alias: '__default__' });
}
async getState() {
diff --git a/test/shared/StorageEmulator.ts b/test/shared/StorageEmulator.ts
index da62bcd54..0e9944aee 100644
--- a/test/shared/StorageEmulator.ts
+++ b/test/shared/StorageEmulator.ts
@@ -15,6 +15,6 @@ export abstract class StorageEmulator {
});
await Promise.all(promises);
- serviceLocator.clearStorageManagerCache();
+ serviceLocator.getStorageInstanceManager().clearCache();
}
}