Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: Fix repeated Storage.open() using the legacy idOrName parameter (#3808)

Jan Buchar43effa186da0
diff --git a/packages/fs-storage/src/cache-helpers.ts b/packages/fs-storage/src/cache-helpers.ts
index d4e3cfc51..640c413b0 100644
--- a/packages/fs-storage/src/cache-helpers.ts
+++ b/packages/fs-storage/src/cache-helpers.ts
@@ -12,6 +12,39 @@ import { type FileSystemStorageClient } from './file-system-storage.js';
const uuidRegex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i;
+/**
+ * A named storage is persisted in a directory named after its *name*, while its *id* lives inside
+ * that directory's `__metadata__.json`. Looking a storage up by id therefore cannot rely on the
+ * directory name — this scans the sibling directories under `baseDirectory` and returns the name of
+ * the directory whose metadata `id` matches, so callers can resolve `Dataset.open("<id>")` (and
+ * `storageExists("<id>")`) for a storage that was originally opened by name.
+ */
+async function resolveDirNameByMetadataId(baseDirectory: string, id: string): Promise<string | undefined> {
+ let directories;
+ try {
+ directories = await opendir(baseDirectory);
+ } catch {
+ return undefined;
+ }
+
+ for await (const directory of directories) {
+ if (!directory.isDirectory()) {
+ continue;
+ }
+
+ try {
+ const fileContent = await readFile(resolve(baseDirectory, directory.name, '__metadata__.json'), 'utf8');
+ if ((JSON.parse(fileContent) as { id?: string }).id === id) {
+ return directory.name;
+ }
+ } catch {
+ // No metadata file (or unreadable) — this directory can't be matched by id.
+ }
+ }
+
+ return undefined;
+}
+
export async function findOrCacheDatasetByPossibleId(client: FileSystemStorageClient, entryNameOrId: string) {
// First check memory cache — match by id, name, or directoryName (which covers alias lookups)
const found = client.datasetClientCache.find(
@@ -25,13 +58,19 @@ export async function findOrCacheDatasetByPossibleId(client: FileSystemStorageCl
return found;
}
- const datasetDir = resolveWithinDirectory(client.datasetsDirectory, entryNameOrId);
+ let datasetDir = resolveWithinDirectory(client.datasetsDirectory, entryNameOrId);
try {
// Check if directory exists
await access(datasetDir);
} catch {
- return undefined;
+ // No directory named after the string — it may be an id of a storage opened by name, whose
+ // directory is named after the name. Fall back to matching the id inside the metadata files.
+ const dirName = await resolveDirNameByMetadataId(client.datasetsDirectory, entryNameOrId);
+ if (dirName === undefined) {
+ return undefined;
+ }
+ datasetDir = resolve(client.datasetsDirectory, dirName);
}
// Access the dataset folder
@@ -130,13 +169,19 @@ export async function findOrCacheKeyValueStoreByPossibleId(client: FileSystemSto
return found;
}
- const keyValueStoreDir = resolveWithinDirectory(client.keyValueStoresDirectory, entryNameOrId);
+ let keyValueStoreDir = resolveWithinDirectory(client.keyValueStoresDirectory, entryNameOrId);
try {
// Check if directory exists
await access(keyValueStoreDir);
} catch {
- return undefined;
+ // No directory named after the string — it may be an id of a storage opened by name, whose
+ // directory is named after the name. Fall back to matching the id inside the metadata files.
+ const dirName = await resolveDirNameByMetadataId(client.keyValueStoresDirectory, entryNameOrId);
+ if (dirName === undefined) {
+ return undefined;
+ }
+ keyValueStoreDir = resolve(client.keyValueStoresDirectory, dirName);
}
// Access the key value store folder
@@ -286,13 +331,19 @@ export async function findRequestQueueByPossibleId(client: FileSystemStorageClie
return found;
}
- const requestQueueDir = resolveWithinDirectory(client.requestQueuesDirectory, entryNameOrId);
+ let requestQueueDir = resolveWithinDirectory(client.requestQueuesDirectory, entryNameOrId);
try {
// Check if directory exists
await access(requestQueueDir);
} catch {
- return undefined;
+ // No directory named after the string — it may be an id of a storage opened by name, whose
+ // directory is named after the name. Fall back to matching the id inside the metadata files.
+ const dirName = await resolveDirNameByMetadataId(client.requestQueuesDirectory, entryNameOrId);
+ if (dirName === undefined) {
+ return undefined;
+ }
+ requestQueueDir = resolve(client.requestQueuesDirectory, dirName);
}
// Access the request queue folder
diff --git a/packages/fs-storage/src/file-system-storage.ts b/packages/fs-storage/src/file-system-storage.ts
index ac427c9d3..fc9e41ff7 100644
--- a/packages/fs-storage/src/file-system-storage.ts
+++ b/packages/fs-storage/src/file-system-storage.ts
@@ -1,5 +1,5 @@
/* eslint-disable import/no-duplicates */
-import { access, readdir, rm } from 'node:fs/promises';
+import { readdir, rm } from 'node:fs/promises';
import { resolve } from 'node:path';
import type * as storage from '@crawlee/types';
@@ -213,20 +213,23 @@ export class FileSystemStorageClient implements storage.StorageClient {
async storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean> {
let clients: { id: string }[];
- let baseDir: string;
+ let findByPossibleId: (
+ client: FileSystemStorageClient,
+ entryNameOrId: string,
+ ) => Promise<{ id: string } | undefined>;
switch (type) {
case 'Dataset':
clients = this.datasetClientCache;
- baseDir = this.datasetsDirectory;
+ findByPossibleId = findOrCacheDatasetByPossibleId;
break;
case 'KeyValueStore':
clients = this.keyValueStoreCache;
- baseDir = this.keyValueStoresDirectory;
+ findByPossibleId = findOrCacheKeyValueStoreByPossibleId;
break;
case 'RequestQueue':
clients = this.requestQueueCache;
- baseDir = this.requestQueuesDirectory;
+ findByPossibleId = findRequestQueueByPossibleId;
break;
default:
return false;
@@ -237,25 +240,14 @@ export class FileSystemStorageClient implements storage.StorageClient {
return true;
}
- // Check if a directory with that ID exists on disk.
- // Only consider directories whose name matches the queried ID — this avoids
- // false positives for alias-created directories (e.g. a directory named 'asdf'
- // created via `{ alias: 'asdf' }` should not make `storageExists('asdf')` return true,
- // since the actual storage ID is a UUID, not the alias string).
- try {
- await access(resolve(baseDir, id));
-
- // If the directory exists but a cached client already owns this directory
- // under a different ID, this is not a match.
- const cachedClients = clients as { id: string; directoryName?: string }[];
- if (cachedClients.some((store) => store.directoryName === id && store.id !== id)) {
- return false;
- }
-
- return true;
- } catch {
- return false;
- }
+ // Resolve any on-disk directory that matches the queried string (by directory name).
+ // The directory is named after the storage's `name ?? id`, so a match does not by itself
+ // prove that the string is the storage's ID — it could just be its name (or an alias).
+ // Read the actual stored metadata and only treat the string as an ID if it equals the
+ // resolved storage's real ID. This prevents a named storage opened via the legacy
+ // `open(idOrName)` signature from being re-resolved as `{ id: name }` on a subsequent run.
+ const resolved = await findByPossibleId(this, id);
+ return resolved?.id === id;
}
async setStatusMessage(message: string, options: storage.SetStatusMessageOptions = {}): Promise<void> {
diff --git a/test/core/storages/storage_aliases.test.ts b/test/core/storages/storage_aliases.test.ts
index 907cb2bf0..db6c9b72b 100644
--- a/test/core/storages/storage_aliases.test.ts
+++ b/test/core/storages/storage_aliases.test.ts
@@ -170,6 +170,56 @@ describe('storage aliases', () => {
/Cannot open storage with name "on-disk" because an alias storage with the same identifier already exists\. If you meant to open the alias storage, use \{ alias: "on-disk" \} instead\./,
);
});
+
+ test('legacy open(idOrName) re-opens the named storage on a subsequent run', async () => {
+ const first = await Dataset.open('named-storage');
+ await first.pushData({ run: 1 });
+
+ // Simulate a fresh process: only the on-disk directory survives.
+ serviceLocator.reset();
+ const client = new FileSystemStorageClient({ localDataDirectory: localStorageDir });
+ serviceLocator.setStorageClient(client);
+
+ // 'named-storage' is the storage's name, not its id, so it must not be resolved as an id.
+ await expect(client.storageExists('named-storage', 'Dataset')).resolves.toBe(false);
+
+ const second = await Dataset.open('named-storage');
+ expect(second.name).toBe('named-storage');
+ await expect(second.getData()).resolves.toMatchObject({ items: [{ run: 1 }] });
+ });
+
+ test('a storage opened by name can be re-opened by its auto-assigned id after a reset', async () => {
+ // `writeMetadata` persists the auto-assigned id to disk so it survives a reset. The
+ // directory is named after the storage's name, not its id.
+ serviceLocator.reset();
+ const firstClient = new FileSystemStorageClient({
+ localDataDirectory: localStorageDir,
+ writeMetadata: true,
+ });
+ serviceLocator.setStorageClient(firstClient);
+
+ const created = await Dataset.open('some-name');
+ const assignedId = created.id;
+ expect(assignedId).not.toBe('some-name');
+ await created.pushData({ run: 1 });
+
+ // Flush background metadata writes to disk (as a real process shutdown would).
+ await firstClient.teardown();
+
+ // Simulate a fresh process: only the on-disk directory survives.
+ serviceLocator.reset();
+ const client = new FileSystemStorageClient({ localDataDirectory: localStorageDir, writeMetadata: true });
+ serviceLocator.setStorageClient(client);
+
+ // Opening by the persisted id must find the storage, even though its directory is named
+ // after the name.
+ await expect(client.storageExists(assignedId, 'Dataset')).resolves.toBe(true);
+
+ const reopened = await Dataset.open(assignedId);
+ expect(reopened.id).toBe(assignedId);
+ expect(reopened.name).toBe('some-name');
+ await expect(reopened.getData()).resolves.toMatchObject({ items: [{ run: 1 }] });
+ });
});
describe('resolveStorageIdentifier', () => {