Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: support alternate storage clients when opening storages (#1901)
Vlad Frangu661e550dcf36
diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.tsindex 3092f85b9..53c148972 100644--- a/packages/core/src/storages/dataset.ts+++ b/packages/core/src/storages/dataset.ts@@ -524,12 +524,17 @@ export class Dataset<Data extends Dictionary = Dictionary> {ow(datasetIdOrName, ow.optional.string);ow(options, ow.object.exactShape({config: ow.optional.object.instanceOf(Configuration),+ storageClient: ow.optional.object,}));+options.config ??= Configuration.getGlobalConfig();- await purgeDefaultStorages();+ options.storageClient ??= options.config.getStorageClient();++ await purgeDefaultStorages(options.config, options.storageClient);+const manager = StorageManager.getManager<Dataset<Data>>(this, options.config);- return manager.openStorage(datasetIdOrName, options.config.getStorageClient());+ return manager.openStorage(datasetIdOrName, options.storageClient);}/**diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.tsindex 6f1ed552f..7f17fea74 100644--- a/packages/core/src/storages/key_value_store.ts+++ b/packages/core/src/storages/key_value_store.ts@@ -409,11 +409,17 @@ export class KeyValueStore {ow(storeIdOrName, ow.optional.any(ow.string, ow.null));ow(options, ow.object.exactShape({config: ow.optional.object.instanceOf(Configuration),+ storageClient: ow.optional.object,}));- await purgeDefaultStorages();++ options.config ??= Configuration.getGlobalConfig();+ options.storageClient ??= options.config.getStorageClient();++ await purgeDefaultStorages(options.config, options.storageClient);+const manager = StorageManager.getManager(this, options.config);- return manager.openStorage(storeIdOrName);+ return manager.openStorage(storeIdOrName, options.storageClient);}/**diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.tsindex 304e17111..101df1fa6 100644--- a/packages/core/src/storages/request_queue.ts+++ b/packages/core/src/storages/request_queue.ts@@ -788,12 +788,17 @@ export class RequestQueue {ow(queueIdOrName, ow.optional.string);ow(options, ow.object.exactShape({config: ow.optional.object.instanceOf(Configuration),+ storageClient: ow.optional.object,}));- await purgeDefaultStorages();+ options.config ??= Configuration.getGlobalConfig();+ options.storageClient ??= options.config.getStorageClient();++ await purgeDefaultStorages(options.config, options.storageClient);+const manager = StorageManager.getManager(this, options.config);- return manager.openStorage(queueIdOrName);+ return manager.openStorage(queueIdOrName, options.storageClient);}}diff --git a/packages/core/src/storages/storage_manager.ts b/packages/core/src/storages/storage_manager.tsindex d220a4111..6d25a6cdf 100644--- a/packages/core/src/storages/storage_manager.ts+++ b/packages/core/src/storages/storage_manager.ts@@ -151,4 +151,9 @@ export interface StorageManagerOptions {* SDK configuration instance, defaults to the static register.*/config?: Configuration;++ /**+ * Optional storage client that should be used to open storages.+ */+ storageClient?: StorageClient;}diff --git a/packages/core/src/storages/utils.ts b/packages/core/src/storages/utils.tsindex 012eaa5e9..cdc0a5338 100644--- a/packages/core/src/storages/utils.ts+++ b/packages/core/src/storages/utils.ts@@ -15,12 +15,12 @@ import { KeyValueStore } from './key_value_store';* this method will make sure the storage is purged only once for a given execution context, so it is safe to call* it multiple times.*/-export async function purgeDefaultStorages(config = Configuration.getGlobalConfig()) {- const client = config.getStorageClient() as StorageClient & { __purged?: boolean };+export async function purgeDefaultStorages(config = Configuration.getGlobalConfig(), client: StorageClient = config.getStorageClient()) {+ const casted = client as StorageClient & { __purged?: boolean };- if (config.get('purgeOnStart') && !client.__purged) {- client.__purged = true;- await client.purge?.();+ if (config.get('purgeOnStart') && !casted.__purged) {+ casted.__purged = true;+ await casted.purge?.();}}diff --git a/packages/core/test/storages/open-storage-with-different-client-should-be-respected.test.ts b/packages/core/test/storages/open-storage-with-different-client-should-be-respected.test.tsnew file mode 100644index 000000000..36281829e--- /dev/null+++ b/packages/core/test/storages/open-storage-with-different-client-should-be-respected.test.ts@@ -0,0 +1,28 @@+import { MemoryStorage } from '@crawlee/memory-storage';+import { Configuration, RequestQueue } from 'crawlee';++const originalClient = Configuration.getStorageClient();+const newClient = new MemoryStorage({ persistStorage: false, writeMetadata: false });+Configuration.useStorageClient(newClient);++afterAll(() => {+ Configuration.useStorageClient(originalClient);+});++describe('Opening a storage with a different storage client should be respected', () => {+ test('opening a RequestQueue with default client from Configuration', async () => {+ const queue = await RequestQueue.open('test-rq-open-client-from-config');++ expect((queue.client as any).client).toBe(newClient);+ });++ test('opening a RequestQueue with a different client', async () => {+ const thirdClient = new MemoryStorage({ persistStorage: false, writeMetadata: false });+ // @ts-expect-error Using this to ensure the test/impl works+ thirdClient._name = 'third-client';++ const queue = await RequestQueue.open('test-rq-open-custom-client', { storageClient: thirdClient });++ expect((queue.client as any).client).toBe(thirdClient);+ });+});