Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Updating signature of openStorage function

Marek Trunkata3b5c1f489de
diff --git a/src/dataset.js b/src/dataset.js
index d8a257850..b66f5e19b 100644
--- a/src/dataset.js
+++ b/src/dataset.js
@@ -636,7 +636,8 @@ const getOrCreateDataset = (datasetIdOrName) => {
* @param {string} [datasetIdOrName]
* ID or name of the dataset to be opened. If `null` or `undefined`,
* the function returns the default dataset associated with the actor run.
- * @param {boolean} [enforceCloud=false]
+ * @param {object} [options]
+ * @param {boolean} [options.forceCloud=false]
* If set to `true` then enforces cloud storage usage. This way is possible to combine local and
* cloud storages when devloping locally.
* @returns {Promise<Dataset>}
@@ -644,11 +645,15 @@ const getOrCreateDataset = (datasetIdOrName) => {
* @name openDataset
* @function
*/
-export const openDataset = (datasetIdOrName, enforceCloud = false) => {
+export const openDataset = (datasetIdOrName, options = {}) => {
checkParamOrThrow(datasetIdOrName, 'datasetIdOrName', 'Maybe String');
+ checkParamOrThrow(options, 'options', 'Object');
ensureTokenOrLocalStorageEnvExists('dataset');
- return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !enforceCloud
+ const { forceCloud = false } = options;
+ checkParamOrThrow(forceCloud, 'options.forceCloud', 'Boolean');
+
+ return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !forceCloud
? openLocalStorage(datasetIdOrName, ENV_VARS.DEFAULT_DATASET_ID, DatasetLocal, datasetsCache)
: openRemoteStorage(datasetIdOrName, ENV_VARS.DEFAULT_DATASET_ID, Dataset, datasetsCache, getOrCreateDataset);
};
diff --git a/src/key_value_store.js b/src/key_value_store.js
index 545a0c451..53bd68900 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -454,7 +454,8 @@ const getOrCreateKeyValueStore = (storeIdOrName) => {
* @param {string} [storeIdOrName]
* ID or name of the key-value store to be opened. If `null` or `undefined`,
* the function returns the default key-value store associated with the actor run.
- * @param {boolean} [enforceCloud=false]
+ * @param {object} [options]
+ * @param {boolean} [options.forceCloud=false]
* If set to `true` then enforces cloud storage usage. This way is possible to combine local and
* cloud storages when devloping locally.
* @returns {Promise<KeyValueStore>}
@@ -462,11 +463,15 @@ const getOrCreateKeyValueStore = (storeIdOrName) => {
* @name openKeyValueStore
* @function
*/
-export const openKeyValueStore = (storeIdOrName, enforceCloud = false) => {
+export const openKeyValueStore = (storeIdOrName, options = {}) => {
checkParamOrThrow(storeIdOrName, 'storeIdOrName', 'Maybe String');
+ checkParamOrThrow(options, 'options', 'Object');
ensureTokenOrLocalStorageEnvExists('key value store');
- return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !enforceCloud
+ const { forceCloud = false } = options;
+ checkParamOrThrow(forceCloud, 'options.forceCloud', 'Boolean');
+
+ return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !forceCloud
? openLocalStorage(storeIdOrName, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, KeyValueStoreLocal, storesCache)
: openRemoteStorage(storeIdOrName, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, KeyValueStore, storesCache, getOrCreateKeyValueStore);
};
diff --git a/src/request_queue.js b/src/request_queue.js
index cf31fc6a1..e1792751e 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -852,18 +852,23 @@ const getOrCreateQueue = (queueIdOrName) => {
* @param {string} [queueIdOrName]
* ID or name of the request queue to be opened. If `null` or `undefined`,
* the function returns the default request queue associated with the actor run.
- * @param {boolean} [enforceCloud=false]
+ * @param {object} [options]
+ * @param {boolean} [options.forceCloud=false]
* If set to `true` then enforces cloud storage usage. This way is possible to combine local and
* cloud storages when devloping locally.
* @returns {Promise<RequestQueue>}
* @memberof module:Apify
* @name openRequestQueue
*/
-export const openRequestQueue = (queueIdOrName, enforceCloud = false) => {
+export const openRequestQueue = (queueIdOrName, options = {}) => {
checkParamOrThrow(queueIdOrName, 'queueIdOrName', 'Maybe String');
+ checkParamOrThrow(options, 'options', 'Object');
ensureTokenOrLocalStorageEnvExists('request queue');
- return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !enforceCloud
+ const { forceCloud = false } = options;
+ checkParamOrThrow(forceCloud, 'options.forceCloud', 'Boolean');
+
+ return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !forceCloud
? openLocalStorage(queueIdOrName, ENV_VARS.DEFAULT_REQUEST_QUEUE_ID, RequestQueueLocal, queuesCache)
: openRemoteStorage(queueIdOrName, ENV_VARS.DEFAULT_REQUEST_QUEUE_ID, RequestQueue, queuesCache, getOrCreateQueue);
};
diff --git a/test/dataset.js b/test/dataset.js
index 15edc40af..8a2dc5143 100644
--- a/test/dataset.js
+++ b/test/dataset.js
@@ -733,7 +733,7 @@ describe('dataset', () => {
mock.expects('openLocalStorage').once();
Apify.openDataset('xxx');
mock.expects('openRemoteStorage').once();
- Apify.openDataset('xxx', true);
+ Apify.openDataset('xxx', { forceCloud: true });
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
process.env[ENV_VARS.TOKEN] = 'xxx';
diff --git a/test/key_value_store.js b/test/key_value_store.js
index f6056aaed..1be952d0d 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -171,7 +171,7 @@ describe('KeyValueStore', () => {
mock.expects('openLocalStorage').once();
Apify.openKeyValueStore('xxx');
mock.expects('openRemoteStorage').once();
- Apify.openKeyValueStore('xxx', true);
+ Apify.openKeyValueStore('xxx', { forceCloud: true });
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
process.env[ENV_VARS.TOKEN] = 'xxx';
diff --git a/test/request_queue.js b/test/request_queue.js
index 3b55e4a62..ae6870f68 100644
--- a/test/request_queue.js
+++ b/test/request_queue.js
@@ -551,7 +551,7 @@ describe('RequestQueue', () => {
mock.expects('openLocalStorage').once();
Apify.openRequestQueue('xxx');
mock.expects('openRemoteStorage').once();
- Apify.openRequestQueue('xxx', true);
+ Apify.openRequestQueue('xxx', { forceCloud: true });
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
process.env[ENV_VARS.TOKEN] = 'xxx';