Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Adding switch enforcing remote store to be used

Marek Trunkat12b18dea3181
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 15aa4a220..f3e05ce1b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,7 @@ xxx
- Fixed `RequestQueue.addRequest()` to gracefully handle invalid URLs
- Renamed `RequestOperationInfo` to `QueueOperationInfo`
- Added `request` field to `QueueOperationInfo`
+- Added option enforcing cloud storage to be used in `openKeyValueStore()`, `openDataset()` and `openRequestQueue()`
0.8.18 / 2018-10-30
===================
diff --git a/src/dataset.js b/src/dataset.js
index e8e7bf10d..d8a257850 100644
--- a/src/dataset.js
+++ b/src/dataset.js
@@ -636,16 +636,19 @@ const getOrCreateDataset = (datasetIdOrName) => {
* @param {string} [datasetIdOrName]
* ID or name of the dataset to be opened. If `null` or `undefined`,
* the function returns the default dataset associated with the actor run.
+ * @param {boolean} [enforceCloud=false]
+ * If set to `true` then enforces cloud storage usage. This way is possible to combine local and
+ * cloud storages when devloping locally.
* @returns {Promise<Dataset>}
* @memberof module:Apify
* @name openDataset
* @function
*/
-export const openDataset = (datasetIdOrName) => {
+export const openDataset = (datasetIdOrName, enforceCloud = false) => {
checkParamOrThrow(datasetIdOrName, 'datasetIdOrName', 'Maybe String');
ensureTokenOrLocalStorageEnvExists('dataset');
- return process.env[ENV_VARS.LOCAL_STORAGE_DIR]
+ return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !enforceCloud
? openLocalStorage(datasetIdOrName, ENV_VARS.DEFAULT_DATASET_ID, DatasetLocal, datasetsCache)
: openRemoteStorage(datasetIdOrName, ENV_VARS.DEFAULT_DATASET_ID, Dataset, datasetsCache, getOrCreateDataset);
};
diff --git a/src/key_value_store.js b/src/key_value_store.js
index a0502ba32..545a0c451 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -454,16 +454,19 @@ const getOrCreateKeyValueStore = (storeIdOrName) => {
* @param {string} [storeIdOrName]
* ID or name of the key-value store to be opened. If `null` or `undefined`,
* the function returns the default key-value store associated with the actor run.
+ * @param {boolean} [enforceCloud=false]
+ * If set to `true` then enforces cloud storage usage. This way is possible to combine local and
+ * cloud storages when devloping locally.
* @returns {Promise<KeyValueStore>}
* @memberof module:Apify
* @name openKeyValueStore
* @function
*/
-export const openKeyValueStore = (storeIdOrName) => {
+export const openKeyValueStore = (storeIdOrName, enforceCloud = false) => {
checkParamOrThrow(storeIdOrName, 'storeIdOrName', 'Maybe String');
ensureTokenOrLocalStorageEnvExists('key value store');
- return process.env[ENV_VARS.LOCAL_STORAGE_DIR]
+ return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !enforceCloud
? openLocalStorage(storeIdOrName, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, KeyValueStoreLocal, storesCache)
: openRemoteStorage(storeIdOrName, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, KeyValueStore, storesCache, getOrCreateKeyValueStore);
};
diff --git a/src/request_queue.js b/src/request_queue.js
index ba8312101..cf31fc6a1 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -852,15 +852,18 @@ const getOrCreateQueue = (queueIdOrName) => {
* @param {string} [queueIdOrName]
* ID or name of the request queue to be opened. If `null` or `undefined`,
* the function returns the default request queue associated with the actor run.
+ * @param {boolean} [enforceCloud=false]
+ * If set to `true` then enforces cloud storage usage. This way is possible to combine local and
+ * cloud storages when devloping locally.
* @returns {Promise<RequestQueue>}
* @memberof module:Apify
* @name openRequestQueue
*/
-export const openRequestQueue = (queueIdOrName) => {
+export const openRequestQueue = (queueIdOrName, enforceCloud = false) => {
checkParamOrThrow(queueIdOrName, 'queueIdOrName', 'Maybe String');
ensureTokenOrLocalStorageEnvExists('request queue');
- return process.env[ENV_VARS.LOCAL_STORAGE_DIR]
+ return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !enforceCloud
? openLocalStorage(queueIdOrName, ENV_VARS.DEFAULT_REQUEST_QUEUE_ID, RequestQueueLocal, queuesCache)
: openRemoteStorage(queueIdOrName, ENV_VARS.DEFAULT_REQUEST_QUEUE_ID, RequestQueue, queuesCache, getOrCreateQueue);
};
diff --git a/test/dataset.js b/test/dataset.js
index e6f3698f9..15edc40af 100644
--- a/test/dataset.js
+++ b/test/dataset.js
@@ -730,6 +730,11 @@ describe('dataset', () => {
mock.expects('openLocalStorage').once();
Apify.openDataset();
+ mock.expects('openLocalStorage').once();
+ Apify.openDataset('xxx');
+ mock.expects('openRemoteStorage').once();
+ Apify.openDataset('xxx', true);
+
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
process.env[ENV_VARS.TOKEN] = 'xxx';
diff --git a/test/key_value_store.js b/test/key_value_store.js
index fa691b15b..f6056aaed 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -168,6 +168,11 @@ describe('KeyValueStore', () => {
mock.expects('openLocalStorage').once();
await Apify.openKeyValueStore();
+ mock.expects('openLocalStorage').once();
+ Apify.openKeyValueStore('xxx');
+ mock.expects('openRemoteStorage').once();
+ Apify.openKeyValueStore('xxx', true);
+
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
process.env[ENV_VARS.TOKEN] = 'xxx';
diff --git a/test/request_queue.js b/test/request_queue.js
index c90953afb..3b55e4a62 100644
--- a/test/request_queue.js
+++ b/test/request_queue.js
@@ -548,6 +548,11 @@ describe('RequestQueue', () => {
mock.expects('openLocalStorage').once();
Apify.openRequestQueue();
+ mock.expects('openLocalStorage').once();
+ Apify.openRequestQueue('xxx');
+ mock.expects('openRemoteStorage').once();
+ Apify.openRequestQueue('xxx', true);
+
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
process.env[ENV_VARS.TOKEN] = 'xxx';