Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Adding switch enforcing remote store to be used
Marek Trunkat12b18dea3181
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 15aa4a220..f3e05ce1b 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -3,6 +3,7 @@ xxx- Fixed `RequestQueue.addRequest()` to gracefully handle invalid URLs- Renamed `RequestOperationInfo` to `QueueOperationInfo`- Added `request` field to `QueueOperationInfo`+- Added option enforcing cloud storage to be used in `openKeyValueStore()`, `openDataset()` and `openRequestQueue()`0.8.18 / 2018-10-30===================diff --git a/src/dataset.js b/src/dataset.jsindex e8e7bf10d..d8a257850 100644--- a/src/dataset.js+++ b/src/dataset.js@@ -636,16 +636,19 @@ const getOrCreateDataset = (datasetIdOrName) => {* @param {string} [datasetIdOrName]* ID or name of the dataset to be opened. If `null` or `undefined`,* the function returns the default dataset associated with the actor run.+ * @param {boolean} [enforceCloud=false]+ * If set to `true` then enforces cloud storage usage. This way is possible to combine local and+ * cloud storages when devloping locally.* @returns {Promise<Dataset>}* @memberof module:Apify* @name openDataset* @function*/-export const openDataset = (datasetIdOrName) => {+export const openDataset = (datasetIdOrName, enforceCloud = false) => {checkParamOrThrow(datasetIdOrName, 'datasetIdOrName', 'Maybe String');ensureTokenOrLocalStorageEnvExists('dataset');- return process.env[ENV_VARS.LOCAL_STORAGE_DIR]+ return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !enforceCloud? openLocalStorage(datasetIdOrName, ENV_VARS.DEFAULT_DATASET_ID, DatasetLocal, datasetsCache): openRemoteStorage(datasetIdOrName, ENV_VARS.DEFAULT_DATASET_ID, Dataset, datasetsCache, getOrCreateDataset);};diff --git a/src/key_value_store.js b/src/key_value_store.jsindex a0502ba32..545a0c451 100644--- a/src/key_value_store.js+++ b/src/key_value_store.js@@ -454,16 +454,19 @@ const getOrCreateKeyValueStore = (storeIdOrName) => {* @param {string} [storeIdOrName]* ID or name of the key-value store to be opened. If `null` or `undefined`,* the function returns the default key-value store associated with the actor run.+ * @param {boolean} [enforceCloud=false]+ * If set to `true` then enforces cloud storage usage. This way is possible to combine local and+ * cloud storages when devloping locally.* @returns {Promise<KeyValueStore>}* @memberof module:Apify* @name openKeyValueStore* @function*/-export const openKeyValueStore = (storeIdOrName) => {+export const openKeyValueStore = (storeIdOrName, enforceCloud = false) => {checkParamOrThrow(storeIdOrName, 'storeIdOrName', 'Maybe String');ensureTokenOrLocalStorageEnvExists('key value store');- return process.env[ENV_VARS.LOCAL_STORAGE_DIR]+ return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !enforceCloud? openLocalStorage(storeIdOrName, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, KeyValueStoreLocal, storesCache): openRemoteStorage(storeIdOrName, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, KeyValueStore, storesCache, getOrCreateKeyValueStore);};diff --git a/src/request_queue.js b/src/request_queue.jsindex ba8312101..cf31fc6a1 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -852,15 +852,18 @@ const getOrCreateQueue = (queueIdOrName) => {* @param {string} [queueIdOrName]* ID or name of the request queue to be opened. If `null` or `undefined`,* the function returns the default request queue associated with the actor run.+ * @param {boolean} [enforceCloud=false]+ * If set to `true` then enforces cloud storage usage. This way is possible to combine local and+ * cloud storages when devloping locally.* @returns {Promise<RequestQueue>}* @memberof module:Apify* @name openRequestQueue*/-export const openRequestQueue = (queueIdOrName) => {+export const openRequestQueue = (queueIdOrName, enforceCloud = false) => {checkParamOrThrow(queueIdOrName, 'queueIdOrName', 'Maybe String');ensureTokenOrLocalStorageEnvExists('request queue');- return process.env[ENV_VARS.LOCAL_STORAGE_DIR]+ return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !enforceCloud? openLocalStorage(queueIdOrName, ENV_VARS.DEFAULT_REQUEST_QUEUE_ID, RequestQueueLocal, queuesCache): openRemoteStorage(queueIdOrName, ENV_VARS.DEFAULT_REQUEST_QUEUE_ID, RequestQueue, queuesCache, getOrCreateQueue);};diff --git a/test/dataset.js b/test/dataset.jsindex e6f3698f9..15edc40af 100644--- a/test/dataset.js+++ b/test/dataset.js@@ -730,6 +730,11 @@ describe('dataset', () => {mock.expects('openLocalStorage').once();Apify.openDataset();+ mock.expects('openLocalStorage').once();+ Apify.openDataset('xxx');+ mock.expects('openRemoteStorage').once();+ Apify.openDataset('xxx', true);+delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];process.env[ENV_VARS.TOKEN] = 'xxx';diff --git a/test/key_value_store.js b/test/key_value_store.jsindex fa691b15b..f6056aaed 100644--- a/test/key_value_store.js+++ b/test/key_value_store.js@@ -168,6 +168,11 @@ describe('KeyValueStore', () => {mock.expects('openLocalStorage').once();await Apify.openKeyValueStore();+ mock.expects('openLocalStorage').once();+ Apify.openKeyValueStore('xxx');+ mock.expects('openRemoteStorage').once();+ Apify.openKeyValueStore('xxx', true);+delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];process.env[ENV_VARS.TOKEN] = 'xxx';diff --git a/test/request_queue.js b/test/request_queue.jsindex c90953afb..3b55e4a62 100644--- a/test/request_queue.js+++ b/test/request_queue.js@@ -548,6 +548,11 @@ describe('RequestQueue', () => {mock.expects('openLocalStorage').once();Apify.openRequestQueue();+ mock.expects('openLocalStorage').once();+ Apify.openRequestQueue('xxx');+ mock.expects('openRemoteStorage').once();+ Apify.openRequestQueue('xxx', true);+delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];process.env[ENV_VARS.TOKEN] = 'xxx';