Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Refactoring ENV variables

Marek Trunkatdb0977db2a04
diff --git a/src/actor.js b/src/actor.js
index 886ddbb81..da333ec89 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -3,7 +3,7 @@ import Promise from 'bluebird';
import log from 'apify-shared/log';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { APIFY_PROXY_VALUE_REGEX } from 'apify-shared/regexs';
-import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES, ACT_TASK_STATUSES, DEFAULT_PROXY_HOSTNAME, DEFAULT_PROXY_PORT } from './constants';
+import { ENV_VARS, LOCAL_ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES, ACT_TASK_STATUSES } from './constants';
import { initializeEvents, stopEvents } from './events';
import { newPromise, apifyClient, addCharsetToContentType } from './utils';
import { maybeStringify } from './key_value_store';
@@ -425,8 +425,8 @@ export const getApifyProxyUrl = (opts = {}) => {
groups,
session,
password = process.env[ENV_VARS.PROXY_PASSWORD],
- hostname = process.env[ENV_VARS.PROXY_HOSTNAME] || DEFAULT_PROXY_HOSTNAME,
- port = parseInt(process.env[ENV_VARS.PROXY_PORT], 10) || DEFAULT_PROXY_PORT,
+ hostname = process.env[ENV_VARS.PROXY_HOSTNAME] || LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME],
+ port = parseInt(process.env[ENV_VARS.PROXY_PORT], 10) || LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT],
// This is used only internaly. Some other function calling this function use different naming for groups and session
// parameters so we need to override this in error messages.
diff --git a/src/dataset.js b/src/dataset.js
index 2c452b629..4e56d261b 100644
--- a/src/dataset.js
+++ b/src/dataset.js
@@ -6,10 +6,10 @@ import Promise from 'bluebird';
import { leftpad } from 'apify-shared/utilities';
import LruCache from 'apify-shared/lru_cache';
import { checkParamOrThrow } from 'apify-client/build/utils';
-import { ENV_VARS, LOCAL_EMULATION_SUBDIRS, MAX_PAYLOAD_SIZE_BYTES } from './constants';
-import { apifyClient, ensureDirExists } from './utils';
+import { ENV_VARS, LOCAL_STORAGE_SUBDIRS, MAX_PAYLOAD_SIZE_BYTES } from './constants';
+import { apifyClient, ensureDirExists, openRemoteStorage, openLocalStorage } from './utils';
-export const LOCAL_EMULATION_SUBDIR = LOCAL_EMULATION_SUBDIRS.datasets;
+export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.datasets;
export const LOCAL_FILENAME_DIGITS = 9;
export const LOCAL_GET_ITEMS_DEFAULT_LIMIT = 250000;
const MAX_OPENED_STORES = 1000;
@@ -364,7 +364,7 @@ export class DatasetLocal {
checkParamOrThrow(datasetId, 'datasetId', 'String');
checkParamOrThrow(localEmulationDir, 'localEmulationDir', 'String');
- this.localEmulationPath = path.resolve(path.join(localEmulationDir, LOCAL_EMULATION_SUBDIR, datasetId));
+ this.localEmulationPath = path.resolve(path.join(localEmulationDir, LOCAL_STORAGE_SUBDIR, datasetId));
this.counter = null;
this.datasetId = datasetId;
this.initializationPromise = this._initialize();
@@ -574,7 +574,7 @@ const getOrCreateDataset = (datasetIdOrName) => {
* ]);
* ```
*
- * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is set, the result of this function
+ * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the result of this function
* is an instance of the `DatasetLocal` class which stores the data in a local directory
* rather than Apify cloud. This is useful for local development and debugging of your actors.
*
@@ -590,38 +590,9 @@ const getOrCreateDataset = (datasetIdOrName) => {
export const openDataset = (datasetIdOrName) => {
checkParamOrThrow(datasetIdOrName, 'datasetIdOrName', 'Maybe String');
- const localEmulationDir = process.env[ENV_VARS.LOCAL_EMULATION_DIR];
-
- let isDefault = false;
- let datasetPromise;
-
- if (!datasetIdOrName) {
- const envVar = ENV_VARS.DEFAULT_DATASET_ID;
-
- // Env var doesn't exist.
- if (!process.env[envVar]) return Promise.reject(new Error(`The '${envVar}' environment variable is not defined.`));
-
- isDefault = true;
- datasetIdOrName = process.env[envVar];
- }
-
- datasetPromise = datasetsCache.get(datasetIdOrName);
-
- // Found in cache.
- if (datasetPromise) return datasetPromise;
-
- // Use local emulation?
- if (localEmulationDir) {
- datasetPromise = Promise.resolve(new DatasetLocal(datasetIdOrName, localEmulationDir));
- } else {
- datasetPromise = isDefault // If true then we know that this is an ID of existing dataset.
- ? Promise.resolve(new Dataset(datasetIdOrName))
- : getOrCreateDataset(datasetIdOrName).then(dataset => (new Dataset(dataset.id)));
- }
-
- datasetsCache.add(datasetIdOrName, datasetPromise);
-
- return datasetPromise;
+ return process.env[ENV_VARS.PLATFORM_STORAGE]
+ ? openRemoteStorage(datasetIdOrName, ENV_VARS.DEFAULT_DATASET_ID, Dataset, datasetsCache, getOrCreateDataset)
+ : openLocalStorage(datasetIdOrName, ENV_VARS.DEFAULT_DATASET_ID, DatasetLocal, datasetsCache);
};
/**
@@ -635,7 +606,7 @@ export const openDataset = (datasetIdOrName) => {
*
* The data is stored in default dataset associated with this actor.
*
- * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is defined, the data gets pushed into local directory.
+ * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is defined, the data gets pushed into local directory.
* This feature is useful for local development and debugging of your actors.
*
* **IMPORTANT**: Do not forget to use the `await` keyword when calling `Apify.pushData()`,
diff --git a/src/index.js b/src/index.js
index a8d71d9e2..8a16f7a93 100644
--- a/src/index.js
+++ b/src/index.js
@@ -27,6 +27,11 @@ if (!isProduction() || process.env[ENV_VARS.LOG_LEVEL] === 'DEBUG') log.isDebugM
// Log as plain text not JSON
log.logJson = false;
+// Just warn user that local emulation will be used.
+if (process.env[ENV_VARS.PLATFORM_STORAGE]) {
+ log.info(`Using local directory for storage as environment variable ${ENV_VARS.PLATFORM_STORAGE}=1 is not set.`);
+}
+
/**
*{include-readme-1}
*{include-readme-2}
diff --git a/src/key_value_store.js b/src/key_value_store.js
index ed7f46ba6..d73fde54e 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -7,10 +7,12 @@ import LruCache from 'apify-shared/lru_cache';
import mime from 'mime';
import { KEY_VALUE_STORE_KEY_REGEX } from 'apify-shared/regexs';
import { checkParamOrThrow, parseBody } from 'apify-client/build/utils';
-import { ENV_VARS, LOCAL_EMULATION_SUBDIRS } from './constants';
-import { addCharsetToContentType, apifyClient, ensureDirExists } from './utils';
+import { ENV_VARS, LOCAL_STORAGE_SUBDIRS } from './constants';
+import {
+ addCharsetToContentType, apifyClient, ensureDirExists, isAtHome, openRemoteStorage, openLocalStorage,
+} from './utils';
-export const LOCAL_EMULATION_SUBDIR = LOCAL_EMULATION_SUBDIRS.keyValueStores;
+export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.keyValueStores;
const MAX_OPENED_STORES = 1000;
const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
const COMMON_LOCAL_FILE_EXTENSIONS = ['bin', 'txt', 'json', 'html', 'xml', 'jpeg', 'png', 'pdf', 'mp3', 'js', 'css', 'csv'];
@@ -200,11 +202,11 @@ export const getFileNameRegexp = (key) => {
* @ignore
*/
export class KeyValueStoreLocal {
- constructor(storeId, localEmulationDir) {
+ constructor(storeId, localStorageDir) {
checkParamOrThrow(storeId, 'storeId', 'String');
- checkParamOrThrow(localEmulationDir, 'localEmulationDir', 'String');
+ checkParamOrThrow(localStorageDir, 'localStorageDir', 'String');
- this.localEmulationPath = path.resolve(path.join(localEmulationDir, LOCAL_EMULATION_SUBDIR, storeId));
+ this.localEmulationPath = path.resolve(path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, storeId));
this.storeId = storeId;
this.initializationPromise = ensureDirExists(this.localEmulationPath);
}
@@ -220,7 +222,7 @@ export class KeyValueStoreLocal {
: null;
})
.catch((err) => {
- throw new Error(`Error reading file '${key}' in directory '${this.localEmulationPath}' referred by ${ENV_VARS.APIFY_LOCAL_EMULATION_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len
+ throw new Error(`Error reading file '${key}' in directory '${this.localEmulationPath}' referred by ${ENV_VARS.LOCAL_STORAGE_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len
});
}
@@ -244,7 +246,7 @@ export class KeyValueStoreLocal {
return deletePromise
.then(() => writeFilePromised(filePath, value))
.catch((err) => {
- throw new Error(`Error writing file '${key}' in directory '${this.localEmulationPath}' referred by ${ENV_VARS.APIFY_LOCAL_EMULATION_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len
+ throw new Error(`Error writing file '${key}' in directory '${this.localEmulationPath}' referred by ${ENV_VARS.LOCAL_STORAGE_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len
});
}
@@ -339,7 +341,6 @@ const getOrCreateKeyValueStore = (storeIdOrName) => {
});
};
-
/**
* Opens a key-value store and returns a promise resolving to an instance
* of the [KeyValueStore](#KeyValueStore) class.
@@ -354,7 +355,7 @@ const getOrCreateKeyValueStore = (storeIdOrName) => {
* await store.setValue('some-key', { foo: 'bar' });
* ```
*
- * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is set, the result of this function
+ * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the result of this function
* is an instance of the `KeyValueStoreLocal` class which stores the records in a local directory
* rather than Apify cloud. This is useful for local development and debugging of your actors.
*
@@ -370,38 +371,9 @@ const getOrCreateKeyValueStore = (storeIdOrName) => {
export const openKeyValueStore = (storeIdOrName) => {
checkParamOrThrow(storeIdOrName, 'storeIdOrName', 'Maybe String');
- const localEmulationDir = process.env[ENV_VARS.LOCAL_EMULATION_DIR];
-
- let isDefault = false;
- let storePromise;
-
- if (!storeIdOrName) {
- const envVar = ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID;
-
- // Env var doesn't exist.
- if (!process.env[envVar]) return Promise.reject(new Error(`The '${envVar}' environment variable is not defined.`));
-
- isDefault = true;
- storeIdOrName = process.env[envVar];
- }
-
- storePromise = storesCache.get(storeIdOrName);
-
- // Found in cache.
- if (storePromise) return storePromise;
-
- // Use local emulation?
- if (localEmulationDir) {
- storePromise = Promise.resolve(new KeyValueStoreLocal(storeIdOrName, localEmulationDir));
- } else {
- storePromise = isDefault // If true then we know that this is an ID of existing store.
- ? Promise.resolve(new KeyValueStore(storeIdOrName))
- : getOrCreateKeyValueStore(storeIdOrName).then(store => (new KeyValueStore(store.id)));
- }
-
- storesCache.add(storeIdOrName, storePromise);
-
- return storePromise;
+ return process.env[ENV_VARS.PLATFORM_STORAGE]
+ ? openRemoteStorage(storeIdOrName, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, KeyValueStore, storesCache, getOrCreateKeyValueStore)
+ : openLocalStorage(storeIdOrName, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, KeyValueStoreLocal, storesCache);
};
/**
@@ -426,7 +398,7 @@ export const openKeyValueStore = (storeIdOrName) => {
* For all other content types, the body is a raw `Buffer`.
* If the record cannot be found, the result is null.
*
- * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is defined,
+ * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is defined,
* the value is read from a that directory rather than the key-value store,
* specifically from a file that has the key as a name.
* file does not exists, the returned value is `null`. The file will get extension based on it's content type.
@@ -460,7 +432,7 @@ export const getValue = key => openKeyValueStore().then(store => store.getValue(
* ```
* In this case, the value must be a string or Buffer.
*
- * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is defined,
+ * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is defined,
* the value is written to that local directory rather than the key-value store on Apify cloud,
* to a file named as the key. This is useful for local development and debugging of your actors.
*
diff --git a/src/puppeteer_live_view_server.js b/src/puppeteer_live_view_server.js
index f1897780b..6bbd485c6 100644
--- a/src/puppeteer_live_view_server.js
+++ b/src/puppeteer_live_view_server.js
@@ -286,14 +286,14 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
constructor() {
super();
- const containerPort = process.env[ENV_VARS.CONTAINER_PORT];
- this.liveViewPort = parseInt(containerPort, 10) || 0;
+ const containerPort = process.env[ENV_VARS.CONTAINER_PORT] || LOCAL_ENV_VARS[ENV_VARS.CONTAINER_PORT];
+ this.liveViewPort = parseInt(containerPort, 10);
if (!(this.liveViewPort >= 0 && this.liveViewPort <= 65535)) {
throw new Error(`Cannot start LiveViewServer - invalid port specified by the ${
ENV_VARS.CONTAINER_PORT} environment variable (was "${containerPort}").`);
}
- this.liveViewUrl = process.env[ENV_VARS.CONTAINER_URL];
+ this.liveViewUrl = process.env[ENV_VARS.CONTAINER_URL] || LOCAL_ENV_VARS[ENV_VARS.CONTAINER_URL];
this.browsers = new Set();
this.browserIdCounter = 0;
@@ -337,10 +337,6 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
return promisifyServerListen(server)(this.liveViewPort)
.then(() => {
- if (!this.liveViewUrl) {
- this.liveViewPort = server.address().port;
- this.liveViewUrl = `http://localhost:${this.liveViewPort}`;
- }
log.info(`Live view is now available at ${this.liveViewUrl}`);
this.httpServer = server;
});
diff --git a/src/request_queue.js b/src/request_queue.js
index 8f25e8e3c..642b55ede 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -9,10 +9,10 @@ import Promise from 'bluebird';
import crypto from 'crypto';
import _ from 'underscore';
import Request from './request';
-import { ENV_VARS, LOCAL_EMULATION_SUBDIRS } from './constants';
-import { ensureDirExists, apifyClient } from './utils';
+import { ENV_VARS, LOCAL_STORAGE_SUBDIRS } from './constants';
+import { ensureDirExists, apifyClient, openRemoteStorage, openLocalStorage } from './utils';
-export const LOCAL_EMULATION_SUBDIR = LOCAL_EMULATION_SUBDIRS.requestQueues;
+export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.requestQueues;
const MAX_OPENED_QUEUES = 1000;
const MAX_CACHED_REQUESTS = 1000 * 1000;
@@ -498,7 +498,7 @@ export class RequestQueueLocal {
checkParamOrThrow(localEmulationDir, 'localEmulationDir', 'String');
this.queueId = queueId;
- this.localEmulationPath = path.resolve(path.join(localEmulationDir, LOCAL_EMULATION_SUBDIR, queueId));
+ this.localEmulationPath = path.resolve(path.join(localEmulationDir, LOCAL_STORAGE_SUBDIR, queueId));
this.localHandledEmulationPath = path.join(this.localEmulationPath, 'handled');
this.localPendingEmulationPath = path.join(this.localEmulationPath, 'pending');
@@ -766,7 +766,7 @@ const getOrCreateQueue = (queueIdOrName) => {
* queue.reclaimRequest(request2);
* ```
*
- * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is defined, the value this function
+ * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is defined, the value this function
* returns is an instance of the `RequestQueueLocal` class which is a local emulation of the request queue.
* This is useful for local development and debugging of the actors.
*
@@ -781,38 +781,9 @@ const getOrCreateQueue = (queueIdOrName) => {
* @ignore
*/
export const openRequestQueue = (queueIdOrName) => {
- const localEmulationDir = process.env[ENV_VARS.LOCAL_EMULATION_DIR];
-
checkParamOrThrow(queueIdOrName, 'queueIdOrName', 'Maybe String');
- let isDefault = false;
- let queuePromise;
-
- if (!queueIdOrName) {
- const envVar = ENV_VARS.DEFAULT_REQUEST_QUEUE_ID;
-
- // Env var doesn't exist.
- if (!process.env[envVar]) return Promise.reject(new Error(`The '${envVar}' environment variable is not defined.`));
-
- isDefault = true;
- queueIdOrName = process.env[envVar];
- }
-
- queuePromise = queuesCache.get(queueIdOrName);
-
- // Found in cache.
- if (queuePromise) return queuePromise;
-
- // Use local emulation?
- if (localEmulationDir) {
- queuePromise = Promise.resolve(new RequestQueueLocal(queueIdOrName, localEmulationDir));
- } else {
- queuePromise = isDefault // If true then we know that this is an ID of existing queue.
- ? Promise.resolve(new RequestQueue(queueIdOrName))
- : getOrCreateQueue(queueIdOrName).then(queue => (new RequestQueue(queue.id)));
- }
-
- queuesCache.add(queueIdOrName, queuePromise);
-
- return queuePromise;
+ return process.env[ENV_VARS.PLATFORM_STORAGE]
+ ? openRemoteStorage(queueIdOrName, ENV_VARS.DEFAULT_REQUEST_QUEUE_ID, RequestQueue, queuesCache, getOrCreateQueue)
+ : openLocalStorage(queueIdOrName, ENV_VARS.DEFAULT_REQUEST_QUEUE_ID, RequestQueueLocal, queuesCache);
};
diff --git a/src/utils.js b/src/utils.js
index b9fc7503d..5f1a3c15f 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -9,7 +9,7 @@ import requestPromise from 'request-promise';
import XRegExp from 'xregexp';
import { delayPromise, getRandomInt } from 'apify-shared/utilities';
import { checkParamOrThrow } from 'apify-client/build/utils';
-import { ENV_VARS, USER_AGENT_LIST } from './constants';
+import { ENV_VARS, LOCAL_ENV_VARS, USER_AGENT_LIST } from './constants';
/* globals process */
@@ -383,6 +383,52 @@ const getRandomUserAgent = () => {
return USER_AGENT_LIST[index];
};
+/**
+ * Helper function to open local storage.
+ *
+ * @ignore
+ */
+export const openLocalStorage = async (idOrName, defaultIdEnvVar, LocalClass, cache) => {
+ const localStorageDir = process.env[ENV_VARS.LOCAL_STORAGE_DIR] || LOCAL_ENV_VARS[ENV_VARS.LOCAL_STORAGE_DIR];
+
+ if (!idOrName) idOrName = process.env[defaultIdEnvVar] || LOCAL_ENV_VARS[defaultIdEnvVar];
+
+ let storagePromise = cache.get(idOrName);
+
+ if (!storagePromise) {
+ storagePromise = Promise.resolve(new LocalClass(idOrName, localStorageDir));
+ cache.add(idOrName, storagePromise);
+ }
+
+ return storagePromise;
+};
+
+/**
+ * Helper function to open remote storage.
+ *
+ * @ignore
+ */
+export const openRemoteStorage = async (idOrName, defaultIdEnvVar, RemoteClass, cache, getOrCreateFunction) => {
+ let isDefault = false;
+
+ if (!idOrName) {
+ isDefault = true;
+ idOrName = process.env[defaultIdEnvVar];
+ if (!idOrName) throw new Error(`The '${defaultIdEnvVar}' environment variable is not defined.`);
+ }
+
+ let storagePromise = cache.get(idOrName);
+
+ if (!storagePromise) {
+ storagePromise = isDefault // If true then we know that this is an ID of existing store.
+ ? Promise.resolve(new RemoteClass(idOrName))
+ : getOrCreateFunction(idOrName).then(storage => (new RemoteClass(storage.id)));
+ cache.add(idOrName, storagePromise);
+ }
+
+ return storagePromise;
+};
+
/**
* A namespace that contains various utilities.
*
diff --git a/test/_helper.js b/test/_helper.js
index 38bfd62fa..c56a813df 100644
--- a/test/_helper.js
+++ b/test/_helper.js
@@ -3,7 +3,9 @@ import path from 'path';
import { expect } from 'chai';
import { ENV_VARS } from '../build/constants';
-export const LOCAL_EMULATION_DIR = path.join(__dirname, '..', 'tmp', 'local-emulation-dir');
+export const LOCAL_STORAGE_DIR = path.join(__dirname, '..', 'tmp', 'local-emulation-dir');
+
+process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
// Log unhandled rejections.
process.on('unhandledRejection', (err) => {
@@ -14,15 +16,15 @@ process.on('unhandledRejection', (err) => {
});
// Immediately ensure that local emulation dir exists.
-fs.ensureDirSync(path.resolve(LOCAL_EMULATION_DIR));
+fs.ensureDirSync(path.resolve(LOCAL_STORAGE_DIR));
-export const emptyLocalEmulationSubdir = (subdir) => {
- const fullPath = path.resolve(path.join(LOCAL_EMULATION_DIR, subdir));
+export const emptyLocalStorageSubdir = (subdir) => {
+ const fullPath = path.resolve(path.join(LOCAL_STORAGE_DIR, subdir));
fs.emptyDirSync(fullPath);
};
-export const expectNotLocalEmulation = () => expect(process.env[ENV_VARS.LOCAL_EMULATION_DIR]).to.be.a('undefined');
+export const expectNotUsingLocalStorage = () => expect(process.env[ENV_VARS.PLATFORM_STORAGE]).to.be.a('undefined');
export const expectDirEmpty = (dirPath) => {
const content = fs.readdirSync(dirPath);
diff --git a/test/actor.js b/test/actor.js
index d3760af37..f5204effd 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -5,7 +5,7 @@ import 'babel-polyfill';
import sinon from 'sinon';
import Promise from 'bluebird';
import { delayPromise } from 'apify-shared/utilities';
-import { ACT_TASK_STATUSES, ENV_VARS, DEFAULT_PROXY_PORT, DEFAULT_PROXY_HOSTNAME } from '../build/constants';
+import { ACT_TASK_STATUSES, ENV_VARS, LOCAL_ENV_VARS, DEFAULT_PROXY_PORT, DEFAULT_PROXY_HOSTNAME } from '../build/constants';
import { ApifyCallError } from '../build/errors';
chai.use(chaiAsPromised);
@@ -616,7 +616,8 @@ describe('Apify.getApifyProxyUrl()', () => {
delete process.env[ENV_VARS.PROXY_HOSTNAME];
delete process.env[ENV_VARS.PROXY_PORT];
- expect(Apify.getApifyProxyUrl({ password: 'xyz' })).to.be.eql(`http://auto:xyz@${DEFAULT_PROXY_HOSTNAME}:${DEFAULT_PROXY_PORT}`);
+ expect(Apify.getApifyProxyUrl({ password: 'xyz' }))
+ .to.be.eql(`http://auto:xyz@${LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME]}:${LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT]}`);
expect(() => Apify.getApifyProxyUrl()).to.throw();
@@ -652,7 +653,8 @@ describe('Apify.getApifyProxyUrl()', () => {
delete process.env[ENV_VARS.PROXY_HOSTNAME];
delete process.env[ENV_VARS.PROXY_PORT];
- expect(Apify.getApifyProxyUrl({ password: 'xyz' })).to.be.eql(`http://auto:xyz@${DEFAULT_PROXY_HOSTNAME}:${DEFAULT_PROXY_PORT}`);
+ expect(Apify.getApifyProxyUrl({ password: 'xyz' }))
+ .to.be.eql(`http://auto:xyz@${LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME]}:${LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT]}`);
expect(() => Apify.getApifyProxyUrl()).to.throw();
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index 8c0e2171a..ea3f6dd93 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -7,7 +7,7 @@ import log from 'apify-shared/log';
import { delayPromise } from 'apify-shared/utilities';
import * as Apify from '../build/index';
import { RequestQueue, RequestQueueLocal } from '../build/request_queue';
-import { LOCAL_EMULATION_DIR } from './_helper';
+import { LOCAL_STORAGE_DIR } from './_helper';
chai.use(chaiAsPromised);
@@ -242,7 +242,7 @@ describe('BasicCrawler', () => {
it('should also support RequestQueueLocal', () => {
const requestQueue = new RequestQueue('xxx');
- const requestQueueLocal = new RequestQueueLocal('xxx', LOCAL_EMULATION_DIR);
+ const requestQueueLocal = new RequestQueueLocal('xxx', LOCAL_STORAGE_DIR);
const handleRequestFunction = () => {};
expect(() => new Apify.BasicCrawler({ handleRequestFunction, requestQueue })).to.not.throw();
diff --git a/test/dataset.js b/test/dataset.js
index 610e8d096..c11629a06 100644
--- a/test/dataset.js
+++ b/test/dataset.js
@@ -4,19 +4,21 @@ import 'babel-polyfill';
import fs from 'fs-extra';
import path from 'path';
import sinon from 'sinon';
-import { leftpad, delayPromise } from 'apify-shared/utilities';
+import { leftpad } from 'apify-shared/utilities';
import { ENV_VARS, MAX_PAYLOAD_SIZE_BYTES } from '../build/constants';
-import { LOCAL_FILENAME_DIGITS, Dataset, DatasetLocal, LOCAL_EMULATION_SUBDIR,
+import { LOCAL_FILENAME_DIGITS, Dataset, DatasetLocal, LOCAL_STORAGE_SUBDIR,
LOCAL_GET_ITEMS_DEFAULT_LIMIT, checkAndSerialize, chunkBySize } from '../build/dataset';
-import { apifyClient } from '../build/utils';
+import * as utils from '../build/utils';
import * as Apify from '../build/index';
-import { LOCAL_EMULATION_DIR, emptyLocalEmulationSubdir, expectNotLocalEmulation, expectDirEmpty, expectDirNonEmpty } from './_helper';
+import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir, expectDirEmpty, expectDirNonEmpty } from './_helper';
+
+const { apifyClient } = utils;
chai.use(chaiAsPromised);
const read = (datasetName, index) => {
const fileName = `${leftpad(index, LOCAL_FILENAME_DIGITS, 0)}.json`;
- const filePath = path.join(LOCAL_EMULATION_DIR, LOCAL_EMULATION_SUBDIR, datasetName, fileName);
+ const filePath = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, datasetName, fileName);
const str = fs.readFileSync(path.resolve(filePath));
return JSON.parse(str);
@@ -25,12 +27,12 @@ const read = (datasetName, index) => {
describe('dataset', () => {
before(() => apifyClient.setOptions({ token: 'xxx' }));
after(() => apifyClient.setOptions({ token: undefined }));
- beforeEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));
- afterEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));
+ beforeEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));
+ afterEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));
describe('local', async () => {
it('should succesfully save data', async () => {
- const dataset = new DatasetLocal('my-dataset', LOCAL_EMULATION_DIR);
+ const dataset = new DatasetLocal('my-dataset', LOCAL_STORAGE_DIR);
await dataset.pushData({ foo: 'bar' });
await dataset.pushData({ foo: 'hotel' });
@@ -45,19 +47,19 @@ describe('dataset', () => {
expect(read('my-dataset', 4)).to.be.eql({ foo: 'from-array-1', arr: [1, 2, 3] });
// Correctly initializes the state.
- const newDataset = new DatasetLocal('my-dataset', LOCAL_EMULATION_DIR);
+ const newDataset = new DatasetLocal('my-dataset', LOCAL_STORAGE_DIR);
await newDataset.pushData({ foo2: 'bar2' });
expect(read('my-dataset', 5)).to.be.eql({ foo2: 'bar2' });
// Delete works.
- const datasetDir = path.join(LOCAL_EMULATION_DIR, LOCAL_EMULATION_SUBDIR, 'my-dataset');
+ const datasetDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-dataset');
expectDirNonEmpty(datasetDir);
await newDataset.delete();
expectDirEmpty(datasetDir);
});
const getLocalDataset = async (data) => {
- const dataset = new DatasetLocal('my-dataset', LOCAL_EMULATION_DIR);
+ const dataset = new DatasetLocal('my-dataset', LOCAL_STORAGE_DIR);
await dataset.pushData(data);
return dataset;
@@ -109,7 +111,7 @@ describe('dataset', () => {
it('getInfo() should work', async () => {
const datasetName = 'stats-dataset';
- const dataset = new DatasetLocal(datasetName, LOCAL_EMULATION_DIR);
+ const dataset = new DatasetLocal(datasetName, LOCAL_STORAGE_DIR);
await Apify.utils.sleep(2);
// Save orig env var since it persists over tests.
@@ -720,84 +722,20 @@ describe('dataset', () => {
});
describe('Apify.openDataset', async () => {
- it('should open a local dataset when process.env[ENV_VARS.LOCAL_EMULATION_DIR] is set', async () => {
- process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;
-
- const dataset = await Apify.openDataset('some-id-2');
- expect(dataset).to.be.instanceof(DatasetLocal);
- expect(dataset).not.to.be.instanceof(Dataset);
-
- delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];
- });
-
- it('should reuse cached dataset instances', async () => {
- process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;
-
- const dataset1 = await Apify.openDataset('some-id-3');
- const dataset2 = await Apify.openDataset('some-id-3');
- const dataset3 = new DatasetLocal('some-id-3', LOCAL_EMULATION_DIR);
-
- expect(dataset1).to.be.instanceof(DatasetLocal);
- expect(dataset2).to.be.instanceof(DatasetLocal);
- expect(dataset3).to.be.instanceof(DatasetLocal);
+ it('should work', () => {
+ const mock = sinon.mock(utils);
- expect(dataset1).to.be.equal(dataset2);
- expect(dataset1).not.to.be.equal(dataset3);
+ delete process.env[ENV_VARS.PLATFORM_STORAGE];
- delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];
+ mock.expects('openLocalStorage').once();
+ Apify.openDataset();
- // Here must be some timeout to don't finish before initialization of dataset finishes.
- // Otherwise we delete the directory and scandir will throw ENOENT: no such file or directory
- await delayPromise(100);
- });
-
- it('should open default dataset when datasetIdOrName is not provided', async () => {
- process.env[ENV_VARS.DEFAULT_DATASET_ID] = 'some-id-4';
- process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;
-
- const dataset = await Apify.openDataset();
- expect(dataset.datasetId).to.be.eql('some-id-4');
- expect(dataset).to.be.instanceof(DatasetLocal);
-
- delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];
- process.env[ENV_VARS.DEFAULT_DATASET_ID] = 'some-id-5';
- expectNotLocalEmulation();
+ process.env[ENV_VARS.PLATFORM_STORAGE] = '1';
- const dataset2 = await Apify.openDataset();
- expect(dataset2.datasetId).to.be.eql('some-id-5');
- expect(dataset2).to.be.instanceof(Dataset);
-
- delete process.env[ENV_VARS.DEFAULT_DATASET_ID];
- });
+ mock.expects('openRemoteStorage').once();
+ Apify.openDataset();
- it('should open remote dataset when process.env[ENV_VARS.LOCAL_EMULATION_DIR] is NOT set', async () => {
- expectNotLocalEmulation();
-
- const mock = sinon.mock(apifyClient.datasets);
-
- // First when used with id it only requests store object.
- mock.expects('getDataset')
- .once()
- .withArgs({ datasetId: 'some-id-6' })
- .returns(Promise.resolve({ id: 'some-id-6' }));
- const dataset = await Apify.openDataset('some-id-6');
- expect(dataset.datasetId).to.be.eql('some-id-6');
- expect(dataset).to.be.instanceof(Dataset);
-
- // Then used with name it requests store object, gets empty response
- // so then it creates dataset.
- mock.expects('getDataset')
- .once()
- .withArgs({ datasetId: 'some-name-7' })
- .returns(Promise.resolve(null));
- mock.expects('getOrCreateDataset')
- .once()
- .withArgs({ datasetName: 'some-name-7' })
- .returns(Promise.resolve({ id: 'some-id-7' }));
-
- const dataset2 = await Apify.openDataset('some-name-7');
- expect(dataset2.datasetId).to.be.eql('some-id-7');
- expect(dataset2).to.be.instanceof(Dataset);
+ delete process.env[ENV_VARS.PLATFORM_STORAGE];
mock.verify();
mock.restore();
@@ -807,7 +745,6 @@ describe('dataset', () => {
describe('pushData', async () => {
it('throws on invalid args', async () => {
process.env[ENV_VARS.DEFAULT_DATASET_ID] = 'some-id-8';
- process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;
const dataErrMsg = 'Parameter "data" of type Array | Object must be provided';
await expect(Apify.pushData()).to.be.rejectedWith(dataErrMsg);
@@ -825,17 +762,20 @@ describe('dataset', () => {
delete process.env[ENV_VARS.DEFAULT_DATASET_ID];
});
- it('throws if DEFAULT_DATASET_ID env var is not defined', async () => {
+ it('throws if DEFAULT_DATASET_ID env var is not defined and we use cloud storage', async () => {
+ process.env[ENV_VARS.PLATFORM_STORAGE] = '1';
+
process.env[ENV_VARS.DEFAULT_DATASET_ID] = '';
await expect(Apify.pushData({ something: 123 })).to.be.rejectedWith(Error);
delete process.env[ENV_VARS.DEFAULT_DATASET_ID];
await expect(Apify.pushData({ something: 123 })).to.be.rejectedWith(Error);
+
+ delete process.env[ENV_VARS.PLATFORM_STORAGE];
});
it('correctly stores records', async () => {
process.env[ENV_VARS.DEFAULT_DATASET_ID] = 'some-id-9';
- process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;
await Apify.pushData({ foo: 'bar' });
await Apify.pushData({ foo: 'hotel' });
@@ -843,7 +783,7 @@ describe('dataset', () => {
expect(read('some-id-9', 1)).to.be.eql({ foo: 'bar' });
expect(read('some-id-9', 2)).to.be.eql({ foo: 'hotel' });
- delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];
+ delete process.env[ENV_VARS.DEFAULT_DATASET_ID];
});
});
diff --git a/test/key_value_store.js b/test/key_value_store.js
index 6df3ebb6f..4678b3c65 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -4,18 +4,20 @@ import 'babel-polyfill';
import sinon from 'sinon';
import path from 'path';
import { ENV_VARS } from '../build/constants';
-import { KeyValueStoreLocal, KeyValueStore, maybeStringify, getFileNameRegexp, LOCAL_EMULATION_SUBDIR } from '../build/key_value_store';
-import { apifyClient } from '../build/utils';
+import { KeyValueStoreLocal, KeyValueStore, maybeStringify, getFileNameRegexp, LOCAL_STORAGE_SUBDIR } from '../build/key_value_store';
+import * as utils from '../build/utils';
import * as Apify from '../build/index';
-import { LOCAL_EMULATION_DIR, emptyLocalEmulationSubdir, expectNotLocalEmulation, expectDirEmpty, expectDirNonEmpty } from './_helper';
+import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir, expectDirEmpty, expectDirNonEmpty } from './_helper';
+
+const { apifyClient } = utils;
chai.use(chaiAsPromised);
describe('KeyValueStore', () => {
before(() => apifyClient.setOptions({ token: 'xxx' }));
after(() => apifyClient.setOptions({ token: undefined }));
- beforeEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));
- afterEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));
+ beforeEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));
+ afterEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));
describe('maybeStringify()', () => {
it('should work', () => {
@@ -54,8 +56,8 @@ describe('KeyValueStore', () => {
describe('local', async () => {
it('should work', async () => {
- const store = new KeyValueStoreLocal('my-store-id', LOCAL_EMULATION_DIR);
- const store2 = new KeyValueStoreLocal('another-store-id', LOCAL_EMULATION_DIR);
+ const store = new KeyValueStoreLocal('my-store-id', LOCAL_STORAGE_DIR);
+ const store2 = new KeyValueStoreLocal('another-store-id', LOCAL_STORAGE_DIR);
const buffer = Buffer.from('some text value');
await store.setValue('key-obj', { foo: 'bar' });
@@ -96,7 +98,7 @@ describe('KeyValueStore', () => {
expect(await store2.getValue('key-badctype')).to.be.eql(buffer);
// Delete works.
- const storeDir = path.join(LOCAL_EMULATION_DIR, LOCAL_EMULATION_SUBDIR, 'my-store-id');
+ const storeDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-store-id');
expectDirNonEmpty(storeDir);
await store.delete();
expectDirEmpty(storeDir);
@@ -158,80 +160,20 @@ describe('KeyValueStore', () => {
});
describe('Apify.openKeyValueStore', async () => {
- it('should open a local store when process.env[ENV_VARS.LOCAL_EMULATION_DIR] is set', async () => {
- process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;
-
- const store = await Apify.openKeyValueStore('some-id-2');
- expect(store).to.be.instanceof(KeyValueStoreLocal);
- expect(store).not.to.be.instanceof(KeyValueStore);
-
- delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];
- });
-
- it('should reuse cached store instances', async () => {
- process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;
-
- const store1 = await Apify.openKeyValueStore('some-id-3');
- const store2 = await Apify.openKeyValueStore('some-id-3');
- const store3 = new KeyValueStoreLocal('some-id-3', LOCAL_EMULATION_DIR);
-
- expect(store1).to.be.instanceof(KeyValueStoreLocal);
- expect(store2).to.be.instanceof(KeyValueStoreLocal);
- expect(store3).to.be.instanceof(KeyValueStoreLocal);
-
- expect(store1).to.be.equal(store2);
- expect(store1).not.to.be.equal(store3);
-
- delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];
- });
-
- it('should open default store when storeIdOrName is not provided', async () => {
- process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = 'some-id-4';
- process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;
-
- const store = await Apify.openKeyValueStore();
- expect(store.storeId).to.be.eql('some-id-4');
- expect(store).to.be.instanceof(KeyValueStoreLocal);
-
- delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];
- process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = 'some-id-5';
- expectNotLocalEmulation();
-
- const store2 = await Apify.openKeyValueStore();
- expect(store2.storeId).to.be.eql('some-id-5');
- expect(store2).to.be.instanceof(KeyValueStore);
+ it('should work', () => {
+ const mock = sinon.mock(utils);
- delete process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];
- });
+ delete process.env[ENV_VARS.PLATFORM_STORAGE];
- it('should open remote store when process.env[ENV_VARS.LOCAL_EMULATION_DIR] is NOT set', async () => {
- expectNotLocalEmulation();
+ mock.expects('openLocalStorage').once();
+ Apify.openKeyValueStore();
- const mock = sinon.mock(apifyClient.keyValueStores);
+ process.env[ENV_VARS.PLATFORM_STORAGE] = '1';
- // First when used with id it only requests store object.
- mock.expects('getStore')
- .once()
- .withArgs({ storeId: 'some-id-6' })
- .returns(Promise.resolve({ id: 'some-id-6' }));
- const store = await Apify.openKeyValueStore('some-id-6');
- expect(store.storeId).to.be.eql('some-id-6');
- expect(store).to.be.instanceof(KeyValueStore);
-
- // Then used with name it requests store object, gets empty response
- // so then it creates dataset.
- mock.expects('getStore')
- .once()
- .withArgs({ storeId: 'some-name-7' })
- .returns(Promise.resolve(null));
- mock.expects('getOrCreateStore')
- .once()
- .withArgs({ storeName: 'some-name-7' })
- .returns(Promise.resolve({ id: 'some-id-7' }));
+ mock.expects('openRemoteStorage').once();
+ Apify.openKeyValueStore();
- const store2 = await Apify.openKeyValueStore('some-name-7');
- expect(store2.storeId).to.be.eql('some-id-7');
- expect(store2).to.be.instanceof(KeyValueStore);
+ delete process.env[ENV_VARS.PLATFORM_STORAGE];
mock.verify();
mock.restore();
@@ -247,7 +189,9 @@ describe('KeyValueStore', () => {
await expect(Apify.getValue(null)).to.be.rejectedWith('Parameter "key" of type String must be provided');
});
- it('throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined', async () => {
+ it('throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined and we use cloud storage', async () => {
+ process.env[ENV_VARS.PLATFORM_STORAGE] = '1';
+
const errMsg = 'The \'APIFY_DEFAULT_KEY_VALUE_STORE_ID\' environment variable is not defined';
process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = '';
@@ -255,12 +199,14 @@ describe('KeyValueStore', () => {
delete process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];
await expect(Apify.getValue('some other key')).to.be.rejectedWith(errMsg);
+
+ delete process.env[ENV_VARS.PLATFORM_STORAGE];
});
});
describe('setValue', async () => {
it('throws on invalid args', async () => {
- process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = '1234';
+ process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = '12345';
await expect(Apify.setValue()).to.be.rejectedWith('Parameter "key" of type String must be provided');
await expect(Apify.setValue('', null)).to.be.rejectedWith('The "key" parameter cannot be empty');
await expect(Apify.setValue('', 'some value')).to.be.rejectedWith('The "key" parameter cannot be empty');
@@ -300,7 +246,7 @@ describe('KeyValueStore', () => {
});
it('throws on invalid characters in key', async () => {
- const store = new KeyValueStoreLocal('my-store-id', LOCAL_EMULATION_DIR);
+ const store = new KeyValueStoreLocal('my-store-id', LOCAL_STORAGE_DIR);
const INVALID_CHARACTERS = '?|\\/"*<>%:';
let counter = 0;
@@ -315,7 +261,9 @@ describe('KeyValueStore', () => {
expect(counter).to.be.eql(INVALID_CHARACTERS.length);
});
- it('throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined', async () => {
+ it('throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined and we use cloud storage', async () => {
+ process.env[ENV_VARS.PLATFORM_STORAGE] = 1;
+
const errMsg = 'The \'APIFY_DEFAULT_KEY_VALUE_STORE_ID\' environment variable is not defined';
process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = '';
@@ -323,6 +271,8 @@ describe('KeyValueStore', () => {
delete process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];
await expect(Apify.setValue('some other key', {})).to.be.rejectedWith(errMsg);
+
+ delete process.env[ENV_VARS.PLATFORM_STORAGE];
});
it('correctly adds charset to content type', async () => {
@@ -401,3 +351,4 @@ describe('KeyValueStore', () => {
});
});
});
+
diff --git a/test/request_queue.js b/test/request_queue.js
index 6d284e8d3..4a2c72b8c 100644
--- a/test/request_queue.js
+++ b/test/request_queue.js
@@ -4,12 +4,13 @@ import _ from 'underscore';
import 'babel-polyfill';
import sinon from 'sinon';
import path from 'path';
-import { delayPromise } from 'apify-shared/utilities';
import * as Apify from '../build/index';
import { ENV_VARS } from '../build/constants';
-import { apifyClient } from '../build/utils';
-import { RequestQueueLocal, RequestQueue, LOCAL_EMULATION_SUBDIR, QUERY_HEAD_MIN_LENGTH } from '../build/request_queue';
-import { emptyLocalEmulationSubdir, LOCAL_EMULATION_DIR, expectNotLocalEmulation, expectDirEmpty, expectDirNonEmpty } from './_helper';
+import * as utils from '../build/utils';
+import { RequestQueueLocal, RequestQueue, LOCAL_STORAGE_SUBDIR, QUERY_HEAD_MIN_LENGTH } from '../build/request_queue';
+import { emptyLocalStorageSubdir, LOCAL_STORAGE_DIR, expectNotUsingLocalStorage, expectDirEmpty, expectDirNonEmpty } from './_helper';
+
+const { apifyClient } = utils;
chai.use(chaiAsPromised);
@@ -20,12 +21,12 @@ const expectRequestsSame = (req1, req2) => {
describe('RequestQueue', () => {
before(() => apifyClient.setOptions({ token: 'xxx' }));
after(() => apifyClient.setOptions({ token: undefined }));
- beforeEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));
- afterEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));
+ beforeEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));
+ afterEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));
describe('local', async () => {
it('should work', async () => {
- const queue = new RequestQueueLocal('my-queue-0', LOCAL_EMULATION_DIR);
+ const queue = new RequestQueueLocal('my-queue-0', LOCAL_STORAGE_DIR);
await queue.addRequest(new Apify.Request({ url: 'http://example.com/first' }));
await queue.addRequest(new Apify.Request({ url: 'http://example.com/middle' }));
@@ -67,14 +68,14 @@ describe('RequestQueue', () => {
expect(await queue.isFinished()).to.be.eql(true);
// Delete it.
- const queueDir = path.join(LOCAL_EMULATION_DIR, LOCAL_EMULATION_SUBDIR, 'my-queue-0');
+ const queueDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-queue-0');
expectDirNonEmpty(queueDir);
await queue.delete();
expectDirEmpty(queueDir);
});
it('supports forefront param in reclaimRequest()', async () => {
- const queue = new RequestQueueLocal('my-queue-1', LOCAL_EMULATION_DIR);
+ const queue = new RequestQueueLocal('my-queue-1', LOCAL_STORAGE_DIR);
await queue.addRequest(new Apify.Request({ url: 'http://example.com/first' }));
await queue.addRequest(new Apify.Request({ url: 'http://example.com/middle' }));
@@ -107,7 +108,7 @@ describe('RequestQueue', () => {
const request3 = new Apify.Request({ url: 'http://example.com/last-but-first' });
// Do something with 3 requests in one queue.
- const queue = new RequestQueueLocal('my-queue-2', LOCAL_EMULATION_DIR);
+ const queue = new RequestQueueLocal('my-queue-2', LOCAL_STORAGE_DIR);
await queue.addRequest(request1);
await queue.addRequest(request2);
await queue.addRequest(request3, { forefront: true });
@@ -118,7 +119,7 @@ describe('RequestQueue', () => {
await queue.markRequestHandled(freshRequest1);
// Now do the same with another queue.
- const anotherQueue = new RequestQueueLocal('my-queue-2', LOCAL_EMULATION_DIR);
+ const anotherQueue = new RequestQueueLocal('my-queue-2', LOCAL_STORAGE_DIR);
expect(await anotherQueue.isEmpty()).to.be.eql(false);
expect(await anotherQueue.isFinished()).to.be.eql(false);
const request3FromAnotherQueue = await anotherQueue.fetchNextRequest();
@@ -136,7 +137,7 @@ describe('RequestQueue', () => {
describe('remote', async () => {
it('should work', async () => {
- expectNotLocalEmulation();
+ expectNotUsingLocalStorage();
const { Request } = Apify;
@@ -258,7 +259,7 @@ describe('RequestQueue', () => {
});
it('should cache requests new locally', async () => {
- expectNotLocalEmulation();
+ expectNotUsingLocalStorage();
const { Request } = Apify;
@@ -296,7 +297,7 @@ describe('RequestQueue', () => {
});
it('should cache requests locally with info if request was already handled', async () => {
- expectNotLocalEmulation();
+ expectNotUsingLocalStorage();
const { Request } = Apify;
@@ -334,7 +335,7 @@ describe('RequestQueue', () => {
});
it('should cache requests from queue head', async () => {
- expectNotLocalEmulation();
+ expectNotUsingLocalStorage();
const { Request } = Apify;
@@ -370,84 +371,20 @@ describe('RequestQueue', () => {
});
describe('Apify.openRequestQueue', async () => {
- it('should open a local request queue when process.env[ENV_VARS.LOCAL_EMULATION_DIR] is set', async () => {
- process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;
-
- const queue = await Apify.openRequestQueue('some-id-2');
- expect(queue).to.be.instanceof(RequestQueueLocal);
- expect(queue).not.to.be.instanceof(RequestQueue);
-
- delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];
- });
-
- it('should reuse cached request queue instances', async () => {
- process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;
-
- const queue1 = await Apify.openRequestQueue('some-id-3');
- const queue2 = await Apify.openRequestQueue('some-id-3');
- const queue3 = new RequestQueueLocal('some-id-3', LOCAL_EMULATION_DIR);
-
- expect(queue1).to.be.instanceof(RequestQueueLocal);
- expect(queue2).to.be.instanceof(RequestQueueLocal);
- expect(queue3).to.be.instanceof(RequestQueueLocal);
-
- expect(queue1).to.be.equal(queue2);
- expect(queue1).not.to.be.equal(queue3);
-
- delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];
-
- // Here must be some timeout to don't finish before initialization of queues finishes.
- // Otherwise we delete the directory and scandir will throw ENOENT: no such file or directory
- await delayPromise(100);
- });
-
- it('should open default request queue when queueIdOrName is not provided', async () => {
- process.env[ENV_VARS.DEFAULT_REQUEST_QUEUE_ID] = 'some-id-4';
- process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;
-
- const queue = await Apify.openRequestQueue();
- expect(queue.queueId).to.be.eql('some-id-4');
- expect(queue).to.be.instanceof(RequestQueueLocal);
-
- delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];
- process.env[ENV_VARS.DEFAULT_REQUEST_QUEUE_ID] = 'some-id-5';
- expectNotLocalEmulation();
-
- const queue2 = await Apify.openRequestQueue();
- expect(queue2.queueId).to.be.eql('some-id-5');
- expect(queue2).to.be.instanceof(RequestQueue);
+ it('should work', () => {
+ const mock = sinon.mock(utils);
- delete process.env[ENV_VARS.DEFAULT_REQUEST_QUEUE_ID];
- });
+ delete process.env[ENV_VARS.PLATFORM_STORAGE];
- it('should open remote queue when process.env[ENV_VARS.LOCAL_EMULATION_DIR] is NOT set', async () => {
- expectNotLocalEmulation();
+ mock.expects('openLocalStorage').once();
+ Apify.openRequestQueue();
- const mock = sinon.mock(apifyClient.requestQueues);
+ process.env[ENV_VARS.PLATFORM_STORAGE] = '1';
- // First when used with id it only requests store object.
- mock.expects('getQueue')
- .once()
- .withArgs({ queueId: 'some-id-6' })
- .returns(Promise.resolve({ id: 'some-id-6' }));
- const queue = await Apify.openRequestQueue('some-id-6');
- expect(queue.queueId).to.be.eql('some-id-6');
- expect(queue).to.be.instanceof(RequestQueue);
-
- // Then used with name it requests store object, gets empty response
- // so then it creates queue.
- mock.expects('getQueue')
- .once()
- .withArgs({ queueId: 'some-name-7' })
- .returns(Promise.resolve(null));
- mock.expects('getOrCreateQueue')
- .once()
- .withArgs({ queueName: 'some-name-7' })
- .returns(Promise.resolve({ id: 'some-id-7' }));
+ mock.expects('openRemoteStorage').once();
+ Apify.openRequestQueue();
- const queue2 = await Apify.openRequestQueue('some-name-7');
- expect(queue2.queueId).to.be.eql('some-id-7');
- expect(queue2).to.be.instanceof(RequestQueue);
+ delete process.env[ENV_VARS.PLATFORM_STORAGE];
mock.verify();
mock.restore();
diff --git a/test/utils.js b/test/utils.js
index fe33cb041..9805a06df 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -6,9 +6,10 @@ import path from 'path';
import os from 'os';
import Promise from 'bluebird';
import requestPromise from 'request-promise';
+import LruCache from 'apify-shared/lru_cache';
import * as utils from '../build/utils';
import Apify from '../build/index';
-import { ENV_VARS } from '../build/constants';
+import { ENV_VARS, LOCAL_ENV_VARS } from '../build/constants';
chai.use(chaiAsPromised);
@@ -525,3 +526,121 @@ describe('Apify.utils.getRandomUserAgent()', () => {
expect(agent.length).to.not.be.eql(0);
});
});
+
+describe('utils.openLocalStorage()', async () => {
+ it('should return item from cache if available and create new one otherwise', async () => {
+ const cache = new LruCache({ maxLength: 5 });
+ class MyStore {}
+
+ expect(cache.length()).to.be.eql(0);
+
+ const store = await utils.openLocalStorage('some-id', 'some-env', MyStore, cache);
+ expect(store).to.be.instanceOf(MyStore);
+ expect(cache.length()).to.be.eql(1);
+
+ const store2 = await utils.openLocalStorage('some-id', 'some-env', MyStore, cache);
+ expect(store2).to.be.equal(store);
+ expect(cache.length()).to.be.eql(1);
+
+ const store3 = await utils.openLocalStorage('some-other-id', 'some-env', MyStore, cache);
+ expect(store3).to.not.be.equal(store);
+ expect(cache.length()).to.be.eql(2);
+ });
+
+ it('should use ID from ENV variable if no parameter is provided', async () => {
+ const cache = new LruCache({ maxLength: 5 });
+ class MyStore {
+ constructor(id) {
+ this.id = id;
+ }
+ }
+
+ process.env['some-env'] = 'id-from-env';
+
+ const store = await utils.openLocalStorage(null, 'some-env', MyStore, cache);
+ expect(store.id).to.eql('id-from-env');
+
+ delete process.env['some-env'];
+ });
+
+ it('should use ID from shared if neither parameter nor ENV var is provided', async () => {
+ const cache = new LruCache({ maxLength: 5 });
+ class MyStore {
+ constructor(id) {
+ this.id = id;
+ }
+ }
+
+ // There is some default in shared constants.
+ const defaultLocalValue = LOCAL_ENV_VARS[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];
+ expect(defaultLocalValue).to.be.a('string');
+ expect(defaultLocalValue).to.have.length.above(1);
+
+ // There is no env var!
+ expect(process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID]).to.be.eql(undefined);
+
+ const store = await utils.openLocalStorage(null, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, MyStore, cache);
+ expect(store.id).to.eql(defaultLocalValue);
+ });
+});
+
+describe('utils.openRemoteStorage()', async () => {
+ it('should return item from cache if available and create new one otherwise', async () => {
+ const cache = new LruCache({ maxLength: 5 });
+ class MyStore {
+ constructor(id) {
+ this.id = id;
+ }
+ }
+
+ delete process.env['some-env'];
+
+ expect(cache.length()).to.be.eql(0);
+
+ const store = await utils.openRemoteStorage('some-id', 'some-env', MyStore, cache, async () => ({ id: 'some-id' }));
+ expect(store.id).to.be.eql('some-id');
+ expect(cache.length()).to.be.eql(1);
+
+ const store2 = await utils.openRemoteStorage('some-id', 'some-env', MyStore, cache, async () => { throw new Error('Should not be called!'); }); // eslint-disable-line
+ expect(store2.id).to.be.eql('some-id');
+ expect(store2).to.be.equal(store);
+ expect(cache.length()).to.be.eql(1);
+
+ const store3 = await utils.openRemoteStorage('some-other-id', 'some-env', MyStore, cache, async () => ({ id: 'some-other-id' }));
+ expect(store3).to.not.be.equal(store);
+ expect(store3.id).to.be.eql('some-other-id');
+ expect(cache.length()).to.be.eql(2);
+ });
+
+ it('should use ID from ENV variable if no parameter is provided', async () => {
+ const cache = new LruCache({ maxLength: 5 });
+ class MyStore {
+ constructor(id) {
+ this.id = id;
+ }
+ }
+
+ process.env['some-env'] = 'id-from-env';
+
+ const store = await utils.openLocalStorage(null, 'some-env', MyStore, cache);
+ expect(store.id).to.eql('id-from-env');
+
+ delete process.env['some-env'];
+ });
+
+ it('should use ID from ENV variable and not call getOrCreateStoreFunction parameter is not provided', async () => {
+ const cache = new LruCache({ maxLength: 5 });
+ class MyStore {
+ constructor(id) {
+ this.id = id;
+ }
+ }
+
+ process.env['some-env'] = 'id-from-env';
+
+ const store = await utils.openRemoteStorage(null, 'some-env', MyStore, cache, async () => { throw new Error('Should not be called!'); }); // eslint-disable-line
+ expect(store.id).to.eql('id-from-env');
+
+ delete process.env['some-env'];
+ });
+});