Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Adding kvstores datasets autoscaledpool etc... (#24)
Marek Trunkáteb4bbfceae53
diff --git a/.gitignore b/.gitignoreindex c54aa7c5c..0bc7a79f4 100644--- a/.gitignore+++ b/.gitignore@@ -11,4 +11,5 @@ pids.ideayarn.lockpackage-lock.json-docs\ No newline at end of file+docs+tmp\ No newline at end of filediff --git a/CHANGELOG.md b/CHANGELOG.mdindex c923fed62..4846955d9 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,8 @@+0.5.0 / 2018-02-08+===================+- Apify.setPromiseDependency / Apify.getPromiseDependency / Apify.getPromisePrototype removed+- Bunch of classes as AutoscaledPool or PuppeteerCrawler added, check documentation+0.4.48 / 2018-02-05===================- Renamed GitHub repodiff --git a/jsdoc/conf.json b/jsdoc/conf.jsonindex a3f7b3939..a2d2f3b47 100644--- a/jsdoc/conf.json+++ b/jsdoc/conf.json@@ -3,8 +3,8 @@"allowUnknownTags": true},"source": {- "include": [ "build" ],- "exclude": [ "build/constants.js", "build/proxy_chain.js" ],+ "include": [ "src" ],+ "exclude": [ "src/constants.js" ],"includePattern": ".+\\.js(doc|x)?$","excludePattern": "(^|\\/|\\\\)_"},diff --git a/package.json b/package.jsonindex 99505218c..f1bf157e1 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apify",- "version": "0.4.48",+ "version": "0.5.0","description": "Apify Actor runtime for JavaScript","main": "build/index.js","keywords": [@@ -35,7 +35,7 @@"scripts": {"build": "rm -rf ./build && babel src --out-dir build","build-doc": "npm run clean && npm run build && node ./node_modules/jsdoc/jsdoc.js --package ./package.json -c ./jsdoc/conf.json -d docs",- "test": "npm run build && mocha --compilers js:babel-core/register --recursive",+ "test": "npm run build && mocha --timeout 60000 --compilers js:babel-core/register --recursive","test-cov": "npm run build && babel-node node_modules/isparta/bin/isparta cover --report html --report text node_modules/.bin/_mocha","prepare": "npm run build","prepublishOnly": "(test $RUNNING_FROM_SCRIPT || (echo \"You must use publish.sh instead of 'npm publish' directly!\"; exit 1)) && npm test && npm run lint",@@ -43,10 +43,13 @@"lint": "npm run build && eslint src test"},"dependencies": {- "apify-client": "^0.1.62",+ "apify-client": "^0.1.69",+ "apify-shared": "0.0.9","bluebird": "^3.5.0","content-type": "^1.0.3",+ "fs-extra": "^5.0.0","proxy-chain": "^0.1.2",+ "request-promise": "^4.2.2","underscore": "^1.8.3"},"devDependencies": {@@ -54,10 +57,12 @@"babel-cli": "^6.6.5","babel-core": "^6.1.21","babel-eslint": "^8.0.2",+ "babel-polyfill": "^6.26.0","babel-preset-es2015": "^6.1.18","babel-preset-stage-0": "^6.1.18","basic-auth-parser": "^0.0.2","chai": "^4.0.2",+ "chai-as-promised": "^7.1.1","eslint": "^4.11.0","eslint-config-airbnb": "^16.1.0","eslint-config-airbnb-base": "^12.1.0",diff --git a/src/actor.js b/src/actor.jsindex cfcce0873..bc3757d19 100644--- a/src/actor.js+++ b/src/actor.js@@ -1,44 +1,16 @@-import fs from 'fs';-import path from 'path';import _ from 'underscore';-import os from 'os';-import contentTypeParser from 'content-type';import Promise from 'bluebird';+import fs from 'fs';import { checkParamOrThrow } from 'apify-client/build/utils';import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES } from './constants';-import { getPromisePrototype, newPromise, nodeifyPromise, newClient, addCharsetToContentType, isDocker } from './utils';-+import { newPromise, apifyClient, addCharsetToContentType } from './utils';-/* global process, Buffer */--/**- * @memberof module:Apify- * @name client- * @instance- * @description <p>A default instance of the `ApifyClient` class provided- * by the {@link https://www.apify.com/docs/sdk/apify-client-js/latest|apify-client} NPM package.- * The instance is created when the `apify` package is first imported- * and it is configured using the `APIFY_API_BASE_URL`, `APIFY_USER_ID` and `APIFY_TOKEN`- * environment variables.- * After that, the instance is used for all underlying calls to the Apify API- * in functions such as <a href="#module-Apify-getValue">Apify.getValue()</a>- * or <a href="#module-Apify-call">Apify.call()</a>.- * The settings of the client can be globally altered by calling the- * <a href="https://www.apify.com/docs/js/apify-client-js/latest#ApifyClient-setOptions"><code>Apify.client.setOptions()</code></a> function.- * Just be careful, it might have undesired effects on other functions provided by this package.- * </p>- */-export const apifyClient = newClient();--const readFilePromised = Promise.promisify(fs.readFile);-const writeFilePromised = Promise.promisify(fs.writeFile);-const unlinkPromised = Promise.promisify(fs.unlink);-const statPromised = Promise.promisify(fs.stat);+/* globals process *//*** Tries to parse a string with date.- * @param str Date string- * @returns Returns either a Date object or undefined+ * Returns either a Date object or undefined+ ** @ignore*/const tryParseDate = (str) => {@@ -46,316 +18,12 @@ const tryParseDate = (str) => {return unix > 0 ? new Date(unix) : undefined;};-const getDefaultStoreIdOrThrow = () => {- const storeId = process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];- if (!storeId) throw new Error(`The '${ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID}' environment variable is not defined.`);- return storeId;-};--const getDefaultDatasetIdOrThrow = () => {- const datasetId = process.env[ENV_VARS.DEFAULT_DATASET_ID];- if (!datasetId) throw new Error(`The '${ENV_VARS.DEFAULT_DATASET_ID}' environment variable is not defined.`);- return datasetId;-};--/**- * @memberof module:Apify- * @function- * @description <p>Gets a value from the default key-value store for the current act run using the Apify API.- * The key-value store is created automatically for each act run- * and its ID is passed by the Actor platform in the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.- * It is used to store input and output of the act under keys named `INPUT` and `OUTPUT`, respectively.- * However, the store can be used for storage of any other values under arbitrary keys.- * </p>- * <p>Example usage</p>- * <pre><code class="language-javascript">const input = await Apify.getValue('INPUT');+ * Returns a new object which contains information parsed from the `APIFY_XXX` environment variables.+ * It has the following properties:*- * console.log('My input:');- * console.dir(input);- * </code></pre>- * <p>- * The result of the function is the body of the record. Bodies with the `application/json`- * content type are automatically parsed to an object.- * Similarly, for `text/plain` content types the body is parsed as `String`.- * For all other content types, the body is a raw `Buffer`.- * If the record cannot be found, the result is null.- * </p>- * <p>- * If the `APIFY_DEV_KEY_VALUE_STORE_DIR` environment variable is defined,- * the value is read from a that directory rather than the key-value store,- * specifically from a file that has the key as a name.- * The directory must exist or an error is thrown. If the file does not exists, the returned value is `null`.- * The file is assumed to have a content type specified in the `APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE`- * environment variable, or `application/json` if not set.- * This feature is useful for local development and debugging of your acts.- * </p>- * @param {String} key Key of the record.- * @param {Function} callback Optional callback.- * @returns {Promise} Returns a promise if no callback was provided.- */-export const getValue = (key, callback = null) => {- if (!key || !_.isString(key)) throw new Error('The "key" parameter must be a non-empty string');-- const devDir = process.env[ENV_VARS.DEV_KEY_VALUE_STORE_DIR];- let promise;-- if (devDir) {- // We're emulating KV store locally in a directory to simplify development- const devContentType = process.env[ENV_VARS.DEV_KEY_VALUE_STORE_CONTENT_TYPE] || 'application/json; charset=utf-8';- const contentType = contentTypeParser.parse(devContentType).type;- const dirPath = path.resolve(devDir);- let filePath;-- promise = newPromise()- .then(() => {- // Check that the directory is really a directory- return statPromised(dirPath)- .then((stats) => {- if (!stats.isDirectory()) throw new Error('The directory is not a directory');- })- .catch((err) => {- if (err.code === 'ENOENT') throw new Error('The directory does not exist');- throw err;- });- })- .then(() => {- // Read file- filePath = path.resolve(dirPath, key);- return readFilePromised(filePath)- .catch((err) => {- if (err.code === 'ENOENT') return null;- throw err;- });- })- .then((data) => {- // Parse file according to the content type- if (data !== null) {- if (contentType === 'application/json') {- try {- data = JSON.parse(data.toString('utf8'));- } catch (e) {- throw new Error(`File cannot be parsed as JSON: ${e.message}`);- }- } else if (contentType === 'text/plain') {- data = data.toString();- }- }- return data;- })- .catch((err) => {- throw new Error(`Error reading file '${key}' in directory '${dirPath}' referred by ${ENV_VARS.DEV_KEY_VALUE_STORE_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len- });- } else {- const storeId = getDefaultStoreIdOrThrow();- const promisePrototype = getPromisePrototype();-- promise = newPromise()- .then(() => {- return apifyClient- .keyValueStores- .getRecord({- storeId,- promise: promisePrototype,- key,- })- .then(output => (output ? output.body : null));- });- }-- return nodeifyPromise(promise, callback);-};---/**- * @memberof module:Apify- * @function- * @description <p>Stores a value in the default key-value store for the current act run using the Apify API.- * The data is stored in the key-value store created specifically for the act run,- * whose ID is defined in the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.- * The function has no result, but throws on invalid args or other errors.</p>- * <pre><code class="language-javascript">await Apify.setValue('OUTPUT', { someValue: 123 });</code></pre>- * <p>- * By default, `value` is converted to JSON and stored with the `application/json; charset=utf-8` content type.- * To store a value with another content type, pass it in the options as follows:- * </p>- * <pre><code class="language-javascript">await Apify.setValue('OUTPUT', 'my text data', { contentType: 'text/plain' });</code></pre>- * <p>- * In this case, the value must be a string or Buffer.- * </p>- * <p>- * If the `APIFY_DEV_KEY_VALUE_STORE_DIR` environment variable is defined,- * the value is written to that local directory rather than the key-value store on Apify cloud,- * to a file named as the key. This is useful for local development and debugging of your acts.- * </p>- * <p>- * **IMPORTANT: Do not forget to use the `await` keyword when calling `Apify.setValue()`,- * otherwise the act process might finish before the value is stored!**- * </p>- * @param key Key of the record- * @param value Value of the record:- * <ul>- * <li>If `null`, the record in the key-value store is deleted.</li>- * <li>If no `options.contentType` is specified, `value` can be any object and it will be stringified to JSON.</li>- * <li>If `options.contentType` is specified, `value` is considered raw data and it must be a String or Buffer.</li>- * </ul>- * For any other value an error will be thrown.- * @param {Object} [options]- * @param {String} [options.contentType] - Sets the MIME content type of the value.- * @param {Function} [callback] Optional callback. Function returns a promise if not provided.- * @returns {Promise} Returns a promise if `callback` was not provided.- */-export const setValue = (key, value, options, callback = null) => {- if (!key || !_.isString(key)) throw new Error('The "key" parameter must be a non-empty string');-- // contentType is optional- if (_.isFunction(options)) {- callback = options;- options = null;- }-- if (typeof (options) !== 'object' && options !== undefined) throw new Error('The "options" parameter must be an object, null or undefined.');- // Make copy of options, don't update what user passed- options = Object.assign({}, options);-- const promisePrototype = getPromisePrototype();-- let storeId = null;-- // Handle emulation of KV store locally in a directory to simplify development- const devDir = process.env[ENV_VARS.DEV_KEY_VALUE_STORE_DIR];- let devDirPath;- let devFilePath;- if (devDir) {- // Get absolute paths- devDirPath = path.resolve(devDir);- devFilePath = path.resolve(devDirPath, key);- } else {- // This would throw if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var was not set- storeId = getDefaultStoreIdOrThrow();- }- const devErrorHandler = (err) => {- throw new Error(`Error writing file '${key}' in directory '${devDirPath}' referred by ${ENV_VARS.DEV_KEY_VALUE_STORE_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len- };-- let innerPromise;-- if (value !== null) {- // Normal case: put record to store- // If contentType is missing, value will be stringified to JSON- if (options.contentType === null || options.contentType === undefined) {- options.contentType = 'application/json';- try {- // Format JSON to simplify debugging, the overheads with compression is negligible- value = JSON.stringify(value, null, 2);- } catch (e) {- // Give more meaningful error message- if (e.message && e.message.indexOf('Invalid string length') >= 0) {- e.message = 'Object is too large';- }- throw new Error(`The "value" parameter cannot be stringified to JSON: ${e.message}`);- }- if (value === undefined) {- throw new Error('The "value" parameter cannot be stringified to JSON.');- }- }-- if (!options.contentType || !_.isString(options.contentType)) {- throw new Error('The "options.contentType" parameter must be a non-empty string, null or undefined.');- }- if (!_.isString(value) && !Buffer.isBuffer(value)) {- throw new Error('The "value" parameter must be a String or Buffer when "contentType" is specified.');- }-- if (devFilePath) {- innerPromise = writeFilePromised(devFilePath, value)- .catch(devErrorHandler);- } else {- // Keep this code in main scope so that simple errors are thrown rather than rejected promise.- innerPromise = apifyClient.keyValueStores.putRecord({- storeId,- promise: promisePrototype,- key,- body: value,- contentType: addCharsetToContentType(options.contentType),- });- }- } else {- // Special case: remove the record from the store- if (options.contentType !== null && options.contentType !== undefined) {- throw new Error('The "options.contentType" parameter must not be used when removing the record.');- }- if (devFilePath) {- innerPromise = unlinkPromised(devFilePath)- .catch(devErrorHandler);- } else {- innerPromise = apifyClient.keyValueStores.deleteRecord({- storeId,- promise: promisePrototype,- key,- });- }- }-- const promise = newPromise().then(() => innerPromise);- return nodeifyPromise(promise, callback);-};--/**- * @memberof module:Apify- * @function- * @description <p>Stores object or an array of objects in the default dataset for the current act run using the Apify API- * The function has no result, but throws on invalid args or other errors.</p>- * <pre><code class="language-javascript">await Apify.pushData(data);</code></pre>- * <p>- * The data is stored in default dataset associated with this act.- * </p>- * <p>- * **IMPORTANT: Do not forget to use the `await` keyword when calling `Apify.pushData()`,- * otherwise the act process might finish before the data is stored!**- * </p>- * @param {Object|Array} data Object or array of objects containing data to by stored in the dataset- * @param {Function} [callback] Optional callback. Function returns a promise if not provided.- * @returns {Promise} Returns a promise if `callback` was not provided.- */-export const pushData = (data, callback = null) => {- if (!data || (!_.isObject(data) && !_.isArray(data))) throw new Error('The "data" parameter must be an object or array');- if (callback && !_.isFunction(callback)) throw new Error('If provided then the "callback" parameter must be a function');-- const promisePrototype = getPromisePrototype();-- let stringifiedData;- try {- // Format JSON to simplify debugging, the overheads with compression is negligible- stringifiedData = JSON.stringify(data, null, 2);- } catch (e) {- throw new Error(`The "data" parameter cannot be stringified to JSON: ${e.message}`);- }- if (stringifiedData === undefined) {- throw new Error('The "data" parameter cannot be stringified to JSON.');- }-- const datasetId = getDefaultDatasetIdOrThrow();-- const innerPromise = apifyClient.datasets.putItems({- datasetId,- promise: promisePrototype,- data,- });--- // TODO: Emulation of sequential store for local development- const promise = newPromise().then(() => innerPromise);- return nodeifyPromise(promise, callback);-};---/**- * @memberof module:Apify- * @function- * @description <p>Returns a new object which contains information parsed from the `APIFY_XXX` environment variables.- * It has the following properties:</p>- * <pre><code class="language-javascript">{+ * ```javascript+ * {* // ID of the act (APIFY_ACT_ID)* actId: String,* @@ -388,11 +56,16 @@ export const pushData = (data, callback = null) => {* // in megabytes (APIFY_MEMORY_MBYTES)* memoryMbytes: Number,* }- * </code></pre>+ * ```* For the list of the `APIFY_XXX` environment variables, see* {@link http://localhost/docs/actor.php#run-env-vars|Actor documentation}.* If some of the variables is not defined or is invalid, the corresponding value in the resulting object will be null.+ ** @returns {Object}+ *+ * @memberof module:Apify+ * @function+ * @name getEnv*/export const getEnv = () => {// NOTE: Don't throw if env vars are invalid to simplify local development and debugging of acts@@ -412,10 +85,9 @@ export const getEnv = () => {};/**- * @memberof module:Apify- * @function- * @description <p>Runs a user function that performs the logic of the act.- * The `Apify.main(userFunct)` function does the following actions:</p>+ * Runs a user function that performs the logic of the act.+ * The `Apify.main(userFunct)` function does the following actions:+ ** <ol>* <li>Invokes the user function passed as the `userFunc` parameter</li>* <li>If the user function returned a promise, waits for it to resolve</li>@@ -423,15 +95,16 @@ export const getEnv = () => {* prints error details to console so that they are stored to the log file</li>* <li>Exits the process</li>* </ol>- * <p>+ ** In the simplest case, the user function is synchronous:- * </p>+ ** ```javascript* Apify.main(() => {* // My synchronous function that returns immediately* });* ```- * <p>If the user function returns a promise, it is considered as asynchronous:</p>+ *+ * If the user function returns a promise, it is considered as asynchronous:* ```javascript* const request = require('request-promise');* Apify.main(() => {@@ -445,7 +118,9 @@ export const getEnv = () => {* });* });* ```- * <p>To simplify your code, you can take advantage of the `async`/`await` keywords:</p>+ *+ * To simplify your code, you can take advantage of the `async`/`await` keywords:+ ** ```javascript* const request = require('request-promise');* Apify.main(async () => {@@ -453,9 +128,15 @@ export const getEnv = () => {* console.log(html);* });* ```+ ** Note that the use of `Apify.main()` in acts is optional;* the function is provided merely for user convenience and acts don't need to use it.+ ** @param userFunc {Function} User function to be executed+ *+ * @memberof module:Apify+ * @function+ * @name main*/export const main = (userFunc) => {if (!userFunc || typeof (userFunc) !== 'function') {@@ -475,7 +156,7 @@ export const main = (userFunc) => {// Set dummy interval to ensure the process will not be killed while awaiting empty promise:// await new Promise(() => {})- // Such a construct is used to for testing of act timeouts and aborts.+ // Such a construct is used for testing of act timeouts and aborts.const intervalId = setInterval(_.noop, 9999999);try {@@ -504,11 +185,10 @@ export const main = (userFunc) => {// TODO: this should rather be called Apify.listeningOnPort() or something like that/**- * @ignore- * @memberof module:Apify- * @function- * @description Notifies Apify runtime that act is listening on port specified by the APIFY_INTERNAL_PORT environment+ * Notifies Apify runtime that act is listening on port specified by the APIFY_INTERNAL_PORT environment* variable and is ready to receive a HTTP request with act input.+ *+ * @ignore*/export const readyFreddy = () => {const watchFileName = process.env[ENV_VARS.WATCH_FILE];@@ -522,69 +202,74 @@ export const readyFreddy = () => {};/**- * @memberof module:Apify- * @function- * @description <p>Executes another act under the current user account, waits for the act finish and fetches its output.</p>- * <p>The result of the function is an object describing the act run, which looks something like this:</p>+ * Executes another act under the current user account, waits for the act finish and fetches its output.+ *+ * The result of the function is an object describing the act run, which looks something like this:+ ** ```json* {- * "id": "ErYkuTTsmKiXccNGT",- * "actId": "E2jjCZBezvAZnX8Rb",- * "userId": "mb7q2dycFBHDhae6A",- * "startedAt": "2017-10-25T14:23:44.376Z",- * "finishedAt": "2017-10-25T14:23:46.723Z",- * "status": "SUCCEEDED",- * "meta": { "origin": "API", "clientIp": "1.2.3.4", "userAgent": null },- * "stats": {- * "netRxBytes": 180,- * "netTxBytes": 0,- * ...- * },- * "options": {- * "build": "latest",- * "timeoutSecs": 0,- * "memoryMbytes": 512,- * "diskMbytes": 1024- * },- * "buildId": "Bwkqk59MCkdexDP34",- * "exitCode": 0,- * "defaultKeyValueStoreId": "ccFfRptZru2uqdQHP",- * "buildNumber": "0.1.2",- * "output": {- * "contentType": "application/json; charset=utf-8",- * "body": { "message": "Hello world!" }- * }+ * "id": "ErYkuTTsmKiXccNGT",+ * "actId": "E2jjCZBezvAZnX8Rb",+ * "userId": "mb7q2dycFBHDhae6A",+ * "startedAt": "2017-10-25T14:23:44.376Z",+ * "finishedAt": "2017-10-25T14:23:46.723Z",+ * "status": "SUCCEEDED",+ * "meta": { "origin": "API", "clientIp": "1.2.3.4", "userAgent": null },+ * "stats": {+ * "netRxBytes": 180,+ * "netTxBytes": 0,+ * ...+ * },+ * "options": {+ * "build": "latest",+ * "timeoutSecs": 0,+ * "memoryMbytes": 512,+ * "diskMbytes": 1024+ * },+ * "buildId": "Bwkqk59MCkdexDP34",+ * "exitCode": 0,+ * "defaultKeyValueStoreId": "ccFfRptZru2uqdQHP",+ * "buildNumber": "0.1.2",+ * "output": {+ * "contentType": "application/json; charset=utf-8",+ * "body": { "message": "Hello world!" }+ * }* }* ```- * <p>Internally, the function calls the {@link https://www.apify.com/docs/api/v2#/reference/acts/runs-collection/run-act|Run act} API endpoint- * and few others.</p>- * <p>Example usage:</p>+ * Internally, the function calls the {@link https://www.apify.com/docs/api/v2#/reference/acts/runs-collection/run-act|Run act} API endpoint+ * and few others.+ *+ * Example usage:+ ** ```javascript* const run = await Apify.call('apify/hello-world', { myInput: 123 });* console.log(`Received message: ${run.output.body.message}`);* ```*- * @param {String} actId - Either `username/act-name` or act ID. If you use the former format,- * beware that the user needs to have username set!- * @param {Object|String|Buffer} [input] - Act input body. If it is an object, it is stringified to- * JSON and the content type set to `application/json; charset=utf-8`.+ * @param {String} actId Either `username/act-name` or act ID.+ * @param {Object|String|Buffer} [input] Act input body. If it is an object, it is stringified to+ * JSON and the content type set to `application/json; charset=utf-8`.* @param {Object} [opts]- * @param {String} [opts.token] - User API token. By default, it is taken from the `APIFY_TOKEN` environment variable.- * @param {String} [opts.build] - Tag or number of act build to be run (e.g. `beta` or `1.2.345`).- * If not provided, the default build tag or number from act configuration is used (typically `latest`).- * @param {String} [opts.contentType] - Content type for the `input`. If not specified,- * `input` is expected to be an object that will be stringified to JSON and content type set to- * `application/json; charset=utf-8`. If `opts.contentType` is specified, then `input` must be a `String` or `Buffer`.- * @param {String} [opts.timeoutSecs] - Time limit for act to finish, in seconds.- * If the limit is reached the resulting run will have the `RUNNING` status.- * By default, there is no timeout.- * @param {String} [opts.fetchOutput] - If `false` then the function does not fetch output of the act. Default is `true`.- * @param {String} [opts.disableBodyParser] - If `true` then the function will not attempt to parse the- * act's output and will return it in a raw `Buffer`. Default is `false`.- * @param {Function} [callback] - Optional callback. Function returns a promise if not provided.- * @returns {Promise} Returns a promise unless `callback` was supplied.+ * @param {String} [opts.token] User API token. By default, it is taken from the `APIFY_TOKEN` environment variable.+ * @param {String} [opts.build] Tag or number of act build to be run (e.g. `beta` or `1.2.345`).+ * If not provided, the default build tag or number from act configuration is used (typically `latest`).+ * @param {String} [opts.contentType] Content type for the `input`. If not specified,+ * `input` is expected to be an object that will be stringified to JSON and content type set to+ * `application/json; charset=utf-8`. If `opts.contentType` is specified, then `input` must be a+ * `String` or `Buffer`.+ * @param {String} [opts.timeoutSecs] Time limit for act to finish, in seconds.+ * If the limit is reached the resulting run will have the `RUNNING` status.+ * By default, there is no timeout.+ * @param {String} [opts.fetchOutput=true] If `false` then the function does not fetch output of the act.+ * @param {String} [opts.disableBodyParser=false] If `true` then the function will not attempt to parse the+ * act's output and will return it in a raw `Buffer`.+ * @returns {Promise}+ *+ * @memberof module:Apify+ * @function+ * @name call*/-export const call = (actId, input, opts = {}, callback) => {+export const call = (actId, input, opts = {}) => {const { acts, keyValueStores } = apifyClient;checkParamOrThrow(actId, 'actId', 'String');@@ -671,57 +356,52 @@ export const call = (actId, input, opts = {}, callback) => {});};- const promise = acts+ return acts.runAct(Object.assign({}, defaultOpts, runActOpts)).then(run => waitForRunToFinish(run));-- return nodeifyPromise(promise, callback);};-/**- * @memberof module:Apify- * @function- * @description Returns memory statistics of the container, which is an object with the following properties:- * ```javascript- * {- * // Total memory available to the act- * totalBytes: Number,+ * Returns a url of Apify Proxy that can be used from Actor acts, web browsers or any other HTTP+ * proxy-enabled applications.*- * // Amount of free memory- * freeBytes: Number,+ * @param {Object} opts+ * @param {String} opts.password User proxy password. By default, it is taken from the `APIFY_PROXY_PASSWORD` environment variable.+ * @param {String} [opts.groups] Proxy groups to be used.+ * @param {String} [opts.session] Session ID that identifies requests that should use the same proxy connection.+ * @returns {String} Returns proxy url.*- * // Amount of memory used (= totalBytes - freeBytes)- * usedBytes: Number,- * }- * ```- *- * @returns {Promise} Returns a promise unless `callback` was supplied.+ * @memberof module:Apify+ * @function+ * @name getApifyProxyUrl*/-export const getMemoryInfo = (callback) => {- // This must be promisified here so that we can Mock it.- const readPromised = Promise.promisify(fs.readFile);- const promise = isDocker()- .then((isDockerVar) => {- if (!isDockerVar) {- const freeBytes = os.freemem();- const totalBytes = os.totalmem();-- return Promise.resolve({ totalBytes, freeBytes, usedBytes: totalBytes - freeBytes });- }-- return Promise- .all([- readPromised('/sys/fs/cgroup/memory/memory.limit_in_bytes'),- readPromised('/sys/fs/cgroup/memory/memory.usage_in_bytes'),- ])- .then(([totalBytesStr, usedBytesStr]) => {- const totalBytes = parseInt(totalBytesStr, 10);- const usedBytes = parseInt(usedBytesStr, 10);-- return { totalBytes, freeBytes: totalBytes - usedBytes, usedBytes };- });- });+export const getApifyProxyUrl = (opts = {}) => {+ const {+ groups,+ session,+ password = process.env[ENV_VARS.PROXY_PASSWORD],+ hostname = process.env[ENV_VARS.PROXY_HOSTNAME],+ port = parseInt(process.env[ENV_VARS.PROXY_PORT], 10),+ } = opts;++ checkParamOrThrow(groups, 'opts.groups', 'Maybe Array');+ checkParamOrThrow(session, 'opts.session', 'Maybe Number | String');+ checkParamOrThrow(password, 'opts.password', 'String');+ checkParamOrThrow(hostname, 'opts.hostname', 'String');+ checkParamOrThrow(port, 'opts.port', 'Number');++ let username;++ if (groups || session) {+ const parts = [];++ if (groups && groups.length) parts.push(`GROUPS-${groups.join('+')}`);+ if (session) parts.push(`SESSION-${session}`);++ username = parts.join(',');+ } else {+ username = 'auto';+ }- return nodeifyPromise(promise, callback);+ return `http://${username}:${password}@${hostname}:${port}`;};diff --git a/src/autoscaled_pool.js b/src/autoscaled_pool.jsnew file mode 100644index 000000000..063eca346--- /dev/null+++ b/src/autoscaled_pool.js@@ -0,0 +1,269 @@+import Promise from 'bluebird';+import { log } from 'apify-shared';+import _ from 'underscore';+import { checkParamOrThrow } from 'apify-client/build/utils';+import { getMemoryInfo, isPromise } from './utils';++const MEM_CHECK_INTERVAL_MILLIS = 200; // This is low to have at least.+const MAYBE_RUN_INTERVAL_MILLIS = 1000;+const MIN_FREE_MEMORY_RATIO = 0.1; // Minimum amount of memory that we keep free.+const DEFAULT_OPTIONS = {+ maxConcurrency: 1000,+ minConcurrency: 1,+};++// These constants defines that in Nth execution memCheckInterval we do:+export const SCALE_UP_INTERVAL = 50;+export const SCALE_UP_MAX_STEP = 10;+export const SCALE_DOWN_INTERVAL = 5;+export const LOG_INFO_INTERVAL = 6 * SCALE_UP_INTERVAL; // This must be multiple of SCALE_UP_INTERVAL++/**+ * Helper function that coverts bytes into human readable MBs.+ *+ * @ignore+ */+const humanReadable = bytes => `${Math.round(bytes / 1024 / 1024)} MB`;++/**+ * AutoscaledPool helps to process asynchronous task in parallel. It scales the number of concurrent tasks based on+ * the available memory. If any of the tasks throws an error then the pool.run() method+ * also throws.+ *+ * Basic usage of AutoscaledPool:+ *+ * ```javascript+ * const pool = new Apify.AutoScaledPool({+ * maxConcurrency: 50,+ * workerFunction: async () => {+ * // ... do some intensive asynchronous operations here and return a promise ...+ * },+ * });+ *+ * await pool.run();+ * ```+ *+ * @param {Object} options+ * @param {Function} options.workerFunction Function we want to call in parallel. This function must either return a+ * promise or null when all the tasks were processed.+ * @param {Number} [options.maxConcurrency=1000] Maximal concurrency.+ * @param {Number} [options.minConcurrency=1] Minimal concurrency.+ * @param {Number} [options.maxMemoryMbytes] Maximum memory available in the system. By default uses the totalMemory from Apify.getMemoryInfo().+ */+export default class AutoscaledPool {+ constructor(opts) {+ const { maxMemoryMbytes, maxConcurrency, minConcurrency, workerFunction } = _.defaults(opts, DEFAULT_OPTIONS);++ checkParamOrThrow(maxMemoryMbytes, 'opts.maxMemoryMbytes', 'Maybe Number');+ checkParamOrThrow(maxConcurrency, 'opts.maxConcurrency', 'Number');+ checkParamOrThrow(minConcurrency, 'opts.minConcurrency', 'Number');+ checkParamOrThrow(workerFunction, 'opts.workerFunction', 'Function');++ // Configuration.+ this.maxMemoryMbytes = maxMemoryMbytes;+ this.maxConcurrency = maxConcurrency;+ this.minConcurrency = Math.min(minConcurrency, maxConcurrency);+ this.workerFunction = workerFunction;++ // State.+ this.promiseCounter = 0;+ this.intervalCounter = 0;+ this.concurrency = minConcurrency;+ this.runningPromises = {};+ this.runningCount = 0;+ this.freeBytesSnapshots = [];++ // Intervals.+ this.memCheckInterval = null;+ this.maybeRunInterval = null;++ // This is resolve function of Promise returned by this.run()+ // which gets resolved once everything is done.+ this.resolve = null;+ this.reject = null;+ }++ /**+ * Runs the autoscaled pool. Returns promise that gets resolved or rejected once+ * all the task got finished or some of them fails.+ *+ * @return {Promise}+ */+ run() {+ const promise = new Promise((resolve, reject) => {+ this.resolve = resolve;+ this.reject = reject;+ this._maybeRunPromise();++ // This is here because if we scale down to let's say 1. Then after each promise is finished+ // this._maybeRunPromise() doesn't trigger another one. So if that 1 instance stucks it results+ // in whole act to stuck and even after scaling up it never triggers another promise.+ this.maybeRunInterval = setInterval(() => this._maybeRunPromise(), MAYBE_RUN_INTERVAL_MILLIS);++ // This interval checks memory and in each call saves current memory stats and in every+ // SCALE_UP_INTERVAL-th/SCALE_DOWN_INTERVAL-th call it may scale up/down based on memory.+ this.memCheckInterval = setInterval(() => this._autoscale(), MEM_CHECK_INTERVAL_MILLIS);+ });++ return promise+ .then(() => this._destroy())+ .catch((err) => {+ this._destroy();++ throw err;+ });+ }++ /**+ * Gets called every MEM_CHECK_INTERVAL_MILLIS and saves number of free bytes in this.freeBytesSnapshots.+ *+ * Every:+ * - SCALE_DOWN_INTERVAL-th call checks memory and possibly scales DOWN by 1.+ * - SCALE_UP_INTERVAL-th call checks memory and possibly scales UP+ * - MEM_INFO_INTERVAL-th call logs statistics about memory.+ *+ * @ignore+ */+ _autoscale() {+ getMemoryInfo()+ .then(({ freeBytes, totalBytes }) => {+ if (this.maxMemoryMbytes) totalBytes = Math.min(totalBytes, this.maxMemoryMbytes);++ this.intervalCounter++;+ this.freeBytesSnapshots = this.freeBytesSnapshots.concat(freeBytes).slice(-SCALE_UP_INTERVAL);++ // Maybe scale down.+ if (+ this.intervalCounter % SCALE_DOWN_INTERVAL === 0+ && (freeBytes / totalBytes < MIN_FREE_MEMORY_RATIO)+ && this.concurrency > this.minConcurrency+ ) {+ this.concurrency--;+ log.debug('AutoscaledPool: scaling down', { concurrency: this.concurrency });+ }++ // Maybe scale up.+ if (+ this.intervalCounter % SCALE_UP_INTERVAL === 0+ && this.concurrency < this.maxConcurrency+ ) {+ const spaceForInstances = this._computeSpaceForInstances(totalBytes, this.intervalCounter % LOG_INFO_INTERVAL);++ if (spaceForInstances > 0) {+ const increaseBy = Math.min(spaceForInstances, SCALE_UP_MAX_STEP);+ const oldConcurrency = this.concurrency;+ this.concurrency = Math.min(this.concurrency + increaseBy, this.maxConcurrency);+ log.debug('AutoscaledPool: scaling up', { oldConcurrency, newConcurrency: this.concurrency });+ }+ }+ });+ }++ /**+ * Gets memory info and computes how much we can scale pool+ * to avoid exceeding the maximum memory.+ *+ * If shouldLogInfo = true then also logs info about memory usage.+ *+ * @ignore+ */+ _computeSpaceForInstances(totalBytes, logInfo) {+ const minFreeBytes = Math.min(...this.freeBytesSnapshots);+ const minFreeRatio = minFreeBytes / totalBytes;+ const maxTakenBytes = totalBytes - minFreeBytes;+ const perInstanceRatio = (maxTakenBytes / totalBytes) / this.concurrency;+ const hasSpaceForInstances = (minFreeRatio - MIN_FREE_MEMORY_RATIO) / perInstanceRatio;++ if (logInfo) {+ log.info('AutoscaledPool: info', {+ concurency: this.concurrency,+ runningCount: this.runningCount,+ freeBytesSnapshots: humanReadable(_.last(this.freeBytesSnapshots)),+ totalBytes: humanReadable(totalBytes),+ minFreeBytes: humanReadable(minFreeBytes),+ minFreePerc: minFreeRatio,+ maxTakenBytes: humanReadable(maxTakenBytes),+ perInstancePerc: perInstanceRatio,+ hasSpaceForInstances,+ });+ }++ return Math.floor(hasSpaceForInstances);+ }++ /**+ * Registers running promise.+ *+ * @ignore+ */+ _addRunningPromise(id, promise) {+ this.runningPromises[id] = promise;+ this.runningCount++;+ }++ /**+ * Removes finished promise.+ *+ * @ignore+ */+ _removeFinishedPromise(id) {+ delete this.runningPromises[id];+ this.runningCount--;+ }++ /**+ * If this.runningCount < this.concurrency then gets new promise from this.workerFunction() and adds it to the pool.+ * If this.workerFunction() returns null and nothing is running then finishes pool.+ *+ * @ignore+ */+ _maybeRunPromise() {+ if (!this.resolve || !this.reject) return;+ if (this.runningCount >= this.concurrency) return;++ const promise = this.workerFunction();++ // We are done.+ if (!promise && this.runningCount === 0) return this.resolve();++ // We are not done but don't want to execute new promise at this point.+ // This may happen when there are less pages in the queue than max concurrency+ // but all of them are being served already.+ if (!promise) return;++ // It's not null so it must be a promise!+ if (!isPromise(promise)) throw new Error('User provided workerFunction must return a Promise.');++ const id = this.promiseCounter;+ this.promiseCounter++;+ this._addRunningPromise(id, promise);++ promise+ .then((data) => {+ this._removeFinishedPromise(id);+ this._maybeRunPromise();++ return data;+ })+ .catch((err) => {+ log.exception(err, 'AutoscaledPool: worker function failed');+ this._removeFinishedPromise(id);+ if (this.reject) this.reject(err);+ });++ this._maybeRunPromise();+ }++ /**+ * Cleanups resources.+ *+ * @ignore+ */+ _destroy() {+ this.resolve = null;+ this.reject = null;++ clearInterval(this.memCheckInterval);+ clearInterval(this.maybeRunInterval);+ }+}diff --git a/src/basic_crawler.js b/src/basic_crawler.jsnew file mode 100644index 000000000..18e7972e9--- /dev/null+++ b/src/basic_crawler.js@@ -0,0 +1,120 @@+import { checkParamOrThrow } from 'apify-client/build/utils';+import _ from 'underscore';+import { log } from 'apify-shared';+import { isPromise, checkParamPrototypeOrThrow } from './utils';+import AutoscaledPool from './autoscaled_pool';+import RequestList from './request_list';++const DEFAULT_OPTIONS = {+ maxRequestRetries: 3,+ handleFailedRequestFunction: ({ request }) => log.error('Request failed', _.pick(request, 'url', 'uniqueKey')),+};++/**+ * BasicCrawler provides a simple framework for parallel crawling of a url list provided by `Apify.RequestList`+ * or a dynamically enqueued requests provided by `Apify.RequestQueue` (TODO).+ *+ * It's simply calling handleRequestFunction for each request from requestList or requestQueue as long as+ * both are not empty. The concurrency is scaled based on available memory using `Apify.AutoscaledPool` and all+ * of it's configuration parameters are supported here.+ *+ * Basic usage of BasicCrawler:+ *+ * ```javascript+ * const request = require('request-promise');+ *+ * const crawler = new Apify.BasicCrawler({+ * urlList,+ * handleRequestFunction: async ({ request }) => {+ * await Apify.pushData({+ * html: await request(request.url),+ * url: request.url,+ * })+ * },+ * });+ *+ * await crawler.run();+ * ```+ *+ * @param {Object} options+ * @param {RequestList} options.requestList List of the requests to be processed.+ * @param {Function} options.handleRequestFunction Function that processes a request. It must return a promise.+ * @param {Function} [options.handleFailedRequestFunction=({ request }) => log.error('Request failed', _.pick(request, 'url', 'uniqueKey'))`]+ * Function to handle requests that failed more then option.maxRequestRetries times.+ * @param {Number} [options.maxRequestRetries=3] How many times request is retried if handleRequestFunction failed.+ * @param {Number} [options.maxMemoryMbytes] Maximal memory available in the system (see `maxMemoryMbytes` parameter of `Apify.AutoscaledPool`).+ * @param {Number} [options.maxConcurrency=1] Minimal concurrency of requests processing (see `maxConcurrency` parameter of `Apify.AutoscaledPool`).+ * @param {Number} [options.minConcurrency=1000] Maximal concurrency of request processing (see `minConcurrency` parameter of `Apify.AutoscaledPool`).+ */+export default class BasicCrawler {+ constructor(opts) {+ const {+ requestList,+ handleRequestFunction,+ handleFailedRequestFunction,+ maxRequestRetries,++ // Autoscaled pool options+ maxMemoryMbytes,+ maxConcurrency,+ minConcurrency,+ } = _.defaults(opts, DEFAULT_OPTIONS);++ checkParamOrThrow(handleRequestFunction, 'opts.handleRequestFunction', 'Function');+ checkParamOrThrow(handleFailedRequestFunction, 'opts.handleFailedRequestFunction', 'Function');+ checkParamOrThrow(maxRequestRetries, 'opts.maxRequestRetries', 'Number');+ // TODO: make this optional once we have the request queue ready.+ checkParamPrototypeOrThrow(requestList, 'opts.requestList', RequestList, 'Apify.RequestList');++ this.requestList = requestList;+ this.handleRequestFunction = handleRequestFunction;+ this.handleFailedRequestFunction = handleFailedRequestFunction;+ this.maxRequestRetries = maxRequestRetries;++ this.autoscaledPool = new AutoscaledPool({+ workerFunction: () => this._workerFunction(),+ maxMemoryMbytes,+ maxConcurrency,+ minConcurrency,+ });+ }++ /**+ * Runs the crawler. Returns promise that gets resolved once all the requests got processed.+ *+ * @return {Promise}+ */+ run() {+ return this.autoscaledPool.run();+ }++ /**+ * Wrapper around handleRequestFunction that catches errors and retries requests.+ *+ * @ignore+ */+ _workerFunction() {+ const request = this.requestList.fetchNextRequest();++ if (!request) return;++ const handlePromise = this.handleRequestFunction({ request });+ if (!isPromise(handlePromise)) throw new Error('User provided handleRequestFunction must return a Promise.');++ return handlePromise.catch((err) => {+ request.pushErrorMessage(err);++ // Retry request.+ if (request.retryCount < this.maxRequestRetries) {+ request.retryCount++;+ this.requestList.reclaimRequest(request);+ return;+ }++ // Mark as failed.+ this.requestList.markRequestHandled(request);++ return this.handleFailedRequestFunction({ request });+ });+ }+}diff --git a/src/constants.js b/src/constants.jsindex 6e5ea00d3..38b94b251 100644--- a/src/constants.js+++ b/src/constants.js@@ -1,12 +1,13 @@+import * as consts from 'apify-shared/consts';-export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'; // eslint-disable-line max-len+consts.DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'; // eslint-disable-line max-len/*** Exit codes for the act process.* The error codes must be in range 1-128, to avoid collision with signal exits* and to ensure Docker will handle them correctly!*/-export const EXIT_CODES = {+consts.EXIT_CODES = {SUCCESS: 0,ERROR_USER_FUNCTION_THREW: 91,ERROR_UNKNOWN: 92,@@ -15,17 +16,19 @@ export const EXIT_CODES = {/*** Dictionary of APIFY_XXX environment variable names.*/-export const ENV_VARS = {+consts.ENV_VARS = {ACT_ID: 'APIFY_ACT_ID',ACT_RUN_ID: 'APIFY_ACT_RUN_ID',USER_ID: 'APIFY_USER_ID',TOKEN: 'APIFY_TOKEN',+ PROXY_PASSWORD: 'APIFY_PROXY_PASSWORD',+ PROXY_HOSTNAME: 'APIFY_PROXY_HOSTNAME',+ PROXY_PORT: 'APIFY_PROXY_PORT',STARTED_AT: 'APIFY_STARTED_AT',TIMEOUT_AT: 'APIFY_TIMEOUT_AT',DEFAULT_KEY_VALUE_STORE_ID: 'APIFY_DEFAULT_KEY_VALUE_STORE_ID',DEFAULT_DATASET_ID: 'APIFY_DEFAULT_DATASET_ID',- DEV_KEY_VALUE_STORE_DIR: 'APIFY_DEV_KEY_VALUE_STORE_DIR',- DEV_KEY_VALUE_STORE_CONTENT_TYPE: 'APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE',+ LOCAL_EMULATION_DIR: 'APIFY_LOCAL_EMULATION_DIR',WATCH_FILE: 'APIFY_WATCH_FILE',API_BASE_URL: 'APIFY_API_BASE_URL',HEADLESS: 'APIFY_HEADLESS',@@ -33,31 +36,9 @@ export const ENV_VARS = {MEMORY_MBYTES: 'APIFY_MEMORY_MBYTES',};-export const KEY_VALUE_STORE_KEYS = {+consts.KEY_VALUE_STORE_KEYS = {INPUT: 'INPUT',OUTPUT: 'OUTPUT',};-/**- * Dictionary of possible values for 'status' field of act2Builds or act2Runs collections.- */-export const ACT_TASK_STATUSES = {- READY: 'READY', // started but not allocated to any worker yet- RUNNING: 'RUNNING', // running on worker- SUCCEEDED: 'SUCCEEDED', // finished and all good- FAILED: 'FAILED', // run or build failed- TIMING_OUT: 'TIMING-OUT', // timing out now- TIMED_OUT: 'TIMED-OUT', // timed out- ABORTING: 'ABORTING', // being aborted by user- ABORTED: 'ABORTED', // aborted by user-};--/**- * An array of act task statuses that are final for the task.- */-export const ACT_TASK_TERMINAL_STATUSES = [- ACT_TASK_STATUSES.SUCCEEDED,- ACT_TASK_STATUSES.FAILED,- ACT_TASK_STATUSES.TIMED_OUT,- ACT_TASK_STATUSES.ABORTED,-];+module.exports = consts;diff --git a/src/dataset.js b/src/dataset.jsnew file mode 100644index 000000000..61ff60b2f--- /dev/null+++ b/src/dataset.js@@ -0,0 +1,206 @@+import fs from 'fs';+import path from 'path';+import _ from 'underscore';+import Promise from 'bluebird';+import { leftpad } from 'apify-shared/utilities';+import LruCache from 'apify-shared/lru_cache';+import { checkParamOrThrow } from 'apify-client/build/utils';+import { ENV_VARS } from './constants';+import { apifyClient, ensureDirExists } from './utils';++export const LOCAL_EMULATION_SUBDIR = 'datasets';+export const LEFTPAD_COUNT = 9; // Used for filename in DatasetLocal.+const MAX_OPENED_STORES = 1000;++const writeFilePromised = Promise.promisify(fs.writeFile);+const readdirPromised = Promise.promisify(fs.readdir);++const { datasets } = apifyClient;+const datasetsCache = new LruCache({ maxLength: MAX_OPENED_STORES }); // Open Datasets are stored here.++/**+ * Dataset class provides easy interface to Apify Dataset storage type. Dataset should be opened using+ * `Apify.openDataset()` function.+ *+ * Basic usage of Dataset:+ *+ * ```javascript+ * const dataset = await Apify.openDataset('my-dataset-id');+ * await dataset.pushData({ foo: 'bar' });+ * ```+ *+ * @param {String} datasetId - ID of the dataset.+ */+export class Dataset {+ constructor(datasetId) {+ checkParamOrThrow(datasetId, 'datasetId', 'String');++ this.datasetId = datasetId;+ }++ /**+ * Stores object or an array of objects in the dataset.+ * The function has no result, but throws on invalid args or other errors.+ *+ * @return {Promise} That resolves when data gets saved into the dataset.+ */+ pushData(data) {+ checkParamOrThrow(data, 'data', 'Array | Object');++ return datasets.putItems({+ datasetId: this.datasetId,+ data,+ });+ }+}++/**+ * This is a local representation of a dataset.+ *+ * @ignore+ */+export class DatasetLocal {+ constructor(datasetId, localEmulationDir) {+ checkParamOrThrow(datasetId, 'datasetId', 'String');+ checkParamOrThrow(localEmulationDir, 'localEmulationDir', 'String');++ this.localEmulationPath = path.resolve(path.join(localEmulationDir, LOCAL_EMULATION_SUBDIR, datasetId));+ this.counter = 0;+ this.datasetId = datasetId;+ this.initializationPromise = this._initialize();+ }++ _initialize() {+ return ensureDirExists(this.localEmulationPath)+ .then(() => readdirPromised(this.localEmulationPath))+ .then((files) => {+ if (files.length) {+ const lastFileNum = files.pop().split('.')[0];++ this.counter = parseInt(lastFileNum, 10);+ }+ });+ }++ pushData(data) {+ checkParamOrThrow(data, 'data', 'Array | Object');++ if (!_.isArray(data)) data = [data];++ return this.initializationPromise+ .then(() => {+ const promises = data.map((item) => {+ this.counter++;++ // Format JSON to simplify debugging, the overheads is negligible+ const itemStr = JSON.stringify(item, null, 2);+ const fileName = `${leftpad(this.counter, LEFTPAD_COUNT, 0)}.json`;+ const filePath = path.join(this.localEmulationPath, fileName);++ return writeFilePromised(filePath, itemStr);+ });++ return Promise.all(promises);+ });+ }+}++/**+ * Helper function that first requests dataset by ID and if dataset doesn't exist then gets him by name.+ *+ * @ignore+ */+const getOrCreateDataset = (datasetIdOrName) => {+ return apifyClient+ .datasets+ .getDataset({ datasetId: datasetIdOrName })+ .then((existingDataset) => {+ if (existingDataset) return existingDataset;++ return apifyClient+ .datasets+ .getOrCreateDataset({ datasetName: datasetIdOrName });+ });+};++/**+ * Opens dataset and returns its object.+ *+ * ```javascript+ * const dataset = await Apify.openDataset('my-dataset-id');+ * await dataset.pushData({ foo: 'bar' });+ * ```+ *+ * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is defined, the value this function+ * returns an instance `DatasetLocal` which is an local emulation of dataset.+ * This is useful for local development and debugging of your acts.+ *+ * @param {string} datasetIdOrName ID or name of the dataset to be opened.+ * @returns {Promise<Dataset>} Returns a promise that resolves to a Dataset object.+ *+ * @memberof module:Apify+ * @name openDataset+ * @instance+ */+export const openDataset = (datasetIdOrName) => {+ checkParamOrThrow(datasetIdOrName, 'datasetIdOrName', 'Maybe String');++ const localEmulationDir = process.env[ENV_VARS.LOCAL_EMULATION_DIR];++ let isDefault = false;+ let datasetPromise;++ if (!datasetIdOrName) {+ const envVar = ENV_VARS.DEFAULT_DATASET_ID;++ // Env var doesn't exist.+ if (!process.env[envVar]) return Promise.reject(new Error(`The '${envVar}' environment variable is not defined.`));++ isDefault = true;+ datasetIdOrName = process.env[envVar];+ }++ datasetPromise = datasetsCache.get(datasetIdOrName);++ // Found in cache.+ if (datasetPromise) return datasetPromise;++ // Use local emulation?+ if (localEmulationDir) {+ datasetPromise = Promise.resolve(new DatasetLocal(datasetIdOrName, localEmulationDir));+ } else {+ datasetPromise = isDefault // If true then we know that this is an ID of existing dataset.+ ? Promise.resolve(new Dataset(datasetIdOrName))+ : getOrCreateDataset(datasetIdOrName).then(dataset => (new Dataset(dataset.id)));+ }++ datasetsCache.add(datasetIdOrName, datasetPromise);++ return datasetPromise;+};++/**+ * Stores object or an array of objects in the default dataset for the current act run using the Apify API+ * Default id of the dataset is in the `APIFY_DEFAULT_DATASET_ID` environment variable+ * The function has no result, but throws on invalid args or other errors.+ *+ * ```javascript+ * await Apify.pushData(data);+ * ```+ *+ * The data is stored in default dataset associated with this act.+ *+ * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is defined, the data gets pushed into local directory.+ * This feature is useful for local development and debugging of your acts.+ *+ * **IMPORTANT**: Do not forget to use the `await` keyword when calling `Apify.pushData()`,+ * otherwise the act process might finish before the data is stored!+ *+ * @param {Object|Array} data Object or array of objects containing data to by stored in the dataset+ * @returns {Promise} Returns a promise that gets resolved once data are saved.+ *+ * @memberof module:Apify+ * @name pushData+ * @instance+ */+export const pushData = item => openDataset().then(dataset => dataset.pushData(item));diff --git a/src/index.js b/src/index.jsindex 1af2686c8..d61aa5c38 100644--- a/src/index.js+++ b/src/index.js@@ -1,89 +1,124 @@import EventEmitter from 'events';+import { log } from 'apify-shared';-import { setPromisesDependency, getPromisesDependency } from './utils';-import { main, readyFreddy, getEnv, getValue, setValue, pushData, apifyClient, call, getMemoryInfo } from './actor';+import { main, readyFreddy, getEnv, call, getApifyProxyUrl } from './actor';+import AutoscaledPool from './autoscaled_pool';+import BasicCrawler from './basic_crawler';+import { pushData, openDataset } from './dataset';+import { getValue, setValue, openKeyValueStore } from './key_value_store';import { launchPuppeteer } from './puppeteer';+import PuppeteerCrawler from './puppeteer_crawler';+import PuppeteerPool from './puppeteer_pool';+import Request from './request';+import RequestList from './request_list';+import SettingsRotator from './settings_rotator';+import { apifyClient, getMemoryInfo, isProduction } from './utils';import { browse, launchWebDriver } from './webdriver';/* globals module */-// Publicly available functions-const Apify = {- main,- getEnv,- getValue,- setValue,- pushData,- call,- readyFreddy,- setPromisesDependency,- getPromisesDependency,- getMemoryInfo,- browse,- launchWebDriver,- launchPuppeteer,- client: apifyClient,- events: new EventEmitter(),-};+// Hide debug log messages when running in production mode.+if (!isProduction()) log.isDebugMode = true;/**- * @module Apify- * @description- * <p>* The `apify` NPM package simplifies development of acts in Apify Actor -* a serverless computing platform that enables execution of arbitrary pieces of code in the cloud.* The package provides helper functions to launch web browsers with proxies,* access the storage etc. Note that the usage of the package is optional,* you can create acts without it.- * </p>- * <p>+ ** For more information about the Apify Actor platform, please go to* {@link https://www.apify.com/docs/actor|Actor documentation}.- * The source code of this package is available on {@link https://github.com/apifytech/apify-js|GitHub}.- * </p>++ * The source code of this package is available on {@link https://github.com/apifytech/apify-runtime-js|GitHub}.** <h2>Example usage</h2>- * <pre><code class="language-javascript">const Apify = require('apify');+ *+ * ```javascript* * Apify.main(async () => {- * // Get input of the act- * const input = await Apify.getValue('INPUT');- * console.dir(input);+ * // Get input of the act+ * const input = await Apify.getValue('INPUT');+ * console.dir(input);* - * // Do something useful here...- * const browser = await Apify.launchPuppeteer();- * const page = await browser.newPage();- * await page.goto('http://www.example.com');- * const pageTitle = await page.title();+ * // Do something useful here...+ * const browser = await Apify.launchPuppeteer();+ * const page = await browser.newPage();+ * await page.goto('http://www.example.com');+ * const pageTitle = await page.title();* - * // Save output- * await Apify.setValue('OUTPUT', { pageTitle });- * });</code></pre>+ * // Save output+ * await Apify.setValue('OUTPUT', { pageTitle });+ * });+ * ```** <h2>Installation</h2>- * <p>This package requires Node.js 6 or higher. It might work with lower versions too,+ *+ * This package requires Node.js 6 or higher. It might work with lower versions too,* but they are neither tested nor supported.- * To install the package locally, run the following command in your Node.js project:</p>+ *+ * To install the package locally, run the following command in your Node.js project:+ ** ```bash* npm install apify --save* ```- * <p>+ ** <h2>Promises vs. callbacks</h2>- * <p>+ ** By default, all asynchronous functions provided by this package return a promise.* However, most of them also accept an optional Node.js-style callback as the last parameter.* If the callback is provided, the return value of the functions is not defined* and the functions only invoke the callback upon completion or error.- * </p>- * <p>- * To set a promise dependency from an external library,- * use the <a href="#module-Apify-setPromisesDependency"><code>Apify.setPromisesDependency()</code></a> function.- * If this function is not called, the runtime defaults to- * native promises if they are available, or it throws an error.- * </p>+ ** <h2>Programmer's reference</h2>- * <p>The following sections describe all functions and properties provided by the `apify` package.+ *+ * The following sections describe all functions and properties provided by the `apify` package.* All of them are instance members exported directly by the main module.- * </p>+ *+ * @module Apify*/-module.exports = Apify;+module.exports = {+ events: new EventEmitter(),++ // Actor+ main,+ getEnv,+ call,+ readyFreddy,+ getMemoryInfo,+ getApifyProxyUrl,++ // Autoscaled pool+ AutoscaledPool,++ // Basic crawler+ BasicCrawler,++ // Dataset+ pushData,+ openDataset,++ // Key value store+ getValue,+ setValue,+ openKeyValueStore,++ // Puppeteer+ launchPuppeteer,+ PuppeteerPool,+ PuppeteerCrawler,++ // Request+ Request,+ RequestList,++ // Settings rotator+ SettingsRotator,++ // Utils+ client: apifyClient,++ // Webdriver+ browse,+ launchWebDriver,+};diff --git a/src/key_value_store.js b/src/key_value_store.jsnew file mode 100644index 000000000..3b49b66e5--- /dev/null+++ b/src/key_value_store.js@@ -0,0 +1,389 @@+import fs from 'fs';+import path from 'path';+import Promise from 'bluebird';+import contentTypeParser from 'content-type';+import LruCache from 'apify-shared/lru_cache';+import { checkParamOrThrow, parseBody } from 'apify-client/build/utils';+import { ENV_VARS } from './constants';+import { addCharsetToContentType, apifyClient, ensureDirExists } from './utils';++export const LOCAL_EMULATION_SUBDIR = 'key-value-stores';+const MAX_OPENED_STORES = 1000;+const LOCAL_FILE_TYPES = [+ { contentType: 'application/octet-stream', extension: 'buffer' },+ { contentType: 'application/json', extension: 'json' },+ { contentType: 'text/plain', extension: 'txt' },+ { contentType: 'image/jpeg', extension: 'jpg' },+ { contentType: 'image/png', extension: 'png' },+];+const DEFAULT_LOCAL_FILE_TYPE = LOCAL_FILE_TYPES[0];++const readFilePromised = Promise.promisify(fs.readFile);+const writeFilePromised = Promise.promisify(fs.writeFile);+const unlinkPromised = Promise.promisify(fs.unlink);++const { keyValueStores } = apifyClient;+const storesCache = new LruCache({ maxLength: MAX_OPENED_STORES }); // Open key-value stores are stored here.++/**+ * Helper function to validate params of *.getValue().+ *+ * @ignore+ */+const validateGetValueParams = (key) => {+ checkParamOrThrow(key, 'key', 'String');+ if (!key) throw new Error('The "key" parameter cannot be empty');+};++/**+ * Helper function to validate params of *.setValue().+ *+ * @ignore+ */+const validateSetValueParams = (key, value, options) => {+ checkParamOrThrow(key, 'key', 'String');+ checkParamOrThrow(options, 'options', 'Object');+ checkParamOrThrow(options.contentType, 'options.contentType', 'String | Null | Undefined');++ if (value === null && options.contentType !== null && options.contentType !== undefined) {+ throw new Error('The "options.contentType" parameter must not be used when removing the record.');+ }++ if (options.contentType) {+ checkParamOrThrow(value, 'value', 'Buffer | String', 'The "value" parameter must be a String or Buffer when "options.contentType" is specified.'); // eslint-disable-line max-len+ }++ if (options.contentType === '') throw new Error('Parameter options.contentType cannot be empty string.');+ if (!key) throw new Error('The "key" parameter cannot be empty');+};++/**+ * Helper function to possibly stringify value if options.contentType is not set.+ *+ * @ignore+ */+const maybeStringify = (value, options) => {+ // If contentType is missing, value will be stringified to JSON+ if (options.contentType === null || options.contentType === undefined) {+ options.contentType = 'application/json';++ try {+ // Format JSON to simplify debugging, the overheads with compression is negligible+ value = JSON.stringify(value, null, 2);+ } catch (e) {+ // Give more meaningful error message+ if (e.message && e.message.indexOf('Invalid string length') >= 0) {+ e.message = 'Object is too large';+ }+ throw new Error(`The "value" parameter cannot be stringified to JSON: ${e.message}`);+ }++ if (value === undefined) {+ throw new Error('The "value" parameter cannot be stringified to JSON.');+ }+ }++ return value;+};++/**+ * KeyValueStore class provides easy interface to Apify key-value store storage type. Key-value store should be opened using+ * `Apify.openKeyValueStore()` function.+ *+ * Basic usage of key-value store:+ *+ * ```javascript+ * const store = await Apify.openKeyValueStore('my-store-id');+ * await store.setValue('some-key', { foo: 'bar' });+ * const value = store.getValue('some-key');+ * ```+ *+ * @param {String} storeId - ID of the store.+ */+export class KeyValueStore {+ constructor(storeId) {+ checkParamOrThrow(storeId, 'storeId', 'String');++ this.storeId = storeId;+ }++ /**+ * Stores object or an array of objects in the dataset.+ * The function has no result, but throws on invalid args or other errors.+ *+ * @param {String} key Record key.+ * @return {Promise}+ */+ getValue(key) {+ validateGetValueParams(key);++ return keyValueStores+ .getRecord({ storeId: this.storeId, key })+ .then(output => (output ? output.body : null));+ }++ /**+ * Stores object or an array of objects in the dataset.+ * The function has no result, but throws on invalid args or other errors.+ *+ * @param {String} key Record key.+ * @param {Object|String|Buffer} value Record value. If content type is not provided then the value is stringified to JSON.+ * @param {Object} [Options]+ * @param {Object} [Options.contentType] Content type of the record.+ * @return {Promise}+ */+ setValue(key, value, options = {}) {+ validateSetValueParams(key, value, options);++ // Make copy of options, don't update what user passed.+ const optionsCopy = Object.assign({}, options);++ // In this case delete the record.+ if (value === null) return keyValueStores.deleteRecord({ storeId: this.storeId, key });++ value = maybeStringify(value, optionsCopy);++ // Keep this code in main scope so that simple errors are thrown rather than rejected promise.+ return keyValueStores.putRecord({+ storeId: this.storeId,+ key,+ body: value,+ contentType: addCharsetToContentType(optionsCopy.contentType),+ });+ }+}++/**+ * This is a local representation of a key-value store.+ *+ * @ignore+ */+export class KeyValueStoreLocal {+ constructor(storeId, localEmulationDir) {+ checkParamOrThrow(storeId, 'storeId', 'String');+ checkParamOrThrow(localEmulationDir, 'localEmulationDir', 'String');++ this.localEmulationPath = path.resolve(path.join(localEmulationDir, LOCAL_EMULATION_SUBDIR, storeId));+ this.storeId = storeId;+ this.initializationPromise = ensureDirExists(this.localEmulationPath);+ }++ getValue(key) {+ validateGetValueParams(key);++ return this.initializationPromise+ .then(() => {+ const filePath = path.resolve(this.localEmulationPath, key);+ const promises = LOCAL_FILE_TYPES.map(({ extension }) => {+ return readFilePromised(`${filePath}.${extension}`).catch(() => null);+ });++ return Promise.all(promises);+ })+ .then((files) => {+ let body = null;++ LOCAL_FILE_TYPES.some(({ contentType }, index) => {+ if (files[index] !== null) {+ body = parseBody(files[index], contentType);++ return true;+ }+ });++ return body;+ })+ .catch((err) => {+ throw new Error(`Error reading file '${key}' in directory '${this.localEmulationPath}' referred by ${ENV_VARS.APIFY_LOCAL_EMULATION_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len+ });+ }++ setValue(key, value, options = {}) {+ validateSetValueParams(key, value, options);++ // Make copy of options, don't update what user passed.+ const optionsCopy = Object.assign({}, options);++ const deletePromisesArr = LOCAL_FILE_TYPES.map(({ extension }) => {+ const filePath = path.resolve(this.localEmulationPath, `${key}.${extension}`);++ return unlinkPromised(filePath)+ .catch((err) => {+ if (err.code !== 'ENOENT') throw err;+ });+ });++ const deletePromise = Promise+ .all(deletePromisesArr);++ // In this case delete the record.+ if (value === null) return deletePromise;++ value = maybeStringify(value, optionsCopy);++ const contentType = contentTypeParser.parse(optionsCopy.contentType).type;+ const { extension } = LOCAL_FILE_TYPES.filter(type => type.contentType === contentType).pop() || DEFAULT_LOCAL_FILE_TYPE;+ const filePath = path.resolve(this.localEmulationPath, `${key}.${extension}`);++ return deletePromise+ .then(() => writeFilePromised(filePath, value))+ .catch((err) => {+ throw new Error(`Error writing file '${key}' in directory '${this.localEmulationPath}' referred by ${ENV_VARS.APIFY_LOCAL_EMULATION_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len+ });+ }+}++/**+ * Helper function that first requests key-value store by ID and if store doesn't exist then gets him by name.+ *+ * @ignore+ */+const getOrCreateKeyValueStore = (storeIdOrName) => {+ return apifyClient+ .keyValueStores+ .getStore({ storeId: storeIdOrName })+ .then((existingStore) => {+ if (existingStore) return existingStore;++ return apifyClient+ .keyValueStores+ .getOrCreateStore({ storeName: storeIdOrName });+ });+};++/**+ * Opens key-value store and returns its object.</p>+ *+ * ```javascript+ * const store = await Apify.openKeyValueStore('my-store-id');+ * await store.setValue('some-key', { foo: 'bar' });+ * ```+ *+ * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is defined, the value this function+ * returns an instance `KeyValueStoreLocal` which is an local emulation of key-value store.+ * This is useful for local development and debugging of your acts.+ *+ * @param {string} storeIdOrName ID or name of the key-value store to be opened.+ * @returns {Promise<KeyValueStore>} Returns a promise that resolves to a KeyValueStore object.+ *+ * @memberof module:Apify+ * @name openKeyValueStore+ * @instance+ */+export const openKeyValueStore = (storeIdOrName) => {+ checkParamOrThrow(storeIdOrName, 'storeIdOrName', 'Maybe String');++ const localEmulationDir = process.env[ENV_VARS.LOCAL_EMULATION_DIR];++ let isDefault = false;+ let storePromise;++ if (!storeIdOrName) {+ const envVar = ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID;++ // Env var doesn't exist.+ if (!process.env[envVar]) return Promise.reject(new Error(`The '${envVar}' environment variable is not defined.`));++ isDefault = true;+ storeIdOrName = process.env[envVar];+ }++ storePromise = storesCache.get(storeIdOrName);++ // Found in cache.+ if (storePromise) return storePromise;++ // Use local emulation?+ if (localEmulationDir) {+ storePromise = Promise.resolve(new KeyValueStoreLocal(storeIdOrName, localEmulationDir));+ } else {+ storePromise = isDefault // If true then we know that this is an ID of existing store.+ ? Promise.resolve(new KeyValueStore(storeIdOrName))+ : getOrCreateKeyValueStore(storeIdOrName).then(store => (new KeyValueStore(store.id)));+ }++ storesCache.add(storeIdOrName, storePromise);++ return storePromise;+};++/**+ * Gets a value from the default key-value store for the current act run using the Apify API.+ * The key-value store is created automatically for each act run+ * and its ID is passed by the Actor platform in the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.+ * It is used to store input and output of the act under keys named `INPUT` and `OUTPUT`, respectively.+ * However, the store can be used for storage of any other values under arbitrary keys.+ *+ * Example usage:+ *+ * ```javascript+ * const input = await Apify.getValue('INPUT');+ *+ * console.log('My input:');+ * console.dir(input);+ * ```+ *+ * The result of the function is the body of the record. Bodies with the `application/json`+ * content type are automatically parsed to an object.+ * Similarly, for `text/plain` content types the body is parsed as `String`.+ * For all other content types, the body is a raw `Buffer`.+ * If the record cannot be found, the result is null.+ *+ * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is defined,+ * the value is read from a that directory rather than the key-value store,+ * specifically from a file that has the key as a name.+ * file does not exists, the returned value is `null`. The file will get extension based on it's content type.+ * This feature is useful for local development and debugging of your acts.+ *+ *+ * @param {String} key Key of the record.+ * @returns {Promise} Returns a promise.+ *+ * @memberof module:Apify+ * @name getValue+ * @instance+ */+export const getValue = key => openKeyValueStore().then(store => store.getValue(key));++/**+ * Stores a value in the default key-value store for the current act run using the Apify API.+ * The data is stored in the key-value store created specifically for the act run,+ * whose ID is defined in the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.+ * The function has no result, but throws on invalid args or other errors.+ *+ * ```javascript+ * await Apify.setValue('OUTPUT', { someValue: 123 });+ * ```+ *+ * By default, `value` is converted to JSON and stored with the `application/json; charset=utf-8` content type.+ * To store a value with another content type, pass it in the options as follows:+ * ```javascript+ * await Apify.setValue('OUTPUT', 'my text data', { contentType: 'text/plain' });+ * ```+ * In this case, the value must be a string or Buffer.+ *+ * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is defined,+ * the value is written to that local directory rather than the key-value store on Apify cloud,+ * to a file named as the key. This is useful for local development and debugging of your acts.+ *+ * **IMPORTANT:** Do not forget to use the `await` keyword when calling `Apify.setValue()`,+ * otherwise the act process might finish before the value is stored!+ *+ * @param key Key of the record+ * @param value Value of the record:+ * <ul>+ * <li>If `null`, the record in the key-value store is deleted.</li>+ * <li>If no `options.contentType` is specified, `value` can be any object and it will be stringified to JSON.</li>+ * <li>If `options.contentType` is specified, `value` is considered raw data and it must be a String or Buffer.</li>+ * </ul>+ * For any other value an error will be thrown.+ *+ * @param {Object} [options]+ * @param {String} [options.contentType] - Sets the MIME content type of the value.+ * @returns {Promise} Returns a promise that resolves to the value.+ *+ * @memberof module:Apify+ * @name setValue+ * @instance+ */+export const setValue = (key, value, options) => openKeyValueStore().then(store => store.setValue(key, value, options));diff --git a/src/puppeteer.js b/src/puppeteer.jsindex 2294b31cb..3aa00f8c6 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -1,19 +1,17 @@import { checkParamOrThrow } from 'apify-client/build/utils';import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';-import { newPromise } from './utils';import { ENV_VARS, DEFAULT_USER_AGENT } from './constants';+import { newPromise } from './utils';/* global process, require */-/**- * @memberof module:Apify- * @function- * @description <p>Launches headless Chrome using Puppeteer pre-configured to work with the Apify Actor platform.+ * Launches headless Chrome using Puppeteer pre-configured to work with the Apify Actor platform.* The function has the same argument and return value as `puppeteer.launch()`.* See {@link https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#puppeteerlaunchoptions|Puppeteer documentation} for more details.- * </p>- * <p>The `launchPuppeteer()` function alters the following Puppeteer options:+ *+ * The `launchPuppeteer()` function alters the following Puppeteer options:+ ** <ul>* <li>Passes the setting from the `APIFY_HEADLESS` environment variable to the `headless` option,* unless it was already defined by the caller. Note that this environment variable is automatically set to `1`@@ -25,22 +23,24 @@ import { ENV_VARS, DEFAULT_USER_AGENT } from './constants';* </li>* <li>Adds `--no-sandbox` to `args` to enable running headless Chrome in a Docker container on the Actor platform.</li>* </ul>- * </p>- * <p>- * To use this function, you need to have the- * {@link https://www.npmjs.com/package/puppeteer|puppeteer}+ *+ * To use this function, you need to have the {@link https://www.npmjs.com/package/puppeteer|puppeteer}* NPM package installed in your project.* For example, you can use the `apify/actor-node-puppeteer` base Docker image for your act - see* {@link https://www.apify.com/docs/actor#base-images|documentation}* for more details.- * </p>+ ** @param {Object} [opts] Optional settings passed to `puppeteer.launch()`. Additionally the object can contain the following fields:- * @param {String} [opts.proxyUrl] - URL to a HTTP proxy server.- * Port number must be specified. Proxy username and password might also be provided.- * For example, `http://bob:[email protected]:1234`.- * @param {String} [opts.userAgent] - Default User-Agent for the browser.- * If not provided, the function sets it to a reasonable default.+ * @param {String} [opts.proxyUrl] URL to a HTTP proxy server.+ * Port number must be specified. Proxy username and password might also be provided.+ * For example, `http://bob:[email protected]:1234`.+ * @param {String} [opts.userAgent] Default User-Agent for the browser.+ * If not provided, the function sets it to a reasonable default.* @returns {Promise} Promise object that resolves to Puppeteer's `Browser` instance.+ *+ * @memberof module:Apify+ * @name launchPuppeteer+ * @instance*/export const launchPuppeteer = (opts) => {if (opts === undefined || opts === null) opts = {};diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsnew file mode 100644index 000000000..c4fb70edb--- /dev/null+++ b/src/puppeteer_crawler.js@@ -0,0 +1,185 @@+import { checkParamOrThrow } from 'apify-client/build/utils';+import _ from 'underscore';+import Promise from 'bluebird';+import BasicCrawler from './basic_crawler';+import PuppeteerPool from './puppeteer_pool';+import { isPromise } from './utils';++const DEFAULT_OPTIONS = {+ gotoFunction: ({ request, page }) => page.goto(request.url),+ pageOpsTimeoutMillis: 30000,+};++/**+ * PuppeteerCrawler provides a simple framework for parallel crawling of a url list provided by Apify.RequestList+ * or a dynamically enqueued requests provided by Apify.RequestQueue (TODO). Each url is opened in Puppeteer (Chrome)+ * browser.+ *+ * For each url in the list or queue it opens a Chrome tab and then calls handlePageFunction. The concurrency is scaled+ * based on available memory using Apify.AutoscaledPool.+ *+ * Basic usage of PuppeteerCrawler:+ *+ * ```javascript+ * const request = require('request-promise');+ *+ * const crawler = new Apify.BasicCrawler({+ * urlList,+ * // Parameter page here is an intance of Puppeteer.Page with page.goto(request.url) already called+ * handlePageFunction: async ({ page, request }) => {+ * await Apify.pushData({+ * title: await page.title(),+ * url: request.url,+ * succeeded: true,+ * })+ * },+ * handleFailedRequestFunction: async ({ request }) => {+ * await Apify.pushData({+ * url: request.url,+ * succeeded: false,+ * errors: request.errorMessages,+ * })+ * },+ * });+ *+ * await crawler.run();+ * ```+ *+ * @param {RequestList} options.requestList List of the requests to be processed. (See `requestList` parameter of `Apify.BasicCrawler`)+ * @param {Function} options.handlePageFunction Function to process each request.+ * @param {Number} [options.pageOpsTimeoutMillis=30000] Timeout for options.handlePagefunction+ * @param {Function} [options.gotoFunction=({ request, page }) => page.goto(request.url)] Overrides default gotoFunction.++ * @param {Function} [options.handleFailedRequestFunction=({ request }) => log.error('Request failed', _.pick(request, 'url', 'uniqueKey'))]+ * Function to handle requests that failed more then option.maxRequestRetries times. (See `handleFailedRequestFunction`+ * parameter of `Apify.BasicCrawler`)+ * @param {Number} [options.maxRequestRetries=3] How many times request is retried if handleRequestFunction failed. (See `maxRequestRetries`+ * parameter of `Apify.BasicCrawler`)+ *+ * @param {Number} [options.maxMemoryMbytes] Maximal memory available in the system (See `maxMemoryMbytes` parameter of `Apify.AutoscaledPool`).+ * @param {Number} [options.maxConcurrency=1] Minimal concurrency of requests processing (See `maxConcurrency` parameter of `Apify.AutoscaledPool`).+ * @param {Number} [options.minConcurrency=1000] Maximal concurrency of request processing (See `minConcurrency` parameter of `Apify.AutoscaledPool`).+ *+ * @param {Number} [options.maxOpenPagesPerInstance=100] Maximal number of opened tabs per browser. If limit is reached then the new+ * browser gets started. (See `maxOpenPagesPerInstance` parameter of `Apify.PuppeteerPool`)+ * @param {Number} [options.abortInstanceAfterRequestCount=150] Maximal number of requests proceeded from one browser. After that browser+ * gets restarted. (See `abortInstanceAfterRequestCount` parameter of+ * `Apify.PuppeteerPool`)+ * @param {Function} [options.launchPuppeteerFunction] Overrides how new Puppeteer instance gets launched. (See `launchPuppeteerFunction` parameter of+ * `Apify.PuppeteerPool`)+ * @param {Number} [options.instanceKillerIntervalMillis=60000] How often opened Puppeteer instances get checked if some of then might be+ * closed. (See `instanceKillerIntervalMillis` parameter of `Apify.PuppeteerPool`)+ * @param {Number} [options.killInstanceAfterMillis=300000] If Puppeteer instance reaches the limit options.abortInstanceAfterRequestCount then it's+ * considered retired and no more tabs will be opened. After the last tab get's closed the+ * whole browser gets closed. This defines limit of inactivity after the browser gets closed+ * even if there are pending tabs. (See `killInstanceAfterMillis` parameter of+ * `Apify.PuppeteerPool`)+ * @param {Object} [options.puppeteerConfig={ dumpio: process.env.NODE_ENV !== 'production', slowMo: 0, args: []}] Configuration of Puppeteer+ * instances. (See `puppeteerConfig` parameter of `Apify.PuppeteerPool`)+ * @param {Boolean} [options.disableProxy=false] Disables proxying thru Apify proxy. (See `disableProxy` parameter of `Apify.PuppeteerPool`)+ * @param {Array} [options.groups] Apify proxy groups to be used. (See `Apify.getApifyProxyUrl()` for more)+ */+export default class PuppeteerCrawler {+ constructor(opts) {+ const {+ handlePageFunction,+ gotoFunction,+ pageOpsTimeoutMillis,++ // Autoscaled pool options+ maxMemoryMbytes,+ maxConcurrency,+ minConcurrency,++ // Basic crawler options+ requestList,+ maxRequestRetries,+ handleFailedRequestFunction,++ // Puppeteer Pool options+ maxOpenPagesPerInstance,+ abortInstanceAfterRequestCount,+ launchPuppeteerFunction,+ instanceKillerIntervalMillis,+ killInstanceAfterMillis,+ groups,+ puppeteerConfig,+ disableProxy,+ } = _.defaults(opts, DEFAULT_OPTIONS);++ checkParamOrThrow(handlePageFunction, 'opts.handlePageFunction', 'Function');+ checkParamOrThrow(handleFailedRequestFunction, 'opts.handleFailedRequestFunction', 'Maybe Function');+ checkParamOrThrow(gotoFunction, 'opts.gotoFunction', 'Function');+ checkParamOrThrow(pageOpsTimeoutMillis, 'opts.pageOpsTimeoutMillis', 'Number');++ this.handlePageFunction = handlePageFunction;+ this.gotoFunction = gotoFunction;+ this.pageOpsTimeoutMillis = pageOpsTimeoutMillis;++ this.puppeteerPool = new PuppeteerPool({+ maxOpenPagesPerInstance,+ abortInstanceAfterRequestCount,+ launchPuppeteerFunction,+ instanceKillerIntervalMillis,+ killInstanceAfterMillis,+ groups,+ puppeteerConfig,+ disableProxy,+ });++ this.basicCrawler = new BasicCrawler({+ // Basic crawler options.+ requestList,+ maxRequestRetries,+ handleRequestFunction: (...args) => this._handleRequestFunction(...args),+ handleFailedRequestFunction,++ // Autoscaled pool options.+ maxMemoryMbytes,+ maxConcurrency,+ minConcurrency,+ });+ }++ /**+ * Runs the crawler. Returns promise that gets resolved once all the requests got processed.+ *+ * @return {Promise}+ */+ run() {+ return this.basicCrawler.run()+ .finally(() => this.puppeteerPool.destroy());+ }++ /**+ * Wrapper around handlePageFunction that opens and closes pages etc.+ *+ * @ignore+ */+ _handleRequestFunction({ request }) {+ let page;++ const handlePagePromise = this.puppeteerPool+ .newPage()+ .then((newPage) => { page = newPage; })+ .then(() => this.gotoFunction({ page, request, puppeteerPool: this.puppeteerPool }))+ .then(() => {+ const promise = this.handlePageFunction({ page, request, puppeteerPool: this.puppeteerPool });++ if (!isPromise(promise)) throw new Error('User provided handlePageFunction must return a Promise.');++ return promise;+ });++ const timeoutPromise = new Promise(resolve => setTimeout(resolve, this.pageOpsTimeoutMillis));++ return Promise+ .race([+ handlePagePromise,+ timeoutPromise.then(() => { throw new Error('PuppeteerCrawler: handlePageFunction timed out'); }),+ ])+ .finally(() => {+ if (page) return page.close();+ });+ }+}diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsnew file mode 100644index 000000000..aebb9907b--- /dev/null+++ b/src/puppeteer_pool.js@@ -0,0 +1,297 @@+import _ from 'underscore';+import log from 'apify-shared/log';+import { checkParamOrThrow } from 'apify-client/build/utils';+import { launchPuppeteer } from './puppeteer';+import { getApifyProxyUrl } from './actor';+import { isProduction } from './utils';++const DEFAULT_PUPPETEER_CONFIG = {+ dumpio: !isProduction(),+ slowMo: 0,+ args: [],+};++const DEFAULT_OPTIONS = {+ maxOpenPagesPerInstance: 100,+ abortInstanceAfterRequestCount: 150,++ // These can't be constants because we need it for unit tests.+ instanceKillerIntervalMillis: 60 * 1000,+ killInstanceAfterMillis: 5 * 60 * 1000,++ // TODO: use settingsRotator()+ launchPuppeteerFunction: ({ groups, puppeteerConfig, disableProxy = false }) => {+ checkParamOrThrow(groups, 'opts.groups', 'Maybe Array');+ checkParamOrThrow(puppeteerConfig, 'opts.puppeteerConfig', 'Maybe Object');+ checkParamOrThrow(disableProxy, 'opts.disableProxy', 'Maybe Boolean');++ const config = Object.assign({}, DEFAULT_PUPPETEER_CONFIG, puppeteerConfig);++ // TODO: is this needed at all? It might be confusing because the feature has the same name+ // as Chrome command line flag, so people will assume it's doing just that.+ // For simplicity I'd just remove it...+ if (config.disableWebSecurity) {+ config.ignoreHTTPSErrors = true;+ config.args.push('--disable-web-security');+ }++ // TODO: Maybe we should move this whole logic directly to Apify.launchPuppeteer().+ // E.g. if process.env.APIFY_PROXY_HOST is defined, then puppeteer should use it with "auto".+ if (!disableProxy) {+ const session = Math.random();++ config.proxyUrl = getApifyProxyUrl({ groups, session });+ }++ return launchPuppeteer(config);+ },+};++/**+ * Internal representation of Puppeteer instance.+ *+ * @ignore+ */+class PuppeteerInstance {+ constructor(id, browserPromise) {+ this.id = id;+ this.activePages = 0;+ this.totalPages = 0;+ this.browserPromise = browserPromise;+ this.lastPageOpenedAt = Date.now();+ this.killed = false;+ }+}++/**+ * PuppeteerPool class provides a pool of Puppeteer (Chrome browser) instances.+ * It rotates them based on it's configuration to change proxies.+ *+ * Example use:+ *+ * ```javascript+ * const puppeteerPool = new PuppeteerPool({ groups: 'some-proxy-group' });+ *+ * const page1 = await puppeteerPool.newPage();+ * const page2 = await puppeteerPool.newPage();+ * const page3 = await puppeteerPool.newPage();+ *+ * // ... do something with pages ...+ *+ * // Close all the browsers.+ * await puppeteerPool.destroy();+ * ```+ *+ * @param {Number} [options.maxOpenPagesPerInstance=100] Maximal number of opened tabs per browser. If limit is reached then the new+ * browser gets started. (See `maxOpenPagesPerInstance` parameter of `Apify.PuppeteerPool`)+ * @param {Number} [options.abortInstanceAfterRequestCount=150] Maximal number of requests proceeded from one browser. After that browser+ * gets restarted. (See `abortInstanceAfterRequestCount` parameter of+ * `Apify.PuppeteerPool`)+ * @param {Function} [options.launchPuppeteerFunction] Overrides how new Puppeteer instance gets launched. (See `launchPuppeteerFunction` parameter of+ * `Apify.PuppeteerPool`)+ * @param {Number} [options.instanceKillerIntervalMillis=60000] How often opened Puppeteer instances get checked if some of then might be+ * closed. (See `instanceKillerIntervalMillis` parameter of `Apify.PuppeteerPool`)+ * @param {Number} [options.killInstanceAfterMillis=300000] If Puppeteer instance reaches the limit options.abortInstanceAfterRequestCount then it's+ * considered retired and no more tabs will be opened. After the last tab get's closed the+ * whole browser gets closed. This defines limit of inactivity after the browser gets closed+ * even if there are pending tabs. (See `killInstanceAfterMillis` parameter of+ * `Apify.PuppeteerPool`)+ * @param {Object} [options.puppeteerConfig={ dumpio: process.env.NODE_ENV !== 'production', slowMo: 0, args: []}] Configuration of Puppeteer+ * instances. (See `puppeteerConfig` parameter of `Apify.PuppeteerPool`)+ * @param {Boolean} [options.disableProxy=false] Disables proxying thru Apify proxy. (See `disableProxy` parameter of `Apify.PuppeteerPool`)+ * @param {Array} [options.groups] Apify proxy groups to be used. (See `Apify.getApifyProxyUrl()` for more)+ */+export default class PuppeteerPool {+ constructor(opts = {}) {+ checkParamOrThrow(opts, 'opts', 'Object');++ const {+ maxOpenPagesPerInstance,+ abortInstanceAfterRequestCount,+ launchPuppeteerFunction,+ instanceKillerIntervalMillis,+ killInstanceAfterMillis,+ } = _.defaults(opts, DEFAULT_OPTIONS);++ checkParamOrThrow(maxOpenPagesPerInstance, 'opts.maxOpenPagesPerInstance', 'Number');+ checkParamOrThrow(abortInstanceAfterRequestCount, 'opts.abortInstanceAfterRequestCount', 'Number');+ checkParamOrThrow(launchPuppeteerFunction, 'opts.launchPuppeteerFunction', 'Function');+ checkParamOrThrow(instanceKillerIntervalMillis, 'opts.instanceKillerIntervalMillis', 'Number');+ checkParamOrThrow(killInstanceAfterMillis, 'opts.killInstanceAfterMillis', 'Number');++ // Config.+ this.maxOpenPagesPerInstance = maxOpenPagesPerInstance;+ this.abortInstanceAfterRequestCount = abortInstanceAfterRequestCount;+ this.killInstanceAfterMillis = killInstanceAfterMillis;+ this.launchPuppeteerFunction = () => launchPuppeteerFunction(opts);++ // State.+ this.browserCounter = 0;+ this.activeInstances = {};+ this.retiredInstances = {};+ this.instanceKillerInterval = setInterval(() => this._killRetiredInstances(), instanceKillerIntervalMillis);+ }++ /**+ * Launches new browser instance.+ *+ * @ignore+ */+ _launchInstance() {+ const id = this.browserCounter++;+ const browserPromise = this.launchPuppeteerFunction();+ const instance = new PuppeteerInstance(id, browserPromise);++ instance+ .browserPromise+ .then((browser) => {+ browser.on('disconnected', () => {+ log.error('PuppeteerPool: Puppeteer sent "disconnect" event. Crashed???', { id });+ this._retireInstance(instance);+ });+ // This one is done manually in Puppeteerpool.newPage() to happen immediately.+ // browser.on('targetcreated', () => instance.activePages++);+ browser.on('targetdestroyed', () => {+ instance.activePages--;++ if (instance.activePages === 0 && this.retiredInstances[id]) this._killInstance(instance);+ });+ })+ .catch((err) => {+ log.exception(err, 'PuppeteerPool: Browser start failed', { id });++ return this._retireInstance(instance);+ });++ this.activeInstances[id] = instance;++ return instance;+ }++ /**+ * Retires some of the instances for example due to to many uses.+ *+ * @ignore+ */+ _retireInstance(instance) {+ const { id } = instance;++ if (!this.activeInstances[id]) return log.warning('PuppeteerPool: browser is retired already', { id });++ log.info('PuppeteerPool: retiring browser', { id });++ this.retiredInstances[id] = instance;+ delete this.activeInstances[id];+ }++ /**+ * Kills all the retired instances that:+ * - have all tabs closed+ * - or are inactive for more then killInstanceAfterMillis.+ *+ * @ignore+ */+ _killRetiredInstances() {+ log.info('PuppeteerPool: retired browsers count', { count: _.values(this.retiredInstances).length });++ _.mapObject(this.retiredInstances, (instance) => {+ // Kill instances that are more than this.killInstanceAfterMillis from last opened page+ if (Date.now() - instance.lastPageOpenedAt > this.killInstanceAfterMillis) this._killInstance(instance);++ instance+ .browserPromise+ .then(browser => browser.pages())+ .then((pages) => {+ if (pages.length === 0) this._killInstance(instance);+ }, (err) => {+ log.exception(err, 'PuppeteerPool: browser.pages() failed', { id: instance.id });+ this._killInstance(instance);+ });+ });+ }++ /**+ * Kills given browser instance.+ *+ * @ignore+ */+ _killInstance(instance) {+ const { id } = instance;++ log.info('PuppeteerPool: killing browser', { id });++ delete this.retiredInstances[id];++ instance+ .browserPromise+ .then((browser) => {+ if (instance.killed) return;++ instance.killed = true;++ return browser.close();+ })+ .catch(err => log.exception(err, 'PuppeteerPool: cannot close the browser instance', { id }));+ }++ /**+ * Opens new tab in one of the browsers and returns promise that resolves to it's Puppeteer.Page.+ *+ * @return {Promise<Puppeteer.Page>}+ */+ newPage() {+ let instance;++ _.mapObject(this.activeInstances, (inst) => {+ if (inst.activePages >= this.maxOpenPagesPerInstance) return;++ instance = inst;+ });++ if (!instance) instance = this._launchInstance();++ instance.lastPageOpenedAt = Date.now();+ instance.totalPages++;+ instance.activePages++;++ if (instance.totalPages >= this.abortInstanceAfterRequestCount) this._retireInstance(instance);++ return instance.browserPromise+ .then(browser => browser.newPage())+ .then((page) => {+ page.on('error', (error) => {+ log.exception(error, 'PuppeteerPool: page crashled');+ page.close();+ });++ // TODO: log console messages page.on('console', message => log.debug(`Chrome console: ${message.text}`));++ return page;+ })+ .catch((err) => {+ log.exception(err, 'PuppeteerPool: browser.newPage() failed', { id: instance.id });+ this._retireInstance(instance);+ throw err;+ });+ }++ /**+ * Closes all the browsers.+ */+ destroy() {+ clearInterval(this.instanceKillerInterval);++ const browserPromises = _+ .values(this.activeInstances)+ .concat(_.values(this.retiredInstances))+ .map(instance => instance.browserPromise);++ const closePromises = browserPromises.map((browserPromise) => {+ return browserPromise.then(browser => browser.close());+ });++ return Promise+ .all(closePromises)+ .catch(err => log.exception(err, 'PuppeteerPool: cannot close the browsers'));+ }+}diff --git a/src/request.js b/src/request.jsnew file mode 100644index 000000000..9d3b2f276--- /dev/null+++ b/src/request.js@@ -0,0 +1,90 @@+import { checkParamOrThrow } from 'apify-client/build/utils';+import { normalizeUrl } from 'apify-shared/utilities';+import _ from 'underscore';++export const computeUniqueKey = (url, keepUrlFragment) => normalizeUrl(url, keepUrlFragment);++/**+ * Requests class defines a request to be processed and stores info about error that occured during the processing.+ *+ * Example use:+ *+ * ```javascript+ * const request = new Apify.Request({+ * url: 'http://example.com',+ * headers: { Accept: 'application/json' },+ * });+ *+ * ...+ *+ * request.userData.foo = 'bar';+ * request.pushErrorMessage(new Error('Request failed!'));+ *+ * ...+ *+ * const foo = request.userData.foo;+ * ```+ *+ * @param {String} url+ * @param {String} [uniqueKey] Unique key indentifying request. In not provided then it's computed as normalized url.+ * @param {String} [method='GET']+ * @param {String|Buffer} [payload] Request payload. If method='GET' then the payload is not allowed.+ * @param {Number} [retryCount=0] How many times the url was retried in a case of exception.+ * @param {String} [errorMessages] Array of error messages from request processing.+ * @param {String} [headers={}] HTTP headers.+ * @param {Object} [userData={}] Custom data that user can assign to request.+ * @param {Boolean} [keepUrlFragment=false] If false then hash part is removed from url when computing `uniqueKey`.+ */+export default class Request {+ constructor({+ url,+ uniqueKey,+ method = 'GET',+ payload = null,+ retryCount = 0,+ errorMessages = null,+ headers = {},+ userData = {},+ keepUrlFragment = false,+ }) {+ checkParamOrThrow(url, 'url', 'String');+ checkParamOrThrow(uniqueKey, 'uniqueKey', 'Maybe String');+ checkParamOrThrow(method, 'method', 'String');+ checkParamOrThrow(payload, 'payload', 'Maybe Buffer | String');+ checkParamOrThrow(retryCount, 'retryCount', 'Number');+ checkParamOrThrow(errorMessages, 'errorMessages', 'Maybe Array');+ checkParamOrThrow(headers, 'headers', 'Object');+ checkParamOrThrow(userData, 'userData', 'Object');++ if (method === 'GET' && payload) throw new Error('Request with GET method cannot have a payload.');++ this.url = url;+ this.uniqueKey = uniqueKey || computeUniqueKey(url, keepUrlFragment);+ this.method = method;+ this.payload = payload;+ this.retryCount = retryCount;+ this.errorMessages = errorMessages;+ this.headers = headers;+ this.userData = userData;+ }++ /**+ * Stores information about processing error of this request.+ *+ * @param {Error|String} errorOrMessage Error object or error message to be stored in request.+ */+ pushErrorMessage(errorOrMessage) {+ if (!_.isString(errorOrMessage) && !(errorOrMessage instanceof Error)) {+ throw new Error('Parameter errorOrMessage must be a String or an instance of Error');+ }++ const message = errorOrMessage instanceof Error+ ? errorOrMessage.message+ : errorOrMessage;++ if (!this.errorMessages) this.errorMessages = [];++ this.errorMessages.push(message);+ }+}+diff --git a/src/request_list.js b/src/request_list.jsnew file mode 100644index 000000000..20d963379--- /dev/null+++ b/src/request_list.js@@ -0,0 +1,354 @@+import { checkParamOrThrow } from 'apify-client/build/utils';+import log from 'apify-shared/log';+import _ from 'underscore';+import requestPromise from 'request-promise';+import { sequentializePromises } from 'apify-shared/utilities';+import Request from './request';++// TODO: better tests+const URL_REGEX = '(http|https)://[\\w-]+(\\.[\\w-]+)+([\\w-.,@?^=%&:/~+#-]*[\\w@?^=%&;/~+#-])?';++/**+ * Helper function that returns the first key from plan object.+ *+ * @ignore+ */+const getFirstKey = (dict) => {+ for (const key in dict) { // eslint-disable-line guard-for-in, no-restricted-syntax+ return key;+ }+};++/**+ * Helper function that validates unique.+ * Throws an error if uniqueKey is not nonempty string.+ *+ * @ignore+ */+const ensureUniqueKeyValid = (uniqueKey) => {+ if (typeof uniqueKey !== 'string' || !uniqueKey) {+ throw new Error('Request object\'s uniqueKey must be a non-empty string');+ }+};++/**+ * RequestList provides way to handle a list of urls to be processed.+ *+ * RequestList has internal state where it remembers handled requests, requests in proggress and also reclaimed requests.+ * State might be persisted in key-value store as shown in example below so if an act get restarted (due to internal+ * error or restart of host machine) then it may be initialized from previous state.+ *+ * Basic usage of RequestList:+ *+ * ```javascript+ * const requestList = new RequestList({+ * sources: [+ * // Seperate requests+ * { url: 'http://www.example.com/page-1', method: 'GET', headers: {} },+ * { url: 'http://www.example.com/page-2', userData: { foo: 'bar' }},+ * // Bulk load of urls from file `http://www.example.com/my-url-list.txt`+ * { pagesFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },+ * ],+ * // Initialize from previous state if act was restarted due to some error+ * state: await Apify.getValue('my-request-list-state'),+ * });+ *+ * await requestList.initialize(); // Load requests.+ *+ * // Save state of the request list every 5 seconds.+ * setInterval(() => {+ * Apify.setValue('my-request-list-state', requestList.getState());+ * }, 5000);+ *+ * // Get requests from list+ * const request1 = requestList.fetchNextRequest();+ * const request2 = requestList.fetchNextRequest();+ * const request3 = requestList.fetchNextRequest();+ *+ * // Mark some of them as handled+ * requestList.markRequestHandled(request1);+ *+ * // If processing fails then reclaim it back to the list+ * requestList.reclaimRequest(request2);+ * ```+ *+ * @param {Array} options.sources Function that processes a request. It must return a promise.+ * ```javascript+ * [+ * // One url+ * { method: 'GET', url: 'http://example.com/a/b' },+ * // Batch import from web hosted file+ * { method: 'POST', requestsFromUrl: 'http://mywebsite.com/urls.txt' },+ * ]+ * ```+ * @param {Object} [options.state] State of the RequestList to be initialized from. It's in the form returned by `requestList.getState()`:+ * ```javascript+ * {+ * nextIndex: 5,+ * nextUniqueKey: 'unique-key-5'+ * inProgress: {+ * 'unique-key-1': true,+ * 'unique-key-4': true,+ * },+ * }+ * ```+ */+export default class RequestList {+ constructor({ sources, state }) {+ checkParamOrThrow(sources, 'options.sources', 'Array');+ checkParamOrThrow(state, 'options.state', 'Maybe Object');++ // We will initialize everything from this state in this.initialize();+ this.initialState = state;++ // Array of all requests from all sources, in the order as they appeared in sources.+ // All requests in the array have distinct uniqueKey!+ this.requests = [];++ // Index to the next item in requests array to fetch. All previous requests are either handled or in progress.+ this.nextIndex = 0;++ // Dictionary, key is Request.uniqueKey, value is corresponding index in the requests array.+ this.uniqueKeyToIndex = {};++ // Dictionary of requests that were returned by fetchNextRequest().+ // The key is uniqueKey, value is true.+ this.inProgress = {};++ // Dictionary of requests for which reclaimRequest() was called.+ // The key is uniqueKey, value is true.+ // Note that reclaimedRequests is always a subset of inProgressRequests!+ this.reclaimed = {};++ this.isLoading = false;+ this.isInitialized = false;++ // We'll load all sources in sequence to ensure that they get loaded in the right order.+ this.loadSourcesPromiseGenerators = sources.map((source) => {+ if (source.requestsFromUrl) return () => this._addRequestsFromUrl(source);++ // TODO: One promise per each item is too much overheads, we could cluster items into single Promise.+ return () => Promise.resolve(this._addRequest(source));+ });+ }++ /**+ * Loads all sources specified.+ *+ * @returns {Promise}+ */+ initialize() {+ if (this.isLoading) {+ throw new Error('RequestList sources are already loading or were loaded.');+ }+ this.isLoading = true;++ return sequentializePromises(this.loadSourcesPromiseGenerators)+ .then(() => {+ const state = this.initialState;++ if (!state) return;++ // Restore state+ if (typeof state.nextIndex !== 'number' || state.nextIndex < 0) {+ throw new Error('The state object is invalid: nextIndex must be a non-negative number.');+ }+ if (state.nextIndex > this.requests.length) {+ throw new Error('The state object is not consistent with RequestList: too few requests loaded.');+ }+ if (state.nextIndex < this.requests.length+ && this.requests[state.nextIndex].uniqueKey !== state.nextUniqueKey) {+ throw new Error('The state object is not consistent with RequestList: the order of URLs seems to have changed.');+ }++ _.keys(state.inProgress).forEach((uniqueKey) => {+ if (typeof this.uniqueKeyToIndex[uniqueKey] !== 'number') {+ throw new Error('The state object is not consistent with RequestList: unknown uniqueKey is present in the state.');+ }+ });++ this.nextIndex = state.nextIndex;+ this.inProgress = state.inProgress;++ // All in-progress requests need to be recrawled+ this.reclaimed = _.clone(this.inProgress);+ })+ .then(() => {+ this.isInitialized = true;+ });+ }++ /**+ * Returns an object representing the state of the RequestList instance. Do not alter the resulting object!+ *+ * @returns Object+ */+ getState() {+ this._ensureIsInitialized();++ return {+ nextIndex: this.nextIndex,+ nextUniqueKey: this.nextIndex < this.requests.length+ ? this.requests[this.nextIndex].uniqueKey+ : null,+ inProgress: this.inProgress,+ };+ }++ /**+ * Returns `true` if the next call to fetchNextRequest() will return null, otherwise it returns `false`.+ * Note that even if the list is empty, there might be some pending requests currently being processed.+ *+ * @returns {boolean}+ */+ isEmpty() {+ this._ensureIsInitialized();++ return !getFirstKey(this.reclaimed) && this.nextIndex >= this.requests.length;+ }++ /**+ * Returns `true` if all requests were already handled and there are no more left.+ *+ * @returns {boolean}+ */+ isFinished() {+ this._ensureIsInitialized();++ return !getFirstKey(this.inProgress) && this.nextIndex >= this.requests.length;+ }++ /**+ * Returns next request which is the reclaimed one if available or next upcoming request otherwise.+ *+ * @returns {Request}+ */+ fetchNextRequest() {+ this._ensureIsInitialized();++ // First return reclaimed requests if any.+ const uniqueKey = getFirstKey(this.reclaimed);+ if (uniqueKey) {+ delete this.reclaimed[uniqueKey];+ const index = this.uniqueKeyToIndex[uniqueKey];+ return this.requests[index];+ }++ // Otherwise return next request.+ if (this.nextIndex < this.requests.length) {+ const request = this.requests[this.nextIndex];+ this.inProgress[request.uniqueKey] = true;+ this.nextIndex++;+ return request;+ }++ return null;+ }++ /**+ * Adds all requests from a file string.+ *+ * @ignore+ */+ _addRequestsFromUrl(source) {+ const sharedOpts = _.omit(source, 'requestsFromUrl', 'regex');+ const {+ requestsFromUrl,+ regex = URL_REGEX,+ } = source;++ return requestPromise.get(requestsFromUrl)+ .then((urlsStr) => {+ const urlsArr = urlsStr.match(new RegExp(regex, 'g'));++ log.info('RequestList: list fetched', {+ requestsFromUrl,+ regex,+ count: urlsArr.length,+ sample: JSON.stringify(urlsArr.slice(0, 5)),+ });++ urlsArr.forEach(url => this._addRequest(_.extend({ url }, sharedOpts)));+ })+ .catch((err) => {+ log.exception(err, 'RequestList: Cannot fetch a request list', { requestsFromUrl, regex });+ throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);+ });+ }++ /**+ * Adds given request.+ * If opts partameter is plain object not instance of an Requests then creates it.+ *+ * @ignore+ */+ _addRequest(opts) {+ const request = opts instanceof Request+ ? opts+ : new Request(opts);++ const { uniqueKey } = request;+ ensureUniqueKeyValid(uniqueKey);++ // Skip requests with duplicate uniqueKey+ if (this.uniqueKeyToIndex[uniqueKey] === undefined) {+ this.uniqueKeyToIndex[uniqueKey] = this.requests.length;+ this.requests.push(request);+ }+ }++ /**+ * Checks that request is not reclaimed and throws an error if so.+ *+ * @ignore+ */+ _ensureInProgressAndNotReclaimed(uniqueKey) {+ if (!this.inProgress[uniqueKey]) {+ throw new Error(`The request is not being processed (uniqueKey: ${uniqueKey})`);+ }+ if (this.reclaimed[uniqueKey]) {+ throw new Error(`The request was already reclaimed (uniqueKey: ${uniqueKey})`);+ }+ }++ /**+ * Throws an error if request list wasn't initialized.+ *+ * @ignore+ */+ _ensureIsInitialized() {+ if (!this.isInitialized) {+ throw new Error('RequestList is not initialized. You must call "await requestList.initialize();" before using it!');+ }+ }++ /**+ * Marks request handled after successfull processing.+ *+ * @param {Request} request+ */+ markRequestHandled(request) {+ const { uniqueKey } = request;++ ensureUniqueKeyValid(uniqueKey);+ this._ensureInProgressAndNotReclaimed(uniqueKey);+ this._ensureIsInitialized();++ delete this.inProgress[uniqueKey];+ }++ /**+ * Reclaims request after unsuccessfull operation. Request will become available for next `this.fetchNextRequest()`.+ *+ * @param {Request} request+ */+ reclaimRequest(request) {+ const { uniqueKey } = request;++ ensureUniqueKeyValid(uniqueKey);+ this._ensureInProgressAndNotReclaimed(uniqueKey);+ this._ensureIsInitialized();++ this.reclaimed[uniqueKey] = true;+ }+}diff --git a/src/request_queue.js b/src/request_queue.jsnew file mode 100644index 000000000..20a749ff4--- /dev/null+++ b/src/request_queue.js@@ -0,0 +1,272 @@+import fs from 'fs';+import fsExtra from 'fs-extra';+import path from 'path';+import { checkParamOrThrow } from 'apify-client/build/utils';+import Promise from 'bluebird';+import _ from 'underscore';+import Request from './request';+import { ensureDirExists, checkParamPrototypeOrThrow } from './utils';++export const LOCAL_EMULATION_SUBDIR = 'request-queues';++const writeFilePromised = Promise.promisify(fs.writeFile);+const readdirPromised = Promise.promisify(fs.readdir);+const readFilePromised = Promise.promisify(fs.readFile);+const moveFilePromised = Promise.promisify(fsExtra.move);++/**+ * Helper function that extracts queue order number from filename.+ *+ * @ignore+ */+const filepathToQueueOrderNo = (filepath) => {+ const int = filepath+ .split(path.sep).pop() // Get filename from path+ .split('.')[0]; // Remove extension++ return parseInt(int, 10);+};++/**+ * Local implementation of RequestQueue.+ *+ * @ignore+ */+export class RequestQueueLocal {+ constructor(queueId, localEmulationDir) {+ checkParamOrThrow(queueId, 'options.queueId', 'String');+ checkParamOrThrow(localEmulationDir, 'localEmulationDir', 'String');++ this.queueId = queueId;+ this.localEmulationPath = path.resolve(path.join(localEmulationDir, LOCAL_EMULATION_SUBDIR, queueId));+ this.localHandledEmulationPath = path.join(this.localEmulationPath, 'handled');+ this.localPendingEmulationPath = path.join(this.localEmulationPath, 'pending');++ this.pendingCount = 0;+ this.inProgressCount = 0;+ this.uniqueKeyToQueueOrderNo = {};+ this.queueOrderNoInProgress = {};++ this.initializationPromise = this._initialize();+ }++ _initialize() {+ return ensureDirExists(this.localEmulationPath)+ .then(() => ensureDirExists(this.localHandledEmulationPath))+ .then(() => ensureDirExists(this.localPendingEmulationPath))+ .then(() => Promise.all([+ readdirPromised(this.localHandledEmulationPath),+ readdirPromised(this.localPendingEmulationPath),+ ]))+ .then(([handled, pending]) => {+ this.pendingCount = pending.length;++ const handledPaths = handled.map(filename => path.join(this.localHandledEmulationPath, filename));+ const pendingPaths = pending.map(filename => path.join(this.localPendingEmulationPath, filename));+ const filePaths = handledPaths.concat(pendingPaths);++ return Promise.mapSeries(filePaths, filepath => this._readFile(filepath));+ });+ }++ _readFile(filepath) {+ return readFilePromised(filepath)+ .then((str) => {+ const request = JSON.parse(str);+ const queueOrderNo = filepathToQueueOrderNo(filepath);++ this.uniqueKeyToQueueOrderNo[request.uniqueKey] = queueOrderNo;+ });+ }++ _getFilePath(queueOrderNo, isHandled = false) {+ const fileName = `${queueOrderNo}.json`;+ const dir = isHandled+ ? this.localHandledEmulationPath+ : this.localPendingEmulationPath;++ return path.join(dir, fileName);+ }++ addRequest(request, opts = {}) {+ checkParamPrototypeOrThrow(request, 'request', Request, 'Apify.Request');+ checkParamOrThrow(opts, 'opts', 'Object');++ const { putInFront = false } = opts;++ checkParamOrThrow(putInFront, 'opts.putInFront', 'Boolean');++ return this.initializationPromise+ .then(() => {+ const sgn = (putInFront ? 1 : 2) * (10 ** 15);+ const base = (10 ** (13)); // Date.now() returns int with 13 numbers.+ // We always add pending count for a case that two pages are insterted at the same millisecond.+ const now = Date.now() + this.pendingCount;+ const queueOrderNo = putInFront+ ? sgn + (base - now)+ : sgn + (base + now);+ const filePath = this._getFilePath(queueOrderNo);++ this.pendingCount++;+ this.uniqueKeyToQueueOrderNo[request.uniqueKey] = queueOrderNo;++ return writeFilePromised(filePath, JSON.stringify(request, null, 4));+ });+ }++ getRequest(uniqueKey) {+ checkParamOrThrow(uniqueKey, 'uniqueKey', 'String');++ return this.initializationPromise+ .then(() => {+ const queueOrderNo = this.uniqueKeyToQueueOrderNo[uniqueKey];++ return this._getRequestByQueueOrderNo(queueOrderNo);+ });+ }++ _getRequestByQueueOrderNo(queueOrderNo) {+ checkParamOrThrow(queueOrderNo, 'queueOrderNo', 'Number');++ return readFilePromised(this._getFilePath(queueOrderNo, false))+ .catch((err) => {+ if (err.code !== 'ENOENT') throw err;++ return readFilePromised(this._getFilePath(queueOrderNo, true));+ })+ .then((str) => {+ if (!str) throw new Error('Request was not found in none of handled and pending directories!');++ const obj = JSON.parse(str);++ return new Request(obj);+ });+ }++ fetchNextRequest() {+ return this.initializationPromise+ .then(() => readdirPromised(this.localPendingEmulationPath))+ .then((files) => {+ let queueOrderNo;++ _.find(files, (filename) => {+ const no = filepathToQueueOrderNo(filename);++ if (!this.queueOrderNoInProgress[no]) {+ queueOrderNo = no;+ return true;+ }+ });++ if (!queueOrderNo) return null;++ this.queueOrderNoInProgress[queueOrderNo] = true;+ this.inProgressCount++;++ return this._getRequestByQueueOrderNo(queueOrderNo);+ });+ }++ markRequestHandled(request) {+ checkParamPrototypeOrThrow(request, 'request', Request, 'Apify.Request');++ return this.initializationPromise+ .then(() => {+ const queueOrderNo = this.uniqueKeyToQueueOrderNo[request.uniqueKey];+ const source = this._getFilePath(queueOrderNo, false);+ const dest = this._getFilePath(queueOrderNo, true);++ if (!this.queueOrderNoInProgress[queueOrderNo]) {+ throw new Error('Cannot mark handled request that is not in progress!');+ }++ return moveFilePromised(source, dest)+ .then(() => {+ this.pendingCount--;+ this.inProgressCount--;+ delete this.queueOrderNoInProgress[queueOrderNo];+ });+ });+ }++ reclaimRequest(request) {+ checkParamPrototypeOrThrow(request, 'request', Request, 'Apify.Request');++ return this.initializationPromise+ .then(() => {+ const queueOrderNo = this.uniqueKeyToQueueOrderNo[request.uniqueKey];++ if (!this.queueOrderNoInProgress[queueOrderNo]) {+ throw new Error('Cannot reclaim request that is not in progress!');+ }++ this.inProgressCount--;++ delete this.queueOrderNoInProgress[queueOrderNo];+ });+ }++ isEmpty() {+ return this.initializationPromise+ .then(() => this.pendingCount === this.inProgressCount);+ }++ isFinished() {+ return this.initializationPromise+ .then(() => this.pendingCount === 0);+ }+}++/**+ * Opens request queue and returns its object.</p>+ *+ * ```javascript+ * const queue = await Apify.openRequestQueue('my-queue-id');+ *+ * await queue.addRequest(new Apify.Request({ url: 'http://example.com/aaa'});+ * await queue.addRequest(new Apify.Request({ url: 'http://example.com/bbb'});+ * await queue.addRequest(new Apify.Request({ url: 'http://example.com/foo/bar'}, { puyInFront: true });+ *+ * // Get requests from queue+ * const request1 = queue.fetchNextRequest();+ * const request2 = queue.fetchNextRequest();+ * const request3 = queue.fetchNextRequest();+ *+ * // Mark some of them as handled+ * queue.markRequestHandled(request1);+ *+ * // If processing fails then reclaim it back to the queue+ * queue.reclaimRequest(request2);+ * ```+ *+ * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is defined, the value this function+ * returns an instance `RequestQueueLocal` which is an local emulation of request queue.+ * This is useful for local development and debugging of your acts.+ *+ * @param {string} queueIdOrName ID or name of the request queue to be opened.+ * @returns {Promise<RequestQueue>} Returns a promise that resolves to a RequestQueue object.+ *+ * @memberof module:Apify+ * @name openRequestQueue+ * @instance+ *+ * @ignore+ */+export const openRequestQueue = (queueIdOrName) => {+ const localEmulationDir = process.env[ENV_VARS.LOCAL_EMULATION_DIR];++ checkParamOrThrow(queueIdOrName, 'queueIdOrName', 'Maybe String');++ // TODO:+ // should reuse instances+ if (!localEmulationDir) {+ return Promise.reject(new Error('We currently don\'t support remote queues! This is only temporary'+ + 'implementation that stores the queue in the key-value store.'));+ }++ const queue = localEmulationDir+ ? new RequestQueueLocal(queueIdOrName, localEmulationDir)+ : new RequestQueueRemote('default-request-queue');++ return Promise.resolve(queue);+};diff --git a/src/settings_rotator.js b/src/settings_rotator.jsnew file mode 100644index 000000000..31bddf224--- /dev/null+++ b/src/settings_rotator.js@@ -0,0 +1,49 @@+import { checkParamOrThrow } from 'apify-client/build/utils';++// TODO: randomWaitBetweenUsagesMillis++/**+ * SettingsRotator rotates settings created by newSettingsFunction based on it's configuration.+ *+ * @param {Object} options+ * @param {Function} options.newSettingsFunction+ * @param {Number} options.maxUsages+ */+export default class SettingsRotator {+ constructor({+ newSettingsFunction,+ maxUsages,+ }) {+ checkParamOrThrow(newSettingsFunction, 'options.newSettingsFunction', 'Function');+ checkParamOrThrow(maxUsages, 'options.maxUsages', 'Number');++ this.maxUsages = maxUsages;+ this.newSettingsFunction = newSettingsFunction;++ this.currentSettings = null;+ this.currentSettingsUsageCount = 0;+ }++ /**+ * Fetches a settings object.+ *+ * @return {*}+ */+ fetchSettings() {+ if (!this.currentSettings || this.currentSettingsUsageCount >= this.maxUsages) {+ this.currentSettings = this.newSettingsFunction();+ this.currentSettingsUsageCount = 0;+ }++ this.currentSettingsUsageCount++;++ return this.currentSettings;+ }++ /**+ * Reclaims settings after use.+ *+ * @param {*} settings+ */+ reclaimSettings(settings) {} // eslint-disable-line+}diff --git a/src/utils.js b/src/utils.jsindex 25dfe1e2e..27d3228f7 100644--- a/src/utils.js+++ b/src/utils.js@@ -1,90 +1,17 @@import Promise from 'bluebird';import contentTypeParser from 'content-type';+import os from 'os';import fs from 'fs';+import fsExtra from 'fs-extra';import ApifyClient from 'apify-client';import { ENV_VARS } from './constants';-// For backward compatibility, re-export functions that were moved to proxy-chain package-// TODO: eventually get rid of this-export { parseUrl, redactUrl, redactParsedUrl } from 'proxy-chain';--/* global process */--let PromisesDependency = Promise;--// TODO: add methods to override console.log() and console.error(), add unit tests for that!--/**- * @memberof module:Apify- * @function- * @description <p>Sets the promise dependency that the package will use wherever promises are returned.- * Passing `null` will force the SDK to use native Promises if they are available.</p>- * <p>Example usage</p>- * <pre><code class="language-javascript">const Promise = require('bluebird');- * const Apify = require('apify');- * - * Apify.setPromisesDependency(Promise);- * </code></pre>- * By default, the package uses the `bluebird` promises.- * @param [Constructor] dep Reference to a Promise constructor- */-export const setPromisesDependency = (dep) => {- if (dep !== null && typeof dep !== 'function') throw new Error('The "dep" parameter must be a function');- PromisesDependency = dep;-};--/**- * @memberof module:Apify- * @function- * @description Gets the promise dependency set by <a href="#module-Apify-setPromisesDependency"><code>Apify.setPromisesDependency</code></a>.- * By default, the package uses the `bluebird` promises.- * @returns {Constructor} Reference to a Promise constructor- */-export const getPromisesDependency = () => {- return PromisesDependency;-};--/**- * Gets a promise dependency set using `setPromisesDependency()`,- * or returns the native `Promise` function, or throws if no native promises are available.- * @returns Promise- * @ignore- */-export const getPromisePrototype = () => {- if (PromisesDependency) {- if (typeof (PromisesDependency.resolve) !== 'function') {- throw new Error('The promise dependency set using Apify.setPromisesDependency() does not define resolve() function.');- }- return PromisesDependency;- }- if (typeof Promise === 'function') return Promise;- throw new Error('Native promises are not available, please call Apify.setPromisesDependency() to set a promise library.');-};--/**- * Returns a result of `Promise.resolve()` using promise library set by `setPromisesDependency()`,- * or using native promises, or throws if no native promises are available.- * @returns {*}- * @ignore- */-export const newPromise = () => {- return getPromisePrototype().resolve();-};--export const nodeifyPromise = (promise, callback) => {- if (!promise) throw new Error('The "promise" parameter must be provided.');-- if (callback) {- promise.then(result => callback(null, result), err => callback(err));- } else {- return promise;- }-};-+const ensureDirPromised = Promise.promisify(fsExtra.ensureDir);/*** Creates an instance of ApifyClient using options as defined in the environment variables.- * This function is exported in order to enable unit testing.+ * This function is exported to enable unit testing.+ ** @returns {*}* @ignore*/@@ -102,12 +29,43 @@ export const newClient = () => {return new ApifyClient(opts);};+/**+ * A default instance of the `ApifyClient` class provided+ * by the {@link https://www.apify.com/docs/sdk/apify-client-js/latest|apify-client} NPM package.+ * The instance is created when the `apify` package is first imported+ * and it is configured using the `APIFY_API_BASE_URL`, `APIFY_USER_ID` and `APIFY_TOKEN`+ * environment variables.+ *+ * After that, the instance is used for all underlying calls to the Apify API+ * in functions such as <a href="#module-Apify-getValue">Apify.getValue()</a>+ * or <a href="#module-Apify-call">Apify.call()</a>.+ * The settings of the client can be globally altered by calling the+ * <a href="https://www.apify.com/docs/js/apify-client-js/latest#ApifyClient-setOptions"><code>Apify.client.setOptions()</code></a> function.+ * Just be careful, it might have undesired effects on other functions provided by this package.+ *+ * @memberof module:Apify+ * @name client+ * @instance+ */+export const apifyClient = newClient();++/**+ * Returns a result of `Promise.resolve()`.+ *+ * @returns {*}+ *+ * @ignore+ */+export const newPromise = () => {+ return Promise.resolve();+};/*** Adds charset=utf-8 to given content type if this parameter is missing.** @param contentType* @returns {string}+ ** @ignore*/export const addCharsetToContentType = (contentType) => {@@ -143,12 +101,103 @@ const createIsDockerPromise = () => {* Returns promise that resolves to true if the code is running in Docker container.* See https://github.com/sindresorhus/is-docker*- * Param forceReset is just internal for unit tests.+ * Parameter forceReset is just internal for unit tests.+ *+ * @return {Promise}*- * @return Promise+ * @memberof module:Apify+ * @name isDocker+ * @instance*/export const isDocker = (forceReset) => {if (!isDockerPromise || forceReset) isDockerPromise = createIsDockerPromise();return isDockerPromise;};++/**+ * Returns memory statistics of the container, which is an object with the following properties:+ *+ * ```javascript+ * {+ * // Total memory available to the act+ * totalBytes: Number,+ * + * // Amount of free memory+ * freeBytes: Number,+ * + * // Amount of memory used (= totalBytes - freeBytes)+ * usedBytes: Number,+ * }+ * ```+ *+ * @returns {Promise} Returns a promise.+ *+ * @memberof module:Apify+ * @name getMemoryInfo+ * @instance+ */+export const getMemoryInfo = () => {+ // module.exports must be here so that we can mock it.+ return module.exports.isDocker()+ .then((isDockerVar) => {+ if (!isDockerVar) {+ const freeBytes = os.freemem();+ const totalBytes = os.totalmem();++ return Promise.resolve({ totalBytes, freeBytes, usedBytes: totalBytes - freeBytes });+ }++ // This must be promisified here so that we can Mock it.+ const readPromised = Promise.promisify(fs.readFile);++ return Promise+ .all([+ readPromised('/sys/fs/cgroup/memory/memory.limit_in_bytes'),+ readPromised('/sys/fs/cgroup/memory/memory.usage_in_bytes'),+ ])+ .then(([totalBytesStr, usedBytesStr]) => {+ const totalBytes = parseInt(totalBytesStr, 10);+ const usedBytes = parseInt(usedBytesStr, 10);++ return { totalBytes, freeBytes: totalBytes - usedBytes, usedBytes };+ });+ });+};++/**+ * Helper function that detrermines if given parameter is an instance of Promise.+ *+ * @ignore+ */+export const isPromise = (maybePromise) => {+ return maybePromise && typeof maybePromise.then === 'function' && typeof maybePromise.catch === 'function';+};++/**+ * Helper function for validation if parameter is an instance of given prototype.+ * TODO: Move this to shared package along with checkParamOrThrow+ *+ * @ignore+ */+export const checkParamPrototypeOrThrow = (paramVal, paramName, prototype, prototypeName, isOptional = false) => {+ if (isOptional && (paramVal === undefined || paramVal === null)) return;++ if (paramVal && !(paramVal instanceof prototype)) {+ throw new Error(`Parameter "${paramName}" must be an instance of ${prototypeName}`);+ }+};++/**+ * Returns true if node is in production environment and false otherwise.+ *+ * @ignore+ */+export const isProduction = () => process.env.NODE_ENV !== 'production';++/**+ * Helper function used for local implementations. Creates dir.+ *+ * @ignore+ */+export const ensureDirExists = path => ensureDirPromised(path);diff --git a/src/webdriver.js b/src/webdriver.jsindex 610d5ba59..0cad1937e 100644--- a/src/webdriver.js+++ b/src/webdriver.js@@ -1,8 +1,7 @@// import { ChromeLauncher } from 'lighthouse/lighthouse-cli/chrome-launcher';import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';import { ENV_VARS } from './constants';-import { newPromise, nodeifyPromise } from './utils';-+import { newPromise } from './utils';/* global process, require */@@ -16,10 +15,10 @@ import { newPromise, nodeifyPromise } from './utils';// TODO: on first use of Apify.browse(), print out the version of Chrome and ChromeDriver-/*** Gets the default options for the browse() function, generated from current process environment* variables. This is function to enable unit testing.+ ** @ignore*/export const getDefaultBrowseOptions = () => {@@ -31,10 +30,10 @@ export const getDefaultBrowseOptions = () => {};};-/*** Represents a single web browser process.* Currently it is just a thin wrapper of Selenium's WebDriver instance.+ ** @ignore*/export class Browser {@@ -77,10 +76,6 @@ export class Browser {this.webDriver = null;}- /**- * Initializes the browser.- * @returns Promise- */_initialize() {let promise = null;@@ -146,29 +141,23 @@ export class Browser {/*** Normalizes arguments for Apify.browse(), fills correctly default values.* The function is exported to allow unit testing.- * @param url Optional string- * @param options Optional object- * @param callback Optional function+ *+ * @param {String} [url]+ * @param {Object} [options]+ ** @ignore*/-export const processBrowseArgs = (url, options, callback) => {- if (typeof (url) === 'object' || typeof (url) === 'function') {- callback = options;+export const processBrowseArgs = (url, options) => {+ if (typeof (url) === 'object') {options = url;url = null;}- if (typeof (options) === 'function') {- callback = options;- options = null;- }options = Object.assign({}, options);options.url = url || options.url || 'about:blank';- callback = callback || null;if (typeof (options.url) !== 'string') throw new Error('Invalid "url" provided.');- if (callback && typeof (callback) !== 'function') throw new Error('Invalid "callback" provided.');- return { options, callback };+ return { options };};/*@@ -178,7 +167,7 @@ OLD INFO FROM README:Apify runtime optionally depends onthe [selenium-webdriver](https://www.npmjs.com/package/selenium-webdriver) package that enablesautomation of a web browser.-The simplest way to launch a new web browser is using the `Apify.browse([url,] [options,] [callback])`+The simplest way to launch a new web browser is using the `Apify.browse([url,] [options)`function. For example:```javascript@@ -224,8 +213,6 @@ The `options` parameter controls settings of the web browser and it has the follThe result of the `Apify.browse()` is a new instance of the `Browser` class,which represents a web browser instance (possibly with multiple windows or tabs).-If you pass a Node.js-style callback the `Browser` instance is passed to it,-otherwise the `Apify.browse()` function returns a promise that resolves to the `Browser` instance.The `Browser` class has the following properties:@@ -259,68 +246,65 @@ await browser.close();// TODO: browse() is only kept for backwards compatibility, get rid of it after no acts are using it!/**- * @memberof module:Apify- * @function- * @description Opens a new web browser, which is attached to Apify debugger so that snapshots are sent to Run console (TODO).+ * Opens a new web browser, which is attached to Apify debugger so that snapshots are sent to Run console (TODO).* Internally, this function calls Selenium WebDrivers's Builder command to create a new WebDriver instance.* (see http://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_Builder.html)* The result of the function is a new instance of the Browser class.- * @param url Optional start URL to open. Defaults to about:blank- * @param options Optional settings, their defaults are provided by the getDefaultBrowseOptions() function.- * @param callback Optional callback.- * @returns Returns a promise if no callback was provided, otherwise the return value is not defined.+ *+ * @param {String} [url] start URL to open. Defaults to about:blank+ * @param {Object} [options] settings, their defaults are provided by the getDefaultBrowseOptions() function.+ * @returns {Promise}+ *+ * @memberof module:Apify+ * @function* @ignore*/-export const browse = (url, options, callback) => {- const args = processBrowseArgs(url, options, callback);-+export const browse = (url, options) => {+ const args = processBrowseArgs(url, options);const browser = new Browser(args.options);- const promise = browser._initialize()++ return browser._initialize().then(() => {return browser.webDriver.get(args.options.url);}).then(() => {return browser;});-- return nodeifyPromise(promise, args.callback);};-/**- * @memberof module:Apify- * @function- * @description <p>Opens a new instance of Chrome web browser+ * Opens a new instance of Chrome web browser* controlled by {@link http://www.seleniumhq.org/projects/webdriver/|Selenium WebDriver}.* The result of the function is the new instance of the* {@link http://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_WebDriver.html|WebDriver}* class.- * </p>- * <p>+ ** If the `APIFY_HEADLESS` environment variable is set to `1`, the function* runs the web browser in headless mode. Note that this environment variable is automatically set to `1` when* in acts running on the Apify Actor cloud platform.- * </p>- * <p>+ ** To use this function, you need to have Google Chrome and* {@link https://sites.google.com/a/chromium.org/chromedriver/|ChromeDriver} installed in your environment.* For example, you can use the `apify/actor-node-chrome` base Docker image for your act - see* {@link https://www.apify.com/docs/actor#base-images|documentation}* for more details.- * </p>+ ** @param {Object} [opts] Optional settings passed to `puppeteer.launch()`. Additionally the object can contain the following fields:* @param {String} [opts.proxyUrl] - URL to a proxy server. Currently only `http://` scheme is supported.* Port number must be specified. For example, `http://example.com:1234`.* @param {String} [opts.userAgent] - Default User-Agent for the browser.- * @param {Function} callback Optional callback.- * @returns {Promise} Returns a promise if no callback was provided, otherwise the return value is not defined.+ * @returns {Promise}+ *+ * @memberof module:Apify+ * @name launchWebDriver+ * @instance*/-export const launchWebDriver = (opts, callback) => {- const args = processBrowseArgs(undefined, opts, callback);+export const launchWebDriver = (opts) => {+ const args = processBrowseArgs(undefined, opts);const browser = new Browser(args.options);// NOTE: eventually get rid of the Browser class- const promise = browser._initialize()+ return browser._initialize().then(() => {// TODO: for some reason this doesn't work, the proxy chain will never shut down!!// BTW this also prevents us from upgrading to mocha 4+@@ -334,11 +318,8 @@ export const launchWebDriver = (opts, callback) => {return browser.webDriver;});-- return nodeifyPromise(promise, args.callback);};-// /**// * Launches a debugging instance of Chrome on port 9222, without Selenium.// * This code is kept here for legacy reasons, it's not used.diff --git a/test/_helper.js b/test/_helper.jsnew file mode 100644index 000000000..8e269480b--- /dev/null+++ b/test/_helper.js@@ -0,0 +1,25 @@+import fs from 'fs-extra';+import path from 'path';+import { expect } from 'chai';+import { ENV_VARS } from '../build/constants';++export const LOCAL_EMULATION_DIR = path.join(__dirname, '..', 'tmp', 'local-emulation-dir');++// Log unhandled rejections.+process.on('unhandledRejection', (err) => {+ console.log('---------------------------------------------------------------------');+ console.log('------------- WARNING: Unhandled promise rejection !!!! -------------');+ console.log('---------------------------------------------------------------------');+ console.log(err);+});++// Immediately ensure that local emulation dir exists.+fs.ensureDirSync(path.resolve(LOCAL_EMULATION_DIR));++export const emptyLocalEmulationSubdir = (subdir) => {+ const fullPath = path.resolve(path.join(LOCAL_EMULATION_DIR, subdir));++ fs.emptyDirSync(fullPath);+};++export const expectNotLocalEmulation = () => expect(process.env[ENV_VARS.LOCAL_EMULATION_DIR]).to.be.a('undefined');diff --git a/test/actor.js b/test/actor.jsindex 2a22da4d6..d47a30a96 100644--- a/test/actor.js+++ b/test/actor.js@@ -1,14 +1,11 @@import fs from 'fs';-import pathModule from 'path';import _ from 'underscore';-import { expect, assert } from 'chai';+import { expect } from 'chai';import sinon from 'sinon';-import os from 'os';import tmp from 'tmp';-import rimraf from 'rimraf';import Promise from 'bluebird';-import { ACT_TASK_STATUSES } from '../build/constants';-import * as utils from '../build/utils';+import { delayPromise } from 'apify-shared/utilities';+import { ACT_TASK_STATUSES, ENV_VARS } from '../build/constants';// NOTE: test use of require() here because this is how its done in actsconst Apify = require('../build/index');@@ -16,7 +13,6 @@ const Apify = require('../build/index');/* global process, describe, it */// TODO: override console.log() to test the error messages (now they are printed to console)-// TODO: test callback version of functions!!!/*let freePorts = [];@@ -306,9 +302,7 @@ describe('Apify.main()', () => {it('on exception in promised user function the process exits with code 91', () => {return testMain({userFunc: () => {- return new Promise((resolve) => {- setTimeout(resolve, 20);- })+ return delayPromise(20).then(() => {throw new Error('Text exception II');});@@ -319,788 +313,6 @@ describe('Apify.main()', () => {});});--describe('Apify.getValue()', () => {- it('throws on invalid args', () => {- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';- const keyErrMsg = 'The "key" parameter must be a non-empty string';- expect(() => { Apify.getValue(); }).to.throw(Error, keyErrMsg);- expect(() => { Apify.getValue({}); }).to.throw(Error, keyErrMsg);- expect(() => { Apify.getValue(''); }).to.throw(Error, keyErrMsg);- expect(() => { Apify.getValue(null); }).to.throw(Error, keyErrMsg);- });-- it('throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined', () => {- const errMsg = 'The \'APIFY_DEFAULT_KEY_VALUE_STORE_ID\' environment variable is not defined';-- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '';- expect(() => { Apify.getValue('KEY'); }).to.throw(Error, errMsg);-- delete process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID;- expect(() => { Apify.getValue('some other key'); }).to.throw(Error, errMsg);- });-- it('supports both promises and callbacks (on success)', () => {- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';- const mock = sinon.mock(Apify.client.keyValueStores);- mock.expects('getRecord')- .twice()- .returns(Promise.resolve(null));-- return Promise.resolve()- .then(() => {- // test promise- return Apify.getValue('INPUT');- })- .then((input) => {- expect(input).to.be.eql(null);- })- .then(() => {- // test callback- return new Promise((resolve, reject) => {- Apify.getValue('INPUT', (err, input) => {- if (err) return reject(err);- resolve(input);- });- });- })- .then((input) => {- expect(input).to.be.eql(null);-- mock.verify();- })- .finally(() => {- mock.restore();- });- });-- it('supports both promises and callbacks (on error)', () => {- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';- const mock = sinon.mock(Apify.client.keyValueStores);- mock.expects('getRecord')- .twice()- .throws(new Error('Test error'));-- return Promise.resolve()- .then(() => {- // test promise- return Apify.getValue('INPUT')- .then(() => {- assert.fail();- })- .catch((err) => {- expect(err.message).to.be.eql('Test error');- });- })- .then(() => {- // test callback- return new Promise((resolve, reject) => {- Apify.getValue('INPUT', (err, input) => {- if (err) return reject(err);- resolve(input);- });- })- .then(() => {- assert.fail();- })- .catch((err) => {- expect(err.message).to.be.eql('Test error');- });- })- .then(() => {- mock.verify();- })- .finally(() => {- mock.restore();- });- });-- it('works with APIFY_DEV_KEY_VALUE_STORE_DIR env var (on success)', () => {- const jsonValue = { test: 123 };- const textValue = 'some text \u00e6\u00f8\u00e5';- const rawValue = Buffer.from('bla bla bla');- const tmpobj = tmp.dirSync();- process.env.APIFY_DEV_KEY_VALUE_STORE_DIR = tmpobj.name;-- // Test that this env var doesn't need to be set- delete process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID;-- return Promise.resolve()- .then(() => {- // Test JSON with default content type- delete process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE;- fs.writeFileSync(pathModule.join(tmpobj.name, 'INPUT'), JSON.stringify(jsonValue));- return Apify.getValue('INPUT');- })- .then((value) => {- expect(value).to.be.eql(jsonValue);- })- .then(() => {- // Test text- fs.writeFileSync(pathModule.join(tmpobj.name, 'TEST_TEXT'), textValue);- process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE = 'text/plain';- return Apify.getValue('TEST_TEXT');- })- .then((value) => {- expect(value).to.be.eql(textValue);- })- .then(() => {- // Test that this env var can be set to anything- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = 'whatever';-- // Test raw data- fs.writeFileSync(pathModule.join(tmpobj.name, 'TEST_RAW'), rawValue);- process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE = 'something/else';- return Apify.getValue('TEST_RAW');- })- .then((value) => {- expect(value).to.be.eql(rawValue);- })- .then(() => {- // Test non-existent key but existing dir- return Apify.getValue('NON_EXISTENT_KEY');- })- .then((value) => {- expect(value).to.be.eql(null);- })- .then(() => {- // Test callback with JSON plus explicit content type- process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE = 'application/json; charset=utf-8';- return new Promise((resolve, reject) => {- Apify.getValue('INPUT', (err, input) => {- if (err) return reject(err);- resolve(input);- });- });- })- .then((value) => {- expect(value).to.be.eql(jsonValue);- })- .finally(() => {- delete process.env.APIFY_DEV_KEY_VALUE_STORE_DIR;- delete process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE;- rimraf.sync(tmpobj.name);- });- });-- it('works with APIFY_DEV_KEY_VALUE_STORE_DIR env var (on error)', () => {- const tmpobj = tmp.dirSync();- process.env.APIFY_DEV_KEY_VALUE_STORE_DIR = tmpobj.name;-- return Promise.resolve()- .then(() => {- // Test invalid JSON- fs.writeFileSync(pathModule.join(tmpobj.name, 'INPUT'), 'something not JSON');- return Apify.getValue('INPUT')- .then(() => {- assert.fail();- })- .catch((err) => {- expect(err.message).to.contain('cannot be parsed as JSON');- });- })- .then(() => {- // Test callback plus non-existent directory- process.env.APIFY_DEV_KEY_VALUE_STORE_DIR = pathModule.join(tmpobj.name, '/blabla/');- return new Promise((resolve, reject) => {- Apify.getValue('INPUT', (err, input) => {- if (err) return reject(err);- resolve(input);- });- })- .then(() => {- assert.fail();- })- .catch((err) => {- expect(err.message).to.contain('The directory does not exist');- });- })- .then(() => {- // Test file instead of dir- const path = pathModule.join(tmpobj.name, '/subfile');- fs.writeFileSync(path, 'some sub file in dir');- process.env.APIFY_DEV_KEY_VALUE_STORE_DIR = path;- return Apify.getValue('INPUT')- .then(() => {- assert.fail();- })- .catch((err) => {- expect(err.message).to.contain('The directory is not a directory');- });- })- .finally(() => {- delete process.env.APIFY_DEV_KEY_VALUE_STORE_DIR;- });- });-});--describe('Apify.setValue()', () => {- it('throws on invalid args', () => {- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';- const keyErrMsg = 'The "key" parameter must be a non-empty string';- expect(() => { Apify.setValue(); }).to.throw(Error, keyErrMsg);- expect(() => { Apify.setValue('', null); }).to.throw(Error, keyErrMsg);- expect(() => { Apify.setValue('', 'some value'); }).to.throw(Error, keyErrMsg);- expect(() => { Apify.setValue({}, 'some value'); }).to.throw(Error, keyErrMsg);- expect(() => { Apify.setValue(123, 'some value'); }).to.throw(Error, keyErrMsg);-- const valueErrMsg = 'The "value" parameter must be a String or Buffer when "contentType" is specified';- expect(() => { Apify.setValue('key', {}, { contentType: 'image/png' }); }).to.throw(Error, valueErrMsg);- expect(() => { Apify.setValue('key', 12345, { contentType: 'image/png' }); }).to.throw(Error, valueErrMsg);- expect(() => { Apify.setValue('key', () => {}, { contentType: 'image/png' }); }).to.throw(Error, valueErrMsg);-- const optsErrMsg = 'The "options" parameter must be an object, null or undefined';- expect(() => { Apify.setValue('key', {}, 123); }).to.throw(Error, optsErrMsg);- expect(() => { Apify.setValue('key', {}, 'bla/bla'); }).to.throw(Error, optsErrMsg);- expect(() => { Apify.setValue('key', {}, true); }).to.throw(Error, optsErrMsg);-- const circularObj = {};- circularObj.xxx = circularObj;- const jsonErrMsg = 'The "value" parameter cannot be stringified to JSON';- expect(() => { Apify.setValue('key', circularObj, null); }).to.throw(Error, jsonErrMsg);- expect(() => { Apify.setValue('key', undefined); }).to.throw(Error, jsonErrMsg);- expect(() => { Apify.setValue('key', () => {}); }).to.throw(Error, jsonErrMsg);- expect(() => { Apify.setValue('key'); }).to.throw(Error, jsonErrMsg);-- const contTypeRedundantErrMsg = 'The "options.contentType" parameter must not be used when removing the record';- expect(() => { Apify.setValue('key', null, { contentType: 'image/png' }); }).to.throw(Error, contTypeRedundantErrMsg);- expect(() => { Apify.setValue('key', null, { contentType: '' }); }).to.throw(Error, contTypeRedundantErrMsg);- expect(() => { Apify.setValue('key', null, { contentType: {} }); }).to.throw(Error, contTypeRedundantErrMsg);-- const contTypeStringErrMsg = 'The "options.contentType" parameter must be a non-empty string, null or undefined';- expect(() => { Apify.setValue('key', 'value', { contentType: 123 }); }).to.throw(Error, contTypeStringErrMsg);- expect(() => { Apify.setValue('key', 'value', { contentType: {} }); }).to.throw(Error, contTypeStringErrMsg);- expect(() => { Apify.setValue('key', 'value', { contentType: new Date() }); }).to.throw(Error, contTypeStringErrMsg);- expect(() => { Apify.setValue('key', 'value', { contentType: '' }); }).to.throw(Error, contTypeStringErrMsg);- });-- it('throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined', () => {- const errMsg = 'The \'APIFY_DEFAULT_KEY_VALUE_STORE_ID\' environment variable is not defined';-- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '';- expect(() => { Apify.setValue('KEY', { something: 123 }); }).to.throw(Error, errMsg);-- delete process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID;- expect(() => { Apify.setValue('KEY', { something: 123 }); }).to.throw(Error, errMsg);- });-- it('supports both promises and callbacks (on success)', () => {- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';- const mock = sinon.mock(Apify.client.keyValueStores);- mock.expects('putRecord')- .exactly(8)- .returns(Promise.resolve(null));-- return Promise.resolve()- .then(() => {- // test promise (no options)- return Apify.setValue('mykey', { someValue: 123 });- })- .then(() => {- // test promise (with options)- return Apify.setValue('mykey', 'value', { contentType: 'text/plain' });- })- .then(() => {- // test promise (null options)- return Apify.setValue('mykey', 'value', null);- })- .then(() => {- // test promise (undefined options)- return Apify.setValue('mykey', 'value', undefined);- })- .then(() => {- // test callback (no options)- return new Promise((resolve, reject) => {- Apify.setValue('mykey', { someValue: 123 }, (err) => {- if (err) return reject(err);- resolve();- });- });- })- .then(() => {- // test callback (with options)- return new Promise((resolve, reject) => {- Apify.setValue('mykey', 'myvalue', { contentType: 'text/plain' }, (err) => {- if (err) return reject(err);- resolve();- });- });- })- .then(() => {- // test callback (null options)- return new Promise((resolve, reject) => {- Apify.setValue('mykey', { someValue: 123 }, null, (err) => {- if (err) return reject(err);- resolve();- });- });- })- .then(() => {- // test callback (undefined options)- return new Promise((resolve, reject) => {- Apify.setValue('mykey', { someValue: 123 }, resolve, (err) => {- if (err) return reject(err);- resolve();- });- });- })- .then(() => {- mock.verify();- })- .finally(() => {- mock.restore();- });- });-- it('supports both promises and callbacks (on error)', () => {- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';- const mock = sinon.mock(Apify.client.keyValueStores);- mock.expects('putRecord')- .twice()- .throws(new Error('Test error'));-- return Promise.resolve()- .then(() => {- // Test promise- return Promise.resolve()- .then(() => {- return Apify.setValue('mykey', { someValue: 1 });- })- .then(() => {- assert.fail();- })- .catch((err) => {- expect(err.message).to.be.eql('Test error');- });- })- .then(() => {- // Test callback- return new Promise((resolve, reject) => {- Apify.setValue('mykey', { someValue: 1 }, (err) => {- if (err) return reject(err);- resolve();- });- })- .then(() => {- assert.fail();- })- .catch((err) => {- expect(err.message).to.be.eql('Test error');- });- })- .then(() => {- mock.verify();- })- .finally(() => {- mock.restore();- });- });-- it('correctly stores object values as JSON', () => {- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';- const storeId = 'mystore';- const key = 'mykey';- const value = { someValue: 123 };-- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = storeId;-- Apify.setPromisesDependency(Promise);-- const mock = sinon.mock(Apify.client.keyValueStores);- mock.expects('putRecord')- .once()- .withArgs({- storeId,- promise: Promise,- key,- body: JSON.stringify(value, null, 2),- contentType: 'application/json; charset=utf-8',- })- .returns(Promise.resolve(null));-- return Promise.resolve()- .then(() => {- return Apify.setValue(key, value);- })- .then(() => {- mock.verify();- })- .finally(() => {- mock.restore();- });- });-- it('correctly adds charset to content type', () => {- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';- const storeId = 'mystore3';- const key = 'mykey2';- const value = 'some string value';-- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = storeId;-- Apify.setPromisesDependency(Promise);-- const mock = sinon.mock(Apify.client.keyValueStores);- mock.expects('putRecord')- .once()- .withArgs({- storeId,- promise: Promise,- key,- body: value,- contentType: 'text/plain; charset=utf-8; foo=bar',- })- .returns(Promise.resolve(null));-- return Promise.resolve()- .then(() => {- return Apify.setValue(key, value, { contentType: 'text/plain; foo=bar' });- })- .then(() => {- mock.verify();- })- .finally(() => {- mock.restore();- });- });-- it('correctly stores raw string values', () => {- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';- const storeId = 'mystore3';- const key = 'mykey2';- const value = 'some string value';- const contentType = 'text/plain; charset=utf-8';-- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = storeId;-- Apify.setPromisesDependency(Promise);-- const mock = sinon.mock(Apify.client.keyValueStores);- mock.expects('putRecord')- .once()- .withArgs({- storeId,- promise: Promise,- key,- body: value,- contentType,- })- .returns(Promise.resolve(null));-- return Promise.resolve()- .then(() => {- return Apify.setValue(key, value, { contentType });- })- .then(() => {- mock.verify();- })- .finally(() => {- mock.restore();- });- });-- it('correctly stores raw Buffer values', () => {- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';- const storeId = 'mystore3';- const key = 'mykey2';- const value = Buffer.from('some text value');- const contentType = 'text/plain; charset=something';-- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = storeId;-- Apify.setPromisesDependency(Promise);-- const mock = sinon.mock(Apify.client.keyValueStores);- mock.expects('putRecord')- .once()- .withArgs({- storeId,- promise: Promise,- key,- body: value,- contentType,- })- .returns(Promise.resolve(null));-- return Promise.resolve()- .then(() => {- return Apify.setValue(key, value, { contentType });- })- .then(() => {- mock.verify();- })- .finally(() => {- mock.restore();- });- });-- it('works with APIFY_DEV_KEY_VALUE_STORE_DIR env var (on success)', () => {- const tmpobj = tmp.dirSync();- const testObj = { 'bla-bla': 123 };- const valueString = 'bla bla some string';- const valueRaw = Buffer.from('some other string that will be raw as ham');- process.env.APIFY_DEV_KEY_VALUE_STORE_DIR = tmpobj.name;-- // Test that this env var doesn't need to be set- delete process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID;-- return Promise.resolve()- .then(() => {- // Test write object- Apify.setValue('TEST_OBJ', testObj);- })- .then(() => {- return Apify.getValue('TEST_OBJ');- })- .then((value) => {- expect(value).to.be.eql(testObj);- })- .then(() => {- // Test removal of value- Apify.setValue('TEST_OBJ', null);- })- .then(() => {- // File must no longer exists- const filePath = pathModule.join(tmpobj.name, '/TEST_OBJ');- const exists = fs.existsSync(filePath);- expect(exists).to.be.eql(false);- })- .then(() => {- // Test that this env var can be set to anything- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = 'whatever';-- // Test write of string with callbacks- return new Promise((resolve, reject) => {- Apify.setValue('TEST_STR', valueString, { contentType: 'text/plain' }, (err, input) => {- if (err) return reject(err);- resolve(input);- });- });- })- .then(() => {- process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE = 'text/plain';- return Apify.getValue('TEST_STR');- })- .then((value) => {- expect(value).to.be.eql(valueString);- })- .then(() => {- // Test write raw buffer- Apify.setValue('TEST_RAW', valueRaw, { contentType: 'something/whatever' });- })- .then(() => {- process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE = 'something/raw';- return Apify.getValue('TEST_RAW');- })- .then((value) => {- expect(value).to.be.eql(valueRaw);- })- .finally(() => {- delete process.env.APIFY_DEV_KEY_VALUE_STORE_DIR;- delete process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE;- // rimraf.sync(tmpobj.name);- });- });-- it('works with APIFY_DEV_KEY_VALUE_STORE_DIR env var (on error)', () => {- const tmpobj = tmp.dirSync();- process.env.APIFY_DEV_KEY_VALUE_STORE_DIR = pathModule.join(tmpobj.name, '/non-existent-dir/');-- return Promise.resolve()- .then(() => {- // Test write object with Promise- Apify.setValue('TEST1', { whatever: 123 })- .then(() => {- assert.fail();- })- .catch((err) => {- expect(err.message).to.contain('ENOENT');- });- })- .then(() => {- // Test callback plus non-existent directory- return new Promise((resolve, reject) => {- Apify.setValue('TEST2', { sometingElse: 456 }, (err, input) => {- if (err) return reject(err);- resolve(input);- });- })- .then(() => {- assert.fail();- })- .catch((err) => {- expect(err.message).to.contain('ENOENT');- });- })- .finally(() => {- delete process.env.APIFY_DEV_KEY_VALUE_STORE_DIR;- delete process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE;- // rimraf.sync(tmpobj.name);- });- });-});--describe('Apify.pushData()', () => {- it('throws on invalid args', () => {- process.env.APIFY_DEFAULT_DATASET_ID = '1234';- const recordErrMsg = 'The "data" parameter must be an object or array';- expect(() => { Apify.pushData(); }).to.throw(Error, recordErrMsg);- expect(() => { Apify.pushData(''); }).to.throw(Error, recordErrMsg);- expect(() => { Apify.pushData(123); }).to.throw(Error, recordErrMsg);- expect(() => { Apify.pushData(true); }).to.throw(Error, recordErrMsg);- expect(() => { Apify.pushData(false); }).to.throw(Error, recordErrMsg);- expect(() => { Apify.pushData(() => {}, () => {}); }).to.throw(Error, jsonErrMsg);-- const circularObj = {};- circularObj.xxx = circularObj;- const jsonErrMsg = 'The "data" parameter cannot be stringified to JSON';- expect(() => { Apify.pushData(circularObj, null); }).to.throw(Error, jsonErrMsg);- });-- it('throws if APIFY_DEFAULT_DATASET_ID env var is not defined', () => {- const errMsg = 'The \'APIFY_DEFAULT_DATASET_ID\' environment variable is not defined';-- process.env.APIFY_DEFAULT_DATASET_ID = '';- expect(() => { Apify.pushData({ something: 123 }); }).to.throw(Error, errMsg);-- delete process.env.APIFY_DEFAULT_DATASET_ID;- expect(() => { Apify.pushData({ something: 123 }); }).to.throw(Error, errMsg);- });-- it('supports both promises and callbacks (on success)', () => {- process.env.APIFY_DEFAULT_DATASET_ID = '1234';- const mock = sinon.mock(Apify.client.datasets);- mock.expects('putItems')- .twice()- .returns(Promise.resolve(null));-- return Promise.resolve()- .then(() => {- // test promise (no options)- return Apify.pushData({ someValue: 123 });- })- .then(() => {- // test callback (no options)- return new Promise((resolve, reject) => {- Apify.pushData({ someValue: 123 }, (err) => {- if (err) return reject(err);- resolve();- });- });- })- .then(() => {- mock.verify();- })- .finally(() => {- mock.restore();- });- });-- it('supports both promises and callbacks (on error)', () => {- process.env.APIFY_DEFAULT_DATASET_ID = '1234';- const mock = sinon.mock(Apify.client.datasets);- mock.expects('putItems')- .twice()- .throws(new Error('Test error'));-- return Promise.resolve()- .then(() => {- // Test promise- return Promise.resolve()- .then(() => {- return Apify.pushData({ someValue: 1 });- })- .then(() => {- assert.fail();- })- .catch((err) => {- expect(err.message).to.be.eql('Test error');- });- })- .then(() => {- // Test callback- return new Promise((resolve, reject) => {- Apify.pushData({ someValue: 1 }, (err) => {- if (err) return reject(err);- resolve();- });- })- .then(() => {- assert.fail();- })- .catch((err) => {- expect(err.message).to.be.eql('Test error');- });- })- .then(() => {- mock.verify();- })- .finally(() => {- mock.restore();- });- });-- it('correctly stores items', () => {- const datasetId = 'mystore';- const value = { someValue: 123 };-- process.env.APIFY_DEFAULT_DATASET_ID = datasetId;-- Apify.setPromisesDependency(Promise);-- const mock = sinon.mock(Apify.client.datasets);- mock.expects('putItems')- .once()- .withArgs({- datasetId,- promise: Promise,- data: value,- })- .returns(Promise.resolve(null));-- return Promise.resolve()- .then(() => {- return Apify.pushData(value);- })- .then(() => {- mock.verify();- })- .finally(() => {- mock.restore();- });- });-- it('correctly stores items when provided as array', () => {- const datasetId = 'mystore';- const value = [{ someValue: 123 }, { someValue: 123 }];-- process.env.APIFY_DEFAULT_DATASET_ID = datasetId;-- Apify.setPromisesDependency(Promise);-- const mock = sinon.mock(Apify.client.datasets);- mock.expects('putItems')- .once()- .withArgs({- datasetId,- promise: Promise,- data: value,- })- .returns(Promise.resolve(null));-- return Promise.resolve()- .then(() => {- return Apify.pushData(value);- })- .then(() => {- mock.verify();- })- .finally(() => {- mock.restore();- });- });-});-describe('Apify.events', () => {it('is there and works as EventEmitter', () => {return new Promise((resolve, reject) => {@@ -1444,99 +656,37 @@ describe('Apify.call()', () => {});});-describe('Apify.getMemoryInfo()', () => {- it('works outside the container', () => {- const osMock = sinon.mock(os);- const utilsMock = sinon.mock(utils);+describe('Apify.getApifyProxyUrl()', () => {+ it('should work', () => {+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';+ process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';+ process.env[ENV_VARS.PROXY_PORT] = 123;- utilsMock- .expects('isDocker')- .once()- .returns(Promise.resolve(false));+ expect(Apify.getApifyProxyUrl({+ session: 'XYZ',+ groups: ['g1', 'g2', 'g3'],+ })).to.be.eql('http://GROUPS-g1+g2+g3,SESSION-XYZ:[email protected]:123');- osMock- .expects('freemem')- .once()- .returns(222);+ expect(Apify.getApifyProxyUrl({+ groups: ['g1', 'g2', 'g3'],+ })).to.be.eql('http://GROUPS-g1+g2+g3:[email protected]:123');- osMock- .expects('totalmem')- .once()- .returns(333);+ expect(Apify.getApifyProxyUrl({+ session: 'XYZ',+ })).to.be.eql('http://SESSION-XYZ:[email protected]:123');- return Apify- .getMemoryInfo()- .then((data) => {- expect(data).to.be.eql({- totalBytes: 333,- freeBytes: 222,- usedBytes: 111,- });+ expect(Apify.getApifyProxyUrl()).to.be.eql('http://auto:[email protected]:123');- utilsMock.restore();- osMock.restore();- });- });-- it('works inside the container', () => {- const utilsMock = sinon.mock(utils);-- utilsMock- .expects('isDocker')- .once()- .returns(Promise.resolve(true));-- sinon- .stub(fs, 'readFile')- .callsFake((path, callback) => {- if (path === '/sys/fs/cgroup/memory/memory.limit_in_bytes') callback(null, '333');- else if (path === '/sys/fs/cgroup/memory/memory.usage_in_bytes') callback(null, '111');- else throw new Error('Invalid path');- });-- return Apify- .getMemoryInfo()- .then((data) => {- expect(data).to.be.eql({- totalBytes: 333,- freeBytes: 222,- usedBytes: 111,- });-- utilsMock.restore();- fs.readFile.restore();- });- });-- it('works with callback', (done) => {- const osMock = sinon.mock(os);- const utilsMock = sinon.mock(utils);-- utilsMock- .expects('isDocker')- .once()- .returns(Promise.resolve(false));-- osMock- .expects('freemem')- .once()- .returns(222);+ delete process.env[ENV_VARS.PROXY_PASSWORD];+ delete process.env[ENV_VARS.PROXY_HOSTNAME];+ delete process.env[ENV_VARS.PROXY_PORT];- osMock- .expects('totalmem')- .once()- .returns(333);+ expect(() => Apify.getApifyProxyUrl()).to.throw();- Apify.getMemoryInfo((err, data) => {- expect(data).to.be.eql({- totalBytes: 333,- freeBytes: 222,- usedBytes: 111,- });-- utilsMock.restore();- osMock.restore();- done();- });+ expect(Apify.getApifyProxyUrl({+ password: 'xyz',+ hostname: 'your.host.com',+ port: 345,+ })).to.be.eql('http://auto:[email protected]:345');});});diff --git a/test/autoscaled_pool.js b/test/autoscaled_pool.jsnew file mode 100644index 000000000..a82852e66--- /dev/null+++ b/test/autoscaled_pool.js@@ -0,0 +1,163 @@+import chai, { expect } from 'chai';+import chaiAsPromised from 'chai-as-promised';+import 'babel-polyfill';+import _ from 'underscore';+import sinon from 'sinon';+import { delayPromise } from 'apify-shared/utilities';+import * as Apify from '../build/index';+import { LOG_INFO_INTERVAL, SCALE_UP_MAX_STEP } from '../build/autoscaled_pool';+import * as utils from '../build/utils';++chai.use(chaiAsPromised);+const toBytes = x => x * 1024 * 1024;++describe('autoscaled_pool', () => {+ it('should work with concurrency 1', async () => {+ const range = _.range(0, 10);+ const result = [];++ const workerFunction = () => {+ if (range.length === 0) return;++ return new Promise((resolve) => {+ const item = range.shift();+ result.push(item);+ setTimeout(resolve, 10);+ });+ };++ const pool = new Apify.AutoscaledPool({+ minConcurrency: 1,+ maxConcurrency: 1,+ workerFunction,+ });++ const startedAt = Date.now();+ await pool.run();++ expect(Date.now() - startedAt).to.be.within(100, 150);+ expect(result).to.be.eql(_.range(0, 10));+ });++ it('should work with concurrency 10', async () => {+ const range = _.range(0, 100);+ const result = [];++ const workerFunction = () => {+ if (range.length === 0) return;++ return new Promise((resolve) => {+ const item = range.shift();+ result.push(item);+ setTimeout(resolve, 10);+ });+ };++ const pool = new Apify.AutoscaledPool({+ minConcurrency: 10,+ maxConcurrency: 10,+ workerFunction,+ });++ const startedAt = Date.now();+ await pool.run();++ expect(Date.now() - startedAt).to.be.within(100, 150);+ expect(result).to.be.eql(_.range(0, 100));+ });++ it('correctly computed space for instances using', async () => {+ const pool = new Apify.AutoscaledPool({+ minConcurrency: 1,+ maxConcurrency: 100,+ workerFunction: async () => {},+ });++ pool.concurrency = 20;+ pool.runningCount = 20;+ pool.freeBytesSnapshots = [+ toBytes(50),+ toBytes(30),+ toBytes(50),+ toBytes(40),+ ];+ const hasSpaceForInstances = pool._computeSpaceForInstances(toBytes(100), true);+ expect(hasSpaceForInstances).to.be.eql(5);++ pool.concurrency = 20;+ pool.runningCount = 20;+ pool.freeBytesSnapshots = [+ toBytes(5),+ toBytes(5),+ toBytes(5),+ toBytes(5),+ ];+ const hasSpaceForInstances2 = pool._computeSpaceForInstances(toBytes(100), true);+ expect(hasSpaceForInstances2).to.be.eql(-2);+ });++ it('should autoscale correctly', async () => {+ const pool = new Apify.AutoscaledPool({+ minConcurrency: 1,+ maxConcurrency: 100,+ workerFunction: async () => {},+ });+ const mock = sinon.mock(utils);++ // Should scale up.+ pool.concurrency = 1;+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;+ mock.expects('getMemoryInfo')+ .once()+ .returns(Promise.resolve({ freeBytes: toBytes(9.99), totalBytes: toBytes(10) }));+ await pool._autoscale();+ expect(pool.concurrency).to.be.eql(1 + SCALE_UP_MAX_STEP);++ // Should not do anything.+ pool.concurrency = 10;+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;+ mock.expects('getMemoryInfo')+ .once()+ .returns(Promise.resolve({ freeBytes: toBytes(1), totalBytes: toBytes(10) }));+ await pool._autoscale();+ expect(pool.concurrency).to.be.eql(10);++ // Should scale down.+ pool.concurrency = 10;+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;+ mock.expects('getMemoryInfo')+ .once()+ .returns(Promise.resolve({ freeBytes: toBytes(0.9), totalBytes: toBytes(10) }));+ await pool._autoscale();+ expect(pool.concurrency).to.be.eql(9);++ mock.verify();+ mock.restore();+ });++ it('should throw when some of the promises throws', async () => {+ let counter = 0;+ const workerFunction = () => {+ counter++;++ if (counter > 100) return;++ if (counter === 100) {+ const err = new Error('some-error');++ return new Promise((resolve, reject) => setTimeout(reject(err), 10));+ }++ return delayPromise(10);+ };++ const pool = new Apify.AutoscaledPool({+ minConcurrency: 10,+ maxConcurrency: 10,+ workerFunction,+ });++ await expect(pool.run()).to.be.rejectedWith('some-error');+ });+});+diff --git a/test/basic_crawler.js b/test/basic_crawler.jsnew file mode 100644index 000000000..7b04044ab--- /dev/null+++ b/test/basic_crawler.js@@ -0,0 +1,116 @@+import chai, { expect } from 'chai';+import chaiAsPromised from 'chai-as-promised';+import _ from 'underscore';+import 'babel-polyfill';+import { delayPromise } from 'apify-shared/utilities';+import * as Apify from '../build/index';++chai.use(chaiAsPromised);++describe('basic_crawler', () => {+ it('runs in parallel thru all the requests', async () => {+ const startedAt = Date.now();+ const sources = _.range(0, 500).map(index => ({ url: `https://example.com/${index}` }));++ const processed = [];+ const requestList = new Apify.RequestList({ sources });+ const handleRequestFunction = async ({ request }) => {+ await delayPromise(10);+ processed.push(_.pick(request, 'url'));+ };++ const basicCrawler = new Apify.BasicCrawler({+ requestList,+ minConcurrency: 25,+ maxConcurrency: 25,+ handleRequestFunction,+ });++ await requestList.initialize();+ await basicCrawler.run();++ expect(processed).to.be.eql(sources);+ expect(Date.now() - startedAt).to.be.within(200, 400);+ });++ it('retries failed requests', async () => {+ const sources = [+ { url: 'http://example.com/1' },+ { url: 'http://example.com/2' },+ { url: 'http://example.com/3' },+ ];+ const processed = {};+ const requestList = new Apify.RequestList({ sources });++ const handleRequestFunction = async ({ request }) => {+ await delayPromise(10);+ processed[request.url] = request;++ if (request.url === 'http://example.com/2') {+ throw Error(`This is ${request.retryCount}th error!`);+ }++ request.userData.foo = 'bar';+ };++ const basicCrawler = new Apify.BasicCrawler({+ requestList,+ maxRequestRetries: 10,+ minConcurrency: 3,+ maxConcurrency: 3,+ handleRequestFunction,+ });++ await requestList.initialize();+ await basicCrawler.run();++ expect(processed['http://example.com/1'].userData.foo).to.be.eql('bar');+ expect(processed['http://example.com/1'].errorMessages).to.be.a('null');+ expect(processed['http://example.com/1'].retryCount).to.be.eql(0);+ expect(processed['http://example.com/3'].userData.foo).to.be.eql('bar');+ expect(processed['http://example.com/3'].errorMessages).to.be.a('null');+ expect(processed['http://example.com/3'].retryCount).to.be.eql(0);++ expect(processed['http://example.com/2'].userData.foo).to.be.a('undefined');+ expect(processed['http://example.com/2'].errorMessages).to.have.lengthOf(11);+ expect(processed['http://example.com/2'].retryCount).to.be.eql(10);+ });++ it('should allow to handle failed requests', async () => {+ const sources = [+ { url: 'http://example.com/1' },+ { url: 'http://example.com/2' },+ { url: 'http://example.com/3' },+ ];+ const processed = {};+ const failed = {};+ const requestList = new Apify.RequestList({ sources });++ const handleRequestFunction = async ({ request }) => {+ await Promise.reject(new Error('some-error'));+ processed[request.url] = request;+ };++ const handleFailedRequestFunction = async ({ request }) => {+ failed[request.url] = request;+ };++ const basicCrawler = new Apify.BasicCrawler({+ requestList,+ handleRequestFunction,+ handleFailedRequestFunction,+ });++ await requestList.initialize();+ await basicCrawler.run();++ expect(failed['http://example.com/1'].errorMessages).to.have.lengthOf(4);+ expect(failed['http://example.com/1'].retryCount).to.be.eql(3);+ expect(failed['http://example.com/2'].errorMessages).to.have.lengthOf(4);+ expect(failed['http://example.com/2'].retryCount).to.be.eql(3);+ expect(failed['http://example.com/3'].errorMessages).to.have.lengthOf(4);+ expect(failed['http://example.com/3'].retryCount).to.be.eql(3);+ expect(_.values(failed)).to.have.length.of(3);+ expect(_.values(processed)).to.have.length.of(0);+ });+});diff --git a/test/constants.js b/test/constants.jsnew file mode 100644index 000000000..026d2de00--- /dev/null+++ b/test/constants.js@@ -0,0 +1,13 @@+import { expect } from 'chai';+import { ACTOR_BASE_DOCKER_IMAGES, BUILD_TAG_LATEST, ENV_VARS } from '../build/constants';++describe('consts', () => {+ it('should contain constants from apify-shared', () => {+ expect(ACTOR_BASE_DOCKER_IMAGES).to.be.a('array');+ expect(BUILD_TAG_LATEST).to.be.a('string');+ });++ it('should contain local constants', () => {+ expect(ENV_VARS).to.be.a('object');+ });+});diff --git a/test/dataset.js b/test/dataset.jsnew file mode 100644index 000000000..bdcb9aff7--- /dev/null+++ b/test/dataset.js@@ -0,0 +1,207 @@+import chai, { expect } from 'chai';+import chaiAsPromised from 'chai-as-promised';+import 'babel-polyfill';+import fs from 'fs-extra';+import path from 'path';+import sinon from 'sinon';+import { leftpad, delayPromise } from 'apify-shared/utilities';+import { ENV_VARS } from '../build/constants';+import { LEFTPAD_COUNT, Dataset, DatasetLocal, LOCAL_EMULATION_SUBDIR } from '../build/dataset';+import { apifyClient } from '../build/utils';+import * as Apify from '../build/index';+import { LOCAL_EMULATION_DIR, emptyLocalEmulationSubdir, expectNotLocalEmulation } from './_helper';++chai.use(chaiAsPromised);++const read = (datasetName, index) => {+ const fileName = `${leftpad(index, LEFTPAD_COUNT, 0)}.json`;+ const filePath = path.join(LOCAL_EMULATION_DIR, LOCAL_EMULATION_SUBDIR, datasetName, fileName);+ const str = fs.readFileSync(path.resolve(filePath));++ return JSON.parse(str);+};++describe('dataset', () => {+ before(() => apifyClient.setOptions({ token: 'xxx' }));+ after(() => apifyClient.setOptions({ token: undefined }));+ beforeEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));+ afterEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));++ describe('local', async () => {+ it('should work', async () => {+ const dataset = new DatasetLocal('my-dataset', LOCAL_EMULATION_DIR);++ await dataset.pushData({ foo: 'bar' });+ await dataset.pushData({ foo: 'hotel' });+ await dataset.pushData([+ { foo: 'from-array-1', arr: [1, 2, 3] },+ { foo: 'from-array-1', arr: [1, 2, 3] },+ ]);++ expect(read('my-dataset', 1)).to.be.eql({ foo: 'bar' });+ expect(read('my-dataset', 2)).to.be.eql({ foo: 'hotel' });+ expect(read('my-dataset', 3)).to.be.eql({ foo: 'from-array-1', arr: [1, 2, 3] });+ expect(read('my-dataset', 4)).to.be.eql({ foo: 'from-array-1', arr: [1, 2, 3] });++ // Correctly initializes the state.+ const newDataset = new DatasetLocal('my-dataset', LOCAL_EMULATION_DIR);+ await newDataset.pushData({ foo2: 'bar2' });+ expect(read('my-dataset', 5)).to.be.eql({ foo2: 'bar2' });+ });+ });++ describe('remote', async () => {+ it('should work', async () => {+ const dataset = new Dataset('some-id');++ const mock = sinon.mock(apifyClient.datasets);++ mock.expects('putItems')+ .once()+ .withArgs({ datasetId: 'some-id', data: { foo: 'bar' } })+ .returns(Promise.resolve(null));++ mock.expects('putItems')+ .once()+ .withArgs({ datasetId: 'some-id', data: [{ foo: 'hotel;' }, { foo: 'restaurant' }] })+ .returns(Promise.resolve(null));++ await dataset.pushData({ foo: 'bar' });+ await dataset.pushData([+ { foo: 'hotel;' },+ { foo: 'restaurant' },+ ]);++ mock.verify();+ mock.restore();+ });+ });++ describe('Apify.openDataset', async () => {+ it('should open a local dataset when process.env[ENV_VARS.LOCAL_EMULATION_DIR] is set', async () => {+ process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;++ const dataset = await Apify.openDataset('some-id-2');+ expect(dataset).to.be.instanceof(DatasetLocal);+ expect(dataset).not.to.be.instanceof(Dataset);++ delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];+ });++ it('should reuse cached dataset instances', async () => {+ process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;++ const dataset1 = await Apify.openDataset('some-id-3');+ const dataset2 = await Apify.openDataset('some-id-3');+ const dataset3 = new DatasetLocal('some-id-3', LOCAL_EMULATION_DIR);++ expect(dataset1).to.be.instanceof(DatasetLocal);+ expect(dataset2).to.be.instanceof(DatasetLocal);+ expect(dataset3).to.be.instanceof(DatasetLocal);++ expect(dataset1).to.be.equal(dataset2);+ expect(dataset1).not.to.be.equal(dataset3);++ delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];++ // Here must be some timeout to don't finish before initialization of dataset finishes.+ // Otherwise we delete the directory and scandir will throw ENOENT: no such file or directory+ await delayPromise(100);+ });++ it('should open default dataset when datasetIdOrName is not provided', async () => {+ process.env[ENV_VARS.DEFAULT_DATASET_ID] = 'some-id-4';+ process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;++ const dataset = await Apify.openDataset();+ expect(dataset.datasetId).to.be.eql('some-id-4');+ expect(dataset).to.be.instanceof(DatasetLocal);++ delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];+ process.env[ENV_VARS.DEFAULT_DATASET_ID] = 'some-id-5';+ expectNotLocalEmulation();++ const dataset2 = await Apify.openDataset();+ expect(dataset2.datasetId).to.be.eql('some-id-5');+ expect(dataset2).to.be.instanceof(Dataset);++ delete process.env[ENV_VARS.DEFAULT_DATASET_ID];+ });++ it('should open remote dataset when process.env[ENV_VARS.LOCAL_EMULATION_DIR] is NOT set', async () => {+ expectNotLocalEmulation();++ const mock = sinon.mock(apifyClient.datasets);++ // First when used with id it only requests store object.+ mock.expects('getDataset')+ .once()+ .withArgs({ datasetId: 'some-id-6' })+ .returns(Promise.resolve({ id: 'some-id-6' }));+ const dataset = await Apify.openDataset('some-id-6');+ expect(dataset.datasetId).to.be.eql('some-id-6');+ expect(dataset).to.be.instanceof(Dataset);++ // Then used with name it requests store object, gets empty response+ // so then it creates dataset.+ mock.expects('getDataset')+ .once()+ .withArgs({ datasetId: 'some-name-7' })+ .returns(Promise.resolve(null));+ mock.expects('getOrCreateDataset')+ .once()+ .withArgs({ datasetName: 'some-name-7' })+ .returns(Promise.resolve({ id: 'some-id-7' }));++ const dataset2 = await Apify.openDataset('some-name-7');+ expect(dataset2.datasetId).to.be.eql('some-id-7');+ expect(dataset2).to.be.instanceof(Dataset);++ mock.verify();+ mock.restore();+ });+ });++ describe('pushData', async () => {+ it('throws on invalid args', async () => {+ process.env[ENV_VARS.DEFAULT_DATASET_ID] = 'some-id-8';+ process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;++ const dataErrMsg = 'Parameter "data" of type Array | Object must be provided';+ await expect(Apify.pushData()).to.be.rejectedWith(dataErrMsg);+ await expect(Apify.pushData('')).to.be.rejectedWith(dataErrMsg);+ await expect(Apify.pushData(123)).to.be.rejectedWith(dataErrMsg);+ await expect(Apify.pushData(true)).to.be.rejectedWith(dataErrMsg);+ await expect(Apify.pushData(false)).to.be.rejectedWith(dataErrMsg);+ await expect(Apify.pushData(() => {})).to.be.rejectedWith(dataErrMsg);++ const circularObj = { xxx: circularObj };+ circularObj.xxx = circularObj;+ const jsonErrMsg = 'Converting circular structure to JSON';+ await expect(Apify.pushData(circularObj)).to.be.rejectedWith(jsonErrMsg);++ delete process.env[ENV_VARS.DEFAULT_DATASET_ID];+ });++ it('throws if DEFAULT_DATASET_ID env var is not defined', async () => {+ process.env[ENV_VARS.DEFAULT_DATASET_ID] = '';+ await expect(Apify.pushData({ something: 123 })).to.be.rejectedWith(Error);++ delete process.env[ENV_VARS.DEFAULT_DATASET_ID];+ await expect(Apify.pushData({ something: 123 })).to.be.rejectedWith(Error);+ });++ it('correctly stores records', async () => {+ process.env[ENV_VARS.DEFAULT_DATASET_ID] = 'some-id-9';+ process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;++ await Apify.pushData({ foo: 'bar' });+ await Apify.pushData({ foo: 'hotel' });++ expect(read('some-id-9', 1)).to.be.eql({ foo: 'bar' });+ expect(read('some-id-9', 2)).to.be.eql({ foo: 'hotel' });++ delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];+ });+ });+});diff --git a/test/key_value_store.js b/test/key_value_store.jsnew file mode 100644index 000000000..5e3b03692--- /dev/null+++ b/test/key_value_store.js@@ -0,0 +1,328 @@+import chai, { expect } from 'chai';+import chaiAsPromised from 'chai-as-promised';+import 'babel-polyfill';+import sinon from 'sinon';+import { ENV_VARS } from '../build/constants';+import { KeyValueStoreLocal, KeyValueStore, LOCAL_EMULATION_SUBDIR } from '../build/key_value_store';+import { apifyClient } from '../build/utils';+import * as Apify from '../build/index';+import { LOCAL_EMULATION_DIR, emptyLocalEmulationSubdir, expectNotLocalEmulation } from './_helper';++chai.use(chaiAsPromised);++describe('key_value_store', () => {+ before(() => apifyClient.setOptions({ token: 'xxx' }));+ after(() => apifyClient.setOptions({ token: undefined }));+ beforeEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));+ afterEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));++ describe('local', async () => {+ it('should work', async () => {+ const store = new KeyValueStoreLocal('my-store-id', LOCAL_EMULATION_DIR);+ const store2 = new KeyValueStoreLocal('another-store-id', LOCAL_EMULATION_DIR);+ const buffer = Buffer.from('some text value');++ await store.setValue('key-obj', { foo: 'bar' });+ await store.setValue('key-string', 'xxxx', { contentType: 'text/plain' });+ await store.setValue('key-buffer', buffer, { contentType: 'image/jpeg' });+ await store2.setValue('key-obj', { foo: 'hotel' });+ await store2.setValue('key-string', 'yyyy', { contentType: 'text/plain' });++ // Try to read store2/key-string.+ expect(await store2.getValue('key-string')).to.be.eql('yyyy');++ // Try to delete store2/key-string with an error.+ try {+ await store2.setValue('key-string', null, { contentType: 'text/plain' });+ throw new Error('This should throw!!!');+ } catch (err) {+ expect(err).to.be.a('error');+ }++ // Try to delete store2/key-string again.+ expect(await store2.getValue('key-string')).to.be.eql('yyyy');++ // Check that it doesn't exist.+ await store2.setValue('key-string', null);++ expect(await store.getValue('key-obj')).to.be.eql({ foo: 'bar' });+ expect(await store.getValue('key-string')).to.be.eql('xxxx');+ expect(await store.getValue('key-buffer')).to.be.eql(buffer);+ expect(await store.getValue('key-nonexist')).to.be.eql(null);+ expect(await store2.getValue('key-obj')).to.be.eql({ foo: 'hotel' });+ });+ });++ describe('remote', async () => {+ it('works', async () => {+ const store = new KeyValueStore('some-id-1');+ const mock = sinon.mock(apifyClient.keyValueStores);+ const record = { foo: 'bar' };+ const recordStr = JSON.stringify(record, null, 2);++ // Set.+ mock.expects('putRecord')+ .once()+ .withArgs({+ storeId: 'some-id-1',+ key: 'key-1',+ body: recordStr,+ contentType: 'application/json; charset=utf-8',+ })+ .returns(Promise.resolve(null));+ await store.setValue('key-1', record);++ // Get.+ mock.expects('getRecord')+ .once()+ .withArgs({+ storeId: 'some-id-1',+ key: 'key-1',+ })+ .returns(Promise.resolve({ body: record, contentType: 'application/json; charset=utf-8' }));+ const response = await store.getValue('key-1');+ expect(response).to.be.eql(record);++ // Delete.+ mock.expects('deleteRecord')+ .once()+ .withArgs({+ storeId: 'some-id-1',+ key: 'key-1',+ })+ .returns(Promise.resolve(null));+ await store.setValue('key-1', null);++ mock.verify();+ mock.restore();+ });+ });++ describe('Apify.openKeyValueStore', async () => {+ it('should open a local store when process.env[ENV_VARS.LOCAL_EMULATION_DIR] is set', async () => {+ process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;++ const store = await Apify.openKeyValueStore('some-id-2');+ expect(store).to.be.instanceof(KeyValueStoreLocal);+ expect(store).not.to.be.instanceof(KeyValueStore);++ delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];+ });++ it('should reuse cached store instances', async () => {+ process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;++ const store1 = await Apify.openKeyValueStore('some-id-3');+ const store2 = await Apify.openKeyValueStore('some-id-3');+ const store3 = new KeyValueStoreLocal('some-id-3', LOCAL_EMULATION_DIR);++ expect(store1).to.be.instanceof(KeyValueStoreLocal);+ expect(store2).to.be.instanceof(KeyValueStoreLocal);+ expect(store3).to.be.instanceof(KeyValueStoreLocal);++ expect(store1).to.be.equal(store2);+ expect(store1).not.to.be.equal(store3);++ delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];+ });++ it('should open default store when storeIdOrName is not provided', async () => {+ process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = 'some-id-4';+ process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;++ const store = await Apify.openKeyValueStore();+ expect(store.storeId).to.be.eql('some-id-4');+ expect(store).to.be.instanceof(KeyValueStoreLocal);++ delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];+ process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = 'some-id-5';+ expectNotLocalEmulation();++ const store2 = await Apify.openKeyValueStore();+ expect(store2.storeId).to.be.eql('some-id-5');+ expect(store2).to.be.instanceof(KeyValueStore);++ delete process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];+ });++ it('should open remote store when process.env[ENV_VARS.LOCAL_EMULATION_DIR] is NOT set', async () => {+ expectNotLocalEmulation();++ const mock = sinon.mock(apifyClient.keyValueStores);++ // First when used with id it only requests store object.+ mock.expects('getStore')+ .once()+ .withArgs({ storeId: 'some-id-6' })+ .returns(Promise.resolve({ id: 'some-id-6' }));+ const store = await Apify.openKeyValueStore('some-id-6');+ expect(store.storeId).to.be.eql('some-id-6');+ expect(store).to.be.instanceof(KeyValueStore);++ // Then used with name it requests store object, gets empty response+ // so then it creates dataset.+ mock.expects('getStore')+ .once()+ .withArgs({ storeId: 'some-name-7' })+ .returns(Promise.resolve(null));+ mock.expects('getOrCreateStore')+ .once()+ .withArgs({ storeName: 'some-name-7' })+ .returns(Promise.resolve({ id: 'some-id-7' }));++ const store2 = await Apify.openKeyValueStore('some-name-7');+ expect(store2.storeId).to.be.eql('some-id-7');+ expect(store2).to.be.instanceof(KeyValueStore);++ mock.verify();+ mock.restore();+ });+ });++ describe('getValue', async () => {+ it('throws on invalid args', async () => {+ process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = '1234';+ await expect(Apify.getValue()).to.be.rejectedWith('Parameter "key" of type String must be provided');+ await expect(Apify.getValue({})).to.be.rejectedWith('Parameter "key" of type String must be provided');+ await expect(Apify.getValue('')).to.be.rejectedWith('The "key" parameter cannot be empty');+ await expect(Apify.getValue(null)).to.be.rejectedWith('Parameter "key" of type String must be provided');+ });++ it('throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined', async () => {+ const errMsg = 'The \'APIFY_DEFAULT_KEY_VALUE_STORE_ID\' environment variable is not defined';++ process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = '';+ await expect(Apify.getValue('KEY')).to.be.rejectedWith(errMsg);++ delete process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];+ await expect(Apify.getValue('some other key')).to.be.rejectedWith(errMsg);+ });+ });++ describe('setValue', async () => {+ it('throws on invalid args', async () => {+ process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = '1234';+ await expect(Apify.setValue()).to.be.rejectedWith('Parameter "key" of type String must be provided');+ await expect(Apify.setValue('', null)).to.be.rejectedWith('The "key" parameter cannot be empty');+ await expect(Apify.setValue('', 'some value')).to.be.rejectedWith('The "key" parameter cannot be empty');+ await expect(Apify.setValue({}, 'some value')).to.be.rejectedWith('Parameter "key" of type String must be provided');+ await expect(Apify.setValue(123, 'some value')).to.be.rejectedWith('Parameter "key" of type String must be provided');++ const valueErrMsg = 'The "value" parameter must be a String or Buffer when "options.contentType" is specified';+ await expect(Apify.setValue('key', {}, { contentType: 'image/png' })).to.be.rejectedWith(valueErrMsg);+ await expect(Apify.setValue('key', 12345, { contentType: 'image/png' })).to.be.rejectedWith(valueErrMsg);+ await expect(Apify.setValue('key', () => {}, { contentType: 'image/png' })).to.be.rejectedWith(valueErrMsg);++ const optsErrMsg = 'Parameter "options" of type Object must be provided';+ await expect(Apify.setValue('key', {}, 123)).to.be.rejectedWith(optsErrMsg);+ await expect(Apify.setValue('key', {}, 'bla/bla')).to.be.rejectedWith(optsErrMsg);+ await expect(Apify.setValue('key', {}, true)).to.be.rejectedWith(optsErrMsg);++ const circularObj = {};+ circularObj.xxx = circularObj;+ const jsonErrMsg = 'The "value" parameter cannot be stringified to JSON';+ await expect(Apify.setValue('key', circularObj)).to.be.rejectedWith(jsonErrMsg);+ await expect(Apify.setValue('key', undefined)).to.be.rejectedWith(jsonErrMsg);+ await expect(Apify.setValue('key', () => {})).to.be.rejectedWith(jsonErrMsg);+ await expect(Apify.setValue('key')).to.be.rejectedWith(jsonErrMsg);++ const contTypeRedundantErrMsg = 'The "options.contentType" parameter must not be used when removing the record';+ await expect(Apify.setValue('key', null, { contentType: 'image/png' })).to.be.rejectedWith(contTypeRedundantErrMsg);+ await expect(Apify.setValue('key', null, { contentType: '' })).to.be.rejectedWith(contTypeRedundantErrMsg);+ await expect(Apify.setValue('key', null, { contentType: {} }))+ .to.be.rejectedWith('Parameter "options.contentType" of type String | Null | Undefined must be provided');++ const contTypeStringErrMsg = 'Parameter "options.contentType" of type String | Null | Undefined must be provided';+ await expect(Apify.setValue('key', 'value', { contentType: 123 })).to.be.rejectedWith(contTypeStringErrMsg);+ await expect(Apify.setValue('key', 'value', { contentType: {} })).to.be.rejectedWith(contTypeStringErrMsg);+ await expect(Apify.setValue('key', 'value', { contentType: new Date() })).to.be.rejectedWith(contTypeStringErrMsg);+ await expect(Apify.setValue('key', 'value', { contentType: '' }))+ .to.be.rejectedWith('Parameter options.contentType cannot be empty string.');+ });++ it('throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined', async () => {+ const errMsg = 'The \'APIFY_DEFAULT_KEY_VALUE_STORE_ID\' environment variable is not defined';++ process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = '';+ await expect(Apify.setValue('KEY', {})).to.be.rejectedWith(errMsg);++ delete process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];+ await expect(Apify.setValue('some other key', {})).to.be.rejectedWith(errMsg);+ });++ it('correctly adds charset to content type', async () => {+ const store = new KeyValueStore('some-id-1');+ const mock = sinon.mock(apifyClient.keyValueStores);++ mock.expects('putRecord')+ .once()+ .withArgs({+ storeId: 'some-id-1',+ key: 'key-1',+ body: 'xxxx',+ contentType: 'text/plain; charset=utf-8',+ })+ .returns(Promise.resolve(null));+ await store.setValue('key-1', 'xxxx', { contentType: 'text/plain' });+ mock.verify();+ mock.restore();+ });++ it('correctly passes object values as JSON', async () => {+ const store = new KeyValueStore('some-id-1');+ const mock = sinon.mock(apifyClient.keyValueStores);+ const record = { foo: 'bar' };+ const recordStr = JSON.stringify(record, null, 2);++ mock.expects('putRecord')+ .once()+ .withArgs({+ storeId: 'some-id-1',+ key: 'key-1',+ body: recordStr,+ contentType: 'application/json; charset=utf-8',+ })+ .returns(Promise.resolve(null));+ await store.setValue('key-1', record);+ mock.verify();+ mock.restore();+ });++ it('correctly passes raw string values', async () => {+ const store = new KeyValueStore('some-id-1');+ const mock = sinon.mock(apifyClient.keyValueStores);++ mock.expects('putRecord')+ .once()+ .withArgs({+ storeId: 'some-id-1',+ key: 'key-1',+ body: 'xxxx',+ contentType: 'text/plain; charset=utf-8',+ })+ .returns(Promise.resolve(null));+ await store.setValue('key-1', 'xxxx', { contentType: 'text/plain; charset=utf-8' });+ mock.verify();+ mock.restore();+ });++ it('correctly passes raw Buffer values', async () => {+ const store = new KeyValueStore('some-id-1');+ const mock = sinon.mock(apifyClient.keyValueStores);+ const value = Buffer.from('some text value');++ mock.expects('putRecord')+ .once()+ .withArgs({+ storeId: 'some-id-1',+ key: 'key-1',+ body: value,+ contentType: 'image/jpeg; charset=something',+ })+ .returns(Promise.resolve(null));+ await store.setValue('key-1', value, { contentType: 'image/jpeg; charset=something' });+ mock.verify();+ mock.restore();+ });+ });+});diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.jsnew file mode 100644index 000000000..9c389450c--- /dev/null+++ b/test/puppeteer_crawler.js@@ -0,0 +1,92 @@+import chai, { expect } from 'chai';+import chaiAsPromised from 'chai-as-promised';+import 'babel-polyfill';+import { delayPromise } from 'apify-shared/utilities';+import { ENV_VARS } from '../build/constants';+import * as Apify from '../build/index';++chai.use(chaiAsPromised);++describe('puppeteer_crawler', () => {+ let prevEnvHeadless;++ before(() => {+ prevEnvHeadless = process.env[ENV_VARS.HEADLESS];+ process.env[ENV_VARS.HEADLESS] = '1';+ });++ after(() => {+ process.env[ENV_VARS.HEADLESS] = prevEnvHeadless;+ });++ it('should work', async () => {+ const sources = [+ { url: 'http://example.com/?q=1' },+ { url: 'http://example.com/?q=2' },+ { url: 'http://example.com/?q=3' },+ { url: 'http://example.com/?q=4' },+ { url: 'http://example.com/?q=5' },+ { url: 'http://example.com/?q=6' },+ ];+ const processed = [];+ const failed = [];+ const requestList = new Apify.RequestList({ sources });+ const handlePageFunction = async ({ page, request }) => {+ await page.waitFor('title');++ request.userData.title = await page.title();++ processed.push(request);+ };++ const puppeteerCrawler = new Apify.PuppeteerCrawler({+ requestList,+ minConcurrency: 1,+ maxConcurrency: 1,+ handlePageFunction,+ handleFailedRequestFunction: ({ request }) => failed.push(request),+ disableProxy: true,+ });++ await requestList.initialize();+ await puppeteerCrawler.run();++ expect(processed).to.have.lengthOf(6);+ expect(failed).to.have.lengthOf(0);++ processed.forEach((request, id) => {+ expect(request.url).to.be.eql(sources[id].url);+ expect(request.userData.title).to.be.eql('Example Domain');+ });+ });++ it('should fail when pageOpsTimeoutMillis gets exceeded', async () => {+ const sources = [+ { url: 'http://example.com/?q=1' },+ ];+ const failed = [];+ const requestList = new Apify.RequestList({ sources });+ const handlePageFunction = async () => {+ await delayPromise(1000);+ };++ const puppeteerCrawler = new Apify.PuppeteerCrawler({+ requestList,+ handlePageFunction,+ handleFailedRequestFunction: ({ request }) => failed.push(request),+ disableProxy: true,+ pageOpsTimeoutMillis: 900,+ });++ await requestList.initialize();+ await puppeteerCrawler.run();++ expect(failed).to.have.lengthOf(1);+ expect(failed[0].retryCount).to.be.eql(3);+ expect(failed[0].errorMessages).to.have.lengthOf(4);+ failed[0].errorMessages.forEach((error) => {+ expect(error).to.be.a('string');+ expect(error).to.include('timed out');+ });+ });+});diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.jsnew file mode 100644index 000000000..b1f4b43c1--- /dev/null+++ b/test/puppeteer_pool.js@@ -0,0 +1,151 @@+import chai, { expect } from 'chai';+import chaiAsPromised from 'chai-as-promised';+import _ from 'underscore';+import 'babel-polyfill';+import { ENV_VARS } from '../build/constants';+import * as Apify from '../build/index';++chai.use(chaiAsPromised);++const shortSleep = (millis = 25) => new Promise(resolve => setTimeout(resolve, millis));++describe('puppeteer_pool', () => {+ let prevEnvHeadless;++ before(() => {+ prevEnvHeadless = process.env[ENV_VARS.HEADLESS];+ process.env[ENV_VARS.HEADLESS] = '1';+ });++ after(() => {+ process.env[ENV_VARS.HEADLESS] = prevEnvHeadless;+ });++ it('should work', async () => {+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';+ process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';+ process.env[ENV_VARS.PROXY_PORT] = 123;++ const pool = new Apify.PuppeteerPool({+ maxOpenPagesPerInstance: 3,+ abortInstanceAfterRequestCount: 5,+ });+ const browsers = [];++ // Open 6 pages 3 in both browsers.+ browsers.push(pool.newPage());+ browsers.push(pool.newPage());+ browsers.push(pool.newPage());+ browsers.push(pool.newPage());+ browsers.push(pool.newPage());+ browsers.push(pool.newPage());+ await Promise.all(browsers);++ expect(_.values(pool.activeInstances).length).to.be.eql(2);+ expect(pool.activeInstances[0].activePages).to.be.eql(3);+ expect(pool.activeInstances[0].totalPages).to.be.eql(3);+ expect(pool.activeInstances[1].activePages).to.be.eql(3);+ expect(pool.activeInstances[1].totalPages).to.be.eql(3);++ // Close 2 pages in a first browser.+ await (await browsers[0]).close();+ await (await browsers[1]).close();+ await shortSleep();++ expect(_.values(pool.activeInstances).length).to.be.eql(2);+ expect(pool.activeInstances[0].activePages).to.be.eql(1);+ expect(pool.activeInstances[0].totalPages).to.be.eql(3);+ expect(pool.activeInstances[1].activePages).to.be.eql(3);+ expect(pool.activeInstances[1].totalPages).to.be.eql(3);++ // Open two more pages in first browser so it reaches 5 and gets retired.+ browsers.push(pool.newPage());+ browsers.push(pool.newPage());+ await Promise.all(browsers);+ expect(_.values(pool.activeInstances).length).to.be.eql(1);+ expect(_.values(pool.retiredInstances).length).to.be.eql(1);+ expect(pool.activeInstances[1].activePages).to.be.eql(3);+ expect(pool.activeInstances[1].totalPages).to.be.eql(3);+ expect(pool.retiredInstances[0].activePages).to.be.eql(3);+ expect(pool.retiredInstances[0].totalPages).to.be.eql(5);++ // Open one more page to see that 3rd browser gets started.+ browsers.push(pool.newPage());+ await Promise.all(browsers);++ expect(_.values(pool.activeInstances).length).to.be.eql(2);+ expect(_.values(pool.retiredInstances).length).to.be.eql(1);+ expect(pool.activeInstances[1].activePages).to.be.eql(3);+ expect(pool.activeInstances[1].totalPages).to.be.eql(3);+ expect(pool.activeInstances[2].activePages).to.be.eql(1);+ expect(pool.activeInstances[2].totalPages).to.be.eql(1);+ expect(pool.retiredInstances[0].activePages).to.be.eql(3);+ expect(pool.retiredInstances[0].totalPages).to.be.eql(5);++ // Kill the remaning 3 pages from the 1st browser to see that it gets closed.+ await (await browsers[2]).close();+ await (await browsers[6]).close();++ // @TODO for some reason it fails here:+ // await (await browsers[7]).close();+ // await shortSleep();+ // expect(_.values(pool.retiredInstances).length).to.be.eql(0);++ // Cleanup everything.+ await pool.destroy();+ delete process.env[ENV_VARS.PROXY_PASSWORD];+ delete process.env[ENV_VARS.PROXY_HOSTNAME];+ delete process.env[ENV_VARS.PROXY_PORT];+ });++ it('kills hanging retired instances', async () => {+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';+ process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';+ process.env[ENV_VARS.PROXY_PORT] = 123;++ const pool = new Apify.PuppeteerPool({+ maxOpenPagesPerInstance: 3,+ abortInstanceAfterRequestCount: 5,+ instanceKillerIntervalMillis: 1000,+ killInstanceAfterMillis: 500,+ });+ const browsers = [];++ // Open 3 pages.+ browsers.push(pool.newPage());+ browsers.push(pool.newPage());+ browsers.push(pool.newPage());+ await Promise.all(browsers);+ expect(_.values(pool.activeInstances).length).to.be.eql(1);+ expect(_.values(pool.retiredInstances).length).to.be.eql(0);++ // Close 2.+ await (await browsers[0]).close();+ await (await browsers[1]).close();+ await shortSleep();+ expect(_.values(pool.activeInstances).length).to.be.eql(1);+ expect(_.values(pool.retiredInstances).length).to.be.eql(0);++ // Open 2 more so that the browser gets retired.+ browsers.push(pool.newPage());+ browsers.push(pool.newPage());+ await Promise.all(browsers);++ // Check that it's retired.+ expect(_.values(pool.activeInstances).length).to.be.eql(0);+ expect(_.values(pool.retiredInstances).length).to.be.eql(1);+ expect(pool.retiredInstances[0].activePages).to.be.eql(3);+ expect(pool.retiredInstances[0].totalPages).to.be.eql(5);++ // Sleep and check that it has been killed.+ await shortSleep(2100);+ expect(_.values(pool.activeInstances).length).to.be.eql(0);+ expect(_.values(pool.retiredInstances).length).to.be.eql(0);++ // Cleanup everything.+ await pool.destroy();+ delete process.env[ENV_VARS.PROXY_PASSWORD];+ delete process.env[ENV_VARS.PROXY_HOSTNAME];+ delete process.env[ENV_VARS.PROXY_PORT];+ });+});diff --git a/test/request.js b/test/request.jsnew file mode 100644index 000000000..68d35d336--- /dev/null+++ b/test/request.js@@ -0,0 +1,35 @@+import { expect } from 'chai';+import { computeUniqueKey } from '../build/request';+import Apify from '../build/index';++describe('Apify.Request', () => {+ it('should not accept invalid values', () => {+ expect(() => new Apify.Request({ url: 1 })).to.throw();+ expect(() => new Apify.Request({ url: 'xxx' })).to.not.throw();+ });++ it('should create unique based on url', () => {+ const url = 'https://user:[email protected]/a/vb/c /d?q=1&q=kjnjkn$lkn#lkmlkml';+ const normalizedUrl = computeUniqueKey(url);+ const request = new Apify.Request({ url });++ expect(request.uniqueKey).to.be.eql(normalizedUrl);+ expect(normalizedUrl).to.not.eql(url);+ });++ it('should should allow to push error messages', () => {+ const request = new Apify.Request({ url: 'http://example.com' });++ expect(request.errorMessages).to.be.eql(null);++ request.pushErrorMessage(new Error('foo'));+ request.pushErrorMessage('bar');++ expect(request.errorMessages).to.be.eql(['foo', 'bar']);+ });++ it('should should allow to have a GET request with payload', () => {+ expect(() => new Apify.Request({ url: 'http://example.com', payload: 'foo' })).to.throw();+ expect(() => new Apify.Request({ url: 'http://example.com', payload: 'foo', method: 'POST' })).to.not.throw();+ });+});diff --git a/test/request_list.js b/test/request_list.jsnew file mode 100644index 000000000..0919e3c5b--- /dev/null+++ b/test/request_list.js@@ -0,0 +1,357 @@+import 'babel-polyfill';+import { expect } from 'chai';+import request from 'request-promise';+import sinon from 'sinon';+import Apify from '../build/index';++describe('Apify.RequestList', () => {+ it('should not accept to pages with same uniqueKey', async () => {+ const requestList = new Apify.RequestList({+ sources: [+ { url: 'https://example.com/1' },+ { url: 'https://example.com/1#same' },+ ],+ });++ await requestList.initialize();++ expect(requestList.isEmpty()).to.be.eql(false);++ const req = requestList.fetchNextRequest();++ expect(req.url).to.be.eql('https://example.com/1');+ expect(requestList.isEmpty()).to.be.eql(true);+ expect(requestList.isFinished()).to.be.eql(false);+ expect(requestList.fetchNextRequest()).to.be.eql(null);++ requestList.markRequestHandled(req);++ expect(requestList.isEmpty()).to.be.eql(true);+ expect(requestList.isFinished()).to.be.eql(true);+ });++ it('must be initialized before using any of the methods', async () => {+ const requestList = new Apify.RequestList({ sources: [{ url: 'https://example.com' }] });+ const requestObj = new Apify.Request({ url: 'https://example.com' });++ expect(() => requestList.isEmpty()).to.throw();+ expect(() => requestList.isFinished()).to.throw();+ expect(() => requestList.getState()).to.throw();+ expect(() => requestList.markRequestHandled(requestObj)).to.throw();+ expect(() => requestList.reclaimRequest(requestObj)).to.throw();+ expect(() => requestList.fetchNextRequest()).to.throw();++ await requestList.initialize();++ expect(() => requestList.isEmpty()).to.not.throw();+ expect(() => requestList.isFinished()).to.not.throw();+ expect(() => requestList.getState()).to.not.throw();+ expect(() => requestList.fetchNextRequest()).to.not.throw();+ expect(() => requestList.reclaimRequest(requestObj)).to.not.throw();+ expect(() => requestList.fetchNextRequest()).to.not.throw();+ expect(() => requestList.markRequestHandled(requestObj)).to.not.throw();+ });++ it('should correctly initialized itself', async () => {+ const sources = [+ { url: 'https://example.com/1' },+ { url: 'https://example.com/2' },+ { url: 'https://example.com/3' },+ { url: 'https://example.com/4' },+ { url: 'https://example.com/5' },+ { url: 'https://example.com/6' },+ { url: 'https://example.com/7' },+ { url: 'https://example.com/8' },+ ];++ const originalList = new Apify.RequestList({ sources });+ await originalList.initialize();++ const r1 = originalList.fetchNextRequest(); // 1+ const r2 = originalList.fetchNextRequest(); // 2+ originalList.fetchNextRequest(); // 3+ const r4 = originalList.fetchNextRequest(); // 4+ const r5 = originalList.fetchNextRequest(); // 5+ originalList.fetchNextRequest(); // 6++ originalList.markRequestHandled(r1);+ originalList.markRequestHandled(r2);+ originalList.markRequestHandled(r4);+ originalList.reclaimRequest(r5);++ const newList = new Apify.RequestList({+ sources,+ state: originalList.getState(),+ });+ await newList.initialize();++ expect(newList.isEmpty()).to.be.eql(false);+ expect(newList.fetchNextRequest().url).to.be.eql('https://example.com/3');+ expect(newList.fetchNextRequest().url).to.be.eql('https://example.com/5');+ expect(newList.fetchNextRequest().url).to.be.eql('https://example.com/6');+ expect(newList.fetchNextRequest().url).to.be.eql('https://example.com/7');+ expect(newList.fetchNextRequest().url).to.be.eql('https://example.com/8');+ expect(newList.isEmpty()).to.be.eql(true);+ });++ it('should correctly load list from hosted files in correct order', async () => {+ const mock = sinon.mock(request);+ const list1 = [+ 'https://example.com',+ 'https://google.com',+ 'https://wired.com',+ ];+ const list2 = [+ 'https://another.com',+ 'https://page.com',+ ];++ mock.expects('get')+ .once()+ .withArgs('http://example.com/list-1')+ .returns(new Promise(resolve => setTimeout(resolve(list1.join('\n')), 100)));++ mock.expects('get')+ .once()+ .withArgs('http://example.com/list-2')+ .returns(Promise.resolve(list2.join('\n')), 0);++ const requestList = new Apify.RequestList({+ sources: [+ { method: 'GET', requestsFromUrl: 'http://example.com/list-1' },+ { method: 'POST', requestsFromUrl: 'http://example.com/list-2' },+ ],+ });++ await requestList.initialize();++ expect(requestList.fetchNextRequest()).to.include({ method: 'GET', url: list1[0] });+ expect(requestList.fetchNextRequest()).to.include({ method: 'GET', url: list1[1] });+ expect(requestList.fetchNextRequest()).to.include({ method: 'GET', url: list1[2] });+ expect(requestList.fetchNextRequest()).to.include({ method: 'POST', url: list2[0] });+ expect(requestList.fetchNextRequest()).to.include({ method: 'POST', url: list2[1] });++ mock.verify();+ mock.restore();+ });++ it('should use regex parameter to parse urls', async () => {+ const mock = sinon.mock(request);+ const listStr = 'kjnjkn"https://example.com/a/b/c?q=1#abc";,"http://google.com/a/b/c";dgg:dd';+ const listArr = ['https://example.com/a/b/c?q=1#abc', 'http://google.com/a/b/c'];++ mock.expects('get')+ .once()+ .withArgs('http://example.com/list-1')+ .returns(Promise.resolve(listStr));++ const requestList = new Apify.RequestList({+ sources: [+ {+ method: 'GET',+ requestsFromUrl: 'http://example.com/list-1',+ },+ ],+ });++ await requestList.initialize();++ expect(requestList.fetchNextRequest()).to.include({ method: 'GET', url: listArr[0] });+ expect(requestList.fetchNextRequest()).to.include({ method: 'GET', url: listArr[1] });++ mock.verify();+ mock.restore();+ });++ it('should correctly handle reclaimed pages', async () => {+ const requestList = new Apify.RequestList({+ sources: [+ { url: 'https://example.com/1' },+ { url: 'https://example.com/2' },+ { url: 'https://example.com/3' },+ { url: 'https://example.com/4' },+ { url: 'https://example.com/5' },+ { url: 'https://example.com/6' },+ ],+ });++ await requestList.initialize();++ //+ // Fetch first 5 urls+ //++ const request1 = requestList.fetchNextRequest();+ const request2 = requestList.fetchNextRequest();+ const request3 = requestList.fetchNextRequest();+ const request4 = requestList.fetchNextRequest();+ const request5 = requestList.fetchNextRequest();++ expect(request1.url).to.be.eql('https://example.com/1');+ expect(request2.url).to.be.eql('https://example.com/2');+ expect(request3.url).to.be.eql('https://example.com/3');+ expect(request4.url).to.be.eql('https://example.com/4');+ expect(request5.url).to.be.eql('https://example.com/5');+ expect(requestList.getState()).to.be.eql({+ inProgress: {+ 'https://example.com/1': true,+ 'https://example.com/2': true,+ 'https://example.com/3': true,+ 'https://example.com/4': true,+ 'https://example.com/5': true,+ },+ nextIndex: 5,+ nextUniqueKey: 'https://example.com/6',+ });+ expect(requestList.isEmpty()).to.be.eql(false);+ expect(requestList.isFinished()).to.be.eql(false);+ expect(requestList.inProgress).to.include(requestList.reclaimed);++ //+ // Mark 1st, 2nd handled+ // Reclaim 3rd 4th+ //++ requestList.markRequestHandled(request1);+ requestList.markRequestHandled(request2);+ requestList.reclaimRequest(request3);+ requestList.reclaimRequest(request4);++ expect(requestList.getState()).to.be.eql({+ inProgress: {+ 'https://example.com/3': true,+ 'https://example.com/4': true,+ 'https://example.com/5': true,+ },+ nextIndex: 5,+ nextUniqueKey: 'https://example.com/6',+ });+ expect(requestList.isEmpty()).to.be.eql(false);+ expect(requestList.isFinished()).to.be.eql(false);+ expect(requestList.inProgress).to.include(requestList.reclaimed);++ //+ // Mark 5th handled+ //++ requestList.markRequestHandled(request5);++ expect(requestList.getState()).to.be.eql({+ inProgress: {+ 'https://example.com/3': true,+ 'https://example.com/4': true,+ },+ nextIndex: 5,+ nextUniqueKey: 'https://example.com/6',+ });+ expect(requestList.isEmpty()).to.be.eql(false);+ expect(requestList.isFinished()).to.be.eql(false);+ expect(requestList.inProgress).to.include(requestList.reclaimed);++ //+ // Fetch 3rd and 4th+ // Mark 4th handled+ //++ const reclaimed3 = requestList.fetchNextRequest();+ expect(reclaimed3.url).to.be.eql('https://example.com/3');+ const reclaimed4 = requestList.fetchNextRequest();+ expect(reclaimed4.url).to.be.eql('https://example.com/4');+ requestList.markRequestHandled(request4);++ expect(requestList.getState()).to.be.eql({+ inProgress: {+ 'https://example.com/3': true,+ },+ nextIndex: 5,+ nextUniqueKey: 'https://example.com/6',+ });+ expect(requestList.isEmpty()).to.be.eql(false);+ expect(requestList.isFinished()).to.be.eql(false);+ expect(requestList.inProgress).to.include(requestList.reclaimed);++ //+ // Mark 3rd handled+ //++ requestList.markRequestHandled(request3);++ expect(requestList.getState()).to.be.eql({+ inProgress: {},+ nextIndex: 5,+ nextUniqueKey: 'https://example.com/6',+ });+ expect(requestList.isEmpty()).to.be.eql(false);+ expect(requestList.isFinished()).to.be.eql(false);+ expect(requestList.inProgress).to.include(requestList.reclaimed);++ //+ // Fetch 6th+ //++ const request6 = requestList.fetchNextRequest();++ expect(request6.url).to.be.eql('https://example.com/6');+ expect(requestList.fetchNextRequest()).to.be.eql(null);+ expect(requestList.getState()).to.be.eql({+ inProgress: {+ 'https://example.com/6': true,+ },+ nextIndex: 6,+ nextUniqueKey: null,+ });+ expect(requestList.isEmpty()).to.be.eql(true);+ expect(requestList.isFinished()).to.be.eql(false);+ expect(requestList.inProgress).to.include(requestList.reclaimed);++ //+ // Reclaim 6th+ //++ requestList.reclaimRequest(request6);++ expect(requestList.getState()).to.be.eql({+ inProgress: {+ 'https://example.com/6': true,+ },+ nextIndex: 6,+ nextUniqueKey: null,+ });+ expect(requestList.isEmpty()).to.be.eql(false);+ expect(requestList.isFinished()).to.be.eql(false);+ expect(requestList.inProgress).to.include(requestList.reclaimed);++ //+ // Fetch 6th+ //++ const reclaimed6 = requestList.fetchNextRequest();++ expect(reclaimed6.url).to.be.eql('https://example.com/6');+ expect(requestList.getState()).to.be.eql({+ inProgress: {+ 'https://example.com/6': true,+ },+ nextIndex: 6,+ nextUniqueKey: null,+ });+ expect(requestList.isEmpty()).to.be.eql(true);+ expect(requestList.isFinished()).to.be.eql(false);+ expect(requestList.inProgress).to.include(requestList.reclaimed);++ //+ // Mark 6th handled+ //++ requestList.markRequestHandled(reclaimed6);++ expect(requestList.getState()).to.be.eql({+ inProgress: {},+ nextIndex: 6,+ nextUniqueKey: null,+ });+ expect(requestList.isEmpty()).to.be.eql(true);+ expect(requestList.isFinished()).to.be.eql(true);+ expect(requestList.inProgress).to.include(requestList.reclaimed);+ });+});diff --git a/test/request_queue.js b/test/request_queue.jsnew file mode 100644index 000000000..607da3e7a--- /dev/null+++ b/test/request_queue.js@@ -0,0 +1,85 @@+import chai, { expect } from 'chai';+import chaiAsPromised from 'chai-as-promised';+import 'babel-polyfill';+import * as Apify from '../build/index';+import { apifyClient } from '../build/utils';+import { RequestQueueLocal, LOCAL_EMULATION_SUBDIR } from '../build/request_queue';+import { emptyLocalEmulationSubdir, LOCAL_EMULATION_DIR } from './_helper';++chai.use(chaiAsPromised);++describe('request_queue', () => {+ before(() => apifyClient.setOptions({ token: 'xxx' }));+ after(() => apifyClient.setOptions({ token: undefined }));+ beforeEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));+ afterEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));++ describe('local', async () => {+ it('should work', async () => {+ const queue = new RequestQueueLocal('my-queue', LOCAL_EMULATION_DIR);++ const request1 = new Apify.Request({ url: 'http://example.com/first' });+ const request2 = new Apify.Request({ url: 'http://example.com/middle' });+ const request3 = new Apify.Request({ url: 'http://example.com/last-but-first' });++ await queue.addRequest(request1);+ await queue.addRequest(request2);+ await queue.addRequest(request3, { putInFront: true });++ expect(await queue.getRequest(request1.uniqueKey)).to.be.eql(request1);+ expect(await queue.getRequest(request2.uniqueKey)).to.be.eql(request2);+ expect(await queue.getRequest(request3.uniqueKey)).to.be.eql(request3);+ expect(await queue.getRequest(request1.uniqueKey)).to.not.be.eql(request3);++ expect(await queue.fetchNextRequest()).to.be.eql(request3);+ expect(await queue.fetchNextRequest()).to.be.eql(request1);+ expect(await queue.fetchNextRequest()).to.be.eql(request2);+ expect(await queue.fetchNextRequest()).to.be.eql(null);++ expect(await queue.isEmpty()).to.be.eql(true);+ expect(await queue.isFinished()).to.be.eql(false);++ await queue.markRequestHandled(request3);+ await queue.reclaimRequest(request1);+ await queue.reclaimRequest(request2);+ expect(await queue.isEmpty()).to.be.eql(false);++ expect(await queue.fetchNextRequest()).to.be.eql(request1);+ expect(await queue.fetchNextRequest()).to.be.eql(request2);+ expect(await queue.fetchNextRequest()).to.be.eql(null);+ await queue.markRequestHandled(request1);+ await queue.markRequestHandled(request2);++ expect(await queue.isEmpty()).to.be.eql(true);+ expect(await queue.isFinished()).to.be.eql(true);+ });++ it('should get initialized from existing dir', async () => {+ const queue = new RequestQueueLocal('my-queue-2', LOCAL_EMULATION_DIR);++ const request1 = new Apify.Request({ url: 'http://example.com/first' });+ const request2 = new Apify.Request({ url: 'http://example.com/middle' });+ const request3 = new Apify.Request({ url: 'http://example.com/last-but-first' });++ await queue.addRequest(request1);+ await queue.addRequest(request2);+ await queue.addRequest(request3, { putInFront: true });++ expect(await queue.fetchNextRequest()).to.be.eql(request3);+ expect(await queue.fetchNextRequest()).to.be.eql(request1);+ await queue.markRequestHandled(request1);++ const anotherQueue = new RequestQueueLocal('my-queue-2', LOCAL_EMULATION_DIR);+ expect(await anotherQueue.isEmpty()).to.be.eql(false);+ expect(await anotherQueue.isFinished()).to.be.eql(false);+ expect(await anotherQueue.fetchNextRequest()).to.be.eql(request3);+ expect(await anotherQueue.fetchNextRequest()).to.be.eql(request2);+ expect(await anotherQueue.isEmpty()).to.be.eql(true);+ expect(await anotherQueue.isFinished()).to.be.eql(false);+ await anotherQueue.markRequestHandled(request2);+ await anotherQueue.markRequestHandled(request3);+ expect(await anotherQueue.isEmpty()).to.be.eql(true);+ expect(await anotherQueue.isFinished()).to.be.eql(true);+ });+ });+});diff --git a/test/settings_rotator.js b/test/settings_rotator.jsnew file mode 100644index 000000000..5e69d6c59--- /dev/null+++ b/test/settings_rotator.js@@ -0,0 +1,20 @@+import 'babel-polyfill';+import _ from 'underscore';+import { expect } from 'chai';+import Apify from '../build/index';++describe('Apify.SettingsRotator', () => {+ it('works', async () => {+ const maxUsages = 12;+ const totalCalls = 167;++ const rotator = new Apify.SettingsRotator({+ newSettingsFunction: () => Math.random(),+ maxUsages,+ });++ const settings = _.range(0, totalCalls).map(() => rotator.fetchSettings());++ expect(_.uniq(settings).length).to.be.eql(Math.ceil(totalCalls / maxUsages));+ });+});diff --git a/test/utils.js b/test/utils.jsindex aad6e435c..2bf2df268 100644--- a/test/utils.js+++ b/test/utils.js@@ -1,40 +1,12 @@-import BluebirdPromise from 'bluebird';import { expect } from 'chai';import sinon from 'sinon';import fs from 'fs';+import os from 'os';import * as utils from '../build/utils';import Apify from '../build/index';/* global process, describe, it */-// TODO: run tests against build scripts too!--describe('Apify.xxxPromisesDependency()', () => {- it('should throw on invalid args', () => {- expect(() => { Apify.setPromisesDependency('test'); }).to.throw(Error);- expect(() => { Apify.setPromisesDependency({}); }).to.throw(Error);- expect(() => { Apify.setPromisesDependency(123); }).to.throw(Error);- expect(() => { Apify.setPromisesDependency(); }).to.throw(Error);- expect(() => { Apify.setPromisesDependency(undefined); }).to.throw(Error);- });-- it('should work as expected', () => {- Apify.setPromisesDependency(null);- expect(Apify.getPromisesDependency()).to.be.a('null');- expect(utils.newPromise()).to.have.property('then');-- // Check native promise- Apify.setPromisesDependency(Promise);- expect(Apify.getPromisesDependency()).to.equal(Promise);- expect(utils.newPromise()).to.have.property('then');-- // Check bluebird- Apify.setPromisesDependency(BluebirdPromise);- expect(Apify.getPromisesDependency()).to.equal(BluebirdPromise);- expect(utils.newPromise()).to.have.property('then');- });-});-describe('utils.newClient()', () => {it('reads environment variables correctly', () => {process.env.APIFY_API_BASE_URL = 'http://www.example.com:1234/path/';@@ -128,3 +100,99 @@ describe('utils.isDocker()', () => {});});});++describe('utils.getMemoryInfo()', () => {+ it('works outside the container', () => {+ const osMock = sinon.mock(os);+ const utilsMock = sinon.mock(utils);++ utilsMock+ .expects('isDocker')+ .once()+ .returns(Promise.resolve(false));++ osMock+ .expects('freemem')+ .once()+ .returns(222);++ osMock+ .expects('totalmem')+ .once()+ .returns(333);++ return Apify+ .getMemoryInfo()+ .then((data) => {+ expect(data).to.be.eql({+ totalBytes: 333,+ freeBytes: 222,+ usedBytes: 111,+ });++ utilsMock.restore();+ osMock.restore();+ });+ });++ it('works inside the container', () => {+ const utilsMock = sinon.mock(utils);++ utilsMock+ .expects('isDocker')+ .once()+ .returns(Promise.resolve(true));++ sinon+ .stub(fs, 'readFile')+ .callsFake((path, callback) => {+ if (path === '/sys/fs/cgroup/memory/memory.limit_in_bytes') callback(null, '333');+ else if (path === '/sys/fs/cgroup/memory/memory.usage_in_bytes') callback(null, '111');+ else throw new Error('Invalid path');+ });++ return Apify+ .getMemoryInfo()+ .then((data) => {+ expect(data).to.be.eql({+ totalBytes: 333,+ freeBytes: 222,+ usedBytes: 111,+ });++ utilsMock.restore();+ fs.readFile.restore();+ });+ });+});++describe('utils.isPromise()', () => {+ it('works', () => {+ const rejected = Promise.reject();++ expect(utils.isPromise(new Promise(resolve => setTimeout(resolve, 1000)))).to.be.eql(true);+ expect(utils.isPromise(Promise.resolve())).to.be.eql(true);+ expect(utils.isPromise(rejected)).to.be.eql(true);+ expect(utils.isPromise(new Date())).to.be.eql(false);+ expect(utils.isPromise(Function)).to.be.eql(false);+ expect(utils.isPromise(() => {})).to.be.eql(false);+ expect(utils.isPromise({ then: () => {} })).to.be.eql(false);++ rejected.catch(() => {});+ });+});++describe('utils.isPromise()', () => {+ it('works', () => {+ expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', Date, 'Date')).to.not.throw();+ expect(() => utils.checkParamPrototypeOrThrow(null, 'param', Function, 'Date', true)).to.not.throw();+ expect(() => utils.checkParamPrototypeOrThrow(undefined, 'param', Function, 'Date', true)).to.not.throw();+ expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', Function, 'Date')).to.throw();+ });+});++describe('utils.newPromise()', () => {+ it('works', () => {+ if (!utils.isPromise(utils.newPromise())) throw new Error('utils.newPromise() must return a promise!');+ });+});diff --git a/test/webdriver.js b/test/webdriver.jsindex 8000faf69..952c89b1e 100644--- a/test/webdriver.js+++ b/test/webdriver.js@@ -81,35 +81,10 @@ describe('getDefaultBrowseOptions()', () => {describe('processBrowseArgs()', () => {it('it handles default parameters well', () => {- const func = () => {};-expect(processBrowseArgs()).to.eql({options: {url: 'about:blank',},- callback: null,- });-- expect(processBrowseArgs(func)).to.eql({- options: {- url: 'about:blank',- },- callback: func,- });-- expect(processBrowseArgs('example.com', func)).to.eql({- options: {- url: 'example.com',- },- callback: func,- });-- expect(processBrowseArgs('example.com', { opt: true }, func)).to.eql({- options: {- url: 'example.com',- opt: true,- },- callback: func,});expect(processBrowseArgs('example.com', { opt: true })).to.eql({@@ -117,51 +92,43 @@ describe('processBrowseArgs()', () => {url: 'example.com',opt: true,},- callback: null,});- expect(processBrowseArgs('example.com', { opt: true }, null)).to.eql({+ expect(processBrowseArgs('example.com', { url: 'another.com' })).to.eql({options: {url: 'example.com',- opt: true,},- callback: null,});- expect(processBrowseArgs('example.com', { url: 'another.com' }, null)).to.eql({+ expect(processBrowseArgs('example.com')).to.eql({options: {url: 'example.com',},- callback: null,});- expect(processBrowseArgs({}, null)).to.eql({+ expect(processBrowseArgs({})).to.eql({options: {url: 'about:blank',},- callback: null,});- expect(processBrowseArgs({ url: 'example.com' }, null)).to.eql({+ expect(processBrowseArgs({ url: 'example.com' })).to.eql({options: {url: 'example.com',},- callback: null,});- expect(processBrowseArgs({ url: 'example.com' }, func)).to.eql({+ expect(processBrowseArgs({ url: 'example.com' })).to.eql({options: {url: 'example.com',},- callback: func,});- expect(processBrowseArgs({ some: 123 }, func)).to.eql({+ expect(processBrowseArgs({ some: 123 })).to.eql({options: {url: 'about:blank',some: 123,},- callback: func,});});});@@ -212,32 +179,6 @@ describe('Apify.launchWebDriver()', function () {});});- it('works with empty options and callback', () => {- let webDriver;- return new Promise((resolve, reject) => {- try {- process.env.APIFY_HEADLESS = '1';- const retVal = Apify.launchWebDriver({}, (err, result) => {- if (err) return reject(err);- webDriver = result;- try {- expect(webDriver.constructor.name).to.eql('Driver');- webDriver.getCurrentUrl()- .then(resolve)- .catch(reject);- } catch (e) {- reject(e);- }- });- assert(!retVal, 'Apify.browse() with callback should return false-ish value');- } catch (e) {- reject(e);- }- }).then(() => {- return webDriver.quit();- });- });-it('works with proxy server', () => {let webDriver;wasProxyCalled = false;@@ -337,35 +278,6 @@ describe('Apify.browse()', function () {});});- it('works with empty options and callback', () => {- let browser;- return new Promise((resolve, reject) => {- try {- process.env.APIFY_HEADLESS = '1';- const retVal = Apify.browse('about:blank', {}, (err, result) => {- if (err) return reject(err);- browser = result;- try {- expect(browser.constructor.name).to.eql('Browser');- browser.webDriver.getCurrentUrl()- .then((url) => {- expect(url).to.eql('about:blank');- resolve();- })- .catch(reject);- } catch (e) {- reject(e);- }- });- assert(!retVal, 'Apify.browse() with callback should return false-ish value');- } catch (e) {- reject(e);- }- }).then(() => {- return browser.close();- });- });-it('works with proxy server', () => {let browser;wasProxyCalled = false;