Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Feature/request queue fixes (#51)
Marek Trunkát92c6565285f0
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex f489449b1..4b25abd92 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,4 +1,10 @@-xxxxxx / xxxxxxxxxxx+xxxxxx / xxxxxxxxxx+- `handleFailedRequestFunction` in both `BasicCrawler` and `PuppeteerCrawler` has now also error object+ available in `ops.error`.+- Request Queue storage type implemented. See documentation for more information.+- `BasicCrawler` and `PuppeteerCrawler` now supports both `RequestList` and `RequestQueue`.+- `AutoscaledPool` was completely redesigned. Check documentation for reference. It still supports previous+ configuration parameters for backwards compatibility but in the future compatibility will break.- `launchPuppeteer()` changes `User-Agent` only when in headless mode or if not using full Google Chrome,to reduce chance of detection of the crawlerdiff --git a/src/autoscaled_pool.js b/src/autoscaled_pool.jsindex 1aa8ea103..fd04a52f7 100644--- a/src/autoscaled_pool.js+++ b/src/autoscaled_pool.js@@ -6,6 +6,9 @@ import { getMemoryInfo, isPromise } from './utils';import { events } from './actor';import { ACTOR_EVENT_NAMES } from './constants';+// TODO: we should scale down less aggressively - ie. scale down only if CPU or memory+// gets exceeded for some longer period of time ...+const MEM_CHECK_INTERVAL_MILLIS = 200; // This is low to have at least.const MIN_FREE_MEMORY_RATIO = 0.1; // Minimum amount of memory that we keep free.const DEFAULT_OPTIONS = {@@ -30,23 +33,27 @@ export const LOG_INFO_INTERVAL = 6 * SCALE_UP_INTERVAL; // This must be multipleconst humanReadable = bytes => `${Math.round(bytes / 1024 / 1024)} MB`;/**- * AutoscaledPool helps to process asynchronous task in parallel. It scales the number of concurrent tasks based on- * the available memory and CPU. If any of the tasks throws an error then the pool.run() method- * also throws.+ * This class manages a pool of asynchronous resource-intensive tasks that are executed in parallel.+ * The pool only starts new tasks if there is enough free memory and CPU capacity.+ * The information about the CPU and memory usage is obtained+ * either from the local system or from the Apify cloud infrastructure in case the process+ * is running on the Apify Actor platform.*- * Autoscaled pool gets finished when the last running task gets resolved and following call of `workerFunction`- * returns null. This behaviour might be changed using `options.finishWhenEmpty` parameter.+ * The auto-scaled pool is started by calling the `run()` function+ * and it finishes when the last running task gets resolved and the next call to+ * the function passed via `isFinishedFunction` resolves to `false`.+ * If any of the tasks throws then the `run()` function also throws.*- * AutoscaledPool tries to start new tasks everytime some of the tasks gets resolved and also in interval- * given by parameter `options.maybeRunIntervalMillis`.+ * The pool evaluates whether is should start a new task every time some of the tasks is finished+ * and also in the interval set by the `options.maybeRunIntervalMillis` parameter.*- * Basic usage of AutoscaledPool:+ * Basic usage of `AutoscaledPool`:** ```javascript* const pool = new Apify.AutoscaledPool({* maxConcurrency: 50,- * workerFunction: () => {- * // ... do some intensive asynchronous operations here and return a promise ...+ * runTaskFunction: () => {+ * // Run some resource-intensive asynchronous operation here and return a promise...* },* });*@@ -54,58 +61,91 @@ const humanReadable = bytes => `${Math.round(bytes / 1024 / 1024)} MB`;* ```** @param {Object} options- * @param {Function} options.workerFunction Function we want to call in parallel. This function must either return a- * promise or null when all the tasks were processed.- * @param {Number} [options.maxConcurrency=1000] Maximal concurrency.- * @param {Number} [options.minConcurrency=1] Minimal concurrency.- * @param {Number} [options.maxMemoryMbytes] Maximum memory available in the system. By default uses the totalMemory from Apify.getMemoryInfo().- * @param {Number} [options.minFreeMemoryRatio=0.2] Minumum ratio of free memory kept in the system.- * @param {number} [options.maybeRunIntervalMillis=1000] Determines how often autoscaled pool tried to call `opts.workerFunction` to get a new task.- * @param {Number} [options.finishWhenEmpty=true] If false then pool stays running even when all tasks are finished and keeps trying to call- * `options.workerFunction` every `options.maybeRunIntervalMillis` milliseconds to get a new task.- * To finish the pool call `pool.finish()`.+ * @param {Function} options.runTaskFunction A function that performs an asynchronous resource-intensive task.+ * The function must either return a+ * promise or `null` if no task is currently available.+ * @param {Function} [opts.isFinishedFunction] A function that is called every time there are no tasks being processed.+ * If it resolves to `true` then the pool's run finishes.+ * If `isFinishedFunction` is not provided then the pool+ * is finished whenever there are no running tasks.+ * @param {Function} [opts.isTaskReadyFunction] A function that indicates if `runTaskFunction` should be called.+ * By default, this function is called every time there is a free capacity for new task.+ * But by overriding this you can throttle+ * number of calls to `runTaskFunction` or to prevent calls to `runTaskFunction` when you know+ * that it would return null.+ * @param {Number} [options.maxConcurrency=1000] Maximum concurrency.+ * @param {Number} [options.minConcurrency=1] Minimum concurrency.+ * @param {Number} [options.maxMemoryMbytes] Maximum memory available in the system. By default the pool+ * uses the `totalMemory` value provided by `Apify.getMemoryInfo()`.+ * @param {Number} [options.minFreeMemoryRatio=0.2] Minimum ratio of free memory kept in the system.+ * @param {number} [options.maybeRunIntervalMillis=1000] Indicates how often should the pool try to call+ * `opts.runTaskFunction` to start a new task.*/export default class AutoscaledPool {constructor(opts) {+ // TODO: remove this when we release v1.0.0+ // For backwards compatibility with opts.workerFunction.+ if (opts.workerFunction) {+ // For backwards compatiblity with opts.finishWhenEmpty and this.finish();+ if (opts.finishWhenEmpty !== undefined) {+ log.warning('Parameter `finishWhenEmpty` of AutoscaledPool is deprecated!!! Use `isFinishedFunction` instead!');+ checkParamOrThrow(opts.finishWhenEmpty, 'opts.finishWhenEmpty', 'Boolean');+ let mayFinish = false;+ opts.isFinishedFunction = () => Promise.resolve(mayFinish);+ this.finish = () => { mayFinish = true; };+ } else {+ opts.isFinishedFunction = () => Promise.resolve(true);+ }++ log.warning('Parameter `workerFunction` of AutoscaledPool is deprecated!!! Use `runTaskFunction` instead!');+ checkParamOrThrow(opts.workerFunction, 'opts.workerFunction', 'Function');+ opts.runTaskFunction = opts.workerFunction;+ opts.isTaskReadyFunction = () => Promise.resolve(true);+ }+const {- maxMemoryMbytes,maxConcurrency,minConcurrency,- workerFunction,+ maxMemoryMbytes,minFreeMemoryRatio,maybeRunIntervalMillis,- finishWhenEmpty,+ runTaskFunction,+ isFinishedFunction,+ isTaskReadyFunction,} = _.defaults(opts, DEFAULT_OPTIONS);- checkParamOrThrow(maxMemoryMbytes, 'opts.maxMemoryMbytes', 'Maybe Number');checkParamOrThrow(maxConcurrency, 'opts.maxConcurrency', 'Number');checkParamOrThrow(minConcurrency, 'opts.minConcurrency', 'Number');+ checkParamOrThrow(maxMemoryMbytes, 'opts.maxMemoryMbytes', 'Maybe Number');checkParamOrThrow(minFreeMemoryRatio, 'opts.minFreeMemoryRatio', 'Number');- checkParamOrThrow(workerFunction, 'opts.workerFunction', 'Function');checkParamOrThrow(maybeRunIntervalMillis, 'opts.maybeRunIntervalMillis', 'Number');- checkParamOrThrow(finishWhenEmpty, 'opts.finishWhenEmpty', 'Boolean');+ checkParamOrThrow(runTaskFunction, 'opts.runTaskFunction', 'Function');+ checkParamOrThrow(isFinishedFunction, 'opts.isFinishedFunction', 'Maybe Function');+ checkParamOrThrow(isTaskReadyFunction, 'opts.isTaskReadyFunction', 'Maybe Function');// Configuration.this.maxMemoryMbytes = maxMemoryMbytes;this.maxConcurrency = maxConcurrency;this.minConcurrency = Math.min(minConcurrency, maxConcurrency);- this.workerFunction = workerFunction;this.minFreeMemoryRatio = minFreeMemoryRatio;this.maybeRunIntervalMillis = maybeRunIntervalMillis;- this.finishWhenEmpty = finishWhenEmpty;+ this.runTaskFunction = runTaskFunction;+ this.isFinishedFunction = isFinishedFunction;+ this.isTaskReadyFunction = isTaskReadyFunction;// State.this.promiseCounter = 0;this.intervalCounter = 0;this.concurrency = minConcurrency;- this.runningPromises = {};this.runningCount = 0;this.freeBytesSnapshots = [];this.isCpuOverloaded = false;+ this.queryingIsTaskReady = false;+ this.queryingIsFinished = false;// Intervals.this.memCheckInterval = null;- this.maybeRunInterval = null;+ this.maybeRunTaskInterval = null;// This is resolve function of Promise returned by this.run()// which gets resolved once everything is done.@@ -114,6 +154,7 @@ export default class AutoscaledPool {this.reject = null;// Connect to actor events for CPU info.+ // TODO: This doesn't work on local machine!this.cpuInfoListener = (data) => {this.isCpuOverloaded = data.isCpuOverloaded;};@@ -121,25 +162,7 @@ export default class AutoscaledPool {}/**- * Closes the pool when `options.finishWhenEmpty=false`.- *- * By default when `options.finishWhenEmpty=true` this function is not needed and pool gets finished- * automatically.- *- * If `options.finishWhenEmpty=false` then pool gets running even after the last task gets resolved and- * `options.workerFunction` returns null. This is usefull when you expect more tasks to be added later.- * In this case you must call `pool.finish()` to finish the pool once all the running tasks are done.- */- finish() {- if (this.runningCount === 0) return this.resolve();-- this.finishWhenEmpty = true;-- return this.poolPromise;- }-- /**- * Runs the autoscaled pool. Returns promise that gets resolved or rejected once+ * Runs the auto-scaled pool. Returns promise that gets resolved or rejected once* all the task got finished or some of them fails.** @return {Promise}@@ -148,20 +171,23 @@ export default class AutoscaledPool {this.poolPromise = new Promise((resolve, reject) => {this.resolve = resolve;this.reject = reject;- this._maybeRunPromise();+ });- // This is here because if we scale down to let's say 1. Then after each promise is finished- // this._maybeRunPromise() doesn't trigger another one. So if that 1 instance stucks it results- // in whole act to stuck and even after scaling up it never triggers another promise.- this.maybeRunInterval = setInterval(() => this._maybeRunPromise(), this.maybeRunIntervalMillis);+ this._maybeRunTask();- // This interval checks memory and in each call saves current memory stats and in every- // SCALE_UP_INTERVAL-th/SCALE_DOWN_INTERVAL-th call it may scale up/down based on memory.- this.memCheckInterval = setInterval(() => this._autoscale(), MEM_CHECK_INTERVAL_MILLIS);- });+ // This is here because if we scale down to let's say 1, then after each promise is finished+ // this._maybeRunTask() doesn't trigger another one. So if that 1 instance gets stuck it results+ // in whole act to stuck and even after scaling up it never triggers another promise.+ this.maybeRunTaskInterval = setInterval(() => this._maybeRunTask(), this.maybeRunIntervalMillis);++ // This interval checks memory and in each call saves current memory stats and in every+ // SCALE_UP_INTERVAL-th/SCALE_DOWN_INTERVAL-th call it may scale up/down based on memory.+ this.memCheckInterval = setInterval(() => this._autoscale(), MEM_CHECK_INTERVAL_MILLIS);return this.poolPromise- .then(() => this._destroy())+ .then(() => {+ this._destroy();+ }).catch((err) => {this._destroy();@@ -180,7 +206,8 @@ export default class AutoscaledPool {* @ignore*/_autoscale() {- getMemoryInfo()+ // Returning promise so that we can await it in unit tests.+ return getMemoryInfo().then(({ freeBytes, totalBytes }) => {if (this.maxMemoryMbytes) totalBytes = Math.min(totalBytes, this.maxMemoryMbytes);@@ -210,7 +237,8 @@ export default class AutoscaledPool {log.debug('AutoscaledPool: scaling up', { oldConcurrency, newConcurrency: this.concurrency });}}- });+ })+ .catch(err => log.exception(err, 'AutoscaledPool._autoscale() function failed'));}/**@@ -225,7 +253,7 @@ export default class AutoscaledPool {const minFreeBytes = Math.min(...this.freeBytesSnapshots);const minFreeRatio = minFreeBytes / totalBytes;const maxTakenBytes = totalBytes - minFreeBytes;- const perInstanceRatio = (maxTakenBytes / totalBytes) / this.concurrency;+ const perInstanceRatio = (maxTakenBytes / totalBytes) / this.runningCount;const hasSpaceForInstances = (minFreeRatio - MIN_FREE_MEMORY_RATIO) / perInstanceRatio;if (logInfo) {@@ -244,72 +272,92 @@ export default class AutoscaledPool {return Math.floor(hasSpaceForInstances);}-/**- * Registers running promise.+ * If number of running task is lower than allowed concurrency and this.isTaskReadyFunction()+ * returns true then starts a new task.*- * @ignore- */- _addRunningPromise(id, promise) {- this.runningPromises[id] = promise;- this.runningCount++;- }-- /**- * Removes finished promise.- *- * @ignore- */- _removeFinishedPromise(id) {- delete this.runningPromises[id];- this.runningCount--;- }-- /**- * If this.runningCount < this.concurrency then gets new promise from this.workerFunction() and adds it to the pool.- * If this.workerFunction() returns null and nothing is running then finishes pool.+ * It doesn't allow multiple concurrent runs of this method.** @ignore*/- _maybeRunPromise() {- if (!this.resolve || !this.reject) return;+ _maybeRunTask() {if (this.runningCount >= this.concurrency) return;+ if (this.queryingIsTaskReady) return;- const promise = this.workerFunction();+ this.queryingIsTaskReady = true;- // We are done.- if (!promise && this.runningCount === 0) {- if (this.finishWhenEmpty) this.resolve();+ const isTaskReadyPromise = this.isTaskReadyFunction+ ? this.isTaskReadyFunction()+ : Promise.resolve(true);- return;- }+ // It's not null so it must be a promise!+ if (!isPromise(isTaskReadyPromise)) throw new Error('User provided isTaskReadyFunction must return a Promise.');++ // We don't want to chain this so no return here!+ isTaskReadyPromise+ .catch((err) => {+ this.queryingIsTaskReady = false;+ log.exception(err, 'AutoscaledPool: isTaskReadyFunction failed');+ })+ .then((isTaskReady) => {+ this.queryingIsTaskReady = false;- // We are not done but don't want to execute new promise at this point.- // This may happen when there are less pages in the queue than max concurrency- // but all of them are being served already.- if (!promise) return;+ if (!isTaskReady) return this._maybeFinish();- // It's not null so it must be a promise!- if (!isPromise(promise)) throw new Error('User provided workerFunction must return a Promise.');+ const taskPromise = this.runTaskFunction();- const id = this.promiseCounter;- this.promiseCounter++;- this._addRunningPromise(id, promise);+ // We are not done but don't want to execute new promise at this point.+ // This may happen when there are less pages in the queue than max concurrency+ // but all of them are being served already.+ if (!taskPromise) return this._maybeFinish();- promise- .then((data) => {- this._removeFinishedPromise(id);- this._maybeRunPromise();+ // It's not null so it must be a promise!+ if (!isPromise(taskPromise)) throw new Error('User provided runTaskFunction must return a Promise.');- return data;+ this.runningCount++;+ this._maybeRunTask();++ return taskPromise+ .then(() => {+ this.runningCount--;+ this._maybeRunTask();+ });}).catch((err) => {- log.exception(err, 'AutoscaledPool: worker function failed');- this._removeFinishedPromise(id);+ log.exception(err, 'AutoscaledPool: runTaskFunction failed');+ this.runningCount--;+ // If is here because we might already rejected this promise.if (this.reject) this.reject(err);});+ }- this._maybeRunPromise();+ /**+ * If there is no running task and this.isFinishedFunction() returns true then closes+ * the pool and resolves the pool dependency..+ *+ * It doesn't allow multiple concurrent runs of this method.+ *+ * @ignore+ */+ _maybeFinish() {+ if (this.queryingIsFinished) return;+ if (this.runningCount > 0) return;+ if (!this.isFinishedFunction) return this.resolve();++ this.queryingIsFinished = true;++ return this+ .isFinishedFunction()+ .then((isFinished) => {+ this.queryingIsFinished = false;++ if (!isFinished) return;+ if (this.resolve) this.resolve();+ })+ .catch((err) => {+ this.queryingIsFinished = false;+ log.exception(err, 'AutoscaledPool: isFinishedFunction failed');+ });}/**@@ -324,6 +372,6 @@ export default class AutoscaledPool {events.removeListener(ACTOR_EVENT_NAMES.CPU_INFO, this.cpuInfoListener);clearInterval(this.memCheckInterval);- clearInterval(this.maybeRunInterval);+ clearInterval(this.maybeRunTaskInterval);}}diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex 1baebc036..9d7d1025f 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -4,6 +4,7 @@ import log from 'apify-shared/log';import { isPromise, checkParamPrototypeOrThrow } from './utils';import AutoscaledPool from './autoscaled_pool';import RequestList from './request_list';+import { RequestQueue } from './request_queue';const DEFAULT_OPTIONS = {maxRequestRetries: 3,@@ -11,14 +12,20 @@ const DEFAULT_OPTIONS = {};/**- * BasicCrawler provides a simple framework for parallel crawling of a url list provided by `Apify.RequestList`- * or a dynamically enqueued requests provided by `Apify.RequestQueue` (TODO).+ * Provides a simple framework for parallel crawling of web pages+ * from a list of URLs managed by the `RequestList` class+ * or dynamically enqueued URLs managed by `RequestQueue`.*- * It's simply calling handleRequestFunction for each request from requestList or requestQueue as long as- * both are not empty. The concurrency is scaled based on available memory using `Apify.AutoscaledPool` and all- * of it's configuration parameters are supported here.+ * `BasicCrawler` simply calls `handleRequestFunction` for each request from `options.requestList` or `options.requestQueue` as long as+ * any of them is not empty. New requests are only handled if there is enough free CPU and memory available,+ * using the functionality provided by the `AutoscaledPool` class.+ * Note that all `AutoscaledPool` configuration options can be passed to `options` parameter of the `BasicCrawler` constructor.*- * Basic usage of BasicCrawler:+ * If both `requestList` and `requestQueue` is used, the instance first+ * processes URLs from the `requestList` and automatically enqueues all of them to `requestQueue` before it starts+ * their processing. This is to guarantee that a single URL is not crawled multiple times.+ *+ * Basic usage:** ```javascript* const rp = require('request-promise');@@ -37,25 +44,27 @@ const DEFAULT_OPTIONS = {* ```** @param {Object} options- * @param {RequestList} options.requestList List of the requests to be processed.- * @param {Function} options.handleRequestFunction Function that processes a request. It must return a promise.+ * @param {RequestList} [options.requestList] Static list of URLs to be processed.+ * @param {RequestQueue} [options.requestQueue] Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.+ * @param {Function} [options.handleRequestFunction] Function that processes a single `Request` object. It must return a promise.* @param {Function} [options.handleFailedRequestFunction=({ request, error }) => log.error('Request failed', _.pick(request, 'url', 'uniqueKey'))`]- * Function to handle requests that failed more then option.maxRequestRetries times.- * @param {Number} [options.maxRequestRetries=3] How many times request is retried if handleRequestFunction failed.- * @param {Number} [options.maxMemoryMbytes] Maximal memory available in the system (see `maxMemoryMbytes` parameter of `Apify.AutoscaledPool`).- * @param {Number} [options.maxConcurrency=1000] Maximal concurrency of request processing (see `maxConcurrency` parameter of `Apify.AutoscaledPool`).- * @param {Number} [options.minConcurrency=1] Minimal concurrency of requests processing (see `minConcurrency` parameter of `Apify.AutoscaledPool`).- * @param {Number} [options.minFreeMemoryRatio=0.2] Minumum ratio of free memory kept in the system.+ * Function that handles requests that failed more then `option.maxRequestRetries` times.+ * @param {Number} [options.maxRequestRetries=3] How many times the request is retried if `handleRequestFunction` failed.+ * @param {Number} [options.maxMemoryMbytes] Maximum memory available in the system (see `AutoscaledPool`).+ * @param {Number} [options.maxConcurrency=1000] Maximum number of request to process in parallel (see `AutoscaledPool`).+ * @param {Number} [options.minConcurrency=1] Minimum number of request to process in parallel (see `AutoscaledPool`).+ * @param {Number} [options.minFreeMemoryRatio=0.2] Minimum ratio of free memory kept in the system.*/export default class BasicCrawler {constructor(opts) {const {requestList,+ requestQueue,handleRequestFunction,handleFailedRequestFunction,maxRequestRetries,- // Autoscaled pool options+ // AutoscaledPool optionsmaxMemoryMbytes,maxConcurrency,minConcurrency,@@ -65,25 +74,32 @@ export default class BasicCrawler {checkParamOrThrow(handleRequestFunction, 'opts.handleRequestFunction', 'Function');checkParamOrThrow(handleFailedRequestFunction, 'opts.handleFailedRequestFunction', 'Function');checkParamOrThrow(maxRequestRetries, 'opts.maxRequestRetries', 'Number');- // TODO: make this optional once we have the request queue ready.- checkParamPrototypeOrThrow(requestList, 'opts.requestList', RequestList, 'Apify.RequestList');+ checkParamPrototypeOrThrow(requestList, 'opts.requestList', RequestList, 'Apify.RequestList', true);+ checkParamPrototypeOrThrow(requestQueue, 'opts.requestQueue', RequestQueue, 'Apify.RequestQueue', true);++ if (!requestList && !requestQueue) {+ throw new Error('At least one of the parameters "opts.requestList" and "opts.requestQueue" must be provided!');+ }this.requestList = requestList;+ this.requestQueue = requestQueue;this.handleRequestFunction = handleRequestFunction;this.handleFailedRequestFunction = handleFailedRequestFunction;this.maxRequestRetries = maxRequestRetries;this.autoscaledPool = new AutoscaledPool({- workerFunction: () => this._workerFunction(),maxMemoryMbytes,maxConcurrency,minConcurrency,minFreeMemoryRatio,+ runTaskFunction: () => this._runTaskFunction(),+ isFinishedFunction: () => this._isFinishedFunction(),+ isTaskReadyFunction: () => this._isTaskReadyFunction(),});}/**- * Runs the crawler. Returns promise that gets resolved once all the requests got processed.+ * Runs the crawler. Returns a promise that gets resolved once all the requests are processed.** @return {Promise}*/@@ -92,36 +108,112 @@ export default class BasicCrawler {}/**- * Wrapper around handleRequestFunction that catches errors and retries requests.+ * Fetches request from either RequestList or RequestQueue. If request comes from a RequestList+ * and RequestQueue is present then enqueues it to the queue first.+ *+ * @ignore+ */+ _fetchNextRequest() {+ if (!this.requestList) return this.requestQueue.fetchNextRequest();++ return this.requestList+ .fetchNextRequest()+ .then((request) => {+ if (!this.requestQueue) return request;+ if (!request) return this.requestQueue.fetchNextRequest();++ return this.requestQueue+ .addRequest(request, { forefront: true })+ .then(() => {+ return Promise+ .all([+ this.requestQueue.fetchNextRequest(),+ this.requestList.markRequestHandled(request),+ ])+ .then(results => results[0]);+ // If requestQueue.addRequest() fails here then we must reclaim it back to+ // the RequestList because probably it's not yet in the queue!+ }, (err) => {+ log.exception(err, 'RequestQueue.addRequest() failed, reclaiming request back to queue', { request });++ // Return null so that we finish immediately.+ return this.requestList+ .reclaimRequest(request)+ .then(() => null);+ });+ });+ }++ /**+ * Wrapper around handleRequestFunction that fetches requests from RequestList/RequestQueue+ * then retries them in a case of an error etc.** @ignore*/- _workerFunction() {- const request = this.requestList.fetchNextRequest();+ _runTaskFunction() {+ const source = this.requestQueue || this.requestList;- if (!request) return;+ return this._fetchNextRequest()+ .then((request) => {+ if (!request) return;- const handlePromise = this.handleRequestFunction({ request });- if (!isPromise(handlePromise)) throw new Error('User provided handleRequestFunction must return a Promise.');+ const handlePromise = this.handleRequestFunction({ request });+ if (!isPromise(handlePromise)) throw new Error('User provided handleRequestFunction must return a Promise.');- return handlePromise+ return handlePromise+ .then(() => source.markRequestHandled(request))+ .catch((error) => {+ request.pushErrorMessage(error);++ // Retry request.+ if (request.retryCount < this.maxRequestRetries) {+ request.retryCount++;++ return source.reclaimRequest(request);+ }++ // Mark as failed.+ return source+ .markRequestHandled(request)+ .then(() => this.handleFailedRequestFunction({ request, error }));+ });+ });+ }++ /**+ * Returns true if some RequestList and RequestQueue have request ready for processing.+ *+ * @ignore+ */+ _isTaskReadyFunction() {+ return Promise+ .resolve().then(() => {- this.requestList.markRequestHandled(request);+ if (!this.requestList) return false;++ return this.requestList.isEmpty();})- .catch((error) => {- request.pushErrorMessage(error);+ .then((isRequestListEmpty) => {+ if (!isRequestListEmpty || !this.requestQueue) return isRequestListEmpty;- // Retry request.- if (request.retryCount < this.maxRequestRetries) {- request.retryCount++;- this.requestList.reclaimRequest(request);- return;- }+ return this.requestQueue.isEmpty();+ })+ .then(areBothEmpty => !areBothEmpty);+ }- // Mark as failed.- this.requestList.markRequestHandled(request);+ /**+ * Returns true if both RequestList and RequestQueue have all requests finished.+ *+ * @ignore+ */+ _isFinishedFunction() {+ const promises = [];- return this.handleFailedRequestFunction({ request, error });- });+ if (this.requestList) promises.push(this.requestList.isFinished());+ if (this.requestQueue) promises.push(this.requestQueue.isFinished());++ return Promise+ .all(promises)+ .then(results => _.all(results));}}diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsindex 97b656d75..76054f86c 100644--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@ -11,20 +11,24 @@ const DEFAULT_OPTIONS = {};/**- * PuppeteerCrawler provides a simple framework for parallel crawling of a url list provided by Apify.RequestList- * or a dynamically enqueued requests provided by Apify.RequestQueue (TODO). Each url is opened in Puppeteer (Chrome)- * browser.+ * Provides a simple framework for parallel crawling of web pages+ * using the headless Chrome with Puppeteer.+ * The URLs of pages to visit are given by `Request` objects that are provided by the `RequestList` class+ * or a dynamically enqueued requests provided by the `RequestQueue` class.*- * For each url in the list or queue it opens a Chrome tab and then calls handlePageFunction. The concurrency is scaled- * based on available memory using Apify.AutoscaledPool.+ * For each `Request` object to crawl the class opens a Chrome tab and then calls+ * the function provided by user in the `handlePageFunction` option. New tasks are only+ * started if there is enough free CPU capacity and memory available,+ * internally using the `AutoscaledPool` instance.*- * Basic usage of PuppeteerCrawler:+ * Basic usage:** ```javascript* const crawler = new Apify.PuppeteerCrawler({* requestList,- * // Parameter page here is an intance of Puppeteer.Page with page.goto(request.url) already called* handlePageFunction: async ({ page, request }) => {+ * // 'page' is an instance of Puppeteer.Page with page.goto(request.url) already called+ * // 'request' is an instance of Request class with information about the page to load* await Apify.pushData({* title: await page.title(),* url: request.url,@@ -43,41 +47,44 @@ const DEFAULT_OPTIONS = {* await crawler.run();* ```*- * @param {RequestList} options.requestList List of the requests to be processed. (See `requestList` parameter of `Apify.BasicCrawler`)- * @param {Function} options.handlePageFunction Function to process each request.- * @param {Number} [options.pageOpsTimeoutMillis=30000] Timeout for options.handlePagefunction- * @param {Function} [options.gotoFunction=({ request, page }) => page.goto(request.url)] Overrides default gotoFunction. This function- * should return a result of page.goto(), ie. Puppeteers Response object.-+ * @param {RequestList} [options.requestList] List of the requests to be processed.+ * See the `requestList` parameter of `BasicCrawler` for more details.+ * @param {RequestList} [options.requestQueue] Queue of the requests to be processed.+ * See the `requestQueue` parameter of `BasicCrawler` for more details.+ * @param {Function} [options.handlePageFunction] Function that is called to process each request.+ * It is passed an object with the following fields:+ * `request` is an instance of the `Request` object with details about the URL to open, HTTP method etc.+ * `page` is an instance of the `Puppeteer.Page` class with `page.goto(request.url)` already called.+ * @param {Number} [options.pageOpsTimeoutMillis=30000] Timeout in which the fuction passed as `options.handlePageFunction` needs to finish.+ * @param {Function} [options.gotoFunction=({ request, page }) => page.goto(request.url)] Overrides the function that opens the request in Puppeteer.+ * This function should return a result of `page.goto()`, i.e. the Puppeteer's `Response` object.* @param {Function} [options.handleFailedRequestFunction=({ request }) => log.error('Request failed', _.pick(request, 'url', 'uniqueKey'))]- * Function to handle requests that failed more then option.maxRequestRetries times. (See `handleFailedRequestFunction`- * parameter of `Apify.BasicCrawler`)- * @param {Number} [options.maxRequestRetries=3] How many times request is retried if handleRequestFunction failed. (See `maxRequestRetries`- * parameter of `Apify.BasicCrawler`)- *- * @param {Number} [options.maxMemoryMbytes] Maximal memory available in the system (See `maxMemoryMbytes` parameter of `Apify.AutoscaledPool`).- * @param {Number} [options.maxConcurrency=1000] Maximal concurrency of request processing (See `maxConcurrency` parameter of `Apify.AutoscaledPool`).- * @param {Number} [options.minConcurrency=1] Minimal concurrency of requests processing (See `minConcurrency` parameter of `Apify.AutoscaledPool`).- * @param {Number} [options.minFreeMemoryRatio=0.2] Minumum ratio of free memory kept in the system.- *- * @param {Number} [options.maxOpenPagesPerInstance=100] Maximal number of opened tabs per browser. If limit is reached then the new- * browser gets started. (See `maxOpenPagesPerInstance` parameter of `Apify.PuppeteerPool`)- * @param {Number} [options.abortInstanceAfterRequestCount=150] Maximal number of requests proceeded from one browser. After that browser- * gets restarted. (See `abortInstanceAfterRequestCount` parameter of- * `Apify.PuppeteerPool`)- * @param {Function} [options.launchPuppeteerFunction] Overrides how new Puppeteer instance gets launched. (See `launchPuppeteerFunction` parameter of- * `Apify.PuppeteerPool`)- * @param {Number} [options.instanceKillerIntervalMillis=60000] How often opened Puppeteer instances get checked if some of then might be- * closed. (See `instanceKillerIntervalMillis` parameter of `Apify.PuppeteerPool`)- * @param {Number} [options.killInstanceAfterMillis=300000] If Puppeteer instance reaches the limit options.abortInstanceAfterRequestCount then it's- * considered retired and no more tabs will be opened. After the last tab get's closed the- * whole browser gets closed. This defines limit of inactivity after the browser gets closed- * even if there are pending tabs. (See `killInstanceAfterMillis` parameter of- * `Apify.PuppeteerPool`)- * @param {Object} [options.puppeteerConfig={ dumpio: process.env.NODE_ENV !== 'production', slowMo: 0, args: []}] Configuration of Puppeteer- * instances. (See `puppeteerConfig` parameter of `Apify.PuppeteerPool`)- * @param {Boolean} [options.disableProxy=false] Disables proxying thru Apify proxy. (See `disableProxy` parameter of `Apify.PuppeteerPool`)- * @param {Array} [options.groups] Apify proxy groups to be used. (See `Apify.getApifyProxyUrl()` for more)+ * Function to handle requests that failed more than `option.maxRequestRetries` times. See the `handleFailedRequestFunction`+ * parameter of `Apify.BasicCrawler` for details.+ * @param {Number} [options.maxRequestRetries=3] How many times each request is retried if `handleRequestFunction` failed. See `maxRequestRetries`+ * parameter of `BasicCrawler`.+ * @param {Number} [options.maxMemoryMbytes] Maximum memory available for crawling. See `maxMemoryMbytes` parameter of `AutoscaledPool`.+ * @param {Number} [options.maxConcurrency=1000] Maximum concurrency of request processing. See `maxConcurrency` parameter of `AutoscaledPool`.+ * @param {Number} [options.minConcurrency=1] Minimum concurrency of requests processing. See `minConcurrency` parameter of `AutoscaledPool`.+ * @param {Number} [options.minFreeMemoryRatio=0.2] Minimum ratio of free memory kept in the system.+ * @param {Number} [options.maxOpenPagesPerInstance=100] Maximum number of opened tabs per browser. If this limit is reached then a new+ * browser instance is started. See `maxOpenPagesPerInstance` parameter of `PuppeteerPool`.+ * @param {Number} [options.abortInstanceAfterRequestCount=150] Maximum number of requests that can be processed by a single browser instance.+ * After this limit is reached the browser is restarted.+ * See `abortInstanceAfterRequestCount` parameter of `PuppeteerPool`.+ * @param {Function} [options.launchPuppeteerFunction] Overrides how new Puppeteer instance gets launched. See `launchPuppeteerFunction` parameter of+ * `PuppeteerPool`.+ * @param {Number} [options.instanceKillerIntervalMillis=60000] How often the launched Puppeteer instances are checked whether they can be+ * closed. See `instanceKillerIntervalMillis` parameter of `PuppeteerPool`.+ * @param {Number} [options.killInstanceAfterMillis=300000] If Puppeteer instance reaches the `options.abortInstanceAfterRequestCount` limit then+ * it is considered retired and no more tabs will be opened. After the last tab is closed+ * the whole browser is closed too. This parameter defines a time limit for inactivity+ * after which the browser is closed even if there are pending tabs. See+ * `killInstanceAfterMillis` parameter of `PuppeteerPool`.+ * @param {Object} [options.puppeteerConfig={ dumpio: process.env.NODE_ENV !== 'production', slowMo: 0, args: []}] Default options for each+ * new Puppeteer instance. See `puppeteerConfig` parameter of `PuppeteerPool`.+ * @param {Boolean} [options.disableProxy=false] Disables proxying through Apify proxy. See `disableProxy` parameter of `PuppeteerPool`.+ * @param {Array} [options.groups] Apify proxy groups to be used. See `Apify.getApifyProxyUrl()` for more details.*/export default class PuppeteerCrawler {constructor(opts) {@@ -94,6 +101,7 @@ export default class PuppeteerCrawler {// Basic crawler optionsrequestList,+ requestQueue,maxRequestRetries,handleFailedRequestFunction,@@ -134,6 +142,7 @@ export default class PuppeteerCrawler {this.basicCrawler = new BasicCrawler({// Basic crawler options.requestList,+ requestQueue,maxRequestRetries,handleRequestFunction: (...args) => this._handleRequestFunction(...args),handleFailedRequestFunction,diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex 845a414b9..a1ebf508a 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -1,5 +1,6 @@import _ from 'underscore';import log from 'apify-shared/log';+import Promise from 'bluebird';import { cryptoRandomObjectId } from 'apify-shared/utilities';import { checkParamOrThrow } from 'apify-client/build/utils';import { launchPuppeteer } from './puppeteer';@@ -66,10 +67,10 @@ class PuppeteerInstance {}/**- * PuppeteerPool class provides a pool of Puppeteer (Chrome browser) instances.- * It rotates them based on it's configuration to change proxies.+ * Provides a pool of Puppeteer (Chrome browser) instances.+ * The class rotates the instances based on its configuration in order to change proxies.*- * Example use:+ * Example usage:** ```javascript* const puppeteerPool = new PuppeteerPool({ groups: 'some-proxy-group' });@@ -84,24 +85,22 @@ class PuppeteerInstance {* await puppeteerPool.destroy();* ```*- * @param {Number} [options.maxOpenPagesPerInstance=100] Maximal number of opened tabs per browser. If limit is reached then the new- * browser gets started. (See `maxOpenPagesPerInstance` parameter of `Apify.PuppeteerPool`)- * @param {Number} [options.abortInstanceAfterRequestCount=150] Maximal number of requests proceeded from one browser. After that browser- * gets restarted. (See `abortInstanceAfterRequestCount` parameter of- * `Apify.PuppeteerPool`)- * @param {Function} [options.launchPuppeteerFunction] Overrides how new Puppeteer instance gets launched. (See `launchPuppeteerFunction` parameter of- * `Apify.PuppeteerPool`)- * @param {Number} [options.instanceKillerIntervalMillis=60000] How often opened Puppeteer instances get checked if some of then might be- * closed. (See `instanceKillerIntervalMillis` parameter of `Apify.PuppeteerPool`)- * @param {Number} [options.killInstanceAfterMillis=300000] If Puppeteer instance reaches the limit options.abortInstanceAfterRequestCount then it's- * considered retired and no more tabs will be opened. After the last tab get's closed the- * whole browser gets closed. This defines limit of inactivity after the browser gets closed- * even if there are pending tabs. (See `killInstanceAfterMillis` parameter of- * `Apify.PuppeteerPool`)+ * @param {Number} [options.maxOpenPagesPerInstance=100] Maximum number of open tabs per browser. If this limit is reached then a new+ * browser will be started.+ * @param {Number} [options.abortInstanceAfterRequestCount=150] Maximum number of requests processed by a single browser.+ * After the limit is reach the browser+ * will be restarted.+ * @param {Function} [options.launchPuppeteerFunction] Overrides the default function to launch a new Puppeteer instance.+ * @param {Number} [options.instanceKillerIntervalMillis=60000] How often opened Puppeteer instances are checked wheter they can be+ * closed.+ * @param {Number} [options.killInstanceAfterMillis=300000] If Puppeteer instance reaches the `options.abortInstanceAfterRequestCount` limit then+ * it is considered retired and no more tabs will be opened. After the last tab is closed the+ * whole browser is closed too. This parameter defines a time limit for inactivity after+ * which the browser is closed even if there are pending open tabs.* @param {Object} [options.puppeteerConfig={ dumpio: process.env.NODE_ENV !== 'production', slowMo: 0, args: []}] Configuration of Puppeteer- * instances. (See `puppeteerConfig` parameter of `Apify.PuppeteerPool`)- * @param {Boolean} [options.disableProxy=false] Disables proxying thru Apify proxy. (See `disableProxy` parameter of `Apify.PuppeteerPool`)- * @param {Array} [options.groups] Apify proxy groups to be used. (See `Apify.getApifyProxyUrl()` for more)+ * instances.+ * @param {Boolean} [options.disableProxy=false] Disables proxying through Apify proxy.+ * @param {Array} [options.groups] Apify proxy groups to be used. See `Apify.getApifyProxyUrl()` for more details.*/export default class PuppeteerPool {constructor(opts = {}) {@@ -228,7 +227,12 @@ export default class PuppeteerPool {delete this.retiredInstances[id];// Ensure that Chrome process will be really killed.- setTimeout(() => childProcess.kill('SIGKILL'), PROCESS_KILL_TIMEOUT_MILLIS);+ setTimeout(() => {+ // This if is here because users reported that it happened+ // that error `TypeError: Cannot read property 'kill' of null` was thrown.+ // Likely Chrome process wasn't started for some error ...+ if (childProcess) childProcess.kill('SIGKILL');+ }, PROCESS_KILL_TIMEOUT_MILLIS);instance.browserPromisediff --git a/src/request.js b/src/request.jsindex 80d5d7ae3..75093edc7 100644--- a/src/request.js+++ b/src/request.js@@ -37,18 +37,23 @@ export const computeUniqueKey = (url, keepUrlFragment) => normalizeUrl(url, keep* @param {Boolean} [opts.keepUrlFragment=false] If false then hash part is removed from url when computing `uniqueKey`.*/export default class Request {- constructor({- id,- url,- uniqueKey,- method = 'GET',- payload = null,- retryCount = 0,- errorMessages = null,- headers = {},- userData = {},- keepUrlFragment = false,- }) {+ constructor(opts = {}) {+ checkParamOrThrow(opts, 'opts', 'Object');++ const {+ id,+ url,+ uniqueKey,+ method = 'GET',+ payload = null,+ retryCount = 0,+ errorMessages = null,+ headers = {},+ userData = {},+ keepUrlFragment = false,+ } = opts;++checkParamOrThrow(id, 'id', 'Maybe String');checkParamOrThrow(url, 'url', 'String');checkParamOrThrow(uniqueKey, 'uniqueKey', 'Maybe String');diff --git a/src/request_list.js b/src/request_list.jsindex 26521c1eb..e1c3c0ddd 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -1,6 +1,7 @@import { checkParamOrThrow } from 'apify-client/build/utils';import log from 'apify-shared/log';import _ from 'underscore';+import Promise from 'bluebird';import requestPromise from 'request-promise';import { sequentializePromises } from 'apify-shared/utilities';import Request from './request';@@ -22,11 +23,12 @@ const ensureUniqueKeyValid = (uniqueKey) => {};/**- * `RequestList` provides way to handle a list of URLs (i.e. requests) to be crawled.+ * Pprovides way to handle a list of URLs to be crawled.+ * Each URL is reprented using an instance of the `Request` class.*- * `RequestList` has internal state where it remembers handled requests, requests in progress and also reclaimed requests.- * State might be persisted in key-value store as shown in the example below so if an act get restarted (due to internal- * error or restart of the host machine) then it may be initialized from previous state.+ * `RequestList` has an internal state where it remembers handled requests, requests in progress and also reclaimed requests.+ * The state might be persisted in a key-value store as shown in the example below so if an act is restarted (due to internal+ * error or restart of the host machine) then the crawling can continue where it left off.** Basic usage of `RequestList`:*@@ -64,6 +66,7 @@ const ensureUniqueKeyValid = (uniqueKey) => {* requestList.reclaimRequest(request2);* ```*+ * @param {Object} options* @param {Array} options.sources Function that processes a request. It must return a promise.* ```javascript* [@@ -86,7 +89,11 @@ const ensureUniqueKeyValid = (uniqueKey) => {* ```*/export default class RequestList {- constructor({ sources, state }) {+ constructor(opts = {}) {+ checkParamOrThrow(opts, 'options', 'Object');++ const { sources, state } = opts;+checkParamOrThrow(sources, 'options.sources', 'Array');checkParamOrThrow(state, 'options.state', 'Maybe Object');@@ -191,50 +198,105 @@ export default class RequestList {* Returns `true` if the next call to fetchNextRequest() will return null, otherwise it returns `false`.* Note that even if the list is empty, there might be some pending requests currently being processed.*- * @returns {boolean}+ * @returns {Promise<boolean>}*/isEmpty() {- this._ensureIsInitialized();+ return Promise+ .resolve()+ .then(() => {+ this._ensureIsInitialized();- return !getFirstKey(this.reclaimed) && this.nextIndex >= this.requests.length;+ return !getFirstKey(this.reclaimed) && this.nextIndex >= this.requests.length;+ });}/*** Returns `true` if all requests were already handled and there are no more left.*- * @returns {boolean}+ * @returns {Promise<boolean>}*/isFinished() {- this._ensureIsInitialized();+ return Promise+ .resolve()+ .then(() => {+ this._ensureIsInitialized();- return !getFirstKey(this.inProgress) && this.nextIndex >= this.requests.length;+ return !getFirstKey(this.inProgress) && this.nextIndex >= this.requests.length;+ });}/*** Returns next request which is the reclaimed one if available or next upcoming request otherwise.*- * @returns {Request}+ * @returns {Promise<Request>}*/fetchNextRequest() {- this._ensureIsInitialized();+ return Promise+ .resolve()+ .then(() => {+ this._ensureIsInitialized();++ // First return reclaimed requests if any.+ const uniqueKey = getFirstKey(this.reclaimed);+ if (uniqueKey) {+ delete this.reclaimed[uniqueKey];+ const index = this.uniqueKeyToIndex[uniqueKey];+ return this.requests[index];+ }- // First return reclaimed requests if any.- const uniqueKey = getFirstKey(this.reclaimed);- if (uniqueKey) {- delete this.reclaimed[uniqueKey];- const index = this.uniqueKeyToIndex[uniqueKey];- return this.requests[index];- }+ // Otherwise return next request.+ if (this.nextIndex < this.requests.length) {+ const request = this.requests[this.nextIndex];+ this.inProgress[request.uniqueKey] = true;+ this.nextIndex++;+ return request;+ }- // Otherwise return next request.- if (this.nextIndex < this.requests.length) {- const request = this.requests[this.nextIndex];- this.inProgress[request.uniqueKey] = true;- this.nextIndex++;- return request;- }+ return null;+ });+ }++ /**+ * Marks request handled after successfull processing.+ *+ * @param {Request} request+ *+ * @returns {Promise}+ */+ markRequestHandled(request) {+ return Promise+ .resolve()+ .then(() => {+ const { uniqueKey } = request;++ ensureUniqueKeyValid(uniqueKey);+ this._ensureInProgressAndNotReclaimed(uniqueKey);+ this._ensureIsInitialized();++ delete this.inProgress[uniqueKey];+ });+ }++ /**+ * Reclaims request to the list if its processing failed.+ * The request will become available in the next `this.fetchNextRequest()`.+ *+ * @param {Request} request+ *+ * @returns {Promise}+ */+ reclaimRequest(request) {+ return Promise+ .resolve()+ .then(() => {+ const { uniqueKey } = request;++ ensureUniqueKeyValid(uniqueKey);+ this._ensureInProgressAndNotReclaimed(uniqueKey);+ this._ensureIsInitialized();- return null;+ this.reclaimed[uniqueKey] = true;+ });}/**@@ -326,34 +388,4 @@ export default class RequestList {throw new Error('RequestList is not initialized. You must call "await requestList.initialize();" before using it!');}}-- /**- * Marks request handled after successfull processing.- *- * @param {Request} request- */- markRequestHandled(request) {- const { uniqueKey } = request;-- ensureUniqueKeyValid(uniqueKey);- this._ensureInProgressAndNotReclaimed(uniqueKey);- this._ensureIsInitialized();-- delete this.inProgress[uniqueKey];- }-- /**- * Reclaims request after unsuccessfull operation. Request will become available for next `this.fetchNextRequest()`.- *- * @param {Request} request- */- reclaimRequest(request) {- const { uniqueKey } = request;-- ensureUniqueKeyValid(uniqueKey);- this._ensureInProgressAndNotReclaimed(uniqueKey);- this._ensureIsInitialized();-- this.reclaimed[uniqueKey] = true;- }}diff --git a/src/request_queue.js b/src/request_queue.jsindex 7bb382f9e..1e4afd677 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -4,6 +4,7 @@ import path from 'path';import { checkParamOrThrow } from 'apify-client/build/utils';import LruCache from 'apify-shared/lru_cache';import ListDictionary from 'apify-shared/list_dictionary';+import { delayPromise } from 'apify-shared/utilities';import Promise from 'bluebird';import crypto from 'crypto';import _ from 'underscore';@@ -13,8 +14,20 @@ import { ensureDirExists, checkParamPrototypeOrThrow, apifyClient } from './utilexport const LOCAL_EMULATION_SUBDIR = 'request-queues';const MAX_OPENED_QUEUES = 1000;++// When requesting queue head we always fetch requestsInProgressCount + QUERY_HEAD_BUFFER number of requestsexport const QUERY_HEAD_BUFFER = 50;+// If queue was modified (request added/updated/deleted) before more than API_PROCESSED_REQUESTS_DELAY_MILLIS+// then we get head query to be consistent.+export const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10 * 1000;++// How many times we try to get queue head with queueModifiedAt older than API_PROCESSED_REQUESTS_DELAY_MILLIS.+export const MAX_QUERIES_FOR_CONSISTENCY = 6;++// TODO: this should be in shared.+export const QUERY_HEAD_MAX_LIMIT = 1000;+const writeFilePromised = Promise.promisify(fs.writeFile);const readdirPromised = Promise.promisify(fs.readdir);const readFilePromised = Promise.promisify(fs.readFile);@@ -23,11 +36,6 @@ const moveFilePromised = Promise.promisify(fsExtra.move);const { requestQueues } = apifyClient;const queuesCache = new LruCache({ maxLength: MAX_OPENED_QUEUES }); // Open queues are stored here.--/**- * @ignore.- */-/*** Helper function to validate params of *.addRequest().* @ignore@@ -66,9 +74,16 @@ const validateMarkRequestHandledParams = (request) => {* Helper function to validate params of *.reclaimRequest().* @ignore*/-const validateReclaimRequestParams = (request) => {+const validateReclaimRequestParams = (request, opts) => {checkParamPrototypeOrThrow(request, 'request', Request, 'Apify.Request');checkParamOrThrow(request.id, 'request.id', 'String');+ checkParamOrThrow(opts, 'opts', 'Object');++ const { forefront = false } = opts;++ checkParamOrThrow(forefront, 'opts.forefront', 'Boolean');++ return { forefront };};/**@@ -79,17 +94,30 @@ const validateReclaimRequestParams = (request) => {*//**- * @ignore.- *- * TODO: fix docs- * Dataset class provides easy interface to Apify Dataset storage type. Dataset should be opened using- * `Apify.openDataset()` function.+ * Provides a simple interface to the Apify Request Queue API,+ * which is used to manage a dynamic queue of web pages to crawl.+ * The instance of this object can be obtained using the+ * `Apify.openRequestQueue()` function.*- * Basic usage of Dataset:+ * Example usage:** ```javascript- * const dataset = await Apify.openDataset('my-dataset-id');- * await dataset.pushData({ foo: 'bar' });+ * const queue = await Apify.openRequestQueue('my-queue-id');+ *+ * await queue.addRequest(new Apify.Request({ url: 'http://example.com/aaa'});+ * await queue.addRequest(new Apify.Request({ url: 'http://example.com/bbb'});+ * await queue.addRequest(new Apify.Request({ url: 'http://example.com/foo/bar'}, { puyInFront: true });+ *+ * // Get requests from queue+ * const request1 = queue.fetchNextRequest();+ * const request2 = queue.fetchNextRequest();+ * const request3 = queue.fetchNextRequest();+ *+ * // Mark some of them as handled+ * queue.markRequestHandled(request1);+ *+ * // If processing fails then reclaim it back to the queue+ * queue.reclaimRequest(request2);* ```** @param {String} queueId - ID of the request queue.@@ -103,6 +131,10 @@ export class RequestQueue {this.requestIdsInProgress = {};this.inProgressCount = 0;this.queryQueueHeadPromise = null;++ // This contains false if we were not able to get queue head with queueModifiedAt older than+ // at least API_PROCESSED_REQUESTS_DELAY_MILLIS.+ this.isHeadConsistent = true;}/**@@ -180,7 +212,7 @@ export class RequestQueue {validateMarkRequestHandledParams(request);if (!this.requestIdsInProgress[request.id]) {- throw new Error('Cannot mark handled request that is not in progress!');+ throw new Error(`Cannot mark request ${request.id} as handled as it is not in progress!`);}if (!request.handledAt) request.handledAt = new Date();@@ -206,8 +238,8 @@ export class RequestQueue {* and to the back of the queue otherwise.* @return {RequestOperationInfo}*/- reclaimRequest(request, { forefront = false }) {- validateReclaimRequestParams(request);+ reclaimRequest(request, opts = {}) {+ const { forefront } = validateReclaimRequestParams(request, opts);return requestQueues.updateRequest({@@ -225,31 +257,34 @@ export class RequestQueue {}/**- * Returns `true` if the next call to fetchNextRequest() will return null, otherwise it returns `false`.+ * Returns `true` if the next call to `fetchNextRequest()` will return `null`, otherwise it returns `false`.* Note that even if the queue is empty, there might be some pending requests currently being processed.*+ * The function might occasionally return a false negative, but it should never return a false positive!+ ** @returns {boolean}*/isEmpty() {return this- ._queryQueueHead()- .then(() => this.queueHeadDict.length() === 0);+ ._ensureHeadIsNonEmpty()+ .then(() => this.isHeadConsistent && this.queueHeadDict.length() === 0);}/*** Returns `true` if all requests were already handled and there are no more left.+ * The function might occasionally return a false negative, but it should never return a false positive!** @returns {boolean}*/isFinished() {return this- ._queryQueueHead()- .then(() => this.inProgressCount === 0 && this.queueHeadDict.length() === 0);+ ._ensureHeadIsNonEmpty()+ .then(() => this.isHeadConsistent && this.inProgressCount === 0 && this.queueHeadDict.length() === 0);}/**- * @ignore.+ * @ignore*/_addToInProgress(requestId) {// Is already there.@@ -260,7 +295,7 @@ export class RequestQueue {}/**- * @ignore.+ * @ignore*/_removeFromInProgress(requestId) {// Is already removed.@@ -271,24 +306,26 @@ export class RequestQueue {}/**- * We always request more items than is in proggress to ensure that something+ * We always request more items than is in progress to ensure that something* falls into head.*- * @ignore.+ * @ignore*/- _ensureHeadIsNonEmpty(limit = this.inProgressCount + QUERY_HEAD_BUFFER) {+ _ensureHeadIsNonEmpty(checkModifiedAt = false, limit = this.inProgressCount + QUERY_HEAD_BUFFER, iteration = 0) {checkParamOrThrow(limit, 'limit', 'Number');// If is nonempty resolve immediately.if (this.queueHeadDict.length()) return Promise.resolve();if (!this.queryQueueHeadPromise) {+ const queryStartedAt = new Date();+this.queryQueueHeadPromise = requestQueues.getHead({limit,queueId: this.queueId,})- .then(({ items }) => {+ .then(({ items, queueModifiedAt }) => {items.forEach(({ id }) => {if (!this.requestIdsInProgress[id]) this.queueHeadDict.add(id, id, false);});@@ -297,21 +334,52 @@ export class RequestQueue {this.queryQueueHeadPromise = null;return {- prevLimit: limit,limitReached: items.length === limit,+ prevLimit: limit,+ queueModifiedAt: new Date(queueModifiedAt),+ queryStartedAt,};});}return this.queryQueueHeadPromise- .then(({ prevLimit, limitReached }) => {+ .then(({ queueModifiedAt, limitReached, prevLimit, queryStartedAt }) => {+ this.isHeadConsistent = true;+// If queue is still empty then it's likely because some of the other calls waiting// for this promise already consumed all the returned requests or the limit was too// low and contained only requests in progress.//// If limit was not reached in the call then there are no more requests to be returned.- if (!this.queueHeadDict.length() && limitReached) {- return this._ensureHeadIsNonEmpty(prevLimit + QUERY_HEAD_BUFFER);+ const shouldRepeatWithHigherLimit = !this.queueHeadDict.length() && limitReached && prevLimit < QUERY_HEAD_MAX_LIMIT;++ // If checkModifiedAt=true then we must ensure that queueModifiedAt is older than+ // queryStartedAt for at least API_PROCESSED_REQUESTS_DELAY_MILLIS.+ const shouldRepeatForConsistency = (+ checkModifiedAt+ && (queryStartedAt - queueModifiedAt > API_PROCESSED_REQUESTS_DELAY_MILLIS)+ && iteration+ );++ if (shouldRepeatWithHigherLimit || shouldRepeatForConsistency) {+ // If we are queriing for consistency then we limit the number of queries to MAX_QUERIES_FOR_CONSISTENCY.+ // If this is reached then we set this.isHeadConsistent=true so that empty() and finished() returns+ // maybe false negative.+ if (!shouldRepeatWithHigherLimit && iteration > MAX_QUERIES_FOR_CONSISTENCY) {+ this.isHeadConsistent = false;+ return;+ }++ const nextLimit = shouldRepeatWithHigherLimit+ ? prevLimit + QUERY_HEAD_MAX_LIMIT+ : prevLimit;++ const delayMillis = shouldRepeatForConsistency+ ? API_PROCESSED_REQUESTS_DELAY_MILLIS+ : 0;++ return delayPromise(delayMillis)+ .then(() => this._ensureHeadIsNonEmpty(checkModifiedAt, nextLimit, iteration + 1));}});}@@ -348,7 +416,7 @@ const uniqueKeyToId = (uniqueKey) => {};/**- * Local implementation of RequestQueue.+ * Local implementation of the `RequestQueue` class.** @ignore*/@@ -408,18 +476,42 @@ export class RequestQueueLocal {return path.join(dir, fileName);}+ _getQueueOrderNo(forefront = false) {+ const sgn = (forefront ? 1 : 2) * (10 ** 15);+ const base = (10 ** (13)); // Date.now() returns int with 13 numbers.+ // We always add pending count for a case that two pages are insterted at the same millisecond.+ const now = Date.now() + this.pendingCount;+ const queueOrderNo = forefront+ ? sgn + (base - now)+ : sgn + (base + now);++ return queueOrderNo;+ }++ _getRequestByQueueOrderNo(queueOrderNo) {+ checkParamOrThrow(queueOrderNo, 'queueOrderNo', 'Number');++ return readFilePromised(this._getFilePath(queueOrderNo, false))+ .catch((err) => {+ if (err.code !== 'ENOENT') throw err;++ return readFilePromised(this._getFilePath(queueOrderNo, true));+ })+ .then((str) => {+ if (!str) throw new Error('Request was not found in none of handled and pending directories!');++ const obj = JSON.parse(str);++ return new Request(obj);+ });+ }+addRequest(request, opts = {}) {const { forefront } = validateAddRequestParams(request, opts);return this.initializationPromise.then(() => {- const sgn = (forefront ? 1 : 2) * (10 ** 15);- const base = (10 ** (13)); // Date.now() returns int with 13 numbers.- // We always add pending count for a case that two pages are insterted at the same millisecond.- const now = Date.now() + this.pendingCount;- const queueOrderNo = forefront- ? sgn + (base - now)- : sgn + (base + now);+ const queueOrderNo = this._getQueueOrderNo(forefront);// Add ID as server does.const requestCopy = JSON.parse(JSON.stringify(request));@@ -438,11 +530,12 @@ export class RequestQueueLocal {this.requestIdToQueueOrderNo[requestCopy.id] = queueOrderNo;if (!requestCopy.handledAt) this.pendingCount++;+const filePath = this._getFilePath(queueOrderNo, !!requestCopy.handledAt);return writeFilePromised(filePath, JSON.stringify(requestCopy, null, 4)).then(() => ({- requestId: requestCopy.id,+ requestId: request.id,wasAlreadyHandled: false,wasAlreadyPresent: false,}));@@ -460,24 +553,6 @@ export class RequestQueueLocal {});}- _getRequestByQueueOrderNo(queueOrderNo) {- checkParamOrThrow(queueOrderNo, 'queueOrderNo', 'Number');-- return readFilePromised(this._getFilePath(queueOrderNo, false))- .catch((err) => {- if (err.code !== 'ENOENT') throw err;-- return readFilePromised(this._getFilePath(queueOrderNo, true));- })- .then((str) => {- if (!str) throw new Error('Request was not found in none of handled and pending directories!');-- const obj = JSON.parse(str);-- return new Request(obj);- });- }-fetchNextRequest() {return this.initializationPromise.then(() => readdirPromised(this.localPendingEmulationPath))@@ -512,7 +587,7 @@ export class RequestQueueLocal {const dest = this._getFilePath(queueOrderNo, true);if (!this.queueOrderNoInProgress[queueOrderNo]) {- throw new Error('Cannot mark handled request that is not in progress!');+ throw new Error(`Cannot mark request ${request.id} handled request that is not in progress!`);}if (!request.handledAt) request.handledAt = new Date();@@ -531,20 +606,27 @@ export class RequestQueueLocal {});}- reclaimRequest(request) {- validateReclaimRequestParams(request);+ reclaimRequest(request, opts = {}) {+ const { forefront } = validateReclaimRequestParams(request, opts);return this.initializationPromise.then(() => {- const queueOrderNo = this.requestIdToQueueOrderNo[request.id];+ const oldQueueOrderNo = this.requestIdToQueueOrderNo[request.id];+ const newQueueOrderNo = this._getQueueOrderNo(forefront);- if (!this.queueOrderNoInProgress[queueOrderNo]) {- throw new Error('Cannot reclaim request that is not in progress!');+ if (!this.queueOrderNoInProgress[oldQueueOrderNo]) {+ throw new Error(`Cannot reclaim request ${request.id} that is not in progress!`);}- this.inProgressCount--;+ this.requestIdToQueueOrderNo[request.id] = newQueueOrderNo;- delete this.queueOrderNoInProgress[queueOrderNo];+ return moveFilePromised(+ this._getFilePath(oldQueueOrderNo),+ this._getFilePath(newQueueOrderNo),+ ).then(() => {+ this.inProgressCount--;+ delete this.queueOrderNoInProgress[oldQueueOrderNo];+ });}).then(() => ({requestId: request.id,@@ -579,7 +661,7 @@ const getOrCreateQueue = (queueIdOrName) => {};/**- * Opens request queue and returns its object.</p>+ * Opens a request queue and returns its `RequestQueue` object.** ```javascript* const queue = await Apify.openRequestQueue('my-queue-id');@@ -601,11 +683,11 @@ const getOrCreateQueue = (queueIdOrName) => {* ```** If the `APIFY_LOCAL_EMULATION_DIR` environment variable is defined, the value this function- * returns an instance `RequestQueueLocal` which is an local emulation of request queue.- * This is useful for local development and debugging of your acts.+ * returns is an instance of the `RequestQueueLocal` class which is a local emulation of the request queue.+ * This is useful for local development and debugging of the acts.** @param {string} queueIdOrName ID or name of the request queue to be opened.- * @returns {Promise<RequestQueue>} Returns a promise that resolves to a RequestQueue object.+ * @returns {Promise<RequestQueue>} Returns a promise that resolves to a `RequestQueue` object.** @memberof module:Apify* @name openRequestQueuediff --git a/src/settings_rotator.js b/src/settings_rotator.jsindex 31bddf224..0e554a190 100644--- a/src/settings_rotator.js+++ b/src/settings_rotator.js@@ -3,17 +3,26 @@ import { checkParamOrThrow } from 'apify-client/build/utils';// TODO: randomWaitBetweenUsagesMillis/**- * SettingsRotator rotates settings created by newSettingsFunction based on it's configuration.+ * Rotates settings created by a user-provided function passed+ * via `newSettingsFunction`.+ * This is useful during web crawling to dynamically change settings and thus+ * avoid detection of the crawler.+ *+ * This class is still work in progress, more features will be added soon.** @param {Object} options* @param {Function} options.newSettingsFunction* @param {Number} options.maxUsages*/export default class SettingsRotator {- constructor({- newSettingsFunction,- maxUsages,- }) {+ constructor(opts) {+ checkParamOrThrow(opts, 'options', 'Object');++ const {+ newSettingsFunction,+ maxUsages,+ } = opts;+checkParamOrThrow(newSettingsFunction, 'options.newSettingsFunction', 'Function');checkParamOrThrow(maxUsages, 'options.maxUsages', 'Number');diff --git a/test/autoscaled_pool.js b/test/autoscaled_pool.jsindex b97cb5502..412adb9ac 100644--- a/test/autoscaled_pool.js+++ b/test/autoscaled_pool.js@@ -17,6 +17,354 @@ describe('AutoscaledPool', () => {const range = _.range(0, 10);const result = [];+ let isFinished = false;++ const runTaskFunction = () => {+ if (range.length === 1) {+ isFinished = true;+ }++ return new Promise((resolve) => {+ const item = range.shift();+ result.push(item);+ setTimeout(resolve, 10);+ });+ };++ const pool = new Apify.AutoscaledPool({+ minConcurrency: 1,+ maxConcurrency: 1,+ runTaskFunction,+ isFinishedFunction: () => Promise.resolve(isFinished),+ isTaskReadyFunction: () => Promise.resolve(!isFinished),+ });+ const startedAt = Date.now();+ await pool.run();++ expect(result).to.be.eql(_.range(0, 10));+ expect(Date.now() - startedAt).to.be.within(100, 150);+ });++ it('should work with concurrency 10', async () => {+ const range = _.range(0, 100);+ const result = [];++ let isFinished = false;++ const runTaskFunction = () => {+ if (range.length === 1) {+ isFinished = true;+ }++ return new Promise((resolve) => {+ const item = range.shift();+ result.push(item);+ setTimeout(resolve, 10);+ });+ };++ const pool = new Apify.AutoscaledPool({+ minConcurrency: 10,+ maxConcurrency: 10,+ runTaskFunction,+ isFinishedFunction: () => Promise.resolve(isFinished),+ isTaskReadyFunction: () => Promise.resolve(!isFinished),+ });++ const startedAt = Date.now();+ await pool.run();++ expect(result).to.be.eql(_.range(0, 100));+ expect(Date.now() - startedAt).to.be.within(100, 150);+ });++ it('correctly computed space for instances using', async () => {+ const pool = new Apify.AutoscaledPool({+ minConcurrency: 1,+ maxConcurrency: 100,+ runTaskFunction: () => Promise.resolve(),+ isFinishedFunction: () => Promise.resolve(false),+ isTaskReadyFunction: () => Promise.resolve(true),+ });++ pool.concurrency = 20;+ pool.runningCount = 20;+ pool.freeBytesSnapshots = [+ toBytes(50),+ toBytes(30),+ toBytes(50),+ toBytes(40),+ ];+ const hasSpaceForInstances = pool._computeSpaceForInstances(toBytes(100), true);+ expect(hasSpaceForInstances).to.be.eql(5);++ pool.concurrency = 20;+ pool.runningCount = 20;+ pool.freeBytesSnapshots = [+ toBytes(5),+ toBytes(5),+ toBytes(5),+ toBytes(5),+ ];+ const hasSpaceForInstances2 = pool._computeSpaceForInstances(toBytes(100), true);+ expect(hasSpaceForInstances2).to.be.eql(-2);+ });++ it('should autoscale correctly', async () => {+ const pool = new Apify.AutoscaledPool({+ minConcurrency: 1,+ maxConcurrency: 100,+ minFreeMemoryRatio: 0.1,+ runTaskFunction: () => Promise.resolve(),+ isFinishedFunction: () => Promise.resolve(false),+ isTaskReadyFunction: () => Promise.resolve(true),+ });+ const mock = sinon.mock(utils);++ // Should scale up.+ pool.concurrency = 1;+ pool.runningCount = 1;+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;+ mock.expects('getMemoryInfo')+ .once()+ .returns(Promise.resolve({ freeBytes: toBytes(9.99), totalBytes: toBytes(10) }));+ await pool._autoscale();+ expect(pool.concurrency).to.be.eql(1 + SCALE_UP_MAX_STEP);++ // Should not do anything.+ pool.concurrency = 10;+ pool.runningCount = 10;+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;+ mock.expects('getMemoryInfo')+ .once()+ .returns(Promise.resolve({ freeBytes: toBytes(1), totalBytes: toBytes(10) }));+ await pool._autoscale();+ expect(pool.concurrency).to.be.eql(10);++ // Should scale down.+ pool.concurrency = 10;+ pool.runningCount = 10;+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;+ mock.expects('getMemoryInfo')+ .once()+ .returns(Promise.resolve({ freeBytes: toBytes(0.9), totalBytes: toBytes(10) }));+ await pool._autoscale();+ expect(pool.concurrency).to.be.eql(9);++ mock.verify();+ mock.restore();+ });++ it('should throw when some of the promises throws', async () => {+ let counter = 0;+ const runTaskFunction = () => {+ counter++;++ if (counter > 100) return;++ if (counter === 100) {+ const err = new Error('some-error');++ return new Promise((resolve, reject) => setTimeout(reject(err), 10));+ }++ return delayPromise(10);+ };++ const pool = new Apify.AutoscaledPool({+ minConcurrency: 10,+ maxConcurrency: 10,+ runTaskFunction,+ isFinishedFunction: () => Promise.resolve(false),+ isTaskReadyFunction: () => Promise.resolve(true),+ });++ await expect(pool.run()).to.be.rejectedWith('some-error');+ });++ it('should throw when runTaskFunction throws', async () => {+ const runTaskFunction = () => {+ throw new Error('some-error');+ };++ const pool = new Apify.AutoscaledPool({+ minConcurrency: 10,+ maxConcurrency: 10,+ runTaskFunction,+ isFinishedFunction: () => Promise.resolve(false),+ isTaskReadyFunction: () => Promise.resolve(true),+ });++ await expect(pool.run()).to.be.rejectedWith('some-error');+ });++ it('should scale down when CPU is overloaded', async () => {+ const pool = new Apify.AutoscaledPool({+ minConcurrency: 1,+ maxConcurrency: 100,+ minFreeMemoryRatio: 0.1,+ runTaskFunction: () => Promise.resolve(),+ isFinishedFunction: () => Promise.resolve(false),+ isTaskReadyFunction: () => Promise.resolve(true),+ });+ const mock = sinon.mock(utils);++ // Should scale up.+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;+ pool.runningCount = pool.concurrency;+ mock.expects('getMemoryInfo')+ .once()+ .returns(Promise.resolve({ freeBytes: toBytes(9.99), totalBytes: toBytes(10) }));+ await pool._autoscale();+ expect(pool.concurrency).to.be.eql(1 + SCALE_UP_MAX_STEP);++ // Should scale up.+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;+ pool.runningCount = pool.concurrency;+ mock.expects('getMemoryInfo')+ .once()+ .returns(Promise.resolve({ freeBytes: toBytes(9.99), totalBytes: toBytes(10) }));+ await pool._autoscale();+ expect(pool.concurrency).to.be.eql(1 + (2 * SCALE_UP_MAX_STEP));++ // Emit CPU overloaded = true event.+ Apify.events.emit(ACTOR_EVENT_NAMES.CPU_INFO, { isCpuOverloaded: true });++ // Should scale up but because of CPU overloaded event it always scales down.+ pool.runningCount = pool.concurrency;+ for (let i = 1; i <= 5; i++) {+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;+ mock.expects('getMemoryInfo')+ .once()+ .returns(Promise.resolve({ freeBytes: toBytes(9.99), totalBytes: toBytes(10) }));+ await pool._autoscale(); //eslint-disable-line+ expect(pool.concurrency).to.be.eql((1 + (2 * SCALE_UP_MAX_STEP)) - i);+ }++ // Emit CPU overloaded = false event.+ Apify.events.emit(ACTOR_EVENT_NAMES.CPU_INFO, { isCpuOverloaded: false });++ // Should scale up again.+ pool.runningCount = pool.concurrency;+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;+ mock.expects('getMemoryInfo')+ .once()+ .returns(Promise.resolve({ freeBytes: toBytes(9.99), totalBytes: toBytes(10) }));+ await pool._autoscale();+ expect(pool.concurrency).to.be.eql((1 + (3 * SCALE_UP_MAX_STEP)) - 5);++ mock.verify();+ mock.restore();+ });++ it('should not handle tasks added after isFinishedFunction returned true', async () => {+ const tasks = [];+ const finished = [];++ let isFinished = false;++ // Start 3 tasks immediately.+ tasks.push(delayPromise(50).then(() => finished.push(0)));+ tasks.push(delayPromise(50).then(() => finished.push(1)));+ tasks.push(delayPromise(50).then(() => finished.push(2)));++ setTimeout(() => {+ isFinished = true;+ }, 200);++ // Add 2 tasks after 500ms.+ setTimeout(() => tasks.push(delayPromise(50).then(() => finished.push(3))), 300);+ setTimeout(() => tasks.push(delayPromise(50).then(() => finished.push(4))), 300);++ // Run the pool and close it after 3s.+ const pool = new Apify.AutoscaledPool({+ maybeRunIntervalMillis: 10,+ minConcurrency: 3,+ runTaskFunction: () => tasks.pop(),+ isFinishedFunction: () => Promise.resolve(isFinished),+ isTaskReadyFunction: () => Promise.resolve(true),+ });+ await pool.run();++ // Check finished tasks.+ expect(finished).to.be.eql([0, 1, 2]);+ });++ it('should not handle tasks added if isTaskReadyFunction returned true', async () => {+ const tasks = [];+ const finished = [];++ let isFinished = false;+ let isTaskReady = true;++ const getTask = (id) => {+ return () => delayPromise(50).then(() => finished.push(id));+ };++ // Start 3 tasks immediately.+ tasks.push(getTask(0));+ tasks.push(getTask(1));+ setTimeout(() => tasks.push(getTask(2)), 150);++ setTimeout(() => {+ isTaskReady = false;+ }, 200);++ // Add 2 tasks after 500ms.+ setTimeout(() => tasks.push(getTask(3)), 300);+ setTimeout(() => tasks.push(getTask(4)), 300);++ // Remove tasks and se isTaskReady=true to accept a new task.+ setTimeout(() => {+ tasks.pop();+ tasks.pop();+ isTaskReady = true;+ }, 390);++ // Add one more task.+ setTimeout(() => tasks.push(getTask(5)), 400);++ // Finish.+ setTimeout(() => {+ isFinished = true;+ }, 500);++ // Run the pool and close it after 3s.+ const pool = new Apify.AutoscaledPool({+ maybeRunIntervalMillis: 10,+ minConcurrency: 3,+ runTaskFunction: () => {+ const task = tasks.shift();++ if (!task) return;++ return task();+ },+ isFinishedFunction: () => Promise.resolve(isFinished),+ isTaskReadyFunction: () => Promise.resolve(isTaskReady),+ });+ await pool.run();++ // Check finished tasks.+ expect(finished).to.be.eql([0, 1, 2, 5]);+ });++ /**+ * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!+ *+ * All the tests below use workerFunction and opts.finishWhenEmpty+ * to test that new version of AutoscaledPool is backwards compatible.+ *+ * TODO: after we release v1.0.0 we can remove this compatibility.+ *+ * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!+ */+++ it('[DEPRECATED] should work with concurrency 1', async () => {+ const range = _.range(0, 10);+ const result = [];+const workerFunction = () => {if (range.length === 0) return;@@ -32,15 +380,14 @@ describe('AutoscaledPool', () => {maxConcurrency: 1,workerFunction,});-const startedAt = Date.now();await pool.run();- expect(Date.now() - startedAt).to.be.within(100, 150);expect(result).to.be.eql(_.range(0, 10));+ expect(Date.now() - startedAt).to.be.within(100, 150);});- it('should work with concurrency 10', async () => {+ it('[DEPRECATED] should work with concurrency 10', async () => {const range = _.range(0, 100);const result = [];@@ -63,11 +410,11 @@ describe('AutoscaledPool', () => {const startedAt = Date.now();await pool.run();- expect(Date.now() - startedAt).to.be.within(100, 150);expect(result).to.be.eql(_.range(0, 100));+ expect(Date.now() - startedAt).to.be.within(100, 150);});- it('correctly computed space for instances using', async () => {+ it('[DEPRECATED] correctly computed space for instances using', async () => {const pool = new Apify.AutoscaledPool({minConcurrency: 1,maxConcurrency: 100,@@ -97,7 +444,7 @@ describe('AutoscaledPool', () => {expect(hasSpaceForInstances2).to.be.eql(-2);});- it('should autoscale correctly', async () => {+ it('[DEPRECATED] should autoscale correctly', async () => {const pool = new Apify.AutoscaledPool({minConcurrency: 1,maxConcurrency: 100,@@ -108,6 +455,7 @@ describe('AutoscaledPool', () => {// Should scale up.pool.concurrency = 1;+ pool.runningCount = 1;pool.intervalCounter = LOG_INFO_INTERVAL - 1;mock.expects('getMemoryInfo').once()@@ -117,6 +465,7 @@ describe('AutoscaledPool', () => {// Should not do anything.pool.concurrency = 10;+ pool.runningCount = 10;pool.intervalCounter = LOG_INFO_INTERVAL - 1;mock.expects('getMemoryInfo').once()@@ -126,6 +475,7 @@ describe('AutoscaledPool', () => {// Should scale down.pool.concurrency = 10;+ pool.runningCount = 10;pool.intervalCounter = LOG_INFO_INTERVAL - 1;mock.expects('getMemoryInfo').once()@@ -137,7 +487,7 @@ describe('AutoscaledPool', () => {mock.restore();});- it('should throw when some of the promises throws', async () => {+ it('[DEPRECATED] should throw when some of the promises throws', async () => {let counter = 0;const workerFunction = () => {counter++;@@ -162,7 +512,7 @@ describe('AutoscaledPool', () => {await expect(pool.run()).to.be.rejectedWith('some-error');});- it('should scale down when CPU is overloaded', async () => {+ it('[DEPRECATED] should scale down when CPU is overloaded', async () => {const pool = new Apify.AutoscaledPool({minConcurrency: 1,maxConcurrency: 100,@@ -173,6 +523,7 @@ describe('AutoscaledPool', () => {// Should scale up.pool.intervalCounter = LOG_INFO_INTERVAL - 1;+ pool.runningCount = pool.concurrency;mock.expects('getMemoryInfo').once().returns(Promise.resolve({ freeBytes: toBytes(9.99), totalBytes: toBytes(10) }));@@ -181,6 +532,7 @@ describe('AutoscaledPool', () => {// Should scale up.pool.intervalCounter = LOG_INFO_INTERVAL - 1;+ pool.runningCount = pool.concurrency;mock.expects('getMemoryInfo').once().returns(Promise.resolve({ freeBytes: toBytes(9.99), totalBytes: toBytes(10) }));@@ -191,6 +543,7 @@ describe('AutoscaledPool', () => {Apify.events.emit(ACTOR_EVENT_NAMES.CPU_INFO, { isCpuOverloaded: true });// Should scale up but because of CPU overloaded event it always scales down.+ pool.runningCount = pool.concurrency;for (let i = 1; i <= 5; i++) {pool.intervalCounter = LOG_INFO_INTERVAL - 1;mock.expects('getMemoryInfo')@@ -204,6 +557,7 @@ describe('AutoscaledPool', () => {Apify.events.emit(ACTOR_EVENT_NAMES.CPU_INFO, { isCpuOverloaded: false });// Should scale up again.+ pool.runningCount = pool.concurrency;pool.intervalCounter = LOG_INFO_INTERVAL - 1;mock.expects('getMemoryInfo').once()@@ -215,7 +569,7 @@ describe('AutoscaledPool', () => {mock.restore();});- it('should not handle tasks added later when opts.finishWhenEmpty is not used', async () => {+ it('[DEPRECATED] should not handle tasks added later when opts.finishWhenEmpty is not used', async () => {const tasks = [];const finished = [];@@ -240,7 +594,7 @@ describe('AutoscaledPool', () => {expect(finished).to.be.eql([0, 1, 2]);});- it('should be possible let pool running forever with opts.finishWhenEmpty=false', async () => {+ it('[DEPRECATED] should be possible let pool running forever with opts.finishWhenEmpty=false', async () => {const tasks = [];const finished = [];diff --git a/test/basic_crawler.js b/test/basic_crawler.jsindex 35321d496..808765d65 100644--- a/test/basic_crawler.js+++ b/test/basic_crawler.js@@ -4,6 +4,7 @@ import _ from 'underscore';import 'babel-polyfill';import { delayPromise } from 'apify-shared/utilities';import * as Apify from '../build/index';+import { RequestQueue } from '../build/request_queue';chai.use(chaiAsPromised);@@ -31,8 +32,8 @@ describe('BasicCrawler', () => {expect(processed).to.be.eql(sources);expect(Date.now() - startedAt).to.be.within(200, 400);- expect(requestList.isFinished()).to.be.eql(true);- expect(requestList.isEmpty()).to.be.eql(true);+ expect(await requestList.isFinished()).to.be.eql(true);+ expect(await requestList.isEmpty()).to.be.eql(true);});it('retries failed requests', async () => {@@ -77,8 +78,8 @@ describe('BasicCrawler', () => {expect(processed['http://example.com/2'].errorMessages).to.have.lengthOf(11);expect(processed['http://example.com/2'].retryCount).to.be.eql(10);- expect(requestList.isFinished()).to.be.eql(true);- expect(requestList.isEmpty()).to.be.eql(true);+ expect(await requestList.isFinished()).to.be.eql(true);+ expect(await requestList.isEmpty()).to.be.eql(true);});it('should allow to handle failed requests', async () => {@@ -119,8 +120,65 @@ describe('BasicCrawler', () => {expect(failed['http://example.com/3'].retryCount).to.be.eql(3);expect(_.values(failed)).to.have.length.of(3);expect(_.values(processed)).to.have.length.of(0);- expect(requestList.isFinished()).to.be.eql(true);- expect(requestList.isEmpty()).to.be.eql(true);+ expect(await requestList.isFinished()).to.be.eql(true);+ expect(await requestList.isEmpty()).to.be.eql(true);errors.forEach(error => expect(error).to.be.an('error'));});++ it('should require at least one of RequestQueue and RequestList', () => {+ const requestList = new Apify.RequestList({ sources: [] });+ const requestQueue = new RequestQueue('xxx');+ const handleRequestFunction = () => {};++ expect(() => new Apify.BasicCrawler({ handleRequestFunction })).to.throw();+ expect(() => new Apify.BasicCrawler({ handleRequestFunction, requestList })).to.not.throw();+ expect(() => new Apify.BasicCrawler({ handleRequestFunction, requestQueue })).to.not.throw();+ expect(() => new Apify.BasicCrawler({ handleRequestFunction, requestQueue, requestList })).to.not.throw();+ });++ it('should fetch first from RequestList and then continue with RequestQueue', async () => {+ const sources = [+ { url: 'http://example.com/1' },+ { url: 'http://example.com/2' },+ { url: 'http://example.com/3' },+ ];+ const processed = {};+ const requestList = new Apify.RequestList({ sources });++ const handleRequestFunction = async ({ request }) => {+ await delayPromise(10);+ processed[request.url] = request;++ if (request.url === 'http://example.com/2') {+ throw Error(`This is ${request.retryCount}th error!`);+ }++ request.userData.foo = 'bar';+ };++ const basicCrawler = new Apify.BasicCrawler({+ requestList,+ maxRequestRetries: 10,+ minConcurrency: 3,+ maxConcurrency: 3,+ handleRequestFunction,+ });++ await requestList.initialize();+ await basicCrawler.run();++ expect(processed['http://example.com/1'].userData.foo).to.be.eql('bar');+ expect(processed['http://example.com/1'].errorMessages).to.be.a('null');+ expect(processed['http://example.com/1'].retryCount).to.be.eql(0);+ expect(processed['http://example.com/3'].userData.foo).to.be.eql('bar');+ expect(processed['http://example.com/3'].errorMessages).to.be.a('null');+ expect(processed['http://example.com/3'].retryCount).to.be.eql(0);++ expect(processed['http://example.com/2'].userData.foo).to.be.a('undefined');+ expect(processed['http://example.com/2'].errorMessages).to.have.lengthOf(11);+ expect(processed['http://example.com/2'].retryCount).to.be.eql(10);++ expect(await requestList.isFinished()).to.be.eql(true);+ expect(await requestList.isEmpty()).to.be.eql(true);+ });});diff --git a/test/request_list.js b/test/request_list.jsindex 8c4a4803e..6ab7c74d6 100644--- a/test/request_list.js+++ b/test/request_list.js@@ -1,9 +1,12 @@import 'babel-polyfill';-import { expect } from 'chai';+import chai, { expect } from 'chai';+import chaiAsPromised from 'chai-as-promised';import request from 'request-promise';import sinon from 'sinon';import Apify from '../build/index';+chai.use(chaiAsPromised);+describe('Apify.RequestList', () => {it('should not accept to pages with same uniqueKey', async () => {const requestList = new Apify.RequestList({@@ -15,41 +18,41 @@ describe('Apify.RequestList', () => {await requestList.initialize();- expect(requestList.isEmpty()).to.be.eql(false);+ expect(await requestList.isEmpty()).to.be.eql(false);- const req = requestList.fetchNextRequest();+ const req = await requestList.fetchNextRequest();expect(req.url).to.be.eql('https://example.com/1');- expect(requestList.isEmpty()).to.be.eql(true);- expect(requestList.isFinished()).to.be.eql(false);- expect(requestList.fetchNextRequest()).to.be.eql(null);+ expect(await requestList.isEmpty()).to.be.eql(true);+ expect(await requestList.isFinished()).to.be.eql(false);+ expect(await requestList.fetchNextRequest()).to.be.eql(null);- requestList.markRequestHandled(req);+ await requestList.markRequestHandled(req);- expect(requestList.isEmpty()).to.be.eql(true);- expect(requestList.isFinished()).to.be.eql(true);+ expect(await requestList.isEmpty()).to.be.eql(true);+ expect(await requestList.isFinished()).to.be.eql(true);});it('must be initialized before using any of the methods', async () => {const requestList = new Apify.RequestList({ sources: [{ url: 'https://example.com' }] });const requestObj = new Apify.Request({ url: 'https://example.com' });- expect(() => requestList.isEmpty()).to.throw();- expect(() => requestList.isFinished()).to.throw();+ await expect(requestList.isEmpty()).to.be.rejectedWith();+ await expect(requestList.isFinished()).to.be.rejectedWith();expect(() => requestList.getState()).to.throw();- expect(() => requestList.markRequestHandled(requestObj)).to.throw();- expect(() => requestList.reclaimRequest(requestObj)).to.throw();- expect(() => requestList.fetchNextRequest()).to.throw();+ await expect(requestList.markRequestHandled(requestObj)).to.be.rejectedWith();+ await expect(requestList.reclaimRequest(requestObj)).to.be.rejectedWith();+ await expect(requestList.fetchNextRequest()).to.be.rejectedWith();await requestList.initialize();- expect(() => requestList.isEmpty()).to.not.throw();- expect(() => requestList.isFinished()).to.not.throw();+ await expect(requestList.isEmpty()).to.not.be.rejectedWith();+ await expect(requestList.isFinished()).to.not.be.rejectedWith();expect(() => requestList.getState()).to.not.throw();- expect(() => requestList.fetchNextRequest()).to.not.throw();- expect(() => requestList.reclaimRequest(requestObj)).to.not.throw();- expect(() => requestList.fetchNextRequest()).to.not.throw();- expect(() => requestList.markRequestHandled(requestObj)).to.not.throw();+ await expect(requestList.fetchNextRequest()).to.not.be.rejectedWith();+ await expect(requestList.reclaimRequest(requestObj)).to.not.be.rejectedWith();+ await expect(requestList.fetchNextRequest()).to.not.be.rejectedWith();+ await expect(requestList.markRequestHandled(requestObj)).to.not.be.rejectedWith();});it('should correctly initialized itself', async () => {@@ -67,17 +70,17 @@ describe('Apify.RequestList', () => {const originalList = new Apify.RequestList({ sources });await originalList.initialize();- const r1 = originalList.fetchNextRequest(); // 1- const r2 = originalList.fetchNextRequest(); // 2- originalList.fetchNextRequest(); // 3- const r4 = originalList.fetchNextRequest(); // 4- const r5 = originalList.fetchNextRequest(); // 5- originalList.fetchNextRequest(); // 6+ const r1 = await originalList.fetchNextRequest(); // 1+ const r2 = await originalList.fetchNextRequest(); // 2+ await originalList.fetchNextRequest(); // 3+ const r4 = await originalList.fetchNextRequest(); // 4+ const r5 = await originalList.fetchNextRequest(); // 5+ await originalList.fetchNextRequest(); // 6- originalList.markRequestHandled(r1);- originalList.markRequestHandled(r2);- originalList.markRequestHandled(r4);- originalList.reclaimRequest(r5);+ await originalList.markRequestHandled(r1);+ await originalList.markRequestHandled(r2);+ await originalList.markRequestHandled(r4);+ await originalList.reclaimRequest(r5);const newList = new Apify.RequestList({sources,@@ -85,13 +88,13 @@ describe('Apify.RequestList', () => {});await newList.initialize();- expect(newList.isEmpty()).to.be.eql(false);- expect(newList.fetchNextRequest().url).to.be.eql('https://example.com/3');- expect(newList.fetchNextRequest().url).to.be.eql('https://example.com/5');- expect(newList.fetchNextRequest().url).to.be.eql('https://example.com/6');- expect(newList.fetchNextRequest().url).to.be.eql('https://example.com/7');- expect(newList.fetchNextRequest().url).to.be.eql('https://example.com/8');- expect(newList.isEmpty()).to.be.eql(true);+ expect(await newList.isEmpty()).to.be.eql(false);+ expect((await newList.fetchNextRequest()).url).to.be.eql('https://example.com/3');+ expect((await newList.fetchNextRequest()).url).to.be.eql('https://example.com/5');+ expect((await newList.fetchNextRequest()).url).to.be.eql('https://example.com/6');+ expect((await newList.fetchNextRequest()).url).to.be.eql('https://example.com/7');+ expect((await newList.fetchNextRequest()).url).to.be.eql('https://example.com/8');+ expect(await newList.isEmpty()).to.be.eql(true);});it('should correctly load list from hosted files in correct order', async () => {@@ -125,11 +128,11 @@ describe('Apify.RequestList', () => {await requestList.initialize();- expect(requestList.fetchNextRequest()).to.include({ method: 'GET', url: list1[0] });- expect(requestList.fetchNextRequest()).to.include({ method: 'GET', url: list1[1] });- expect(requestList.fetchNextRequest()).to.include({ method: 'GET', url: list1[2] });- expect(requestList.fetchNextRequest()).to.include({ method: 'POST', url: list2[0] });- expect(requestList.fetchNextRequest()).to.include({ method: 'POST', url: list2[1] });+ expect(await requestList.fetchNextRequest()).to.include({ method: 'GET', url: list1[0] });+ expect(await requestList.fetchNextRequest()).to.include({ method: 'GET', url: list1[1] });+ expect(await requestList.fetchNextRequest()).to.include({ method: 'GET', url: list1[2] });+ expect(await requestList.fetchNextRequest()).to.include({ method: 'POST', url: list2[0] });+ expect(await requestList.fetchNextRequest()).to.include({ method: 'POST', url: list2[1] });mock.verify();mock.restore();@@ -156,8 +159,8 @@ describe('Apify.RequestList', () => {await requestList.initialize();- expect(requestList.fetchNextRequest()).to.include({ method: 'GET', url: listArr[0] });- expect(requestList.fetchNextRequest()).to.include({ method: 'GET', url: listArr[1] });+ expect(await requestList.fetchNextRequest()).to.include({ method: 'GET', url: listArr[0] });+ expect(await requestList.fetchNextRequest()).to.include({ method: 'GET', url: listArr[1] });mock.verify();mock.restore();@@ -181,7 +184,7 @@ describe('Apify.RequestList', () => {await requestList.initialize();- expect(requestList.fetchNextRequest()).to.eql(null);+ expect(await requestList.fetchNextRequest()).to.eql(null);mock.verify();mock.restore();@@ -205,11 +208,11 @@ describe('Apify.RequestList', () => {// Fetch first 5 urls//- const request1 = requestList.fetchNextRequest();- const request2 = requestList.fetchNextRequest();- const request3 = requestList.fetchNextRequest();- const request4 = requestList.fetchNextRequest();- const request5 = requestList.fetchNextRequest();+ const request1 = await requestList.fetchNextRequest();+ const request2 = await requestList.fetchNextRequest();+ const request3 = await requestList.fetchNextRequest();+ const request4 = await requestList.fetchNextRequest();+ const request5 = await requestList.fetchNextRequest();expect(request1.url).to.be.eql('https://example.com/1');expect(request2.url).to.be.eql('https://example.com/2');@@ -227,8 +230,8 @@ describe('Apify.RequestList', () => {nextIndex: 5,nextUniqueKey: 'https://example.com/6',});- expect(requestList.isEmpty()).to.be.eql(false);- expect(requestList.isFinished()).to.be.eql(false);+ expect(await requestList.isEmpty()).to.be.eql(false);+ expect(await requestList.isFinished()).to.be.eql(false);expect(requestList.inProgress).to.include(requestList.reclaimed);//@@ -236,10 +239,10 @@ describe('Apify.RequestList', () => {// Reclaim 3rd 4th//- requestList.markRequestHandled(request1);- requestList.markRequestHandled(request2);- requestList.reclaimRequest(request3);- requestList.reclaimRequest(request4);+ await requestList.markRequestHandled(request1);+ await requestList.markRequestHandled(request2);+ await requestList.reclaimRequest(request3);+ await requestList.reclaimRequest(request4);expect(requestList.getState()).to.be.eql({inProgress: {@@ -250,15 +253,15 @@ describe('Apify.RequestList', () => {nextIndex: 5,nextUniqueKey: 'https://example.com/6',});- expect(requestList.isEmpty()).to.be.eql(false);- expect(requestList.isFinished()).to.be.eql(false);+ expect(await requestList.isEmpty()).to.be.eql(false);+ expect(await requestList.isFinished()).to.be.eql(false);expect(requestList.inProgress).to.include(requestList.reclaimed);//// Mark 5th handled//- requestList.markRequestHandled(request5);+ await requestList.markRequestHandled(request5);expect(requestList.getState()).to.be.eql({inProgress: {@@ -268,8 +271,8 @@ describe('Apify.RequestList', () => {nextIndex: 5,nextUniqueKey: 'https://example.com/6',});- expect(requestList.isEmpty()).to.be.eql(false);- expect(requestList.isFinished()).to.be.eql(false);+ expect(await requestList.isEmpty()).to.be.eql(false);+ expect(await requestList.isFinished()).to.be.eql(false);expect(requestList.inProgress).to.include(requestList.reclaimed);//@@ -277,11 +280,11 @@ describe('Apify.RequestList', () => {// Mark 4th handled//- const reclaimed3 = requestList.fetchNextRequest();+ const reclaimed3 = await requestList.fetchNextRequest();expect(reclaimed3.url).to.be.eql('https://example.com/3');- const reclaimed4 = requestList.fetchNextRequest();+ const reclaimed4 = await requestList.fetchNextRequest();expect(reclaimed4.url).to.be.eql('https://example.com/4');- requestList.markRequestHandled(request4);+ await requestList.markRequestHandled(request4);expect(requestList.getState()).to.be.eql({inProgress: {@@ -290,33 +293,33 @@ describe('Apify.RequestList', () => {nextIndex: 5,nextUniqueKey: 'https://example.com/6',});- expect(requestList.isEmpty()).to.be.eql(false);- expect(requestList.isFinished()).to.be.eql(false);+ expect(await requestList.isEmpty()).to.be.eql(false);+ expect(await requestList.isFinished()).to.be.eql(false);expect(requestList.inProgress).to.include(requestList.reclaimed);//// Mark 3rd handled//- requestList.markRequestHandled(request3);+ await requestList.markRequestHandled(request3);expect(requestList.getState()).to.be.eql({inProgress: {},nextIndex: 5,nextUniqueKey: 'https://example.com/6',});- expect(requestList.isEmpty()).to.be.eql(false);- expect(requestList.isFinished()).to.be.eql(false);+ expect(await requestList.isEmpty()).to.be.eql(false);+ expect(await requestList.isFinished()).to.be.eql(false);expect(requestList.inProgress).to.include(requestList.reclaimed);//// Fetch 6th//- const request6 = requestList.fetchNextRequest();+ const request6 = await requestList.fetchNextRequest();expect(request6.url).to.be.eql('https://example.com/6');- expect(requestList.fetchNextRequest()).to.be.eql(null);+ expect(await requestList.fetchNextRequest()).to.be.eql(null);expect(requestList.getState()).to.be.eql({inProgress: {'https://example.com/6': true,@@ -324,15 +327,15 @@ describe('Apify.RequestList', () => {nextIndex: 6,nextUniqueKey: null,});- expect(requestList.isEmpty()).to.be.eql(true);- expect(requestList.isFinished()).to.be.eql(false);+ expect(await requestList.isEmpty()).to.be.eql(true);+ expect(await requestList.isFinished()).to.be.eql(false);expect(requestList.inProgress).to.include(requestList.reclaimed);//// Reclaim 6th//- requestList.reclaimRequest(request6);+ await requestList.reclaimRequest(request6);expect(requestList.getState()).to.be.eql({inProgress: {@@ -341,15 +344,15 @@ describe('Apify.RequestList', () => {nextIndex: 6,nextUniqueKey: null,});- expect(requestList.isEmpty()).to.be.eql(false);- expect(requestList.isFinished()).to.be.eql(false);+ expect(await requestList.isEmpty()).to.be.eql(false);+ expect(await requestList.isFinished()).to.be.eql(false);expect(requestList.inProgress).to.include(requestList.reclaimed);//// Fetch 6th//- const reclaimed6 = requestList.fetchNextRequest();+ const reclaimed6 = await requestList.fetchNextRequest();expect(reclaimed6.url).to.be.eql('https://example.com/6');expect(requestList.getState()).to.be.eql({@@ -359,23 +362,23 @@ describe('Apify.RequestList', () => {nextIndex: 6,nextUniqueKey: null,});- expect(requestList.isEmpty()).to.be.eql(true);- expect(requestList.isFinished()).to.be.eql(false);+ expect(await requestList.isEmpty()).to.be.eql(true);+ expect(await requestList.isFinished()).to.be.eql(false);expect(requestList.inProgress).to.include(requestList.reclaimed);//// Mark 6th handled//- requestList.markRequestHandled(reclaimed6);+ await requestList.markRequestHandled(reclaimed6);expect(requestList.getState()).to.be.eql({inProgress: {},nextIndex: 6,nextUniqueKey: null,});- expect(requestList.isEmpty()).to.be.eql(true);- expect(requestList.isFinished()).to.be.eql(true);+ expect(await requestList.isEmpty()).to.be.eql(true);+ expect(await requestList.isFinished()).to.be.eql(true);expect(requestList.inProgress).to.include(requestList.reclaimed);});});diff --git a/test/request_queue.js b/test/request_queue.jsindex e552c0d8b..8b25f6cda 100644--- a/test/request_queue.js+++ b/test/request_queue.js@@ -66,6 +66,34 @@ describe('RequestQueue', () => {expect(await queue.isFinished()).to.be.eql(true);});+ it('supports forefront param in reclaimRequest()', async () => {+ const queue = new RequestQueueLocal('my-queue', LOCAL_EMULATION_DIR);++ await queue.addRequest(new Apify.Request({ url: 'http://example.com/first' }));+ await queue.addRequest(new Apify.Request({ url: 'http://example.com/middle' }));+ await queue.addRequest(new Apify.Request({ url: 'http://example.com/third' }));++ const request1ForFirstTime = await queue.fetchNextRequest();+ expect(request1ForFirstTime.url).to.be.eql('http://example.com/first');++ // Return it to the front.+ await queue.reclaimRequest(request1ForFirstTime, { forefront: true });++ const request1ForSecondTime = await queue.fetchNextRequest();+ expect(request1ForSecondTime.url).to.be.eql('http://example.com/first');++ // Now put it at the back.+ await queue.reclaimRequest(request1ForSecondTime);++ const request2 = await queue.fetchNextRequest();+ const request3 = await queue.fetchNextRequest();+ const request1 = await queue.fetchNextRequest();++ expect(request1.url).to.be.eql('http://example.com/first');+ expect(request2.url).to.be.eql('http://example.com/middle');+ expect(request3.url).to.be.eql('http://example.com/third');+ });+it('should get initialized from existing dir', async () => {const request1 = new Apify.Request({ url: 'http://example.com/first' });const request2 = new Apify.Request({ url: 'http://example.com/middle' });