Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Docusaurus (#205)

Ondra Urban5d9fd097011d
diff --git a/build-docs-bck/index.html b/build-docs-bck/index.html
new file mode 100644
index 000000000..fa6f5e1df
--- /dev/null
+++ b/build-docs-bck/index.html
@@ -0,0 +1,5030 @@
+<main id="main">
+ <div class="line">
+ <div class="wrap">
+ <div class="docs-wrap cf">
+ <nav class="docs-menu mt-none">
+ <div>
+ <ul><li><a href="#Apify">Apify</a><ul><li><a href="#module-Apify-events">events</a></li><li><a href="#module-Apify-client">client</a></li><li><a href="#module-Apify-openRequestQueue">openRequestQueue</a></li><li><a href="#module-Apify-call">call</a></li><li><a href="#module-Apify-getApifyProxyUrl">getApifyProxyUrl</a></li><li><a href="#module-Apify-getEnv">getEnv</a></li><li><a href="#module-Apify-main">main</a></li><li><a href="#module-Apify-getMemoryInfo">getMemoryInfo</a></li><li><a href="#module-Apify-getValue">getValue</a></li><li><a href="#module-Apify-isAtHome">isAtHome</a></li><li><a href="#module-Apify-isDocker">isDocker</a></li><li><a href="#module-Apify-launchPuppeteer">launchPuppeteer</a></li><li><a href="#module-Apify-launchWebDriver">launchWebDriver</a></li><li><a href="#module-Apify-openDataset">openDataset</a></li><li><a href="#module-Apify-openKeyValueStore">openKeyValueStore</a></li><li><a href="#module-Apify-pushData">pushData</a></li><li><a href="#module-Apify-setValue">setValue</a></li></ul></li><li><a href="#AutoscaledPool">AutoscaledPool</a><ul><li><a href="#AutoscaledPool-abort">abort</a></li><li><a href="#AutoscaledPool-run">run</a></li></ul></li><li><a href="#BasicCrawler">BasicCrawler</a><ul><li><a href="#BasicCrawler-abort">abort</a></li><li><a href="#BasicCrawler-run">run</a></li></ul></li><li><a href="#CheerioCrawler">CheerioCrawler</a><ul><li><a href="#CheerioCrawler-abort">abort</a></li><li><a href="#CheerioCrawler-run">run</a></li></ul></li><li><a href="#Dataset">Dataset</a><ul><li><a href="#Dataset-delete">delete</a></li><li><a href="#Dataset-forEach">forEach</a></li><li><a href="#Dataset-getData">getData</a></li><li><a href="#Dataset-getInfo">getInfo</a></li><li><a href="#Dataset-map">map</a></li><li><a href="#Dataset-pushData">pushData</a></li><li><a href="#Dataset-reduce">reduce</a></li></ul></li><li><a href="#KeyValueStore">KeyValueStore</a><ul><li><a href="#KeyValueStore-delete">delete</a></li><li><a href="#KeyValueStore-getValue">getValue</a></li><li><a href="#KeyValueStore-setValue">setValue</a></li></ul></li><li><a href="#PseudoUrl">PseudoUrl</a><ul><li><a href="#PseudoUrl-createRequest">createRequest</a></li><li><a href="#PseudoUrl-matches">matches</a></li></ul></li><li><a href="#PuppeteerCrawler">PuppeteerCrawler</a><ul><li><a href="#PuppeteerCrawler-abort">abort</a></li><li><a href="#PuppeteerCrawler-run">run</a></li></ul></li><li><a href="#PuppeteerPool">PuppeteerPool</a><ul><li><a href="#PuppeteerPool-destroy">destroy</a></li><li><a href="#PuppeteerPool-newPage">newPage</a></li><li><a href="#PuppeteerPool-retire">retire</a></li></ul></li><li><a href="#Request">Request</a><ul><li><a href="#Request-pushErrorMessage">pushErrorMessage</a></li></ul></li><li><a href="#RequestList">RequestList</a><ul><li><a href="#RequestList-fetchNextRequest">fetchNextRequest</a></li><li><a href="#RequestList-getState">getState</a></li><li><a href="#RequestList-initialize">initialize</a></li><li><a href="#RequestList-isEmpty">isEmpty</a></li><li><a href="#RequestList-isFinished">isFinished</a></li><li><a href="#RequestList-length">length</a></li><li><a href="#RequestList-markRequestHandled">markRequestHandled</a></li><li><a href="#RequestList-reclaimRequest">reclaimRequest</a></li></ul></li><li><a href="#RequestQueue">RequestQueue</a><ul><li><a href="#RequestQueue-addRequest">addRequest</a></li><li><a href="#RequestQueue-delete">delete</a></li><li><a href="#RequestQueue-fetchNextRequest">fetchNextRequest</a></li><li><a href="#RequestQueue-getRequest">getRequest</a></li><li><a href="#RequestQueue-isEmpty">isEmpty</a></li><li><a href="#RequestQueue-isFinished">isFinished</a></li><li><a href="#RequestQueue-markRequestHandled">markRequestHandled</a></li><li><a href="#RequestQueue-reclaimRequest">reclaimRequest</a></li></ul></li><li><a href="#SettingsRotator">SettingsRotator</a><ul><li><a href="#SettingsRotator-fetchSettings">fetchSettings</a></li><li><a href="#SettingsRotator-reclaimSettings">reclaimSettings</a></li></ul></li><li><a href="#utils">utils</a><ul><li><a href="#utils-URL-NO-COMMAS-REGEX">URL_NO_COMMAS_REGEX</a></li><li><a href="#utils-URL-WITH-COMMAS-REGEX">URL_WITH_COMMAS_REGEX</a></li><li><a href="#utils-downloadListOfUrls">downloadListOfUrls</a></li><li><a href="#utils-extractUrls">extractUrls</a></li><li><a href="#utils-getRandomUserAgent">getRandomUserAgent</a></li><li><a href="#utils-sleep">sleep</a></li></ul></li><li><a href="#utils-puppeteer">puppeteer</a><ul><li><a href="#utils-puppeteer-blockResources">blockResources</a></li><li><a href="#utils-puppeteer-cacheResponses">cacheResponses</a></li><li><a href="#utils-puppeteer-compileScript">compileScript</a></li><li><a href="#utils-puppeteer-enqueueLinks">enqueueLinks</a></li><li><a href="#utils-puppeteer-hideWebDriver">hideWebDriver</a></li><li><a href="#utils-puppeteer-injectFile">injectFile</a></li><li><a href="#utils-puppeteer-injectJQuery">injectJQuery</a></li><li><a href="#utils-puppeteer-injectUnderscore">injectUnderscore</a></li></ul></li><li><a href="#Globals">Globals</a><ul><li><a href="#ActorRun">ActorRun</a></li><li><a href="#ApifyCallError">ApifyCallError</a></li><li><a href="#LaunchPuppeteerOptions">LaunchPuppeteerOptions</a></li><li><a href="#PaginationList">PaginationList</a></li><li><a href="#RequestOperationInfo">RequestOperationInfo</a></li></ul></li></ul>
+ </div>
+ </nav>
+ <div class="docs-content mleft">
+
+<div id="Apify">
+
+<header>
+<h1>The scalable web crawling and scraping library for JavaScript 0.8.14 <a href="https://www.apify.com/docs/sdk" target="_blank" rel="noopener noreferrer"><i class="fa fa-github"></i></a></h1>
+
+</header>
+
+<article>
+ <div class="container-overview">
+<div class="description"><p>{include-readme-1}
+{include-readme-2}
+{include-readme-3}</p>
+<h2>Programmer's reference</h2>
+
+<p>The following sections describe all functions and properties provided by the <code>apify</code> package.
+All of them are instance members exported directly by the main module.</p></div>
+<section id="Apify" class="sub-sec">
+</section>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Members (3)</h3>
+<div class="members">
+ <h4 class="name" id="module-Apify-events"><span class="type-signature">(static) </span>events<span class="type-signature"></span></h4>
+<div class="description">
+ <p>Gets an instance of Node.js' <a href="#https-nodejs-org-api-events-events-class-eventemitter">EventEmitter</a> class
+that emits various events from the SDK or the Apify platform.
+The event emitter is initialized by calling <a href="#module-Apify-main"><code>Apify.main()</code></a> function.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>Apify.main(async () => {
+ &nbsp;
+ Apify.events.on('cpuInfo', (data) => {
+ if (data.isCpuOverloaded) console.log('Oh no, the CPU is overloaded!');
+ });
+. &nbsp;
+});</code></pre><p>The following table shows all currently emitted events:</p>
+<table class="table table-bordered table-condensed">
+ <thead>
+ <tr>
+ <th>Event name</th>
+ <th>Data</th>
+ <th>Description</th>
+ </thead>
+ <tbody>
+ <tr>
+ <td><code>cpuInfo</code></td>
+ <td><code>{ &quot;isCpuOverloaded&quot;: Boolean }</code></td>
+ <td>
+ The event is emitted approximately every second
+ and it indicates whether the actor is using the maximum of available CPU resources.
+ If that's the case, the actor should not add more workload.
+ For example, this event is used by the <a href="#AutoscaledPool">AutoscaledPool</a> class.
+ </td>
+ </tr>
+ <tr>
+ <td><code>migrating</code></td>
+ <td>None</td>
+ <td>
+ Emitted when the actor running on the Apify platform is going to be migrated to another worker server soon.
+ You can use it to persist the state of the actor and abort the run, to speed up migration.
+ For example, this is used by the <a href="#RequestList">RequestList</a> class.
+ </td>
+ </tr>
+ <tr>
+ <td><code>persistState</code></td>
+ <td><code>{ &quot;isMigrating&quot;: Boolean }</code></td>
+ <td>
+ Emitted in regular intervals to notify all components of Apify SDK that it is time to persist
+ their state, in order to avoid repeating all work when the actor restarts.
+ This event is automatically emitted together with the <code>migrating</code> event,
+ in which case the <code>isMigrating</code> flag is set to <code>true</code>. Otherwise the flag is <code>false</code>.
+ </td>
+ </tr>
+ </tbody>
+</table>
+ </div>
+</div>
+<div class="members">
+ <h4 class="name" id="module-Apify-client"><span class="type-signature"></span>client<span class="type-signature"></span></h4>
+<div class="description">
+ <p>Gets the default instance of the <code>ApifyClient</code> class provided
+by the <a href="https://www.apify.com/docs/sdk/apify-client-js/latest" target="_blank">apify-client</a> NPM package.
+The instance is created automatically by the Apify SDK
+and it is configured using the <code>APIFY_API_BASE_URL</code>, <code>APIFY_USER_ID</code> and <code>APIFY_TOKEN</code>
+environment variables.</p>
+<p>The instance is used for all underlying calls to the Apify API
+in functions such as <a href="#module-Apify-getValue">Apify.getValue()</a>
+or <a href="#module-Apify-call">Apify.call()</a>.
+The settings of the client can be globally altered by calling the
+<a href="https://www.apify.com/docs/sdk/apify-client-js/latest#ApifyClient-setOptions"><code>Apify.client.setOptions()</code></a> function.
+Beware that altering these settings might have unintended effects on the entire Apify SDK package.</p>
+ </div>
+</div>
+<div class="members">
+ <h4 class="name" id="module-Apify-openRequestQueue"><span class="type-signature"></span>openRequestQueue<span class="type-signature"></span></h4>
+<div class="description">
+ <p>Opens a request queue and returns a promise resolving to an instance
+of the <a href="#RequestQueue"><code>RequestQueue</code></a> class.</p>
+<p><code>RequestQueue</code> represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
+The queue is used for deep crawling of websites, where you start with several URLs and then
+recursively follow links to other pages. The data structure supports both breadth-first
+and depth-first crawling orders.</p>
+<p>For more details and code examples, see the <a href="#RequestQueue"><code>RequestQueue</code></a> class.</p>
+ </div>
+</div>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Methods (14)</h3>
+<section id="module-Apify-call" class="sub-sec">
+<h4>call<span class="signature">(actId, input<span class="signature-attributes">opt</span>, options<span class="signature-attributes">opt</span>)</span><span class="type-signature"> &rarr; {Promise.&lt;<a href="#ActorRun">ActorRun</a>&gt;}</span></h4>
+<div class="description">
+ <p>Runs an actor on the Apify platform using the current user account (determined by the <code>APIFY_TOKEN</code> environment variable),
+waits for the actor to finish and fetches its output.</p>
+<p>By passing the <code>waitSecs</code> option you can reduce the maximum amount of time to wait for the run to finish.
+If the value is less than or equal to zero, the function returns immediately after the run is started.</p>
+<p>The result of the function is an <a href="#ActorRun"><code>ActorRun</code></a> object
+that contains details about the actor run and its output (if any).
+If the actor run failed, the function fails with <a href="#ApifyCallError"><code>ApifyCallError</code></a> exception.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>const run = await Apify.call('apify/hello-world', { myInput: 123 });
+console.log(`Received message: ${run.output.body.message}`);</code></pre><p>Internally, the <code>call()</code> function calls the
+<a href="https://www.apify.com/docs/api/v2#/reference/actors/run-collection/run-actor" target="_blank">Run actor</a>
+Apify API endpoint and few others to obtain the output.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>actId</code></b>
+(
+<span class="param-type">String</span>
+
+)
+-
+Either <code>username/actor-name</code> or actor ID.
+</li>
+<li>
+<b><code>input</code></b>
+(
+<span class="param-type">Object</span>
+|
+
+<span class="param-type">String</span>
+|
+
+<span class="param-type">Buffer</span>
+
+)
+&lt;optional>
+-
+Input for the actor. If it is an object, it will be stringified to
+ JSON and its content type is set to <code>application/json; charset=utf-8</code>.
+ Otherwise the <code>options.contentType</code> parameter must be provided.
+</li>
+<li>
+<b><code>options</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+-
+Object with settings
+<div class="params">
+<ul>
+<li>
+<b><code>contentType</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+Content type for the <code>input</code>. If not specified,
+ <code>input</code> is expected to be an object that will be stringified to JSON and content type set to
+ <code>application/json; charset=utf-8</code>. If <code>options.contentType</code> is specified, then <code>input</code> must be a
+ <code>String</code> or <code>Buffer</code>.
+</li>
+<li>
+<b><code>token</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+User API token that is used to run the actor. By default, it is taken from the <code>APIFY_TOKEN</code> environment variable.
+</li>
+<li>
+<b><code>memory</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Memory in megabytes which will be allocated for the new actor run.
+</li>
+<li>
+<b><code>build</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+Tag or number of the actor build to run (e.g. <code>beta</code> or <code>1.2.345</code>).
+ If not provided, the run uses build tag or number from the default actor run configuration (typically <code>latest</code>).
+</li>
+<li>
+<b><code>waitSecs</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+Maximum time to wait for the actor run to finish, in seconds.
+ If the limit is reached, the returned promise is resolved to a run object that will have
+ status <code>READY</code> or <code>RUNNING</code> and it will not contain the actor run output.
+ If <code>waitSecs</code> is null or undefined, the function waits for the actor to finish (default behavior).
+</li>
+<li>
+<b><code>fetchOutput</code></b>
+(
+<span class="param-type">Boolean</span>
+
+)
+&lt;optional>
+-
+If <code>false</code> then the function does not fetch output of the actor.
+Defaults to <code>true</code>.
+</li>
+<li>
+<b><code>disableBodyParser</code></b>
+(
+<span class="param-type">Boolean</span>
+
+)
+&lt;optional>
+-
+If <code>true</code> then the function will not attempt to parse the
+ actor's output and will return it in a raw <code>Buffer</code>.
+Defaults to <code>false</code>.
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<h5>Throws:</h5>
+<dl>
+ <dt>
+ <div class="param-desc">
+ <p>If the run did not succeed, e.g. if it failed or timed out.</p>
+ </div>
+ </dt>
+ <dd></dd>
+ <dt>
+ <dl>
+ <dt>
+ Type
+ </dt>
+ <dd>
+ (
+<span class="param-type"><a href="#ApifyCallError">ApifyCallError</a></span>
+
+)
+</dd>
+ </dl>
+ </dt>
+ <dd></dd>
+</dl>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;<a href="#ActorRun">ActorRun</a>&gt;</span>
+
+)
+- Returns a promise that resolves to an instance
+of <a href="#ActorRun"><code>ActorRun</code></a>. If the actor run fails, the promise is rejected
+with <a href="#ApifyCallError"><code>ApifyCallError</code></a>.
+
+</li>
+</section>
+<section id="module-Apify-getApifyProxyUrl" class="sub-sec">
+<h4>getApifyProxyUrl<span class="signature">(opts)</span><span class="type-signature"> &rarr; {String}</span></h4>
+<div class="description">
+ <p>Constructs the URL to the Apify Proxy using the specified settings.
+The proxy URL can be used from Apify actors, web browsers or any other HTTP
+proxy-enabled applications.</p>
+<p>For more information, see
+the <a href="https://my.apify.com/proxy">Apify Proxy</a> page in the app
+or the <a href="https://www.apify.com/docs/proxy">documentation</a>.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>opts</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+<div class="params">
+<ul>
+<li>
+<b><code>password</code></b>
+(
+<span class="param-type">String</span>
+
+)
+-
+User's password for the proxy.
+By default, it is taken from the <code>APIFY_PROXY_PASSWORD</code> environment variable,
+which is automatically set by the system when running the actors on the Apify cloud.
+</li>
+<li>
+<b><code>groups</code></b>
+(
+<span class="param-type">Array.&lt;String&gt;</span>
+
+)
+&lt;optional>
+-
+Array of Apify Proxy groups to be used.
+If not provided, the proxy will select the groups automatically.
+</li>
+<li>
+<b><code>session</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+Apify Proxy session identifier to be used by the Chrome browser.
+All HTTP requests going through the proxy with the same session identifier
+will use the same target proxy server (i.e. the same IP address).
+The identifier can only contain the following characters: <code>0-9</code>, <code>a-z</code>, <code>A-Z</code>, <code>&quot;.&quot;</code>, <code>&quot;_&quot;</code> and <code>&quot;~&quot;</code>.
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">String</span>
+
+)
+- Returns the proxy URL, e.g. <code>http://auto:[email protected]:8000</code>.
+
+</li>
+</section>
+<section id="module-Apify-getEnv" class="sub-sec">
+<h4>getEnv<span class="signature">()</span><span class="type-signature"> &rarr; {Object}</span></h4>
+<div class="description">
+ <p>Returns a new object which contains information parsed from the <code>APIFY_XXX</code> environment variables.
+It has the following properties:</p>
+<pre class="prettyprint source lang-javascript"><code>{
+ // ID of the actor (APIFY_ACT_ID)
+ actId: String,
+&nbsp;
+ // ID of the actor run (APIFY_ACT_RUN_ID)
+ actRunId: String,
+&nbsp;
+ // ID of the user who started the actor - note that it might be
+ // different than the owner of the actor (APIFY_USER_ID)
+ userId: String,
+&nbsp;
+ // Authentication token representing privileges given to the actor run,
+ // it can be passed to various Apify APIs (APIFY_TOKEN).
+ token: String,
+&nbsp;
+ // Date when the actor was started (APIFY_STARTED_AT)
+ startedAt: Date,
+&nbsp;
+ // Date when the actor will time out (APIFY_TIMEOUT_AT)
+ timeoutAt: Date,
+&nbsp;
+ // ID of the key-value store where input and output data of this
+ // actor is stored (APIFY_DEFAULT_KEY_VALUE_STORE_ID)
+ defaultKeyValueStoreId: String,
+&nbsp;
+ // ID of the dataset where input and output data of this
+ // actor is stored (APIFY_DEFAULT_DATASET_ID)
+ defaultDatasetId: String,
+&nbsp;
+ // Amount of memory allocated for the actor,
+ // in megabytes (APIFY_MEMORY_MBYTES)
+ memoryMbytes: Number,
+}</code></pre><p>For the list of the <code>APIFY_XXX</code> environment variables, see
+<a href="http://localhost/docs/actor.php#run-env-vars" target="_blank">Actor documentation</a>.
+If some of the variables are not defined or are invalid, the corresponding value in the resulting object will be null.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Object</span>
+
+)
+</li>
+</section>
+<section id="module-Apify-main" class="sub-sec">
+<h4>main<span class="signature">(userFunc)</span><span class="type-signature"></span></h4>
+<div class="description">
+ <p>Runs the main user function that performs the job of the actor.</p>
+<p><code>Apify.main()</code> is especially useful when you're running your code in an actor on the Apify platform.
+Note that its use is optional - the function is provided merely for your convenience.</p>
+<p>The function performs the following actions:</p>
+<ol>
+ <li>When running on the Apify platform (i.e. <code>APIFY_IS_AT_HOME</code> environment variable is set),
+ it sets up a connection to listen for platform events.
+ For example, to get a notification about an imminent migration to another server.
+ See <a href="#module-Apify-events"><code>Apify.events</code></a> for details.
+ </li>
+ <li>It checks that either <code>APIFY_TOKEN</code> or <code>APIFY_LOCAL_STORAGE_DIR</code> environment variable
+ is defined. If not, the functions sets <code>APIFY_LOCAL_STORAGE_DIR</code> to <code>./apify_storage</code>
+ inside the current working directory. This is to simplify running code examples.
+ </li>
+ <li>It invokes the user function passed as the <code>userFunc</code> parameter.</li>
+ <li>If the user function returned a promise, waits for it to resolve.</li>
+ <li>If the user function throws an exception or some other error is encountered,
+ prints error details to console so that they are stored to the log.</li>
+ <li>Exits the Node.js process, with zero exit code on success and non-zero on errors.</li>
+</ol>
+
+<p>The user function can be synchronous:</p>
+<pre class="prettyprint source lang-javascript"><code>Apify.main(() => {
+ // My synchronous function that returns immediately
+ console.log('Hello world from actor!');
+});</code></pre><p>If the user function returns a promise, it is considered asynchronous:</p>
+<pre class="prettyprint source lang-javascript"><code>const request = require('request-promise');
+
+Apify.main(() => {
+ // My asynchronous function that returns a promise
+ return request('http://www.example.com').then((html) => {
+ console.log(html);
+ });
+});</code></pre><p>To simplify your code, you can take advantage of the <code>async</code>/<code>await</code> keywords:</p>
+<pre class="prettyprint source lang-javascript"><code>const request = require('request-promise');
+
+Apify.main(async () => {
+ // My asynchronous function
+ const html = await request('http://www.example.com');
+ console.log(html);
+});</code></pre>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>userFunc</code></b>
+(
+<span class="param-type">function</span>
+
+)
+-
+User function to be executed. If it returns a promise,
+the promise will be awaited. The user function is called with no arguments.
+</li>
+</ul>
+</div>
+</section>
+<section id="module-Apify-getMemoryInfo" class="sub-sec">
+<h4>getMemoryInfo<span class="signature">()</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Returns memory statistics of the process and the system, which is an object with the following properties:</p>
+<pre class="prettyprint source lang-javascript"><code>{
+ // Total memory available in the system or container
+ totalBytes: Number,
+ &nbsp;
+ // Amount of free memory in the system or container
+ freeBytes: Number,
+ &nbsp;
+ // Amount of memory used (= totalBytes - freeBytes)
+ usedBytes: Number,
+ // Amount of memory used the current Node.js process
+ mainProcessBytes: Number,
+ // Amount of memory used by child processes of the current Node.js process
+ childProcessesBytes: Number,
+}</code></pre><p>If the process runs inside of Docker, the <code>getMemoryInfo</code> gets container memory limits,
+otherwise it gets system memory limits.</p>
+<p>Beware that the function is quite inefficient because it spawns a new process.
+Therefore you shouldn't call it too often, like more than once per second.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+- Returns a promise.
+
+</li>
+</section>
+<section id="module-Apify-getValue" class="sub-sec">
+<h4>getValue<span class="signature">(key)</span><span class="type-signature"> &rarr; {Promise.&lt;Object&gt;}</span></h4>
+<div class="description">
+ <p>Gets a value from the default <a href="#KeyValueStore"><code>KeyValueStore</code></a> associated with the current actor run.</p>
+<p>This is just a convenient shortcut for <a href="#KeyValueStore-getValue"><code>KeyValueStore.getValue()</code></a>.
+For example, calling the following code:</p>
+<pre class="prettyprint source lang-javascript"><code>const input = await Apify.getValue('INPUT');</code></pre><p>is equivalent to:</p>
+<pre class="prettyprint source lang-javascript"><code>const store = await Apify.openKeyValueStore();
+await store.getValue('INPUT');</code></pre><p>To store the value to the default-key value store, you can use the <code>Apify.setValue()</code> function.</p>
+<p>For more information, see <code>Apify.openKeyValueStore()</code>
+and <a href="#KeyValueStore-getValue"><code>KeyValueStore.getValue()</code></a>.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>key</code></b>
+(
+<span class="param-type">String</span>
+
+)
+-
+Unique record key.
+</li>
+</ul>
+</div>
+<dl class="details">
+ <dt class="tag-see">See:</dt>
+ <dd class="tag-see">
+ <ul>
+ <li><a href="#KeyValueStore">KeyValueStore</a></li>
+ </ul>
+ </dd>
+ </dl>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;Object&gt;</span>
+
+)
+- Returns a promise that resolves once the record is stored.
+
+</li>
+</section>
+<section id="module-Apify-isAtHome" class="sub-sec">
+<h4>isAtHome<span class="signature">()</span><span class="type-signature"> &rarr; {Boolean}</span></h4>
+<div class="description">
+ <p>Returns <code>true</code> when code is running on Apify platform and <code>false</code> otherwise (for example locally).</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Boolean</span>
+
+)
+</li>
+</section>
+<section id="module-Apify-isDocker" class="sub-sec">
+<h4>isDocker<span class="signature">()</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Returns promise that resolves to true if the code is running in a Docker container.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="module-Apify-launchPuppeteer" class="sub-sec">
+<h4>launchPuppeteer<span class="signature">(opts<span class="signature-attributes">opt</span>)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Launches headless Chrome using Puppeteer pre-configured to work within the Apify platform.
+The function has the same argument and the return value as <code>puppeteer.launch()</code>.
+See <a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#puppeteerlaunchoptions" target="_blank">
+Puppeteer documentation</a> for more details.</p>
+<p>The <code>launchPuppeteer()</code> function alters the following Puppeteer options:</p>
+<ul>
+ <li>
+ Passes the setting from the <code>APIFY_HEADLESS</code> environment variable to the <code>headless</code> option,
+ unless it was already defined by the caller or <code>APIFY_XVFB</code> environment variable is set to <code>1</code>.
+ Note that Apify Actor cloud platform automatically sets <code>APIFY_HEADLESS=1</code> to all running actors.
+ </li>
+ <li>
+ Takes the <code>proxyUrl</code> option, checks it and adds it to <code>args</code> as <code>--proxy-server=XXX</code>.
+ If the proxy uses authentication, the function sets up an anonymous proxy HTTP
+ to make the proxy work with headless Chrome. For more information, read the
+ <a href="https://blog.apify.com/how-to-make-headless-chrome-and-puppeteer-use-a-proxy-server-with-authentication-249a21a79212"
+ target="_blank">blog post about proxy-chain library</a>.
+ </li>
+ <li>
+ If <code>opts.useApifyProxy</code> is <code>true</code> then the function generates a URL of
+ <a href="https://www.apify.com/docs/proxy" target="_blank">Apify Proxy</a>
+ based on <code>opts.apifyProxyGroups</code> and <code>opts.apifyProxySession</code> and passes it as <code>opts.proxyUrl</code>.
+ </li>
+ <li>
+ The function adds <code>--no-sandbox</code> to <code>args</code> to enable running headless Chrome in a Docker container on the Apify platform.
+ </li>
+</ul>
+
+<p>To use this function, you need to have the <a href="https://www.npmjs.com/package/puppeteer" target="_blank">puppeteer</a>
+NPM package installed in your project.
+When running on the Apify cloud platform, you can achieve that simply
+by using the <code>apify/actor-node-chrome</code> base Docker image for your actor - see
+<a href="https://www.apify.com/docs/actor#base-images" target="_blank">Apify Actor documentation</a>
+for details.</p>
+<p>For an example of usage, see the <a href="https://www.apify.com/apify/example-puppeteer">apify/example-puppeteer</a> actor.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>opts</code></b>
+(
+<span class="param-type"><a href="#LaunchPuppeteerOptions">LaunchPuppeteerOptions</a></span>
+
+)
+&lt;optional>
+-
+Optional settings passed to <code>puppeteer.launch()</code>. Additionally the object can
+ contain the following fields:
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+- Promise object that resolves to Puppeteer's <code>Browser</code> instance.
+
+</li>
+</section>
+<section id="module-Apify-launchWebDriver" class="sub-sec">
+<h4>launchWebDriver<span class="signature">(opts<span class="signature-attributes">opt</span>)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Opens a new instance of Chrome web browser
+controlled by <a href="http://www.seleniumhq.org/projects/webdriver/" target="_blank">Selenium WebDriver</a>.
+The result of the function is the new instance of the
+<a href="#http-seleniumhq-github-io-selenium-docs-api-javascript-module-selenium-webdriver-index-exports-WebDriver" target="_blank">
+WebDriver</a>
+class.</p>
+<p>To use this function, you need to have Google Chrome and
+<a href="https://sites.google.com/a/chromium.org/chromedriver/" target="_blank">ChromeDriver</a> installed in your environment.
+For example, you can use the <code>apify/actor-node-chrome</code> base Docker image for your actor - see
+<a href="https://www.apify.com/docs/actor#base-images" target="_blank">documentation</a>
+for more details.</p>
+<p>For an example of usage, see the <a href="https://www.apify.com/apify/example-selenium">apify/example-selenium</a> actor.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>opts</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+-
+Optional settings passed to <code>puppeteer.launch()</code>. Additionally the object can contain the following fields:
+<div class="params">
+<ul>
+<li>
+<b><code>proxyUrl</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+URL to a proxy server. Currently only <code>http://</code> scheme is supported.
+Port number must be specified. For example, <code>http://example.com:1234</code>.
+</li>
+<li>
+<b><code>headless</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+Indicates that the browser will be started in headless mode.
+If the option is not defined, and the <code>APIFY_HEADLESS</code> environment variable has value <code>1</code>
+and <code>APIFY_XVFB</code> is NOT <code>1</code>, the value defaults to <code>true</code>, otherwise it will be <code>false</code>.
+</li>
+<li>
+<b><code>userAgent</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+User-Agent for the browser.
+If not provided, the function sets it to a reasonable default.
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="module-Apify-openDataset" class="sub-sec">
+<h4>openDataset<span class="signature">(datasetIdOrName<span class="signature-attributes">opt</span>)</span><span class="type-signature"> &rarr; {Promise.&lt;<a href="#Dataset">Dataset</a>&gt;}</span></h4>
+<div class="description">
+ <p>Opens a dataset and returns a promise resolving to an instance of the <a href="#Dataset"><code>Dataset</code></a> class.</p>
+<p>Datasets are used to store structured data where each object stored has the same attributes,
+such as online store products or real estate offers.
+The actual data is stored either on the local filesystem or in the cloud.</p>
+<p>For more details and code examples, see the <a href="#Dataset"><code>Dataset</code></a> class.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>datasetIdOrName</code></b>
+(
+<span class="param-type">string</span>
+
+)
+&lt;optional>
+-
+ID or name of the dataset to be opened. If <code>null</code> or <code>undefined</code>,
+ the function returns the default dataset associated with the actor run.
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;<a href="#Dataset">Dataset</a>&gt;</span>
+
+)
+- Returns a promise that resolves to an instance of the <code>Dataset</code> class.
+
+</li>
+</section>
+<section id="module-Apify-openKeyValueStore" class="sub-sec">
+<h4>openKeyValueStore<span class="signature">(storeIdOrName<span class="signature-attributes">opt</span>)</span><span class="type-signature"> &rarr; {Promise.&lt;<a href="#KeyValueStore">KeyValueStore</a>&gt;}</span></h4>
+<div class="description">
+ <p>Opens a key-value store and returns a promise resolving to an instance of the <a href="#KeyValueStore"><code>KeyValueStore</code></a> class.</p>
+<p>Key-value stores are used to store records or files, along with their MIME content type.
+The records are stored and retrieved using a unique key.
+The actual data is stored either on a local filesystem or in the Apify cloud.</p>
+<p>For more details and code examples, see the <a href="#KeyValueStore"><code>KeyValueStore</code></a> class.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>storeIdOrName</code></b>
+(
+<span class="param-type">string</span>
+
+)
+&lt;optional>
+-
+ID or name of the key-value store to be opened. If <code>null</code> or <code>undefined</code>,
+ the function returns the default key-value store associated with the actor run.
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;<a href="#KeyValueStore">KeyValueStore</a>&gt;</span>
+
+)
+- Returns a promise that resolves to an instance of the <code>KeyValueStore</code> class.
+
+</li>
+</section>
+<section id="module-Apify-pushData" class="sub-sec">
+<h4>pushData<span class="signature">(data)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Stores an object or an array of objects to the default <a href="#Dataset"><code>Dataset</code></a> of the current actor run.</p>
+<p>This is just a convenient shortcut for <a href="#Dataset-pushData"><code>Dataset.pushData()</code></a>.
+For example, calling the following code:</p>
+<pre class="prettyprint source lang-javascript"><code>await Apify.pushData({ myValue: 123 });</code></pre><p>is equivalent to:</p>
+<pre class="prettyprint source lang-javascript"><code>const dataset = await Apify.openDataset();
+await dataset.pushData({ myValue: 123 });</code></pre><p>For more information, see <code>Apify.openDataset()</code> and <a href="#Dataset-pushData"><code>Dataset.pushData()</code></a></p>
+<p><strong>IMPORTANT</strong>: Make sure to use the <code>await</code> keyword when calling <code>pushData()</code>,
+otherwise the actor process might finish before the data is stored!</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>data</code></b>
+(
+<span class="param-type">Object</span>
+|
+
+<span class="param-type">Array</span>
+
+)
+-
+Object or array of objects containing data to be stored in the default dataset.
+The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
+</li>
+</ul>
+</div>
+<dl class="details">
+ <dt class="tag-see">See:</dt>
+ <dd class="tag-see">
+ <ul>
+ <li><a href="#Dataset">Dataset</a></li>
+ </ul>
+ </dd>
+ </dl>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+- Returns a promise that resolves once the data is saved.
+
+</li>
+</section>
+<section id="module-Apify-setValue" class="sub-sec">
+<h4>setValue<span class="signature">(key, value, options<span class="signature-attributes">opt</span>)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Stores or deletes a value in the default <a href="#KeyValueStore"><code>KeyValueStore</code></a> associated with the current actor run.</p>
+<p>This is just a convenient shortcut for <a href="#KeyValueStore-setValue"><code>KeyValueStore.setValue()</code></a>.
+For example, calling the following code:</p>
+<pre class="prettyprint source lang-javascript"><code>await Apify.setValue('OUTPUT', { foo: &quot;bar&quot; });</code></pre><p>is equivalent to:</p>
+<pre class="prettyprint source lang-javascript"><code>const store = await Apify.openKeyValueStore();
+await store.setValue('OUTPUT', { foo: &quot;bar&quot; });</code></pre><p>To get a value from the default-key value store, you can use the <code>Apify.getValue()</code> function.</p>
+<p>For more information, see <code>Apify.openKeyValueStore()</code>
+and <a href="#KeyValueStore-setValue"><code>KeyValueStore.setValue()</code></a>.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>key</code></b>
+(
+<span class="param-type">String</span>
+
+)
+-
+Unique record key.
+</li>
+<li>
+<b><code>value</code></b>
+(
+<span class="param-type">Object</span>
+|
+
+<span class="param-type">String</span>
+|
+
+<span class="param-type">Buffer</span>
+
+)
+-
+Record data, which can be one of the following values:
+ <ul>
+ <li>If <code>null</code>, the record in the key-value store is deleted.</li>
+ <li>If no <code>options.contentType</code> is specified, <code>value</code> can be any JavaScript object and it will be stringified to JSON.</li>
+ <li>If <code>options.contentType</code> is specified, <code>value</code> is considered raw data and it must be a <code>String</code>
+ or <a href="#https-nodejs-org-api-buffer"><code>Buffer</code></a>.</li>
+ </ul>
+ For any other value an error will be thrown.
+</li>
+<li>
+<b><code>options</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+<div class="params">
+<ul>
+<li>
+<b><code>contentType</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+Specifies a custom MIME content type of the record.
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<dl class="details">
+ <dt class="tag-see">See:</dt>
+ <dd class="tag-see">
+ <ul>
+ <li><a href="#KeyValueStore">KeyValueStore</a></li>
+ </ul>
+ </dd>
+ </dl>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+- Returns a promise that resolves once the value is stored or deleted.
+
+</li>
+</section>
+</div>
+</article>
+
+</div>
+<div id="AutoscaledPool">
+
+<header>
+<h2>
+ AutoscaledPool
+ </h2>
+<div class="class-description"><p>Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
+The pool only starts new tasks if there is enough free CPU and memory available
+and the Javascript event loop is not blocked.</p>
+<p>The information about the CPU and memory usage is obtained by the <code>Snapshotter</code> class,
+which makes regular snapshots of system resources that may be either local
+or from the Apify cloud infrastructure in case the process is running on the Apify platform.
+Meaningful data gathered from these snapshots is provided to <code>AutoscaledPool</code> by the <code>SystemStatus</code> class.</p>
+<p>Before running the pool, you need to implement the following three functions:
+<code>runTaskFunction()</code>,
+<code>isTaskReadyFunction()</code> and
+<code>isFinishedFunction()</code>.</p>
+<p>The auto-scaled pool is started by calling the <a href="#AutoscaledPool-run"><code>run()</code></a> function.
+The pool periodically queries the <code>isTaskReadyFunction()</code> function
+for more tasks, managing optimal concurrency, until the function resolves to <code>false</code>. The pool then queries
+the <code>isFinishedFunction()</code>. If it resolves to <code>true</code>, the run finishes. If it resolves to <code>false</code>, it assumes
+there will be more tasks available later and keeps querying for tasks, until finally both the
+<code>isTaskReadyFunction()</code> and <code>isFinishedFunction()</code> functions resolve to <code>true</code>. If any of the tasks throws
+then the <code>run()</code> function rejects the promise with an error.</p>
+<p>The pool evaluates whether it should start a new task every time one of the tasks finishes
+and also in the interval set by the <code>options.maybeRunIntervalSecs</code> parameter.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>const pool = new Apify.AutoscaledPool({
+ maxConcurrency: 50,
+ runTaskFunction: async () => {
+ // Run some resource-intensive asynchronous operation here.
+ },
+ isTaskReadyFunction: async () => {
+ // Tell the pool whether more tasks are ready to be processed. (true / false)
+ },
+ isFinishedFunction: async () => {
+ // Tell the pool whether it should finish or wait for more tasks to become available. (true / false)
+ }
+});
+
+await pool.run();</code></pre></div>
+</header>
+
+<article>
+ <div class="container-overview">
+<section id="AutoscaledPool" class="sub-sec">
+<h2>Constructor</h2>
+<h4>new AutoscaledPool<span class="signature">(options)</span><span class="type-signature"></span></h4>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>options</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+<div class="params">
+<ul>
+<li>
+<b><code>runTaskFunction</code></b>
+(
+<span class="param-type">function</span>
+
+)
+-
+A function that performs an asynchronous resource-intensive task.
+ The function must either be labeled <code>async</code> or return a promise.
+</li>
+<li>
+<b><code>isTaskReadyFunction</code></b>
+(
+<span class="param-type">function</span>
+
+)
+-
+A function that indicates whether <code>runTaskFunction</code> should be called.
+ This function is called every time there is free capacity for a new task and it should
+ indicate whether it should start or not by resolving to either <code>true</code> or `false.
+ Besides its obvious use, it is also useful for task throttling to save resources.
+</li>
+<li>
+<b><code>isFinishedFunction</code></b>
+(
+<span class="param-type">function</span>
+
+)
+-
+A function that is called only when there are no tasks to be processed.
+ If it resolves to <code>true</code> then the pool's run finishes. Being called only
+ when there are no tasks being processed means that as long as <code>isTaskReadyFunction()</code>
+ keeps resolving to <code>true</code>, <code>isFinishedFunction()</code> will never be called.
+ To abort a run, use the <code>pool.abort()</code> method.
+</li>
+<li>
+<b><code>minConcurrency</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Minimum number of tasks running in parallel.
+Defaults to <code>1</code>.
+</li>
+<li>
+<b><code>maxConcurrency</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Maximum number of tasks running in parallel.
+Defaults to <code>1000</code>.
+</li>
+<li>
+<b><code>desiredConcurrencyRatio</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Minimum level of desired concurrency to reach before more scaling up is allowed.
+Defaults to <code>0.95</code>.
+</li>
+<li>
+<b><code>scaleUpStepRatio</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Defines the fractional amount of desired concurrency to be added with each scaling up.
+ The minimum scaling step is one.
+Defaults to <code>0.05</code>.
+</li>
+<li>
+<b><code>scaleDownStepRatio</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Defines the amount of desired concurrency to be subtracted with each scaling down.
+ The minimum scaling step is one.
+Defaults to <code>0.05</code>.
+</li>
+<li>
+<b><code>maybeRunIntervalSecs</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Indicates how often the pool should call the <code>runTaskFunction()</code> to start a new task, in seconds.
+ This has no effect on starting new tasks immediately after a task completes.
+Defaults to <code>0.5</code>.
+</li>
+<li>
+<b><code>loggingIntervalSecs</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Specifies a period in which the instance logs its state, in seconds.
+ Set to <code>null</code> to disable periodic logging.
+Defaults to <code>60</code>.
+</li>
+<li>
+<b><code>autoscaleIntervalSecs</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Defines in seconds how often the pool should attempt to adjust the desired concurrency
+ based on the latest system status. Setting it lower than 1 might have a severe impact on performance.
+ We suggest using a value from 5 to 20.
+Defaults to <code>10</code>.
+</li>
+<li>
+<b><code>snapshotterOptions</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Options to be passed down to the <code>Snapshotter</code> constructor. This is useful for fine-tuning
+ the snapshot intervals and history.
+ See <a href="https://github.com/apifytech/apify-js/blob/develop/src/autoscaling/snapshotter.js">Snapshotter</a> source code for more details.
+</li>
+<li>
+<b><code>systemStatusOptions</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Options to be passed down to the <code>SystemStatus</code> constructor. This is useful for fine-tuning
+ the system status reports. If a custom snapshotter is set in the options, it will be used
+ by the pool.
+ See <a href="https://github.com/apifytech/apify-js/blob/develop/src/autoscaling/system_status.js">SystemStatus</a> source code for more details.
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+</section>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Methods (2)</h3>
+<section id="AutoscaledPool-abort" class="sub-sec">
+<h4>abort<span class="signature">()</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Aborts the run of the auto-scaled pool, discards all currently running tasks and destroys it.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="AutoscaledPool-run" class="sub-sec">
+<h4>run<span class="signature">()</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Runs the auto-scaled pool. Returns a promise that gets resolved or rejected once
+all the tasks are finished or one of them fails.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+</div>
+</article>
+
+</div>
+<div id="BasicCrawler">
+
+<header>
+<h2>
+ BasicCrawler
+ </h2>
+<div class="class-description"><p>Provides a simple framework for the parallel crawling of web pages,
+whose URLs are fed either from a static list
+or from a dynamic queue of URLs.</p>
+<p><code>BasicCrawler</code> invokes the user-provided <code>handleRequestFunction</code> for each <a href="#Request"><code>Request</code></a>
+object, which corresponds to a single URL to crawl.
+The <code>Request</code> objects are fed from the <a href="#RequestList"><code>RequestList</code></a> or <a href="#RequestQueue"><code>RequestQueue</code></a>
+instances provided by the <code>requestList</code> or <code>requestQueue</code> constructor options, respectively.</p>
+<p>If both <code>requestList</code> and <code>requestQueue</code> is used, the instance first
+processes URLs from the <code>RequestList</code> and automatically enqueues all of them to <code>RequestQueue</code> before it starts
+their processing. This ensures that a single URL is not crawled multiple times.</p>
+<p>The crawler finishes if there are no more <code>Request</code> objects to crawl.</p>
+<p>New requests are only launched if there is enough free CPU and memory available,
+using the functionality provided by the <a href="#AutoscaledPool"><code>AutoscaledPool</code></a> class.
+All <code>AutoscaledPool</code> configuration options can be passed to the <code>autoscaledPoolOptions</code> parameter
+of the <code>CheerioCrawler</code> constructor.
+For user convenience, the <code>minConcurrency</code> and <code>maxConcurrency</code> options are available directly in the constructor.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>const rp = require('request-promise');
+
+// Prepare a list of URLs to crawl
+const requestList = new Apify.RequestList({
+ sources: [
+ { url: 'http://www.example.com/page-1' },
+ { url: 'http://www.example.com/page-2' },
+ ],
+});
+await requestList.initialize();
+
+// Crawl the URLs
+const crawler = new Apify.BasicCrawler({
+ requestList,
+ handleRequestFunction: async ({ request }) => {
+ // 'request' contains an instance of the Request class
+ // Here we simply fetch the HTML of the page and store it to a dataset
+ await Apify.pushData({
+ url: request.url,
+ html: await rp(request.url),
+ })
+ },
+});
+
+await crawler.run();</code></pre></div>
+</header>
+
+<article>
+ <div class="container-overview">
+<section id="BasicCrawler" class="sub-sec">
+<h2>Constructor</h2>
+<h4>new BasicCrawler<span class="signature">(options)</span><span class="type-signature"></span></h4>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>options</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+<div class="params">
+<ul>
+<li>
+<b><code>handleRequestFunction</code></b>
+(
+<span class="param-type">function</span>
+
+)
+-
+User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
+<p> The function that receives an object as argument, with the following field:</p>
+ <ul>
+ <li><code>request</code>: the <a href="#Request"><code>Request</code></a> object representing the URL to crawl</li>
+ </ul>
+
+<p> The function must return a promise.</p>
+</li>
+<li>
+<b><code>requestList</code></b>
+(
+<span class="param-type"><a href="#RequestList">RequestList</a></span>
+
+)
+-
+Static list of URLs to be processed.
+ Either <code>RequestList</code> or <code>RequestQueue</code> must be provided.
+</li>
+<li>
+<b><code>requestQueue</code></b>
+(
+<span class="param-type"><a href="#RequestQueue">RequestQueue</a></span>
+
+)
+-
+Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
+ Either RequestList or RequestQueue must be provided.
+</li>
+<li>
+<b><code>handleFailedRequestFunction</code></b>
+(
+<span class="param-type">function</span>
+
+)
+&lt;optional>
+-
+Function that handles requests that failed more then <code>option.maxRequestRetries</code> times.
+ See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/basic_crawler.js#L11">GitHub</a> for default behavior.
+</li>
+<li>
+<b><code>maxRequestRetries</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+How many times the request is retried if <code>handleRequestFunction</code> failed.
+Defaults to <code>3</code>.
+</li>
+<li>
+<b><code>maxRequestsPerCrawl</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
+ Always set this value in order to prevent infinite loops in misconfigured crawlers.
+ Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
+</li>
+<li>
+<b><code>autoscaledPoolOptions</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+-
+Custom options passed to the underlying <a href="#AutoscaledPool"><code>AutoscaledPool</code></a> instance constructor.
+ Note that the <code>runTaskFunction</code>, <code>isTaskReadyFunction</code> and <code>isFinishedFunction</code> options
+ are provided by <code>BasicCrawler</code> and cannot be overridden.
+</li>
+<li>
+<b><code>minConcurrency</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+-
+Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding <code>AutoscaledPool</code> option.
+Defaults to <code>1</code>.
+</li>
+<li>
+<b><code>maxConcurrency</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+-
+Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding <code>AutoscaledPool</code> option.
+Defaults to <code>1000</code>.
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<dl class="details">
+ <dt class="tag-see">See:</dt>
+ <dd class="tag-see">
+ <ul>
+ <li><a href="#CheerioCrawler">CheerioCrawler</a></li>
+<li><a href="#PuppeteerCrawler">PuppeteerCrawler</a></li>
+ </ul>
+ </dd>
+ </dl>
+</section>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Methods (2)</h3>
+<section id="BasicCrawler-abort" class="sub-sec">
+<h4>abort<span class="signature">()</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Aborts the crawler by preventing additional requests and terminating the running ones.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="BasicCrawler-run" class="sub-sec">
+<h4>run<span class="signature">()</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Runs the crawler. Returns a promise that gets resolved once all the requests are processed.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+</div>
+</article>
+
+</div>
+<div id="CheerioCrawler">
+
+<header>
+<h2>
+ CheerioCrawler
+ </h2>
+<div class="class-description"><p>Provides a framework for the parallel crawling of web pages using plain HTTP requests and
+<a href="https://www.npmjs.com/package/cheerio">cheerio</a> HTML parser.</p>
+<p><code>CheerioCrawler</code> downloads each URL using a plain HTTP request,
+parses the HTML content using cheerio and then
+invokes the user-provided <code>handlePageFunction</code> to extract page data
+using a <a href="https://jquery.com/">jQuery</a>-like interface to parsed HTML DOM.</p>
+<p>The source URLs are represented using <code>Request</code> objects that
+are fed from the <a href="#RequestList"><code>RequestList</code></a> or <a href="#RequestQueue"><code>RequestQueue</code></a>
+instances provided by the <code>requestList</code> or <code>requestQueue</code> constructor options, respectively.</p>
+<p>If both <code>requestList</code> and <code>requestQueue</code> is used, the instance first
+processes URLs from the <code>RequestList</code> and automatically enqueues all of them to <code>RequestQueue</code> before it starts
+their processing. This ensures that a single URL is not crawled multiple times.</p>
+<p>The crawler finishes if there are no more <code>Request</code> objects to crawl.</p>
+<p>By default, <code>CheerioCrawler</code> downloads HTML using the <a href="https://www.npmjs.com/package/request-promise">request-promise</a> NPM package.
+You can override this behavior by setting the <code>requestFunction</code> option.</p>
+<p>New requests are only started if there is enough free CPU and memory available,
+using the functionality provided by the <a href="#AutoscaledPool"><code>AutoscaledPool</code></a> class.
+All <code>AutoscaledPool</code> configuration options can be passed to the <code>autoscaledPoolOptions</code> parameter
+of the <code>CheerioCrawler</code> constructor.
+For user convenience, the <code>minConcurrency</code> and <code>maxConcurrency</code> options are available directly.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>// Prepare a list of URLs to crawl
+const requestList = new Apify.RequestList({
+ sources: [
+ { url: 'http://www.example.com/page-1' },
+ { url: 'http://www.example.com/page-2' },
+ ],
+});
+await requestList.initialize();
+
+// Crawl the URLs
+const crawler = new Apify.CheerioCrawler({
+ requestList,
+ handlePageFunction: async ({ $, html, request }) => {
+const data = [];
+// Do some data extraction from the page with Cheerio.
+ $('.some-collection').each((index, el) => {
+ data.push({ title: $(el).find('.some-title').text() });
+ });
+// Save the data to dataset.
+ await Apify.pushData({
+ url: request.url,
+ html,
+ data,
+ })
+ },
+});
+
+await crawler.run();</code></pre></div>
+</header>
+
+<article>
+ <div class="container-overview">
+<section id="CheerioCrawler" class="sub-sec">
+<h2>Constructor</h2>
+<h4>new CheerioCrawler<span class="signature">(options)</span><span class="type-signature"></span></h4>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>options</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+<div class="params">
+<ul>
+<li>
+<b><code>handlePageFunction</code></b>
+(
+<span class="param-type">function</span>
+
+)
+-
+User-provided function that performs the logic of the crawler. It is called for each page
+ loaded and parsed by the crawler.
+<p> The function that receives an object as argument, with the following three fields:</p>
+ <ul>
+ <li><code>$</code>: the Cheerio object</li>
+ <li><code>html</code>: the raw HTML</li>
+ <li><code>request</code>: the <a href="#Request"><code>Request</code></a> object representing the URL to crawl</li>
+ </ul>
+
+<p> If the function returns a promise, it is awaited.</p>
+</li>
+<li>
+<b><code>requestList</code></b>
+(
+<span class="param-type"><a href="#RequestList">RequestList</a></span>
+
+)
+-
+Static list of URLs to be processed.
+ Either RequestList or RequestQueue must be provided.
+</li>
+<li>
+<b><code>requestQueue</code></b>
+(
+<span class="param-type"><a href="#RequestQueue">RequestQueue</a></span>
+
+)
+-
+Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
+ Either RequestList or RequestQueue must be provided.
+</li>
+<li>
+<b><code>requestFunction</code></b>
+(
+<span class="param-type">function</span>
+
+)
+&lt;optional>
+-
+Overrides the function that performs the HTTP request to get the raw HTML needed for Cheerio.
+ See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/cheerio_crawler.js#L264">GitHub</a> for default behavior.
+</li>
+<li>
+<b><code>handlePageTimeoutSecs</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Timeout in which the function passed as <code>options.handlePageFunction</code> needs to finish, given in seconds.
+Defaults to <code>300</code>.
+</li>
+<li>
+<b><code>requestTimeoutSecs</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Timeout in which the function passed as <code>options.requestFunction</code> needs to finish, given in seconds.
+Defaults to <code>30</code>.
+</li>
+<li>
+<b><code>ignoreSslErrors</code></b>
+(
+<span class="param-type">Boolean</span>
+
+)
+&lt;optional>
+-
+If set to true, SSL certificate errors will be ignored. This is dependent on using the default
+ request function. If using a custom request function, user needs to implement this functionality.
+Defaults to <code>false</code>.
+</li>
+<li>
+<b><code>handleFailedRequestFunction</code></b>
+(
+<span class="param-type">function</span>
+
+)
+&lt;optional>
+-
+Function that handles requests that failed more then <code>option.maxRequestRetries</code> times.
+ See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/cheerio_crawler.js#L13">GitHub</a> for default behavior.
+</li>
+<li>
+<b><code>maxRequestRetries</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+How many times the request is retried if either <code>requestFunction</code> or <code>handlePageFunction</code> failed.
+Defaults to <code>3</code>.
+</li>
+<li>
+<b><code>maxRequestsPerCrawl</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
+ Always set this value in order to prevent infinite loops in misconfigured crawlers.
+ Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
+</li>
+<li>
+<b><code>autoscaledPoolOptions</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+-
+Custom options passed to the underlying <a href="#AutoscaledPool"><code>AutoscaledPool</code></a> instance constructor.
+ Note that the <code>runTaskFunction</code>, <code>isTaskReadyFunction</code> and <code>isFinishedFunction</code> options
+ are provided by <code>CheerioCrawler</code> and cannot be overridden.
+</li>
+<li>
+<b><code>minConcurrency</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+-
+Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding <code>AutoscaledPool</code> option.
+Defaults to <code>1</code>.
+</li>
+<li>
+<b><code>maxConcurrency</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+-
+Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding <code>AutoscaledPool</code> option.
+Defaults to <code>1000</code>.
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<dl class="details">
+ <dt class="tag-see">See:</dt>
+ <dd class="tag-see">
+ <ul>
+ <li><a href="#BasicCrawler">BasicCrawler</a></li>
+<li><a href="#PuppeteerCrawler">PuppeteerCrawler</a></li>
+ </ul>
+ </dd>
+ </dl>
+</section>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Methods (2)</h3>
+<section id="CheerioCrawler-abort" class="sub-sec">
+<h4>abort<span class="signature">()</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Aborts the crawler by preventing crawls of additional pages and terminating the running ones.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="CheerioCrawler-run" class="sub-sec">
+<h4>run<span class="signature">()</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Runs the crawler. Returns promise that gets resolved once all the requests got processed.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+</div>
+</article>
+
+</div>
+<div id="Dataset">
+
+<header>
+<h2>
+ Dataset
+ </h2>
+<div class="class-description"><p>The <code>Dataset</code> class represents a store for structured data where each object stored has the same attributes,
+such as online store products or real estate offers. You can imagine it as a table,
+where each object is a row and its attributes are columns.
+Dataset is an append-only storage - you can only add new records to it but you cannot modify or remove existing records.
+Typically it is used to store crawling results.</p>
+<p>Do not instantiate this class directly, use the
+<code>Apify.openDataset()</code> function instead.</p>
+<p><code>Dataset</code> stores its data either on local disk or in the Apify cloud,
+depending on whether the <code>APIFY_LOCAL_STORAGE_DIR</code> or <code>APIFY_TOKEN</code> environment variable is set.</p>
+<p>If the <code>APIFY_LOCAL_STORAGE_DIR</code> environment variable is set, the data is stored in
+the local directory in the following files:</p>
+<pre class="prettyprint source"><code>[APIFY_LOCAL_STORAGE_DIR]/datasets/[DATASET_ID]/[INDEX].json</code></pre><p>Note that <code>[DATASET_ID]</code> is the name or ID of the dataset. The default dataset has ID <code>default</code>,
+unless you override it by setting the <code>APIFY_DEFAULT_DATASET_ID</code> environment variable.
+Each dataset item is stored as a separate JSON file, where <code>[INDEX]</code> is a zero-based index of the item in the dataset.</p>
+<p>If the <code>APIFY_TOKEN</code> environment variable is provided instead, the data is stored
+in the <a href="https://www.apify.com/docs/storage#dataset">Apify Dataset</a> cloud storage.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>// Write a single row to the default dataset
+await Apify.pushData({ col1: 123, col2: 'val2' });
+
+// Open a named dataset
+const dataset = await Apify.openDataset('some-name');
+
+// Write a single row
+await dataset.pushData({ foo: 'bar' });
+
+// Write multiple rows
+await dataset.pushData([
+ { foo: 'bar2', col2: 'val2' },
+ { col3: 123 },
+]);</code></pre></div>
+</header>
+
+<article>
+ <div class="container-overview">
+<section id="Dataset" class="sub-sec">
+<h2>Constructor</h2>
+<h4>new Dataset<span class="signature">()</span><span class="type-signature"></span></h4>
+</section>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Methods (7)</h3>
+<section id="Dataset-delete" class="sub-sec">
+<h4>delete<span class="signature">()</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Removes the dataset either from the Apify cloud storage or from the local directory,
+depending on the mode of operation.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="Dataset-forEach" class="sub-sec">
+<h4>forEach<span class="signature">(iteratee, opts, index)</span><span class="type-signature"> &rarr; {Promise.&lt;undefined&gt;}</span></h4>
+<div class="description">
+ <p>Iterates over dataset items, yielding each in turn to an <code>iteratee</code> function.
+Each invocation of <code>iteratee</code> is called with three arguments: <code>(element, index)</code>.</p>
+<p>If <code>iteratee</code> returns a Promise then it is awaited before a next call.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>iteratee</code></b>
+(
+<span class="param-type">function</span>
+
+)
+-
+</li>
+<li>
+<b><code>opts</code></b>
+(
+<span class="param-type">Opts</span>
+
+)
+-
+</li>
+<li>
+<b><code>options.offset</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Number of array elements that should be skipped at the start.
+Defaults to <code>0</code>.
+</li>
+<li>
+<b><code>options.desc</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+If <code>1</code> then the objects are sorted by <code>createdAt</code> in descending order.
+</li>
+<li>
+<b><code>options.fields</code></b>
+(
+<span class="param-type">Array</span>
+
+)
+&lt;optional>
+-
+If provided then returned objects will only contain specified keys
+</li>
+<li>
+<b><code>options.unwind</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+If provided then objects will be unwound based on provided field.
+</li>
+<li>
+<b><code>options.limit</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+How many items to load in one request.
+Defaults to <code>250000</code>.
+</li>
+<li>
+<b><code>index</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+-
+[description]
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;undefined&gt;</span>
+
+)
+</li>
+</section>
+<section id="Dataset-getData" class="sub-sec">
+<h4>getData<span class="signature">(options)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Returns items in the dataset based on the provided parameters.</p>
+<p>If format is <code>json</code> then the function doesn't return an array of records but <a href="#PaginationList"><code>PaginationList</code></a> instead.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>options</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+<div class="params">
+<ul>
+<li>
+<b><code>format</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+Format of the items, possible values are: <code>json</code>, <code>csv</code>, <code>xlsx</code>, <code>html</code>, <code>xml</code> and <code>rss</code>.
+Defaults to <code>'json'</code>.
+</li>
+<li>
+<b><code>offset</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Number of array elements that should be skipped at the start.
+Defaults to <code>0</code>.
+</li>
+<li>
+<b><code>limit</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Maximum number of array elements to return.
+Defaults to <code>250000</code>.
+</li>
+<li>
+<b><code>desc</code></b>
+(
+<span class="param-type">Boolean</span>
+
+)
+&lt;optional>
+-
+If <code>true</code> then the objects are sorted by <code>createdAt</code> in descending order.
+ Otherwise they are sorted in ascending order.
+</li>
+<li>
+<b><code>fields</code></b>
+(
+<span class="param-type">Array</span>
+
+)
+&lt;optional>
+-
+An array of field names that will be included in the result. If omitted, all fields are included in the results.
+</li>
+<li>
+<b><code>unwind</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+Specifies a name of the field in the result objects that will be used to unwind the resulting objects.
+ By default, the results are returned as they are.
+</li>
+<li>
+<b><code>disableBodyParser</code></b>
+(
+<span class="param-type">Boolean</span>
+
+)
+&lt;optional>
+-
+If <code>true</code> then response from API will not be parsed.
+</li>
+<li>
+<b><code>attachment</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+If <code>true</code> then the response will define the <code>Content-Disposition: attachment</code> HTTP header, forcing a web
+ browser to download the file rather than to display it. By default, this header is not present.
+</li>
+<li>
+<b><code>delimiter</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+A delimiter character for CSV files, only used if <code>format</code> is <code>csv</code>.
+ You might need to URL-encode the character (e.g. use <code>%09</code> for tab or <code>%3B</code> for semicolon).
+Defaults to <code>','</code>.
+</li>
+<li>
+<b><code>bom</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+All responses are encoded in UTF-8 encoding. By default, the CSV files are prefixed with the UTF-8 Byte
+ Order Mark (BOM), while JSON, JSONL, XML, HTML and RSS files are not. If you want to override this default
+ behavior, set <code>bom</code> option to <code>true</code> to include the BOM, or set <code>bom</code> to <code>false</code> to skip it.
+</li>
+<li>
+<b><code>xmlRoot</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+Overrides the default root element name of the XML output. By default, the root element is <code>results</code>.
+</li>
+<li>
+<b><code>xmlRow</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+Overrides the default element name that wraps each page or page function result object in XML output.
+ By default, the element name is <code>page</code> or <code>result</code>, depending on the value of the <code>simplified</code> option.
+</li>
+<li>
+<b><code>skipHeaderRow</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+If set to <code>1</code> then header row in csv format is skipped.
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="Dataset-getInfo" class="sub-sec">
+<h4>getInfo<span class="signature">(opts)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Returns an object containing general information about the dataset.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>opts</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+<h5>Example</h5>
+<pre class="prettyprint"><code>{
+ "id": "WkzbQMuFYuamGv3YF",
+ "name": "d7b9MDYsbtX5L7XAj",
+ "userId": "wRsJZtadYvn4mBZmm",
+ "createdAt": "2015-12-12T07:34:14.202Z",
+ "modifiedAt": "2015-12-13T08:36:13.202Z",
+ "accessedAt": "2015-12-14T08:36:13.202Z",
+ "itemsCount": 0
+}</code></pre>
+</section>
+<section id="Dataset-map" class="sub-sec">
+<h4>map<span class="signature">(iteratee, opts, index)</span><span class="type-signature"> &rarr; {Promise.&lt;Array&gt;}</span></h4>
+<div class="description">
+ <p>Produces a new array of values by mapping each value in list through a transformation function (<code>iteratee</code>).
+Each invocation of <code>iteratee</code> is called with three arguments: <code>(element, index)</code>.</p>
+<p>If <code>iteratee</code> returns a <code>Promise</code> then it's awaited before a next call.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>iteratee</code></b>
+(
+<span class="param-type">function</span>
+
+)
+-
+</li>
+<li>
+<b><code>opts</code></b>
+(
+<span class="param-type">Opts</span>
+
+)
+-
+</li>
+<li>
+<b><code>options.offset</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Number of array elements that should be skipped at the start.
+Defaults to <code>0</code>.
+</li>
+<li>
+<b><code>options.desc</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+If 1 then the objects are sorted by createdAt in descending order.
+</li>
+<li>
+<b><code>options.fields</code></b>
+(
+<span class="param-type">Array</span>
+
+)
+&lt;optional>
+-
+If provided then returned objects will only contain specified keys
+</li>
+<li>
+<b><code>options.unwind</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+If provided then objects will be unwound based on provided field.
+</li>
+<li>
+<b><code>options.limit</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+How many items to load in one request.
+Defaults to <code>250000</code>.
+</li>
+<li>
+<b><code>index</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+-
+[description]
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;Array&gt;</span>
+
+)
+</li>
+</section>
+<section id="Dataset-pushData" class="sub-sec">
+<h4>pushData<span class="signature">(data)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Stores an object or an array of objects to the dataset.
+The function returns a promise that resolves when the operation finishes.
+It has no result, but throws on invalid args or other errors.</p>
+<p><strong>IMPORTANT</strong>: Make sure to use the <code>await</code> keyword when calling <code>pushData()</code>,
+otherwise the actor process might finish before the data is stored!</p>
+<p>The size of the data is limited by the receiving API and therefore <code>pushData</code> will only
+allow objects whose JSON representation is smaller than 9MB. When an array is passed,
+none of the included objects
+may be larger than 9MB, but the array itself may be of any size.</p>
+<p>The function internally
+chunks the array into separate items and pushes them sequentially.
+The chunking process is stable (keeps order of data), but it does not provide a transaction
+safety mechanism. Therefore, in the event of an uploading error (after several automatic retries),
+the function's promise will reject and the dataset will be left in a state where some of
+the items have already been saved to the dataset while other items from the source array were not.
+To overcome this limitation, the developer may, for example, read the last item saved in the dataset
+and re-attempt the save of the data from this item onwards to prevent duplicates.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>data</code></b>
+(
+<span class="param-type">Object</span>
+|
+
+<span class="param-type">Array</span>
+
+)
+-
+Object or array of objects containing data to be stored in the default dataset.
+The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+- Returns a promise that resolves once the data is saved.
+
+</li>
+</section>
+<section id="Dataset-reduce" class="sub-sec">
+<h4>reduce<span class="signature">(iteratee, memo, opts, index)</span><span class="type-signature"> &rarr; {Promise.&lt;*&gt;}</span></h4>
+<div class="description">
+ <p>Boils down a list of values into a single value.</p>
+<p>Memo is the initial state of the reduction, and each successive step of it should be returned by <code>iteratee</code>.
+The <code>iteratee</code> is passed three arguments: the <code>memo</code>, then the value and index of the iteration.</p>
+<p>If no <code>memo</code> is passed to the initial invocation of reduce, the <code>iteratee</code> is not invoked on the first element of the list.
+The first element is instead passed as the memo in the invocation of the <code>iteratee</code> on the next element in the list.</p>
+<p>If <code>iteratee</code> returns a <code>Promise</code> then it's awaited before a next call.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>iteratee</code></b>
+(
+<span class="param-type">function</span>
+
+)
+-
+</li>
+<li>
+<b><code>memo</code></b>
+(
+<span class="param-type">*</span>
+
+)
+-
+</li>
+<li>
+<b><code>opts</code></b>
+(
+<span class="param-type">Opts</span>
+
+)
+-
+</li>
+<li>
+<b><code>options.offset</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Number of array elements that should be skipped at the start.
+Defaults to <code>0</code>.
+</li>
+<li>
+<b><code>options.desc</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+If 1 then the objects are sorted by createdAt in descending order.
+</li>
+<li>
+<b><code>options.fields</code></b>
+(
+<span class="param-type">Array</span>
+
+)
+&lt;optional>
+-
+If provided then returned objects will only contain specified keys
+</li>
+<li>
+<b><code>options.unwind</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+If provided then objects will be unwound based on provided field.
+</li>
+<li>
+<b><code>options.limit</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+How many items to load in one request.
+Defaults to <code>250000</code>.
+</li>
+<li>
+<b><code>index</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+-
+[description]
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;*&gt;</span>
+
+)
+</li>
+</section>
+</div>
+</article>
+
+</div>
+<div id="KeyValueStore">
+
+<header>
+<h2>
+ KeyValueStore
+ </h2>
+<div class="class-description"><p>The <code>KeyValueStore</code> class represents a key-value store, a simple data storage that is used
+for saving and reading data records or files. Each data record is
+represented by a unique key and associated with a MIME content type. Key-value stores are ideal
+for saving screenshots, actor inputs and outputs, web pages, PDFs or to persist the state of crawlers.</p>
+<p>Do not instantiate this class directly, use the
+<code>Apify.openKeyValueStore()</code> function instead.</p>
+<p>Each actor run is associated with a default key-value store, which is created exclusively
+for the run. By convention, the actor input and output are stored into the
+default key-value store under the <code>INPUT</code> and <code>OUTPUT</code> key, respectively.
+Typically, input and output are JSON files, although it can be any other format.
+To access the default key-value store directly, you can use the
+<code>Apify.getValue()</code>
+and <code>Apify.setValue()</code> convenience functions.</p>
+<p><code>KeyValueStore</code> stores its data either on local disk or in the Apify cloud,
+depending on whether the <code>APIFY_LOCAL_STORAGE_DIR</code> or <code>APIFY_TOKEN</code> environment variable is set.</p>
+<p>If the <code>APIFY_LOCAL_STORAGE_DIR</code> environment variable is set, the data is stored in
+the local directory in the following files:</p>
+<pre class="prettyprint source"><code>[APIFY_LOCAL_STORAGE_DIR]/key_value_stores/[STORE_ID]/[KEY].[EXT]</code></pre><p>Note that <code>[STORE_ID]</code> is the name or ID of the key-value store. The default key value store has ID <code>default</code>,
+unless you override it by setting the <code>APIFY_DEFAULT_KEY_VALUE_STORE_ID</code> environment variable.
+The <code>[KEY]</code> is the key of the record and <code>[EXT]</code> corresponds to the MIME content type of the data value.</p>
+<p>If the <code>APIFY_TOKEN</code> environment variable is provided instead, the data is stored
+in the <a href="https://www.apify.com/docs/storage#dataset">Apify Dataset</a> cloud storage.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>// Get actor input from the default key-value store
+const input = await Apify.getValue('INPUT');
+
+// Write actor output to the default key-value store.
+await Apify.setValue('OUTPUT', { myResult: 123 });
+
+// Open a named key-value store
+const store = await Apify.openKeyValueStore('some-name');
+
+// Write a record. JavaScript object is automatically converted to JSON,
+// strings and binary buffers are stored as they are
+await store.setValue('some-key', { foo: 'bar' });
+
+// Read a record. Note that JSON is automatically parsed to a JavaScript object,
+// text data returned as a string and other data is returned as binary buffer
+const value = await store.getValue('some-key');
+// Delete record
+await store.delete('some-key');</code></pre></div>
+</header>
+
+<article>
+ <div class="container-overview">
+<section id="KeyValueStore" class="sub-sec">
+<h2>Constructor</h2>
+<h4>new KeyValueStore<span class="signature">()</span><span class="type-signature"></span></h4>
+<dl class="details">
+ <dt class="tag-see">See:</dt>
+ <dd class="tag-see">
+ <ul>
+ <li><code>Apify.getValue()</code></li>
+<li><code>Apify.setValue()</code></li>
+ </ul>
+ </dd>
+ </dl>
+</section>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Methods (3)</h3>
+<section id="KeyValueStore-delete" class="sub-sec">
+<h4>delete<span class="signature">()</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Removes the key-value store either from the Apify cloud storage or from the local directory,
+depending on the mode of operation.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="KeyValueStore-getValue" class="sub-sec">
+<h4>getValue<span class="signature">(key)</span><span class="type-signature"> &rarr; {Promise.&lt;Object&gt;}</span></h4>
+<div class="description">
+ <p>Gets a value from the key-value store.</p>
+<p>The function returns a promise that resolves to the record value,
+whose JavaScript type depends on the MIME content type of the record.
+Records with the <code>application/json</code>
+content type are automatically parsed and returned as a JavaScript object.
+Similarly, records with <code>text/plain</code> content types are returned as a string.
+For all other content types, the value is returned as a raw
+<a href="#https-nodejs-org-api-buffer"><code>Buffer</code></a> instance.</p>
+<p>If the record does not exist, the functions resolves to <code>null</code>.</p>
+<p>To save or delete a value in the key-value store, use the
+<a href="#KeyValueStore-setValue"><code>KeyValueStore.setValue()</code></a> function.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>const store = await Apify.openKeyValueStore('my-screenshots');
+const buffer = await store.getValue('screenshot1.png');</code></pre>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>key</code></b>
+(
+<span class="param-type">String</span>
+
+)
+-
+Key of the record.
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;Object&gt;</span>
+
+)
+- Returns a promise that resolves to an object, string
+ or <a href="#https-nodejs-org-api-buffer"><code>Buffer</code></a>, depending
+ on the MIME content type of the record.
+
+</li>
+</section>
+<section id="KeyValueStore-setValue" class="sub-sec">
+<h4>setValue<span class="signature">(key, value, options<span class="signature-attributes">opt</span>)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Saves or deletes a record in the key-value store.
+The function returns a promise that resolves once the record has been saved or deleted.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>const store = await Apify.openKeyValueStore('my-store');
+await store.setValue('RESULTS', 'my text data', { contentType: 'text/plain' });</code></pre><p>By default, <code>value</code> is converted to JSON and stored with the
+<code>application/json; charset=utf-8</code> MIME content type.
+To store the value with another content type, pass it in the options as follows:</p>
+<pre class="prettyprint source lang-javascript"><code>const store = await Apify.openKeyValueStore('my-store');
+await store.setValue('RESULTS', 'my text data', { contentType: 'text/plain' });</code></pre><p>If you set custom content type, <code>value</code> must be either a string or
+<a href="#https-nodejs-org-api-buffer"><code>Buffer</code></a>, otherwise an error will be thrown.</p>
+<p>If <code>value</code> is null, the record is deleted instead. Note that the <code>setValue()</code> function succeeds
+regardless whether the record existed or not.</p>
+<p>To retrieve a value from the key-value store, use the
+<a href="#KeyValueStore-setValue"><code>KeyValueStore.getValue()</code></a> function.</p>
+<p><strong>IMPORTANT:</strong> Always make sure to use the <code>await</code> keyword when calling <code>setValue()</code>,
+otherwise the actor process might finish before the value is stored!</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>key</code></b>
+(
+<span class="param-type">String</span>
+
+)
+-
+Unique record key.
+</li>
+<li>
+<b><code>value</code></b>
+(
+<span class="param-type">Object</span>
+|
+
+<span class="param-type">String</span>
+|
+
+<span class="param-type">Buffer</span>
+
+)
+-
+Record data, which can be one of the following values:
+ <ul>
+ <li>If <code>null</code>, the record in the key-value store is deleted.</li>
+ <li>If no <code>options.contentType</code> is specified, <code>value</code> can be any JavaScript object and it will be stringified to JSON.</li>
+ <li>If <code>options.contentType</code> is specified, <code>value</code> is considered raw data and it must be a <code>String</code>
+ or <a href="#https-nodejs-org-api-buffer"><code>Buffer</code></a>.</li>
+ </ul>
+ For any other value an error will be thrown.
+</li>
+<li>
+<b><code>options</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+<div class="params">
+<ul>
+<li>
+<b><code>contentType</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+Specifies a custom MIME content type of the record.
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+- Returns a promise that resolves once the value is stored or deleted.
+
+</li>
+</section>
+</div>
+</article>
+
+</div>
+<div id="PseudoUrl">
+
+<header>
+<h2>
+ PseudoUrl
+ </h2>
+<div class="class-description"><p>Represents a pseudo URL (PURL) - an URL pattern used by web crawlers
+to specify which URLs should the crawler visit.
+This class is used by the <code>Apify.utils.puppeteer.enqueueLinks()</code> function.</p>
+<p>A PURL is simply a URL with special directives enclosed in <code>[]</code> brackets.
+Currently, the only supported directive is <code>[regexp]</code>,
+which defines a JavaScript-style regular expression to match against the URL.</p>
+<p>For example, a PURL <code>http://www.example.com/pages/[(\w|-)*]</code> will match all of the following URLs:</p>
+<ul>
+ <li><code>http://www.example.com/pages/</code></li>
+ <li><code>http://www.example.com/pages/my-awesome-page</code></li>
+ <li><code>http://www.example.com/pages/something</code></li>
+</ul>
+
+<p>If either <code>[</code> or <code>]</code> is part of the normal query string, it must be encoded as <code>[\x5B]</code> or <code>[\x5D]</code>,
+respectively. For example, the following PURL:</p>
+<pre class="prettyprint source"><code>http://www.example.com/search?do[\x5B]load[\x5D]=1</code></pre><p>will match the URL:</p>
+<pre class="prettyprint source"><code>http://www.example.com/search?do[load]=1</code></pre><p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>const purl = new Apify.PseudoUrl('http://www.example.com/pages/[(\w|-)*]');
+
+if (purl.matches('http://www.example.com/pages/my-awesome-page')) console.log('Match!');</code></pre></div>
+</header>
+
+<article>
+ <div class="container-overview">
+<section id="PseudoUrl" class="sub-sec">
+<h2>Constructor</h2>
+<h4>new PseudoUrl<span class="signature">(purl, requestTemplate)</span><span class="type-signature"></span></h4>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>purl</code></b>
+(
+<span class="param-type">String</span>
+
+)
+-
+Pseudo URL.
+</li>
+<li>
+<b><code>requestTemplate</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+-
+Options for the new <a href="#Request"><code>Request</code></a> instances created for matching URLs.
+</li>
+</ul>
+</div>
+<dl class="details">
+ <dt class="tag-see">See:</dt>
+ <dd class="tag-see">
+ <ul>
+ <li><a href="#Request"><code>Request</code></a></li>
+<li><a href="#Request"><code>Request</code></a></li>
+ </ul>
+ </dd>
+ </dl>
+</section>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Methods (2)</h3>
+<section id="PseudoUrl-createRequest" class="sub-sec">
+<h4>createRequest<span class="signature">(url)</span><span class="type-signature"> &rarr; {<a href="#Request">Request</a>}</span></h4>
+<div class="description">
+ <p>Creates a Request object from requestTemplate and given URL.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>url</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type"><a href="#Request">Request</a></span>
+
+)
+</li>
+</section>
+<section id="PseudoUrl-matches" class="sub-sec">
+<h4>matches<span class="signature">(url)</span><span class="type-signature"> &rarr; {Boolean}</span></h4>
+<div class="description">
+ <p>Determines whether a URL matches this pseudo-URL pattern.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>url</code></b>
+(
+<span class="param-type">String</span>
+
+)
+-
+URL to be matched.
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Boolean</span>
+
+)
+- Returns <code>true</code> if given URL matches pseudo URL.
+
+</li>
+</section>
+</div>
+</article>
+
+</div>
+<div id="PuppeteerCrawler">
+
+<header>
+<h2>
+ PuppeteerCrawler
+ </h2>
+<div class="class-description"><p>Provides a simple framework for parallel crawling of web pages
+using headless Chrome with <a href="https://github.com/GoogleChrome/puppeteer">Puppeteer</a>.
+The URLs of pages to visit are given by <code>Request</code> objects that are fed from a list (see <code>RequestList</code> class)
+or from a dynamic queue (see <code>RequestQueue</code> class).</p>
+<p><code>PuppeteerCrawler</code> opens a new Chrome page (i.e. tab) for each <code>Request</code> object to crawl
+and then calls the function provided by user as the <code>handlePageFunction</code> option.
+New tasks are only started if there is enough free CPU and memory available,
+using the <code>AutoscaledPool</code> class internally.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>const crawler = new Apify.PuppeteerCrawler({
+ requestList,
+ handlePageFunction: async ({ page, request }) => {
+ // This function is called to extract data from a single web page
+ // 'page' is an instance of Puppeteer.Page with page.goto(request.url) already called
+ // 'request' is an instance of Request class with information about the page to load
+ await Apify.pushData({
+ title: await page.title(),
+ url: request.url,
+ succeeded: true,
+ })
+ },
+ handleFailedRequestFunction: async ({ request }) => {
+ // This function is called when crawling of a request failed too many time
+ await Apify.pushData({
+ url: request.url,
+ succeeded: false,
+ errors: request.errorMessages,
+ })
+ },
+});
+
+await crawler.run();</code></pre></div>
+</header>
+
+<article>
+ <div class="container-overview">
+<section id="PuppeteerCrawler" class="sub-sec">
+<h2>Constructor</h2>
+<h4>new PuppeteerCrawler<span class="signature">(options)</span><span class="type-signature"></span></h4>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>options</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+<div class="params">
+<ul>
+<li>
+<b><code>handlePageFunction</code></b>
+(
+<span class="param-type">function</span>
+
+)
+-
+Function that is called to process each request.
+ It is passed an object with the following fields:
+ <code>request</code> is an instance of the <code>Request</code> object with details about the URL to open, HTTP method etc.
+ <code>page</code> is an instance of the <code>Puppeteer.Page</code> class with <code>page.goto(request.url)</code> already called.
+</li>
+<li>
+<b><code>requestList</code></b>
+(
+<span class="param-type"><a href="#RequestList">RequestList</a></span>
+
+)
+-
+List of the requests to be processed.
+ Either RequestList or RequestQueue must be provided.
+ See the <code>requestList</code> parameter of <code>BasicCrawler</code> for more details.
+</li>
+<li>
+<b><code>requestQueue</code></b>
+(
+<span class="param-type"><a href="#RequestQueue">RequestQueue</a></span>
+
+)
+-
+Queue of the requests to be processed.
+ Either RequestList or RequestQueue must be provided.
+ See the <code>requestQueue</code> parameter of <code>BasicCrawler</code> for more details.
+</li>
+<li>
+<b><code>handlePageTimeoutSecs</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Timeout in which the function passed as <code>options.handlePageFunction</code> needs to finish, in seconds.
+Defaults to <code>300</code>.
+</li>
+<li>
+<b><code>gotoFunction</code></b>
+(
+<span class="param-type">function</span>
+
+)
+&lt;optional>
+-
+Overrides the function that opens the request in Puppeteer. The function should return a result of Puppeteer's
+ <a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#pagegotourl-options">page.goto()</a> function,
+ i.e. a promise resolving to the <a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-response">Response</a> object.
+<p> For example, this is useful if you need to extend the page load timeout or select different criteria
+ to determine that the navigation succeeded.</p>
+<p> Note that a single page object is only used to process a single request and it is closed afterwards.</p>
+<p> See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/puppeteer_crawler.js#L9">GitHub</a> for default behavior.</p>
+</li>
+<li>
+<b><code>handleFailedRequestFunction</code></b>
+(
+<span class="param-type">function</span>
+
+)
+&lt;optional>
+-
+Function to handle requests that failed more than <code>option.maxRequestRetries</code> times. See the <code>handleFailedRequestFunction</code>
+ parameter of <code>Apify.BasicCrawler</code> for details.
+ See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/puppeteer_crawler.js#L13">GitHub</a> for default behavior.
+</li>
+<li>
+<b><code>maxRequestRetries</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Indicates how many times each request is retried if <code>handleRequestFunction</code> failed.
+ See <code>maxRequestRetries</code> parameter of <code>BasicCrawler</code>.
+Defaults to <code>3</code>.
+</li>
+<li>
+<b><code>maxRequestsPerCrawl</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
+ Always set this value in order to prevent infinite loops in misconfigured crawlers.
+ Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
+ See <code>maxRequestsPerCrawl</code> parameter of <code>BasicCrawler</code>.
+</li>
+<li>
+<b><code>maxOpenPagesPerInstance</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Maximum number of opened tabs per browser. If this limit is reached then a new
+ browser instance is started. See <code>maxOpenPagesPerInstance</code> parameter of <code>PuppeteerPool</code>.
+Defaults to <code>50</code>.
+</li>
+<li>
+<b><code>retireInstanceAfterRequestCount</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Maximum number of requests that can be processed by a single browser instance.
+ After the limit is reached the browser will be retired and new requests will
+ be handled by a new browser instance.
+ See <code>retireInstanceAfterRequestCount</code> parameter of <code>PuppeteerPool</code>.
+Defaults to <code>100</code>.
+</li>
+<li>
+<b><code>instanceKillerIntervalMillis</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+How often the launched Puppeteer instances are checked whether they can be
+ closed. See <code>instanceKillerIntervalMillis</code> parameter of <code>PuppeteerPool</code>.
+Defaults to <code>60000</code>.
+</li>
+<li>
+<b><code>killInstanceAfterMillis</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+If Puppeteer instance reaches the <code>options.retireInstanceAfterRequestCount</code> limit then
+ it is considered retired and no more tabs will be opened. After the last tab is closed
+ the whole browser is closed too. This parameter defines a time limit for inactivity
+ after which the browser is closed even if there are pending tabs. See
+ <code>killInstanceAfterMillis</code> parameter of <code>PuppeteerPool</code>.
+Defaults to <code>300000</code>.
+</li>
+<li>
+<b><code>launchPuppeteerFunction</code></b>
+(
+<span class="param-type">function</span>
+
+)
+&lt;optional>
+-
+Overrides the default function to launch a new Puppeteer instance.
+ See <code>launchPuppeteerFunction</code> parameter of <code>PuppeteerPool</code>.
+ See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/puppeteer_crawler.js#L9">GitHub</a> for default behavior.
+</li>
+<li>
+<b><code>launchPuppeteerOptions</code></b>
+(
+<span class="param-type"><a href="#LaunchPuppeteerOptions">LaunchPuppeteerOptions</a></span>
+
+)
+&lt;optional>
+-
+Options used by <code>Apify.launchPuppeteer()</code> to start new Puppeteer instances.
+ See <code>launchPuppeteerOptions</code> parameter of <code>PuppeteerPool</code>.
+</li>
+<li>
+<b><code>autoscaledPoolOptions</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+-
+Custom options passed to the underlying <a href="#AutoscaledPool"><code>AutoscaledPool</code></a> instance constructor.
+ Note that the <code>runTaskFunction</code>, <code>isTaskReadyFunction</code> and <code>isFinishedFunction</code> options
+ are provided by <code>PuppeteerCrawler</code> and should not be overridden.
+</li>
+<li>
+<b><code>minConcurrency</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+-
+Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding <code>AutoscaledPool</code> option.
+Defaults to <code>1</code>.
+</li>
+<li>
+<b><code>maxConcurrency</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+-
+Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding <code>AutoscaledPool</code> option.
+Defaults to <code>1000</code>.
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<dl class="details">
+ <dt class="tag-see">See:</dt>
+ <dd class="tag-see">
+ <ul>
+ <li><a href="#CheerioCrawler">CheerioCrawler</a></li>
+<li><a href="#BasicCrawler">BasicCrawler</a></li>
+ </ul>
+ </dd>
+ </dl>
+</section>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Methods (2)</h3>
+<section id="PuppeteerCrawler-abort" class="sub-sec">
+<h4>abort<span class="signature">()</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Stops the crawler by preventing crawls of additional pages and terminating the running ones.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="PuppeteerCrawler-run" class="sub-sec">
+<h4>run<span class="signature">()</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Runs the crawler. Returns promise that gets resolved once all the requests got processed.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+</div>
+</article>
+
+</div>
+<div id="PuppeteerPool">
+
+<header>
+<h2>
+ PuppeteerPool
+ </h2>
+<div class="class-description"><p>Manages a pool of Chrome browser instances controlled using <a href="https://github.com/GoogleChrome/puppeteer">Puppeteer</a>.
+<code>PuppeteerPool</code> reuses Chrome instances and tabs using specific
+browser rotation and retirement policies.
+This is useful in order to facilitate rotation of proxies, cookies
+or other settings in order to prevent detection of your web scraping bot,
+access web pages from various countries etc.
+Additionally, the reuse of browser instances instances speeds up crawling,
+and the retirement of instances helps mitigate effects of memory leaks in Chrome.</p>
+<p><code>PuppeteerPool</code> is internally used by the <a href="#PuppeteerCrawler"><code>PuppeteerCrawler</code></a> class.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>const puppeteerPool = new PuppeteerPool({
+ launchPuppeteerFunction: () => {
+ // Use a new proxy with a new IP address for each new Chrome instance
+ return Apify.launchPuppeteer({
+ apifyProxySession: Math.random(),
+ });
+ },
+});
+
+const page1 = await puppeteerPool.newPage();
+const page2 = await puppeteerPool.newPage();
+const page3 = await puppeteerPool.newPage();
+
+// ... do something with the pages ...
+
+// Close all browsers.
+await puppeteerPool.destroy();</code></pre></div>
+</header>
+
+<article>
+ <div class="container-overview">
+<section id="PuppeteerPool" class="sub-sec">
+<h2>Constructor</h2>
+<h4>new PuppeteerPool<span class="signature">()</span><span class="type-signature"></span></h4>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>options.maxOpenPagesPerInstance</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Maximum number of open pages (i.e. tabs) per browser. When this limit is reached, new pages are loaded in a new browser instance.
+Defaults to <code>50</code>.
+</li>
+<li>
+<b><code>options.retireInstanceAfterRequestCount</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Maximum number of requests that can be processed by a single browser instance.
+ After the limit is reached, the browser is retired and new requests are
+ be handled by a new browser instance.
+Defaults to <code>100</code>.
+</li>
+<li>
+<b><code>options.instanceKillerIntervalMillis</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Indicates how often opened Puppeteer instances are checked whether they can be closed.
+Defaults to <code>60000</code>.
+</li>
+<li>
+<b><code>options.killInstanceAfterMillis</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+When Puppeteer instance reaches the <code>options.retireInstanceAfterRequestCount</code> limit then
+ it is considered retired and no more tabs will be opened. After the last tab is closed the
+ whole browser is closed too. This parameter defines a time limit between the last tab was opened and
+ before the browser is closed even if there are pending open tabs.
+Defaults to <code>300000</code>.
+</li>
+<li>
+<b><code>options.launchPuppeteerFunction</code></b>
+(
+<span class="param-type">function</span>
+
+)
+&lt;optional>
+-
+Overrides the default function to launch a new <code>Puppeteer</code> instance.
+Defaults to <code>launchPuppeteerOptions&nbsp;=>&nbsp;Apify.launchPuppeteer(launchPuppeteerOptions)</code>.
+</li>
+<li>
+<b><code>options.launchPuppeteerOptions</code></b>
+(
+<span class="param-type"><a href="#LaunchPuppeteerOptions">LaunchPuppeteerOptions</a></span>
+
+)
+&lt;optional>
+-
+Options used by <code>Apify.launchPuppeteer()</code> to start new Puppeteer instances.
+</li>
+<li>
+<b><code>options.recycleDiskCache</code></b>
+(
+<span class="param-type">Boolean</span>
+
+)
+&lt;optional>
+-
+Enables recycling of disk cache directories by Chrome instances.
+ When a browser instance is closed, its disk cache directory is not deleted but it's used by a newly opened browser instance.
+ This is useful to reduce amount of data that needs to be downloaded to speed up crawling and reduce proxy usage.
+ Note that the new browser starts with empty cookies, local storage etc. so this setting doesn't affect anonymity of your crawler.
+<p> Beware that the disk cache directories can consume a lot of disk space.
+ To limit the space consumed, you can pass the <code>--disk-cache-size=X</code> argument to <code>options.launchPuppeteerOptions.args</code>,
+ where <code>X</code> is the approximate maximum number of bytes for disk cache.</p>
+<p> <em>IMPORTANT:</em> Currently this feature only works in headful mode, because of a bug in Chromium.</p>
+<p> The <code>options.recycleDiskCache</code> setting should not be used together with <code>--disk-cache-dir</code> argument in <code>options.launchPuppeteerOptions.args</code>.</p>
+</li>
+</ul>
+</div>
+</section>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Methods (3)</h3>
+<section id="PuppeteerPool-destroy" class="sub-sec">
+<h4>destroy<span class="signature">()</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Closes all the browsers.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="PuppeteerPool-newPage" class="sub-sec">
+<h4>newPage<span class="signature">()</span><span class="type-signature"> &rarr; {Promise.&lt;Puppeteer.Page&gt;}</span></h4>
+<div class="description">
+ <p>Opens new tab in one of the browsers and returns promise that resolves to its Puppeteer.Page.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;Puppeteer.Page&gt;</span>
+
+)
+</li>
+</section>
+<section id="PuppeteerPool-retire" class="sub-sec">
+<h4>retire<span class="signature">(browser)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Manually retires a Puppeteer Browser instance from the pool. The browser will continue
+to process open pages so that they may gracefully finish. This is unlike browser.close()
+which will forcibly terminate the browser and all open pages will be closed.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>browser</code></b>
+(
+<span class="param-type">Puppeteer.Browser</span>
+
+)
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+</div>
+</article>
+
+</div>
+<div id="Request">
+
+<header>
+<h2>
+ Request
+ </h2>
+<div class="class-description"><p>Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
+The <code>Request</code> object also stores information about errors that occurred during processing of the request.</p>
+<p>Each <code>Request</code> instance has the <code>uniqueKey</code> property, which is be either specified
+manually in constructor or generated automatically from the URL. Two requests with the same <code>uniqueKey</code>
+are considered as pointing to the same web page. This behavior applies to all Apify SDK classes,
+such as <a href="#RequestList"><code>RequestList</code></a>, <a href="#RequestQueue"><code>RequestQueue</code></a> or <a href="#PuppeteerCrawler"><code>PuppeteerCrawler</code></a>.</p>
+<p>Example use:</p>
+<pre class="prettyprint source lang-javascript"><code>const request = new Apify.Request({
+ url: 'http://www.example.com',
+ headers: { Accept: 'application/json' },
+});
+
+...
+
+request.userData.foo = 'bar';
+request.pushErrorMessage(new Error('Request failed!'));
+
+...
+
+const foo = request.userData.foo;</code></pre></div>
+</header>
+
+<article>
+ <div class="container-overview">
+<section id="Request" class="sub-sec">
+<h2>Constructor</h2>
+<h4>new Request<span class="signature">(opts)</span><span class="type-signature"></span></h4>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>opts</code></b>
+(
+<span class="param-type">object</span>
+
+)
+<div class="params">
+<ul>
+<li>
+<b><code>url</code></b>
+(
+<span class="param-type">String</span>
+
+)
+-
+URL of the web page to crawl.
+</li>
+<li>
+<b><code>uniqueKey</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+A unique key identifying the request.
+Two requests with the same <code>uniqueKey</code> are considered as pointing to the same URL.
+<p>If <code>uniqueKey</code> is not provided, then it is automatically generated by normalizing the URL.
+For example, the URL of <code>HTTP://www.EXAMPLE.com/something/</code> will be generated the <code>uniqueKey</code>
+of <code>http://www.example.com/something</code>. The <code>keepUrlFragment</code> option
+determines whether URL hash fragment is included in the <code>uniqueKey</code> or not. Beware that the HTTP method
+and payload is not included in the <code>uniqueKey</code>, so requests to the same URL but with different
+HTTP methods or different POST payloads are all considered equal.</p>
+<p>You can set <code>uniqueKey</code> property to arbitrary non-empty text value in order
+to override the default behavior and specify which URLs shall be considered equal.</p>
+</li>
+<li>
+<b><code>method</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+Defaults to <code>'GET'</code>.
+</li>
+<li>
+<b><code>payload</code></b>
+(
+<span class="param-type">String</span>
+|
+
+<span class="param-type">Buffer</span>
+
+)
+&lt;optional>
+-
+HTTP request payload, e.g. for POST requests.
+</li>
+<li>
+<b><code>retryCount</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+&lt;optional>
+-
+Indicates how many times the URL was retried in a case of error.
+Defaults to <code>0</code>.
+</li>
+<li>
+<b><code>errorMessages</code></b>
+(
+<span class="param-type">Array.&lt;String&gt;</span>
+
+)
+&lt;optional>
+-
+An array of error messages from request processing.
+</li>
+<li>
+<b><code>headers</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+HTTP headers.
+Defaults to <code>{}</code>.
+</li>
+<li>
+<b><code>userData</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+-
+Custom user data assigned to the request.
+Defaults to <code>{}</code>.
+</li>
+<li>
+<b><code>keepUrlFragment</code></b>
+(
+<span class="param-type">Boolean</span>
+
+)
+&lt;optional>
+-
+If <code>false</code> then hash part is removed from the URL when computing the <code>uniqueKey</code> property.
+ For example, this causes the <code>http://www.example.com#foo</code> and <code>http://www.example.com#bar</code> URLs
+ to have the same <code>uniqueKey</code> of <code>http://www.example.com</code> and thus the URLs are considered equal.
+ Note that this option only has effect if <code>uniqueKey</code> is not set.
+Defaults to <code>false</code>.
+</li>
+<li>
+<b><code>ignoreErrors</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+If <code>true</code> then errors in processing of this will be ignored.
+ For example, the request won't be retried in a case of an error for example.
+Defaults to <code>false</code>.
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<dl class="details">
+ <dt class="tag-see">See:</dt>
+ <dd class="tag-see">
+ <ul>
+ <li><a href="#RequestList">RequestList</a></li>
+<li><a href="#RequestQueue">RequestQueue</a></li>
+ </ul>
+ </dd>
+ </dl>
+</section>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Methods (1)</h3>
+<section id="Request-pushErrorMessage" class="sub-sec">
+<h4>pushErrorMessage<span class="signature">(errorOrMessage)</span><span class="type-signature"></span></h4>
+<div class="description">
+ <p>Stores information about an error occurred during processing of this request.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>errorOrMessage</code></b>
+(
+<span class="param-type">Error</span>
+|
+
+<span class="param-type">String</span>
+
+)
+-
+Error object or error message to be stored in request.
+</li>
+</ul>
+</div>
+</section>
+</div>
+</article>
+
+</div>
+<div id="RequestList">
+
+<header>
+<h2>
+ RequestList
+ </h2>
+<div class="class-description"><p>Represents a static list of URLs to crawl.
+The URLs can be provided either in code or parsed from a text file hosted on the web.</p>
+<p>Each URL is represented using an instance of the <a href="#Request"><code>Request</code></a> class.
+The list can only contain unique URLs. More precisely, it can only contain <code>Request</code> instances
+with distinct <code>uniqueKey</code> properties. By default, <code>uniqueKey</code> is generated from the URL, but it can also be overridden.
+To add a single URL multiple times to the list,
+corresponding <code>Request</code> objects will need to have different <code>uniqueKey</code> properties.
+You can use the <code>keepDuplicateUrls</code> option to do this for you.</p>
+<p>Once you create an instance of <code>RequestList</code>, you need to call <a href="#RequestList-initialize"><code>initialize()</code></a>
+before the instance can be used. After that, no more URLs can be added to the list.</p>
+<p><code>RequestList</code> is used by <a href="#BasicCrawler"><code>BasicCrawler</code></a>, <a href="#CheerioCrawler"><code>CheerioCrawler</code></a>
+and <a href="#PuppeteerCrawler"><code>PuppeteerCrawler</code></a> as a source of URLs to crawl.
+Unlike <a href="#RequestQueue"><code>RequestQueue</code></a>, <code>RequestList</code> is static but it can contain even millions of URLs.</p>
+<p><code>RequestList</code> has an internal state where it stores information which requests were handled,
+which are in progress or which were reclaimed.
+The state might be automatically persisted to the default key-value store by setting the <code>persistStateKey</code> option
+so that if the Node.js process is restarted,
+the crawling can continue where it left off. For more details, see <a href="#KeyValueStore"><code>KeyValueStore</code></a>.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>const requestList = new Apify.RequestList({
+ sources: [
+ // Separate requests
+ { url: 'http://www.example.com/page-1', method: 'GET', headers: {} },
+ { url: 'http://www.example.com/page-2', userData: { foo: 'bar' }},
+// Bulk load of URLs from file `http://www.example.com/my-url-list.txt`
+ // Note that all URLs must start with http:// or https://
+ { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },
+ ],
+ persistStateKey: 'my-crawling-state'
+});
+
+// This call loads and parses the URLs from the remote file.
+await requestList.initialize();
+
+// Get requests from list
+const request1 = await requestList.fetchNextRequest();
+const request2 = await requestList.fetchNextRequest();
+const request3 = await requestList.fetchNextRequest();
+
+// Mark some of them as handled
+await requestList.markRequestHandled(request1);
+
+// If processing fails then reclaim it back to the list
+await requestList.reclaimRequest(request2);</code></pre></div>
+</header>
+
+<article>
+ <div class="container-overview">
+<section id="RequestList" class="sub-sec">
+<h2>Constructor</h2>
+<h4>new RequestList<span class="signature">(options)</span><span class="type-signature"></span></h4>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>options</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+<div class="params">
+<ul>
+<li>
+<b><code>sources</code></b>
+(
+<span class="param-type">Array</span>
+
+)
+-
+An array of sources for the <code>RequestList</code>. Its contents can either be just plain objects,
+ defining at least the 'url' property or instances of the <a href="#Request"><code>Request</code></a> class.
+ Additionally a <code>requestsFromUrl</code> property may be used instead of <code>url</code>,
+ which will instruct the <code>RequestList</code> to download the sources from the given remote location.
+ The URLs will be parsed from the received response.
+<pre class="prettyprint source lang-javascript"><code>[
+ // One URL
+ { method: 'GET', url: 'http://example.com/a/b' },
+ // Batch import of URLs from a file hosted on the web
+ { method: 'POST', requestsFromUrl: 'http://example.com/urls.txt' },
+]</code></pre>
+</li>
+<li>
+<b><code>persistStateKey</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+Identifies the key in the default key-value store under which the <code>RequestList</code> persists its state.
+ If this is set then <code>RequestList</code>
+ persists its state in regular intervals and loads the state from there in case it is restarted
+ due to an error or system reboot.
+</li>
+<li>
+<b><code>state</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+-
+The state object that the <code>RequestList</code> will be initialized from.
+ It is in the form as returned by <code>RequestList.getState()</code>, such as follows:
+<pre class="prettyprint source lang-javascript"><code>{
+ nextIndex: 5,
+ nextUniqueKey: 'unique-key-5'
+ inProgress: {
+ 'unique-key-1': true,
+ 'unique-key-4': true,
+ },
+}</code></pre><p> Note that the preferred (and simpler) way to persist the state of crawling of the <code>RequestList</code>
+ is to use the <code>persistStateKey</code> parameter instead.</p>
+</li>
+<li>
+<b><code>keepDuplicateUrls</code></b>
+(
+<span class="param-type">Boolean</span>
+
+)
+&lt;optional>
+-
+By default, <code>RequestList</code> will deduplicate the provided URLs. Default deduplication is based
+ on the <code>uniqueKey</code> property of passed source <a href="#Request">Request</a> objects. If the property is not present,
+ it is generated by normalizing the URL. If present, it is kept intact. In any case, only one request per <code>uniqueKey</code> is added
+ to the <code>RequestList</code> resulting in removing of duplicate URLs / unique keys.
+ Setting <code>keepDuplicateUrls</code> to <code>true</code> will append an additional identifier to the <code>uniqueKey</code>
+ of each request that does not already include a <code>uniqueKey</code>. Therefore, duplicate
+ URLs will be kept in the list. It does not protect the user from having duplicates in user set
+ <code>uniqueKey</code>s however. It is the user's responsibility to ensure uniqueness of their unique keys,
+ if they wish to keep more than just a single copy in the <code>RequestList</code>.
+Defaults to <code>false</code>.
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+</section>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Methods (8)</h3>
+<section id="RequestList-fetchNextRequest" class="sub-sec">
+<h4>fetchNextRequest<span class="signature">()</span><span class="type-signature"> &rarr; {Promise.&lt;<a href="#Request">Request</a>&gt;}</span></h4>
+<div class="description">
+ <p>Gets the next <code>Request</code> to process. First, the function gets a request previously reclaimed
+using <a href="#RequestList-reclaimRequest"><code>reclaimRequest()</code></a> function, if there is any.
+Otherwise it gets a next request from the sources.</p>
+<p>The function gets <code>null</code> if there are no more
+requests to process.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;<a href="#Request">Request</a>&gt;</span>
+
+)
+</li>
+</section>
+<section id="RequestList-getState" class="sub-sec">
+<h4>getState<span class="signature">()</span><span class="type-signature"></span></h4>
+<div class="description">
+ <p>Returns an object representing the internal state of the <code>RequestList</code> instance.
+Note that the objects fields can change in future releases.</p>
+</div>
+<h5>Returns:</h5>
+</section>
+<section id="RequestList-initialize" class="sub-sec">
+<h4>initialize<span class="signature">()</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Loads all remote sources of URLs and potentially starts periodic state persistence.
+This function must be called before you can start using the instance in a meaningful way.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="RequestList-isEmpty" class="sub-sec">
+<h4>isEmpty<span class="signature">()</span><span class="type-signature"> &rarr; {Promise.&lt;Boolean&gt;}</span></h4>
+<div class="description">
+ <p>Resolves to <code>true</code> if the next call to <a href="#RequestList-fetchNextRequest"><code>fetchNextRequest()</code></a>
+will return <code>null</code>, otherwise it resolves to <code>false</code>.
+Note that even if the list is empty, there might be some pending requests currently being processed.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;Boolean&gt;</span>
+
+)
+</li>
+</section>
+<section id="RequestList-isFinished" class="sub-sec">
+<h4>isFinished<span class="signature">()</span><span class="type-signature"> &rarr; {Promise.&lt;boolean&gt;}</span></h4>
+<div class="description">
+ <p>Returns <code>true</code> if all requests were already handled and there are no more left.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;boolean&gt;</span>
+
+)
+</li>
+</section>
+<section id="RequestList-length" class="sub-sec">
+<h4>length<span class="signature">()</span><span class="type-signature"></span></h4>
+<div class="description">
+ <p>Returns the total number of unique requests present in the <code>RequestList</code>.</p>
+</div>
+</section>
+<section id="RequestList-markRequestHandled" class="sub-sec">
+<h4>markRequestHandled<span class="signature">(request)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Marks request as handled after successful processing.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>request</code></b>
+(
+<span class="param-type"><a href="#Request">Request</a></span>
+
+)
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="RequestList-reclaimRequest" class="sub-sec">
+<h4>reclaimRequest<span class="signature">(request)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Reclaims request to the list if its processing failed.
+The request will become available in the next <code>this.fetchNextRequest()</code>.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>request</code></b>
+(
+<span class="param-type"><a href="#Request">Request</a></span>
+
+)
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+</div>
+</article>
+
+</div>
+<div id="RequestQueue">
+
+<header>
+<h2>
+ RequestQueue
+ </h2>
+<div class="class-description"><p>Represents a queue of URLs to crawl, which is used for deep crawling of websites
+where you start with several URLs and then recursively
+follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.</p>
+<p>Each URL is represented using an instance of the <a href="#Request"><code>Request</code></a> class.
+The queue can only contain unique URLs. More precisely, it can only contain <code>Request</code> instances
+with distinct <code>uniqueKey</code> properties. By default, <code>uniqueKey</code> is generated from the URL, but it can also be overridden.
+To add a single URL multiple times to the queue,
+corresponding <code>Request</code> objects will need to have different <code>uniqueKey</code> properties.</p>
+<p>Do not instantiate this class directly, use the
+<code>Apify.openRequestQueue()</code> function instead.</p>
+<p><code>RequestQueue</code> is used by <a href="#BasicCrawler"><code>BasicCrawler</code></a>, <a href="#CheerioCrawler"><code>CheerioCrawler</code></a>
+and <a href="#PuppeteerCrawler"><code>PuppeteerCrawler</code></a> as a source of URLs to crawl.
+Unlike <a href="#RequestList"><code>RequestList</code></a>, <code>RequestQueue</code> supports dynamic adding and removing of requests.
+On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.</p>
+<p><code>RequestQueue</code> stores its data either on local disk or in the Apify cloud,
+depending on whether the <code>APIFY_LOCAL_STORAGE_DIR</code> or <code>APIFY_TOKEN</code> environment variable is set.</p>
+<p>If the <code>APIFY_LOCAL_STORAGE_DIR</code> environment variable is set, the queue data is stored in
+that local directory as follows:</p>
+<pre class="prettyprint source"><code>[APIFY_LOCAL_STORAGE_DIR]/request_queues/[QUEUE_ID]/[STATE]/[NUMBER].json</code></pre><p>Note that <code>[QUEUE_ID]</code> is the name or ID of the request queue. The default queue has ID <code>default</code>,
+unless you override it by setting the <code>APIFY_DEFAULT_REQUEST_QUEUE_ID</code> environment variable.
+Each request in the queue is stored as a separate JSON file, where <code>[STATE]</code> is either <code>handled</code> or <code>pending</code>,
+and <code>[NUMBER]</code> is an integer indicating the position of the request in the queue.</p>
+<p>If the <code>APIFY_TOKEN</code> environment variable is provided instead, the data is stored
+in the <a href="https://www.apify.com/docs/storage#queue">Apify Request Queue</a> cloud storage.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>// Open the default request queue associated with the actor run
+const queue = await Apify.openRequestQueue();
+
+// Open a named request queue
+const queueWithName = await Apify.openRequestQueue('some-name');
+
+// Enqueue few requests
+await queue.addRequest(new Apify.Request({ url: 'http://example.com/aaa'}));
+await queue.addRequest(new Apify.Request({ url: 'http://example.com/bbb'}));
+await queue.addRequest(new Apify.Request({ url: 'http://example.com/foo/bar'}), { forefront: true });
+
+// Get requests from queue
+const request1 = await queue.fetchNextRequest();
+const request2 = await queue.fetchNextRequest();
+const request3 = await queue.fetchNextRequest();
+
+// Mark a request as handled
+await queue.markRequestHandled(request1);
+
+// If processing of a request fails then reclaim it back to the queue, so that it's crawled again
+await queue.reclaimRequest(request2);</code></pre></div>
+</header>
+
+<article>
+ <div class="container-overview">
+<section id="RequestQueue" class="sub-sec">
+<h2>Constructor</h2>
+<h4>new RequestQueue<span class="signature">()</span><span class="type-signature"></span></h4>
+</section>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Methods (8)</h3>
+<section id="RequestQueue-addRequest" class="sub-sec">
+<h4>addRequest<span class="signature">(request, opts<span class="signature-attributes">opt</span>)</span><span class="type-signature"> &rarr; {<a href="#RequestOperationInfo">RequestOperationInfo</a>}</span></h4>
+<div class="description">
+ <p>Adds a request to the queue.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>request</code></b>
+(
+<span class="param-type"><a href="#Request">Request</a></span>
+
+)
+-
+Request object
+</li>
+<li>
+<b><code>opts</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+<div class="params">
+<ul>
+<li>
+<b><code>forefront</code></b>
+(
+<span class="param-type">Boolean</span>
+
+)
+&lt;optional>
+-
+If <code>true</code>, the request will be added to the foremost position in the queue.
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type"><a href="#RequestOperationInfo">RequestOperationInfo</a></span>
+
+)
+</li>
+</section>
+<section id="RequestQueue-delete" class="sub-sec">
+<h4>delete<span class="signature">()</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Removes the queue either from the Apify cloud storage or from the local directory,
+depending on the mode of operation.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="RequestQueue-fetchNextRequest" class="sub-sec">
+<h4>fetchNextRequest<span class="signature">()</span><span class="type-signature"> &rarr; {Promise.&lt;<a href="#Request">Request</a>&gt;}</span></h4>
+<div class="description">
+ <p>Returns next request in the queue to be processed.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;<a href="#Request">Request</a>&gt;</span>
+
+)
+</li>
+</section>
+<section id="RequestQueue-getRequest" class="sub-sec">
+<h4>getRequest<span class="signature">(requestId)</span><span class="type-signature"> &rarr; {Promise.&lt;<a href="#Request">Request</a>&gt;}</span></h4>
+<div class="description">
+ <p>Gets the request from the queue specified by ID.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>requestId</code></b>
+(
+<span class="param-type">String</span>
+
+)
+-
+Request ID
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;<a href="#Request">Request</a>&gt;</span>
+
+)
+</li>
+</section>
+<section id="RequestQueue-isEmpty" class="sub-sec">
+<h4>isEmpty<span class="signature">()</span><span class="type-signature"> &rarr; {Promise.&lt;Boolean&gt;}</span></h4>
+<div class="description">
+ <p>Resolves to <code>true</code> if the next call to <code>fetchNextRequest()</code> will return <code>null</code>, otherwise it resolves to <code>false</code>.
+Note that even if the queue is empty, there might be some pending requests currently being processed.</p>
+<p>Due to the nature of distributed storage systems,
+the function might occasionally return a false negative, but it should never return a false positive!</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;Boolean&gt;</span>
+
+)
+</li>
+</section>
+<section id="RequestQueue-isFinished" class="sub-sec">
+<h4>isFinished<span class="signature">()</span><span class="type-signature"> &rarr; {Promise.&lt;Boolean&gt;}</span></h4>
+<div class="description">
+ <p>Resolves to <code>true</code> if all requests were already handled and there are no more left.
+Due to the nature of distributed storage systems,
+the function might occasionally return a false negative, but it will never return a false positive.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;Boolean&gt;</span>
+
+)
+</li>
+</section>
+<section id="RequestQueue-markRequestHandled" class="sub-sec">
+<h4>markRequestHandled<span class="signature">(request)</span><span class="type-signature"> &rarr; {Promise.&lt;<a href="#RequestOperationInfo">RequestOperationInfo</a>&gt;}</span></h4>
+<div class="description">
+ <p>Marks request handled after successfull processing.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>request</code></b>
+(
+<span class="param-type"><a href="#Request">Request</a></span>
+
+)
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;<a href="#RequestOperationInfo">RequestOperationInfo</a>&gt;</span>
+
+)
+</li>
+</section>
+<section id="RequestQueue-reclaimRequest" class="sub-sec">
+<h4>reclaimRequest<span class="signature">(request, opts<span class="signature-attributes">opt</span>)</span><span class="type-signature"> &rarr; {Promise.&lt;<a href="#RequestOperationInfo">RequestOperationInfo</a>&gt;}</span></h4>
+<div class="description">
+ <p>Reclaims failed request back to the queue,
+so that it can be processed later again.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>request</code></b>
+(
+<span class="param-type"><a href="#Request">Request</a></span>
+
+)
+</li>
+<li>
+<b><code>opts</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+&lt;optional>
+<div class="params">
+<ul>
+<li>
+<b><code>forefront</code></b>
+(
+<span class="param-type">Boolean</span>
+
+)
+&lt;optional>
+-
+If true then requests gets returned to the begining of the queue
+ and to the back of the queue otherwise.
+Defaults to <code>false</code>.
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;<a href="#RequestOperationInfo">RequestOperationInfo</a>&gt;</span>
+
+)
+</li>
+</section>
+</div>
+</article>
+
+</div>
+<div id="SettingsRotator">
+
+<header>
+<h2>
+ SettingsRotator
+ </h2>
+<div class="class-description"><p>Rotates settings created by a user-provided function passed
+via <code>newSettingsFunction</code>.
+This is useful during web crawling to dynamically change settings and thus
+avoid detection of the crawler.</p>
+<p>This class is still work in progress, more features will be added soon.</p></div>
+</header>
+
+<article>
+ <div class="container-overview">
+<section id="SettingsRotator" class="sub-sec">
+<h2>Constructor</h2>
+<h4>new SettingsRotator<span class="signature">(options)</span><span class="type-signature"></span></h4>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>options</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+<div class="params">
+<ul>
+<li>
+<b><code>newSettingsFunction</code></b>
+(
+<span class="param-type">function</span>
+
+)
+</li>
+<li>
+<b><code>maxUsages</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+</section>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Methods (2)</h3>
+<section id="SettingsRotator-fetchSettings" class="sub-sec">
+<h4>fetchSettings<span class="signature">()</span><span class="type-signature"> &rarr; {*}</span></h4>
+<div class="description">
+ <p>Fetches a settings object.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">*</span>
+
+)
+</li>
+</section>
+<section id="SettingsRotator-reclaimSettings" class="sub-sec">
+<h4>reclaimSettings<span class="signature">(settings)</span><span class="type-signature"></span></h4>
+<div class="description">
+ <p>Reclaims settings after use.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>settings</code></b>
+(
+<span class="param-type">*</span>
+
+)
+</li>
+</ul>
+</div>
+</section>
+</div>
+</article>
+
+</div>
+<div id="utils">
+
+<header>
+<h2>
+ utils
+ </h2>
+</header>
+
+<article>
+ <div class="container-overview">
+<div class="description"><p>A namespace that contains various utilities.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>const Apify = require('apify);
+
+...
+
+// Sleep 1.5 seconds
+await Apify.utils.sleep(1500);</code></pre></div>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Members (2)</h3>
+<div class="members">
+ <h4 class="name" id="utils-URL-NO-COMMAS-REGEX"><span class="type-signature">(static, constant) </span>URL_NO_COMMAS_REGEX<span class="type-signature"></span></h4>
+<div class="description">
+ <p>Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters
+and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).</p>
+ </div>
+</div>
+<div class="members">
+ <h4 class="name" id="utils-URL-WITH-COMMAS-REGEX"><span class="type-signature">(static, constant) </span>URL_WITH_COMMAS_REGEX<span class="type-signature"></span></h4>
+<div class="description">
+ <p>Regular expression that, in addition to the default regular expression URL_NO_COMMAS_REGEX, supports matching commas in URL path and query.
+Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.</p>
+ </div>
+</div>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Methods (4)</h3>
+<section id="utils-downloadListOfUrls" class="sub-sec">
+<h4>downloadListOfUrls<span class="signature">(url, encoding<span class="signature-attributes">opt</span>, urlRegExp<span class="signature-attributes">opt</span>)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Returns a promise that resolves to an array of urls parsed from the resource available at the provided url.
+Optionally, custom regular expression and encoding may be provided.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>url</code></b>
+(
+<span class="param-type">String</span>
+
+)
+-
+</li>
+<li>
+<b><code>encoding</code></b>
+(
+<span class="param-type">String</span>
+
+)
+&lt;optional>
+-
+Defaults to <code>'utf8'</code>.
+</li>
+<li>
+<b><code>urlRegExp</code></b>
+(
+<span class="param-type">RegExp</span>
+
+)
+&lt;optional>
+-
+Defaults to <code>URL_NO_COMMAS_REGEX</code>.
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="utils-extractUrls" class="sub-sec">
+<h4>extractUrls<span class="signature">(string, urlRegExp<span class="signature-attributes">opt</span>)</span><span class="type-signature"> &rarr; {Array}</span></h4>
+<div class="description">
+ <p>Collects all URLs in an arbitrary string to an array, optionally using a custom regular expression.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>string</code></b>
+(
+<span class="param-type">String</span>
+
+)
+-
+</li>
+<li>
+<b><code>urlRegExp</code></b>
+(
+<span class="param-type">RegExp</span>
+
+)
+&lt;optional>
+-
+Defaults to <code>URL_NO_COMMAS_REGEX</code>.
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Array</span>
+
+)
+</li>
+</section>
+<section id="utils-getRandomUserAgent" class="sub-sec">
+<h4>getRandomUserAgent<span class="signature">()</span><span class="type-signature"> &rarr; {String}</span></h4>
+<div class="description">
+ <p>Returns a randomly selected User-Agent header out of a list of the most common headers.</p>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">String</span>
+
+)
+</li>
+</section>
+<section id="utils-sleep" class="sub-sec">
+<h4>sleep<span class="signature">(millis)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Returns a promise that resolves after a specific period of time. This is useful to implement waiting
+in your code, e.g. to prevent overloading of target website or to avoid bot detection.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>const Apify = require('apify);
+
+...
+
+// Sleep 1.5 seconds
+await Apify.utils.sleep(1500);</code></pre>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>millis</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+-
+Period of time to sleep, in milliseconds. If not a positive number, the returned promise resolves immediately.
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+</div>
+</article>
+
+</div>
+<div id="utils-puppeteer">
+
+<header>
+<h2>
+ utils.puppeteer
+ </h2>
+</header>
+
+<article>
+ <div class="container-overview">
+<div class="description"><p>A namespace that contains various Puppeteer utilities.</p>
+<p><strong>Example usage:</strong></p>
+<pre class="prettyprint source lang-javascript"><code>const Apify = require('apify');
+
+// Open https://www.example.com in Puppeteer
+const browser = await Apify.launchPuppeteer();
+const page = await browser.newPage();
+await page.goto('https://www.example.com');
+
+// Inject jQuery into a page
+await Apify.utils.puppeteer.injectJQuery(page);</code></pre></div>
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Methods (8)</h3>
+<section id="utils-puppeteer-blockResources" class="sub-sec">
+<h4>blockResources<span class="signature">(page, resourceTypes)</span><span class="type-signature"> &rarr; {Promise.&lt;void&gt;}</span></h4>
+<div class="description">
+ <p>Forces the browser tab to block loading certain page resources,
+using the <code>Page.setRequestInterception(value)</code> method.
+This is useful to speed up crawling of websites.</p>
+<p>The resource types to block can be controlled using the <code>resourceTypes</code> parameter,
+which indicates the types of resources as they are perceived by the rendering engine.
+The following resource types are currently supported:
+<code>document</code>, <code>stylesheet</code>, <code>image</code>, <code>media</code>, <code>font</code>, <code>script</code>, <code>texttrack</code>, <code>xhr</code>, <code>fetch</code>,
+<code>eventsource</code>, <code>websocket</code>, <code>manifest</code>, <code>other</code>.
+For more details, see Puppeteer's
+<a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#requestresourcetype">Request.resourceType() documentation</a>.</p>
+<p>By default, the function blocks these resource types: <code>stylesheet</code>, <code>font</code>, <code>image</code>, <code>media</code>.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>page</code></b>
+(
+<span class="param-type">Page</span>
+
+)
+-
+Puppeteer's <code>Page</code> object
+</li>
+<li>
+<b><code>resourceTypes</code></b>
+(
+<span class="param-type">Array.&lt;String&gt;</span>
+
+)
+-
+Array of resource types to block.
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;void&gt;</span>
+
+)
+</li>
+</section>
+<section id="utils-puppeteer-cacheResponses" class="sub-sec">
+<h4>cacheResponses<span class="signature">(page, cache, responseUrlRules)</span><span class="type-signature"> &rarr; {Promise.&lt;void&gt;}</span></h4>
+<div class="description">
+ <p>Enables caching of intercepted responses into provided object. Automaticaly enables request interception in puppeteer.
+<em>IMPORTANT</em>: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
+ This issue should be resolved or atleast mitigated in future iterations of this feature.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>page</code></b>
+(
+<span class="param-type">Page</span>
+
+)
+-
+Puppeteer's <code>Page</code> object
+</li>
+<li>
+<b><code>cache</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+-
+Object in which responses are stored
+</li>
+<li>
+<b><code>responseUrlRules</code></b>
+(
+<span class="param-type">Array.&lt;(String|RegExp)&gt;</span>
+
+)
+-
+List of rules that are used to check if response should be cached.
+ String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.&lt;void&gt;</span>
+
+)
+</li>
+</section>
+<section id="utils-puppeteer-compileScript" class="sub-sec">
+<h4>compileScript<span class="signature">(scriptString)</span><span class="type-signature"> &rarr; {function}</span></h4>
+<div class="description">
+ <p>Compiles a Puppeteer script into an async function that may be executed at any time
+by providing it with the following object:</p>
+<pre class="prettyprint source"><code>{
+ page: Puppeteer.Page,
+ request: Apify.Request,
+}</code></pre><p>The function is compiled by using the scriptString parameter as the function's body,
+so any limitations to function bodies apply. Return value of the function is the return
+value of the function body = scriptString parameter.</p>
+<p>As a security measure, no globals such as 'process' or 'require' are accessible
+from within the function body. Note that the function does not provide a safe
+sandbox and even though globals are not easily accessible, malicious code may
+still execute in the main process via prototype manipulation. Therefore you
+should only use this function to execute sanitized or safe code.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>scriptString</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">function</span>
+
+)
+- async ({ page, request }) =&gt; { scriptString }
+
+</li>
+</section>
+<section id="utils-puppeteer-enqueueLinks" class="sub-sec">
+<h4>enqueueLinks<span class="signature">(page, selector, pseudoUrls, requestQueue)</span><span class="type-signature"> &rarr; {Promise.<Array.&lt;<a href="#RequestOperationInfo">RequestOperationInfo</a>&gt;}</span></h4>
+<div class="description">
+ <p>Finds HTML elements matching a CSS selector, clicks the elements and if a redirect is triggered and destination URL matches
+one of the provided pseudo-URLs, then the function enqueues that URL to a given request queue.
+To create a Request object function uses <code>requestTemplate</code> from a matching Pseudo-URL.</p>
+<p><em>WARNING</em>: This is work in progress. Currently the function doesn't click elements and only takes their <code>href</code> attribute and so
+ is working only for link (<code>a</code>) elements, but not for buttons or JavaScript links.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>page</code></b>
+(
+<span class="param-type">Page</span>
+
+)
+-
+Puppeteer <a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page">Page</a> object.
+</li>
+<li>
+<b><code>selector</code></b>
+(
+<span class="param-type">String</span>
+
+)
+-
+CSS selector matching elements to be clicked.
+</li>
+<li>
+<b><code>pseudoUrls</code></b>
+(
+<span class="param-type">Array.&lt;<a href="#PseudoUrl">PseudoUrl</a>&gt;</span>
+
+)
+-
+An array of pseudo-URLs matching the URLs to be enqueued.
+</li>
+<li>
+<b><code>requestQueue</code></b>
+(
+<span class="param-type"><a href="#RequestQueue">RequestQueue</a></span>
+
+)
+-
+Request queue object where URLs will be enqueued.
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise.<Array.&lt;<a href="#RequestOperationInfo">RequestOperationInfo</a>&gt;</span>
+
+)
+- Promise that resolves to an array of <a href="#RequestOperationInfo"><code>RequestOperationInfo</code></a> objects.
+
+</li>
+</section>
+<section id="utils-puppeteer-hideWebDriver" class="sub-sec">
+<h4>hideWebDriver<span class="signature">(page)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Hides certain Puppeteer fingerprints from the page, in order to help avoid detection of the crawler.
+The function should be called on a newly-created page object before navigating to the target crawled page.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>page</code></b>
+(
+<span class="param-type">Page</span>
+
+)
+-
+Puppeteer <a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page">Page</a> object.
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="utils-puppeteer-injectFile" class="sub-sec">
+<h4>injectFile<span class="signature">(page, filePath)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Injects a JavaScript file into a Puppeteer page.
+Unlike Puppeteer's <code>addScriptTag</code> function, this function works on pages
+with arbitrary Cross-Origin Resource Sharing (CORS) policies.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>page</code></b>
+(
+<span class="param-type">Page</span>
+
+)
+-
+Puppeteer <a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page">Page</a> object.
+</li>
+<li>
+<b><code>filePath</code></b>
+(
+<span class="param-type">String</span>
+
+)
+-
+File path
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="utils-puppeteer-injectJQuery" class="sub-sec">
+<h4>injectJQuery<span class="signature">(page)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Injects <a href="https://jquery.com/">jQuery</a> library into a Puppeteer page.
+jQuery is often useful for various web scraping and crawling tasks,
+e.g. to extract data from HTML elements using CSS selectors.</p>
+<p>Beware that the injected jQuery object will be set to the <code>window.$</code> variable and thus it might cause conflicts with
+libraries included by the page that use the same variable (e.g. another version of jQuery).</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>page</code></b>
+(
+<span class="param-type">Page</span>
+
+)
+-
+Puppeteer <a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page">Page</a> object.
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+<section id="utils-puppeteer-injectUnderscore" class="sub-sec">
+<h4>injectUnderscore<span class="signature">(page)</span><span class="type-signature"> &rarr; {Promise}</span></h4>
+<div class="description">
+ <p>Injects <a href="https://underscorejs.org/">Underscore.js</a> library into a Puppeteer page.
+Beware that the injected Underscore object will be set to the <code>window._</code> variable and thus it might cause conflicts with
+libraries included by the page that use the same variable.</p>
+</div>
+<h5>Parameters:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>page</code></b>
+(
+<span class="param-type">Page</span>
+
+)
+-
+Puppeteer <a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page">Page</a> object.
+</li>
+</ul>
+</div>
+<h5>Returns:</h5>
+<li>
+ (
+<span class="param-type">Promise</span>
+
+)
+</li>
+</section>
+</div>
+</article>
+
+</div>
+<div id="Globals">
+
+<header>
+<h2>
+ Globals
+ </h2>
+</header>
+
+<article>
+ <div class="container-overview">
+</div>
+<div class="subsection">
+ <h3 class="subsection-title">Type Definitions</h3>
+<div class="members">
+ <h4 class="name" id="ActorRun">ActorRun</h4>
+<div class="description">
+ <p>Represents information about an actor run, as returned by the
+<code>Apify.call()</code> function.
+The object is almost equivalent to the JSON response
+of the
+<a href="https://www.apify.com/docs/api/v2#/reference/actors/run-collection/run-actor">Actor run</a>
+Apify API endpoint and extended with certain fields.
+For more details, see <a href="https://www.apify.com/docs/actor#run">Runs</a> in Apify actor documentation.</p>
+ </div>
+<h5 class="subsection-title">Properties:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>id</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+Actor run ID
+</li>
+<li>
+<b><code>actId</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+Actor ID
+</li>
+<li>
+<b><code>startedAt</code></b>
+(
+<span class="param-type">Date</span>
+
+)
+</td>
+-
+Time when the actor run started
+</li>
+<li>
+<b><code>finishedAt</code></b>
+(
+<span class="param-type">Date</span>
+
+)
+</td>
+-
+Time when the actor run finished. Contains <code>null</code> for running actors.
+</li>
+<li>
+<b><code>status</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+Status of the run. For possible values, see
+ <a href="https://www.apify.com/docs/actor#run-lifecycle">Run lifecycle</a> in Apify actor documentation.
+</li>
+<li>
+<b><code>meta</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+</td>
+-
+Actor run meta-data. For example:
+<pre class="prettyprint source lang-javascript"><code> {
+ &quot;origin&quot;: &quot;API&quot;,
+ &quot;clientIp&quot;: &quot;1.2.3.4&quot;,
+ &quot;userAgent&quot;: &quot;ApifyClient/0.2.13 (Linux; Node/v8.11.3)&quot;
+ }</code></pre>
+</li>
+<li>
+<b><code>stats</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+</td>
+-
+An object containing various actor run statistics. For example:
+<pre class="prettyprint source lang-javascript"><code> {
+ &quot;inputBodyLen&quot;: 22,
+ &quot;restartCount&quot;: 0,
+ &quot;workersUsed&quot;: 1,
+ }</code></pre><p> Beware that object fields might change in future releases.</p>
+</li>
+<li>
+<b><code>options</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+</td>
+-
+Actor run options. For example:
+<pre class="prettyprint source lang-javascript"><code> {
+ &quot;build&quot;: &quot;latest&quot;,
+ &quot;timeoutSecs&quot;: 0,
+ &quot;memoryMbytes&quot;: 256,
+ &quot;diskMbytes&quot;: 512
+ }</code></pre>
+</li>
+<li>
+<b><code>buildId</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+ID of the actor build used for the run. For details, see
+ <a href="https://www.apify.com/docs/actor#build">Builds</a> in Apify actor documentation.
+</li>
+<li>
+<b><code>buildNumber</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+Number of the actor build used for the run. For example, <code>0.0.10</code>.
+</li>
+<li>
+<b><code>exitCode</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+</td>
+-
+Exit code of the actor run process. It's <code>null</code> if actor is still running.
+</li>
+<li>
+<b><code>defaultKeyValueStoreId</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+ID of the default key-value store associated with the actor run. See <a href="#KeyValueStore"><code>KeyValueStore</code></a> for details.
+</li>
+<li>
+<b><code>defaultDatasetId</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+ID of the default dataset associated with the actor run. See <a href="#Dataset"><code>Dataset</code></a> for details.
+</li>
+<li>
+<b><code>defaultRequestQueueId</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+ID of the default request queue associated with the actor run. See <a href="#RequestQueue"><code>RequestQueue</code></a> for details.
+</li>
+<li>
+<b><code>containerUrl</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+URL on which the web server running inside actor run's Docker container can be accessed.
+ For more details, see <a href="https://www.apify.com/docs/actor#container-web-server">Container web server</a>
+ in Apify actor documentation.
+</li>
+<li>
+<b><code>output</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+</td>
+-
+Contains output of the actor run. The value is <code>null</code> or <code>undefined</code> in case the actor is still running,
+ or if you pass <code>false</code> to the <code>fetchOutput</code> option of <code>Apify.call()</code>.
+<p> For example:</p>
+<pre class="prettyprint source lang-javascript"><code> {
+ &quot;contentType&quot;: &quot;application/json; charset=utf-8&quot;,
+ &quot;body&quot;: {
+ &quot;message&quot;: &quot;Hello world!&quot;
+ }
+ }</code></pre>
+</li>
+
+</ul>
+</div>
+</div>
+<div class="members">
+ <h4 class="name" id="ApifyCallError">ApifyCallError</h4>
+<div class="description">
+ <p>The class represents exceptions thrown
+by the <code>Apify.call()</code> function.</p>
+ </div>
+<h5 class="subsection-title">Properties:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>message</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+Error message
+</li>
+<li>
+<b><code>name</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+Contains <code>ApifyCallError</code>
+</li>
+<li>
+<b><code>run</code></b>
+(
+<span class="param-type"><a href="#ActorRun">ActorRun</a></span>
+
+)
+</td>
+-
+Object representing the failed actor run.
+</li>
+
+</ul>
+</div>
+</div>
+<div class="members">
+ <h4 class="name" id="LaunchPuppeteerOptions">LaunchPuppeteerOptions</h4>
+<div class="description">
+ <p>Represents options passed to the <code>Apify.launchPuppeteer()</code>
+function.</p>
+ </div>
+<h5 class="subsection-title">Properties:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>opts.proxyUrl</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+URL to a HTTP proxy server. It must define the port number,
+ and it might also contain proxy username and password.
+<p> For example: <code>http://bob:[email protected]:1234</code>.</p>
+</li>
+<li>
+<b><code>opts.userAgent</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+The <code>User-Agent</code> HTTP header used by the browser.
+ If not provided, the function sets <code>User-Agent</code> to a reasonable default
+ to reduce the chance of detection of the crawler.
+</li>
+<li>
+<b><code>opts.useChrome</code></b>
+(
+<span class="param-type">Boolean</span>
+
+)
+</td>
+-
+If <code>true</code> and <code>opts.executablePath</code> is not set,
+ Puppeteer will launch full Google Chrome browser available on the machine
+ rather than the bundled Chromium. The path to Chrome executable
+ is taken from the <code>APIFY_CHROME_EXECUTABLE_PATH</code> environment variable if provided,
+ or defaults to the typical Google Chrome executable location specific for the operating system.
+ By default, this option is <code>false</code>.
+Defaults to <code>false</code>.
+</li>
+<li>
+<b><code>opts.useApifyProxy</code></b>
+(
+<span class="param-type">Boolean</span>
+
+)
+</td>
+-
+If set to <code>true</code>, Puppeteer will be configured to use
+ <a href="https://my.apify.com/proxy" target="_blank">Apify Proxy</a> for all connections.
+ For more information, see the <a href="https://www.apify.com/docs/proxy">documentation</a>
+Defaults to <code>false</code>.
+</li>
+<li>
+<b><code>opts.apifyProxyGroups</code></b>
+(
+<span class="param-type">Array.&lt;String&gt;</span>
+
+)
+</td>
+-
+An array of proxy groups to be used
+ by the <a href="https://www.apify.com/docs/proxy" target="_blank">Apify Proxy</a>.
+ Only applied if the <code>useApifyProxy</code> option is <code>true</code>.
+</li>
+<li>
+<b><code>opts.apifyProxySession</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+Apify Proxy session identifier to be used by all the Chrome browsers.
+ All HTTP requests going through the proxy with the same session identifier
+ will use the same target proxy server (i.e. the same IP address).
+ The identifier can only contain the following characters: <code>0-9</code>, <code>a-z</code>, <code>A-Z</code>, <code>&quot;.&quot;</code>, <code>&quot;_&quot;</code> and <code>&quot;~&quot;</code>.
+ Only applied if the <code>useApifyProxy</code> option is <code>true</code>.
+</li>
+<li>
+<b><code>opts.liveView</code></b>
+(
+<span class="param-type">Boolean</span>
+
+)
+</td>
+-
+If set to <code>true</code>, a PuppeteerLiveViewServer will be started to enable
+ screenshot and html capturing of visited pages using PuppeteerLiveViewBrowser.
+Defaults to <code>false</code>.
+</li>
+<li>
+<b><code>opts.liveViewOptions</code></b>
+(
+<span class="param-type">Object</span>
+
+)
+</td>
+-
+Settings for <code>PuppeteerLiveViewBrowser</code> started using <code>launchPuppeteer()</code>.
+<div class="params">
+<ul>
+<li>
+<b><code>id</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+Custom ID of a browser instance in live view.
+</li>
+<li>
+<b><code>screenshotTimeoutMillis</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+</td>
+-
+Time in milliseconds before a screenshot capturing
+ will time out and the actor continues with execution. Screenshot capturing pauses execution within the given page.
+</li>
+
+</ul>
+</div>
+</li>
+
+</ul>
+</div>
+</div>
+<div class="members">
+ <h4 class="name" id="PaginationList">PaginationList</h4>
+<div class="description">
+ <p>Represents one page of data items from the <a href="#Dataset"><code>Dataset</code></a>.
+For more details, see <a href="#Dataset-getData"><code>Dataset.getData()</code></a>.</p>
+ </div>
+<h5 class="subsection-title">Properties:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>items</code></b>
+(
+<span class="param-type">Array</span>
+
+)
+</td>
+-
+Array of returned items.
+</li>
+<li>
+<b><code>total</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+</td>
+-
+Total number of object.
+</li>
+<li>
+<b><code>offset</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+</td>
+-
+Number of Request objects that was skipped at the start.
+</li>
+<li>
+<b><code>count</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+</td>
+-
+Number of returned objects.
+</li>
+<li>
+<b><code>limit</code></b>
+(
+<span class="param-type">Number</span>
+
+)
+</td>
+-
+Requested limit on the number of items.
+</li>
+
+</ul>
+</div>
+</div>
+<div class="members">
+ <h4 class="name" id="RequestOperationInfo">RequestOperationInfo</h4>
+<div class="description">
+ <p>A helper class that is used to report results from the
+<code>Apify.utils.puppeteer.enqueueLinks()</code> function.</p>
+ </div>
+<h5 class="subsection-title">Properties:</h5>
+<div class="params">
+<ul>
+<li>
+<b><code>wasAlreadyPresent</code></b>
+(
+<span class="param-type">Boolean</span>
+
+)
+</td>
+-
+Indicates if request was already present in the queue.
+</li>
+<li>
+<b><code>wasAlreadyHandled</code></b>
+(
+<span class="param-type">Boolean</span>
+
+)
+</td>
+-
+Indicates if request was already marked as handled.
+</li>
+<li>
+<b><code>requestId</code></b>
+(
+<span class="param-type">String</span>
+
+)
+</td>
+-
+The ID of the added request
+</li>
+
+</ul>
+</div>
+</div>
+</div>
+</article>
+
+</div>
+ </div>
+ </div>
+ </div>
+ </div>
+</main>
diff --git a/docs/api/Apify.md b/docs/api/Apify.md
new file mode 100644
index 000000000..98bf5f060
--- /dev/null
+++ b/docs/api/Apify.md
@@ -0,0 +1,725 @@
+---
+id: apify
+title: Apify
+---
+<a name="module_Apify"></a>
+
+The following section describes all functions and properties provided by the `apify` package,
+except individual classes and namespaces that have their separate, detailed, documentation pages
+accessible from the left sidebar.
+
+
+* [Apify](#module_Apify)
+ * [`.call(actId, [input], [options])`](#module_Apify.call) ⇒ [<code>Promise&lt;ActorRun&gt;</code>](../typedefs/actorrun)
+ * [`.client`](#module_Apify.client)
+ * [`.events`](#module_Apify.events)
+ * [`.getApifyProxyUrl(options)`](#module_Apify.getApifyProxyUrl) ⇒ <code>String</code>
+ * [`.getEnv()`](#module_Apify.getEnv) ⇒ <code>Object</code>
+ * [`.getMemoryInfo()`](#module_Apify.getMemoryInfo) ⇒ <code>Promise&lt;Object&gt;</code>
+ * [`.getValue(key)`](#module_Apify.getValue) ⇒ <code>Promise&lt;Object&gt;</code>
+ * [`.isAtHome()`](#module_Apify.isAtHome) ⇒ <code>Boolean</code>
+ * [`.isDocker()`](#module_Apify.isDocker) ⇒ <code>Promise</code>
+ * [`.launchPuppeteer([options])`](#module_Apify.launchPuppeteer) ⇒ <code>Promise&lt;Browser&gt;</code>
+ * [`.launchWebDriver([options])`](#module_Apify.launchWebDriver) ⇒ <code>Promise</code>
+ * [`.main(userFunc)`](#module_Apify.main)
+ * [`.openDataset([datasetIdOrName])`](#module_Apify.openDataset) ⇒ [<code>Promise&lt;Dataset&gt;</code>](dataset)
+ * [`.openKeyValueStore([storeIdOrName])`](#module_Apify.openKeyValueStore) ⇒ [<code>Promise&lt;KeyValueStore&gt;</code>](keyvaluestore)
+ * [`.openRequestQueue`](#module_Apify.openRequestQueue) ⇒ [<code>Promise&lt;RequestQueue&gt;</code>](requestqueue)
+ * [`.pushData(item)`](#module_Apify.pushData) ⇒ <code>Promise</code>
+ * [`.setValue(key, value, [options])`](#module_Apify.setValue) ⇒ <code>Promise</code>
+
+<a name="module_Apify.call"></a>
+
+## `Apify.call(actId, [input], [options])` ⇒ [<code>Promise&lt;ActorRun&gt;</code>](../typedefs/actorrun)
+Runs an actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable),
+waits for the actor to finish and fetches its output.
+
+By passing the `waitSecs` option you can reduce the maximum amount of time to wait for the run to finish.
+If the value is less than or equal to zero, the function returns immediately after the run is started.
+
+The result of the function is an [`ActorRun`](../typedefs/actorrun) object
+that contains details about the actor run and its output (if any).
+If the actor run failed, the function fails with [`ApifyCallError`](../typedefs/apifycallerror) exception.
+
+**Example usage:**
+
+```javascript
+const run = await Apify.call('apify/hello-world', { myInput: 123 });
+console.log(`Received message: ${run.output.body.message}`);
+```
+
+Internally, the `call()` function calls the
+<a href="https://www.apify.com/docs/api/v2#/reference/actors/run-collection/run-actor" target="_blank">Run actor</a>
+Apify API endpoint and few others to obtain the output.
+
+**Throws**:
+
+- [<code>ApifyCallError</code>](../typedefs/apifycallerror) If the run did not succeed, e.g. if it failed or timed out.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>actId</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Either <code>username/actor-name</code> or actor ID.</p>
+</td></tr><tr>
+<td><code>[input]</code></td><td><code>Object</code> | <code>String</code> | <code>Buffer</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Input for the actor. If it is an object, it will be stringified to
+ JSON and its content type set to <code>application/json; charset=utf-8</code>.
+ Otherwise the <code>options.contentType</code> parameter must be provided.</p>
+</td></tr><tr>
+<td><code>[options]</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Object with the settings below:</p>
+</td></tr><tr>
+<td><code>[options.contentType]</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Content type for the <code>input</code>. If not specified,
+ <code>input</code> is expected to be an object that will be stringified to JSON and content type set to
+ <code>application/json; charset=utf-8</code>. If <code>options.contentType</code> is specified, then <code>input</code> must be a
+ <code>String</code> or <code>Buffer</code>.</p>
+</td></tr><tr>
+<td><code>[options.token]</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>User API token that is used to run the actor. By default, it is taken from the <code>APIFY_TOKEN</code> environment variable.</p>
+</td></tr><tr>
+<td><code>[options.memory]</code></td><td><code>Number</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Memory in megabytes which will be allocated for the new actor run.
+ If not provided, the run uses memory of the default actor run configuration.</p>
+</td></tr><tr>
+<td><code>[options.build]</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Tag or number of the actor build to run (e.g. <code>beta</code> or <code>1.2.345</code>).
+ If not provided, the run uses build tag or number from the default actor run configuration (typically <code>latest</code>).</p>
+</td></tr><tr>
+<td><code>[options.waitSecs]</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Maximum time to wait for the actor run to finish, in seconds.
+ If the limit is reached, the returned promise is resolved to a run object that will have
+ status <code>READY</code> or <code>RUNNING</code> and it will not contain the actor run output.
+ If <code>waitSecs</code> is null or undefined, the function waits for the actor to finish (default behavior).</p>
+</td></tr><tr>
+<td><code>[options.fetchOutput]</code></td><td><code>Boolean</code></td><td><code>true</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If <code>false</code> then the function does not fetch output of the actor.</p>
+</td></tr><tr>
+<td><code>[options.disableBodyParser]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If <code>true</code> then the function will not attempt to parse the
+ actor&#39;s output and will return it in a raw <code>Buffer</code>.</p>
+</td></tr></tbody>
+</table>
+<a name="module_Apify.client"></a>
+
+## `Apify.client`
+Gets the default instance of the `ApifyClient` class provided
+<a href="https://www.apify.com/docs/sdk/apify-client-js/latest"
+target="_blank">apify-client</a> by the NPM package.
+The instance is created automatically by the Apify SDK and it is configured using the
+`APIFY_API_BASE_URL`, `APIFY_USER_ID` and `APIFY_TOKEN` environment variables.
+
+The instance is used for all underlying calls to the Apify API in functions such as
+[`Apify.getValue()`](#module_Apify.getValue) or [`Apify.call()`](#module_Apify.call).
+The settings of the client can be globally altered by calling the
+<a href="https://www.apify.com/docs/sdk/apify-client-js/latest#ApifyClient-setOptions"
+target="_blank">`Apify.client.setOptions()`</a> function.
+Beware that altering these settings might have unintended effects on the entire Apify SDK package.
+
+<a name="module_Apify.events"></a>
+
+## `Apify.events`
+Gets an instance of a Node.js'
+<a href="https://nodejs.org/api/events.html#events_class_eventemitter" target="_blank">EventEmitter</a>
+class that emits various events from the SDK or the Apify platform.
+The event emitter is initialized by calling the [`Apify.main()`](#module_Apify.main) function.
+
+**Example usage:**
+
+```javascript
+Apify.events.on('cpuInfo', (data) => {
+ if (data.isCpuOverloaded) console.log('Oh no, the CPU is overloaded!');
+});
+```
+
+The following table shows all currently emitted events:
+<table>
+ <thead>
+ <tr>
+ <th>Event name</th>
+ <th>Data</th>
+ </thead>
+ <tbody>
+ <tr>
+ <td><code>cpuInfo</code></td>
+ <td><code>{ "isCpuOverloaded": Boolean }</code></td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ The event is emitted approximately every second
+ and it indicates whether the actor is using the maximum of available CPU resources.
+ If that's the case, the actor should not add more workload.
+ For example, this event is used by the <a href="autoscaledpool"><code>AutoscaledPool</code></a> class.
+ </td>
+ </tr>
+ <tr>
+ <td><code>migrating</code></td>
+ <td>None</td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ Emitted when the actor running on the Apify platform is going to be migrated to another worker server soon.
+ You can use it to persist the state of the actor and abort the run, to speed up migration.
+ For example, this is used by the <a href="requestlist"><code>RequestList</code></a> class.
+ </td>
+ </tr>
+ <tr>
+ <td><code>persistState</code></td>
+ <td><code>{ "isMigrating": Boolean }</code></td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ Emitted in regular intervals to notify all components of Apify SDK that it is time to persist
+ their state, in order to avoid repeating all work when the actor restarts.
+ This event is automatically emitted together with the <code>migrating</code> event,
+ in which case the <code>isMigrating</code> flag is set to <code>true</code>. Otherwise the flag is <code>false</code>.
+ </td>
+ </tr>
+ </tbody>
+</table>
+
+<a name="module_Apify.getApifyProxyUrl"></a>
+
+## `Apify.getApifyProxyUrl(options)` ⇒ <code>String</code>
+Constructs an Apify Proxy URL using the specified settings.
+The proxy URL can be used from Apify actors, web browsers or any other HTTP
+proxy-enabled applications.
+
+For more information, see
+the <a href="https://my.apify.com/proxy" target="_blank">Apify Proxy</a> page in the app
+or the <a href="https://www.apify.com/docs/proxy" target="_blank">documentation</a>.
+
+**Returns**: <code>String</code> - Returns the proxy URL, e.g. `http://auto:[email protected]:8000`.
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>options</code></td><td><code>Object</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Object with the settings below:</p>
+</td></tr><tr>
+<td><code>[options.password]</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>User&#39;s password for the proxy.
+ By default, it is taken from the <code>APIFY_PROXY_PASSWORD</code> environment variable,
+ which is automatically set by the system when running the actors on the Apify cloud,
+ or when using the <a href="https://github.com/apifytech/apify-cli" target="_blank">Apify CLI</a>
+ package and the user previously logged in (called <code>apify login</code>).</p>
+</td></tr><tr>
+<td><code>[options.groups]</code></td><td><code>Array&lt;String&gt;</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Array of Apify Proxy groups to be used.
+ If not provided, the proxy will select the groups automatically.</p>
+</td></tr><tr>
+<td><code>[options.session]</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Apify Proxy session identifier to be used by the Chrome browser.
+ All HTTP requests going through the proxy with the same session identifier
+ will use the same target proxy server (i.e. the same IP address), unless using Residential proxies.
+ The identifier can only contain the following characters: <code>0-9</code>, <code>a-z</code>, <code>A-Z</code>, <code>&quot;.&quot;</code>, <code>&quot;_&quot;</code> and <code>&quot;~&quot;</code>.</p>
+</td></tr></tbody>
+</table>
+<a name="module_Apify.getEnv"></a>
+
+## `Apify.getEnv()` ⇒ <code>Object</code>
+Returns a new object which contains information parsed from the `APIFY_XXX` environment variables.
+It has the following properties:
+
+```javascript
+{
+ // ID of the actor (APIFY_ACT_ID)
+ actId: String,
+ // ID of the actor run (APIFY_ACT_RUN_ID)
+ actRunId: String,
+ // ID of the user who started the actor - note that it might be
+ // different than the owner of the actor (APIFY_USER_ID)
+ userId: String,
+ // Authentication token representing privileges given to the actor run,
+ // it can be passed to various Apify APIs (APIFY_TOKEN).
+ token: String,
+ // Date when the actor was started (APIFY_STARTED_AT)
+ startedAt: Date,
+ // Date when the actor will time out (APIFY_TIMEOUT_AT)
+ timeoutAt: Date,
+ // ID of the key-value store where input and output data of this
+ // actor is stored (APIFY_DEFAULT_KEY_VALUE_STORE_ID)
+ defaultKeyValueStoreId: String,
+ // ID of the dataset where input and output data of this
+ // actor is stored (APIFY_DEFAULT_DATASET_ID)
+ defaultDatasetId: String,
+ // Amount of memory allocated for the actor,
+ // in megabytes (APIFY_MEMORY_MBYTES)
+ memoryMbytes: Number,
+}
+```
+For the list of the `APIFY_XXX` environment variables, see
+<a href="https://www.apify.com/docs/actor#run-env-vars" target="_blank">Actor documentation</a>.
+If some of the variables are not defined or are invalid, the corresponding value in the resulting object will be null.
+
+<a name="module_Apify.getMemoryInfo"></a>
+
+## `Apify.getMemoryInfo()` ⇒ <code>Promise&lt;Object&gt;</code>
+Returns memory statistics of the process and the system, which is an object with the following properties:
+
+```javascript
+{
+ // Total memory available in the system or container
+ totalBytes: Number,
+ // Amount of free memory in the system or container
+ freeBytes: Number,
+ // Amount of memory used (= totalBytes - freeBytes)
+ usedBytes: Number,
+ // Amount of memory used the current Node.js process
+ mainProcessBytes: Number,
+ // Amount of memory used by child processes of the current Node.js process
+ childProcessesBytes: Number,
+}
+```
+
+If the process runs inside of Docker, the `getMemoryInfo` gets container memory limits,
+otherwise it gets system memory limits.
+
+Beware that the function is quite inefficient because it spawns a new process.
+Therefore you shouldn't call it too often, like more than once per second.
+
+<a name="module_Apify.getValue"></a>
+
+## `Apify.getValue(key)` ⇒ <code>Promise&lt;Object&gt;</code>
+Gets a value from the default [`KeyValueStore`](keyvaluestore) associated with the current actor run.
+
+This is just a convenient shortcut for [`keyValueStore.getValue()`](keyvaluestore#KeyValueStore+getValue).
+For example, calling the following code:
+```javascript
+const input = await Apify.getValue('INPUT');
+```
+
+is equivalent to:
+```javascript
+const store = await Apify.openKeyValueStore();
+await store.getValue('INPUT');
+```
+
+To store the value to the default-key value store, you can use the [`Apify.setValue()`](#module_Apify.setValue) function.
+
+For more information, see [`Apify.openKeyValueStore()`](#module_Apify.openKeyValueStore)
+and [`keyValueStore.getValue()`](keyvaluestore#KeyValueStore+getValue).
+
+**Returns**: <code>Promise&lt;Object&gt;</code> - Returns a promise that resolves once the record is stored.
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>key</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Unique record key.</p>
+</td></tr></tbody>
+</table>
+<a name="module_Apify.isAtHome"></a>
+
+## `Apify.isAtHome()` ⇒ <code>Boolean</code>
+Returns `true` when code is running on Apify platform and `false` otherwise (for example locally).
+
+<a name="module_Apify.isDocker"></a>
+
+## `Apify.isDocker()` ⇒ <code>Promise</code>
+Returns a `Promise` that resolves to true if the code is running in a Docker container.
+
+<a name="module_Apify.launchPuppeteer"></a>
+
+## `Apify.launchPuppeteer([options])` ⇒ <code>Promise&lt;Browser&gt;</code>
+Launches headless Chrome using Puppeteer pre-configured to work within the Apify platform.
+The function has the same argument and the return value as `puppeteer.launch()`.
+See <a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#puppeteerlaunchoptions" target="_blank">
+Puppeteer documentation</a> for more details.
+
+The `launchPuppeteer()` function alters the following Puppeteer options:
+
+<ul>
+ <li>
+ Passes the setting from the <code>APIFY_HEADLESS</code> environment variable to the <code>headless</code> option,
+ unless it was already defined by the caller or <code>APIFY_XVFB</code> environment variable is set to <code>1</code>.
+ Note that Apify Actor cloud platform automatically sets <code>APIFY_HEADLESS=1</code> to all running actors.
+ </li>
+ <li>
+ Takes the <code>proxyUrl</code> option, checks it and adds it to <code>args</code> as <code>--proxy-server=XXX</code>.
+ If the proxy uses authentication, the function sets up an anonymous proxy HTTP
+ to make the proxy work with headless Chrome. For more information, read the
+ <a href="https://blog.apify.com/how-to-make-headless-chrome-and-puppeteer-use-a-proxy-server-with-authentication-249a21a79212"
+ target="_blank">blog post about proxy-chain library</a>.
+ </li>
+ <li>
+ If <code>options.useApifyProxy</code> is <code>true</code> then the function generates a URL of
+ <a href="https://www.apify.com/docs/proxy" target="_blank">Apify Proxy</a>
+ based on <code>options.apifyProxyGroups</code> and <code>options.apifyProxySession</code> and passes it as <code>options.proxyUrl</code>.
+ </li>
+ <li>
+ The function adds <code>--no-sandbox</code> to <code>args</code> to enable running
+ headless Chrome in a Docker container on the Apify platform.
+ </li>
+</ul>
+
+To use this function, you need to have the <a href="https://www.npmjs.com/package/puppeteer" target="_blank">puppeteer</a>
+NPM package installed in your project.
+When running on the Apify cloud platform, you can achieve that simply
+by using the `apify/actor-node-chrome` base Docker image for your actor - see
+<a href="https://www.apify.com/docs/actor#base-images" target="_blank">Apify Actor documentation</a>
+for details.
+
+For an example of usage, see the [Synchronous run Example](../examples/synchronousrun) or the [Puppeteer proxy Example](../examples/puppeteerproxy)
+
+**Returns**: <code>Promise&lt;Browser&gt;</code> - Promise that resolves to Puppeteer's `Browser` instance.
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>[options]</code></td><td><code><a href="../typedefs/launchpuppeteeroptions">LaunchPuppeteerOptions</a></code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Optional settings passed to <code>puppeteer.launch()</code>. In addition to
+ <a href="https://pptr.dev/#?product=Puppeteer&show=api-puppeteerlaunchoptions" target="_blank">Puppeteer&#39;s options</a>
+ the object may contain our own <a href="../typedefs/launchpuppeteeroptions"><code>LaunchPuppeteerOptions</code></a> that enable additional features.</p>
+</td></tr></tbody>
+</table>
+<a name="module_Apify.launchWebDriver"></a>
+
+## `Apify.launchWebDriver([options])` ⇒ <code>Promise</code>
+Opens a new instance of Chrome web browser
+controlled by <a href="http://www.seleniumhq.org/projects/webdriver/" target="_blank">Selenium WebDriver</a>.
+The result of the function is the new instance of the
+<a href="http://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_WebDriver.html" target="_blank">
+WebDriver</a>
+class.
+
+To use this function, you need to have Google Chrome and
+<a href="https://sites.google.com/a/chromium.org/chromedriver/" target="_blank">ChromeDriver</a> installed in your environment.
+For example, you can use the `apify/actor-node-chrome` base Docker image for your actor - see
+<a href="https://www.apify.com/docs/actor#base-images" target="_blank">documentation</a>
+for more details.
+
+For an example of usage, see the <a href="https://www.apify.com/apify/example-selenium" target="_blank">apify/example-selenium</a> actor.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>[options]</code></td><td><code>Object</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Optional settings passed to WebDriver. Additionally the object can contain the following fields:</p>
+</td></tr><tr>
+<td><code>[options.proxyUrl]</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>URL to a proxy server. Currently only <code>http://</code> scheme is supported.
+Port number must be specified. For example, <code>http://example.com:1234</code>.</p>
+</td></tr><tr>
+<td><code>[options.headless]</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Indicates that the browser will be started in headless mode.
+If the option is not defined, and the <code>APIFY_HEADLESS</code> environment variable has value <code>1</code>
+and <code>APIFY_XVFB</code> is NOT <code>1</code>, the value defaults to <code>true</code>, otherwise it will be <code>false</code>.</p>
+</td></tr><tr>
+<td><code>[options.userAgent]</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>User-Agent for the browser.
+If not provided, the function sets it to a reasonable default.</p>
+</td></tr></tbody>
+</table>
+<a name="module_Apify.main"></a>
+
+## `Apify.main(userFunc)`
+Runs the main user function that performs the job of the actor.
+
+`Apify.main()` is especially useful when you're running your code in an actor on the Apify platform.
+Note that its use is optional - the function is provided merely for your convenience.
+
+The function performs the following actions:
+
+<ol>
+ <li>When running on the Apify platform (i.e. <code>APIFY_IS_AT_HOME</code> environment variable is set),
+ it sets up a connection to listen for platform events.
+ For example, to get a notification about an imminent migration to another server.
+ See <a href="apify#apifyevents"><code>Apify.events</code></a> for details.
+ </li>
+ <li>It checks that either <code>APIFY_TOKEN</code> or <code>APIFY_LOCAL_STORAGE_DIR</code> environment variable
+ is defined. If not, the functions sets <code>APIFY_LOCAL_STORAGE_DIR</code> to <code>./apify_storage</code>
+ inside the current working directory. This is to simplify running code examples.
+ </li>
+ <li>It invokes the user function passed as the <code>userFunc</code> parameter.</li>
+ <li>If the user function returned a promise, waits for it to resolve.</li>
+ <li>If the user function throws an exception or some other error is encountered,
+ prints error details to console so that they are stored to the log.</li>
+ <li>Exits the Node.js process, with zero exit code on success and non-zero on errors.</li>
+</ol>
+
+The user function can be synchronous:
+
+```javascript
+Apify.main(() => {
+ // My synchronous function that returns immediately
+ console.log('Hello world from actor!');
+});
+```
+
+If the user function returns a promise, it is considered asynchronous:
+```javascript
+const request = require('request-promise');
+
+Apify.main(() => {
+ // My asynchronous function that returns a promise
+ return request('http://www.example.com').then((html) => {
+ console.log(html);
+ });
+});
+```
+
+To simplify your code, you can take advantage of the `async`/`await` keywords:
+
+```javascript
+const request = require('request-promise');
+
+Apify.main(async () => {
+ // My asynchronous function
+ const html = await request('http://www.example.com');
+ console.log(html);
+});
+```
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>userFunc</code></td><td><code>function</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>User function to be executed. If it returns a promise,
+the promise will be awaited. The user function is called with no arguments.</p>
+</td></tr></tbody>
+</table>
+<a name="module_Apify.openDataset"></a>
+
+## `Apify.openDataset([datasetIdOrName])` ⇒ [<code>Promise&lt;Dataset&gt;</code>](dataset)
+Opens a dataset and returns a promise resolving to an instance of the [`Dataset`](dataset) class.
+
+Datasets are used to store structured data where each object stored has the same attributes,
+such as online store products or real estate offers.
+The actual data is stored either on the local filesystem or in the cloud.
+
+For more details and code examples, see the [`Dataset`](dataset) class.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>[datasetIdOrName]</code></td><td><code>string</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>ID or name of the dataset to be opened. If <code>null</code> or <code>undefined</code>,
+ the function returns the default dataset associated with the actor run.</p>
+</td></tr></tbody>
+</table>
+<a name="module_Apify.openKeyValueStore"></a>
+
+## `Apify.openKeyValueStore([storeIdOrName])` ⇒ [<code>Promise&lt;KeyValueStore&gt;</code>](keyvaluestore)
+Opens a key-value store and returns a promise resolving to an instance of the [`KeyValueStore`](keyvaluestore) class.
+
+Key-value stores are used to store records or files, along with their MIME content type.
+The records are stored and retrieved using a unique key.
+The actual data is stored either on a local filesystem or in the Apify cloud.
+
+For more details and code examples, see the [`KeyValueStore`](keyvaluestore) class.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>[storeIdOrName]</code></td><td><code>string</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>ID or name of the key-value store to be opened. If <code>null</code> or <code>undefined</code>,
+ the function returns the default key-value store associated with the actor run.</p>
+</td></tr></tbody>
+</table>
+<a name="module_Apify.openRequestQueue"></a>
+
+## `Apify.openRequestQueue` ⇒ [<code>Promise&lt;RequestQueue&gt;</code>](requestqueue)
+Opens a request queue and returns a promise resolving to an instance
+of the [`RequestQueue`](requestqueue) class.
+
+[`RequestQueue`](requestqueue) represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
+The queue is used for deep crawling of websites, where you start with several URLs and then
+recursively follow links to other pages. The data structure supports both breadth-first
+and depth-first crawling orders.
+
+For more details and code examples, see the [`RequestQueue`](requestqueue) class.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>[queueIdOrName]</code></td><td><code>string</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>ID or name of the request queue to be opened. If <code>null</code> or <code>undefined</code>,
+ the function returns the default request queue associated with the actor run.</p>
+</td></tr></tbody>
+</table>
+<a name="module_Apify.pushData"></a>
+
+## `Apify.pushData(item)` ⇒ <code>Promise</code>
+Stores an object or an array of objects to the default [`Dataset`](dataset) of the current actor run.
+
+This is just a convenient shortcut for [`dataset.pushData()`](dataset#Dataset+pushData).
+For example, calling the following code:
+```javascript
+await Apify.pushData({ myValue: 123 });
+```
+
+is equivalent to:
+```javascript
+const dataset = await Apify.openDataset();
+await dataset.pushData({ myValue: 123 });
+```
+
+For more information, see [`Apify.openDataset()`](apify#module_Apify.openDataset) and [`dataset.pushData()`](dataset#Dataset+pushData)
+
+**IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
+otherwise the actor process might finish before the data are stored!
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>item</code></td><td><code>Object</code> | <code>Array</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Object or array of objects containing data to be stored in the default dataset.
+The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.</p>
+</td></tr></tbody>
+</table>
+<a name="module_Apify.setValue"></a>
+
+## `Apify.setValue(key, value, [options])` ⇒ <code>Promise</code>
+Stores or deletes a value in the default [`KeyValueStore`](keyvaluestore) associated with the current actor run.
+
+This is just a convenient shortcut for [`keyValueStore.setValue()`](keyvaluestore#KeyValueStore+setValue).
+For example, calling the following code:
+```javascript
+await Apify.setValue('OUTPUT', { foo: "bar" });
+```
+
+is equivalent to:
+```javascript
+const store = await Apify.openKeyValueStore();
+await store.setValue('OUTPUT', { foo: "bar" });
+```
+
+To get a value from the default-key value store, you can use the [`Apify.getValue()`](#module_Apify.getValue) function.
+
+For more information, see [`Apify.openKeyValueStore()`](#module_Apify.openKeyValueStore)
+and [`keyValueStore.getValue()`](keyvaluestore#KeyValueStore+getValue).
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>key</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Unique record key.</p>
+</td></tr><tr>
+<td><code>value</code></td><td><code>Object</code> | <code>String</code> | <code>Buffer</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Record data, which can be one of the following values:
+ <ul>
+ <li>If <code>null</code>, the record in the key-value store is deleted.</li>
+ <li>If no <code>options.contentType</code> is specified, <code>value</code> can be any JavaScript object and it will be stringified to JSON.</li>
+ <li>If <code>options.contentType</code> is specified, <code>value</code> is considered raw data and it must be a <code>String</code>
+ or <a href="https://nodejs.org/api/buffer.html" target="_blank"><code>Buffer</code></a>.</li>
+ </ul>
+ For any other value an error will be thrown.</p>
+</td></tr><tr>
+<td><code>[options]</code></td><td><code>Object</code></td>
+</tr>
+<tr>
+<td colspan="3"></td></tr><tr>
+<td><code>[options.contentType]</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Specifies a custom MIME content type of the record.</p>
+</td></tr></tbody>
+</table>
diff --git a/docs/api/AutoscaledPool.md b/docs/api/AutoscaledPool.md
new file mode 100644
index 000000000..29670e93e
--- /dev/null
+++ b/docs/api/AutoscaledPool.md
@@ -0,0 +1,169 @@
+---
+id: autoscaledpool
+title: AutoscaledPool
+---
+<a name="AutoscaledPool"></a>
+
+Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
+The pool only starts new tasks if there is enough free CPU and memory available
+and the Javascript event loop is not blocked.
+
+The information about the CPU and memory usage is obtained by the [`Snapshotter`](snapshotter) class,
+which makes regular snapshots of system resources that may be either local
+or from the Apify cloud infrastructure in case the process is running on the Apify platform.
+Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the [`SystemStatus`](systemstatus) class.
+
+Before running the pool, you need to implement the following three functions:
+[`runTaskFunction()`](#new_AutoscaledPool_new),
+[`isTaskReadyFunction()`](#new_AutoscaledPool_new) and
+[`isFinishedFunction()`](#new_AutoscaledPool_new).
+
+The auto-scaled pool is started by calling the [`run()`](#AutoscaledPool+run) function.
+The pool periodically queries the [`isTaskReadyFunction()`](#new_AutoscaledPool_new) function
+for more tasks, managing optimal concurrency, until the function resolves to `false`. The pool then queries
+the [`isFinishedFunction()`](#new_AutoscaledPool_new). If it resolves to `true`, the run finishes after all running tasks complete.
+If it resolves to `false`, it assumes there will be more tasks available later and keeps periodically querying for tasks.
+If any of the tasks throws then the [`run()`](#AutoscaledPool+run) function rejects the promise with an error.
+
+The pool evaluates whether it should start a new task every time one of the tasks finishes
+and also in the interval set by the `options.maybeRunIntervalSecs` parameter.
+
+**Example usage:**
+
+```javascript
+const pool = new Apify.AutoscaledPool({
+ maxConcurrency: 50,
+ runTaskFunction: async () => {
+ // Run some resource-intensive asynchronous operation here.
+ },
+ isTaskReadyFunction: async () => {
+ // Tell the pool whether more tasks are ready to be processed.
+ // Return true or false
+ },
+ isFinishedFunction: async () => {
+ // Tell the pool whether it should finish
+ // or wait for more tasks to become available.
+ // Return true or false
+ }
+});
+
+await pool.run();
+```
+
+
+* [AutoscaledPool](autoscaledpool)
+ * [`new AutoscaledPool(options)`](#new_AutoscaledPool_new)
+ * [`.run()`](#AutoscaledPool+run) ⇒ <code>Promise</code>
+ * [`.abort()`](#AutoscaledPool+abort) ⇒ <code>Promise</code>
+
+<a name="new_AutoscaledPool_new"></a>
+
+## `new AutoscaledPool(options)`
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>options</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>All <code>AutoscaledPool</code> parameters are passed
+ via an options object with the following keys.</p>
+</td></tr><tr>
+<td><code>options.runTaskFunction</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>A function that performs an asynchronous resource-intensive task.
+ The function must either be labeled <code>async</code> or return a promise.</p>
+</td></tr><tr>
+<td><code>options.isTaskReadyFunction</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>A function that indicates whether <code>runTaskFunction</code> should be called.
+ This function is called every time there is free capacity for a new task and it should
+ indicate whether it should start a new task or not by resolving to either <code>true</code> or `false.
+ Besides its obvious use, it is also useful for task throttling to save resources.</p>
+</td></tr><tr>
+<td><code>options.isFinishedFunction</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>A function that is called only when there are no tasks to be processed.
+ If it resolves to <code>true</code> then the pool&#39;s run finishes. Being called only
+ when there are no tasks being processed means that as long as <code>isTaskReadyFunction()</code>
+ keeps resolving to <code>true</code>, <code>isFinishedFunction()</code> will never be called.
+ To abort a run, use the <a href="#AutoscaledPool+abort"><code>abort()</code></a> method.</p>
+</td></tr><tr>
+<td><code>[options.minConcurrency]</code></td><td><code>Number</code></td><td><code>1</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Minimum number of tasks running in parallel.</p>
+</td></tr><tr>
+<td><code>[options.maxConcurrency]</code></td><td><code>Number</code></td><td><code>1000</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Maximum number of tasks running in parallel.</p>
+</td></tr><tr>
+<td><code>[options.desiredConcurrencyRatio]</code></td><td><code>Number</code></td><td><code>0.95</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Minimum level of desired concurrency to reach before more scaling up is allowed.</p>
+</td></tr><tr>
+<td><code>[options.scaleUpStepRatio]</code></td><td><code>Number</code></td><td><code>0.05</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Defines the fractional amount of desired concurrency to be added with each scaling up.
+ The minimum scaling step is one.</p>
+</td></tr><tr>
+<td><code>[options.scaleDownStepRatio]</code></td><td><code>Number</code></td><td><code>0.05</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Defines the amount of desired concurrency to be subtracted with each scaling down.
+ The minimum scaling step is one.</p>
+</td></tr><tr>
+<td><code>[options.maybeRunIntervalSecs]</code></td><td><code>Number</code></td><td><code>0.5</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Indicates how often the pool should call the <code>runTaskFunction()</code> to start a new task, in seconds.
+ This has no effect on starting new tasks immediately after a task completes.</p>
+</td></tr><tr>
+<td><code>[options.loggingIntervalSecs]</code></td><td><code>Number</code></td><td><code>60</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Specifies a period in which the instance logs its state, in seconds.
+ Set to <code>null</code> to disable periodic logging.</p>
+</td></tr><tr>
+<td><code>[options.autoscaleIntervalSecs]</code></td><td><code>Number</code></td><td><code>10</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Defines in seconds how often the pool should attempt to adjust the desired concurrency
+ based on the latest system status. Setting it lower than 1 might have a severe impact on performance.
+ We suggest using a value from 5 to 20.</p>
+</td></tr><tr>
+<td><code>[options.snapshotterOptions]</code></td><td><code>Number</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Options to be passed down to the <a href="snapshotter"><code>Snapshotter</code></a> constructor. This is useful for fine-tuning
+ the snapshot intervals and history.</p>
+</td></tr><tr>
+<td><code>[options.systemStatusOptions]</code></td><td><code>Number</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Options to be passed down to the <a href="systemstatus"><code>SystemStatus</code></a> constructor. This is useful for fine-tuning
+ the system status reports. If a custom snapshotter is set in the options, it will be used
+ by the pool.</p>
+</td></tr></tbody>
+</table>
+<a name="AutoscaledPool+run"></a>
+
+## `autoscaledPool.run()` ⇒ <code>Promise</code>
+Runs the auto-scaled pool. Returns a promise that gets resolved or rejected once
+all the tasks are finished or one of them fails.
+
+<a name="AutoscaledPool+abort"></a>
+
+## `autoscaledPool.abort()` ⇒ <code>Promise</code>
+Aborts the run of the auto-scaled pool, discards all currently running tasks and destroys it.
+
diff --git a/docs/api/BasicCrawler.md b/docs/api/BasicCrawler.md
new file mode 100644
index 000000000..14cfb889a
--- /dev/null
+++ b/docs/api/BasicCrawler.md
@@ -0,0 +1,152 @@
+---
+id: basiccrawler
+title: BasicCrawler
+---
+<a name="BasicCrawler"></a>
+
+Provides a simple framework for parallel crawling of web pages,
+whose URLs are fed either from a static list
+or from a dynamic queue of URLs.
+
+`BasicCrawler` invokes the user-provided [`handleRequestFunction()`](#new_BasicCrawler_new)
+for each [`Request`](request) object, which represents a single URL to crawl.
+The [`Request`](request) objects are fed from the [`RequestList`](requestlist) or the [`RequestQueue`](requestqueue)
+instances provided by the [`requestList`](#new_BasicCrawler_new) or [`requestQueue`](#new_BasicCrawler_new)
+constructor options, respectively.
+
+If both [`requestList`](#new_BasicCrawler_new) and [`requestQueue`](#new_BasicCrawler_new) options are used,
+the instance first processes URLs from the [`RequestList`](requestlist) and automatically enqueues all of them
+to [`RequestQueue`](requestqueue) before it starts their processing. This ensures that a single URL is not crawled multiple times.
+
+The crawler finishes if there are no more [`Request`](request) objects to crawl.
+
+New requests are only dispatched when there is enough free CPU and memory available,
+using the functionality provided by the [`AutoscaledPool`](autoscaledpool) class.
+All [`AutoscaledPool`](autoscaledpool) configuration options can be passed to the `autoscaledPoolOptions`
+parameter of the `BasicCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+[`AutoscaledPool`](autoscaledpool) options are available directly in the `BasicCrawler` constructor.
+
+**Example usage:**
+
+```javascript
+const rp = require('request-promise');
+
+// Prepare a list of URLs to crawl
+const requestList = new Apify.RequestList({
+ sources: [
+ { url: 'http://www.example.com/page-1' },
+ { url: 'http://www.example.com/page-2' },
+ ],
+});
+await requestList.initialize();
+
+// Crawl the URLs
+const crawler = new Apify.BasicCrawler({
+ requestList,
+ handleRequestFunction: async ({ request }) => {
+ // 'request' contains an instance of the Request class
+ // Here we simply fetch the HTML of the page and store it to a dataset
+ await Apify.pushData({
+ url: request.url,
+ html: await rp(request.url),
+ })
+ },
+});
+
+await crawler.run();
+```
+
+
+* [BasicCrawler](basiccrawler)
+ * [`new BasicCrawler(options)`](#new_BasicCrawler_new)
+ * [`.run()`](#BasicCrawler+run) ⇒ <code>Promise</code>
+ * [`.abort()`](#BasicCrawler+abort) ⇒ <code>Promise</code>
+
+<a name="new_BasicCrawler_new"></a>
+
+## `new BasicCrawler(options)`
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>options</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>All <code>BasicCrawler</code> parameters are passed
+ via an options object with the following keys:</p>
+</td></tr><tr>
+<td><code>options.handleRequestFunction</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>User-provided function that performs the logic of the crawler. It is called for each URL to crawl.</p>
+<p> The function receives the following object as an argument:</p>
+<pre><code> {
+ request: Request
+ }
+</code></pre><p> With the <a href="request"><code>Request</code></a> object representing the URL to crawl.
+ The function must return a promise.</p>
+</td></tr><tr>
+<td><code>options.requestList</code></td><td><code><a href="requestlist">RequestList</a></code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Static list of URLs to be processed.
+ Either <a href="requestlist"><code>RequestList</code></a> or <a href="requestqueue"><code>RequestQueue</code></a> must be provided.</p>
+</td></tr><tr>
+<td><code>options.requestQueue</code></td><td><code><a href="requestqueue">RequestQueue</a></code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
+ Either <a href="requestlist"><code>RequestList</code></a> or <a href="requestqueue"><code>RequestQueue</code></a> must be provided.</p>
+</td></tr><tr>
+<td><code>[options.handleFailedRequestFunction]</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Function that handles requests that failed more then <code>options.maxRequestRetries</code> times.
+ See source code on
+ <a href="https://github.com/apifytech/apify-js/blob/master/src/basic_crawler.js#L11" target="_blank">GitHub</a>
+ for default behavior.</p>
+</td></tr><tr>
+<td><code>[options.maxRequestRetries]</code></td><td><code>Number</code></td><td><code>3</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Indicates how many times the request is retried if <a href="#new_BasicCrawler_new"><code>handleRequestFunction()</code></a> fails.</p>
+</td></tr><tr>
+<td><code>[options.maxRequestsPerCrawl]</code></td><td><code>Number</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
+ Always set this value in order to prevent infinite loops in misconfigured crawlers.
+ Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.</p>
+</td></tr><tr>
+<td><code>[options.autoscaledPoolOptions]</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Custom options passed to the underlying <a href="autoscaledpool"><code>AutoscaledPool</code></a> constructor.
+ Note that the <code>runTaskFunction</code>, <code>isTaskReadyFunction</code> and <code>isFinishedFunction</code> options
+ are provided by <code>BasicCrawler</code> and cannot be overridden.</p>
+</td></tr><tr>
+<td><code>[options.minConcurrency]</code></td><td><code>Object</code></td><td><code>1</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding <a href="autoscaledpool"><code>AutoscaledPool</code></a> option.</p>
+</td></tr><tr>
+<td><code>[options.maxConcurrency]</code></td><td><code>Object</code></td><td><code>1000</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding <a href="autoscaledpool"><code>AutoscaledPool</code></a> option.</p>
+</td></tr></tbody>
+</table>
+<a name="BasicCrawler+run"></a>
+
+## `basicCrawler.run()` ⇒ <code>Promise</code>
+Runs the crawler. Returns a promise that gets resolved once all the requests are processed.
+
+<a name="BasicCrawler+abort"></a>
+
+## `basicCrawler.abort()` ⇒ <code>Promise</code>
+Aborts the crawler by preventing additional requests and terminating the running ones.
+
diff --git a/docs/api/CheerioCrawler.md b/docs/api/CheerioCrawler.md
new file mode 100644
index 000000000..e42c06a07
--- /dev/null
+++ b/docs/api/CheerioCrawler.md
@@ -0,0 +1,187 @@
+---
+id: cheeriocrawler
+title: CheerioCrawler
+---
+<a name="CheerioCrawler"></a>
+
+Provides a framework for the parallel crawling of web pages using plain HTTP requests and
+<a href="https://www.npmjs.com/package/cheerio" target="_blank">cheerio</a> HTML parser.
+
+`CheerioCrawler` downloads each URL using a plain HTTP request,
+parses the HTML content using <a href="https://www.npmjs.com/package/cheerio" target="_blank">cheerio</a>
+and then invokes the user-provided [`handlePageFunction()`](#new_CheerioCrawler_new) to extract page data
+using a <a href="https://jquery.com/" target="_blank">jQuery</a>-like interface to the parsed HTML DOM.
+
+The source URLs are represented using [`Request`](request) objects that are fed from
+[`RequestList`](requestlist) or [`RequestQueue`](requestqueue) instances provided by the [`requestList`](#new_CheerioCrawler_new)
+or [`requestQueue`](#new_CheerioCrawler_new) constructor options, respectively.
+
+If both [`requestList`](#new_CheerioCrawler_new) and [`requestQueue`](#new_CheerioCrawler_new) are used,
+the instance first processes URLs from the [`RequestList`](requestlist) and automatically enqueues all of them
+to [`RequestQueue`](requestqueue) before it starts their processing. This ensures that a single URL is not crawled multiple times.
+
+The crawler finishes when there are no more [`Request`](request) objects to crawl.
+
+By default, `CheerioCrawler` downloads HTML using the
+<a href="https://www.npmjs.com/package/request-promise" target="_blank">request-promise</a> NPM package.
+You can override this behavior by setting the `requestFunction` option.
+
+New requests are only dispatched when there is enough free CPU and memory available,
+using the functionality provided by the [`AutoscaledPool`](autoscaledpool) class.
+All [`AutoscaledPool`](autoscaledpool) configuration options can be passed to the `autoscaledPoolOptions`
+parameter of the `CheerioCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+[`AutoscaledPool`](autoscaledpool) options are available directly in the `CheerioCrawler` constructor.
+
+**Example usage:**
+
+```javascript
+// Prepare a list of URLs to crawl
+const requestList = new Apify.RequestList({
+ sources: [
+ { url: 'http://www.example.com/page-1' },
+ { url: 'http://www.example.com/page-2' },
+ ],
+});
+await requestList.initialize();
+
+// Crawl the URLs
+const crawler = new Apify.CheerioCrawler({
+ requestList,
+ handlePageFunction: async ({ $, html, request }) => {
+
+ const data = [];
+
+ // Do some data extraction from the page with Cheerio.
+ $('.some-collection').each((index, el) => {
+ data.push({ title: $(el).find('.some-title').text() });
+ });
+
+ // Save the data to dataset.
+ await Apify.pushData({
+ url: request.url,
+ html,
+ data,
+ })
+ },
+});
+
+await crawler.run();
+```
+
+
+* [CheerioCrawler](cheeriocrawler)
+ * [`new CheerioCrawler(options)`](#new_CheerioCrawler_new)
+ * [`.run()`](#CheerioCrawler+run) ⇒ <code>Promise</code>
+ * [`.abort()`](#CheerioCrawler+abort) ⇒ <code>Promise</code>
+
+<a name="new_CheerioCrawler_new"></a>
+
+## `new CheerioCrawler(options)`
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>options</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>All <code>CheerioCrawler</code> parameters are passed
+ via an options object with the following keys:</p>
+</td></tr><tr>
+<td><code>options.handlePageFunction</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>User-provided function that performs the logic of the crawler. It is called for each page
+ loaded and parsed by the crawler.</p>
+<p> The function receives the following object as an argument:</p>
+<pre><code> {
+ $: Cheerio, // the Cheerio object with parsed HTML
+ html: String // the raw HTML of the page
+ request: Request
+ }
+</code></pre><p> With the <a href="request"><code>Request</code></a> object representing the URL to crawl.
+ If the function returns a promise, it is awaited.</p>
+</td></tr><tr>
+<td><code>options.requestList</code></td><td><code><a href="requestlist">RequestList</a></code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Static list of URLs to be processed.
+ Either <a href="requestlist"><code>RequestList</code></a> or <a href="requestqueue"><code>RequestQueue</code></a> must be provided.</p>
+</td></tr><tr>
+<td><code>options.requestQueue</code></td><td><code><a href="requestqueue">RequestQueue</a></code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
+ Either <a href="requestlist"><code>RequestList</code></a> or <a href="requestqueue"><code>RequestQueue</code></a> must be provided.</p>
+</td></tr><tr>
+<td><code>[options.requestFunction]</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Overrides the function that performs the HTTP request to get the raw HTML needed for Cheerio.
+ See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/cheerio_crawler.js#L264">GitHub</a> for default behavior.</p>
+</td></tr><tr>
+<td><code>[options.handlePageTimeoutSecs]</code></td><td><code>Number</code></td><td><code>300</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Timeout in which the function passed as <code>options.handlePageFunction</code> needs to finish, given in seconds.</p>
+</td></tr><tr>
+<td><code>[options.requestTimeoutSecs]</code></td><td><code>Number</code></td><td><code>30</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Timeout in which the function passed as <code>options.requestFunction</code> needs to finish, given in seconds.</p>
+</td></tr><tr>
+<td><code>[options.ignoreSslErrors]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If set to true, SSL certificate errors will be ignored. This is dependent on using the default
+ request function. If using a custom <code>options.requestFunction</code>, user needs to implement this functionality.</p>
+</td></tr><tr>
+<td><code>[options.handleFailedRequestFunction]</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Function that handles requests that failed more then <code>option.maxRequestRetries</code> times.
+ See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/cheerio_crawler.js#L13">GitHub</a> for default behavior.</p>
+</td></tr><tr>
+<td><code>[options.maxRequestRetries]</code></td><td><code>Number</code></td><td><code>3</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Indicates how many times the request is retried if either <code>requestFunction</code> or <code>handlePageFunction</code> fails.</p>
+</td></tr><tr>
+<td><code>[options.maxRequestsPerCrawl]</code></td><td><code>Number</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
+ Always set this value in order to prevent infinite loops in misconfigured crawlers.
+ Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.</p>
+</td></tr><tr>
+<td><code>[options.autoscaledPoolOptions]</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Custom options passed to the underlying <a href="autoscaledpool"><code>AutoscaledPool</code></a> constructor.
+ Note that the <code>runTaskFunction</code>, <code>isTaskReadyFunction</code> and <code>isFinishedFunction</code> options
+ are provided by <code>CheerioCrawler</code> and cannot be overridden.</p>
+</td></tr><tr>
+<td><code>[options.minConcurrency]</code></td><td><code>Object</code></td><td><code>1</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding <a href="autoscaledpool"><code>AutoscaledPool</code></a> option.</p>
+</td></tr><tr>
+<td><code>[options.maxConcurrency]</code></td><td><code>Object</code></td><td><code>1000</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding <a href="autoscaledpool"><code>AutoscaledPool</code></a> option.</p>
+</td></tr></tbody>
+</table>
+<a name="CheerioCrawler+run"></a>
+
+## `cheerioCrawler.run()` ⇒ <code>Promise</code>
+Runs the crawler. Returns promise that gets resolved once all the requests got processed.
+
+<a name="CheerioCrawler+abort"></a>
+
+## `cheerioCrawler.abort()` ⇒ <code>Promise</code>
+Aborts the crawler by preventing crawls of additional pages and terminating the running ones.
+
diff --git a/docs/api/Dataset.md b/docs/api/Dataset.md
new file mode 100644
index 000000000..b6f04de72
--- /dev/null
+++ b/docs/api/Dataset.md
@@ -0,0 +1,387 @@
+---
+id: dataset
+title: Dataset
+---
+<a name="Dataset"></a>
+
+The `Dataset` class represents a store for structured data where each object stored has the same attributes,
+such as online store products or real estate offers. You can imagine it as a table,
+where each object is a row and its attributes are columns.
+Dataset is an append-only storage - you can only add new records to it but you cannot modify or remove existing records.
+Typically it is used to store crawling results.
+
+Do not instantiate this class directly, use the
+[`Apify.openDataset()`](apify#module_Apify.openDataset) function instead.
+
+`Dataset` stores its data either on local disk or in the Apify cloud,
+depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.
+
+If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in
+the local directory in the following files:
+```
+{APIFY_LOCAL_STORAGE_DIR}/datasets/{DATASET_ID}/{INDEX}.json
+```
+Note that `{DATASET_ID}` is the name or ID of the dataset. The default dataset has ID: `default`,
+unless you override it by setting the `APIFY_DEFAULT_DATASET_ID` environment variable.
+Each dataset item is stored as a separate JSON file, where `{INDEX}` is a zero-based index of the item in the dataset.
+
+If the `APIFY_TOKEN` environment variable is provided instead, the data are stored
+in the <a href="https://www.apify.com/docs/storage#dataset" target="_blank">Apify Dataset</a> cloud storage.
+
+**Example usage:**
+
+```javascript
+// Write a single row to the default dataset
+await Apify.pushData({ col1: 123, col2: 'val2' });
+
+// Open a named dataset
+const dataset = await Apify.openDataset('some-name');
+
+// Write a single row
+await dataset.pushData({ foo: 'bar' });
+
+// Write multiple rows
+await dataset.pushData([
+ { foo: 'bar2', col2: 'val2' },
+ { col3: 123 },
+]);
+```
+
+
+* [Dataset](dataset)
+ * [`.pushData(data)`](#Dataset+pushData) ⇒ <code>Promise</code>
+ * [`.getData([options])`](#Dataset+getData) ⇒ <code>Promise</code>
+ * [`.getInfo()`](#Dataset+getInfo) ⇒ <code>Promise&lt;Object&gt;</code>
+ * [`.forEach(iteratee, [options], [index])`](#Dataset+forEach) ⇒ <code>Promise</code>
+ * [`.map(iteratee, options)`](#Dataset+map) ⇒ <code>Promise&lt;Array&gt;</code>
+ * [`.reduce(iteratee, memo, options)`](#Dataset+reduce) ⇒ <code>Promise&lt;\*&gt;</code>
+ * [`.delete()`](#Dataset+delete) ⇒ <code>Promise</code>
+
+<a name="Dataset+pushData"></a>
+
+## `dataset.pushData(data)` ⇒ <code>Promise</code>
+Stores an object or an array of objects to the dataset.
+The function returns a promise that resolves when the operation finishes.
+It has no result, but throws on invalid args or other errors.
+
+**IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
+otherwise the actor process might finish before the data is stored!
+
+The size of the data is limited by the receiving API and therefore `pushData()` will only
+allow objects whose JSON representation is smaller than 9MB. When an array is passed,
+none of the included objects
+may be larger than 9MB, but the array itself may be of any size.
+
+The function internally
+chunks the array into separate items and pushes them sequentially.
+The chunking process is stable (keeps order of data), but it does not provide a transaction
+safety mechanism. Therefore, in the event of an uploading error (after several automatic retries),
+the function's Promise will reject and the dataset will be left in a state where some of
+the items have already been saved to the dataset while other items from the source array were not.
+To overcome this limitation, the developer may, for example, read the last item saved in the dataset
+and re-attempt the save of the data from this item onwards to prevent duplicates.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>data</code></td><td><code>Object</code> | <code>Array</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Object or array of objects containing data to be stored in the default dataset.
+The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.</p>
+</td></tr></tbody>
+</table>
+<a name="Dataset+getData"></a>
+
+## `dataset.getData([options])` ⇒ <code>Promise</code>
+Returns items in the dataset based on the provided parameters.
+
+If format is `json` then the function doesn't return an array of records but [`PaginationList`](../typedefs/paginationlist) instead.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>[options]</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>All <code>getData()</code> parameters are passed
+ via an options object with the following keys:</p>
+</td></tr><tr>
+<td><code>[options.format]</code></td><td><code>String</code></td><td><code>&#x27;json&#x27;</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Format of the items, possible values are: <code>json</code>, <code>csv</code>, <code>xlsx</code>, <code>html</code>, <code>xml</code> and <code>rss</code>.</p>
+</td></tr><tr>
+<td><code>[options.offset]</code></td><td><code>Number</code></td><td><code>0</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Number of array elements that should be skipped at the start.</p>
+</td></tr><tr>
+<td><code>[options.limit]</code></td><td><code>Number</code></td><td><code>250000</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Maximum number of array elements to return.</p>
+</td></tr><tr>
+<td><code>[options.desc]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If <code>true</code> then the objects are sorted by <code>createdAt</code> in descending order.
+ Otherwise they are sorted in ascending order.</p>
+</td></tr><tr>
+<td><code>[options.fields]</code></td><td><code>Array</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>An array of field names that will be included in the result. If omitted, all fields are included in the results.</p>
+</td></tr><tr>
+<td><code>[options.unwind]</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Specifies a name of the field in the result objects that will be used to unwind the resulting objects.
+ By default, the results are returned as they are.</p>
+</td></tr><tr>
+<td><code>[options.disableBodyParser]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If <code>true</code> then response from API will not be parsed.</p>
+</td></tr><tr>
+<td><code>[options.attachment]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If <code>true</code> then the response will define the <code>Content-Disposition: attachment</code> HTTP header, forcing a web
+ browser to download the file rather than to display it. By default, this header is not present.</p>
+</td></tr><tr>
+<td><code>[options.delimiter]</code></td><td><code>String</code></td><td><code>&#x27;,&#x27;</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>A delimiter character for CSV files, only used if <code>format</code> is <code>csv</code>.
+ You might need to URL-encode the character (e.g. use <code>%09</code> for tab or <code>%3B</code> for semicolon).</p>
+</td></tr><tr>
+<td><code>[options.bom]</code></td><td><code>Boolean</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>All responses are encoded in UTF-8 encoding. By default, the CSV files are prefixed with the UTF-8 Byte
+ Order Mark (BOM), while JSON, JSONL, XML, HTML and RSS files are not. If you want to override this default
+ behavior, set <code>bom</code> option to <code>true</code> to include the BOM, or set <code>bom</code> to <code>false</code> to skip it.</p>
+</td></tr><tr>
+<td><code>[options.xmlRoot]</code></td><td><code>String</code></td><td><code>&#x27;results&#x27;</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Overrides the default root element name of the XML output. By default, the root element is <code>results</code>.</p>
+</td></tr><tr>
+<td><code>[options.xmlRow]</code></td><td><code>String</code></td><td><code>&#x27;page&#x27;</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Overrides the default element name that wraps each page or page function result object in XML output.
+ By default, the element name is <code>page</code> or <code>result</code>, depending on the value of the <code>simplified</code> option.</p>
+</td></tr><tr>
+<td><code>[options.skipHeaderRow]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If set to <code>true</code> then header row in CSV format is skipped.</p>
+</td></tr></tbody>
+</table>
+<a name="Dataset+getInfo"></a>
+
+## `dataset.getInfo()` ⇒ <code>Promise&lt;Object&gt;</code>
+Returns an object containing general information about the dataset.
+
+**Example:**
+```
+{
+ "id": "WkzbQMuFYuamGv3YF",
+ "name": "d7b9MDYsbtX5L7XAj",
+ "userId": "wRsJZtadYvn4mBZmm",
+ "createdAt": "2015-12-12T07:34:14.202Z",
+ "modifiedAt": "2015-12-13T08:36:13.202Z",
+ "accessedAt": "2015-12-14T08:36:13.202Z",
+ "itemsCount": 0
+}
+```
+
+<a name="Dataset+forEach"></a>
+
+## `dataset.forEach(iteratee, [options], [index])` ⇒ <code>Promise</code>
+Iterates over dataset items, yielding each in turn to an `iteratee()` function.
+Each invocation of `iteratee()` is called with two arguments: `(element, index)`.
+
+If the `iteratee()` returns a Promise then it is awaited before a next call.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>iteratee</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"></td></tr><tr>
+<td><code>[options]</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>All <code>forEach()</code> parameters are passed
+ via an options object with the following keys:</p>
+</td></tr><tr>
+<td><code>[options.offset]</code></td><td><code>Number</code></td><td><code>0</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Number of array elements that should be skipped at the start.</p>
+</td></tr><tr>
+<td><code>[options.desc]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If <code>true</code> then the objects are sorted by <code>createdAt</code> in descending order.</p>
+</td></tr><tr>
+<td><code>[options.fields]</code></td><td><code>Array</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>If provided then returned objects will only contain specified keys.</p>
+</td></tr><tr>
+<td><code>[options.unwind]</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>If provided then objects will be unwound based on provided field.</p>
+</td></tr><tr>
+<td><code>[options.limit]</code></td><td><code>Number</code></td><td><code>250000</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>How many items to load in one request.</p>
+</td></tr><tr>
+<td><code>[index]</code></td><td><code>Number</code></td><td><code>0</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Controls the initial index number passed to the <code>iteratee()</code>.</p>
+</td></tr></tbody>
+</table>
+<a name="Dataset+map"></a>
+
+## `dataset.map(iteratee, options)` ⇒ <code>Promise&lt;Array&gt;</code>
+Produces a new array of values by mapping each value in list through a transformation function `iteratee()`.
+Each invocation of `iteratee()` is called with two arguments: `(element, index)`.
+
+If `iteratee` returns a `Promise` then it's awaited before a next call.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>iteratee</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"></td></tr><tr>
+<td><code>options</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>All <code>map()</code> parameters are passed
+ via an options object with the following keys:</p>
+</td></tr><tr>
+<td><code>[options.offset]</code></td><td><code>Number</code></td><td><code>0</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Number of array elements that should be skipped at the start.</p>
+</td></tr><tr>
+<td><code>[options.desc]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If <code>true</code> then the objects are sorted by createdAt in descending order.</p>
+</td></tr><tr>
+<td><code>[options.fields]</code></td><td><code>Array</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>If provided then returned objects will only contain specified keys</p>
+</td></tr><tr>
+<td><code>[options.unwind]</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>If provided then objects will be unwound based on provided field.</p>
+</td></tr><tr>
+<td><code>[options.limit]</code></td><td><code>Number</code></td><td><code>250000</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>How many items to load in one request.</p>
+</td></tr></tbody>
+</table>
+<a name="Dataset+reduce"></a>
+
+## `dataset.reduce(iteratee, memo, options)` ⇒ <code>Promise&lt;\*&gt;</code>
+Reduces a list of values down to a single value.
+
+Memo is the initial state of the reduction, and each successive step of it should be returned by `iteratee()`.
+The `iteratee()` is passed three arguments: the `memo`, then the `value` and `index` of the iteration.
+
+If no `memo` is passed to the initial invocation of reduce, the `iteratee()` is not invoked on the first element of the list.
+The first element is instead passed as the memo in the invocation of the `iteratee()` on the next element in the list.
+
+If `iteratee()` returns a `Promise` then it's awaited before a next call.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>iteratee</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"></td></tr><tr>
+<td><code>memo</code></td><td><code>*</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Initial state of the reduction.</p>
+</td></tr><tr>
+<td><code>options</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>All <code>reduce()</code> parameters are passed
+ via an options object with the following keys:</p>
+</td></tr><tr>
+<td><code>[options.offset]</code></td><td><code>Number</code></td><td><code>0</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Number of array elements that should be skipped at the start.</p>
+</td></tr><tr>
+<td><code>[options.desc]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If <code>true</code> then the objects are sorted by createdAt in descending order.</p>
+</td></tr><tr>
+<td><code>[options.fields]</code></td><td><code>Array</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>If provided then returned objects will only contain specified keys</p>
+</td></tr><tr>
+<td><code>[options.unwind]</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>If provided then objects will be unwound based on provided field.</p>
+</td></tr><tr>
+<td><code>[options.limit]</code></td><td><code>Number</code></td><td><code>250000</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>How many items to load in one request.</p>
+</td></tr></tbody>
+</table>
+<a name="Dataset+delete"></a>
+
+## `dataset.delete()` ⇒ <code>Promise</code>
+Removes the dataset either from the Apify cloud storage or from the local directory,
+depending on the mode of operation.
+
diff --git a/docs/api/KeyValueStore.md b/docs/api/KeyValueStore.md
new file mode 100644
index 000000000..ac211f9cf
--- /dev/null
+++ b/docs/api/KeyValueStore.md
@@ -0,0 +1,183 @@
+---
+id: keyvaluestore
+title: KeyValueStore
+---
+<a name="KeyValueStore"></a>
+
+The `KeyValueStore` class represents a key-value store, a simple data storage that is used
+for saving and reading data records or files. Each data record is
+represented by a unique key and associated with a MIME content type. Key-value stores are ideal
+for saving screenshots, actor inputs and outputs, web pages, PDFs or to persist the state of crawlers.
+
+Do not instantiate this class directly, use the
+[`Apify.openKeyValueStore()`](apify#module_Apify.openKeyValueStore) function instead.
+
+Each actor run is associated with a default key-value store, which is created exclusively
+for the run. By convention, the actor input and output are stored into the
+default key-value store under the `INPUT` and `OUTPUT` key, respectively.
+Typically, input and output are JSON files, although it can be any other format.
+To access the default key-value store directly, you can use the
+[`Apify.getValue()`](apify#module_Apify.getValue)
+and [`Apify.setValue()`](apify#module_Apify.setValue) convenience functions.
+
+`KeyValueStore` stores its data either on local disk or in the Apify cloud,
+depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.
+
+If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in
+the local directory in the following files:
+```
+{APIFY_LOCAL_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}
+```
+Note that `{STORE_ID}` is the name or ID of the key-value store. The default key value store has ID: `default`,
+unless you override it by setting the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
+The `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.
+
+If the `APIFY_TOKEN` environment variable is provided instead, the data is stored in the
+<a href="https://www.apify.com/docs/storage#key-value-store" target="_blank">Apify Key-Value Store</a>
+cloud storage.
+
+**Example usage:**
+
+```javascript
+// Get actor input from the default key-value store
+const input = await Apify.getValue('INPUT');
+
+// Write actor output to the default key-value store.
+await Apify.setValue('OUTPUT', { myResult: 123 });
+
+// Open a named key-value store
+const store = await Apify.openKeyValueStore('some-name');
+
+// Write a record. JavaScript object is automatically converted to JSON,
+// strings and binary buffers are stored as they are
+await store.setValue('some-key', { foo: 'bar' });
+
+// Read a record. Note that JSON is automatically parsed to a JavaScript object,
+// text data returned as a string and other data is returned as binary buffer
+const value = await store.getValue('some-key');
+
+ // Delete record
+await store.delete('some-key');
+```
+
+
+* [KeyValueStore](keyvaluestore)
+ * [`.getValue(key)`](#KeyValueStore+getValue) ⇒ <code>Promise&lt;(Object\|String\|Buffer)&gt;</code>
+ * [`.setValue(key, value, [options])`](#KeyValueStore+setValue) ⇒ <code>Promise</code>
+ * [`.delete()`](#KeyValueStore+delete) ⇒ <code>Promise</code>
+
+<a name="KeyValueStore+getValue"></a>
+
+## `keyValueStore.getValue(key)` ⇒ <code>Promise&lt;(Object\|String\|Buffer)&gt;</code>
+Gets a value from the key-value store.
+
+The function returns a `Promise` that resolves to the record value,
+whose JavaScript type depends on the MIME content type of the record.
+Records with the `application/json`
+content type are automatically parsed and returned as a JavaScript object.
+Similarly, records with `text/plain` content types are returned as a string.
+For all other content types, the value is returned as a raw
+<a href="https://nodejs.org/api/buffer.html" target="_blank"><code>Buffer</code></a> instance.
+
+If the record does not exist, the function resolves to `null`.
+
+To save or delete a value in the key-value store, use the
+[`setValue`](#KeyValueStore+setValue) function.
+
+**Example usage:**
+
+```javascript
+const store = await Apify.openKeyValueStore('my-screenshots');
+const buffer = await store.getValue('screenshot1.png');
+```
+
+**Returns**: <code>Promise&lt;(Object\|String\|Buffer)&gt;</code> - Returns a promise that resolves to an object, string
+ or <a href="https://nodejs.org/api/buffer.html" target="_blank"><code>Buffer</code></a>, depending
+ on the MIME content type of the record.
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>key</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Unique key of the record.</p>
+</td></tr></tbody>
+</table>
+<a name="KeyValueStore+setValue"></a>
+
+## `keyValueStore.setValue(key, value, [options])` ⇒ <code>Promise</code>
+Saves or deletes a record in the key-value store.
+The function returns a promise that resolves once the record has been saved or deleted.
+
+**Example usage:**
+
+```javascript
+const store = await Apify.openKeyValueStore('my-store');
+await store.setValue('RESULTS', 'my text data', { contentType: 'text/plain' });
+```
+
+By default, `value` is converted to JSON and stored with the
+`application/json; charset=utf-8` MIME content type.
+To store the value with another content type, pass it in the options as follows:
+```javascript
+const store = await Apify.openKeyValueStore('my-store');
+await store.setValue('RESULTS', 'my text data', { contentType: 'text/plain' });
+```
+If you set custom content type, `value` must be either a string or
+<a href="https://nodejs.org/api/buffer.html" target="_blank"><code>Buffer</code></a>, otherwise an error will be thrown.
+
+If `value` is `null`, the record is deleted instead. Note that the `setValue()` function succeeds
+regardless whether the record existed or not.
+
+To retrieve a value from the key-value store, use the
+[`getValue`](#KeyValueStore+getValue) function.
+
+**IMPORTANT:** Always make sure to use the `await` keyword when calling `setValue()`,
+otherwise the actor process might finish before the value is stored!
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>key</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Unique key of the record.</p>
+</td></tr><tr>
+<td><code>value</code></td><td><code>Object</code> | <code>String</code> | <code>Buffer</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Record data, which can be one of the following values:
+ <ul>
+ <li>If <code>null</code>, the record in the key-value store is deleted.</li>
+ <li>If no <code>options.contentType</code> is specified, <code>value</code> can be any JavaScript object and it will be stringified to JSON.</li>
+ <li>If <code>options.contentType</code> is specified, <code>value</code> is considered raw data and it must be either a <code>String</code>
+ or <a href="https://nodejs.org/api/buffer.html" target="_blank"><code>Buffer</code></a>.</li>
+ </ul>
+ For any other value an error will be thrown.</p>
+</td></tr><tr>
+<td><code>[options]</code></td><td><code>Object</code></td>
+</tr>
+<tr>
+<td colspan="3"></td></tr><tr>
+<td><code>[options.contentType]</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Specifies a custom MIME content type of the record.</p>
+</td></tr></tbody>
+</table>
+<a name="KeyValueStore+delete"></a>
+
+## `keyValueStore.delete()` ⇒ <code>Promise</code>
+Removes the key-value store either from the Apify cloud storage or from the local directory,
+depending on the mode of operation.
+
diff --git a/docs/api/PseudoUrl.md b/docs/api/PseudoUrl.md
new file mode 100644
index 000000000..abd4f761f
--- /dev/null
+++ b/docs/api/PseudoUrl.md
@@ -0,0 +1,106 @@
+---
+id: pseudourl
+title: PseudoUrl
+---
+<a name="PseudoUrl"></a>
+
+Represents a pseudo URL (PURL) - an URL pattern used by web crawlers
+to specify which URLs should the crawler visit.
+This class is used by the [utils.puppeteer.enqueueLinks()](puppeteer#puppeteer.enqueueLinks) function.
+
+A PURL is simply a URL with special directives enclosed in `[]` brackets.
+Currently, the only supported directive is `[RegExp]`,
+which defines a JavaScript-style regular expression to match against the URL.
+
+For example, a PURL `http://www.example.com/pages/[(\w|-)*]` will match all of the following URLs:
+
+<ul>
+ <li><code>http://www.example.com/pages/</code></li>
+ <li><code>http://www.example.com/pages/my-awesome-page</code></li>
+ <li><code>http://www.example.com/pages/something</code></li>
+</ul>
+
+If either `[` or `]` is part of the normal query string, it must be encoded as `[\x5B]` or `[\x5D]`,
+respectively. For example, the following PURL:
+```http
+http://www.example.com/search?do[\x5B]load[\x5D]=1
+```
+will match the URL:
+```http
+http://www.example.com/search?do[load]=1
+```
+
+**Example usage:**
+
+```javascript
+const purl = new Apify.PseudoUrl('http://www.example.com/pages/[(\w|-)*]');
+
+if (purl.matches('http://www.example.com/pages/my-awesome-page')) console.log('Match!');
+```
+
+
+* [PseudoUrl](pseudourl)
+ * [`new PseudoUrl(purl, requestTemplate)`](#new_PseudoUrl_new)
+ * [`.matches(url)`](#PseudoUrl+matches) ⇒ <code>Boolean</code>
+ * [`.createRequest(url)`](#PseudoUrl+createRequest) ⇒ [<code>Request</code>](request)
+
+<a name="new_PseudoUrl_new"></a>
+
+## `new PseudoUrl(purl, requestTemplate)`
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>purl</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Pseudo URL.</p>
+</td></tr><tr>
+<td><code>requestTemplate</code></td><td><code>Object</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Options for the new <a href="request"><code>Request</code></a> instances created for matching URLs.</p>
+</td></tr></tbody>
+</table>
+<a name="PseudoUrl+matches"></a>
+
+## `pseudoUrl.matches(url)` ⇒ <code>Boolean</code>
+Determines whether a URL matches this pseudo-URL pattern.
+
+**Returns**: <code>Boolean</code> - Returns `true` if given URL matches pseudo URL.
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>url</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>URL to be matched.</p>
+</td></tr></tbody>
+</table>
+<a name="PseudoUrl+createRequest"></a>
+
+## `pseudoUrl.createRequest(url)` ⇒ [<code>Request</code>](request)
+Creates a Request object from requestTemplate and given URL.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>url</code></td><td><code>String</code></td>
+</tr>
+<tr>
+</tr></tbody>
+</table>
diff --git a/docs/api/PuppeteerCrawler.md b/docs/api/PuppeteerCrawler.md
new file mode 100644
index 000000000..05530adb7
--- /dev/null
+++ b/docs/api/PuppeteerCrawler.md
@@ -0,0 +1,222 @@
+---
+id: puppeteercrawler
+title: PuppeteerCrawler
+---
+<a name="PuppeteerCrawler"></a>
+
+Provides a simple framework for parallel crawling of web pages
+using headless Chrome with <a href="https://github.com/GoogleChrome/puppeteer" target="_blank">Puppeteer</a>.
+
+The source URLs are represented using [`Request`](request) objects that are fed from
+[`RequestList`](requestlist) or [`RequestQueue`](requestqueue) instances provided by the [`requestList`](#new_PuppeteerCrawler_new)
+or [`requestQueue`](#new_PuppeteerCrawler_new) constructor options, respectively.
+
+If both [`requestList`](#new_PuppeteerCrawler_new) and [`requestQueue`](#new_PuppeteerCrawler_new) are used,
+the instance first processes URLs from the [`RequestList`](requestlist) and automatically enqueues all of them
+to [`RequestQueue`](requestqueue) before it starts their processing. This ensures that a single URL is not crawled multiple times.
+
+The crawler finishes when there are no more [`Request`](request) objects to crawl.
+
+`PuppeteerCrawler` opens a new Chrome page (i.e. tab) for each [`Request`](request) object to crawl
+and then calls the function provided by user as the [`handlePageFunction()`](#new_PuppeteerCrawler_new) option.
+
+New pages are only opened when there is enough free CPU and memory available,
+using the functionality provided by the [`AutoscaledPool`](autoscaledpool) class.
+All [`AutoscaledPool`](autoscaledpool) configuration options can be passed to the `autoscaledPoolOptions`
+parameter of the `PuppeteerCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+[`AutoscaledPool`](autoscaledpool) options are available directly in the `PuppeteerCrawler` constructor.
+
+Note that the pool of Puppeteer instances is internally managed by
+the [`PuppeteerPool`](puppeteerpool) class. Many constructor options
+such as `maxOpenPagesPerInstance` or `launchPuppeteerFunction` are passed directly
+to [`PuppeteerPool`](puppeteerpool) constructor.
+
+**Example usage:**
+
+```javascript
+const crawler = new Apify.PuppeteerCrawler({
+ requestList,
+ handlePageFunction: async ({ page, request }) => {
+ // This function is called to extract data from a single web page
+ // 'page' is an instance of Puppeteer.Page with page.goto(request.url) already called
+ // 'request' is an instance of Request class with information about the page to load
+ await Apify.pushData({
+ title: await page.title(),
+ url: request.url,
+ succeeded: true,
+ })
+ },
+ handleFailedRequestFunction: async ({ request }) => {
+ // This function is called when crawling of a request failed too many time
+ await Apify.pushData({
+ url: request.url,
+ succeeded: false,
+ errors: request.errorMessages,
+ })
+ },
+});
+
+await crawler.run();
+```
+
+
+* [PuppeteerCrawler](puppeteercrawler)
+ * [`new PuppeteerCrawler(options)`](#new_PuppeteerCrawler_new)
+ * [`.run()`](#PuppeteerCrawler+run) ⇒ <code>Promise</code>
+ * [`.abort()`](#PuppeteerCrawler+abort) ⇒ <code>Promise</code>
+
+<a name="new_PuppeteerCrawler_new"></a>
+
+## `new PuppeteerCrawler(options)`
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>options</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>All <code>PuppeteerCrawler</code> parameters are passed
+ via an options object with the following keys:</p>
+</td></tr><tr>
+<td><code>options.handlePageFunction</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Function that is called to process each request.
+ It is passed an object with the following fields:</p>
+<pre><code> {
+ request: Request,
+ page: Page,
+ puppeteerPool: PuppeteerPool
+ }
+</code></pre><p> <code>request</code> is an instance of the <a href="request"><code>Request</code></a> object with details about the URL to open, HTTP method etc.
+ <code>page</code> is an instance of the <code>Puppeteer</code>
+ <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a>
+ class with <code>page.goto(request.url)</code> already called.
+ <code>puppeteerPool</code> is an instance of the <a href="puppeteerpool"><code>PuppeteerPool</code></a> used by this <code>PuppeteerCrawler</code>.</p>
+</td></tr><tr>
+<td><code>options.requestList</code></td><td><code><a href="requestlist">RequestList</a></code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Static list of URLs to be processed.
+ Either <a href="requestlist"><code>RequestList</code></a> or <a href="requestqueue"><code>RequestQueue</code></a> must be provided.</p>
+</td></tr><tr>
+<td><code>options.requestQueue</code></td><td><code><a href="requestqueue">RequestQueue</a></code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
+ Either <a href="requestlist"><code>RequestList</code></a> or <a href="requestqueue"><code>RequestQueue</code></a> must be provided.</p>
+</td></tr><tr>
+<td><code>[options.handlePageTimeoutSecs]</code></td><td><code>Number</code></td><td><code>300</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Timeout in which the function passed as <code>options.handlePageFunction</code> needs to finish, in seconds.</p>
+</td></tr><tr>
+<td><code>[options.gotoFunction]</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Overrides the function that opens the page in Puppeteer. The function should return the result of Puppeteer&#39;s
+ <a href="https://pptr.dev/#?product=Puppeteer&show=api-pagegotourl-options" target="_blank">page.goto()</a> function,
+ i.e. a <code>Promise</code> resolving to the <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-response" target="_blank">Response</a> object.</p>
+<p> This is useful if you need to extend the page load timeout or select different criteria
+ to determine that the navigation succeeded.</p>
+<p> Note that a single page object is only used to process a single request and it is closed afterwards.</p>
+<p> See source code on
+ <a href="https://github.com/apifytech/apify-js/blob/master/src/puppeteer_crawler.js#L9" target="_blank">GitHub</a>
+ for default behavior.</p>
+</td></tr><tr>
+<td><code>[options.handleFailedRequestFunction]</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Function to handle requests that failed more than <code>option.maxRequestRetries</code> times.
+ See source code on
+ <a href="https://github.com/apifytech/apify-js/blob/master/src/puppeteer_crawler.js#L11" target="_blank">GitHub</a>
+ for default behavior.</p>
+</td></tr><tr>
+<td><code>[options.maxRequestRetries]</code></td><td><code>Number</code></td><td><code>3</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Indicates how many times the request is retried if either <code>handlePageFunction()</code> or <code>gotoFunction()</code> fails.</p>
+</td></tr><tr>
+<td><code>[options.maxRequestsPerCrawl]</code></td><td><code>Number</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
+ Always set this value in order to prevent infinite loops in misconfigured crawlers.
+ Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.</p>
+</td></tr><tr>
+<td><code>[options.maxOpenPagesPerInstance]</code></td><td><code>Number</code></td><td><code>50</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Maximum number of opened tabs per browser. If this limit is reached then a new
+ browser instance is started. See <code>maxOpenPagesPerInstance</code> parameter of <a href="puppeteerpool"><code>PuppeteerPool</code></a>.</p>
+</td></tr><tr>
+<td><code>[options.retireInstanceAfterRequestCount]</code></td><td><code>Number</code></td><td><code>100</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Maximum number of requests that can be processed by a single browser instance.
+ After the limit is reached the browser will be retired and new requests will
+ be handled by a new browser instance.
+ See <code>retireInstanceAfterRequestCount</code> parameter of <a href="puppeteerpool"><code>PuppeteerPool</code></a>.</p>
+</td></tr><tr>
+<td><code>[options.instanceKillerIntervalMillis]</code></td><td><code>Number</code></td><td><code>60000</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Indicates how often are the open Puppeteer instances checked whether they can be closed.
+ See <code>instanceKillerIntervalMillis</code> parameter of <a href="puppeteerpool"><code>PuppeteerPool</code></a>.</p>
+</td></tr><tr>
+<td><code>[options.killInstanceAfterMillis]</code></td><td><code>Number</code></td><td><code>300000</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If Puppeteer instance reaches the <code>options.retireInstanceAfterRequestCount</code> limit then
+ it is considered retired and no more tabs will be opened. After the last tab is closed
+ the whole browser is closed too. This parameter defines a time limit for inactivity
+ after which the browser is closed even if there are pending tabs. See
+ <code>killInstanceAfterMillis</code> parameter of <a href="puppeteerpool"><code>PuppeteerPool</code></a>.</p>
+</td></tr><tr>
+<td><code>[options.launchPuppeteerFunction]</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Overrides the default function to launch a new Puppeteer instance.
+ See <code>launchPuppeteerFunction</code> parameter of <a href="puppeteerpool"><code>PuppeteerPool</code></a>.
+ See source code on
+ <a href="https://github.com/apifytech/apify-js/blob/master/src/puppeteer_pool.js#L28" target="_blank">GitHub</a>
+ for default behavior.</p>
+</td></tr><tr>
+<td><code>[options.launchPuppeteerOptions]</code></td><td><code><a href="../typedefs/launchpuppeteeroptions">LaunchPuppeteerOptions</a></code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Options used by <a href="apify#module_Apify.launchPuppeteer"><code>Apify.launchPuppeteer()</code></a> to start new Puppeteer instances.
+ See <code>launchPuppeteerOptions</code> parameter of <a href="puppeteerpool"><code>PuppeteerPool</code></a> and <a href="../typedefs/launchpuppeteeroptions"><code>LaunchPuppeteerOptions</code></a>.</p>
+</td></tr><tr>
+<td><code>[options.autoscaledPoolOptions]</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Custom options passed to the underlying <a href="autoscaledpool"><code>AutoscaledPool</code></a> instance constructor.
+ Note that the <code>runTaskFunction</code>, <code>isTaskReadyFunction</code> and <code>isFinishedFunction</code> options
+ are provided by <code>PuppeteerCrawler</code> and should not be overridden.</p>
+</td></tr><tr>
+<td><code>[options.minConcurrency]</code></td><td><code>Object</code></td><td><code>1</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding <a href="autoscaledpool"><code>AutoscaledPool</code></a> option.</p>
+</td></tr><tr>
+<td><code>[options.maxConcurrency]</code></td><td><code>Object</code></td><td><code>1000</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding <a href="autoscaledpool"><code>AutoscaledPool</code></a> option.</p>
+</td></tr></tbody>
+</table>
+<a name="PuppeteerCrawler+run"></a>
+
+## `puppeteerCrawler.run()` ⇒ <code>Promise</code>
+Runs the crawler. Returns promise that gets resolved once all the requests got processed.
+
+<a name="PuppeteerCrawler+abort"></a>
+
+## `puppeteerCrawler.abort()` ⇒ <code>Promise</code>
+Aborts the crawler by preventing crawls of additional pages and terminating the running ones.
+
diff --git a/docs/api/PuppeteerPool.md b/docs/api/PuppeteerPool.md
new file mode 100644
index 000000000..31033dfd2
--- /dev/null
+++ b/docs/api/PuppeteerPool.md
@@ -0,0 +1,151 @@
+---
+id: puppeteerpool
+title: PuppeteerPool
+---
+<a name="PuppeteerPool"></a>
+
+Manages a pool of Chrome browser instances controlled using
+<a href="https://github.com/GoogleChrome/puppeteer" target="_blank">Puppeteer</a>.
+
+`PuppeteerPool` reuses Chrome instances and tabs using specific browser rotation and retirement policies.
+This is useful in order to facilitate rotation of proxies, cookies
+or other settings in order to prevent detection of your web scraping bot,
+access web pages from various countries etc.
+
+Additionally, the reuse of browser instances instances speeds up crawling,
+and the retirement of instances helps mitigate effects of memory leaks in Chrome.
+
+`PuppeteerPool` is internally used by the [`PuppeteerCrawler`](puppeteercrawler) class.
+
+**Example usage:**
+
+```javascript
+const puppeteerPool = new PuppeteerPool({
+ launchPuppeteerFunction: () => {
+ // Use a new proxy with a new IP address for each new Chrome instance
+ return Apify.launchPuppeteer({
+ apifyProxySession: Math.random(),
+ });
+ },
+});
+
+const page1 = await puppeteerPool.newPage();
+const page2 = await puppeteerPool.newPage();
+const page3 = await puppeteerPool.newPage();
+
+// ... do something with the pages ...
+
+// Close all browsers.
+await puppeteerPool.destroy();
+```
+
+
+* [PuppeteerPool](puppeteerpool)
+ * [`new PuppeteerPool([options])`](#new_PuppeteerPool_new)
+ * [`.newPage()`](#PuppeteerPool+newPage) ⇒ <code>Promise&lt;Page&gt;</code>
+ * [`.destroy()`](#PuppeteerPool+destroy) ⇒ <code>Promise</code>
+ * [`.retire(browser)`](#PuppeteerPool+retire) ⇒ <code>Promise</code>
+
+<a name="new_PuppeteerPool_new"></a>
+
+## `new PuppeteerPool([options])`
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>[options]</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>All <code>PuppeteerPool</code> parameters are passed
+ via an options object with the following keys:</p>
+</td></tr><tr>
+<td><code>[options.maxOpenPagesPerInstance]</code></td><td><code>Number</code></td><td><code>50</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Maximum number of open pages (i.e. tabs) per browser. When this limit is reached, new pages are loaded in a new browser instance.</p>
+</td></tr><tr>
+<td><code>[options.retireInstanceAfterRequestCount]</code></td><td><code>Number</code></td><td><code>100</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Maximum number of requests that can be processed by a single browser instance.
+ After the limit is reached, the browser is retired and new requests are
+ handled by a new browser instance.</p>
+</td></tr><tr>
+<td><code>[options.instanceKillerIntervalMillis]</code></td><td><code>Number</code></td><td><code>60000</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Indicates how often are the open Puppeteer instances checked whether they can be closed.</p>
+</td></tr><tr>
+<td><code>[options.killInstanceAfterMillis]</code></td><td><code>Number</code></td><td><code>300000</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>When Puppeteer instance reaches the <code>options.retireInstanceAfterRequestCount</code> limit then
+ it is considered retired and no more tabs will be opened. After the last tab is closed the
+ whole browser is closed too. This parameter defines a time limit between the last tab was opened and
+ before the browser is closed even if there are pending open tabs.</p>
+</td></tr><tr>
+<td><code>[options.launchPuppeteerFunction]</code></td><td><code>function</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Overrides the default function to launch a new <code>Puppeteer</code> instance.
+ See source code on
+ <a href="https://github.com/apifytech/apify-js/blob/master/src/puppeteer_pool.js#L28" target="_blank">GitHub</a>
+ for default behavior.</p>
+</td></tr><tr>
+<td><code>[options.launchPuppeteerOptions]</code></td><td><code><a href="../typedefs/launchpuppeteeroptions">LaunchPuppeteerOptions</a></code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Options used by <code>Apify.launchPuppeteer()</code> to start new Puppeteer instances.
+ See <a href="../typedefs/launchpuppeteeroptions"><code>LaunchPuppeteerOptions</code></a>.</p>
+</td></tr><tr>
+<td><code>[options.recycleDiskCache]</code></td><td><code>Boolean</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Enables recycling of disk cache directories by Chrome instances.
+ When a browser instance is closed, its disk cache directory is not deleted but it&#39;s used by a newly opened browser instance.
+ This is useful to reduce amount of data that needs to be downloaded to speed up crawling and reduce proxy usage.
+ Note that the new browser starts with empty cookies, local storage etc. so this setting doesn&#39;t affect anonymity of your crawler.</p>
+<p> Beware that the disk cache directories can consume a lot of disk space.
+ To limit the space consumed, you can pass the <code>--disk-cache-size=X</code> argument to <code>options.launchPuppeteerOptions.args</code>,
+ where <code>X</code> is the approximate maximum number of bytes for disk cache.</p>
+<p> <em>IMPORTANT:</em> Currently this feature only works in <strong>headful</strong> mode, because of a bug in Chromium.</p>
+<p> The <code>options.recycleDiskCache</code> setting should not be used together with <code>--disk-cache-dir</code> argument in <code>options.launchPuppeteerOptions.args</code>.</p>
+</td></tr></tbody>
+</table>
+<a name="PuppeteerPool+newPage"></a>
+
+## `puppeteerPool.newPage()` ⇒ <code>Promise&lt;Page&gt;</code>
+Opens new tab in one of the browsers in the pool and returns a `Promise`
+that resolves to an instance of a Puppeteer
+<a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a>.
+
+<a name="PuppeteerPool+destroy"></a>
+
+## `puppeteerPool.destroy()` ⇒ <code>Promise</code>
+Closes all open browsers.
+
+<a name="PuppeteerPool+retire"></a>
+
+## `puppeteerPool.retire(browser)` ⇒ <code>Promise</code>
+Manually retires a Puppeteer
+<a href="https://pptr.dev/#?product=Puppeteer&show=api-class-browser" target="_blank"><code>Browser</code></a>
+instance from the pool. The browser will continue to process open pages so that they may gracefully finish.
+This is unlike `browser.close()` which will forcibly terminate the browser and all open pages will be closed.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>browser</code></td><td><code>Browser</code></td>
+</tr>
+<tr>
+</tr></tbody>
+</table>
diff --git a/docs/api/Request.md b/docs/api/Request.md
new file mode 100644
index 000000000..a65fe9c8f
--- /dev/null
+++ b/docs/api/Request.md
@@ -0,0 +1,200 @@
+---
+id: request
+title: Request
+---
+<a name="Request"></a>
+
+Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
+The `Request` object also stores information about errors that occurred during processing of the request.
+
+Each `Request` instance has the `uniqueKey` property, which can be either specified
+manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
+are considered as pointing to the same web resource. This behavior applies to all Apify SDK classes,
+such as [`RequestList`](requestlist), [`RequestQueue`](requestqueue) or [`PuppeteerCrawler`](puppeteercrawler).
+
+Example use:
+
+```javascript
+const request = new Apify.Request({
+ url: 'http://www.example.com',
+ headers: { Accept: 'application/json' },
+});
+
+...
+
+request.userData.foo = 'bar';
+request.pushErrorMessage(new Error('Request failed!'));
+
+...
+
+const foo = request.userData.foo;
+```
+
+**Properties**
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>id</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Request ID</p>
+</td></tr><tr>
+<td><code>url</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>URL of the web page to crawl.</p>
+</td></tr><tr>
+<td><code>uniqueKey</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>A unique key identifying the request.
+ Two requests with the same <code>uniqueKey</code> are considered as pointing to the same URL.</p>
+</td></tr><tr>
+<td><code>method</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>HTTP method, e.g. <code>GET</code> or <code>POST</code>.</p>
+</td></tr><tr>
+<td><code>payload</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>HTTP request payload, e.g. for POST requests.</p>
+</td></tr><tr>
+<td><code>retryCount</code></td><td><code>Number</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Indicates the number of times the crawling of the request has been retried on error.</p>
+</td></tr><tr>
+<td><code>errorMessages</code></td><td><code>Array&lt;String&gt;</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>An array of error messages from request processing.</p>
+</td></tr><tr>
+<td><code>headers</code></td><td><code>Object</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Object with HTTP headers. Key is header name, value is the value.</p>
+</td></tr><tr>
+<td><code>userData</code></td><td><code>Object</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Custom user data assigned to the request.</p>
+</td></tr><tr>
+<td><code>ignoreErrors</code></td><td><code>Boolean</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If <code>true</code> then errors in processing of this request are ignored.
+ For example, the request won&#39;t be retried in a case of an error.</p>
+</td></tr><tr>
+<td><code>handledAt</code></td><td><code>Date</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Indicates the time when the request has been processed.
+ Is <code>null</code> if the request has not been crawled yet.</p>
+</td></tr></tbody>
+</table>
+
+* [Request](request)
+ * [`new Request(options)`](#new_Request_new)
+ * [`.pushErrorMessage(errorOrMessage)`](#Request+pushErrorMessage)
+
+<a name="new_Request_new"></a>
+
+## `new Request(options)`
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>options</code></td><td><code>object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>All <code>Request</code> parameters are passed
+ via an options object with the following keys:</p>
+</td></tr><tr>
+<td><code>options.url</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>URL of the web page to crawl.</p>
+</td></tr><tr>
+<td><code>[options.uniqueKey]</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>A unique key identifying the request.
+Two requests with the same <code>uniqueKey</code> are considered as pointing to the same URL.</p>
+<p>If <code>uniqueKey</code> is not provided, then it is automatically generated by normalizing the URL.
+For example, the URL of <code>HTTP://www.EXAMPLE.com/something/</code> will produce the <code>uniqueKey</code>
+of <code>http://www.example.com/something</code>.</p>
+<p>The <code>keepUrlFragment</code> option determines whether URL hash fragment is included in the <code>uniqueKey</code> or not.
+Beware that the HTTP method and payload are not included in the <code>uniqueKey</code>,
+so requests to the same URL but with different HTTP methods or different POST payloads
+are all considered equal and will be deduplicated by <a href="requestlist"><code>RequestList</code></a> and <a href="requestqueue"><code>RequestQueue</code></a>.</p>
+<p>Pass an arbitrary non-empty text value to the <code>uniqueKey</code> property
+to override the default behavior and specify which URLs shall be considered equal.</p>
+</td></tr><tr>
+<td><code>[options.method]</code></td><td><code>String</code></td><td><code>&#x27;GET&#x27;</code></td>
+</tr>
+<tr>
+<td colspan="3"></td></tr><tr>
+<td><code>[options.payload]</code></td><td><code>String</code> | <code>Buffer</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>HTTP request payload, e.g. for POST requests.</p>
+</td></tr><tr>
+<td><code>[options.headers]</code></td><td><code>Object</code></td><td><code>{}</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>HTTP headers in the following format:</p>
+<pre><code> {
+ Accept: &#39;text/html&#39;,
+ &#39;Content-Type&#39;: &#39;application/json&#39;
+ }
+</code></pre></td></tr><tr>
+<td><code>[options.userData]</code></td><td><code>Object</code></td><td><code>{}</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Custom user data assigned to the request. Use this to save any request related data to the
+ request&#39;s scope, keeping them accessible on retries, failures etc.</p>
+</td></tr><tr>
+<td><code>[options.keepUrlFragment]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If <code>false</code> then the hash part of a URL is removed when computing the <code>uniqueKey</code> property.
+ For example, this causes the <code>http://www.example.com#foo</code> and <code>http://www.example.com#bar</code> URLs
+ to have the same <code>uniqueKey</code> of <code>http://www.example.com</code> and thus the URLs are considered equal.
+ Note that this option only has an effect if <code>uniqueKey</code> is not set.</p>
+</td></tr><tr>
+<td><code>[options.ignoreErrors]</code></td><td><code>String</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If <code>true</code> then errors in processing of this request will be ignored.
+ For example, the request won&#39;t be retried in a case of an error.</p>
+</td></tr></tbody>
+</table>
+<a name="Request+pushErrorMessage"></a>
+
+## `request.pushErrorMessage(errorOrMessage)`
+Stores information about an error that occurred during processing of this request.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>errorOrMessage</code></td><td><code>Error</code> | <code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Error object or error message to be stored in the request.</p>
+</td></tr></tbody>
+</table>
diff --git a/docs/api/RequestList.md b/docs/api/RequestList.md
new file mode 100644
index 000000000..5c1e4fe7e
--- /dev/null
+++ b/docs/api/RequestList.md
@@ -0,0 +1,223 @@
+---
+id: requestlist
+title: RequestList
+---
+<a name="RequestList"></a>
+
+Represents a static list of URLs to crawl.
+The URLs can be provided either in code or parsed from a text file hosted on the web.
+
+Each URL is represented using an instance of the [`Request`](request) class.
+The list can only contain unique URLs. More precisely, it can only contain `Request` instances
+with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
+To add a single URL to the list multiple times, corresponding [`Request`](request) objects will need to have different
+`uniqueKey` properties. You can use the `keepDuplicateUrls` option to do this for you when initializing the
+`RequestList` from sources.
+
+Once you create an instance of `RequestList`, you need to call the [`initialize`](#RequestList+initialize) function
+before the instance can be used. After that, no more URLs can be added to the list.
+
+`RequestList` is used by [`BasicCrawler`](basiccrawler), [`CheerioCrawler`](cheeriocrawler)
+and [`PuppeteerCrawler`](puppeteercrawler) as a source of URLs to crawl.
+Unlike [`RequestQueue`](requestqueue), `RequestList` is static but it can contain even millions of URLs.
+
+`RequestList` has an internal state where it stores information about which requests were already handled,
+which are in progress and which were reclaimed. The state might be automatically persisted to the default
+key-value store by setting the `persistStateKey` option o that if the Node.js process is restarted,
+the crawling can continue where it left off. For more details, see [`KeyValueStore`](keyvaluestore).
+
+**Example usage:**
+
+```javascript
+const requestList = new Apify.RequestList({
+ sources: [
+ // Separate requests
+ { url: 'http://www.example.com/page-1', method: 'GET', headers: {} },
+ { url: 'http://www.example.com/page-2', userData: { foo: 'bar' }},
+
+ // Bulk load of URLs from file `http://www.example.com/my-url-list.txt`
+ // Note that all URLs must start with http:// or https://
+ { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },
+ ],
+ persistStateKey: 'my-crawling-state'
+});
+
+// This call loads and parses the URLs from the remote file.
+await requestList.initialize();
+
+// Get requests from list
+const request1 = await requestList.fetchNextRequest();
+const request2 = await requestList.fetchNextRequest();
+const request3 = await requestList.fetchNextRequest();
+
+// Mark some of them as handled
+await requestList.markRequestHandled(request1);
+
+// If processing fails then reclaim it back to the list
+await requestList.reclaimRequest(request2);
+```
+
+
+* [RequestList](requestlist)
+ * [`new RequestList(options)`](#new_RequestList_new)
+ * [`.initialize()`](#RequestList+initialize) ⇒ <code>Promise</code>
+ * [`.getState()`](#RequestList+getState) ⇒ <code>Object</code>
+ * [`.isEmpty()`](#RequestList+isEmpty) ⇒ <code>Promise&lt;Boolean&gt;</code>
+ * [`.isFinished()`](#RequestList+isFinished) ⇒ <code>Promise&lt;Boolean&gt;</code>
+ * [`.fetchNextRequest()`](#RequestList+fetchNextRequest) ⇒ [<code>Promise&lt;Request&gt;</code>](request)
+ * [`.markRequestHandled(request)`](#RequestList+markRequestHandled) ⇒ <code>Promise</code>
+ * [`.reclaimRequest(request)`](#RequestList+reclaimRequest) ⇒ <code>Promise</code>
+ * [`.length()`](#RequestList+length) ⇒ <code>Number</code>
+
+<a name="new_RequestList_new"></a>
+
+## `new RequestList(options)`
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>options</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>All <code>RequestList</code> parameters are passed
+ via an options object with the following keys:</p>
+</td></tr><tr>
+<td><code>options.sources</code></td><td><code>Array</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>An array of sources for the <code>RequestList</code>. Its contents can either be just plain objects,
+ defining at least the &#39;url&#39; property or instances of the <a href="request"><code>Request</code></a> class.
+ Additionally a <code>requestsFromUrl</code> property may be used instead of <code>url</code>,
+ which will instruct the <code>RequestList</code> to download the sources from the given remote location.
+ The URLs will be parsed from the received response.</p>
+<pre><code>[
+ // One URL
+ { method: &#39;GET&#39;, url: &#39;http://example.com/a/b&#39; },
+ // Batch import of URLs from a file hosted on the web
+ { method: &#39;POST&#39;, requestsFromUrl: &#39;http://example.com/urls.txt&#39; },
+]
+</code></pre></td></tr><tr>
+<td><code>[options.persistStateKey]</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Identifies the key in the default key-value store under which the <code>RequestList</code> persists its
+ state. State represents a position of the last scraped request in the list. If this is set then <code>RequestList</code>
+ persists this state in regular intervals and loads the state from there in case it is restarted due to an error or system reboot.</p>
+<p> <strong>IMPORTANT:</strong>
+ The state represents only a pointer/index of a request in the list. The full contents of the list are not persisted.
+ Therefore, the user is responsible for either persisting the list on her own or a consistent order of the requests
+ in the list must be preserved. This is especially important when downloading a live list of URLs from a remote resource,
+ since the change in the source URL list would change the resulting <code>RequestList</code> on error/restart and the state
+ of crawling will be impossible to restore, resulting in an error.</p>
+</td></tr><tr>
+<td><code>[options.state]</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>The state object that the <code>RequestList</code> will be initialized from.
+ It is in the form as returned by <code>RequestList.getState()</code>, such as follows:</p>
+<pre><code>{
+ nextIndex: 5,
+ nextUniqueKey: &#39;unique-key-5&#39;
+ inProgress: {
+ &#39;unique-key-1&#39;: true,
+ &#39;unique-key-4&#39;: true,
+ },
+}
+</code></pre><p> Note that the preferred (and simpler) way to persist the state of crawling of the <code>RequestList</code>
+ is to use the <code>persistStateKey</code> parameter instead.</p>
+</td></tr><tr>
+<td><code>[options.keepDuplicateUrls]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>By default, <code>RequestList</code> will deduplicate the provided URLs. Default deduplication is based
+ on the <code>uniqueKey</code> property of passed source <a href="request"><code>Request</code></a> objects.</p>
+<p> If the property is not present, it is generated by normalizing the URL. If present, it is kept intact.
+ In any case, only one request per <code>uniqueKey</code> is added to the <code>RequestList</code> resulting in removal
+ of duplicate URLs / unique keys.</p>
+<p> Setting <code>keepDuplicateUrls</code> to <code>true</code> will append an additional identifier to the <code>uniqueKey</code>
+ of each request that does not already include a <code>uniqueKey</code>. Therefore, duplicate
+ URLs will be kept in the list. It does not protect the user from having duplicates in user set
+ <code>uniqueKey</code>s however. It is the user&#39;s responsibility to ensure uniqueness of their unique keys,
+ if they wish to keep more than just a single copy in the <code>RequestList</code>.</p>
+</td></tr></tbody>
+</table>
+<a name="RequestList+initialize"></a>
+
+## `requestList.initialize()` ⇒ <code>Promise</code>
+Loads all remote sources of URLs and potentially starts periodic state persistence.
+This function must be called before you can start using the instance in a meaningful way.
+
+<a name="RequestList+getState"></a>
+
+## `requestList.getState()` ⇒ <code>Object</code>
+Returns an object representing the internal state of the `RequestList` instance.
+Note that the object's fields can change in future releases.
+
+<a name="RequestList+isEmpty"></a>
+
+## `requestList.isEmpty()` ⇒ <code>Promise&lt;Boolean&gt;</code>
+Resolves to `true` if the next call to [`fetchNextRequest`](#RequestList+fetchNextRequest) function
+would return `null`, otherwise it resolves to `false`.
+Note that even if the list is empty, there might be some pending requests currently being processed.
+
+<a name="RequestList+isFinished"></a>
+
+## `requestList.isFinished()` ⇒ <code>Promise&lt;Boolean&gt;</code>
+Returns `true` if all requests were already handled and there are no more left.
+
+<a name="RequestList+fetchNextRequest"></a>
+
+## `requestList.fetchNextRequest()` ⇒ [<code>Promise&lt;Request&gt;</code>](request)
+Gets the next [`Request`](request) to process. First, the function gets a request previously reclaimed
+using the [`reclaimRequest`](#RequestList+reclaimRequest) function, if there is any.
+Otherwise it gets the next request from sources.
+
+The function's `Promise` resolves to `null` if there are no more
+requests to process.
+
+<a name="RequestList+markRequestHandled"></a>
+
+## `requestList.markRequestHandled(request)` ⇒ <code>Promise</code>
+Marks request as handled after successful processing.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>request</code></td><td><code><a href="request">Request</a></code></td>
+</tr>
+<tr>
+</tr></tbody>
+</table>
+<a name="RequestList+reclaimRequest"></a>
+
+## `requestList.reclaimRequest(request)` ⇒ <code>Promise</code>
+Reclaims request to the list if its processing failed.
+The request will become available in the next `this.fetchNextRequest()`.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>request</code></td><td><code><a href="request">Request</a></code></td>
+</tr>
+<tr>
+</tr></tbody>
+</table>
+<a name="RequestList+length"></a>
+
+## `requestList.length()` ⇒ <code>Number</code>
+Returns the total number of unique requests present in the `RequestList`.
+
diff --git a/docs/api/RequestQueue.md b/docs/api/RequestQueue.md
new file mode 100644
index 000000000..82640eadd
--- /dev/null
+++ b/docs/api/RequestQueue.md
@@ -0,0 +1,202 @@
+---
+id: requestqueue
+title: RequestQueue
+---
+<a name="RequestQueue"></a>
+
+Represents a queue of URLs to crawl, which is used for deep crawling of websites
+where you start with several URLs and then recursively
+follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
+
+Each URL is represented using an instance of the [`Request`](request) class.
+The queue can only contain unique URLs. More precisely, it can only contain [`Request`](request) instances
+with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
+To add a single URL multiple times to the queue,
+corresponding [`Request`](request) objects will need to have different `uniqueKey` properties.
+
+Do not instantiate this class directly, use the
+[`Apify.openRequestQueue()`](apify#module_Apify.openRequestQueue) function instead.
+
+`RequestQueue` is used by [`BasicCrawler`](basiccrawler), [`CheerioCrawler`](cheeriocrawler)
+and [`PuppeteerCrawler`](puppeteercrawler) as a source of URLs to crawl.
+Unlike [`RequestList`](requestlist), `RequestQueue` supports dynamic adding and removing of requests.
+On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
+
+`RequestQueue` stores its data either on local disk or in the Apify Cloud,
+depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variable is set.
+
+If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the queue data is stored in
+that local directory as follows:
+```
+{APIFY_LOCAL_STORAGE_DIR}/request_queues/{QUEUE_ID}/{STATE}/{NUMBER}.json
+```
+Note that `{QUEUE_ID}` is the name or ID of the request queue. The default queue has ID: `default`,
+unless you override it by setting the `APIFY_DEFAULT_REQUEST_QUEUE_ID` environment variable.
+Each request in the queue is stored as a separate JSON file, where `{STATE}` is either `handled` or `pending`,
+and `{NUMBER}` is an integer indicating the position of the request in the queue.
+
+If the `APIFY_TOKEN` environment variable is provided instead, the data is stored
+in the <a href="https://www.apify.com/docs/storage#queue" target="_blank">Apify Request Queue</a> cloud storage.
+
+**Example usage:**
+
+```javascript
+// Open the default request queue associated with the actor run
+const queue = await Apify.openRequestQueue();
+
+// Open a named request queue
+const queueWithName = await Apify.openRequestQueue('some-name');
+
+// Enqueue few requests
+await queue.addRequest(new Apify.Request({ url: 'http://example.com/aaa' }));
+await queue.addRequest(new Apify.Request({ url: 'http://example.com/bbb' }));
+await queue.addRequest(new Apify.Request({ url: 'http://example.com/foo/bar' }), { forefront: true });
+
+// Get requests from queue
+const request1 = await queue.fetchNextRequest();
+const request2 = await queue.fetchNextRequest();
+const request3 = await queue.fetchNextRequest();
+
+// Mark a request as handled
+await queue.markRequestHandled(request1);
+
+// If processing of a request fails then reclaim it back to the queue, so that it's crawled again
+await queue.reclaimRequest(request2);
+```
+
+
+* [RequestQueue](requestqueue)
+ * [`.addRequest(request, [options])`](#RequestQueue+addRequest) ⇒ [<code>RequestOperationInfo</code>](../typedefs/requestoperationinfo)
+ * [`.getRequest(requestId)`](#RequestQueue+getRequest) ⇒ [<code>Promise&lt;Request&gt;</code>](request)
+ * [`.fetchNextRequest()`](#RequestQueue+fetchNextRequest) ⇒ [<code>Promise&lt;Request&gt;</code>](request)
+ * [`.markRequestHandled(request)`](#RequestQueue+markRequestHandled) ⇒ [<code>Promise&lt;RequestOperationInfo&gt;</code>](../typedefs/requestoperationinfo)
+ * [`.reclaimRequest(request, [options])`](#RequestQueue+reclaimRequest) ⇒ [<code>Promise&lt;RequestOperationInfo&gt;</code>](../typedefs/requestoperationinfo)
+ * [`.isEmpty()`](#RequestQueue+isEmpty) ⇒ <code>Promise&lt;Boolean&gt;</code>
+ * [`.isFinished()`](#RequestQueue+isFinished) ⇒ <code>Promise&lt;Boolean&gt;</code>
+ * [`.delete()`](#RequestQueue+delete) ⇒ <code>Promise</code>
+
+<a name="RequestQueue+addRequest"></a>
+
+## `requestQueue.addRequest(request, [options])` ⇒ [<code>RequestOperationInfo</code>](../typedefs/requestoperationinfo)
+Adds a request to the queue.
+
+If a request with the same `uniqueKey` property is already present in the queue,
+it will not be updated. You can find out whether this happened from the resulting
+[`RequestOperationInfo`](../typedefs/requestoperationinfo) object.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>request</code></td><td><code><a href="request">Request</a></code> | <code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Request object, or an Object to construct a Request from.</p>
+</td></tr><tr>
+<td><code>[options]</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"></td></tr><tr>
+<td><code>[options.forefront]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If <code>true</code>, the request will be added to the foremost position in the queue.</p>
+</td></tr></tbody>
+</table>
+<a name="RequestQueue+getRequest"></a>
+
+## `requestQueue.getRequest(requestId)` ⇒ [<code>Promise&lt;Request&gt;</code>](request)
+Gets the request from the queue specified by ID.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>requestId</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Request ID</p>
+</td></tr></tbody>
+</table>
+<a name="RequestQueue+fetchNextRequest"></a>
+
+## `requestQueue.fetchNextRequest()` ⇒ [<code>Promise&lt;Request&gt;</code>](request)
+Returns next request in the queue to be processed.
+
+<a name="RequestQueue+markRequestHandled"></a>
+
+## `requestQueue.markRequestHandled(request)` ⇒ [<code>Promise&lt;RequestOperationInfo&gt;</code>](../typedefs/requestoperationinfo)
+Marks request handled after successful processing.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>request</code></td><td><code><a href="request">Request</a></code></td>
+</tr>
+<tr>
+</tr></tbody>
+</table>
+<a name="RequestQueue+reclaimRequest"></a>
+
+## `requestQueue.reclaimRequest(request, [options])` ⇒ [<code>Promise&lt;RequestOperationInfo&gt;</code>](../typedefs/requestoperationinfo)
+Reclaims failed request back to the queue,
+so that it can be processed later again.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>request</code></td><td><code><a href="request">Request</a></code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"></td></tr><tr>
+<td><code>[options]</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"></td></tr><tr>
+<td><code>[options.forefront]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If <code>true</code> then requests get returned to the start of the queue
+ and to the back of the queue otherwise.</p>
+</td></tr></tbody>
+</table>
+<a name="RequestQueue+isEmpty"></a>
+
+## `requestQueue.isEmpty()` ⇒ <code>Promise&lt;Boolean&gt;</code>
+Resolves to `true` if the next call to [`fetchNextRequest`](#RequestQueue+fetchNextRequest) would return `null`, otherwise it resolves to `false`.
+Note that even if the queue is empty, there might be some pending requests currently being processed.
+
+Due to the nature of distributed storage systems,
+the function might occasionally return a false negative, but it should never return a false positive!
+
+<a name="RequestQueue+isFinished"></a>
+
+## `requestQueue.isFinished()` ⇒ <code>Promise&lt;Boolean&gt;</code>
+Resolves to `true` if all requests were already handled and there are no more left.
+Due to the nature of distributed storage systems,
+the function might occasionally return a false negative, but it will never return a false positive.
+
+<a name="RequestQueue+delete"></a>
+
+## `requestQueue.delete()` ⇒ <code>Promise</code>
+Removes the queue either from the Apify Cloud storage or from the local directory,
+depending on the mode of operation.
+
diff --git a/docs/api/Snapshotter.md b/docs/api/Snapshotter.md
new file mode 100644
index 000000000..11bc84905
--- /dev/null
+++ b/docs/api/Snapshotter.md
@@ -0,0 +1,148 @@
+---
+id: snapshotter
+title: Snapshotter
+---
+<a name="Snapshotter"></a>
+
+Creates snapshots of system resources at given intervals and marks the resource
+as either overloaded or not during the last interval. Keeps a history of the snapshots.
+It tracks the following resources: Memory, EventLoop and CPU.
+The class is used by the [`AutoscaledPool`](autoscaledpool) class.
+
+There are differences in behavior when running locally and on the Apify platform,
+but those differences are handled internally by the class and do not affect its interface.
+
+Memory becomes overloaded if its current use exceeds the `maxUsedMemoryRatio` option.
+It's computed using the total memory available to the container when running on
+the Apify platform and a quarter of total system memory when running locally.
+Max total memory may be overridden by using the `APIFY_MEMORY_MBYTES` environment variable.
+
+Event loop becomes overloaded if it slows down by more than the `maxBlockedMillis` option.
+
+CPU tracking is available only on the Apify platform and the CPU overloaded event is read
+directly off the container and is not configurable.
+
+
+* [Snapshotter](snapshotter)
+ * [`new Snapshotter([options])`](#new_Snapshotter_new)
+ * [`.start()`](#Snapshotter+start) ⇒ <code>Promise</code>
+ * [`.stop()`](#Snapshotter+stop) ⇒ <code>Promise</code>
+ * [`.getMemorySample([sampleDurationMillis])`](#Snapshotter+getMemorySample) ⇒ <code>Array</code>
+ * [`.getEventLoopSample([sampleDurationMillis])`](#Snapshotter+getEventLoopSample) ⇒ <code>Array</code>
+ * [`.getCpuSample([sampleDurationMillis])`](#Snapshotter+getCpuSample) ⇒ <code>Array</code>
+
+<a name="new_Snapshotter_new"></a>
+
+## `new Snapshotter([options])`
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>[options]</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>All <code>Snapshotter</code> parameters are passed
+ via an options object with the following keys:</p>
+</td></tr><tr>
+<td><code>[options.eventLoopSnapshotIntervalSecs]</code></td><td><code>Number</code></td><td><code>0.5</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Defines the interval of measuring the event loop response time.</p>
+</td></tr><tr>
+<td><code>[options.maxBlockedMillis]</code></td><td><code>Number</code></td><td><code>50</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Maximum allowed delay of the event loop in milliseconds.
+ Exceeding this limit overloads the event loop.</p>
+</td></tr><tr>
+<td><code>[options.memorySnapshotIntervalSecs]</code></td><td><code>Number</code></td><td><code>1</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Defines the interval of measuring memory consumption.
+ The measurement itself is resource intensive (25 - 50ms async).
+ Therefore, setting this interval below 1 second is not recommended.</p>
+</td></tr><tr>
+<td><code>[options.maxUsedMemoryRatio]</code></td><td><code>Number</code></td><td><code>0.7</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Defines the maximum ratio of total memory that can be used.
+ Exceeding this limit overloads the memory.</p>
+</td></tr><tr>
+<td><code>[options.snapshotHistorySecs]</code></td><td><code>Number</code></td><td><code>60</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Sets the interval in seconds for which a history of resource snapshots
+ will be kept. Increasing this to very high numbers will affect performance.</p>
+</td></tr></tbody>
+</table>
+<a name="Snapshotter+start"></a>
+
+## `snapshotter.start()` ⇒ <code>Promise</code>
+Starts capturing snapshots at configured intervals.
+
+<a name="Snapshotter+stop"></a>
+
+## `snapshotter.stop()` ⇒ <code>Promise</code>
+Stops all resource capturing.
+
+<a name="Snapshotter+getMemorySample"></a>
+
+## `snapshotter.getMemorySample([sampleDurationMillis])` ⇒ <code>Array</code>
+Returns a sample of latest memory snapshots, with the size of the sample defined
+by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>[sampleDurationMillis]</code></td><td><code>Number</code></td>
+</tr>
+<tr>
+</tr></tbody>
+</table>
+<a name="Snapshotter+getEventLoopSample"></a>
+
+## `snapshotter.getEventLoopSample([sampleDurationMillis])` ⇒ <code>Array</code>
+Returns a sample of latest event loop snapshots, with the size of the sample defined
+by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>[sampleDurationMillis]</code></td><td><code>Number</code></td>
+</tr>
+<tr>
+</tr></tbody>
+</table>
+<a name="Snapshotter+getCpuSample"></a>
+
+## `snapshotter.getCpuSample([sampleDurationMillis])` ⇒ <code>Array</code>
+Returns a sample of latest CPU snapshots, with the size of the sample defined
+by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>[sampleDurationMillis]</code></td><td><code>Number</code></td>
+</tr>
+<tr>
+</tr></tbody>
+</table>
diff --git a/docs/api/SystemStatus.md b/docs/api/SystemStatus.md
new file mode 100644
index 000000000..76d0a9cda
--- /dev/null
+++ b/docs/api/SystemStatus.md
@@ -0,0 +1,110 @@
+---
+id: systemstatus
+title: SystemStatus
+---
+<a name="SystemStatus"></a>
+
+Provides a simple interface to reading system status from a [`Snapshotter`](snapshotter) instance.
+It only exposes two functions [`getCurrentStatus()`](#SystemStatus+getCurrentStatus)
+and [`getHistoricalStatus()`](#SystemStatus+getHistoricalStatus).
+The system status is calculated using a weighted average of overloaded
+messages in the snapshots, with the weights being the time intervals
+between the snapshots. Each resource is calculated separately
+and the system is overloaded whenever at least one resource is overloaded.
+The class is used by the [`AutoscaledPool`](autoscaledpool) class.
+
+[`getCurrentStatus()`](#SystemStatus+getCurrentStatus)
+returns a boolean that represents the current status of the system.
+The length of the current timeframe in seconds is configurable
+by the `currentHistorySecs` option and represents the max age
+of snapshots to be considered for the calculation.
+
+[`getHistoricalStatus()`](#SystemStatus+getHistoricalStatus)
+returns a boolean that represents the long-term status
+of the system. It considers the full snapshot history available
+in the [`Snapshotter`](snapshotter) instance.
+
+
+* [SystemStatus](systemstatus)
+ * [`new SystemStatus([options])`](#new_SystemStatus_new)
+ * [`.getCurrentStatus()`](#SystemStatus+getCurrentStatus) ⇒ <code>Object</code>
+ * [`.getHistoricalStatus()`](#SystemStatus+getHistoricalStatus) ⇒ <code>Object</code>
+
+<a name="new_SystemStatus_new"></a>
+
+## `new SystemStatus([options])`
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>[options]</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>All <code>SystemStatus</code> parameters are passed
+ via an options object with the following keys:</p>
+</td></tr><tr>
+<td><code>[options.currentHistorySecs]</code></td><td><code>Number</code></td><td><code>5</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Defines max age of snapshots used in the
+ <a href="#SystemStatus+getCurrentStatus"><code>getCurrentStatus()</code></a> measurement.</p>
+</td></tr><tr>
+<td><code>[options.maxMemoryOverloadedRatio]</code></td><td><code>Number</code></td><td><code>0.2</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Sets the maximum ratio of overloaded snapshots in a memory sample.
+ If the sample exceeds this ratio, the system will be overloaded.</p>
+</td></tr><tr>
+<td><code>[options.maxEventLoopOverloadedRatio]</code></td><td><code>Number</code></td><td><code>0.02</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Sets the maximum ratio of overloaded snapshots in an event loop sample.
+ If the sample exceeds this ratio, the system will be overloaded.</p>
+</td></tr><tr>
+<td><code>[options.maxCpuOverloadedRatio]</code></td><td><code>Number</code></td><td><code>0.1</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Sets the maximum ratio of overloaded snapshots in a CPU sample.
+ If the sample exceeds this ratio, the system will be overloaded.</p>
+</td></tr></tbody>
+</table>
+<a name="SystemStatus+getCurrentStatus"></a>
+
+## `systemStatus.getCurrentStatus()` ⇒ <code>Object</code>
+Returns an object with the following structure:
+
+```javascript
+{
+ isSystemIdle: Boolean,
+ memInfo: Object,
+ eventLoopInfo: Object,
+ cpuInfo: Object
+}
+```
+
+Where the `isSystemIdle` property is set to `false` if the system
+has been overloaded in the last `options.currentHistorySecs` seconds,
+and `true` otherwise.
+
+<a name="SystemStatus+getHistoricalStatus"></a>
+
+## `systemStatus.getHistoricalStatus()` ⇒ <code>Object</code>
+Returns an object with the following structure:
+
+```javascript
+{
+ isSystemIdle: Boolean,
+ memInfo: Object,
+ eventLoopInfo: Object,
+ cpuInfo: Object
+}
+```
+
+Where the `isSystemIdle` property is set to `false` if the system
+has been overloaded in the full history of the [`Snapshotter`](snapshotter)
+(which is configurable in the [`Snapshotter`](snapshotter)) and `true` otherwise.
+
diff --git a/docs/api/puppeteer.md b/docs/api/puppeteer.md
new file mode 100644
index 000000000..19030386c
--- /dev/null
+++ b/docs/api/puppeteer.md
@@ -0,0 +1,279 @@
+---
+id: puppeteer
+title: utils.puppeteer
+---
+<a name="puppeteer"></a>
+
+A namespace that contains various Puppeteer utilities.
+
+**Example usage:**
+
+```javascript
+const Apify = require('apify');
+const { puppeteer } = Apify.utils;
+
+// Open https://www.example.com in Puppeteer
+const browser = await Apify.launchPuppeteer();
+const page = await browser.newPage();
+await page.goto('https://www.example.com');
+
+// Inject jQuery into a page
+await puppeteer.injectJQuery(page);
+```
+
+
+* [`puppeteer`](#puppeteer) : <code>object</code>
+ * [`.hideWebDriver(page)`](#puppeteer.hideWebDriver) ⇒ <code>Promise</code>
+ * [`.injectFile(page, filePath)`](#puppeteer.injectFile) ⇒ <code>Promise</code>
+ * [`.injectJQuery(page)`](#puppeteer.injectJQuery) ⇒ <code>Promise</code>
+ * [`.injectUnderscore(page)`](#puppeteer.injectUnderscore) ⇒ <code>Promise</code>
+ * [`.enqueueLinks(page, selector, pseudoUrls, requestQueue)`](#puppeteer.enqueueLinks) ⇒ <code>Promise&lt;Array&lt;RequestOperationInfo&gt;&gt;</code>
+ * [`.blockResources(page, resourceTypes)`](#puppeteer.blockResources) ⇒ <code>Promise</code>
+ * [`.cacheResponses(page, cache, responseUrlRules)`](#puppeteer.cacheResponses) ⇒ <code>Promise</code>
+ * [`.compileScript(scriptString, context)`](#puppeteer.compileScript) ⇒ <code>function</code>
+
+<a name="puppeteer.hideWebDriver"></a>
+
+## `puppeteer.hideWebDriver(page)` ⇒ <code>Promise</code>
+Hides certain Puppeteer fingerprints from the page, in order to help avoid detection of the crawler.
+The function should be called on a newly-created page object before navigating to the target crawled page.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>page</code></td><td><code>Page</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.</p>
+</td></tr></tbody>
+</table>
+<a name="puppeteer.injectFile"></a>
+
+## `puppeteer.injectFile(page, filePath)` ⇒ <code>Promise</code>
+Injects a JavaScript file into a Puppeteer page.
+Unlike Puppeteer's `addScriptTag` function, this function works on pages
+with arbitrary Cross-Origin Resource Sharing (CORS) policies.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>page</code></td><td><code>Page</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.</p>
+</td></tr><tr>
+<td><code>filePath</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>File path</p>
+</td></tr></tbody>
+</table>
+<a name="puppeteer.injectJQuery"></a>
+
+## `puppeteer.injectJQuery(page)` ⇒ <code>Promise</code>
+Injects the <a href="https://jquery.com/" target="_blank"><code>jQuery</code></a> library into a Puppeteer page.
+jQuery is often useful for various web scraping and crawling tasks,
+e.g. to extract data from HTML elements using CSS selectors.
+
+Beware that the injected jQuery object will be set to the `window.$` variable and thus it might cause conflicts with
+libraries included by the page that use the same variable (e.g. another version of jQuery).
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>page</code></td><td><code>Page</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.</p>
+</td></tr></tbody>
+</table>
+<a name="puppeteer.injectUnderscore"></a>
+
+## `puppeteer.injectUnderscore(page)` ⇒ <code>Promise</code>
+Injects the <a href="https://underscorejs.org/" target="_blank"><code>Underscore.js</code></a> library into a Puppeteer page.
+Beware that the injected Underscore object will be set to the `window._` variable and thus it might cause conflicts with
+libraries included by the page that use the same variable.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>page</code></td><td><code>Page</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Puppeteer <a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page">Page</a> object.</p>
+</td></tr></tbody>
+</table>
+<a name="puppeteer.enqueueLinks"></a>
+
+## `puppeteer.enqueueLinks(page, selector, pseudoUrls, requestQueue)` ⇒ <code>Promise&lt;Array&lt;RequestOperationInfo&gt;&gt;</code>
+Finds HTML elements matching a CSS selector, clicks the elements and if a redirect is triggered and destination URL matches
+one of the provided [`PseudoUrl`](pseudourl)s, then the function enqueues that URL to a given request queue.
+To create a Request object function uses `requestTemplate` from a matching [`PseudoUrl`](pseudourl).
+
+*WARNING*: This is work in progress. Currently the function doesn't click elements and only takes their `href` attribute and so
+ is working only for link (`a`) elements, but not for buttons or JavaScript links.
+
+**Returns**: <code>Promise&lt;Array&lt;RequestOperationInfo&gt;&gt;</code> - Promise that resolves to an array of [`RequestOperationInfo`](../typedefs/requestoperationinfo) objects.
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>page</code></td><td><code>Page</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.</p>
+</td></tr><tr>
+<td><code>selector</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>CSS selector matching elements to be clicked.</p>
+</td></tr><tr>
+<td><code>pseudoUrls</code></td><td><code><a href="pseudourl">Array&lt;PseudoUrl&gt;</a></code></td>
+</tr>
+<tr>
+<td colspan="3"><p>An array of <a href="pseudourl"><code>PseudoUrl</code></a>s matching the URLs to be enqueued.</p>
+</td></tr><tr>
+<td><code>requestQueue</code></td><td><code><a href="requestqueue">RequestQueue</a></code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Request queue object where URLs will be enqueued.</p>
+</td></tr></tbody>
+</table>
+<a name="puppeteer.blockResources"></a>
+
+## `puppeteer.blockResources(page, resourceTypes)` ⇒ <code>Promise</code>
+Forces the browser tab to block loading certain page resources,
+using the `Page.setRequestInterception(value)` method.
+This is useful to speed up crawling of websites.
+
+The resource types to block can be controlled using the `resourceTypes` parameter,
+which indicates the types of resources as they are perceived by the rendering engine.
+The following resource types are currently supported:
+`document`, `stylesheet`, `image`, `media`, `font`, `script`, `texttrack`, `xhr`, `fetch`,
+`eventsource`, `websocket`, `manifest`, `other`.
+For more details, see Puppeteer's
+<a href="https://pptr.dev/#?product=Puppeteer&show=api-requestresourcetype" target="_blank">Request.resourceType() documentation</a>.
+
+By default, the function blocks these resource types: `stylesheet`, `font`, `image`, `media`.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>page</code></td><td><code>Page</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.</p>
+</td></tr><tr>
+<td><code>resourceTypes</code></td><td><code>Array&lt;String&gt;</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Array of resource types to block.</p>
+</td></tr></tbody>
+</table>
+<a name="puppeteer.cacheResponses"></a>
+
+## `puppeteer.cacheResponses(page, cache, responseUrlRules)` ⇒ <code>Promise</code>
+Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.
+*IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
+ This issue should be resolved or atleast mitigated in future iterations of this feature.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>page</code></td><td><code>Page</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.</p>
+</td></tr><tr>
+<td><code>cache</code></td><td><code>Object</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Object in which responses are stored</p>
+</td></tr><tr>
+<td><code>responseUrlRules</code></td><td><code>Array&lt;(String|RegExp)&gt;</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>List of rules that are used to check if the response should be cached.
+ String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).</p>
+</td></tr></tbody>
+</table>
+<a name="puppeteer.compileScript"></a>
+
+## `puppeteer.compileScript(scriptString, context)` ⇒ <code>function</code>
+Compiles a Puppeteer script into an async function that may be executed at any time
+by providing it with the following object:
+```
+{
+ page: Page,
+ request: Request,
+}
+```
+Where `page` is a Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a>
+and `request` is a [`Request`](request).
+
+The function is compiled by using the `scriptString` parameter as the function's body,
+so any limitations to function bodies apply. Return value of the compiled function
+is the return value of the function body = the `scriptString` parameter.
+
+As a security measure, no globals such as `process` or `require` are accessible
+from within the function body. Note that the function does not provide a safe
+sandbox and even though globals are not easily accessible, malicious code may
+still execute in the main process via prototype manipulation. Therefore you
+should only use this function to execute sanitized or safe code.
+
+Custom context may also be provided using the `context` parameter. To improve security,
+make sure to only pass the really necessary objects to the context. Preferably making
+secured copies beforehand.
+
+**Returns**: <code>function</code> - `async ({ page, request }) => { scriptString }`
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>scriptString</code></td><td><code>String</code></td>
+</tr>
+<tr>
+</tr><tr>
+<td><code>context</code></td><td><code>Object</code></td>
+</tr>
+<tr>
+</tr></tbody>
+</table>
diff --git a/docs/api/utils.md b/docs/api/utils.md
new file mode 100644
index 000000000..bd1cec175
--- /dev/null
+++ b/docs/api/utils.md
@@ -0,0 +1,125 @@
+---
+id: utils
+title: utils
+---
+<a name="utils"></a>
+
+A namespace that contains various utilities.
+
+**Example usage:**
+
+```javascript
+const Apify = require('apify');
+
+...
+
+// Sleep 1.5 seconds
+await Apify.utils.sleep(1500);
+```
+
+
+* [`utils`](#utils) : <code>object</code>
+ * [`.URL_NO_COMMAS_REGEX`](#utils.URL_NO_COMMAS_REGEX)
+ * [`.URL_WITH_COMMAS_REGEX`](#utils.URL_WITH_COMMAS_REGEX)
+ * [`.sleep(millis)`](#utils.sleep) ⇒ <code>Promise</code>
+ * [`.downloadListOfUrls(url, [encoding], [urlRegExp])`](#utils.downloadListOfUrls) ⇒ <code>Promise&lt;Array&lt;String&gt;&gt;</code>
+ * [`.extractUrls(string, [urlRegExp])`](#utils.extractUrls) ⇒ <code>Array&lt;String&gt;</code>
+ * [`.getRandomUserAgent()`](#utils.getRandomUserAgent) ⇒ <code>String</code>
+
+<a name="utils.URL_NO_COMMAS_REGEX"></a>
+
+## `utils.URL_NO_COMMAS_REGEX`
+Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters
+and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
+
+<a name="utils.URL_WITH_COMMAS_REGEX"></a>
+
+## `utils.URL_WITH_COMMAS_REGEX`
+Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.
+Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
+
+<a name="utils.sleep"></a>
+
+## `utils.sleep(millis)` ⇒ <code>Promise</code>
+Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting
+in your code, e.g. to prevent overloading of target website or to avoid bot detection.
+
+**Example usage:**
+
+```
+const Apify = require('apify');
+
+...
+
+// Sleep 1.5 seconds
+await Apify.utils.sleep(1500);
+```
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>millis</code></td><td><code>Number</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Period of time to sleep, in milliseconds. If not a positive number, the returned promise resolves immediately.</p>
+</td></tr></tbody>
+</table>
+<a name="utils.downloadListOfUrls"></a>
+
+## `utils.downloadListOfUrls(url, [encoding], [urlRegExp])` ⇒ <code>Promise&lt;Array&lt;String&gt;&gt;</code>
+Returns a promise that resolves to an array of urls parsed from the resource available at the provided url.
+Optionally, custom regular expression and encoding may be provided.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>url</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+</tr><tr>
+<td><code>[encoding]</code></td><td><code>String</code></td><td><code>&#x27;utf8&#x27;</code></td>
+</tr>
+<tr>
+</tr><tr>
+<td><code>[urlRegExp]</code></td><td><code>RegExp</code></td><td><code>URL_NO_COMMAS_REGEX</code></td>
+</tr>
+<tr>
+</tr></tbody>
+</table>
+<a name="utils.extractUrls"></a>
+
+## `utils.extractUrls(string, [urlRegExp])` ⇒ <code>Array&lt;String&gt;</code>
+Collects all URLs in an arbitrary string to an array, optionally using a custom regular expression.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>string</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+</tr><tr>
+<td><code>[urlRegExp]</code></td><td><code>RegExp</code></td><td><code>URL_NO_COMMAS_REGEX</code></td>
+</tr>
+<tr>
+</tr></tbody>
+</table>
+<a name="utils.getRandomUserAgent"></a>
+
+## `utils.getRandomUserAgent()` ⇒ <code>String</code>
+Returns a randomly selected User-Agent header out of a list of the most common headers.
+
diff --git a/docs/examples/BasicCrawler.md b/docs/examples/BasicCrawler.md
new file mode 100644
index 000000000..c3b521b84
--- /dev/null
+++ b/docs/examples/BasicCrawler.md
@@ -0,0 +1,61 @@
+---
+id: basiccrawler
+title: Basic Crawler
+---
+
+This is the most basic example of the Apify SDK, which demonstrates some of its
+elementary tools, such as the
+[BasicCrawler](https://www.apify.com/docs/sdk/apify-runtime-js/latest#BasicCrawler)
+and [RequestList](https://www.apify.com/docs/sdk/apify-runtime-js/latest#RequestList) classes.
+The script just downloads several web pages with plain HTTP requests (using the
+[request-promise](https://www.npmjs.com/package/request-promise) library)
+and stores their raw HTML and URL to the default dataset.
+In local configuration, the data will be stored as JSON files in `./apify_storage/datasets/default`.
+```javascript
+const Apify = require('apify');
+const requestPromise = require('request-promise');
+
+// Apify.main() function wraps the crawler logic (it is optional).
+Apify.main(async () => {
+ // Create and initialize an instance of the RequestList class that contains
+ // a list of URLs to crawl. Here we use just a few hard-coded URLs.
+ const requestList = new Apify.RequestList({
+ sources: [
+ { url: 'http://www.google.com/' },
+ { url: 'http://www.example.com/' },
+ { url: 'http://www.bing.com/' },
+ { url: 'http://www.wikipedia.com/' },
+ ],
+ });
+ await requestList.initialize();
+
+ // Create a BasicCrawler - the simplest crawler that enables
+ // users to implement the crawling logic themselves.
+ const crawler = new Apify.BasicCrawler({
+
+ // Let the crawler fetch URLs from our list.
+ requestList,
+
+ // This function will be called for each URL to crawl.
+ // The 'request' option is an instance of the Request class, which contains
+ // information such as URL and HTTP method, as supplied by the RequestList.
+ handleRequestFunction: async ({ request }) => {
+ console.log(`Processing ${request.url}...`);
+
+ // Fetch the page HTML
+ const html = await requestPromise(request.url);
+
+ // Store the HTML and URL to the default dataset.
+ await Apify.pushData({
+ url: request.url,
+ html,
+ });
+ },
+ });
+
+ // Run the crawler and wait for it to finish.
+ await crawler.run();
+
+ console.log('Crawler finished.');
+});
+```
\ No newline at end of file
diff --git a/docs/examples/CallActor.md b/docs/examples/CallActor.md
new file mode 100644
index 000000000..6a67b5d5f
--- /dev/null
+++ b/docs/examples/CallActor.md
@@ -0,0 +1,48 @@
+---
+id: callactor
+title: Call Actor
+---
+
+This example demonstrates how to start an Apify actor using
+<a href="https://www.apify.com/docs/sdk/apify-runtime-js/latest#module-Apify-call"><code>Apify.call()</code></a>
+and how to call Apify API using
+<a href="https://www.apify.com/docs/sdk/apify-runtime-js/latest#module-Apify-client"><code>Apify.client</code></a>.
+The script extracts the current Bitcoin prices from Kraken.com
+and sends them to your email using the [apify/send-mail](https://www.apify.com/apify/send-mail) actor.
+
+To make the example work, you'll need an [Apify Account](https://my.apify.com/).
+Go to [Account - Integrations](https://my.apify.com/account#/integrations) page to obtain your API token
+and set it to the `APIFY_TOKEN` environment variable, or run the script using the CLI.
+If you deploy this actor to the Apify platform then you can set up a scheduler for early
+morning. Don't miss the chance of your life to get rich!
+```javascript
+const Apify = require('apify');
+
+Apify.main(async () => {
+ // Launch the web browser.
+ const browser = await Apify.launchPuppeteer();
+
+ console.log('Obtaining email address...');
+ const user = await Apify.client.users.getUser();
+
+ // Load Kraken.com charts and get last traded price of BTC
+ console.log('Extracting data from kraken.com...');
+ const page = await browser.newPage();
+ await page.goto('https://www.kraken.com/charts');
+ const tradedPricesHtml = await page.$eval('#ticker-top ul', el => el.outerHTML);
+
+ // Send prices to your email. For that, you can use an actor we already
+ // have available on the platform under the name: apify/send-mail.
+ // The second parameter to the Apify.call() invocation is the actor's
+ // desired input. You can find the required input parameters by checking
+ // the actor's documentation page: https://www.apify.com/apify/send-mail
+ console.log(`Sending email to ${user.email}...`);
+ await Apify.call('apify/send-mail', {
+ to: user.email,
+ subject: 'Kraken.com BTC',
+ html: `<h1>Kraken.com BTC</h1>${tradedPricesHtml}`,
+ });
+
+ console.log('Email sent. Good luck!');
+});
+```
\ No newline at end of file
diff --git a/docs/examples/CheerioCrawler.md b/docs/examples/CheerioCrawler.md
new file mode 100644
index 000000000..637935fb0
--- /dev/null
+++ b/docs/examples/CheerioCrawler.md
@@ -0,0 +1,86 @@
+---
+id: cheeriocrawler
+title: Cheerio Crawler
+---
+
+This example demonstrates how to use [CheerioCrawler](https://www.apify.com/docs/sdk/apify-runtime-js/latest#CheerioCrawler)
+to crawl a list of URLs from an external file,
+load each URL using a plain HTTP request, parse the HTML using [cheerio](https://www.npmjs.com/package/cheerio)
+and extract some data from it: the page title and all H1 tags.
+```javascript
+const Apify = require('apify');
+
+// Apify.utils contains various utilities, e.g. for logging.
+// Here we turn off the logging of unimportant messages.
+const { log } = Apify.utils;
+log.setLevel(log.LEVELS.WARNING);
+
+// A link to a list of Fortune 500 companies' websites available on GitHub.
+const CSV_LINK = 'https://gist.githubusercontent.com/hrbrmstr/ae574201af3de035c684/raw/f1000.csv';
+
+// Apify.main() function wraps the crawler logic (it is optional).
+Apify.main(async () => {
+ // Create an instance of the RequestList class that contains a list of URLs to crawl.
+ // Here we download and parse the list of URLs from an external file.
+ const requestList = new Apify.RequestList({
+ sources: [{ requestsFromUrl: CSV_LINK }],
+ });
+ await requestList.initialize();
+
+ // Create an instance of the CheerioCrawler class - a crawler
+ // that automatically loads the URLs and parses their HTML using the cheerio library.
+ const crawler = new Apify.CheerioCrawler({
+ // Let the crawler fetch URLs from our list.
+ requestList,
+
+ // The crawler downloads and processes the web pages in parallel, with a concurrency
+ // automatically managed based on the available system memory and CPU (see AutoscaledPool class).
+ // Here we define some hard limits for the concurrency.
+ minConcurrency: 10,
+ maxConcurrency: 50,
+
+ // On error, retry each page at most once.
+ maxRequestRetries: 1,
+
+ // Increase the timeout for processing of each page.
+ handlePageTimeoutSecs: 60,
+
+ // This function will be called for each URL to crawl.
+ // It accepts a single parameter, which is an object with the following fields:
+ // - request: an instance of the Request class with information such as URL and HTTP method
+ // - html: contains raw HTML of the page
+ // - $: the cheerio object containing parsed HTML
+ handlePageFunction: async ({ request, html, $ }) => {
+ console.log(`Processing ${request.url}...`);
+
+ // Extract data from the page using cheerio.
+ const title = $('title').text();
+ const h1texts = [];
+ $('h1').each((index, el) => {
+ h1texts.push({
+ text: $(el).text(),
+ });
+ });
+
+ // Store the results to the default dataset. In local configuration,
+ // the data will be stored as JSON files in ./apify_storage/datasets/default
+ await Apify.pushData({
+ url: request.url,
+ title,
+ h1texts,
+ html,
+ });
+ },
+
+ // This function is called if the page processing failed more than maxRequestRetries+1 times.
+ handleFailedRequestFunction: async ({ request }) => {
+ console.log(`Request ${request.url} failed twice.`);
+ },
+ });
+
+ // Run the crawler and wait for it to finish.
+ await crawler.run();
+
+ console.log('Crawler finished.');
+});
+```
\ No newline at end of file
diff --git a/docs/examples/HelloWorld.md b/docs/examples/HelloWorld.md
new file mode 100644
index 000000000..c65a3d02f
--- /dev/null
+++ b/docs/examples/HelloWorld.md
@@ -0,0 +1,28 @@
+---
+id: helloworld
+title: Hello World
+---
+
+Run the following example to perform a recursive crawl of a website using Puppeteer.
+```javascript
+const Apify = require('apify');
+
+Apify.main(async () => {
+ const requestQueue = await Apify.openRequestQueue();
+ await requestQueue.addRequest(new Apify.Request({ url: 'https://www.iana.org/' }));
+ const pseudoUrls = [new Apify.PseudoUrl('https://www.iana.org/[.*]')];
+
+ const crawler = new Apify.PuppeteerCrawler({
+ requestQueue,
+ handlePageFunction: async ({ request, page }) => {
+ const title = await page.title();
+ console.log(`Title of ${request.url}: ${title}`);
+ await Apify.utils.puppeteer.enqueueLinks(page, 'a', pseudoUrls, requestQueue);
+ },
+ maxRequestsPerCrawl: 100,
+ maxConcurrency: 10,
+ });
+
+ await crawler.run();
+});
+```
\ No newline at end of file
diff --git a/docs/examples/PuppeteerCrawler.md b/docs/examples/PuppeteerCrawler.md
new file mode 100644
index 000000000..e914edeb3
--- /dev/null
+++ b/docs/examples/PuppeteerCrawler.md
@@ -0,0 +1,93 @@
+---
+id: puppeteercrawler
+title: Puppeteer Crawler
+---
+
+This example demonstrates how to use [PuppeteerCrawler](https://www.apify.com/docs/sdk/apify-runtime-js/latest#PuppeteerCrawler)
+in combination with [RequestList](https://www.apify.com/docs/sdk/apify-runtime-js/latest#RequestList)
+and [RequestQueue](https://www.apify.com/docs/sdk/apify-runtime-js/latest#RequestQueue) to recursively scrape the
+Hacker News website (https://news.ycombinator.com) using headless Chrome / Puppeteer.
+The crawler starts with a single URL, finds links to next pages,
+enqueues them and continues until no more desired links are available.
+The results are stored to the default dataset. In local configuration, the results are stored as JSON files in `./apify_storage/datasets/default`
+```javascript
+const Apify = require('apify');
+
+Apify.main(async () => {
+ // Create and initialize an instance of the RequestList class that contains the start URL.
+ const requestList = new Apify.RequestList({
+ sources: [
+ { url: 'http://www.ycombinator.com/' },
+ ],
+ });
+ await requestList.initialize();
+
+ // Apify.openRequestQueue() is a factory to get a preconfigured RequestQueue instance.
+ const requestQueue = await Apify.openRequestQueue();
+
+ // Create an instance of the PuppeteerCrawler class - a crawler
+ // that automatically loads the URLs in headless Chrome / Puppeteer.
+ const crawler = new Apify.PuppeteerCrawler({
+ // The crawler will first fetch start URLs from the RequestList
+ // and then the newly discovered URLs from the RequestQueue
+ requestList,
+ requestQueue,
+
+ // Run Puppeteer in headless mode. If you set headless to false, you'll see the scraping
+ // browsers showing up on your screen. This is great for debugging.
+ launchPuppeteerOptions: { headless: true },
+
+ // This function will be called for each URL to crawl.
+ // Here you can write the Puppeteer scripts you are familiar with,
+ // with the exception that browsers and pages are automatically managed by the Apify SDK.
+ // The function accepts a single parameter, which is an object with the following fields:
+ // - request: an instance of the Request class with information such as URL and HTTP method
+ // - page: Puppeteer's Page object (see https://pptr.dev/#show=api-class-page)
+ handlePageFunction: async ({ request, page }) => {
+ console.log(`Processing ${request.url}...`);
+
+ // A function to be evaluated by Puppeteer within the browser context.
+ const pageFunction = ($posts) => {
+ const data = [];
+
+ // We're getting the title, rank and URL of each post on Hacker News.
+ $posts.forEach(($post) => {
+ data.push({
+ title: $post.querySelector('.title a').innerText,
+ rank: $post.querySelector('.rank').innerText,
+ href: $post.querySelector('.title a').href,
+ });
+ });
+
+ return data;
+ };
+ const data = await page.$$eval('.athing', pageFunction);
+
+ // Store the results to the default dataset.
+ await Apify.pushData(data);
+
+ // Find the link to the next page using Puppeteer functions.
+ let nextHref;
+ try {
+ nextHref = await page.$eval('.morelink', el => el.href);
+ } catch (err) {
+ console.log(`${request.url} is the last page!`);
+ return;
+ }
+
+ // Enqueue the link to the RequestQueue
+ await requestQueue.addRequest(new Apify.Request({ url: nextHref }));
+ },
+
+ // This function is called if the page processing failed more than maxRequestRetries+1 times.
+ handleFailedRequestFunction: async ({ request }) => {
+ console.log(`Request ${request.url} failed too many times`);
+ },
+ });
+
+ // Run the crawler and wait for it to finish.
+ await crawler.run();
+
+ console.log('Crawler finished.');
+});
+```
\ No newline at end of file
diff --git a/docs/examples/PuppeteerSitemap.md b/docs/examples/PuppeteerSitemap.md
new file mode 100644
index 000000000..a573f3e9c
--- /dev/null
+++ b/docs/examples/PuppeteerSitemap.md
@@ -0,0 +1,34 @@
+---
+id: puppeteersitemap
+title: Puppeteer Sitemap
+---
+
+This example demonstrates how to use `PuppeteerCrawler` to crawl a list of web pages
+specified in a sitemap. The crawler extracts page title and URL from each page
+and stores them as a record to the default dataset.
+In local configuration, the results are stored as JSON files in `./apify_storage/datasets/default`
+```javascript
+const Apify = require('apify');
+
+Apify.main(async () => {
+ const requestList = new Apify.RequestList({
+ sources: [{ requestsFromUrl: 'https://edition.cnn.com/sitemaps/cnn/news.xml' }],
+ });
+ await requestList.initialize();
+
+ const crawler = new Apify.PuppeteerCrawler({
+ requestList,
+ handlePageFunction: async ({ page, request }) => {
+ console.log(`Processing ${request.url}...`);
+ await Apify.pushData({
+ url: request.url,
+ title: await page.title(),
+ html: await page.content(),
+ });
+ },
+ });
+
+ await crawler.run();
+ console.log('Done.');
+});
+```
\ No newline at end of file
diff --git a/docs/examples/PuppeteerWithProxy.md b/docs/examples/PuppeteerWithProxy.md
new file mode 100644
index 000000000..670dea0a3
--- /dev/null
+++ b/docs/examples/PuppeteerWithProxy.md
@@ -0,0 +1,39 @@
+---
+id: puppeteerwithproxy
+title: Puppeteer With Proxy
+---
+
+This example demonstrates how to load pages in headless Chrome / Puppeteer
+over [Apify Proxy](https://www.apify.com/docs/proxy).
+To make it work, you'll need an Apify Account
+that has access to the proxy.
+The proxy password is available on the [Proxy](https://my.apify.com/proxy) page in the app.
+Just set it to the `APIFY_PROXY_PASSWORD` environment variable
+or run the script using the CLI.
+```javascript
+const Apify = require('apify');
+
+Apify.main(async () => {
+ // Apify.launchPuppeteer() is similar to Puppeteer's launch() function.
+ // It accepts the same parameters and returns a preconfigured Puppeteer.Browser instance.
+ // Moreover, it accepts several additional options, such as useApifyProxy.
+ const options = {
+ useApifyProxy: true,
+ };
+ const browser = await Apify.launchPuppeteer(options);
+
+ console.log('Running Puppeteer script...');
+
+ // Proceed with a plain Puppeteer script.
+ const page = await browser.newPage();
+ const url = 'https://en.wikipedia.org/wiki/Main_Page';
+ await page.goto(url);
+ const title = await page.title();
+
+ console.log(`Page title: ${title}`);
+
+ // Cleaning up after yourself is always good.
+ await browser.close();
+ console.log('Puppeteer closed.');
+});
+```
\ No newline at end of file
diff --git a/docs/examples/Screenshots.md b/docs/examples/Screenshots.md
new file mode 100644
index 000000000..b16571445
--- /dev/null
+++ b/docs/examples/Screenshots.md
@@ -0,0 +1,65 @@
+---
+id: screenshots
+title: Screenshots
+---
+
+This example demonstrates how to read and write
+data to the default key-value store using
+<a href="https://www.apify.com/docs/sdk/apify-runtime-js/latest#module-Apify-getValue"><code>Apify.getValue()</code></a>
+and
+<a href="https://www.apify.com/docs/sdk/apify-runtime-js/latest#module-Apify-setValue"><code>Apify.setValue()</code></a>.
+The script crawls a list of URLs using Puppeteer,
+captures a screenshot of each page and saves it to the store. The list of URLs is
+provided as actor input that is also read from the store.
+
+In local configuration, the input is stored in the default key-value store's directory as a JSON file at
+`./apify_storage/key_value_stores/default/INPUT.json`. You need to create the file and set it with the following content:
+
+```json
+{ "sources": [{ "url": "https://www.google.com" }, { "url": "https://www.duckduckgo.com" }] }
+```
+
+On the Apify cloud platform, the input can be either set manually
+in the UI app or passed as the POST payload to the
+[Run actor API call](https://www.apify.com/docs/api/v2#/reference/actors/run-collection/run-actor).
+For more details, see [Input and output](https://www.apify.com/docs/actor#input-output)
+in the Apify Actor documentation.
+```javascript
+const Apify = require('apify');
+
+Apify.main(async () => {
+ // Read the actor input configuration containing the URLs for the screenshot.
+ // By convention, the input is present in the actor's default key-value store under the "INPUT" key.
+ const input = await Apify.getValue('INPUT');
+ if (!input) throw new Error('Have you passed the correct INPUT ?');
+
+ const { sources } = input;
+
+ const requestList = new Apify.RequestList({ sources });
+ await requestList.initialize();
+
+ const crawler = new Apify.PuppeteerCrawler({
+ requestList,
+ handlePageFunction: async ({ page, request }) => {
+ console.log(`Processing ${request.url}...`);
+
+ // This is a Puppeteer function that takes a screenshot of the page and returns its buffer.
+ const screenshotBuffer = await page.screenshot();
+
+ // The record key may only include the following characters: a-zA-Z0-9!-_.'()
+ const key = request.url.replace(/[:/]/g, '_');
+
+ // Save the screenshot. Choosing the right content type will automatically
+ // assign the local file the right extension, in this case .png.
+ // The screenshots will be stored in ./apify_storage/key_value_stores/default/
+ await Apify.setValue(key, screenshotBuffer, { contentType: 'image/png' });
+ console.log(`Screenshot of ${request.url} saved.`);
+ },
+ });
+
+ // Run crawler.
+ await crawler.run();
+
+ console.log('Crawler finished.');
+});
+```
\ No newline at end of file
diff --git a/docs/examples/SynchronousRun.md b/docs/examples/SynchronousRun.md
new file mode 100644
index 000000000..817be68d6
--- /dev/null
+++ b/docs/examples/SynchronousRun.md
@@ -0,0 +1,48 @@
+---
+id: synchronousrun
+title: Synchronous Run
+---
+
+This example shows a quick actor that has a run time of just a few seconds.
+It opens a <a href="http://goldengatebridge75.org/news/webcam.html" target="_blank">web page</a>
+that contains a webcam stream from the Golden Gate Bridge, takes a screenshot of the page and saves it as output.
+
+This actor
+can be invoked synchronously using a single HTTP request to directly obtain its output as a reponse, using the
+<a href="https://www.apify.com/docs/api/v2#/reference/actors/run-actor-synchronously/without-input" target="_blank">Run actor synchronously</a>
+Apify API endpoint. The example is also shared as the
+<a href="https://www.apify.com/apify/example-golden-gate-webcam" target="_blank">apify/example-golden-gate-webcam</a>
+actor in the Apify library, so you can test it directly there simply by sending a POST request to
+```http
+https://api.apify.com/v2/acts/apify~example-golden-gate-webcam/run-sync?token=[YOUR_API_TOKEN]
+```
+```javascript
+const Apify = require('apify');
+
+Apify.main(async () => {
+ // Launch web browser.
+ const browser = await Apify.launchPuppeteer();
+
+ // Load http://goldengatebridge75.org/news/webcam.html and get an IFRAME with the webcam stream
+ console.log('Opening web page...');
+ const page = await browser.newPage();
+ await page.goto('http://goldengatebridge75.org/news/webcam.html');
+ const iframe = (await page.frames()).pop();
+
+ // Get webcam image element handle.
+ const imageElementHandle = await iframe.$('.VideoColm img');
+
+ // Give the webcam image some time to load.
+ console.log('Waiting for page to load...');
+ await Apify.utils.sleep(3000);
+
+ // Get a screenshot of that image.
+ const imageBuffer = await imageElementHandle.screenshot();
+ console.log('Screenshot captured.');
+
+ // Save the screenshot as the actor's output. By convention, similarly to "INPUT",
+ // the actor's output is stored in the default key-value store under the "OUTPUT" key.
+ await Apify.setValue('OUTPUT', imageBuffer, { contentType: 'image/jpeg' });
+ console.log('Actor finished.');
+});
+```
\ No newline at end of file
diff --git a/docs/guides/data_storage.md b/docs/guides/data_storage.md
new file mode 100644
index 000000000..3c63a6266
--- /dev/null
+++ b/docs/guides/data_storage.md
@@ -0,0 +1,190 @@
+---
+id: datastorage
+title: Data Storage
+---
+
+The Apify SDK has several data storage types that are useful for specific tasks.
+The data is stored either on local disk to a directory defined by the `APIFY_LOCAL_STORAGE_DIR` environment variable,
+or on the <a href="https://www.apify.com" target="_blank">Apify Cloud</a> under the user account identified by the API token defined by the `APIFY_TOKEN` environment variable.
+If neither of these variables is defined, by default Apify SDK sets `APIFY_LOCAL_STORAGE_DIR`
+to `./apify_storage` in the current working directory and prints a warning.
+
+Typically, you will be developing the code on your local computer and thus set the `APIFY_LOCAL_STORAGE_DIR` environment variable.
+Once the code is ready, you will deploy it to the Apify Cloud, where it will automatically
+set the `APIFY_TOKEN` environment variable and thus use cloud storage.
+No code changes are needed.
+
+**Related links**
+
+* <a href="https://www.apify.com/docs/storage" target="_blank">Apify cloud storage documentation</a>
+* <a href="https://my.apify.com/storage" target="_blank">View storage in Apify app</a>
+* <a href="https://www.apify.com/docs/api/v2#/reference/key-value-stores" target="_blank">API reference</a>
+
+## Key-value store
+
+The key-value store is used for saving and reading data records or files.
+Each data record is represented by a unique key and associated with a MIME content type.
+Key-value stores are ideal for saving screenshots of web pages, PDFs or to persist the state of crawlers.
+
+Each actor run is associated with a **default key-value store**, which is created exclusively for the actor run.
+By convention, the actor run input and output is stored in the default key-value store
+under the `INPUT` and `OUTPUT` key, respectively. Typically the input and output is a JSON file,
+although it can be any other format.
+
+In the Apify SDK, the key-value store is represented by the
+[`KeyValueStore`](../api/keyvaluestore)
+class.
+In order to simplify access to the default key-value store, the SDK also provides
+[`Apify.getValue()`](../api/apify#module_Apify.getValue)
+and [`Apify.setValue()`](../api/apify#module_Apify.setValue) functions.
+
+In local configuration, the data is stored in the directory specified by the `APIFY_LOCAL_STORAGE_DIR` environment variable as follows:
+
+```
+{APIFY_LOCAL_STORAGE_DIR}/key_value_stores/{STORE_ID}/{KEY}.{EXT}
+```
+
+Note that `{STORE_ID}` is the name or ID of the key-value store.
+The default key value store has ID `default`, unless you override it by setting the `APIFY_DEFAULT_KEY_VALUE_STORE_ID`
+environment variable.
+The `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the
+data value.
+
+The following code demonstrates basic operations of key-value stores:
+
+```javascript
+// Get actor input from the default key-value store
+const input = await Apify.getValue('INPUT');
+
+// Write actor output to the default key-value store.
+await Apify.setValue('OUTPUT', { myResult: 123 });
+
+// Open a named key-value store
+const store = await Apify.openKeyValueStore('some-name');
+
+// Write record. JavaScript object is automatically converted to JSON,
+// strings and binary buffers are stored as they are
+await store.setValue('some-key', { foo: 'bar' });
+
+// Read record. Note that JSON is automatically parsed to a JavaScript object,
+// text data returned as a string and other data is returned as binary buffer
+const value = await store.getValue('some-key');
+
+// Delete record
+await store.delete('some-key');
+```
+
+To see a real-world example of how to get the input from the key-value store, see the
+[Screenshots](../examples/screenshots) example.
+
+
+## Dataset
+
+Datasets are used to store structured data
+where each object stored has the same attributes, such as online store products or real estate offers.
+You can imagine a dataset as a table, where each object is a row and its attributes are columns.
+Dataset is an append-only storage - you can only add new records to it but you cannot modify or remove
+existing records.
+
+When the dataset is stored in the <a href="https://www.apify.com" target="_blank">Apify Cloud</a>,
+you can export its data to the following formats: HTML, JSON, CSV, Excel, XML and RSS.
+The datasets are displayed on the actor run details page and in the <a href="https://my.apify.com/storage" target="_blank">Storage</a>
+section in the Apify app. The actual data is exported using the
+<a href="https://www.apify.com/docs/api/v2#/reference/datasets/item-collection/get-items" target="_blank">Get dataset items</a> Apify API endpoint.
+This way you can easily share crawling results.
+
+Each actor run is associated with a **default dataset**, which is created exclusively for the actor run.
+Typically, it is used to store crawling results specific for the actor run. Its usage is optional.
+
+In the Apify SDK, the dataset is represented by the [`Dataset`](../api/dataset) class.
+In order to simplify writes to the default dataset, the SDK also provides the
+[`Apify.pushData()`](../api/apify#module_Apify.pushData) function.
+
+In local configuration, the data is stored in the directory specified by the `APIFY_LOCAL_STORAGE_DIR` environment variable as follows:
+
+```
+{APIFY_LOCAL_STORAGE_DIR}/datasets/{DATASET_ID}/{INDEX}.json
+```
+
+Note that `{DATASET_ID}` is the name or ID of the dataset.
+The default dataset has ID `default`, unless you override it by setting the `APIFY_DEFAULT_DATASET_ID`
+environment variable.
+Each dataset item is stored as a separate JSON file,
+where `{INDEX}` is a zero-based index of the item in the dataset.
+
+The following code demonstrates basic operations of the dataset:
+
+```javascript
+// Write a single row to the default dataset
+await Apify.pushData({ col1: 123, col2: 'val2' });
+
+// Open a named dataset
+const dataset = await Apify.openDataset('some-name');
+
+// Write a single row
+await dataset.pushData({ foo: 'bar' });
+
+// Write multiple rows
+await dataset.pushData([
+ { foo: 'bar2', col2: 'val2' },
+ { col3: 123 },
+]);
+```
+
+To see how to use the dataset to store crawler results, see the
+[Cheerio Crawler](../examples/cheeriocrawler) example.
+
+
+## Request queue
+
+The request queue is a storage of URLs to crawl.
+The queue is used for the deep crawling of websites, where you start with
+several URLs and then recursively follow links to other pages.
+The data structure supports both breadth-first and depth-first crawling orders.
+
+Each actor run is associated with a **default request queue**, which is created exclusively for the actor run.
+Typically, it is used to store URLs to crawl in the specific actor run. Its usage is optional.
+
+In Apify SDK, the request queue is represented by the [`RequestQueue`](../api/requestqueue) class.
+
+In local configuration, the request queue data is stored in the directory specified by the `APIFY_LOCAL_STORAGE_DIR` environment variable as follows:
+
+```
+{APIFY_LOCAL_STORAGE_DIR}/request_queues/{QUEUE_ID}/{STATE}/{NUMBER}.json
+```
+
+Note that `{QUEUE_ID}` is the name or ID of the request queue.
+The default queue has ID `default`, unless you override it by setting the `APIFY_DEFAULT_REQUEST_QUEUE_ID`
+environment variable.
+Each request in the queue is stored as a separate JSON file,
+where `{STATE}` is either `handled` or `pending`,
+and `{NUMBER}` is an integer indicating the position of the request in the queue.
+
+The following code demonstrates basic operations of the request queue:
+
+```javascript
+// Open the default request queue associated with the actor run
+const queue = await Apify.openRequestQueue();
+
+// Open a named request queue
+const queueWithName = await Apify.openRequestQueue('some-name');
+
+// Enqueue few requests
+await queue.addRequest(new Apify.Request({ url: 'http://example.com/aaa'}));
+await queue.addRequest(new Apify.Request({ url: 'http://example.com/bbb'}));
+await queue.addRequest(new Apify.Request({ url: 'http://example.com/foo/bar'}), { forefront: true });
+
+// Get requests from queue
+const request1 = await queue.fetchNextRequest();
+const request2 = await queue.fetchNextRequest();
+const request3 = await queue.fetchNextRequest();
+
+// Mark a request as handled
+await queue.markRequestHandled(request1);
+
+// If processing fails then reclaim the request back to the queue, so that it's crawled again
+await queue.reclaimRequest(request2);
+```
+
+To see how to use the request queue with a crawler, see the
+[Puppeteer Crawler](../examples/puppeteercrawler) example.
diff --git a/docs/guides/environment_variables.md b/docs/guides/environment_variables.md
new file mode 100644
index 000000000..052d5e873
--- /dev/null
+++ b/docs/guides/environment_variables.md
@@ -0,0 +1,78 @@
+---
+id: environmentvariables
+title: Environment Variables
+---
+
+The following table shows the basic environment variables used by Apify SDK:
+
+<table>
+ <thead>
+ <tr>
+ <th>Environment variable</th>
+ <th>Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><code>APIFY_LOCAL_STORAGE_DIR</code></td>
+ <td>
+ Defines the path to a local directory where
+ <a href="../api/keyvaluestore">key-value stores</a>,
+ <a href="../api/requestlist">request lists</a>
+ and <a href="../api/requestqueue">request queues</a> store their data.
+ Typically it is set to <code>./apify_storage</code>.
+ If omitted, you should define
+ the <code>APIFY_TOKEN</code> environment variable instead.
+ </td>
+ </tr>
+ <tr>
+ <td><code>APIFY_TOKEN</code></td>
+ <td>
+ The API token for your Apify Account. It is used to access the Apify API, e.g. to access cloud storage or to run an actor in the Apify Cloud.
+ You can find your API token on the <a href="https://my.apify.com/account#intergrations" target="_blank">Account - Integrations</a> page.
+ If omitted, you should define the <code>APIFY_LOCAL_STORAGE_DIR</code> environment variable instead.
+ </td>
+ </tr>
+ <tr>
+ <td><code>APIFY_PROXY_PASSWORD</code></td>
+ <td>
+ Optional password to <a href="https://www.apify.com/docs/proxy" target="_blank">Apify Proxy</a> for IP address rotation.
+ If you have have an Apify Account, you can find the password on the
+ <a href="https://my.apify.com/proxy" target="_blank">Proxy page</a> in the Apify app.
+ This feature is optional. You can use your own proxies or no proxies at all.
+ </td>
+ </tr>
+ <tr>
+ <td><code>APIFY_HEADLESS</code></td>
+ <td>
+ If set to <code>1</code>, web browsers launched by Apify SDK will run in the headless
+ mode. You can still override this setting in the code, e.g. by
+ passing the <code>headless: true</code> option to the
+ <a href="../api/apify#module_Apify.launchPuppeteer"><code>Apify.launchPuppeteer()</code></a>
+ function. But having this setting in an environment variable allows you to develop
+ the crawler locally in headful mode to simplify the debugging, and only run the crawler in headless
+ mode once you deploy it to the Apify Cloud.
+ By default, the browsers are launched in headful mode, i.e. with windows.
+ </td>
+ </tr>
+ <tr>
+ <td><code>APIFY_LOG_LEVEL</code></td>
+ <td>
+ Specifies the minimum log level, which can be one of the following values (in order of severity):
+ <code>DEBUG</code>, <code>INFO</code>, <code>WARNING</code>, <code>SOFT_FAIL</code> and <code>ERROR</code>.
+ By default, the log level is set to <code>INFO</code>, which means that <code>DEBUG</code> messages
+ are not printed to console.
+ </td>
+ </tr>
+ <tr>
+ <td><code>APIFY_MEMORY_MBYTES</code></td>
+ <td>
+ Sets the amount of system memory in megabytes to be used by the
+ <a href="../api/autoscaledpool">autoscaled pool</a>.
+ It is used to limit the number of concurrently running tasks. By default, the max amount of memory
+ to be used is set to one quarter of total system memory, i. e. on a system with 8192 MB of memory,
+ the autoscaling feature will only use up to 2048 MB of memory.
+ </td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/guides/getting_started.md b/docs/guides/getting_started.md
new file mode 100644
index 000000000..59c2c89b2
--- /dev/null
+++ b/docs/guides/getting_started.md
@@ -0,0 +1,115 @@
+---
+id: gettingstarted
+title: Getting Started
+---
+The Apify SDK requires <a href="https://nodejs.org/en/" target="_blank">Node.js</a> 8 or later.
+
+## Local stand-alone usage
+
+Add Apify SDK to any Node.js project by running:
+
+```bash
+npm install apify --save
+```
+
+Run the following example to perform a recursive crawl of a website using Puppeteer.
+
+```javascript
+const Apify = require('apify');
+
+Apify.main(async () => {
+ const requestQueue = await Apify.openRequestQueue();
+ await requestQueue.addRequest({ url: 'https://www.iana.org/' });
+ const pseudoUrls = [new Apify.PseudoUrl('https://www.iana.org/[.*]')];
+
+ const crawler = new Apify.PuppeteerCrawler({
+ requestQueue,
+ handlePageFunction: async ({ request, page }) => {
+ const title = await page.title();
+ console.log(`Title of ${request.url}: ${title}`);
+ await Apify.utils.puppeteer.enqueueLinks(page, 'a', pseudoUrls, requestQueue);
+ },
+ maxRequestsPerCrawl: 100,
+ maxConcurrency: 10,
+ });
+
+ await crawler.run();
+});
+```
+
+When you run the example, you should see Apify SDK automating a Chrome browser.
+
+![Chrome Scrape](../../img/chrome_scrape.gif)
+
+By default, Apify SDK stores data to
+`./apify_storage` in the current working directory.
+You can override this behavior by setting either the
+`APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variable.
+For details, see [Environment variables](environmentvariables)
+and [Data storage](datastorage).
+
+## Local usage with Apify command-line interface (CLI)
+
+To avoid the need to set the environment variables manually,
+to create a boilerplate of your project,
+and to enable pushing and running your code on the
+<a href="https://www.apify.com" target="_blank">Apify Cloud</a>,
+you can use the
+<a href="https://github.com/apifytech/apify-cli" target="_blank">Apify command-line interface</a>
+(CLI) tool.
+
+Install the CLI by running:
+
+```bash
+npm -g install apify-cli
+```
+
+You might need to run the above command with `sudo`, depending on how crazy your configuration is.
+
+Now create a boilerplate of your new web crawling project by running:
+
+```bash
+apify create my-hello-world
+```
+
+The CLI will prompt you to select a project boilerplate template - just pick "Hello world".
+The tool will create a directory called `my-hello-world` with a Node.js project files.
+You can run the project as follows:
+
+```bash
+cd my-hello-world
+apify run
+```
+
+By default, the crawling data will be stored in a local directory at `./apify_storage`.
+For example, the input JSON file for the actor is expected to be in the default key-value store
+in `./apify_storage/key_value_stores/default/INPUT.json`.
+
+Now you can easily deploy your code to the Apify Cloud by running:
+
+```bash
+apify login
+```
+```bash
+apify push
+```
+
+Your script will be uploaded to the Apify Cloud and built there so that it can be run.
+For more information, view the
+<a href="https://www.apify.com/docs/cli" target="_blank">Apify CLI</a>
+and
+<a href="https://www.apify.com/docs/actor" target="_blank">Apify Actor</a>
+documentation.
+
+## Usage on the Apify Cloud
+
+You can also develop your web scraping project
+in an online code editor directly on the
+<a href="https://www.apify.com" target="_blank">Apify Cloud</a>.
+You'll need to have an Apify Account.
+Go to <a href="https://my.apify.com/actors" target="_blank">Actors</a>,
+page in the app, click <i>Create new</i> and then go to the
+<i>Source</i> tab and start writing your code or paste one of the examples from the Examples section.
+
+For more information, view the
+<a href="https://www.apify.com/docs/actor#quick-start" target="_blank">Apify actors quick start guide</a>.
diff --git a/docs/guides/motivation.md b/docs/guides/motivation.md
new file mode 100644
index 000000000..644940b5d
--- /dev/null
+++ b/docs/guides/motivation.md
@@ -0,0 +1,26 @@
+---
+id: motivation
+title: Motivation
+---
+
+Thanks to tools like <a href="https://github.com/GoogleChrome/puppeteer" target="_blank">Puppeteer</a> or
+<a href="https://www.npmjs.com/package/cheerio" target="_blank">Cheerio</a>,
+it is easy to write Node.js code to extract data from web pages.
+But eventually things will get complicated. For example, when you try to:
+
+* Perform a deep crawl of an entire website using a persistent queue of URLs.
+* Run your scraping code on a list of 100k URLs in a CSV file,
+ without losing any data when your code crashes.
+* Rotate proxies to hide your browser origin.
+* Schedule the code to run periodically and send notification on errors.
+* Disable browser fingerprinting protections used by websites.
+
+Python has <a href="https://scrapy.org/" target="_blank">Scrapy</a> for these tasks, but there was no
+such library for **JavaScript, the language of the web**.
+The use of JavaScript is natural,
+since the same language is used to write the scripts as well as the data extraction code running in a browser.
+
+The goal of the Apify SDK is to fill this gap and provide a toolbox
+for generic web scraping, crawling and automation tasks in JavaScript.
+So don't reinvent the wheel every time you need data from the web,
+and focus on writing code specific to the target website, rather than developing commonalities.
diff --git a/docs/guides/puppeteer_live_view.md b/docs/guides/puppeteer_live_view.md
new file mode 100644
index 000000000..d3fc9e4d3
--- /dev/null
+++ b/docs/guides/puppeteer_live_view.md
@@ -0,0 +1,41 @@
+---
+id: puppeteerliveview
+title: Puppeteer Live View
+---
+
+Apify SDK enables real-time view of launched Puppeteer browser instances and their open tabs,
+including screenshots of pages and snapshots of HTML.
+This is useful for debugging your crawlers that run in headless mode.
+
+The live view dashboard is run on a web server that is started on a port specified
+by the `APIFY_CONTAINER_PORT` environment variable (typically 4321).
+To enable live view, pass the `liveView: true` option to
+[`Apify.launchPuppeteer()`](../api/apify#module_Apify.launchPuppeteer):
+
+```js
+const browser = Apify.launchPuppeteer({ liveView: true });
+```
+
+or to [`PuppeteerCrawler`](../api/puppeteercrawler#new_PuppeteerCrawler_new) constructor as follows:
+
+```js
+const crawler = new PuppeteerCrawler({
+ launchPuppeteerOptions: { liveView: true },
+ // other options
+})
+```
+
+To simplify debugging, you may also want to add the
+`{ slowMo: 300 }` option to slow down all browser operation.
+See <a href="https://pptr.dev/#?product=Puppeteer&show=api-puppeteerlaunchoptions" target="_blank">Puppeteer documentation</a> for details.
+
+Once live view is enabled, you can open `http://localhost:4321` and you will see a page like this:
+
+![Puppeteer Live View Dashboard](../../img/puppeteer-live-view-dashboard.png)
+
+Click on the magnifying glass icon to view page detail, showing page screenshot and raw HTML:
+
+![Puppeteer Live View Detail](../../img/puppeteer-live-view-detail.png)
+
+For more information, read the <a href="https://kb.apify.com/actor/debugging-your-actors-with-live-view" target="_blank">Debugging your actors with Live View</a>
+article in Apify Knowledge Base.
diff --git a/docs/guides/what_is_an_actor.md b/docs/guides/what_is_an_actor.md
new file mode 100644
index 000000000..ae3dfc5b2
--- /dev/null
+++ b/docs/guides/what_is_an_actor.md
@@ -0,0 +1,23 @@
+---
+id: whatisanactor
+title: What is an Actor
+---
+When you deploy your script to the <a href="https://www.apify.com/" target="_blank">Apify Cloud</a>,
+it becomes an actor.
+An actor is a serverless microservice that accepts an input and produces an output.
+It can run for a few seconds, hours or even infinitely.
+An actor can perform anything from a simple action such as filling out a web form or sending an email,
+to complex operations such as crawling an entire website and removing duplicates from a large dataset.
+
+To run an actor, you need to have an <a href="https://my.apify.com/)" target="_blank">Apify Account</a>.
+Actors can be shared in the <a href="https://www.apify.com/library?&type=acts)" target="_blank">Apify Library</a>
+so that other people can use them.
+But don't worry, if you share your actor in the library
+and somebody uses it, it runs under their account, not yours.
+
+**Related links**
+
+* <a href="https://www.apify.com/library?&type=acts" target="_blank">Library of existing actors</a>
+* <a href="https://www.apify.com/docs/actor" target="_blank">Documentation</a>
+* <a href="https://my.apify.com/actors" target="_blank">View actors in Apify app</a>
+* <a href="https://www.apify.com/docs/api/v2#/reference/actors" target="_blank">API reference</a>
diff --git a/docs/hello-world-browsers.gif b/docs/hello-world-browsers.gif
deleted file mode 100644
index 9aceffa50..000000000
Binary files a/docs/hello-world-browsers.gif and /dev/null differ
diff --git a/docs/jsdoc-conf.json b/docs/jsdoc-conf.json
deleted file mode 100644
index e90719e02..000000000
--- a/docs/jsdoc-conf.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "tags": {
- "allowUnknownTags": true
- },
- "source": {
- "include": [ "src" ],
- "exclude": [ "src/constants.js" ],
- "includePattern": ".+\\.js(doc|x)?$",
- "excludePattern": "(^|\\/|\\\\)_"
- },
- "plugins": [
- "./node_modules/jsdoc-export-default-interop/dist/index.js",
- "plugins/markdown"
- ],
- "templates": {
- "cleverLinks": false,
- "monospaceLinks": false,
- "default": {
- "outputSourceFiles": false
- }
- },
- "opts": {
- "template": "./node_modules/apify-jsdoc-template",
- "encoding": "utf8",
- "destination": ".docs",
- "recurse": true
- }
-}
diff --git a/docs/typedefs/ActorRun.md b/docs/typedefs/ActorRun.md
new file mode 100644
index 000000000..91044c6fe
--- /dev/null
+++ b/docs/typedefs/ActorRun.md
@@ -0,0 +1,137 @@
+---
+id: actorrun
+title: ActorRun
+---
+<a name="ActorRun"></a>
+
+Represents information about an actor run, as returned by the
+[`Apify.call()`](../api/apify#module_Apify.call) function.
+The object is almost equivalent to the JSON response
+of the
+<a href="https://www.apify.com/docs/api/v2#/reference/actors/run-collection/run-actor" target="_blank">Actor run</a>
+Apify API endpoint and extended with certain fields.
+For more details, see
+<a href="https://www.apify.com/docs/actor#run" target="_blank">Runs.</a>
+
+**Properties**
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>id</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Actor run ID</p>
+</td></tr><tr>
+<td><code>actId</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Actor ID</p>
+</td></tr><tr>
+<td><code>startedAt</code></td><td><code>Date</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Time when the actor run started</p>
+</td></tr><tr>
+<td><code>finishedAt</code></td><td><code>Date</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Time when the actor run finished. Contains <code>null</code> for running actors.</p>
+</td></tr><tr>
+<td><code>status</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Status of the run. For possible values, see
+ <a href="https://www.apify.com/docs/actor#run-lifecycle" target="_blank">Run lifecycle</a>
+ in Apify actor documentation.</p>
+</td></tr><tr>
+<td><code>meta</code></td><td><code>Object</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Actor run meta-data. For example:</p>
+<pre><code> {
+ &quot;origin&quot;: &quot;API&quot;,
+ &quot;clientIp&quot;: &quot;1.2.3.4&quot;,
+ &quot;userAgent&quot;: &quot;ApifyClient/0.2.13 (Linux; Node/v8.11.3)&quot;
+ }
+</code></pre></td></tr><tr>
+<td><code>stats</code></td><td><code>Object</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>An object containing various actor run statistics. For example:</p>
+<pre><code> {
+ &quot;inputBodyLen&quot;: 22,
+ &quot;restartCount&quot;: 0,
+ &quot;workersUsed&quot;: 1,
+ }
+</code></pre><p> Beware that object fields might change in future releases.</p>
+</td></tr><tr>
+<td><code>options</code></td><td><code>Object</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Actor run options. For example:</p>
+<pre><code> {
+ &quot;build&quot;: &quot;latest&quot;,
+ &quot;timeoutSecs&quot;: 0,
+ &quot;memoryMbytes&quot;: 256,
+ &quot;diskMbytes&quot;: 512
+ }
+</code></pre></td></tr><tr>
+<td><code>buildId</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>ID of the actor build used for the run. For details, see
+ <a href="https://www.apify.com/docs/actor#build" target="_blank">Builds</a>
+ in Apify actor documentation.</p>
+</td></tr><tr>
+<td><code>buildNumber</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Number of the actor build used for the run. For example, <code>0.0.10</code>.</p>
+</td></tr><tr>
+<td><code>exitCode</code></td><td><code>Number</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Exit code of the actor run process. It&#39;s <code>null</code> if actor is still running.</p>
+</td></tr><tr>
+<td><code>defaultKeyValueStoreId</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>ID of the default key-value store associated with the actor run. See <a href="../api/keyvaluestore"><code>KeyValueStore</code></a> for details.</p>
+</td></tr><tr>
+<td><code>defaultDatasetId</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>ID of the default dataset associated with the actor run. See <a href="../api/dataset"><code>Dataset</code></a> for details.</p>
+</td></tr><tr>
+<td><code>defaultRequestQueueId</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>ID of the default request queue associated with the actor run. See <a href="../api/requestqueue"><code>RequestQueue</code></a> for details.</p>
+</td></tr><tr>
+<td><code>containerUrl</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>URL on which the web server running inside actor run&#39;s Docker container can be accessed.
+ For more details, see
+ <a href="https://www.apify.com/docs/actor#container-web-server" target="_blank">Container web server</a>
+ in Apify actor documentation.</p>
+</td></tr><tr>
+<td><code>output</code></td><td><code>Object</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Contains output of the actor run. The value is <code>null</code> or <code>undefined</code> in case the actor is still running,
+ or if you pass <code>false</code> to the <code>fetchOutput</code> option of <a href="../api/apify#module_Apify.call"><code>Apify.call()</code></a>.</p>
+<p> For example:</p>
+<pre><code> {
+ &quot;contentType&quot;: &quot;application/json; charset=utf-8&quot;,
+ &quot;body&quot;: {
+ &quot;message&quot;: &quot;Hello world!&quot;
+ }
+ }
+</code></pre></td></tr></tbody>
+</table>
diff --git a/docs/typedefs/ApifyCallError.md b/docs/typedefs/ApifyCallError.md
new file mode 100644
index 000000000..1e52eeb6d
--- /dev/null
+++ b/docs/typedefs/ApifyCallError.md
@@ -0,0 +1,34 @@
+---
+id: apifycallerror
+title: ApifyCallError
+---
+<a name="ApifyCallError"></a>
+
+The class represents exceptions thrown
+by the [`Apify.call()`](../api/apify#module_Apify.call) function.
+
+**Properties**
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>message</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Error message</p>
+</td></tr><tr>
+<td><code>name</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Contains <code>ApifyCallError</code></p>
+</td></tr><tr>
+<td><code>run</code></td><td><code><a href="../typedefs/actorrun">ActorRun</a></code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Object representing the failed actor run.</p>
+</td></tr></tbody>
+</table>
diff --git a/docs/typedefs/LaunchPuppeteerOptions.md b/docs/typedefs/LaunchPuppeteerOptions.md
new file mode 100644
index 000000000..19ec8e8cb
--- /dev/null
+++ b/docs/typedefs/LaunchPuppeteerOptions.md
@@ -0,0 +1,90 @@
+---
+id: launchpuppeteeroptions
+title: LaunchPuppeteerOptions
+---
+<a name="LaunchPuppeteerOptions"></a>
+
+Represents options passed to the
+[`Apify.launchPuppeteer()`](../api/apify#module_Apify.launchPuppeteer)
+function.
+
+**Properties**
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>[proxyUrl]</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>URL to a HTTP proxy server. It must define the port number,
+ and it may also contain proxy username and password.</p>
+<p> Example: <code>http://bob:[email protected]:1234</code>.</p>
+</td></tr><tr>
+<td><code>[userAgent]</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>The <code>User-Agent</code> HTTP header used by the browser.
+ If not provided, the function sets <code>User-Agent</code> to a reasonable default
+ to reduce the chance of detection of the crawler.</p>
+</td></tr><tr>
+<td><code>[useChrome]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If <code>true</code> and <code>executablePath</code> is not set,
+ Puppeteer will launch full Google Chrome browser available on the machine
+ rather than the bundled Chromium. The path to Chrome executable
+ is taken from the <code>APIFY_CHROME_EXECUTABLE_PATH</code> environment variable if provided,
+ or defaults to the typical Google Chrome executable location specific for the operating system.
+ By default, this option is <code>false</code>.</p>
+</td></tr><tr>
+<td><code>[useApifyProxy]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If set to <code>true</code>, Puppeteer will be configured to use
+ <a href="https://my.apify.com/proxy" target="_blank">Apify Proxy</a> for all connections.
+ For more information, see the <a href="https://www.apify.com/docs/proxy" target="_blank">documentation</a></p>
+</td></tr><tr>
+<td><code>[apifyProxyGroups]</code></td><td><code>Array&lt;String&gt;</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>An array of proxy groups to be used
+ by the <a href="https://www.apify.com/docs/proxy" target="_blank">Apify Proxy</a>.
+ Only applied if the <code>useApifyProxy</code> option is <code>true</code>.</p>
+</td></tr><tr>
+<td><code>[apifyProxySession]</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Apify Proxy session identifier to be used by all the Chrome browsers.
+ All HTTP requests going through the proxy with the same session identifier
+ will use the same target proxy server (i.e. the same IP address).
+ The identifier can only contain the following characters: <code>0-9</code>, <code>a-z</code>, <code>A-Z</code>, <code>&quot;.&quot;</code>, <code>&quot;_&quot;</code> and <code>&quot;~&quot;</code>.
+ Only applied if the <code>useApifyProxy</code> option is <code>true</code>.</p>
+</td></tr><tr>
+<td><code>[liveView]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If set to <code>true</code>, a <code>PuppeteerLiveViewServer</code> will be started to enable
+ screenshot and html capturing of visited pages using <code>PuppeteerLiveViewBrowser</code>.</p>
+</td></tr><tr>
+<td><code>[liveViewOptions]</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Settings for <code>PuppeteerLiveViewBrowser</code> started using <code>launchPuppeteer()</code>.</p>
+</td></tr><tr>
+<td><code>[liveViewOptions.liveViewId]</code></td><td><code>String</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Custom ID of a browser instance in live view.</p>
+</td></tr><tr>
+<td><code>[liveViewOptions.screenshotTimeoutMillis]</code></td><td><code>Number</code></td><td><code>3000</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Time in milliseconds before a screenshot capturing
+ will time out and the actor continues with execution.
+ Screenshot capturing pauses execution within the given page.</p>
+</td></tr></tbody>
+</table>
diff --git a/docs/typedefs/PaginationList.md b/docs/typedefs/PaginationList.md
new file mode 100644
index 000000000..7ec86964c
--- /dev/null
+++ b/docs/typedefs/PaginationList.md
@@ -0,0 +1,44 @@
+---
+id: paginationlist
+title: PaginationList
+---
+<a name="PaginationList"></a>
+
+Represents one page of data items from the [`Dataset`](../api/dataset).
+For more details, see [`Dataset.getData()`](../api/dataset#Dataset+getData).
+
+**Properties**
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>items</code></td><td><code>Array</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Array of returned items.</p>
+</td></tr><tr>
+<td><code>total</code></td><td><code>Number</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Total number of object.</p>
+</td></tr><tr>
+<td><code>offset</code></td><td><code>Number</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Number of Request objects that was skipped at the start.</p>
+</td></tr><tr>
+<td><code>count</code></td><td><code>Number</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Number of returned objects.</p>
+</td></tr><tr>
+<td><code>limit</code></td><td><code>Number</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Requested limit on the number of items.</p>
+</td></tr></tbody>
+</table>
diff --git a/docs/typedefs/RequestOperationInfo.md b/docs/typedefs/RequestOperationInfo.md
new file mode 100644
index 000000000..507f68a3c
--- /dev/null
+++ b/docs/typedefs/RequestOperationInfo.md
@@ -0,0 +1,34 @@
+---
+id: requestoperationinfo
+title: RequestOperationInfo
+---
+<a name="RequestOperationInfo"></a>
+
+A helper class that is used to report results from the
+[`Apify.utils.puppeteer.enqueueLinks()`](../api/puppeteer#puppeteer.enqueueLinks) function.
+
+**Properties**
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>wasAlreadyPresent</code></td><td><code>Boolean</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Indicates if request was already present in the queue.</p>
+</td></tr><tr>
+<td><code>wasAlreadyHandled</code></td><td><code>Boolean</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Indicates if request was already marked as handled.</p>
+</td></tr><tr>
+<td><code>requestId</code></td><td><code>String</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>The ID of the added request</p>
+</td></tr></tbody>
+</table>
diff --git a/examples/hello_world.js b/examples/hello_world.js
index b1c833107..79588c7e6 100644
--- a/examples/hello_world.js
+++ b/examples/hello_world.js
@@ -1,8 +1,12 @@
+/**
+ * Run the following example to perform a recursive crawl of a website using Puppeteer.
+ */
+
const Apify = require('apify');
Apify.main(async () => {
const requestQueue = await Apify.openRequestQueue();
- await requestQueue.addRequest(new Apify.Request({ url: 'https://www.iana.org/' }));
+ await requestQueue.addRequest({ url: 'https://www.iana.org/' });
const pseudoUrls = [new Apify.PseudoUrl('https://www.iana.org/[.*]')];
const crawler = new Apify.PuppeteerCrawler({
diff --git a/examples/puppeteer_proxy.js b/examples/puppeteer_with_proxy.js
similarity index 100%
rename from examples/puppeteer_proxy.js
rename to examples/puppeteer_with_proxy.js
diff --git a/examples/synchronous_run.js b/examples/synchronous_run.js
index f00143df6..9ce8e4af5 100644
--- a/examples/synchronous_run.js
+++ b/examples/synchronous_run.js
@@ -1,15 +1,17 @@
/**
* This example shows a quick actor that has a run time of just a few seconds.
- * It opens a [web page](http://goldengatebridge75.org/news/webcam.html) that contains a webcam stream from the Golden Gate
- * Bridge, takes a screenshot of the page and saves it as output.
+ * It opens a <a href="http://goldengatebridge75.org/news/webcam.html" target="_blank">web page</a>
+ * that contains a webcam stream from the Golden Gate Bridge, takes a screenshot of the page and saves it as output.
*
* This actor
- * can be invoked synchronously using a single HTTP request to directly obtain its output as a reponse,
- * using the [Run actor synchronously](https://www.apify.com/docs/api/v2#/reference/actors/run-actor-synchronously/without-input)
- * Apify API endpoint.
- * The example is also shared as the [apify/example-golden-gate-webcam](https://www.apify.com/apify/example-golden-gate-webcam)
+ * can be invoked synchronously using a single HTTP request to directly obtain its output as a reponse, using the
+ * <a href="https://www.apify.com/docs/api/v2#/reference/actors/run-actor-synchronously/without-input" target="_blank">Run actor synchronously</a>
+ * Apify API endpoint. The example is also shared as the
+ * <a href="https://www.apify.com/apify/example-golden-gate-webcam" target="_blank">apify/example-golden-gate-webcam</a>
* actor in the Apify library, so you can test it directly there simply by sending a POST request to
+ * ```http
* https://api.apify.com/v2/acts/apify~example-golden-gate-webcam/run-sync?token=[YOUR_API_TOKEN]
+ * ```
*/
const Apify = require('apify');
diff --git a/src/actor.js b/src/actor.js
index 99939b4a2..4a3890596 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -32,39 +32,31 @@ const tryParseDate = (str) => {
* {
* // ID of the actor (APIFY_ACT_ID)
* actId: String,
- * &nbsp;
* // ID of the actor run (APIFY_ACT_RUN_ID)
* actRunId: String,
- * &nbsp;
* // ID of the user who started the actor - note that it might be
* // different than the owner of the actor (APIFY_USER_ID)
* userId: String,
- * &nbsp;
* // Authentication token representing privileges given to the actor run,
* // it can be passed to various Apify APIs (APIFY_TOKEN).
* token: String,
- * &nbsp;
* // Date when the actor was started (APIFY_STARTED_AT)
* startedAt: Date,
- * &nbsp;
* // Date when the actor will time out (APIFY_TIMEOUT_AT)
* timeoutAt: Date,
- * &nbsp;
* // ID of the key-value store where input and output data of this
* // actor is stored (APIFY_DEFAULT_KEY_VALUE_STORE_ID)
* defaultKeyValueStoreId: String,
- * &nbsp;
* // ID of the dataset where input and output data of this
* // actor is stored (APIFY_DEFAULT_DATASET_ID)
* defaultDatasetId: String,
- * &nbsp;
* // Amount of memory allocated for the actor,
* // in megabytes (APIFY_MEMORY_MBYTES)
* memoryMbytes: Number,
* }
* ```
* For the list of the `APIFY_XXX` environment variables, see
- * <a href="http://localhost/docs/actor.php#run-env-vars" target="_blank">Actor documentation</a>.
+ * <a href="https://www.apify.com/docs/actor#run-env-vars" target="_blank">Actor documentation</a>.
* If some of the variables are not defined or are invalid, the corresponding value in the resulting object will be null.
*
* @returns {Object}
@@ -102,13 +94,13 @@ export const getEnv = () => {
* <li>When running on the Apify platform (i.e. <code>APIFY_IS_AT_HOME</code> environment variable is set),
* it sets up a connection to listen for platform events.
* For example, to get a notification about an imminent migration to another server.
- * See <a href="#module-Apify-events"><code>Apify.events</code></a> for details.
+ * See <a href="apify#apifyevents"><code>Apify.events</code></a> for details.
* </li>
* <li>It checks that either <code>APIFY_TOKEN</code> or <code>APIFY_LOCAL_STORAGE_DIR</code> environment variable
* is defined. If not, the functions sets <code>APIFY_LOCAL_STORAGE_DIR</code> to <code>./apify_storage</code>
* inside the current working directory. This is to simplify running code examples.
* </li>
- * <li>It invokes the user function passed as the `userFunc` parameter.</li>
+ * <li>It invokes the user function passed as the <code>userFunc</code> parameter.</li>
* <li>If the user function returned a promise, waits for it to resolve.</li>
* <li>If the user function throws an exception or some other error is encountered,
* prints error details to console so that they are stored to the log.</li>
@@ -214,9 +206,9 @@ export const main = (userFunc) => {
* By passing the `waitSecs` option you can reduce the maximum amount of time to wait for the run to finish.
* If the value is less than or equal to zero, the function returns immediately after the run is started.
*
- * The result of the function is an {@linkcode ActorRun} object
+ * The result of the function is an {@link ActorRun} object
* that contains details about the actor run and its output (if any).
- * If the actor run failed, the function fails with {@linkcode ApifyCallError} exception.
+ * If the actor run failed, the function fails with {@link ApifyCallError} exception.
*
* **Example usage:**
*
@@ -233,10 +225,10 @@ export const main = (userFunc) => {
* Either `username/actor-name` or actor ID.
* @param {Object|String|Buffer} [input]
* Input for the actor. If it is an object, it will be stringified to
- * JSON and its content type is set to `application/json; charset=utf-8`.
+ * JSON and its content type set to `application/json; charset=utf-8`.
* Otherwise the `options.contentType` parameter must be provided.
* @param {Object} [options]
- * Object with settings
+ * Object with the settings below:
* @param {String} [options.contentType]
* Content type for the `input`. If not specified,
* `input` is expected to be an object that will be stringified to JSON and content type set to
@@ -246,6 +238,7 @@ export const main = (userFunc) => {
* User API token that is used to run the actor. By default, it is taken from the `APIFY_TOKEN` environment variable.
* @param {Number} [options.memory]
* Memory in megabytes which will be allocated for the new actor run.
+ * If not provided, the run uses memory of the default actor run configuration.
* @param {String} [options.build]
* Tag or number of the actor build to run (e.g. `beta` or `1.2.345`).
* If not provided, the run uses build tag or number from the default actor run configuration (typically `latest`).
@@ -259,29 +252,27 @@ export const main = (userFunc) => {
* @param {Boolean} [options.disableBodyParser=false]
* If `true` then the function will not attempt to parse the
* actor's output and will return it in a raw `Buffer`.
- * @returns {Promise<ActorRun>} Returns a promise that resolves to an instance
- * of {@linkcode ActorRun}. If the actor run fails, the promise is rejected
- * with {@linkcode ApifyCallError}.
+ * @returns {Promise<ActorRun>}
* @throws {ApifyCallError} If the run did not succeed, e.g. if it failed or timed out.
*
* @memberof module:Apify
* @function
* @name call
*/
-export const call = (actId, input, opts = {}) => {
+export const call = (actId, input, options = {}) => {
const { acts, keyValueStores } = apifyClient;
checkParamOrThrow(actId, 'actId', 'String');
- checkParamOrThrow(opts, 'opts', 'Object');
+ checkParamOrThrow(options, 'opts', 'Object');
// Common options.
- const { token } = opts;
+ const { token } = options;
checkParamOrThrow(token, 'token', 'Maybe String');
const defaultOpts = { actId };
if (token) defaultOpts.token = token;
// RunAct() options.
- const { build, memory } = opts;
+ const { build, memory } = options;
const runActOpts = {};
checkParamOrThrow(build, 'build', 'Maybe String');
checkParamOrThrow(memory, 'memory', 'Maybe Number');
@@ -289,18 +280,18 @@ export const call = (actId, input, opts = {}) => {
if (memory) runActOpts.memory = memory;
if (input) {
- input = maybeStringify(input, opts);
+ input = maybeStringify(input, options);
checkParamOrThrow(input, 'input', 'Buffer|String');
- checkParamOrThrow(opts.contentType, 'contentType', 'String');
+ checkParamOrThrow(options.contentType, 'contentType', 'String');
- if (opts.contentType) runActOpts.contentType = addCharsetToContentType(opts.contentType);
+ if (options.contentType) runActOpts.contentType = addCharsetToContentType(options.contentType);
runActOpts.body = input;
}
// GetAct() options.
- const { timeoutSecs, fetchOutput = true } = opts;
- let { waitSecs } = opts;
+ const { timeoutSecs, fetchOutput = true } = options;
+ let { waitSecs } = options;
// TODO: This is bad, timeoutSecs should be used! Fix this and mark as breaking change!!!
// The comments should be * @param {Number} [options.timeoutSecs]
// * Time limit for the finish of the actor run, in seconds.
@@ -315,7 +306,7 @@ export const call = (actId, input, opts = {}) => {
const waitUntil = typeof waitSecs === 'number' ? Date.now() + (waitSecs * 1000) : null;
// GetRecord() options.
- const { disableBodyParser } = opts;
+ const { disableBodyParser } = options;
checkParamOrThrow(disableBodyParser, 'disableBodyParser', 'Maybe Boolean');
// Adds run.output field to given run and returns it.
@@ -362,12 +353,13 @@ export const call = (actId, input, opts = {}) => {
/**
* Represents information about an actor run, as returned by the
- * {@linkcode Apify#call|Apify.call()} function.
+ * [`Apify.call()`](../api/apify#module_Apify.call) function.
* The object is almost equivalent to the JSON response
* of the
- * [Actor run](https://www.apify.com/docs/api/v2#/reference/actors/run-collection/run-actor)
+ * <a href="https://www.apify.com/docs/api/v2#/reference/actors/run-collection/run-actor" target="_blank">Actor run</a>
* Apify API endpoint and extended with certain fields.
- * For more details, see [Runs](https://www.apify.com/docs/actor#run) in Apify actor documentation.
+ * For more details, see
+ * <a href="https://www.apify.com/docs/actor#run" target="_blank">Runs.</a>
*
* @typedef {Object} ActorRun
* @property {String} id
@@ -380,10 +372,11 @@ export const call = (actId, input, opts = {}) => {
* Time when the actor run finished. Contains `null` for running actors.
* @property {String} status
* Status of the run. For possible values, see
- * [Run lifecycle](https://www.apify.com/docs/actor#run-lifecycle) in Apify actor documentation.
+ * <a href="https://www.apify.com/docs/actor#run-lifecycle" target="_blank">Run lifecycle</a>
+ * in Apify actor documentation.
* @property {Object} meta
* Actor run meta-data. For example:
- * ```javascript
+ * ```
* {
* "origin": "API",
* "clientIp": "1.2.3.4",
@@ -392,7 +385,7 @@ export const call = (actId, input, opts = {}) => {
* ```
* @property {Object} stats
* An object containing various actor run statistics. For example:
- * ```javascript
+ * ```
* {
* "inputBodyLen": 22,
* "restartCount": 0,
@@ -402,7 +395,7 @@ export const call = (actId, input, opts = {}) => {
* Beware that object fields might change in future releases.
* @property {Object} options
* Actor run options. For example:
- * ```javascript
+ * ```
* {
* "build": "latest",
* "timeoutSecs": 0,
@@ -412,27 +405,29 @@ export const call = (actId, input, opts = {}) => {
* ```
* @property {String} buildId
* ID of the actor build used for the run. For details, see
- * [Builds](https://www.apify.com/docs/actor#build) in Apify actor documentation.
+ * <a href="https://www.apify.com/docs/actor#build" target="_blank">Builds</a>
+ * in Apify actor documentation.
* @property {String} buildNumber
* Number of the actor build used for the run. For example, `0.0.10`.
* @property {Number} exitCode
* Exit code of the actor run process. It's `null` if actor is still running.
* @property {String} defaultKeyValueStoreId
- * ID of the default key-value store associated with the actor run. See {@linkcode KeyValueStore} for details.
+ * ID of the default key-value store associated with the actor run. See [`KeyValueStore`](../api/keyvaluestore) for details.
* @property {String} defaultDatasetId
- * ID of the default dataset associated with the actor run. See {@linkcode Dataset} for details.
+ * ID of the default dataset associated with the actor run. See [`Dataset`](../api/dataset) for details.
* @property {String} defaultRequestQueueId
- * ID of the default request queue associated with the actor run. See {@linkcode RequestQueue} for details.
+ * ID of the default request queue associated with the actor run. See [`RequestQueue`](../api/requestqueue) for details.
* @property {String} containerUrl
* URL on which the web server running inside actor run's Docker container can be accessed.
- * For more details, see [Container web server](https://www.apify.com/docs/actor#container-web-server)
+ * For more details, see
+ * <a href="https://www.apify.com/docs/actor#container-web-server" target="_blank">Container web server</a>
* in Apify actor documentation.
* @property {Object} output
* Contains output of the actor run. The value is `null` or `undefined` in case the actor is still running,
- * or if you pass `false` to the `fetchOutput` option of {@linkcode Apify#call|Apify.call()}.
+ * or if you pass `false` to the `fetchOutput` option of [`Apify.call()`](../api/apify#module_Apify.call).
*
* For example:
- * ```javascript
+ * ```
* {
* "contentType": "application/json; charset=utf-8",
* "body": {
@@ -444,24 +439,27 @@ export const call = (actId, input, opts = {}) => {
/**
- * Constructs the URL to the Apify Proxy using the specified settings.
+ * Constructs an Apify Proxy URL using the specified settings.
* The proxy URL can be used from Apify actors, web browsers or any other HTTP
* proxy-enabled applications.
*
* For more information, see
- * the <a href="https://my.apify.com/proxy">Apify Proxy</a> page in the app
- * or the <a href="https://www.apify.com/docs/proxy">documentation</a>.
+ * the <a href="https://my.apify.com/proxy" target="_blank">Apify Proxy</a> page in the app
+ * or the <a href="https://www.apify.com/docs/proxy" target="_blank">documentation</a>.
*
- * @param {Object} opts
- * @param {String} opts.password User's password for the proxy.
- * By default, it is taken from the `APIFY_PROXY_PASSWORD` environment variable,
- * which is automatically set by the system when running the actors on the Apify cloud.
- * @param {String[]} [opts.groups] Array of Apify Proxy groups to be used.
- * If not provided, the proxy will select the groups automatically.
- * @param {String} [opts.session] Apify Proxy session identifier to be used by the Chrome browser.
- * All HTTP requests going through the proxy with the same session identifier
- * will use the same target proxy server (i.e. the same IP address).
- * The identifier can only contain the following characters: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
+ * @param {Object} options
+ * Object with the settings below:
+ * @param {String} [options.password] User's password for the proxy.
+ * By default, it is taken from the `APIFY_PROXY_PASSWORD` environment variable,
+ * which is automatically set by the system when running the actors on the Apify cloud,
+ * or when using the <a href="https://github.com/apifytech/apify-cli" target="_blank">Apify CLI</a>
+ * package and the user previously logged in (called `apify login`).
+ * @param {String[]} [options.groups] Array of Apify Proxy groups to be used.
+ * If not provided, the proxy will select the groups automatically.
+ * @param {String} [options.session] Apify Proxy session identifier to be used by the Chrome browser.
+ * All HTTP requests going through the proxy with the same session identifier
+ * will use the same target proxy server (i.e. the same IP address), unless using Residential proxies.
+ * The identifier can only contain the following characters: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
*
* @returns {String} Returns the proxy URL, e.g. `http://auto:[email protected]:8000`.
*
@@ -469,16 +467,16 @@ export const call = (actId, input, opts = {}) => {
* @function
* @name getApifyProxyUrl
*/
-export const getApifyProxyUrl = (opts = {}) => {
+export const getApifyProxyUrl = (options = {}) => {
// For backwards compatibility.
// TODO: remove this when we release v1.0.0
- if (!opts.groups && opts.apifyProxyGroups) {
+ if (!options.groups && options.apifyProxyGroups) {
log.warning('Parameter `apifyProxyGroups` of Apify.getApifyProxyUrl() is deprecated!!! Use `groups` instead!');
- opts.groups = opts.apifyProxyGroups;
+ options.groups = options.apifyProxyGroups;
}
- if (!opts.session && opts.apifyProxySession) {
+ if (!options.session && options.apifyProxySession) {
log.warning('Parameter `apifyProxySession` of Apify.getApifyProxyUrl() is deprecated!!! Use `session` instead!');
- opts.session = opts.apifyProxySession;
+ options.session = options.apifyProxySession;
}
const {
@@ -492,7 +490,7 @@ export const getApifyProxyUrl = (opts = {}) => {
// parameters so we need to override this in error messages.
groupsParamName = 'opts.groups',
sessionParamName = 'opts.session',
- } = opts;
+ } = options;
const getMissingParamErrorMgs = (param, env) => `Apify Proxy ${param} must be provided as parameter or "${env}" environment variable!`;
const throwInvalidProxyValueError = (param) => {
diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.js
index f68c6e4bf..33f4f7a75 100644
--- a/src/autoscaling/autoscaled_pool.js
+++ b/src/autoscaling/autoscaled_pool.js
@@ -21,23 +21,22 @@ const DEFAULT_OPTIONS = {
* The pool only starts new tasks if there is enough free CPU and memory available
* and the Javascript event loop is not blocked.
*
- * The information about the CPU and memory usage is obtained by the `Snapshotter` class,
+ * The information about the CPU and memory usage is obtained by the {@link Snapshotter} class,
* which makes regular snapshots of system resources that may be either local
* or from the Apify cloud infrastructure in case the process is running on the Apify platform.
- * Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the `SystemStatus` class.
+ * Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@link SystemStatus} class.
*
* Before running the pool, you need to implement the following three functions:
- * {@link AutoscaledPool#runTaskFunction|`runTaskFunction()`},
- * {@link AutoscaledPool#isTaskReadyFunction|`isTaskReadyFunction()`} and
- * {@link AutoscaledPool#isFinishedFunction|`isFinishedFunction()`}.
+ * [`runTaskFunction()`](#new_AutoscaledPool_new),
+ * [`isTaskReadyFunction()`](#new_AutoscaledPool_new) and
+ * [`isFinishedFunction()`](#new_AutoscaledPool_new).
*
- * The auto-scaled pool is started by calling the {@link AutoscaledPool#run|`run()`} function.
- * The pool periodically queries the `isTaskReadyFunction()` function
+ * The auto-scaled pool is started by calling the [`run()`](#AutoscaledPool+run) function.
+ * The pool periodically queries the [`isTaskReadyFunction()`](#new_AutoscaledPool_new) function
* for more tasks, managing optimal concurrency, until the function resolves to `false`. The pool then queries
- * the `isFinishedFunction()`. If it resolves to `true`, the run finishes. If it resolves to `false`, it assumes
- * there will be more tasks available later and keeps querying for tasks, until finally both the
- * `isTaskReadyFunction()` and `isFinishedFunction()` functions resolve to `true`. If any of the tasks throws
- * then the `run()` function rejects the promise with an error.
+ * the [`isFinishedFunction()`](#new_AutoscaledPool_new). If it resolves to `true`, the run finishes after all running tasks complete.
+ * If it resolves to `false`, it assumes there will be more tasks available later and keeps periodically querying for tasks.
+ * If any of the tasks throws then the [`run()`](#AutoscaledPool+run) function rejects the promise with an error.
*
* The pool evaluates whether it should start a new task every time one of the tasks finishes
* and also in the interval set by the `options.maybeRunIntervalSecs` parameter.
@@ -64,7 +63,8 @@ const DEFAULT_OPTIONS = {
* await pool.run();
* ```
*
- * @param {Object} options
+ * @param {Object} options All `AutoscaledPool` parameters are passed
+ * via an options object with the following keys.
* @param {Function} options.runTaskFunction
* A function that performs an asynchronous resource-intensive task.
* The function must either be labeled `async` or return a promise.
@@ -72,7 +72,7 @@ const DEFAULT_OPTIONS = {
* @param {Function} options.isTaskReadyFunction
* A function that indicates whether `runTaskFunction` should be called.
* This function is called every time there is free capacity for a new task and it should
- * indicate whether it should start or not by resolving to either `true` or `false.
+ * indicate whether it should start a new task or not by resolving to either `true` or `false.
* Besides its obvious use, it is also useful for task throttling to save resources.
*
* @param {Function} options.isFinishedFunction
@@ -80,7 +80,7 @@ const DEFAULT_OPTIONS = {
* If it resolves to `true` then the pool's run finishes. Being called only
* when there are no tasks being processed means that as long as `isTaskReadyFunction()`
* keeps resolving to `true`, `isFinishedFunction()` will never be called.
- * To abort a run, use the `pool.abort()` method.
+ * To abort a run, use the [`abort()`](#AutoscaledPool+abort) method.
*
* @param {Number} [options.minConcurrency=1]
* Minimum number of tasks running in parallel.
@@ -105,16 +105,14 @@ const DEFAULT_OPTIONS = {
* based on the latest system status. Setting it lower than 1 might have a severe impact on performance.
* We suggest using a value from 5 to 20.
* @param {Number} [options.snapshotterOptions]
- * Options to be passed down to the `Snapshotter` constructor. This is useful for fine-tuning
+ * Options to be passed down to the {@link Snapshotter} constructor. This is useful for fine-tuning
* the snapshot intervals and history.
- * See <a href="https://github.com/apifytech/apify-js/blob/develop/src/autoscaling/snapshotter.js">Snapshotter</a> source code for more details.
* @param {Number} [options.systemStatusOptions]
- * Options to be passed down to the `SystemStatus` constructor. This is useful for fine-tuning
+ * Options to be passed down to the {@link SystemStatus} constructor. This is useful for fine-tuning
* the system status reports. If a custom snapshotter is set in the options, it will be used
* by the pool.
- * See <a href="https://github.com/apifytech/apify-js/blob/develop/src/autoscaling/system_status.js">SystemStatus</a> source code for more details.
*/
-export default class AutoscaledPool {
+class AutoscaledPool {
constructor(options = {}) {
const {
maxConcurrency,
@@ -400,3 +398,5 @@ export default class AutoscaledPool {
if (this.snapshotter) await this.snapshotter.stop();
}
}
+
+export default AutoscaledPool;
diff --git a/src/autoscaling/snapshotter.js b/src/autoscaling/snapshotter.js
index d929d23a4..75132cee9 100644
--- a/src/autoscaling/snapshotter.js
+++ b/src/autoscaling/snapshotter.js
@@ -18,22 +18,23 @@ const DEFAULT_OPTIONS = {
* Creates snapshots of system resources at given intervals and marks the resource
* as either overloaded or not during the last interval. Keeps a history of the snapshots.
* It tracks the following resources: Memory, EventLoop and CPU.
- * The class is used by the {@linkcode AutoscaledPool} class.
+ * The class is used by the {@link AutoscaledPool} class.
*
* There are differences in behavior when running locally and on the Apify platform,
* but those differences are handled internally by the class and do not affect its interface.
*
- * Memory becomes overloaded if its current use exceeds the `minFreeMemoryRatio` option.
+ * Memory becomes overloaded if its current use exceeds the `maxUsedMemoryRatio` option.
* It's computed using the total memory available to the container when running on
* the Apify platform and a quarter of total system memory when running locally.
- * Use of total memory may be overridden by using the `maxBlockedRatio` option.
+ * Max total memory may be overridden by using the `APIFY_MEMORY_MBYTES` environment variable.
*
* Event loop becomes overloaded if it slows down by more than the `maxBlockedMillis` option.
*
* CPU tracking is available only on the Apify platform and the CPU overloaded event is read
* directly off the container and is not configurable.
*
- * @param {Object} options
+ * @param {Object} [options] All `Snapshotter` parameters are passed
+ * via an options object with the following keys:
* @param {Number} [options.eventLoopSnapshotIntervalSecs=0.5]
* Defines the interval of measuring the event loop response time.
* @param {Number} [options.maxBlockedMillis=50]
@@ -41,17 +42,16 @@ const DEFAULT_OPTIONS = {
* Exceeding this limit overloads the event loop.
* @param {Number} [options.memorySnapshotIntervalSecs=1]
* Defines the interval of measuring memory consumption.
- * The measurement itself is resource intensive (25 - 50ms async),
- * therefore, setting this interval below 1 second is not recommended.
+ * The measurement itself is resource intensive (25 - 50ms async).
+ * Therefore, setting this interval below 1 second is not recommended.
* @param {Number} [options.maxUsedMemoryRatio=0.7]
- * Defines the maximum ratio of memory that can be used.
+ * Defines the maximum ratio of total memory that can be used.
* Exceeding this limit overloads the memory.
* @param {Number} [options.snapshotHistorySecs=60]
* Sets the interval in seconds for which a history of resource snapshots
* will be kept. Increasing this to very high numbers will affect performance.
- * @ignore
*/
-export default class Snapshotter {
+class Snapshotter {
constructor(options = {}) {
const {
eventLoopSnapshotIntervalSecs,
@@ -82,7 +82,6 @@ export default class Snapshotter {
/**
* Starts capturing snapshots at configured intervals.
* @return {Promise}
- * @ignore
*/
async start() {
// Ensure max memory is correctly computed.
@@ -105,7 +104,6 @@ export default class Snapshotter {
/**
* Stops all resource capturing.
* @return {Promise}
- * @ignore
*/
async stop() {
betterClearInterval(this.eventLoopInterval);
@@ -119,8 +117,7 @@ export default class Snapshotter {
* Returns a sample of latest memory snapshots, with the size of the sample defined
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
* @param {Number} [sampleDurationMillis]
- * @return {Array} sample
- * @ignore
+ * @return {Array}
*/
getMemorySample(sampleDurationMillis) {
return this._getSample(this.memorySnapshots, sampleDurationMillis);
@@ -130,8 +127,7 @@ export default class Snapshotter {
* Returns a sample of latest event loop snapshots, with the size of the sample defined
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
* @param {Number} [sampleDurationMillis]
- * @return {Array} sample
- * @ignore
+ * @return {Array}
*/
getEventLoopSample(sampleDurationMillis) {
return this._getSample(this.eventLoopSnapshots, sampleDurationMillis);
@@ -141,8 +137,7 @@ export default class Snapshotter {
* Returns a sample of latest CPU snapshots, with the size of the sample defined
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
* @param {Number} [sampleDurationMillis]
- * @return {Array} sample
- * @ignore
+ * @return {Array}
*/
getCpuSample(sampleDurationMillis) {
return this._getSample(this.cpuSnapshots, sampleDurationMillis);
@@ -152,7 +147,7 @@ export default class Snapshotter {
* Finds the latest snapshots by sampleDurationMillis in the provided array.
* @param {Array} snapshots
* @param {Number} [sampleDurationMillis]
- * @return {Array} sample
+ * @return {Array}
* @ignore
*/
_getSample(snapshots, sampleDurationMillis) { // eslint-disable-line class-methods-use-this
@@ -262,3 +257,5 @@ export default class Snapshotter {
snapshots.splice(0, oldCount);
}
}
+
+export default Snapshotter;
diff --git a/src/autoscaling/system_status.js b/src/autoscaling/system_status.js
index ce36e2662..74c118ac1 100644
--- a/src/autoscaling/system_status.js
+++ b/src/autoscaling/system_status.js
@@ -11,26 +11,31 @@ const DEFAULT_OPTIONS = {
};
/**
- * Provides a simple interface to reading system status from a Snapshotter instance.
- * It only exposes two functions `getCurrentStatus()` and `getHistoricalStatus()`.
+ * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
+ * It only exposes two functions [`getCurrentStatus()`](#SystemStatus+getCurrentStatus)
+ * and [`getHistoricalStatus()`](#SystemStatus+getHistoricalStatus).
* The system status is calculated using a weighted average of overloaded
* messages in the snapshots, with the weights being the time intervals
* between the snapshots. Each resource is calculated separately
* and the system is overloaded whenever at least one resource is overloaded.
- * The class is used by the {@linkcode AutoscaledPool} class.
+ * The class is used by the {@link AutoscaledPool} class.
*
- * `getCurrentStatus()` returns a boolean that represents the current status of the system.
+ * [`getCurrentStatus()`](#SystemStatus+getCurrentStatus)
+ * returns a boolean that represents the current status of the system.
* The length of the current timeframe in seconds is configurable
- * by the currentHistorySecs option and represents the max age
+ * by the `currentHistorySecs` option and represents the max age
* of snapshots to be considered for the calculation.
*
- * `getHistoricalStatus()` returns a boolean that represents the long-term status
+ * [`getHistoricalStatus()`](#SystemStatus+getHistoricalStatus)
+ * returns a boolean that represents the long-term status
* of the system. It considers the full snapshot history available
- * in the Snapshotter instance.
+ * in the {@link Snapshotter} instance.
*
- * @param {Object} options
+ * @param {Object} [options] All `SystemStatus` parameters are passed
+ * via an options object with the following keys:
* @param {Number} [options.currentHistorySecs=5]
- * Defines max age of snapshots used in the `isOk()` measurement.
+ * Defines max age of snapshots used in the
+ * [`getCurrentStatus()`](#SystemStatus+getCurrentStatus) measurement.
* @param {Number} [options.maxMemoryOverloadedRatio=0.2]
* Sets the maximum ratio of overloaded snapshots in a memory sample.
* If the sample exceeds this ratio, the system will be overloaded.
@@ -40,9 +45,8 @@ const DEFAULT_OPTIONS = {
* @param {Number} [options.maxCpuOverloadedRatio=0.1]
* Sets the maximum ratio of overloaded snapshots in a CPU sample.
* If the sample exceeds this ratio, the system will be overloaded.
- * @ignore
*/
-export default class SystemStatus {
+class SystemStatus {
constructor(options = {}) {
const {
currentHistorySecs,
@@ -67,28 +71,49 @@ export default class SystemStatus {
}
/**
- * Returns true if the system has not been overloaded in the last
- * currentHistorySecs seconds, otherwise returns false.
- * @return {Boolean}
- * @ignore
+ * Returns an object with the following structure:
+ *
+ * ```javascript
+ * {
+ * isSystemIdle: Boolean,
+ * memInfo: Object,
+ * eventLoopInfo: Object,
+ * cpuInfo: Object
+ * }
+ * ```
+ *
+ * Where the `isSystemIdle` property is set to `false` if the system
+ * has been overloaded in the last `options.currentHistorySecs` seconds,
+ * and `true` otherwise.
+ * @return {Object}
*/
getCurrentStatus() {
return this._isSystemIdle(this.currentHistorySecs);
}
/**
- * Returns true if the system has not been overloaded in the full
- * history of the snapshotter (which is configurable in the snapshotter).
- * @return {Boolean}
- * @ignore
+ * Returns an object with the following structure:
+ *
+ * ```javascript
+ * {
+ * isSystemIdle: Boolean,
+ * memInfo: Object,
+ * eventLoopInfo: Object,
+ * cpuInfo: Object
+ * }
+ * ```
+ *
+ * Where the `isSystemIdle` property is set to `false` if the system
+ * has been overloaded in the full history of the {@link Snapshotter}
+ * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
+ * @return {Object}
*/
getHistoricalStatus() {
return this._isSystemIdle();
}
/**
- * Returns true if the system has been overloaded
- * in the last sampleDurationMillis.
+ * Returns a system status object.
*
* @param {Number} [sampleDurationMillis]
* @return {Boolean}
@@ -172,3 +197,5 @@ export default class SystemStatus {
};
}
}
+
+export default SystemStatus;
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index ae926b7d3..188fcf2f6 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -16,26 +16,27 @@ const DEFAULT_OPTIONS = {
};
/**
- * Provides a simple framework for the parallel crawling of web pages,
+ * Provides a simple framework for parallel crawling of web pages,
* whose URLs are fed either from a static list
* or from a dynamic queue of URLs.
*
- * `BasicCrawler` invokes the user-provided `handleRequestFunction` for each {@link Request|`Request`}
- * object, which corresponds to a single URL to crawl.
- * The `Request` objects are fed from the {@link RequestList|`RequestList`} or {@link RequestQueue|`RequestQueue`}
- * instances provided by the `requestList` or `requestQueue` constructor options, respectively.
+ * `BasicCrawler` invokes the user-provided [`handleRequestFunction()`](#new_BasicCrawler_new)
+ * for each {@link Request} object, which represents a single URL to crawl.
+ * The {@link Request} objects are fed from the {@link RequestList} or the {@link RequestQueue}
+ * instances provided by the [`requestList`](#new_BasicCrawler_new) or [`requestQueue`](#new_BasicCrawler_new)
+ * constructor options, respectively.
*
- * If both `requestList` and `requestQueue` is used, the instance first
- * processes URLs from the `RequestList` and automatically enqueues all of them to `RequestQueue` before it starts
- * their processing. This ensures that a single URL is not crawled multiple times.
+ * If both [`requestList`](#new_BasicCrawler_new) and [`requestQueue`](#new_BasicCrawler_new) options are used,
+ * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
+ * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
*
- * The crawler finishes if there are no more `Request` objects to crawl.
+ * The crawler finishes if there are no more {@link Request} objects to crawl.
*
- * New requests are only launched if there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool|`AutoscaledPool`} class.
- * All `AutoscaledPool` configuration options can be passed to the `autoscaledPoolOptions` parameter
- * of the `CheerioCrawler` constructor.
- * For user convenience, the `minConcurrency` and `maxConcurrency` options are available directly in the constructor.
+ * New requests are only dispatched when there is enough free CPU and memory available,
+ * using the functionality provided by the {@link AutoscaledPool} class.
+ * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
+ * parameter of the `BasicCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+ * {@link AutoscaledPool} options are available directly in the `BasicCrawler` constructor.
*
* **Example usage:**
*
@@ -67,46 +68,47 @@ const DEFAULT_OPTIONS = {
* await crawler.run();
* ```
*
- * @param {Object} options
+ * @param {Object} options All `BasicCrawler` parameters are passed
+ * via an options object with the following keys:
* @param {Function} options.handleRequestFunction
* User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
*
- * The function that receives an object as argument, with the following field:
- *
- * <ul>
- * <li>`request`: the {@link Request|`Request`} object representing the URL to crawl</li>
- * </ul>
- *
+ * The function receives the following object as an argument:
+ * ```
+ * {
+ * request: Request
+ * }
+ * ```
+ * With the {@link Request} object representing the URL to crawl.
* The function must return a promise.
* @param {RequestList} options.requestList
* Static list of URLs to be processed.
- * Either `RequestList` or `RequestQueue` must be provided.
+ * Either {@link RequestList} or {@link RequestQueue} must be provided.
* @param {RequestQueue} options.requestQueue
* Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
- * Either RequestList or RequestQueue must be provided.
+ * Either {@link RequestList} or {@link RequestQueue} must be provided.
* @param {Function} [options.handleFailedRequestFunction]
- * Function that handles requests that failed more then `option.maxRequestRetries` times.
- * See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/basic_crawler.js#L11">GitHub</a> for default behavior.
+ * Function that handles requests that failed more then `options.maxRequestRetries` times.
+ * See source code on
+ * <a href="https://github.com/apifytech/apify-js/blob/master/src/basic_crawler.js#L11" target="_blank">GitHub</a>
+ * for default behavior.
* @param {Number} [options.maxRequestRetries=3]
- * How many times the request is retried if `handleRequestFunction` failed.
+ * Indicates how many times the request is retried if [`handleRequestFunction()`](#new_BasicCrawler_new) fails.
* @param {Number} [options.maxRequestsPerCrawl]
* Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
* Always set this value in order to prevent infinite loops in misconfigured crawlers.
* Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
* @param {Object} [options.autoscaledPoolOptions]
- * Custom options passed to the underlying {@link AutoscaledPool|`AutoscaledPool`} instance constructor.
+ * Custom options passed to the underlying {@link AutoscaledPool} constructor.
* Note that the `runTaskFunction`, `isTaskReadyFunction` and `isFinishedFunction` options
* are provided by `BasicCrawler` and cannot be overridden.
* @param {Object} [options.minConcurrency=1]
- * Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding `AutoscaledPool` option.
+ * Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
* @param {Object} [options.maxConcurrency=1000]
- * Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding `AutoscaledPool` option.
- *
- * @see {@link CheerioCrawler}
- * @see {@link PuppeteerCrawler}
+ * Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
*/
-export default class BasicCrawler {
- constructor(opts) {
+class BasicCrawler {
+ constructor(options) {
const {
requestList,
requestQueue,
@@ -119,18 +121,18 @@ export default class BasicCrawler {
// AutoscaledPool shorthands
minConcurrency,
maxConcurrency,
- } = _.defaults(opts, DEFAULT_OPTIONS);
+ } = _.defaults(options, DEFAULT_OPTIONS);
- checkParamPrototypeOrThrow(requestList, 'opts.requestList', RequestList, 'Apify.RequestList', true);
- checkParamPrototypeOrThrow(requestQueue, 'opts.requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue', true);
- checkParamOrThrow(handleRequestFunction, 'opts.handleRequestFunction', 'Function');
- checkParamOrThrow(handleFailedRequestFunction, 'opts.handleFailedRequestFunction', 'Function');
- checkParamOrThrow(maxRequestRetries, 'opts.maxRequestRetries', 'Number');
- checkParamOrThrow(maxRequestsPerCrawl, 'opts.maxRequestsPerCrawl', 'Maybe Number');
- checkParamOrThrow(autoscaledPoolOptions, 'opts.autoscaledPoolOptions', 'Object');
+ checkParamPrototypeOrThrow(requestList, 'options.requestList', RequestList, 'Apify.RequestList', true);
+ checkParamPrototypeOrThrow(requestQueue, 'options.requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue', true);
+ checkParamOrThrow(handleRequestFunction, 'options.handleRequestFunction', 'Function');
+ checkParamOrThrow(handleFailedRequestFunction, 'options.handleFailedRequestFunction', 'Function');
+ checkParamOrThrow(maxRequestRetries, 'options.maxRequestRetries', 'Number');
+ checkParamOrThrow(maxRequestsPerCrawl, 'options.maxRequestsPerCrawl', 'Maybe Number');
+ checkParamOrThrow(autoscaledPoolOptions, 'options.autoscaledPoolOptions', 'Object');
if (!requestList && !requestQueue) {
- throw new Error('At least one of the parameters "opts.requestList" and "opts.requestQueue" must be provided!');
+ throw new Error('At least one of the parameters "options.requestList" and "options.requestQueue" must be provided!');
}
this.requestList = requestList;
@@ -325,3 +327,5 @@ export default class BasicCrawler {
return this.handleFailedRequestFunction({ request, error }); // This function prints an error message.
}
}
+
+export default BasicCrawler;
diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.js
index 9bbff394e..b25924491 100644
--- a/src/cheerio_crawler.js
+++ b/src/cheerio_crawler.js
@@ -19,31 +19,32 @@ const DEFAULT_OPTIONS = {
/**
* Provides a framework for the parallel crawling of web pages using plain HTTP requests and
- * [cheerio](https://www.npmjs.com/package/cheerio) HTML parser.
+ * <a href="https://www.npmjs.com/package/cheerio" target="_blank">cheerio</a> HTML parser.
*
* `CheerioCrawler` downloads each URL using a plain HTTP request,
- * parses the HTML content using cheerio and then
- * invokes the user-provided `handlePageFunction` to extract page data
- * using a [jQuery](https://jquery.com/)-like interface to parsed HTML DOM.
+ * parses the HTML content using <a href="https://www.npmjs.com/package/cheerio" target="_blank">cheerio</a>
+ * and then invokes the user-provided [`handlePageFunction()`](#new_CheerioCrawler_new) to extract page data
+ * using a <a href="https://jquery.com/" target="_blank">jQuery</a>-like interface to the parsed HTML DOM.
*
- * The source URLs are represented using `Request` objects that
- * are fed from the {@link RequestList|`RequestList`} or {@link RequestQueue|`RequestQueue`}
- * instances provided by the `requestList` or `requestQueue` constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from
+ * {@link RequestList} or {@link RequestQueue} instances provided by the [`requestList`](#new_CheerioCrawler_new)
+ * or [`requestQueue`](#new_CheerioCrawler_new) constructor options, respectively.
*
- * If both `requestList` and `requestQueue` is used, the instance first
- * processes URLs from the `RequestList` and automatically enqueues all of them to `RequestQueue` before it starts
- * their processing. This ensures that a single URL is not crawled multiple times.
+ * If both [`requestList`](#new_CheerioCrawler_new) and [`requestQueue`](#new_CheerioCrawler_new) are used,
+ * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
+ * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
*
- * The crawler finishes if there are no more `Request` objects to crawl.
+ * The crawler finishes when there are no more {@link Request} objects to crawl.
*
- * By default, `CheerioCrawler` downloads HTML using the [request-promise](https://www.npmjs.com/package/request-promise) NPM package.
+ * By default, `CheerioCrawler` downloads HTML using the
+ * <a href="https://www.npmjs.com/package/request-promise" target="_blank">request-promise</a> NPM package.
* You can override this behavior by setting the `requestFunction` option.
*
- * New requests are only started if there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool|`AutoscaledPool`} class.
- * All `AutoscaledPool` configuration options can be passed to the `autoscaledPoolOptions` parameter
- * of the `CheerioCrawler` constructor.
- * For user convenience, the `minConcurrency` and `maxConcurrency` options are available directly.
+ * New requests are only dispatched when there is enough free CPU and memory available,
+ * using the functionality provided by the {@link AutoscaledPool} class.
+ * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
+ * parameter of the `CheerioCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+ * {@link AutoscaledPool} options are available directly in the `CheerioCrawler` constructor.
*
* **Example usage:**
*
@@ -81,27 +82,28 @@ const DEFAULT_OPTIONS = {
* await crawler.run();
* ```
*
- * @param {Object} options
+ * @param {Object} options All `CheerioCrawler` parameters are passed
+ * via an options object with the following keys:
* @param {Function} options.handlePageFunction
* User-provided function that performs the logic of the crawler. It is called for each page
* loaded and parsed by the crawler.
*
- * The function that receives an object as argument, with the following three fields:
- *
- * <ul>
- * <li>`$`: the Cheerio object</li>
- * <li>`html`: the raw HTML</li>
- * <li>`request`: the {@link Request|`Request`} object representing the URL to crawl</li>
- * </ul>
- *
+ * The function receives the following object as an argument:
+ * ```
+ * {
+ * $: Cheerio, // the Cheerio object with parsed HTML
+ * html: String // the raw HTML of the page
+ * request: Request
+ * }
+ * ```
+ * With the {@link Request} object representing the URL to crawl.
* If the function returns a promise, it is awaited.
- *
* @param {RequestList} options.requestList
* Static list of URLs to be processed.
- * Either RequestList or RequestQueue must be provided.
+ * Either {@link RequestList} or {@link RequestQueue} must be provided.
* @param {RequestQueue} options.requestQueue
* Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
- * Either RequestList or RequestQueue must be provided.
+ * Either {@link RequestList} or {@link RequestQueue} must be provided.
* @param {Function} [options.requestFunction]
* Overrides the function that performs the HTTP request to get the raw HTML needed for Cheerio.
* See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/cheerio_crawler.js#L264">GitHub</a> for default behavior.
@@ -111,30 +113,27 @@ const DEFAULT_OPTIONS = {
* Timeout in which the function passed as `options.requestFunction` needs to finish, given in seconds.
* @param {Boolean} [options.ignoreSslErrors=false]
* If set to true, SSL certificate errors will be ignored. This is dependent on using the default
- * request function. If using a custom request function, user needs to implement this functionality.
+ * request function. If using a custom `options.requestFunction`, user needs to implement this functionality.
* @param {Function} [options.handleFailedRequestFunction]
* Function that handles requests that failed more then `option.maxRequestRetries` times.
* See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/cheerio_crawler.js#L13">GitHub</a> for default behavior.
* @param {Number} [options.maxRequestRetries=3]
- * How many times the request is retried if either `requestFunction` or `handlePageFunction` failed.
+ * Indicates how many times the request is retried if either `requestFunction` or `handlePageFunction` fails.
* @param {Number} [options.maxRequestsPerCrawl]
* Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
* Always set this value in order to prevent infinite loops in misconfigured crawlers.
* Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
* @param {Object} [options.autoscaledPoolOptions]
- * Custom options passed to the underlying {@link AutoscaledPool|`AutoscaledPool`} instance constructor.
+ * Custom options passed to the underlying {@link AutoscaledPool} constructor.
* Note that the `runTaskFunction`, `isTaskReadyFunction` and `isFinishedFunction` options
* are provided by `CheerioCrawler` and cannot be overridden.
* @param {Object} [options.minConcurrency=1]
- * Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding `AutoscaledPool` option.
+ * Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
* @param {Object} [options.maxConcurrency=1000]
- * Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding `AutoscaledPool` option.
- *
- * @see {@link BasicCrawler}
- * @see {@link PuppeteerCrawler}
+ * Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
*/
-export default class CheerioCrawler {
- constructor(opts = {}) {
+class CheerioCrawler {
+ constructor(options = {}) {
const {
requestFunction,
handlePageFunction,
@@ -153,13 +152,13 @@ export default class CheerioCrawler {
maxRequestsPerCrawl,
handleFailedRequestFunction,
autoscaledPoolOptions,
- } = _.defaults(opts, DEFAULT_OPTIONS);
+ } = _.defaults(options, DEFAULT_OPTIONS);
- checkParamOrThrow(handlePageFunction, 'opts.handlePageFunction', 'Function');
- checkParamOrThrow(requestFunction, 'opts.requestFunction', 'Maybe Function');
- checkParamOrThrow(requestTimeoutSecs, 'opts.requestTimeoutSecs', 'Number');
- checkParamOrThrow(handlePageTimeoutSecs, 'opts.handlePageTimeoutSecs', 'Number');
- checkParamOrThrow(ignoreSslErrors, 'opts.ignoreSslErrors', 'Boolean');
+ checkParamOrThrow(handlePageFunction, 'options.handlePageFunction', 'Function');
+ checkParamOrThrow(requestFunction, 'options.requestFunction', 'Maybe Function');
+ checkParamOrThrow(requestTimeoutSecs, 'options.requestTimeoutSecs', 'Number');
+ checkParamOrThrow(handlePageTimeoutSecs, 'options.handlePageTimeoutSecs', 'Number');
+ checkParamOrThrow(ignoreSslErrors, 'options.ignoreSslErrors', 'Boolean');
this.ignoreSslErrors = ignoreSslErrors;
@@ -271,3 +270,5 @@ export default class CheerioCrawler {
});
}
}
+
+export default CheerioCrawler;
diff --git a/src/dataset.js b/src/dataset.js
index 7da3e046c..e8e7bf10d 100644
--- a/src/dataset.js
+++ b/src/dataset.js
@@ -96,8 +96,8 @@ export const chunkBySize = (items, limitBytes) => {
};
/**
- * Represents one page of data items from the {@linkcode Dataset}.
- * For more details, see {@linkcode Dataset#getData|Dataset.getData()}.
+ * Represents one page of data items from the [`Dataset`](../api/dataset).
+ * For more details, see [`Dataset.getData()`](../api/dataset#Dataset+getData).
*
* @typedef {Object} PaginationList
* @property {Array} items
@@ -120,22 +120,22 @@ export const chunkBySize = (items, limitBytes) => {
* Typically it is used to store crawling results.
*
* Do not instantiate this class directly, use the
- * {@link Apify#openDataset|`Apify.openDataset()`} function instead.
+ * [`Apify.openDataset()`](apify#module_Apify.openDataset) function instead.
*
* `Dataset` stores its data either on local disk or in the Apify cloud,
- * depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variable is set.
+ * depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.
*
* If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in
* the local directory in the following files:
* ```
- * [APIFY_LOCAL_STORAGE_DIR]/datasets/[DATASET_ID]/[INDEX].json
+ * {APIFY_LOCAL_STORAGE_DIR}/datasets/{DATASET_ID}/{INDEX}.json
* ```
- * Note that `[DATASET_ID]` is the name or ID of the dataset. The default dataset has ID `default`,
+ * Note that `{DATASET_ID}` is the name or ID of the dataset. The default dataset has ID: `default`,
* unless you override it by setting the `APIFY_DEFAULT_DATASET_ID` environment variable.
- * Each dataset item is stored as a separate JSON file, where `[INDEX]` is a zero-based index of the item in the dataset.
+ * Each dataset item is stored as a separate JSON file, where `{INDEX}` is a zero-based index of the item in the dataset.
*
- * If the `APIFY_TOKEN` environment variable is provided instead, the data is stored
- * in the [Apify Dataset](https://www.apify.com/docs/storage#dataset) cloud storage.
+ * If the `APIFY_TOKEN` environment variable is provided instead, the data are stored
+ * in the <a href="https://www.apify.com/docs/storage#dataset" target="_blank">Apify Dataset</a> cloud storage.
*
* **Example usage:**
*
@@ -174,7 +174,7 @@ export class Dataset {
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
* otherwise the actor process might finish before the data is stored!
*
- * The size of the data is limited by the receiving API and therefore `pushData` will only
+ * The size of the data is limited by the receiving API and therefore `pushData()` will only
* allow objects whose JSON representation is smaller than 9MB. When an array is passed,
* none of the included objects
* may be larger than 9MB, but the array itself may be of any size.
@@ -183,14 +183,14 @@ export class Dataset {
* chunks the array into separate items and pushes them sequentially.
* The chunking process is stable (keeps order of data), but it does not provide a transaction
* safety mechanism. Therefore, in the event of an uploading error (after several automatic retries),
- * the function's promise will reject and the dataset will be left in a state where some of
+ * the function's Promise will reject and the dataset will be left in a state where some of
* the items have already been saved to the dataset while other items from the source array were not.
* To overcome this limitation, the developer may, for example, read the last item saved in the dataset
* and re-attempt the save of the data from this item onwards to prevent duplicates.
*
* @param {Object|Array} data Object or array of objects containing data to be stored in the default dataset.
* The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
- * @returns {Promise} Returns a promise that resolves once the data is saved.
+ * @return {Promise}
*/
pushData(data) {
checkParamOrThrow(data, 'data', 'Array | Object');
@@ -223,16 +223,17 @@ export class Dataset {
/**
* Returns items in the dataset based on the provided parameters.
*
- * If format is `json` then the function doesn't return an array of records but {@linkcode PaginationList} instead.
+ * If format is `json` then the function doesn't return an array of records but {@link PaginationList} instead.
*
- * @param {Object} options
+ * @param {Object} [options] All `getData()` parameters are passed
+ * via an options object with the following keys:
* @param {String} [options.format='json']
* Format of the items, possible values are: `json`, `csv`, `xlsx`, `html`, `xml` and `rss`.
* @param {Number} [options.offset=0]
* Number of array elements that should be skipped at the start.
* @param {Number} [options.limit=250000]
* Maximum number of array elements to return.
- * @param {Boolean} [options.desc]
+ * @param {Boolean} [options.desc=false]
* If `true` then the objects are sorted by `createdAt` in descending order.
* Otherwise they are sorted in ascending order.
* @param {Array} [options.fields]
@@ -240,30 +241,30 @@ export class Dataset {
* @param {String} [options.unwind]
* Specifies a name of the field in the result objects that will be used to unwind the resulting objects.
* By default, the results are returned as they are.
- * @param {Boolean} [options.disableBodyParser]
+ * @param {Boolean} [options.disableBodyParser=false]
* If `true` then response from API will not be parsed.
- * @param {Number} [options.attachment]
+ * @param {Boolean} [options.attachment=false]
* If `true` then the response will define the `Content-Disposition: attachment` HTTP header, forcing a web
* browser to download the file rather than to display it. By default, this header is not present.
* @param {String} [options.delimiter=',']
* A delimiter character for CSV files, only used if `format` is `csv`.
* You might need to URL-encode the character (e.g. use `%09` for tab or `%3B` for semicolon).
- * @param {Number} [options.bom]
+ * @param {Boolean} [options.bom]
* All responses are encoded in UTF-8 encoding. By default, the CSV files are prefixed with the UTF-8 Byte
* Order Mark (BOM), while JSON, JSONL, XML, HTML and RSS files are not. If you want to override this default
* behavior, set `bom` option to `true` to include the BOM, or set `bom` to `false` to skip it.
- * @param {String} [options.xmlRoot]
+ * @param {String} [options.xmlRoot='results']
* Overrides the default root element name of the XML output. By default, the root element is `results`.
- * @param {String} [options.xmlRow]
+ * @param {String} [options.xmlRow='page']
* Overrides the default element name that wraps each page or page function result object in XML output.
* By default, the element name is `page` or `result`, depending on the value of the `simplified` option.
- * @param {Number} [options.skipHeaderRow]
- * If set to `1` then header row in csv format is skipped.
+ * @param {Boolean} [options.skipHeaderRow=false]
+ * If set to `true` then header row in CSV format is skipped.
* @return {Promise}
*/
- getData(opts = {}) {
+ getData(options = {}) {
const { datasetId } = this;
- const params = Object.assign({ datasetId }, opts);
+ const params = Object.assign({ datasetId }, options);
return datasets.getItems(params);
}
@@ -271,7 +272,8 @@ export class Dataset {
/**
* Returns an object containing general information about the dataset.
*
- * @example
+ * **Example:**
+ * ```
* {
* "id": "WkzbQMuFYuamGv3YF",
* "name": "d7b9MDYsbtX5L7XAj",
@@ -281,39 +283,40 @@ export class Dataset {
* "accessedAt": "2015-12-14T08:36:13.202Z",
* "itemsCount": 0
* }
+ * ```
*
- * @param {Object} opts
- * @returns {Promise}
+ * @returns {Promise<Object>}
*/
- getInfo(opts = {}) {
+ getInfo(options = {}) {
const { datasetId } = this;
- const params = Object.assign({ datasetId }, opts);
+ const params = Object.assign({ datasetId }, options);
return datasets.getDataset(params);
}
/**
- * Iterates over dataset items, yielding each in turn to an `iteratee` function.
- * Each invocation of `iteratee` is called with three arguments: `(element, index)`.
+ * Iterates over dataset items, yielding each in turn to an `iteratee()` function.
+ * Each invocation of `iteratee()` is called with two arguments: `(element, index)`.
*
- * If `iteratee` returns a Promise then it is awaited before a next call.
+ * If the `iteratee()` returns a Promise then it is awaited before a next call.
*
* @param {Function} iteratee
- * @param {Opts} opts
- * @param {Number} [options.offset=0] - Number of array elements that should be skipped at the start.
- * @param {Number} [options.desc] - If `1` then the objects are sorted by `createdAt` in descending order.
- * @param {Array} [options.fields] - If provided then returned objects will only contain specified keys
- * @param {String} [options.unwind] - If provided then objects will be unwound based on provided field.
- * @param {Number} [options.limit=250000] - How many items to load in one request.
- * @param {Number} index [description]
- * @return {Promise<undefined>}
+ * @param {Object} [options] All `forEach()` parameters are passed
+ * via an options object with the following keys:
+ * @param {Number} [options.offset=0] Number of array elements that should be skipped at the start.
+ * @param {Boolean} [options.desc=false] If `true` then the objects are sorted by `createdAt` in descending order.
+ * @param {Array} [options.fields] If provided then returned objects will only contain specified keys.
+ * @param {String} [options.unwind] If provided then objects will be unwound based on provided field.
+ * @param {Number} [options.limit=250000] How many items to load in one request.
+ * @param {Number} [index=0] Controls the initial index number passed to the `iteratee()`.
+ * @return {Promise}
*/
- forEach(iteratee, opts = {}, index = 0) {
- if (!opts.offset) opts.offset = 0;
- if (opts.format && opts.format !== 'json') throw new Error('Dataset.forEach/map/reduce() support only a "json" format.');
+ forEach(iteratee, options = {}, index = 0) {
+ if (!options.offset) options.offset = 0;
+ if (options.format && options.format !== 'json') throw new Error('Dataset.forEach/map/reduce() support only a "json" format.');
return this
- .getData(opts)
+ .getData(options)
.then(({ items, total, limit, offset }) => {
return Promise
.mapSeries(items, item => iteratee(item, index++))
@@ -322,7 +325,7 @@ export class Dataset {
if (newOffset >= total) return undefined;
- const newOpts = Object.assign({}, opts, {
+ const newOpts = Object.assign({}, options, {
offset: newOffset,
});
@@ -332,22 +335,22 @@ export class Dataset {
}
/**
- * Produces a new array of values by mapping each value in list through a transformation function (`iteratee`).
- * Each invocation of `iteratee` is called with three arguments: `(element, index)`.
+ * Produces a new array of values by mapping each value in list through a transformation function `iteratee()`.
+ * Each invocation of `iteratee()` is called with two arguments: `(element, index)`.
*
* If `iteratee` returns a `Promise` then it's awaited before a next call.
*
* @param {Function} iteratee
- * @param {Opts} opts
- * @param {Number} [options.offset=0] - Number of array elements that should be skipped at the start.
- * @param {Number} [options.desc] - If 1 then the objects are sorted by createdAt in descending order.
- * @param {Array} [options.fields] - If provided then returned objects will only contain specified keys
- * @param {String} [options.unwind] - If provided then objects will be unwound based on provided field.
- * @param {Number} [options.limit=250000] - How many items to load in one request.
- * @param {Number} index [description]
+ * @param {Object} options All `map()` parameters are passed
+ * via an options object with the following keys:
+ * @param {Number} [options.offset=0] Number of array elements that should be skipped at the start.
+ * @param {Boolean} [options.desc=false] If `true` then the objects are sorted by createdAt in descending order.
+ * @param {Array} [options.fields] If provided then returned objects will only contain specified keys
+ * @param {String} [options.unwind] If provided then objects will be unwound based on provided field.
+ * @param {Number} [options.limit=250000] How many items to load in one request.
* @return {Promise<Array>}
*/
- map(iteratee, opts) {
+ map(iteratee, options) {
const result = [];
const wrappedFunc = (item, index) => {
@@ -358,33 +361,33 @@ export class Dataset {
};
return this
- .forEach(wrappedFunc, opts)
+ .forEach(wrappedFunc, options)
.then(() => result);
}
/**
- * Boils down a list of values into a single value.
+ * Reduces a list of values down to a single value.
*
- * Memo is the initial state of the reduction, and each successive step of it should be returned by `iteratee`.
- * The `iteratee` is passed three arguments: the `memo`, then the value and index of the iteration.
+ * Memo is the initial state of the reduction, and each successive step of it should be returned by `iteratee()`.
+ * The `iteratee()` is passed three arguments: the `memo`, then the `value` and `index` of the iteration.
*
- * If no `memo` is passed to the initial invocation of reduce, the `iteratee` is not invoked on the first element of the list.
- * The first element is instead passed as the memo in the invocation of the `iteratee` on the next element in the list.
+ * If no `memo` is passed to the initial invocation of reduce, the `iteratee()` is not invoked on the first element of the list.
+ * The first element is instead passed as the memo in the invocation of the `iteratee()` on the next element in the list.
*
- * If `iteratee` returns a `Promise` then it's awaited before a next call.
+ * If `iteratee()` returns a `Promise` then it's awaited before a next call.
*
* @param {Function} iteratee
- * @param {*} memo
- * @param {Opts} opts
- * @param {Number} [options.offset=0] - Number of array elements that should be skipped at the start.
- * @param {Number} [options.desc] - If 1 then the objects are sorted by createdAt in descending order.
- * @param {Array} [options.fields] - If provided then returned objects will only contain specified keys
- * @param {String} [options.unwind] - If provided then objects will be unwound based on provided field.
- * @param {Number} [options.limit=250000] - How many items to load in one request.
- * @param {Number} index [description]
+ * @param {*} memo Initial state of the reduction.
+ * @param {Object} options All `reduce()` parameters are passed
+ * via an options object with the following keys:
+ * @param {Number} [options.offset=0] Number of array elements that should be skipped at the start.
+ * @param {Boolean} [options.desc=false] If `true` then the objects are sorted by createdAt in descending order.
+ * @param {Array} [options.fields] If provided then returned objects will only contain specified keys
+ * @param {String} [options.unwind] If provided then objects will be unwound based on provided field.
+ * @param {Number} [options.limit=250000] How many items to load in one request.
* @return {Promise<*>}
*/
- reduce(iteratee, memo, opts) {
+ reduce(iteratee, memo, options) {
let currentMemo = memo;
const wrappedFunc = (item, index) => {
@@ -401,7 +404,7 @@ export class Dataset {
};
return this
- .forEach(wrappedFunc, opts)
+ .forEach(wrappedFunc, options)
.then(() => currentMemo);
}
@@ -622,22 +625,20 @@ const getOrCreateDataset = (datasetIdOrName) => {
/**
- * Opens a dataset and returns a promise resolving to an instance of the {@linkcode Dataset} class.
+ * Opens a dataset and returns a promise resolving to an instance of the {@link Dataset} class.
*
* Datasets are used to store structured data where each object stored has the same attributes,
* such as online store products or real estate offers.
* The actual data is stored either on the local filesystem or in the cloud.
*
- * For more details and code examples, see the {@linkcode Dataset} class.
+ * For more details and code examples, see the {@link Dataset} class.
*
* @param {string} [datasetIdOrName]
* ID or name of the dataset to be opened. If `null` or `undefined`,
* the function returns the default dataset associated with the actor run.
* @returns {Promise<Dataset>}
- * Returns a promise that resolves to an instance of the `Dataset` class.
* @memberof module:Apify
* @name openDataset
- * @instance
* @function
*/
export const openDataset = (datasetIdOrName) => {
@@ -650,9 +651,9 @@ export const openDataset = (datasetIdOrName) => {
};
/**
- * Stores an object or an array of objects to the default {@linkcode Dataset} of the current actor run.
+ * Stores an object or an array of objects to the default {@link Dataset} of the current actor run.
*
- * This is just a convenient shortcut for {@link Dataset#pushData|`Dataset.pushData()`}.
+ * This is just a convenient shortcut for [`dataset.pushData()`](dataset#Dataset+pushData).
* For example, calling the following code:
* ```javascript
* await Apify.pushData({ myValue: 123 });
@@ -664,19 +665,17 @@ export const openDataset = (datasetIdOrName) => {
* await dataset.pushData({ myValue: 123 });
* ```
*
- * For more information, see {@link Apify.openDataset|`Apify.openDataset()`} and {@link Dataset#pushData|`Dataset.pushData()`}
+ * For more information, see [`Apify.openDataset()`](apify#module_Apify.openDataset) and [`dataset.pushData()`](dataset#Dataset+pushData)
*
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
- * otherwise the actor process might finish before the data is stored!
+ * otherwise the actor process might finish before the data are stored!
*
- * @param {Object|Array} data Object or array of objects containing data to be stored in the default dataset.
+ * @param {Object|Array} item Object or array of objects containing data to be stored in the default dataset.
* The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
- * @returns {Promise} Returns a promise that resolves once the data is saved.
- * @see {@link Dataset}
+ * @returns {Promise}
*
* @memberof module:Apify
* @name pushData
- * @instance
* @function
*/
export const pushData = item => openDataset().then(dataset => dataset.pushData(item));
diff --git a/src/errors.js b/src/errors.js
index f5fcb2543..cc72bbd98 100644
--- a/src/errors.js
+++ b/src/errors.js
@@ -2,7 +2,7 @@ export const APIFY_CALL_ERROR_NAME = 'ApifyCallError';
/**
* The class represents exceptions thrown
- * by the {@linkcode Apify#call|Apify.call()} function.
+ * by the [`Apify.call()`](../api/apify#module_Apify.call) function.
*
* @typedef {Object} ApifyCallError
* @property {String} message
diff --git a/src/events.js b/src/events.js
index cb785b1b2..6433a18b0 100644
--- a/src/events.js
+++ b/src/events.js
@@ -25,59 +25,60 @@ let eventsWs = null;
let persistStateInterval = null;
/**
- * Gets an instance of Node.js' <a href="https://nodejs.org/api/events.html#events_class_eventemitter">EventEmitter</a> class
- * that emits various events from the SDK or the Apify platform.
- * The event emitter is initialized by calling <a href="#module-Apify-main"><code>Apify.main()</code></a> function.
+ * Gets an instance of a Node.js'
+ * <a href="https://nodejs.org/api/events.html#events_class_eventemitter" target="_blank">EventEmitter</a>
+ * class that emits various events from the SDK or the Apify platform.
+ * The event emitter is initialized by calling the [`Apify.main()`](#module_Apify.main) function.
*
* **Example usage:**
*
* ```javascript
- * Apify.main(async () => {
- * &nbsp;
- * Apify.events.on('cpuInfo', (data) => {
- * if (data.isCpuOverloaded) console.log('Oh no, the CPU is overloaded!');
- * });
- *. &nbsp;
+ * Apify.events.on('cpuInfo', (data) => {
+ * if (data.isCpuOverloaded) console.log('Oh no, the CPU is overloaded!');
* });
* ```
*
* The following table shows all currently emitted events:
- *
- * <table class="table table-bordered table-condensed">
+ * <table>
* <thead>
* <tr>
* <th>Event name</th>
* <th>Data</th>
- * <th>Description</th>
* </thead>
* <tbody>
* <tr>
- * <td>`cpuInfo`</td>
- * <td>`{ "isCpuOverloaded": Boolean }`</td>
- * <td>
+ * <td><code>cpuInfo</code></td>
+ * <td><code>{ "isCpuOverloaded": Boolean }</code></td>
+ * </tr>
+ * <tr>
+ * <td colspan="2">
* The event is emitted approximately every second
* and it indicates whether the actor is using the maximum of available CPU resources.
* If that's the case, the actor should not add more workload.
- * For example, this event is used by the <a href="#AutoscaledPool">AutoscaledPool</a> class.
+ * For example, this event is used by the <a href="autoscaledpool"><code>AutoscaledPool</code></a> class.
* </td>
* </tr>
* <tr>
- * <td>`migrating`</td>
+ * <td><code>migrating</code></td>
* <td>None</td>
- * <td>
+ * </tr>
+ * <tr>
+ * <td colspan="2">
* Emitted when the actor running on the Apify platform is going to be migrated to another worker server soon.
* You can use it to persist the state of the actor and abort the run, to speed up migration.
- * For example, this is used by the <a href="#RequestList">RequestList</a> class.
+ * For example, this is used by the <a href="requestlist"><code>RequestList</code></a> class.
* </td>
* </tr>
* <tr>
- * <td>`persistState`</td>
- * <td>`{ "isMigrating": Boolean }`</td>
- * <td>
+ * <td><code>persistState</code></td>
+ * <td><code>{ "isMigrating": Boolean }</code></td>
+ * </tr>
+ * <tr>
+ * <td colspan="2">
* Emitted in regular intervals to notify all components of Apify SDK that it is time to persist
* their state, in order to avoid repeating all work when the actor restarts.
- * This event is automatically emitted together with the `migrating` event,
- * in which case the `isMigrating` flag is set to `true`. Otherwise the flag is `false`.
+ * This event is automatically emitted together with the <code>migrating</code> event,
+ * in which case the <code>isMigrating</code> flag is set to <code>true</code>. Otherwise the flag is <code>false</code>.
* </td>
* </tr>
* </tbody>
diff --git a/src/index.js b/src/index.js
index bfd4a5b0e..a669058a6 100644
--- a/src/index.js
+++ b/src/index.js
@@ -32,14 +32,9 @@ if (process.env[EMULATION_ENV_VAR]) {
}
/**
- *{include-readme-1}
- *{include-readme-2}
- *{include-readme-3}
- *
- * <h2>Programmer's reference</h2>
- *
- * The following sections describe all functions and properties provided by the `apify` package.
- * All of them are instance members exported directly by the main module.
+ * The following section describes all functions and properties provided by the `apify` package,
+ * except individual classes and namespaces that have their separate, detailed, documentation pages
+ * accessible from the left sidebar.
*
* @module Apify
*/
diff --git a/src/key_value_store.js b/src/key_value_store.js
index 8a0bc08b7..a0502ba32 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -99,30 +99,31 @@ export const maybeStringify = (value, options) => {
* for saving screenshots, actor inputs and outputs, web pages, PDFs or to persist the state of crawlers.
*
* Do not instantiate this class directly, use the
- * {@link Apify#openKeyValueStore|`Apify.openKeyValueStore()`} function instead.
+ * [`Apify.openKeyValueStore()`](apify#module_Apify.openKeyValueStore) function instead.
*
* Each actor run is associated with a default key-value store, which is created exclusively
* for the run. By convention, the actor input and output are stored into the
* default key-value store under the `INPUT` and `OUTPUT` key, respectively.
* Typically, input and output are JSON files, although it can be any other format.
* To access the default key-value store directly, you can use the
- * {@linkcode Apify.getValue|Apify.getValue()}
- * and {@linkcode Apify.getValue|Apify.setValue()} convenience functions.
+ * [`Apify.getValue()`](apify#module_Apify.getValue)
+ * and [`Apify.setValue()`](apify#module_Apify.setValue) convenience functions.
*
* `KeyValueStore` stores its data either on local disk or in the Apify cloud,
- * depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variable is set.
+ * depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.
*
* If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in
* the local directory in the following files:
* ```
- * [APIFY_LOCAL_STORAGE_DIR]/key_value_stores/[STORE_ID]/[KEY].[EXT]
+ * {APIFY_LOCAL_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}
* ```
- * Note that `[STORE_ID]` is the name or ID of the key-value store. The default key value store has ID `default`,
+ * Note that `{STORE_ID}` is the name or ID of the key-value store. The default key value store has ID: `default`,
* unless you override it by setting the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
- * The `[KEY]` is the key of the record and `[EXT]` corresponds to the MIME content type of the data value.
+ * The `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.
*
- * If the `APIFY_TOKEN` environment variable is provided instead, the data is stored
- * in the [Apify Dataset](https://www.apify.com/docs/storage#dataset) cloud storage.
+ * If the `APIFY_TOKEN` environment variable is provided instead, the data is stored in the
+ * <a href="https://www.apify.com/docs/storage#key-value-store" target="_blank">Apify Key-Value Store</a>
+ * cloud storage.
*
* **Example usage:**
*
@@ -148,8 +149,6 @@ export const maybeStringify = (value, options) => {
* await store.delete('some-key');
* ```
* @hideconstructor
- * @see {@linkcode Apify.getValue|Apify.getValue()}
- * @see {@linkcode Apify.setValue|Apify.setValue()}
*/
export class KeyValueStore {
constructor(storeId, storeName) {
@@ -163,18 +162,18 @@ export class KeyValueStore {
/**
* Gets a value from the key-value store.
*
- * The function returns a promise that resolves to the record value,
+ * The function returns a `Promise` that resolves to the record value,
* whose JavaScript type depends on the MIME content type of the record.
* Records with the `application/json`
* content type are automatically parsed and returned as a JavaScript object.
* Similarly, records with `text/plain` content types are returned as a string.
* For all other content types, the value is returned as a raw
- * <a href="https://nodejs.org/api/buffer.html"><code>Buffer</code></a> instance.
+ * <a href="https://nodejs.org/api/buffer.html" target="_blank"><code>Buffer</code></a> instance.
*
- * If the record does not exist, the functions resolves to `null`.
+ * If the record does not exist, the function resolves to `null`.
*
* To save or delete a value in the key-value store, use the
- * {@linkcode KeyValueStore#setValue|KeyValueStore.setValue()} function.
+ * {@link KeyValueStore#setValue} function.
*
* **Example usage:**
*
@@ -184,10 +183,10 @@ export class KeyValueStore {
* ```
*
* @param {String} key
- * Key of the record.
- * @returns {Promise<Object>}
+ * Unique key of the record.
+ * @returns {Promise<Object|String|Buffer>}
* Returns a promise that resolves to an object, string
- * or <a href="https://nodejs.org/api/buffer.html"><code>Buffer</code></a>, depending
+ * or <a href="https://nodejs.org/api/buffer.html" target="_blank"><code>Buffer</code></a>, depending
* on the MIME content type of the record.
*/
getValue(key) {
@@ -220,32 +219,32 @@ export class KeyValueStore {
* await store.setValue('RESULTS', 'my text data', { contentType: 'text/plain' });
* ```
* If you set custom content type, `value` must be either a string or
- * <a href="https://nodejs.org/api/buffer.html"><code>Buffer</code></a>, otherwise an error will be thrown.
+ * <a href="https://nodejs.org/api/buffer.html" target="_blank"><code>Buffer</code></a>, otherwise an error will be thrown.
*
- * If `value` is null, the record is deleted instead. Note that the `setValue()` function succeeds
+ * If `value` is `null`, the record is deleted instead. Note that the `setValue()` function succeeds
* regardless whether the record existed or not.
*
* To retrieve a value from the key-value store, use the
- * {@linkcode KeyValueStore#setValue|KeyValueStore.getValue()} function.
+ * {@link KeyValueStore#getValue} function.
*
* **IMPORTANT:** Always make sure to use the `await` keyword when calling `setValue()`,
* otherwise the actor process might finish before the value is stored!
*
* @param {String} key
- * Unique record key.
+ * Unique key of the record.
* @param {Object|String|Buffer} value
* Record data, which can be one of the following values:
* <ul>
* <li>If `null`, the record in the key-value store is deleted.</li>
* <li>If no `options.contentType` is specified, `value` can be any JavaScript object and it will be stringified to JSON.</li>
- * <li>If `options.contentType` is specified, `value` is considered raw data and it must be a `String`
- * or <a href="https://nodejs.org/api/buffer.html"><code>Buffer</code></a>.</li>
+ * <li>If `options.contentType` is specified, `value` is considered raw data and it must be either a `String`
+ * or <a href="https://nodejs.org/api/buffer.html" target="_blank"><code>Buffer</code></a>.</li>
* </ul>
* For any other value an error will be thrown.
* @param {Object} [options]
* @param {String} [options.contentType]
* Specifies a custom MIME content type of the record.
- * @returns {Promise} Returns a promise that resolves once the value is stored or deleted.
+ * @returns {Promise}
*
*/
setValue(key, value, options = {}) {
@@ -444,22 +443,20 @@ const getOrCreateKeyValueStore = (storeIdOrName) => {
/**
- * Opens a key-value store and returns a promise resolving to an instance of the {@linkcode KeyValueStore} class.
+ * Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
*
* Key-value stores are used to store records or files, along with their MIME content type.
* The records are stored and retrieved using a unique key.
* The actual data is stored either on a local filesystem or in the Apify cloud.
*
- * For more details and code examples, see the {@linkcode KeyValueStore} class.
+ * For more details and code examples, see the {@link KeyValueStore} class.
*
* @param {string} [storeIdOrName]
* ID or name of the key-value store to be opened. If `null` or `undefined`,
* the function returns the default key-value store associated with the actor run.
* @returns {Promise<KeyValueStore>}
- * Returns a promise that resolves to an instance of the `KeyValueStore` class.
* @memberof module:Apify
* @name openKeyValueStore
- * @instance
* @function
*/
export const openKeyValueStore = (storeIdOrName) => {
@@ -471,11 +468,12 @@ export const openKeyValueStore = (storeIdOrName) => {
: openRemoteStorage(storeIdOrName, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, KeyValueStore, storesCache, getOrCreateKeyValueStore);
};
+//
/**
- * Gets a value from the default {@linkcode KeyValueStore} associated with the current actor run.
+ * Gets a value from the default {@link KeyValueStore} associated with the current actor run.
*
- * This is just a convenient shortcut for {@link KeyValueStore#getValue|`KeyValueStore.getValue()`}.
+ * This is just a convenient shortcut for [`keyValueStore.getValue()`](keyvaluestore#KeyValueStore+getValue).
* For example, calling the following code:
* ```javascript
* const input = await Apify.getValue('INPUT');
@@ -487,29 +485,27 @@ export const openKeyValueStore = (storeIdOrName) => {
* await store.getValue('INPUT');
* ```
*
- * To store the value to the default-key value store, you can use the {@link Apify#setValue|`Apify.setValue()`} function.
+ * To store the value to the default-key value store, you can use the [`Apify.setValue()`](#module_Apify.setValue) function.
*
- * For more information, see {@link Apify.openKeyValueStore|`Apify.openKeyValueStore()`}
- * and {@link KeyValueStore#getValue|`KeyValueStore.getValue()`}.
+ * For more information, see [`Apify.openKeyValueStore()`](#module_Apify.openKeyValueStore)
+ * and [`keyValueStore.getValue()`](keyvaluestore#KeyValueStore+getValue).
*
* @param {String} key
* Unique record key.
* @returns {Promise<Object>}
* Returns a promise that resolves once the record is stored.
- * @see {@link KeyValueStore}
*
* @memberof module:Apify
* @name getValue
- * @instance
* @function
*/
export const getValue = key => openKeyValueStore().then(store => store.getValue(key));
/**
- * Stores or deletes a value in the default {@linkcode KeyValueStore} associated with the current actor run.
+ * Stores or deletes a value in the default {@link KeyValueStore} associated with the current actor run.
*
- * This is just a convenient shortcut for {@link KeyValueStore#setValue|`KeyValueStore.setValue()`}.
+ * This is just a convenient shortcut for [`keyValueStore.setValue()`](keyvaluestore#KeyValueStore+setValue).
* For example, calling the following code:
* ```javascript
* await Apify.setValue('OUTPUT', { foo: "bar" });
@@ -521,10 +517,10 @@ export const getValue = key => openKeyValueStore().then(store => store.getValue(
* await store.setValue('OUTPUT', { foo: "bar" });
* ```
*
- * To get a value from the default-key value store, you can use the {@link Apify#getValue|`Apify.getValue()`} function.
+ * To get a value from the default-key value store, you can use the [`Apify.getValue()`](#module_Apify.getValue) function.
*
- * For more information, see {@link Apify.openKeyValueStore|`Apify.openKeyValueStore()`}
- * and {@link KeyValueStore#setValue|`KeyValueStore.setValue()`}.
+ * For more information, see [`Apify.openKeyValueStore()`](#module_Apify.openKeyValueStore)
+ * and [`keyValueStore.getValue()`](keyvaluestore#KeyValueStore+getValue).
*
* @param {String} key
* Unique record key.
@@ -534,18 +530,15 @@ export const getValue = key => openKeyValueStore().then(store => store.getValue(
* <li>If `null`, the record in the key-value store is deleted.</li>
* <li>If no `options.contentType` is specified, `value` can be any JavaScript object and it will be stringified to JSON.</li>
* <li>If `options.contentType` is specified, `value` is considered raw data and it must be a `String`
- * or <a href="https://nodejs.org/api/buffer.html"><code>Buffer</code></a>.</li>
+ * or <a href="https://nodejs.org/api/buffer.html" target="_blank"><code>Buffer</code></a>.</li>
* </ul>
* For any other value an error will be thrown.
* @param {Object} [options]
* @param {String} [options.contentType]
* Specifies a custom MIME content type of the record.
- * @returns {Promise} Returns a promise that resolves once the value is stored or deleted.
- * @see {@link KeyValueStore}
- *
+ * @return {Promise}
* @memberof module:Apify
* @name setValue
- * @instance
* @function
*/
export const setValue = (key, value, options) => openKeyValueStore().then(store => store.setValue(key, value, options));
diff --git a/src/pseudo_url.js b/src/pseudo_url.js
index c0fc3a6ec..e425babd4 100644
--- a/src/pseudo_url.js
+++ b/src/pseudo_url.js
@@ -52,27 +52,27 @@ const parsePurl = (purl) => {
/**
* Represents a pseudo URL (PURL) - an URL pattern used by web crawlers
* to specify which URLs should the crawler visit.
- * This class is used by the {@linkcode enqueueLinks|Apify.utils.puppeteer.enqueueLinks()} function.
+ * This class is used by the [utils.puppeteer.enqueueLinks()](puppeteer#puppeteer.enqueueLinks) function.
*
* A PURL is simply a URL with special directives enclosed in `[]` brackets.
- * Currently, the only supported directive is `[regexp]`,
+ * Currently, the only supported directive is `[RegExp]`,
* which defines a JavaScript-style regular expression to match against the URL.
*
* For example, a PURL `http://www.example.com/pages/[(\w|-)*]` will match all of the following URLs:
*
* <ul>
- * <li>`http://www.example.com/pages/`</li>
- * <li>`http://www.example.com/pages/my-awesome-page`</li>
- * <li>`http://www.example.com/pages/something`</li>
+ * <li><code>http://www.example.com/pages/</code></li>
+ * <li><code>http://www.example.com/pages/my-awesome-page</code></li>
+ * <li><code>http://www.example.com/pages/something</code></li>
* </ul>
*
* If either `[` or `]` is part of the normal query string, it must be encoded as `[\x5B]` or `[\x5D]`,
* respectively. For example, the following PURL:
- * ```
+ * ```http
* http://www.example.com/search?do[\x5B]load[\x5D]=1
* ```
* will match the URL:
- * ```
+ * ```http
* http://www.example.com/search?do[load]=1
* ```
*
@@ -87,11 +87,9 @@ const parsePurl = (purl) => {
* @param {String} purl
* Pseudo URL.
* @param {Object} requestTemplate
- * Options for the new {@linkcode Request} instances created for matching URLs.
- * @see {@linkcode Request}
- * @see {@linkcode Request}
+ * Options for the new {@link Request} instances created for matching URLs.
*/
-export default class PseudoUrl {
+class PseudoUrl {
constructor(purl, requestTemplate = {}) {
checkParamOrThrow(purl, 'purl', 'String');
checkParamOrThrow(requestTemplate, 'requestTemplate', 'Object');
@@ -124,3 +122,5 @@ export default class PseudoUrl {
return new Request(Object.assign({ url }, this.requestTemplate));
}
}
+
+export default PseudoUrl;
diff --git a/src/puppeteer.js b/src/puppeteer.js
index 46d2ce2da..ffab34443 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -16,49 +16,51 @@ const LAUNCH_PUPPETEER_LOG_OMIT_OPTS = [
];
/**
- * Represents options passed to the {@linkcode Apify#launchPuppeteer|Apify.launchPuppeteer()}
+ * Represents options passed to the
+ * [`Apify.launchPuppeteer()`](../api/apify#module_Apify.launchPuppeteer)
* function.
* @typedef {Object} LaunchPuppeteerOptions
- * @property {String} [opts.proxyUrl]
+ * @property {String} [proxyUrl]
* URL to a HTTP proxy server. It must define the port number,
- * and it might also contain proxy username and password.
+ * and it may also contain proxy username and password.
*
- * For example: `http://bob:[email protected]:1234`.
- * @property {String} [opts.userAgent]
+ * Example: `http://bob:[email protected]:1234`.
+ * @property {String} [userAgent]
* The `User-Agent` HTTP header used by the browser.
* If not provided, the function sets `User-Agent` to a reasonable default
* to reduce the chance of detection of the crawler.
- * @property {Boolean} [opts.useChrome=false]
- * If `true` and `opts.executablePath` is not set,
+ * @property {Boolean} [useChrome=false]
+ * If `true` and `executablePath` is not set,
* Puppeteer will launch full Google Chrome browser available on the machine
* rather than the bundled Chromium. The path to Chrome executable
* is taken from the `APIFY_CHROME_EXECUTABLE_PATH` environment variable if provided,
* or defaults to the typical Google Chrome executable location specific for the operating system.
* By default, this option is `false`.
- * @property {Boolean} [opts.useApifyProxy=false]
+ * @property {Boolean} [useApifyProxy=false]
* If set to `true`, Puppeteer will be configured to use
* <a href="https://my.apify.com/proxy" target="_blank">Apify Proxy</a> for all connections.
- * For more information, see the <a href="https://www.apify.com/docs/proxy">documentation</a>
- * @property {String[]} [opts.apifyProxyGroups]
+ * For more information, see the <a href="https://www.apify.com/docs/proxy" target="_blank">documentation</a>
+ * @property {String[]} [apifyProxyGroups]
* An array of proxy groups to be used
* by the <a href="https://www.apify.com/docs/proxy" target="_blank">Apify Proxy</a>.
* Only applied if the `useApifyProxy` option is `true`.
- * @property {String} [opts.apifyProxySession]
+ * @property {String} [apifyProxySession]
* Apify Proxy session identifier to be used by all the Chrome browsers.
* All HTTP requests going through the proxy with the same session identifier
* will use the same target proxy server (i.e. the same IP address).
* The identifier can only contain the following characters: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
* Only applied if the `useApifyProxy` option is `true`.
- * @property {Boolean} [opts.liveView=false]
- * If set to `true`, a PuppeteerLiveViewServer will be started to enable
- * screenshot and html capturing of visited pages using PuppeteerLiveViewBrowser.
- * @property {Object} [opts.liveViewOptions]
+ * @property {Boolean} [liveView=false]
+ * If set to `true`, a `PuppeteerLiveViewServer` will be started to enable
+ * screenshot and html capturing of visited pages using `PuppeteerLiveViewBrowser`.
+ * @property {Object} [liveViewOptions]
* Settings for `PuppeteerLiveViewBrowser` started using `launchPuppeteer()`.
- * @property {String} [opts.liveViewOptions.id]
+ * @property {String} [liveViewOptions.liveViewId]
* Custom ID of a browser instance in live view.
- * @property {Number} [opts.liveViewOptions.screenshotTimeoutMillis]
+ * @property {Number} [liveViewOptions.screenshotTimeoutMillis=3000]
* Time in milliseconds before a screenshot capturing
- * will time out and the actor continues with execution. Screenshot capturing pauses execution within the given page.
+ * will time out and the actor continues with execution.
+ * Screenshot capturing pauses execution within the given page.
*/
/**
@@ -132,24 +134,25 @@ const getPuppeteerOrThrow = () => {
*
* <ul>
* <li>
- * Passes the setting from the `APIFY_HEADLESS` environment variable to the `headless` option,
- * unless it was already defined by the caller or `APIFY_XVFB` environment variable is set to `1`.
- * Note that Apify Actor cloud platform automatically sets `APIFY_HEADLESS=1` to all running actors.
+ * Passes the setting from the <code>APIFY_HEADLESS</code> environment variable to the <code>headless</code> option,
+ * unless it was already defined by the caller or <code>APIFY_XVFB</code> environment variable is set to <code>1</code>.
+ * Note that Apify Actor cloud platform automatically sets <code>APIFY_HEADLESS=1</code> to all running actors.
* </li>
* <li>
- * Takes the `proxyUrl` option, checks it and adds it to `args` as `--proxy-server=XXX`.
+ * Takes the <code>proxyUrl</code> option, checks it and adds it to <code>args</code> as <code>--proxy-server=XXX</code>.
* If the proxy uses authentication, the function sets up an anonymous proxy HTTP
* to make the proxy work with headless Chrome. For more information, read the
* <a href="https://blog.apify.com/how-to-make-headless-chrome-and-puppeteer-use-a-proxy-server-with-authentication-249a21a79212"
* target="_blank">blog post about proxy-chain library</a>.
* </li>
* <li>
- * If `opts.useApifyProxy` is `true` then the function generates a URL of
+ * If <code>options.useApifyProxy</code> is <code>true</code> then the function generates a URL of
* <a href="https://www.apify.com/docs/proxy" target="_blank">Apify Proxy</a>
- * based on `opts.apifyProxyGroups` and `opts.apifyProxySession` and passes it as `opts.proxyUrl`.
+ * based on <code>options.apifyProxyGroups</code> and <code>options.apifyProxySession</code> and passes it as <code>options.proxyUrl</code>.
* </li>
* <li>
- * The function adds `--no-sandbox` to `args` to enable running headless Chrome in a Docker container on the Apify platform.
+ * The function adds <code>--no-sandbox</code> to <code>args</code> to enable running
+ * headless Chrome in a Docker container on the Apify platform.
* </li>
* </ul>
*
@@ -160,29 +163,29 @@ const getPuppeteerOrThrow = () => {
* <a href="https://www.apify.com/docs/actor#base-images" target="_blank">Apify Actor documentation</a>
* for details.
*
- * For an example of usage, see the <a href="https://www.apify.com/apify/example-puppeteer">apify/example-puppeteer</a> actor.
+ * For an example of usage, see the [Synchronous run Example](../examples/synchronousrun) or the [Puppeteer proxy Example](../examples/puppeteerproxy)
*
- * @param {LaunchPuppeteerOptions} [opts]
- * Optional settings passed to `puppeteer.launch()`. Additionally the object can
- * contain the following fields:
- * @returns {Promise}
- * Promise object that resolves to Puppeteer's `Browser` instance.
+ * @param {LaunchPuppeteerOptions} [options]
+ * Optional settings passed to `puppeteer.launch()`. In addition to
+ * <a href="https://pptr.dev/#?product=Puppeteer&show=api-puppeteerlaunchoptions" target="_blank">Puppeteer's options</a>
+ * the object may contain our own [`LaunchPuppeteerOptions`](../typedefs/launchpuppeteeroptions) that enable additional features.
+ * @returns {Promise<Browser>}
+ * Promise that resolves to Puppeteer's `Browser` instance.
* @memberof module:Apify
* @name launchPuppeteer
- * @instance
* @function
*/
-export const launchPuppeteer = (opts = {}) => {
- checkParamOrThrow(opts, 'opts', 'Object');
- checkParamOrThrow(opts.args, 'opts.args', 'Maybe [String]');
- checkParamOrThrow(opts.proxyUrl, 'opts.proxyUrl', 'Maybe String');
- checkParamOrThrow(opts.useApifyProxy, 'opts.useApifyProxy', 'Maybe Boolean');
- checkParamOrThrow(opts.liveView, 'opts.liveView', 'Maybe Boolean');
- checkParamOrThrow(opts.liveViewOptions, 'opts.liveViewOptoins', 'Maybe Object');
- if (opts.useApifyProxy && opts.proxyUrl) throw new Error('Cannot combine "opts.useApifyProxy" with "opts.proxyUrl"!');
+export const launchPuppeteer = (options = {}) => {
+ checkParamOrThrow(options, 'options', 'Object');
+ checkParamOrThrow(options.args, 'options.args', 'Maybe [String]');
+ checkParamOrThrow(options.proxyUrl, 'options.proxyUrl', 'Maybe String');
+ checkParamOrThrow(options.useApifyProxy, 'options.useApifyProxy', 'Maybe Boolean');
+ checkParamOrThrow(options.liveView, 'options.liveView', 'Maybe Boolean');
+ checkParamOrThrow(options.liveViewOptions, 'options.liveViewOptoins', 'Maybe Object');
+ if (options.useApifyProxy && options.proxyUrl) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrl"!');
const puppeteer = getPuppeteerOrThrow();
- const optsCopy = Object.assign({}, opts);
+ const optsCopy = Object.assign({}, options);
optsCopy.args = optsCopy.args || [];
optsCopy.args.push('--no-sandbox');
@@ -197,8 +200,8 @@ export const launchPuppeteer = (opts = {}) => {
optsCopy.proxyUrl = getApifyProxyUrl({
groups: optsCopy.apifyProxyGroups,
session: optsCopy.apifyProxySession,
- groupsParamName: 'opts.apifyProxyGroups',
- sessionParamName: 'opts.apifyProxySession',
+ groupsParamName: 'options.apifyProxyGroups',
+ sessionParamName: 'options.apifyProxySession',
});
}
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index e8662e68d..e8be5268a 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -18,20 +18,31 @@ const PAGE_CLOSE_TIMEOUT_MILLIS = 30000;
/**
* Provides a simple framework for parallel crawling of web pages
- * using headless Chrome with [Puppeteer](https://github.com/GoogleChrome/puppeteer).
- * The URLs of pages to visit are given by {@linkcode Request} objects that are fed from
- * a static list (see {@linkcode RequestList} class)
- * or from a dynamic queue (see {@linkcode RequestQueue} class).
+ * using headless Chrome with <a href="https://github.com/GoogleChrome/puppeteer" target="_blank">Puppeteer</a>.
*
- * `PuppeteerCrawler` opens a new Chrome page (i.e. tab) for each `Request` object to crawl
- * and then calls the function provided by user as the `handlePageFunction` option.
- * New tasks are only started if there is enough free CPU and memory available,
- * using the {@linkcode AutoscaledPool} class internally.
+ * The source URLs are represented using {@link Request} objects that are fed from
+ * {@link RequestList} or {@link RequestQueue} instances provided by the [`requestList`](#new_PuppeteerCrawler_new)
+ * or [`requestQueue`](#new_PuppeteerCrawler_new) constructor options, respectively.
+ *
+ * If both [`requestList`](#new_PuppeteerCrawler_new) and [`requestQueue`](#new_PuppeteerCrawler_new) are used,
+ * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
+ * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ *
+ * The crawler finishes when there are no more {@link Request} objects to crawl.
+ *
+ * `PuppeteerCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
+ * and then calls the function provided by user as the [`handlePageFunction()`](#new_PuppeteerCrawler_new) option.
+ *
+ * New pages are only opened when there is enough free CPU and memory available,
+ * using the functionality provided by the {@link AutoscaledPool} class.
+ * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
+ * parameter of the `PuppeteerCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+ * {@link AutoscaledPool} options are available directly in the `PuppeteerCrawler` constructor.
*
* Note that the pool of Puppeteer instances is internally managed by
- * the {@linkcode PuppeteerPool} class. Many constructor options
+ * the {@link PuppeteerPool} class. Many constructor options
* such as `maxOpenPagesPerInstance` or `launchPuppeteerFunction` are passed directly
- * to `PuppeteerPool` constructor.
+ * to {@link PuppeteerPool} constructor.
*
* **Example usage:**
*
@@ -60,87 +71,98 @@ const PAGE_CLOSE_TIMEOUT_MILLIS = 30000;
*
* await crawler.run();
* ```
- * @param {Object} options
+ * @param {Object} options All `PuppeteerCrawler` parameters are passed
+ * via an options object with the following keys:
* @param {Function} options.handlePageFunction
* Function that is called to process each request.
* It is passed an object with the following fields:
- * `request` is an instance of the `Request` object with details about the URL to open, HTTP method etc.
- * `page` is an instance of the `Puppeteer.Page` class with `page.goto(request.url)` already called.
+ *
+ * ```
+ * {
+ * request: Request,
+ * page: Page,
+ * puppeteerPool: PuppeteerPool
+ * }
+ * ```
+ *
+ * `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
+ * `page` is an instance of the `Puppeteer`
+ * <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a>
+ * class with `page.goto(request.url)` already called.
+ * `puppeteerPool` is an instance of the {@link PuppeteerPool} used by this `PuppeteerCrawler`.
* @param {RequestList} options.requestList
- * List of the requests to be processed.
- * Either RequestList or RequestQueue must be provided.
- * See the `requestList` parameter of `BasicCrawler` for more details.
+ * Static list of URLs to be processed.
+ * Either {@link RequestList} or {@link RequestQueue} must be provided.
* @param {RequestQueue} options.requestQueue
- * Queue of the requests to be processed.
- * Either RequestList or RequestQueue must be provided.
- * See the `requestQueue` parameter of `BasicCrawler` for more details.
+ * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
+ * Either {@link RequestList} or {@link RequestQueue} must be provided.
* @param {Number} [options.handlePageTimeoutSecs=300]
* Timeout in which the function passed as `options.handlePageFunction` needs to finish, in seconds.
* @param {Function} [options.gotoFunction]
- * Overrides the function that opens the request in Puppeteer. The function should return a result of Puppeteer's
- * <a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#pagegotourl-options">page.goto()</a> function,
- * i.e. a promise resolving to the <a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-response">Response</a> object.
+ * Overrides the function that opens the page in Puppeteer. The function should return the result of Puppeteer's
+ * <a href="https://pptr.dev/#?product=Puppeteer&show=api-pagegotourl-options" target="_blank">page.goto()</a> function,
+ * i.e. a `Promise` resolving to the <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-response" target="_blank">Response</a> object.
*
- * For example, this is useful if you need to extend the page load timeout or select different criteria
+ * This is useful if you need to extend the page load timeout or select different criteria
* to determine that the navigation succeeded.
*
* Note that a single page object is only used to process a single request and it is closed afterwards.
*
- * See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/puppeteer_crawler.js#L9">GitHub</a> for default behavior.
+ * See source code on
+ * <a href="https://github.com/apifytech/apify-js/blob/master/src/puppeteer_crawler.js#L9" target="_blank">GitHub</a>
+ * for default behavior.
* @param {Function} [options.handleFailedRequestFunction]
- * Function to handle requests that failed more than `option.maxRequestRetries` times. See the `handleFailedRequestFunction`
- * parameter of `Apify.BasicCrawler` for details.
- * See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/puppeteer_crawler.js#L13">GitHub</a> for default behavior.
+ * Function to handle requests that failed more than `option.maxRequestRetries` times.
+ * See source code on
+ * <a href="https://github.com/apifytech/apify-js/blob/master/src/puppeteer_crawler.js#L11" target="_blank">GitHub</a>
+ * for default behavior.
* @param {Number} [options.maxRequestRetries=3]
- * Indicates how many times each request is retried if `handleRequestFunction` failed.
- * See `maxRequestRetries` parameter of `BasicCrawler`.
+ * Indicates how many times the request is retried if either `handlePageFunction()` or `gotoFunction()` fails.
* @param {Number} [options.maxRequestsPerCrawl]
* Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
* Always set this value in order to prevent infinite loops in misconfigured crawlers.
* Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
- * See `maxRequestsPerCrawl` parameter of `BasicCrawler`.
* @param {Number} [options.maxOpenPagesPerInstance=50]
* Maximum number of opened tabs per browser. If this limit is reached then a new
- * browser instance is started. See `maxOpenPagesPerInstance` parameter of `PuppeteerPool`.
+ * browser instance is started. See `maxOpenPagesPerInstance` parameter of {@link PuppeteerPool}.
* @param {Number} [options.retireInstanceAfterRequestCount=100]
* Maximum number of requests that can be processed by a single browser instance.
* After the limit is reached the browser will be retired and new requests will
* be handled by a new browser instance.
- * See `retireInstanceAfterRequestCount` parameter of `PuppeteerPool`.
+ * See `retireInstanceAfterRequestCount` parameter of {@link PuppeteerPool}.
* @param {Number} [options.instanceKillerIntervalMillis=60000]
- * How often the launched Puppeteer instances are checked whether they can be
- * closed. See `instanceKillerIntervalMillis` parameter of `PuppeteerPool`.
+ * Indicates how often are the open Puppeteer instances checked whether they can be closed.
+ * See `instanceKillerIntervalMillis` parameter of {@link PuppeteerPool}.
* @param {Number} [options.killInstanceAfterMillis=300000]
* If Puppeteer instance reaches the `options.retireInstanceAfterRequestCount` limit then
* it is considered retired and no more tabs will be opened. After the last tab is closed
* the whole browser is closed too. This parameter defines a time limit for inactivity
* after which the browser is closed even if there are pending tabs. See
- * `killInstanceAfterMillis` parameter of `PuppeteerPool`.
+ * `killInstanceAfterMillis` parameter of {@link PuppeteerPool}.
* @param {Function} [options.launchPuppeteerFunction]
* Overrides the default function to launch a new Puppeteer instance.
- * See `launchPuppeteerFunction` parameter of `PuppeteerPool`.
- * See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/puppeteer_crawler.js#L9">GitHub</a> for default behavior.
+ * See `launchPuppeteerFunction` parameter of {@link PuppeteerPool}.
+ * See source code on
+ * <a href="https://github.com/apifytech/apify-js/blob/master/src/puppeteer_pool.js#L28" target="_blank">GitHub</a>
+ * for default behavior.
* @param {LaunchPuppeteerOptions} [options.launchPuppeteerOptions]
- * Options used by `Apify.launchPuppeteer()` to start new Puppeteer instances.
- * See `launchPuppeteerOptions` parameter of {@linkcode PuppeteerPool}.
+ * Options used by [`Apify.launchPuppeteer()`](apify#module_Apify.launchPuppeteer) to start new Puppeteer instances.
+ * See `launchPuppeteerOptions` parameter of {@link PuppeteerPool} and [`LaunchPuppeteerOptions`](../typedefs/launchpuppeteeroptions).
* @param {Object} [options.autoscaledPoolOptions]
- * Custom options passed to the underlying {@link AutoscaledPool|`AutoscaledPool`} instance constructor.
+ * Custom options passed to the underlying {@link AutoscaledPool} instance constructor.
* Note that the `runTaskFunction`, `isTaskReadyFunction` and `isFinishedFunction` options
* are provided by `PuppeteerCrawler` and should not be overridden.
* @param {Object} [options.minConcurrency=1]
- * Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding `AutoscaledPool` option.
+ * Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
* @param {Object} [options.maxConcurrency=1000]
- * Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding `AutoscaledPool` option.
- *
- * @see {@link CheerioCrawler}
- * @see {@link BasicCrawler}
+ * Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
*/
-export default class PuppeteerCrawler {
- constructor(opts) {
+class PuppeteerCrawler {
+ constructor(options) {
// For backwards compatibility, in the future we can remove this...
- if (!opts.retireInstanceAfterRequestCount && opts.abortInstanceAfterRequestCount) {
+ if (!options.retireInstanceAfterRequestCount && options.abortInstanceAfterRequestCount) {
log.warning('PuppeteerCrawler: Parameter `abortInstanceAfterRequestCount` is deprecated! Use `retireInstanceAfterRequestCount` instead!');
- opts.retireInstanceAfterRequestCount = opts.abortInstanceAfterRequestCount;
+ options.retireInstanceAfterRequestCount = options.abortInstanceAfterRequestCount;
}
const {
@@ -169,11 +191,11 @@ export default class PuppeteerCrawler {
killInstanceAfterMillis,
launchPuppeteerFunction,
launchPuppeteerOptions,
- } = _.defaults(opts, DEFAULT_OPTIONS);
+ } = _.defaults(options, DEFAULT_OPTIONS);
- checkParamOrThrow(handlePageFunction, 'opts.handlePageFunction', 'Function');
- checkParamOrThrow(handleFailedRequestFunction, 'opts.handleFailedRequestFunction', 'Maybe Function');
- checkParamOrThrow(gotoFunction, 'opts.gotoFunction', 'Function');
+ checkParamOrThrow(handlePageFunction, 'options.handlePageFunction', 'Function');
+ checkParamOrThrow(handleFailedRequestFunction, 'options.handleFailedRequestFunction', 'Maybe Function');
+ checkParamOrThrow(gotoFunction, 'options.gotoFunction', 'Function');
this.handlePageFunction = handlePageFunction;
this.gotoFunction = gotoFunction;
@@ -232,7 +254,7 @@ export default class PuppeteerCrawler {
}
/**
- * Stops the crawler by preventing crawls of additional pages and terminating the running ones.
+ * Aborts the crawler by preventing crawls of additional pages and terminating the running ones.
*
* @return {Promise}
*/
@@ -274,3 +296,6 @@ export default class PuppeteerCrawler {
}
}
}
+
+
+export default PuppeteerCrawler;
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index 4f697862e..51d5eb40b 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -66,16 +66,18 @@ class PuppeteerInstance {
}
/**
- * Manages a pool of Chrome browser instances controlled using [Puppeteer](https://github.com/GoogleChrome/puppeteer).
- * `PuppeteerPool` reuses Chrome instances and tabs using specific
- * browser rotation and retirement policies.
+ * Manages a pool of Chrome browser instances controlled using
+ * <a href="https://github.com/GoogleChrome/puppeteer" target="_blank">Puppeteer</a>.
+ *
+ * `PuppeteerPool` reuses Chrome instances and tabs using specific browser rotation and retirement policies.
* This is useful in order to facilitate rotation of proxies, cookies
* or other settings in order to prevent detection of your web scraping bot,
* access web pages from various countries etc.
+ *
* Additionally, the reuse of browser instances instances speeds up crawling,
* and the retirement of instances helps mitigate effects of memory leaks in Chrome.
*
- * `PuppeteerPool` is internally used by the {@link PuppeteerCrawler|`PuppeteerCrawler`} class.
+ * `PuppeteerPool` is internally used by the {@link PuppeteerCrawler} class.
*
* **Example usage:**
*
@@ -98,24 +100,29 @@ class PuppeteerInstance {
* // Close all browsers.
* await puppeteerPool.destroy();
* ```
- *
+ * @param {Object} [options] All `PuppeteerPool` parameters are passed
+ * via an options object with the following keys:
* @param {Number} [options.maxOpenPagesPerInstance=50]
* Maximum number of open pages (i.e. tabs) per browser. When this limit is reached, new pages are loaded in a new browser instance.
* @param {Number} [options.retireInstanceAfterRequestCount=100]
* Maximum number of requests that can be processed by a single browser instance.
* After the limit is reached, the browser is retired and new requests are
- * be handled by a new browser instance.
+ * handled by a new browser instance.
* @param {Number} [options.instanceKillerIntervalMillis=60000]
- * Indicates how often opened Puppeteer instances are checked whether they can be closed.
+ * Indicates how often are the open Puppeteer instances checked whether they can be closed.
* @param {Number} [options.killInstanceAfterMillis=300000]
* When Puppeteer instance reaches the `options.retireInstanceAfterRequestCount` limit then
* it is considered retired and no more tabs will be opened. After the last tab is closed the
* whole browser is closed too. This parameter defines a time limit between the last tab was opened and
* before the browser is closed even if there are pending open tabs.
- * @param {Function} [options.launchPuppeteerFunction=launchPuppeteerOptions&nbsp;=>&nbsp;Apify.launchPuppeteer(launchPuppeteerOptions)]
+ * @param {Function} [options.launchPuppeteerFunction]
* Overrides the default function to launch a new `Puppeteer` instance.
+ * See source code on
+ * <a href="https://github.com/apifytech/apify-js/blob/master/src/puppeteer_pool.js#L28" target="_blank">GitHub</a>
+ * for default behavior.
* @param {LaunchPuppeteerOptions} [options.launchPuppeteerOptions]
* Options used by `Apify.launchPuppeteer()` to start new Puppeteer instances.
+ * See [`LaunchPuppeteerOptions`](../typedefs/launchpuppeteeroptions).
* @param {Boolean} [options.recycleDiskCache]
* Enables recycling of disk cache directories by Chrome instances.
* When a browser instance is closed, its disk cache directory is not deleted but it's used by a newly opened browser instance.
@@ -126,11 +133,11 @@ class PuppeteerInstance {
* To limit the space consumed, you can pass the `--disk-cache-size=X` argument to `options.launchPuppeteerOptions.args`,
* where `X` is the approximate maximum number of bytes for disk cache.
*
- * *IMPORTANT:* Currently this feature only works in headful mode, because of a bug in Chromium.
+ * *IMPORTANT:* Currently this feature only works in **headful** mode, because of a bug in Chromium.
*
* The `options.recycleDiskCache` setting should not be used together with `--disk-cache-dir` argument in `options.launchPuppeteerOptions.args`.
*/
-export default class PuppeteerPool {
+class PuppeteerPool {
constructor(opts = {}) {
checkParamOrThrow(opts, 'opts', 'Object');
@@ -368,9 +375,11 @@ export default class PuppeteerPool {
}
/**
- * Opens new tab in one of the browsers and returns promise that resolves to its Puppeteer.Page.
+ * Opens new tab in one of the browsers in the pool and returns a `Promise`
+ * that resolves to an instance of a Puppeteer
+ * <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a>.
*
- * @return {Promise<Puppeteer.Page>}
+ * @return {Promise<Page>}
*/
async newPage() {
let instance;
@@ -413,7 +422,7 @@ export default class PuppeteerPool {
}
/**
- * Closes all the browsers.
+ * Closes all open browsers.
* @return {Promise}
*/
destroy() {
@@ -458,7 +467,7 @@ export default class PuppeteerPool {
/**
* Finds a PuppeteerInstance given a Puppeteer Browser running in the instance.
- * @param {Puppeteer.Browser} browser
+ * @param {Browser} browser
* @return {Promise}
* @ignore
*/
@@ -478,10 +487,11 @@ export default class PuppeteerPool {
}
/**
- * Manually retires a Puppeteer Browser instance from the pool. The browser will continue
- * to process open pages so that they may gracefully finish. This is unlike browser.close()
- * which will forcibly terminate the browser and all open pages will be closed.
- * @param {Puppeteer.Browser} browser
+ * Manually retires a Puppeteer
+ * <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-browser" target="_blank"><code>Browser</code></a>
+ * instance from the pool. The browser will continue to process open pages so that they may gracefully finish.
+ * This is unlike `browser.close()` which will forcibly terminate the browser and all open pages will be closed.
+ * @param {Browser} browser
* @return {Promise}
*/
retire(browser) {
@@ -492,3 +502,5 @@ export default class PuppeteerPool {
});
}
}
+
+export default PuppeteerPool;
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index 7bed9a760..affd0fe25 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -16,9 +16,10 @@ const readFilePromised = Promise.promisify(fs.readFile);
* Hides certain Puppeteer fingerprints from the page, in order to help avoid detection of the crawler.
* The function should be called on a newly-created page object before navigating to the target crawled page.
*
- * @param {Page} page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
+ * @param {Page} page
+ * Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.
* @return {Promise}
- * @memberof utils.puppeteer
+ * @memberOf puppeteer
*/
const hideWebDriver = async (page) => {
checkParamOrThrow(page, 'page', 'Object');
@@ -57,10 +58,11 @@ const hideWebDriver = async (page) => {
* Unlike Puppeteer's `addScriptTag` function, this function works on pages
* with arbitrary Cross-Origin Resource Sharing (CORS) policies.
*
- * @param {Page} page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
+ * @param {Page} page
+ * Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.
* @param {String} filePath File path
* @return {Promise}
- * @memberof utils.puppeteer
+ * @memberOf puppeteer
*/
const injectFile = async (page, filePath) => {
checkParamOrThrow(page, 'page', 'Object');
@@ -72,16 +74,17 @@ const injectFile = async (page, filePath) => {
};
/**
- * Injects [jQuery](https://jquery.com/) library into a Puppeteer page.
+ * Injects the <a href="https://jquery.com/" target="_blank"><code>jQuery</code></a> library into a Puppeteer page.
* jQuery is often useful for various web scraping and crawling tasks,
* e.g. to extract data from HTML elements using CSS selectors.
*
* Beware that the injected jQuery object will be set to the `window.$` variable and thus it might cause conflicts with
* libraries included by the page that use the same variable (e.g. another version of jQuery).
*
- * @param {Page} page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
+ * @param {Page} page
+ * Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.
* @return {Promise}
- * @memberof utils.puppeteer
+ * @memberOf puppeteer
*/
const injectJQuery = (page) => {
checkParamOrThrow(page, 'page', 'Object');
@@ -91,13 +94,13 @@ const injectJQuery = (page) => {
};
/**
- * Injects [Underscore.js](https://underscorejs.org/) library into a Puppeteer page.
+ * Injects the <a href="https://underscorejs.org/" target="_blank"><code>Underscore.js</code></a> library into a Puppeteer page.
* Beware that the injected Underscore object will be set to the `window._` variable and thus it might cause conflicts with
* libraries included by the page that use the same variable.
*
* @param {Page} page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
* @return {Promise}
- * @memberof utils.puppeteer
+ * @memberOf puppeteer
*/
const injectUnderscore = (page) => {
checkParamOrThrow(page, 'page', 'Object');
@@ -130,28 +133,28 @@ const enqueueRequestsFromClickableElements = async (page, selector, purls, reque
/**
* Finds HTML elements matching a CSS selector, clicks the elements and if a redirect is triggered and destination URL matches
- * one of the provided pseudo-URLs, then the function enqueues that URL to a given request queue.
- * To create a Request object function uses `requestTemplate` from a matching Pseudo-URL.
+ * one of the provided {@link PseudoUrl}s, then the function enqueues that URL to a given request queue.
+ * To create a Request object function uses `requestTemplate` from a matching {@link PseudoUrl}.
*
* *WARNING*: This is work in progress. Currently the function doesn't click elements and only takes their `href` attribute and so
* is working only for link (`a`) elements, but not for buttons or JavaScript links.
*
* @param {Page} page
- * Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
+ * Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.
* @param {String} selector
* CSS selector matching elements to be clicked.
* @param {PseudoUrl[]} pseudoUrls
- * An array of pseudo-URLs matching the URLs to be enqueued.
+ * An array of {@link PseudoUrl}s matching the URLs to be enqueued.
* @param {RequestQueue} requestQueue
* Request queue object where URLs will be enqueued.
* @return {Promise<RequestOperationInfo[]>}
- * Promise that resolves to an array of {@linkcode RequestOperationInfo} objects.
- * @memberof utils.puppeteer
+ * Promise that resolves to an array of {@link RequestOperationInfo} objects.
+ * @memberOf puppeteer
*/
-const enqueueLinks = async (page, selector, purls, requestQueue) => {
+const enqueueLinks = async (page, selector, pseudoUrls, requestQueue) => {
checkParamOrThrow(page, 'page', 'Object');
checkParamOrThrow(selector, 'selector', 'String');
- checkParamOrThrow(purls, 'purls', 'Array');
+ checkParamOrThrow(pseudoUrls, 'purls', 'Array');
checkParamPrototypeOrThrow(requestQueue, 'requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue');
/* istanbul ignore next */
@@ -160,7 +163,7 @@ const enqueueLinks = async (page, selector, purls, requestQueue) => {
const requests = [];
urls.forEach((url) => {
- purls
+ pseudoUrls
.filter(purl => purl.matches(url))
.forEach(purl => requests.push(purl.createRequest(url)));
});
@@ -179,14 +182,15 @@ const enqueueLinks = async (page, selector, purls, requestQueue) => {
* `document`, `stylesheet`, `image`, `media`, `font`, `script`, `texttrack`, `xhr`, `fetch`,
* `eventsource`, `websocket`, `manifest`, `other`.
* For more details, see Puppeteer's
- * <a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#requestresourcetype">Request.resourceType() documentation</a>.
+ * <a href="https://pptr.dev/#?product=Puppeteer&show=api-requestresourcetype" target="_blank">Request.resourceType() documentation</a>.
*
* By default, the function blocks these resource types: `stylesheet`, `font`, `image`, `media`.
*
- * @param {Page} page Puppeteer's `Page` object
- * @param {Array<String>} resourceTypes Array of resource types to block.
- * @return {Promise<void>}
- * @memberof utils.puppeteer
+ * @param {Page} page
+ * Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.
+ * @param {String[]} resourceTypes Array of resource types to block.
+ * @return {Promise}
+ * @memberOf puppeteer
*/
const blockResources = async (page, resourceTypes = ['stylesheet', 'font', 'image', 'media']) => {
await page.setRequestInterception(true);
@@ -198,15 +202,18 @@ const blockResources = async (page, resourceTypes = ['stylesheet', 'font', 'imag
};
/**
- * Enables caching of intercepted responses into provided object. Automaticaly enables request interception in puppeteer.
+ * Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.
* *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
* This issue should be resolved or atleast mitigated in future iterations of this feature.
- * @param {Page} page Puppeteer's `Page` object
- * @param {Object} cache Object in which responses are stored
- * @param {Array<String|RegExp>} responseUrlRules List of rules that are used to check if response should be cached.
+ * @param {Page} page
+ * Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.
+ * @param {Object} cache
+ * Object in which responses are stored
+ * @param {Array<String|RegExp>} responseUrlRules
+ * List of rules that are used to check if the response should be cached.
* String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
- * @return {Promise<void>}
- * @memberof utils.puppeteer
+ * @return {Promise}
+ * @memberOf puppeteer
*/
const cacheResponses = async (page, cache, responseUrlRules) => {
checkParamOrThrow(page, 'page', 'Object');
@@ -261,30 +268,38 @@ const cacheResponses = async (page, cache, responseUrlRules) => {
* by providing it with the following object:
* ```
* {
- * page: Puppeteer.Page,
- * request: Apify.Request,
+ * page: Page,
+ * request: Request,
* }
* ```
- * The function is compiled by using the scriptString parameter as the function's body,
- * so any limitations to function bodies apply. Return value of the function is the return
- * value of the function body = scriptString parameter.
+ * Where `page` is a Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a>
+ * and `request` is a {@link Request}.
+ *
+ * The function is compiled by using the `scriptString` parameter as the function's body,
+ * so any limitations to function bodies apply. Return value of the compiled function
+ * is the return value of the function body = the `scriptString` parameter.
*
- * As a security measure, no globals such as 'process' or 'require' are accessible
+ * As a security measure, no globals such as `process` or `require` are accessible
* from within the function body. Note that the function does not provide a safe
* sandbox and even though globals are not easily accessible, malicious code may
* still execute in the main process via prototype manipulation. Therefore you
* should only use this function to execute sanitized or safe code.
*
+ * Custom context may also be provided using the `context` parameter. To improve security,
+ * make sure to only pass the really necessary objects to the context. Preferably making
+ * secured copies beforehand.
+ *
* @param {String} scriptString
- * @return {Function} async ({ page, request }) => { scriptString }
- * @memberof utils.puppeteer
+ * @param {Object} context
+ * @return {Function} `async ({ page, request }) => { scriptString }`
+ * @memberOf puppeteer
*/
-const compileScript = (scriptString) => {
+const compileScript = (scriptString, context = Object.create(null)) => {
const funcString = `async ({ page, request }) => {${scriptString}}`;
let func;
try {
- func = vm.runInNewContext(funcString, Object.create(null)); // "Secure" the context by removing prototypes.
+ func = vm.runInNewContext(funcString, context); // "Secure" the context by removing prototypes, unless custom context is provided.
} catch (err) {
log.exception(err, 'Cannot compile script!');
throw err;
@@ -303,6 +318,7 @@ const compileScript = (scriptString) => {
*
* ```javascript
* const Apify = require('apify');
+ * const { puppeteer } = Apify.utils;
*
* // Open https://www.example.com in Puppeteer
* const browser = await Apify.launchPuppeteer();
@@ -310,10 +326,9 @@ const compileScript = (scriptString) => {
* await page.goto('https://www.example.com');
*
* // Inject jQuery into a page
- * await Apify.utils.puppeteer.injectJQuery(page);
+ * await puppeteer.injectJQuery(page);
* ```
- * @namespace utils.puppeteer
- * @name utils.puppeteer
+ * @namespace puppeteer
*/
export const puppeteerUtils = {
hideWebDriver,
diff --git a/src/request.js b/src/request.js
index 413866c3b..304083906 100644
--- a/src/request.js
+++ b/src/request.js
@@ -8,10 +8,10 @@ export const computeUniqueKey = (url, keepUrlFragment) => normalizeUrl(url, keep
* Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
* The `Request` object also stores information about errors that occurred during processing of the request.
*
- * Each `Request` instance has the `uniqueKey` property, which is be either specified
- * manually in constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
- * are considered as pointing to the same web page. This behavior applies to all Apify SDK classes,
- * such as {@linkcode RequestList}, {@linkcode RequestQueue} or {@linkcode PuppeteerCrawler}.
+ * Each `Request` instance has the `uniqueKey` property, which can be either specified
+ * manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
+ * are considered as pointing to the same web resource. This behavior applies to all Apify SDK classes,
+ * such as {@link RequestList}, {@link RequestQueue} or {@link PuppeteerCrawler}.
*
* Example use:
*
@@ -30,42 +30,45 @@ export const computeUniqueKey = (url, keepUrlFragment) => normalizeUrl(url, keep
*
* const foo = request.userData.foo;
* ```
- * @class
- * @param {object} opts
- * @param {String} opts.url URL of the web page to crawl.
- * @param {String} [opts.uniqueKey] A unique key identifying the request.
+ * @param {object} options All `Request` parameters are passed
+ * via an options object with the following keys:
+ * @param {String} options.url URL of the web page to crawl.
+ * @param {String} [options.uniqueKey] A unique key identifying the request.
* Two requests with the same `uniqueKey` are considered as pointing to the same URL.
*
* If `uniqueKey` is not provided, then it is automatically generated by normalizing the URL.
- * For example, the URL of `HTTP://www.EXAMPLE.com/something/` will be generated the `uniqueKey`
- * of `http://www.example.com/something`. The `keepUrlFragment` option
- * determines whether URL hash fragment is included in the `uniqueKey` or not. Beware that the HTTP method
- * and payload is not included in the `uniqueKey`, so requests to the same URL but with different
- * HTTP methods or different POST payloads are all considered equal.
+ * For example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `uniqueKey`
+ * of `http://www.example.com/something`.
+ *
+ * The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
+ * Beware that the HTTP method and payload are not included in the `uniqueKey`,
+ * so requests to the same URL but with different HTTP methods or different POST payloads
+ * are all considered equal and will be deduplicated by {@link RequestList} and {@link RequestQueue}.
*
* Pass an arbitrary non-empty text value to the `uniqueKey` property
* to override the default behavior and specify which URLs shall be considered equal.
- * @param {String} [opts.method='GET']
- * @param {String|Buffer} [opts.payload]
+ * @param {String} [options.method='GET']
+ * @param {String|Buffer} [options.payload]
* HTTP request payload, e.g. for POST requests.
- * @param {Number} [opts.retryCount=0]
- * Indicates how many times the URL was retried in a case of error.
- * @param {String[]} [opts.errorMessages]
- * An array of error messages from request processing.
- * @param {Object} [opts.headers={}]
- * HTTP headers.
- * @param {Object} [opts.userData={}]
- * Custom user data assigned to the request.
- * @param {Boolean} [opts.keepUrlFragment=false]
- * If `false` then hash part is removed from the URL when computing the `uniqueKey` property.
+ * @param {Object} [options.headers={}]
+ * HTTP headers in the following format:
+ * ```
+ * {
+ * Accept: 'text/html',
+ * 'Content-Type': 'application/json'
+ * }
+ * ```
+ * @param {Object} [options.userData={}]
+ * Custom user data assigned to the request. Use this to save any request related data to the
+ * request's scope, keeping them accessible on retries, failures etc.
+ * @param {Boolean} [options.keepUrlFragment=false]
+ * If `false` then the hash part of a URL is removed when computing the `uniqueKey` property.
* For example, this causes the `http://www.example.com#foo` and `http://www.example.com#bar` URLs
* to have the same `uniqueKey` of `http://www.example.com` and thus the URLs are considered equal.
- * Note that this option only has effect if `uniqueKey` is not set.
- * @param {String} [opts.ignoreErrors=false]
- * If `true` then errors in processing of this will be ignored.
- * For example, the request won't be retried in a case of an error for example.
- * @see {@link RequestList}
- * @see {@link RequestQueue}
+ * Note that this option only has an effect if `uniqueKey` is not set.
+ * @param {String} [options.ignoreErrors=false]
+ * If `true` then errors in processing of this request will be ignored.
+ * For example, the request won't be retried in a case of an error.
*
* @property {String} id
* Request ID
@@ -87,15 +90,15 @@ export const computeUniqueKey = (url, keepUrlFragment) => normalizeUrl(url, keep
* @property {Object} userData
* Custom user data assigned to the request.
* @property {Boolean} ignoreErrors
- * If `true` then errors in processing of this are ignored.
- * For example, the request won't be retried in a case of an error for example.
+ * If `true` then errors in processing of this request are ignored.
+ * For example, the request won't be retried in a case of an error.
* @property {Date} handledAt
* Indicates the time when the request has been processed.
- * Is `null` if the request has not been crawler yet.
+ * Is `null` if the request has not been crawled yet.
*/
-export default class Request {
- constructor(opts = {}) {
- checkParamOrThrow(opts, 'opts', 'Object');
+class Request {
+ constructor(options = {}) {
+ checkParamOrThrow(options, 'options', 'Object');
const {
id,
@@ -110,7 +113,7 @@ export default class Request {
keepUrlFragment = false,
ignoreErrors = false,
handledAt = null,
- } = opts;
+ } = options;
checkParamOrThrow(id, 'id', 'Maybe String');
@@ -141,9 +144,9 @@ export default class Request {
}
/**
- * Stores information about an error occurred during processing of this request.
+ * Stores information about an error that occurred during processing of this request.
*
- * @param {Error|String} errorOrMessage Error object or error message to be stored in request.
+ * @param {Error|String} errorOrMessage Error object or error message to be stored in the request.
*/
pushErrorMessage(errorOrMessage) {
if (!_.isString(errorOrMessage) && !(errorOrMessage instanceof Error)) {
@@ -159,3 +162,5 @@ export default class Request {
this.errorMessages.push(message);
}
}
+
+export default Request;
diff --git a/src/request_list.js b/src/request_list.js
index bc6b467ae..428511394 100644
--- a/src/request_list.js
+++ b/src/request_list.js
@@ -24,25 +24,24 @@ const ensureUniqueKeyValid = (uniqueKey) => {
* Represents a static list of URLs to crawl.
* The URLs can be provided either in code or parsed from a text file hosted on the web.
*
- * Each URL is represented using an instance of the {@link Request|`Request`} class.
+ * Each URL is represented using an instance of the {@link Request} class.
* The list can only contain unique URLs. More precisely, it can only contain `Request` instances
* with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
- * To add a single URL multiple times to the list,
- * corresponding `Request` objects will need to have different `uniqueKey` properties.
- * You can use the `keepDuplicateUrls` option to do this for you.
+ * To add a single URL to the list multiple times, corresponding {@link Request} objects will need to have different
+ * `uniqueKey` properties. You can use the `keepDuplicateUrls` option to do this for you when initializing the
+ * `RequestList` from sources.
*
- * Once you create an instance of `RequestList`, you need to call {@link RequestList#initialize|`initialize()`}
+ * Once you create an instance of `RequestList`, you need to call the {@link RequestList#initialize} function
* before the instance can be used. After that, no more URLs can be added to the list.
*
- * `RequestList` is used by {@link BasicCrawler|`BasicCrawler`}, {@link CheerioCrawler|`CheerioCrawler`}
- * and {@link PuppeteerCrawler|`PuppeteerCrawler`} as a source of URLs to crawl.
- * Unlike {@link RequestQueue|`RequestQueue`}, `RequestList` is static but it can contain even millions of URLs.
+ * `RequestList` is used by {@link BasicCrawler}, {@link CheerioCrawler}
+ * and {@link PuppeteerCrawler} as a source of URLs to crawl.
+ * Unlike {@link RequestQueue}, `RequestList` is static but it can contain even millions of URLs.
*
- * `RequestList` has an internal state where it stores information which requests were handled,
- * which are in progress or which were reclaimed.
- * The state might be automatically persisted to the default key-value store by setting the `persistStateKey` option
- * so that if the Node.js process is restarted,
- * the crawling can continue where it left off. For more details, see {@link KeyValueStore|`KeyValueStore`}.
+ * `RequestList` has an internal state where it stores information about which requests were already handled,
+ * which are in progress and which were reclaimed. The state might be automatically persisted to the default
+ * key-value store by setting the `persistStateKey` option o that if the Node.js process is restarted,
+ * the crawling can continue where it left off. For more details, see {@link KeyValueStore}.
*
* **Example usage:**
*
@@ -75,15 +74,16 @@ const ensureUniqueKeyValid = (uniqueKey) => {
* await requestList.reclaimRequest(request2);
* ```
*
- * @param {Object} options
+ * @param {Object} options All `RequestList` parameters are passed
+ * via an options object with the following keys:
* @param {Array} options.sources
* An array of sources for the `RequestList`. Its contents can either be just plain objects,
- * defining at least the 'url' property or instances of the {@link Request|`Request`} class.
+ * defining at least the 'url' property or instances of the {@link Request} class.
* Additionally a `requestsFromUrl` property may be used instead of `url`,
* which will instruct the `RequestList` to download the sources from the given remote location.
* The URLs will be parsed from the received response.
*
- * ```javascript
+ * ```
* [
* // One URL
* { method: 'GET', url: 'http://example.com/a/b' },
@@ -92,15 +92,21 @@ const ensureUniqueKeyValid = (uniqueKey) => {
* ]
* ```
* @param {String} [options.persistStateKey]
- * Identifies the key in the default key-value store under which the `RequestList` persists its state.
- * If this is set then `RequestList`
- * persists its state in regular intervals and loads the state from there in case it is restarted
- * due to an error or system reboot.
+ * Identifies the key in the default key-value store under which the `RequestList` persists its
+ * state. State represents a position of the last scraped request in the list. If this is set then `RequestList`
+ * persists this state in regular intervals and loads the state from there in case it is restarted due to an error or system reboot.
+ *
+ * **IMPORTANT:**
+ * The state represents only a pointer/index of a request in the list. The full contents of the list are not persisted.
+ * Therefore, the user is responsible for either persisting the list on her own or a consistent order of the requests
+ * in the list must be preserved. This is especially important when downloading a live list of URLs from a remote resource,
+ * since the change in the source URL list would change the resulting `RequestList` on error/restart and the state
+ * of crawling will be impossible to restore, resulting in an error.
* @param {Object} [options.state]
* The state object that the `RequestList` will be initialized from.
* It is in the form as returned by `RequestList.getState()`, such as follows:
*
- * ```javascript
+ * ```
* {
* nextIndex: 5,
* nextUniqueKey: 'unique-key-5'
@@ -113,23 +119,25 @@ const ensureUniqueKeyValid = (uniqueKey) => {
*
* Note that the preferred (and simpler) way to persist the state of crawling of the `RequestList`
* is to use the `persistStateKey` parameter instead.
- *
* @param {Boolean} [options.keepDuplicateUrls=false]
* By default, `RequestList` will deduplicate the provided URLs. Default deduplication is based
- * on the `uniqueKey` property of passed source {@link Request} objects. If the property is not present,
- * it is generated by normalizing the URL. If present, it is kept intact. In any case, only one request per `uniqueKey` is added
- * to the `RequestList` resulting in removing of duplicate URLs / unique keys.
+ * on the `uniqueKey` property of passed source {@link Request} objects.
+ *
+ * If the property is not present, it is generated by normalizing the URL. If present, it is kept intact.
+ * In any case, only one request per `uniqueKey` is added to the `RequestList` resulting in removal
+ * of duplicate URLs / unique keys.
+ *
* Setting `keepDuplicateUrls` to `true` will append an additional identifier to the `uniqueKey`
* of each request that does not already include a `uniqueKey`. Therefore, duplicate
* URLs will be kept in the list. It does not protect the user from having duplicates in user set
* `uniqueKey`s however. It is the user's responsibility to ensure uniqueness of their unique keys,
* if they wish to keep more than just a single copy in the `RequestList`.
*/
-export default class RequestList {
- constructor(opts = {}) {
- checkParamOrThrow(opts, 'options', 'Object');
+class RequestList {
+ constructor(options = {}) {
+ checkParamOrThrow(options, 'options', 'Object');
- const { sources, persistStateKey, state, keepDuplicateUrls = false } = opts;
+ const { sources, persistStateKey, state, keepDuplicateUrls = false } = options;
checkParamOrThrow(sources, 'options.sources', 'Array');
checkParamOrThrow(state, 'options.state', 'Maybe Object');
@@ -266,9 +274,9 @@ export default class RequestList {
/**
* Returns an object representing the internal state of the `RequestList` instance.
- * Note that the objects fields can change in future releases.
+ * Note that the object's fields can change in future releases.
*
- * @returns Object
+ * @returns {Object}
*/
getState() {
this._ensureIsInitialized();
@@ -283,8 +291,8 @@ export default class RequestList {
}
/**
- * Resolves to `true` if the next call to {@link RequestList#fetchNextRequest|`fetchNextRequest()`}
- * will return `null`, otherwise it resolves to `false`.
+ * Resolves to `true` if the next call to {@link RequestList#fetchNextRequest} function
+ * would return `null`, otherwise it resolves to `false`.
* Note that even if the list is empty, there might be some pending requests currently being processed.
*
* @returns {Promise<Boolean>}
@@ -302,7 +310,7 @@ export default class RequestList {
/**
* Returns `true` if all requests were already handled and there are no more left.
*
- * @returns {Promise<boolean>}
+ * @returns {Promise<Boolean>}
*/
isFinished() {
return Promise
@@ -315,11 +323,11 @@ export default class RequestList {
}
/**
- * Gets the next `Request` to process. First, the function gets a request previously reclaimed
- * using {@link RequestList#reclaimRequest|`reclaimRequest()`} function, if there is any.
- * Otherwise it gets a next request from the sources.
+ * Gets the next {@link Request} to process. First, the function gets a request previously reclaimed
+ * using the {@link RequestList#reclaimRequest} function, if there is any.
+ * Otherwise it gets the next request from sources.
*
- * The function gets `null` if there are no more
+ * The function's `Promise` resolves to `null` if there are no more
* requests to process.
*
* @returns {Promise<Request>}
@@ -497,8 +505,12 @@ export default class RequestList {
/**
* Returns the total number of unique requests present in the `RequestList`.
+ *
+ * @returns {Number}
*/
length() {
return this.requests.length;
}
}
+
+export default RequestList;
diff --git a/src/request_queue.js b/src/request_queue.js
index 619a8eee5..345dd9142 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -115,7 +115,7 @@ const getRequestId = (uniqueKey) => {
/**
* A helper class that is used to report results from the
- * {@linkcode enqueueLinks|Apify.utils.puppeteer.enqueueLinks()} function.
+ * [`Apify.utils.puppeteer.enqueueLinks()`](../api/puppeteer#puppeteer.enqueueLinks) function.
*
* @typedef {Object} RequestOperationInfo
* @property {Boolean} wasAlreadyPresent Indicates if request was already present in the queue.
@@ -128,35 +128,35 @@ const getRequestId = (uniqueKey) => {
* where you start with several URLs and then recursively
* follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
*
- * Each URL is represented using an instance of the {@link Request|`Request`} class.
- * The queue can only contain unique URLs. More precisely, it can only contain `Request` instances
+ * Each URL is represented using an instance of the {@link Request} class.
+ * The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
* with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
* To add a single URL multiple times to the queue,
- * corresponding `Request` objects will need to have different `uniqueKey` properties.
+ * corresponding {@link Request} objects will need to have different `uniqueKey` properties.
*
* Do not instantiate this class directly, use the
- * {@link Apify#openRequestQueue|`Apify.openRequestQueue()`} function instead.
+ * [`Apify.openRequestQueue()`](apify#module_Apify.openRequestQueue) function instead.
*
- * `RequestQueue` is used by {@link BasicCrawler|`BasicCrawler`}, {@link CheerioCrawler|`CheerioCrawler`}
- * and {@link PuppeteerCrawler|`PuppeteerCrawler`} as a source of URLs to crawl.
- * Unlike {@link RequestList|`RequestList`}, `RequestQueue` supports dynamic adding and removing of requests.
+ * `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}
+ * and {@link PuppeteerCrawler} as a source of URLs to crawl.
+ * Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
* On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
*
- * `RequestQueue` stores its data either on local disk or in the Apify cloud,
+ * `RequestQueue` stores its data either on local disk or in the Apify Cloud,
* depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variable is set.
*
* If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the queue data is stored in
* that local directory as follows:
* ```
- * [APIFY_LOCAL_STORAGE_DIR]/request_queues/[QUEUE_ID]/[STATE]/[NUMBER].json
+ * {APIFY_LOCAL_STORAGE_DIR}/request_queues/{QUEUE_ID}/{STATE}/{NUMBER}.json
* ```
- * Note that `[QUEUE_ID]` is the name or ID of the request queue. The default queue has ID `default`,
+ * Note that `{QUEUE_ID}` is the name or ID of the request queue. The default queue has ID: `default`,
* unless you override it by setting the `APIFY_DEFAULT_REQUEST_QUEUE_ID` environment variable.
- * Each request in the queue is stored as a separate JSON file, where `[STATE]` is either `handled` or `pending`,
- * and `[NUMBER]` is an integer indicating the position of the request in the queue.
+ * Each request in the queue is stored as a separate JSON file, where `{STATE}` is either `handled` or `pending`,
+ * and `{NUMBER}` is an integer indicating the position of the request in the queue.
*
* If the `APIFY_TOKEN` environment variable is provided instead, the data is stored
- * in the [Apify Request Queue](https://www.apify.com/docs/storage#queue) cloud storage.
+ * in the <a href="https://www.apify.com/docs/storage#queue" target="_blank">Apify Request Queue</a> cloud storage.
*
* **Example usage:**
*
@@ -209,17 +209,17 @@ export class RequestQueue {
/**
* Adds a request to the queue.
*
- * If the request with the same `Request.uniqueKey` property is already present in the queue,
- * it will not be updated. You can find out this happened from the resulting
- * {@linkcode RequestOperationInfo} object.
+ * If a request with the same `uniqueKey` property is already present in the queue,
+ * it will not be updated. You can find out whether this happened from the resulting
+ * {@link RequestOperationInfo} object.
*
- * @param {Request|Object} request Request object, or object to construct a Request
- * @param {Object} [opts]
- * @param {Boolean} [opts.forefront] If `true`, the request will be added to the foremost position in the queue.
+ * @param {Request|Object} request Request object, or an Object to construct a Request from.
+ * @param {Object} [options]
+ * @param {Boolean} [options.forefront=false] If `true`, the request will be added to the foremost position in the queue.
* @return {RequestOperationInfo}
*/
- addRequest(request, opts = {}) {
- const { forefront, request: newRequest } = validateAddRequestParams(request, opts);
+ addRequest(request, options = {}) {
+ const { forefront, request: newRequest } = validateAddRequestParams(request, options);
if (newRequest) {
request = newRequest;
@@ -306,7 +306,7 @@ export class RequestQueue {
}
/**
- * Marks request handled after successfull processing.
+ * Marks request handled after successful processing.
*
* @param {Request} request
* @return {Promise<RequestOperationInfo>}
@@ -338,14 +338,14 @@ export class RequestQueue {
* so that it can be processed later again.
*
* @param {Request} request
- * @param {Object} [opts]
- * @param {Boolean} [opts.forefront=false]
- * If true then requests gets returned to the begining of the queue
+ * @param {Object} [options]
+ * @param {Boolean} [options.forefront=false]
+ * If `true` then requests get returned to the start of the queue
* and to the back of the queue otherwise.
* @return {Promise<RequestOperationInfo>}
*/
- reclaimRequest(request, opts = {}) {
- const { forefront } = validateReclaimRequestParams(request, opts);
+ reclaimRequest(request, options = {}) {
+ const { forefront } = validateReclaimRequestParams(request, options);
return requestQueues
.updateRequest({
@@ -364,7 +364,7 @@ export class RequestQueue {
}
/**
- * Resolves to `true` if the next call to `fetchNextRequest()` will return `null`, otherwise it resolves to `false`.
+ * Resolves to `true` if the next call to {@link RequestQueue#fetchNextRequest} would return `null`, otherwise it resolves to `false`.
* Note that even if the queue is empty, there might be some pending requests currently being processed.
*
* Due to the nature of distributed storage systems,
@@ -519,7 +519,7 @@ export class RequestQueue {
}
/**
- * Removes the queue either from the Apify cloud storage or from the local directory,
+ * Removes the queue either from the Apify Cloud storage or from the local directory,
* depending on the mode of operation.
*
* @return {Promise}
@@ -822,23 +822,21 @@ const getOrCreateQueue = (queueIdOrName) => {
/**
* Opens a request queue and returns a promise resolving to an instance
- * of the {@link RequestQueue|`RequestQueue`} class.
+ * of the {@link RequestQueue} class.
*
- * `RequestQueue` represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
+ * {@link RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
* The queue is used for deep crawling of websites, where you start with several URLs and then
* recursively follow links to other pages. The data structure supports both breadth-first
* and depth-first crawling orders.
*
- * For more details and code examples, see the {@link RequestQueue|`RequestQueue`} class.
+ * For more details and code examples, see the {@link RequestQueue} class.
*
* @param {string} [queueIdOrName]
* ID or name of the request queue to be opened. If `null` or `undefined`,
* the function returns the default request queue associated with the actor run.
* @returns {Promise<RequestQueue>}
- * Returns a promise that resolves to an instance of the `RequestQueue` class.
* @memberof module:Apify
* @name openRequestQueue
- * @instance
*/
export const openRequestQueue = (queueIdOrName) => {
checkParamOrThrow(queueIdOrName, 'queueIdOrName', 'Maybe String');
diff --git a/src/settings_rotator.js b/src/settings_rotator.js
index 0e554a190..8d8debc28 100644
--- a/src/settings_rotator.js
+++ b/src/settings_rotator.js
@@ -13,6 +13,7 @@ import { checkParamOrThrow } from 'apify-client/build/utils';
* @param {Object} options
* @param {Function} options.newSettingsFunction
* @param {Number} options.maxUsages
+ * @ignore
*/
export default class SettingsRotator {
constructor(opts) {
diff --git a/src/utils.js b/src/utils.js
index 1ff430dfe..556c7b414 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -22,7 +22,7 @@ import { USER_AGENT_LIST } from './constants';
*/
const URL_NO_COMMAS_REGEX = XRegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+.~#?&//=\\(\\)]*)?', 'gi'); // eslint-disable-line
/**
- * Regular expression that, in addition to the default regular expression URL_NO_COMMAS_REGEX, supports matching commas in URL path and query.
+ * Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.
* Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
* @memberOf utils
*/
@@ -54,21 +54,20 @@ export const newClient = () => {
/**
* Gets the default instance of the `ApifyClient` class provided
- * by the <a href="https://www.apify.com/docs/sdk/apify-client-js/latest" target="_blank">apify-client</a> NPM package.
- * The instance is created automatically by the Apify SDK
- * and it is configured using the `APIFY_API_BASE_URL`, `APIFY_USER_ID` and `APIFY_TOKEN`
- * environment variables.
- *
- * The instance is used for all underlying calls to the Apify API
- * in functions such as <a href="#module-Apify-getValue">Apify.getValue()</a>
- * or <a href="#module-Apify-call">Apify.call()</a>.
+ * <a href="https://www.apify.com/docs/sdk/apify-client-js/latest"
+ * target="_blank">apify-client</a> by the NPM package.
+ * The instance is created automatically by the Apify SDK and it is configured using the
+ * `APIFY_API_BASE_URL`, `APIFY_USER_ID` and `APIFY_TOKEN` environment variables.
+ *
+ * The instance is used for all underlying calls to the Apify API in functions such as
+ * [`Apify.getValue()`](#module_Apify.getValue) or [`Apify.call()`](#module_Apify.call).
* The settings of the client can be globally altered by calling the
- * <a href="https://www.apify.com/docs/sdk/apify-client-js/latest#ApifyClient-setOptions"><code>Apify.client.setOptions()</code></a> function.
+ * <a href="https://www.apify.com/docs/sdk/apify-client-js/latest#ApifyClient-setOptions"
+ * target="_blank">`Apify.client.setOptions()`</a> function.
* Beware that altering these settings might have unintended effects on the entire Apify SDK package.
*
* @memberof module:Apify
* @name client
- * @instance
*/
export const apifyClient = newClient();
@@ -121,13 +120,12 @@ const createIsDockerPromise = () => {
};
/**
- * Returns promise that resolves to true if the code is running in a Docker container.
+ * Returns a `Promise` that resolves to true if the code is running in a Docker container.
*
* @return {Promise}
*
* @memberof module:Apify
* @name isDocker
- * @instance
* @function
*/
export const isDocker = (forceReset) => {
@@ -184,10 +182,8 @@ export const weightedAvg = (arrValues, arrWeights) => {
* {
* // Total memory available in the system or container
* totalBytes: Number,
- * &nbsp;
* // Amount of free memory in the system or container
* freeBytes: Number,
- * &nbsp;
* // Amount of memory used (= totalBytes - freeBytes)
* usedBytes: Number,
* // Amount of memory used the current Node.js process
@@ -203,11 +199,10 @@ export const weightedAvg = (arrValues, arrWeights) => {
* Beware that the function is quite inefficient because it spawns a new process.
* Therefore you shouldn't call it too often, like more than once per second.
*
- * @returns {Promise} Returns a promise.
+ * @returns {Promise<Object>}
*
* @memberof module:Apify
* @name getMemoryInfo
- * @instance
* @function
*/
export const getMemoryInfo = async () => {
@@ -333,19 +328,18 @@ export const createTimeoutPromise = (timeoutMillis, errorMessage) => {
*
* @memberof module:Apify
* @name isAtHome
- * @instance
* @function
*/
export const isAtHome = () => !!process.env[ENV_VARS.IS_AT_HOME];
/**
- * Returns a promise that resolves after a specific period of time. This is useful to implement waiting
+ * Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting
* in your code, e.g. to prevent overloading of target website or to avoid bot detection.
*
* **Example usage:**
*
- * ```javascript
- * const Apify = require('apify);
+ * ```
+ * const Apify = require('apify');
*
* ...
*
@@ -367,7 +361,7 @@ const sleep = (millis) => {
* @param {String} url
* @param {String} [encoding='utf8']
* @param {RegExp} [urlRegExp=URL_NO_COMMAS_REGEX]
- * @returns {Promise}
+ * @returns {Promise<String[]>}
* @memberOf utils
*/
const downloadListOfUrls = ({ url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX }) => {
@@ -386,7 +380,7 @@ const downloadListOfUrls = ({ url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_
* Collects all URLs in an arbitrary string to an array, optionally using a custom regular expression.
* @param {String} string
* @param {RegExp} [urlRegExp=URL_NO_COMMAS_REGEX]
- * @returns {Array}
+ * @returns {String[]}
* @memberOf utils
*/
const extractUrls = ({ string, urlRegExp = URL_NO_COMMAS_REGEX }) => {
@@ -467,7 +461,7 @@ export const ensureTokenOrLocalStorageEnvExists = (storageName) => {
* **Example usage:**
*
* ```javascript
- * const Apify = require('apify);
+ * const Apify = require('apify');
*
* ...
*
diff --git a/src/webdriver.js b/src/webdriver.js
index 1f1317dad..620a07c68 100644
--- a/src/webdriver.js
+++ b/src/webdriver.js
@@ -292,25 +292,24 @@ export const browse = (url, options) => {
* <a href="https://www.apify.com/docs/actor#base-images" target="_blank">documentation</a>
* for more details.
*
- * For an example of usage, see the <a href="https://www.apify.com/apify/example-selenium">apify/example-selenium</a> actor.
+ * For an example of usage, see the <a href="https://www.apify.com/apify/example-selenium" target="_blank">apify/example-selenium</a> actor.
*
- * @param {Object} [opts] Optional settings passed to `puppeteer.launch()`. Additionally the object can contain the following fields:
- * @param {String} [opts.proxyUrl] - URL to a proxy server. Currently only `http://` scheme is supported.
+ * @param {Object} [options] Optional settings passed to WebDriver. Additionally the object can contain the following fields:
+ * @param {String} [options.proxyUrl] - URL to a proxy server. Currently only `http://` scheme is supported.
* Port number must be specified. For example, `http://example.com:1234`.
- * @param {String} [opts.headless] - Indicates that the browser will be started in headless mode.
+ * @param {String} [options.headless] - Indicates that the browser will be started in headless mode.
* If the option is not defined, and the `APIFY_HEADLESS` environment variable has value `1`
* and `APIFY_XVFB` is NOT `1`, the value defaults to `true`, otherwise it will be `false`.
- * @param {String} [opts.userAgent] - User-Agent for the browser.
+ * @param {String} [options.userAgent] - User-Agent for the browser.
* If not provided, the function sets it to a reasonable default.
* @returns {Promise}
*
* @memberof module:Apify
* @name launchWebDriver
- * @instance
* @function
*/
-export const launchWebDriver = (opts) => {
- const args = processBrowseArgs(undefined, opts);
+export const launchWebDriver = (options) => {
+ const args = processBrowseArgs(undefined, options);
const browser = new Browser(args.options);
// NOTE: eventually get rid of the Browser class
diff --git a/test/puppeteer.js b/test/puppeteer.js
index 53003aa66..c6a383a46 100644
--- a/test/puppeteer.js
+++ b/test/puppeteer.js
@@ -200,8 +200,8 @@ describe('Apify.launchPuppeteer()', () => {
.withArgs({
session: 'xxx',
groups: ['yyy'],
- groupsParamName: 'opts.apifyProxyGroups',
- sessionParamName: 'opts.apifyProxySession',
+ groupsParamName: 'options.apifyProxyGroups',
+ sessionParamName: 'options.apifyProxySession',
})
.returns(null); // Return null so that it doesn't start proxy-chain
diff --git a/website/README.md b/website/README.md
new file mode 100644
index 000000000..f3da77ff3
--- /dev/null
+++ b/website/README.md
@@ -0,0 +1,193 @@
+This website was created with [Docusaurus](https://docusaurus.io/).
+
+# What's In This Document
+
+* [Get Started in 5 Minutes](#get-started-in-5-minutes)
+* [Directory Structure](#directory-structure)
+* [Editing Content](#editing-content)
+* [Adding Content](#adding-content)
+* [Full Documentation](#full-documentation)
+
+# Get Started in 5 Minutes
+
+1. Make sure all the dependencies for the website are installed:
+
+```sh
+# Install dependencies
+$ yarn
+```
+2. Run your dev server:
+
+```sh
+# Start the site
+$ yarn start
+```
+
+## Directory Structure
+
+Your project file structure should look something like this
+
+```
+my-docusaurus/
+ docs/
+ doc-1.md
+ doc-2.md
+ doc-3.md
+ website/
+ blog/
+ 2016-3-11-oldest-post.md
+ 2017-10-24-newest-post.md
+ core/
+ node_modules/
+ pages/
+ static/
+ css/
+ img/
+ package.json
+ sidebar.json
+ siteConfig.js
+```
+
+# Editing Content
+
+## Editing an existing docs page
+
+Edit docs by navigating to `docs/` and editing the corresponding document:
+
+`docs/doc-to-be-edited.md`
+
+```markdown
+---
+id: page-needs-edit
+title: This Doc Needs To Be Edited
+---
+
+Edit me...
+```
+
+For more information about docs, click [here](https://docusaurus.io/docs/en/navigation)
+
+## Editing an existing blog post
+
+Edit blog posts by navigating to `website/blog` and editing the corresponding post:
+
+`website/blog/post-to-be-edited.md`
+```markdown
+---
+id: post-needs-edit
+title: This Blog Post Needs To Be Edited
+---
+
+Edit me...
+```
+
+For more information about blog posts, click [here](https://docusaurus.io/docs/en/adding-blog)
+
+# Adding Content
+
+## Adding a new docs page to an existing sidebar
+
+1. Create the doc as a new markdown file in `/docs`, example `docs/newly-created-doc.md`:
+
+```md
+---
+id: newly-created-doc
+title: This Doc Needs To Be Edited
+---
+
+My new content here..
+```
+
+1. Refer to that doc's ID in an existing sidebar in `website/sidebar.json`:
+
+```javascript
+// Add newly-created-doc to the Getting Started category of docs
+{
+ "docs": {
+ "Getting Started": [
+ "quick-start",
+ "newly-created-doc" // new doc here
+ ],
+ ...
+ },
+ ...
+}
+```
+
+For more information about adding new docs, click [here](https://docusaurus.io/docs/en/navigation)
+
+## Adding a new blog post
+
+1. Make sure there is a header link to your blog in `website/siteConfig.js`:
+
+`website/siteConfig.js`
+```javascript
+headerLinks: [
+ ...
+ { blog: true, label: 'Blog' },
+ ...
+]
+```
+
+2. Create the blog post with the format `YYYY-MM-DD-My-Blog-Post-Title.md` in `website/blog`:
+
+`website/blog/2018-05-21-New-Blog-Post.md`
+
+```markdown
+---
+author: Frank Li
+authorURL: https://twitter.com/foobarbaz
+authorFBID: 503283835
+title: New Blog Post
+---
+
+Lorem Ipsum...
+```
+
+For more information about blog posts, click [here](https://docusaurus.io/docs/en/adding-blog)
+
+## Adding items to your site's top navigation bar
+
+1. Add links to docs, custom pages or external links by editing the headerLinks field of `website/siteConfig.js`:
+
+`website/siteConfig.js`
+```javascript
+{
+ headerLinks: [
+ ...
+ /* you can add docs */
+ { doc: 'my-examples', label: 'Examples' },
+ /* you can add custom pages */
+ { page: 'help', label: 'Help' },
+ /* you can add external links */
+ { href: 'https://github.com/facebook/Docusaurus', label: 'GitHub' },
+ ...
+ ],
+ ...
+}
+```
+
+For more information about the navigation bar, click [here](https://docusaurus.io/docs/en/navigation)
+
+## Adding custom pages
+
+1. Docusaurus uses React components to build pages. The components are saved as .js files in `website/pages/en`:
+1. If you want your page to show up in your navigation header, you will need to update `website/siteConfig.js` to add to the `headerLinks` element:
+
+`website/siteConfig.js`
+```javascript
+{
+ headerLinks: [
+ ...
+ { page: 'my-new-custom-page', label: 'My New Custom Page' },
+ ...
+ ],
+ ...
+}
+```
+
+For more information about custom pages, click [here](https://docusaurus.io/docs/en/custom-pages).
+
+# Full Documentation
+
+Full documentation can be found on the [website](https://docusaurus.io/).
diff --git a/website/blog/2016-03-11-blog-post.md b/website/blog/2016-03-11-blog-post.md
new file mode 100755
index 000000000..cf2ba2960
--- /dev/null
+++ b/website/blog/2016-03-11-blog-post.md
@@ -0,0 +1,18 @@
+---
+title: Blog Title
+author: Blog Author
+authorURL: http://twitter.com/
+authorFBID: 100002976521003
+---
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus elementum massa eget nulla aliquet sagittis. Proin odio tortor, vulputate ut odio in, ultrices ultricies augue. Cras ornare ultrices lorem malesuada iaculis. Etiam sit amet libero tempor, pulvinar mauris sed, sollicitudin sapien.
+
+<!--truncate-->
+
+Mauris vestibulum ullamcorper nibh, ut semper purus pulvinar ut. Donec volutpat orci sit amet mauris malesuada, non pulvinar augue aliquam. Vestibulum ultricies at urna ut suscipit. Morbi iaculis, erat at imperdiet semper, ipsum nulla sodales erat, eget tincidunt justo dui quis justo. Pellentesque dictum bibendum diam at aliquet. Sed pulvinar, dolor quis finibus ornare, eros odio facilisis erat, eu rhoncus nunc dui sed ex. Nunc gravida dui massa, sed ornare arcu tincidunt sit amet. Maecenas efficitur sapien neque, a laoreet libero feugiat ut.
+
+Nulla facilisi. Maecenas sodales nec purus eget posuere. Sed sapien quam, pretium a risus in, porttitor dapibus erat. Sed sit amet fringilla ipsum, eget iaculis augue. Integer sollicitudin tortor quis ultricies aliquam. Suspendisse fringilla nunc in tellus cursus, at placerat tellus scelerisque. Sed tempus elit a sollicitudin rhoncus. Nulla facilisi. Morbi nec dolor dolor. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Cras et aliquet lectus. Pellentesque sit amet eros nisi. Quisque ac sapien in sapien congue accumsan. Nullam in posuere ante. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Proin lacinia leo a nibh fringilla pharetra.
+
+Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Proin venenatis lectus dui, vel ultrices ante bibendum hendrerit. Aenean egestas feugiat dui id hendrerit. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Curabitur in tellus laoreet, eleifend nunc id, viverra leo. Proin vulputate non dolor vel vulputate. Curabitur pretium lobortis felis, sit amet finibus lorem suscipit ut. Sed non mollis risus. Duis sagittis, mi in euismod tincidunt, nunc mauris vestibulum urna, at euismod est elit quis erat. Phasellus accumsan vitae neque eu placerat. In elementum arcu nec tellus imperdiet, eget maximus nulla sodales. Curabitur eu sapien eget nisl sodales fermentum.
+
+Phasellus pulvinar ex id commodo imperdiet. Praesent odio nibh, sollicitudin sit amet faucibus id, placerat at metus. Donec vitae eros vitae tortor hendrerit finibus. Interdum et malesuada fames ac ante ipsum primis in faucibus. Quisque vitae purus dolor. Duis suscipit ac nulla et finibus. Phasellus ac sem sed dui dictum gravida. Phasellus eleifend vestibulum facilisis. Integer pharetra nec enim vitae mattis. Duis auctor, lectus quis condimentum bibendum, nunc dolor aliquam massa, id bibendum orci velit quis magna. Ut volutpat nulla nunc, sed interdum magna condimentum non. Sed urna metus, scelerisque vitae consectetur a, feugiat quis magna. Donec dignissim ornare nisl, eget tempor risus malesuada quis.
diff --git a/website/blog/2017-04-10-blog-post-two.md b/website/blog/2017-04-10-blog-post-two.md
new file mode 100644
index 000000000..3ab4637bd
--- /dev/null
+++ b/website/blog/2017-04-10-blog-post-two.md
@@ -0,0 +1,18 @@
+---
+title: New Blog Post
+author: Blog Author
+authorURL: http://twitter.com/
+authorFBID: 100002976521003
+---
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus elementum massa eget nulla aliquet sagittis. Proin odio tortor, vulputate ut odio in, ultrices ultricies augue. Cras ornare ultrices lorem malesuada iaculis. Etiam sit amet libero tempor, pulvinar mauris sed, sollicitudin sapien.
+
+<!--truncate-->
+
+Mauris vestibulum ullamcorper nibh, ut semper purus pulvinar ut. Donec volutpat orci sit amet mauris malesuada, non pulvinar augue aliquam. Vestibulum ultricies at urna ut suscipit. Morbi iaculis, erat at imperdiet semper, ipsum nulla sodales erat, eget tincidunt justo dui quis justo. Pellentesque dictum bibendum diam at aliquet. Sed pulvinar, dolor quis finibus ornare, eros odio facilisis erat, eu rhoncus nunc dui sed ex. Nunc gravida dui massa, sed ornare arcu tincidunt sit amet. Maecenas efficitur sapien neque, a laoreet libero feugiat ut.
+
+Nulla facilisi. Maecenas sodales nec purus eget posuere. Sed sapien quam, pretium a risus in, porttitor dapibus erat. Sed sit amet fringilla ipsum, eget iaculis augue. Integer sollicitudin tortor quis ultricies aliquam. Suspendisse fringilla nunc in tellus cursus, at placerat tellus scelerisque. Sed tempus elit a sollicitudin rhoncus. Nulla facilisi. Morbi nec dolor dolor. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Cras et aliquet lectus. Pellentesque sit amet eros nisi. Quisque ac sapien in sapien congue accumsan. Nullam in posuere ante. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Proin lacinia leo a nibh fringilla pharetra.
+
+Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Proin venenatis lectus dui, vel ultrices ante bibendum hendrerit. Aenean egestas feugiat dui id hendrerit. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Curabitur in tellus laoreet, eleifend nunc id, viverra leo. Proin vulputate non dolor vel vulputate. Curabitur pretium lobortis felis, sit amet finibus lorem suscipit ut. Sed non mollis risus. Duis sagittis, mi in euismod tincidunt, nunc mauris vestibulum urna, at euismod est elit quis erat. Phasellus accumsan vitae neque eu placerat. In elementum arcu nec tellus imperdiet, eget maximus nulla sodales. Curabitur eu sapien eget nisl sodales fermentum.
+
+Phasellus pulvinar ex id commodo imperdiet. Praesent odio nibh, sollicitudin sit amet faucibus id, placerat at metus. Donec vitae eros vitae tortor hendrerit finibus. Interdum et malesuada fames ac ante ipsum primis in faucibus. Quisque vitae purus dolor. Duis suscipit ac nulla et finibus. Phasellus ac sem sed dui dictum gravida. Phasellus eleifend vestibulum facilisis. Integer pharetra nec enim vitae mattis. Duis auctor, lectus quis condimentum bibendum, nunc dolor aliquam massa, id bibendum orci velit quis magna. Ut volutpat nulla nunc, sed interdum magna condimentum non. Sed urna metus, scelerisque vitae consectetur a, feugiat quis magna. Donec dignissim ornare nisl, eget tempor risus malesuada quis.
diff --git a/website/blog/2017-09-25-testing-rss.md b/website/blog/2017-09-25-testing-rss.md
new file mode 100644
index 000000000..b7ff8129c
--- /dev/null
+++ b/website/blog/2017-09-25-testing-rss.md
@@ -0,0 +1,11 @@
+---
+title: Adding RSS Support - RSS Truncation Test
+author: Eric Nakagawa
+authorURL: http://twitter.com/ericnakagawa
+authorFBID: 661277173
+---
+1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+
+This should be truncated.
+<!--truncate-->
+This line should never render in XML.
diff --git a/website/blog/2017-09-26-adding-rss.md b/website/blog/2017-09-26-adding-rss.md
new file mode 100644
index 000000000..eeb4f0477
--- /dev/null
+++ b/website/blog/2017-09-26-adding-rss.md
@@ -0,0 +1,10 @@
+---
+title: Adding RSS Support
+author: Eric Nakagawa
+authorURL: http://twitter.com/ericnakagawa
+authorFBID: 661277173
+---
+
+This is a test post.
+
+A whole bunch of other information.
diff --git a/website/blog/2017-10-24-new-version-1.0.0.md b/website/blog/2017-10-24-new-version-1.0.0.md
new file mode 100644
index 000000000..60761c02d
--- /dev/null
+++ b/website/blog/2017-10-24-new-version-1.0.0.md
@@ -0,0 +1,8 @@
+---
+title: New Version 1.0.0
+author: Eric Nakagawa
+authorURL: http://twitter.com/ericnakagawa
+authorFBID: 661277173
+---
+
+This blog post will test file name parsing issues when periods are present.
diff --git a/website/core/Footer.js b/website/core/Footer.js
new file mode 100644
index 000000000..899d61837
--- /dev/null
+++ b/website/core/Footer.js
@@ -0,0 +1,98 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+const React = require('react');
+
+class Footer extends React.Component {
+ docUrl(doc, language) {
+ const baseUrl = this.props.config.baseUrl;
+ return `${baseUrl}docs/${language ? `${language}/` : ''}${doc}`;
+ }
+
+ pageUrl(doc, language) {
+ const baseUrl = this.props.config.baseUrl;
+ return baseUrl + (language ? `${language}/` : '') + doc;
+ }
+
+ render() {
+ return (
+ <footer className="nav-footer" id="footer">
+ <section className="sitemap">
+ <a href={this.props.config.baseUrl} className="nav-home">
+ {this.props.config.footerIcon && (
+ <img
+ src={this.props.config.baseUrl + this.props.config.footerIcon}
+ alt={this.props.config.title}
+ width="66"
+ height="58"
+ />
+ )}
+ </a>
+ <div>
+ <h5>Docs</h5>
+ <a href={this.docUrl('guides/motivation.html', this.props.language)}>
+ Guide
+ </a>
+ <a href={this.docUrl('examples/basiccrawler.html', this.props.language)}>
+ Examples
+ </a>
+ <a href={this.docUrl('api/apify.html', this.props.language)}>
+ Reference
+ </a>
+ </div>
+ <div>
+ <h5>Community</h5>
+ {/*<a href={this.pageUrl('users.html', this.props.language)}>*/}
+ {/*User Showcase*/}
+ {/*</a>*/}
+ <a
+ href="https://stackoverflow.com/questions/tagged/apify"
+ target="_blank"
+ rel="noreferrer noopener">
+ Stack Overflow
+ </a>
+ {/*<a href="https://discordapp.com/">Project Chat</a>*/}
+ <a
+ href="https://twitter.com/apify"
+ target="_blank"
+ rel="noreferrer noopener">
+ Twitter
+ </a>
+ <a
+ href="https://www.facebook.com/apifytech"
+ target="_blank"
+ rel="noreferrer noopener">
+ Facebook
+ </a>
+ </div>
+ <div>
+ <h5>More</h5>
+ <a href="https://www.apify.com" target="_blank">Apify Cloud Platform</a>
+ <a href="https://docusaurus.io" target="_blank">Docusaurus</a>
+ <a href={this.props.config.repoUrl} target="_blank">GitHub</a>
+ </div>
+ </section>
+
+ {/*<a*/}
+ {/*href="https://docusaurus.io"*/}
+ {/*target="_blank"*/}
+ {/*rel="noreferrer noopener"*/}
+ {/*className="fbOpenSource">*/}
+ {/*<img*/}
+ {/*src={`${this.props.config.baseUrl}img/docusaurus.svg`}*/}
+ {/*alt="Docusaurus"*/}
+ {/*width="80"*/}
+ {/*height="80"*/}
+ {/*/>*/}
+ {/*</a>*/}
+ <section className="copyright">{this.props.config.copyright}</section>
+ </footer>
+ );
+ }
+}
+
+module.exports = Footer;
diff --git a/website/i18n/en.json b/website/i18n/en.json
new file mode 100644
index 000000000..0217bbeb2
--- /dev/null
+++ b/website/i18n/en.json
@@ -0,0 +1,136 @@
+{
+ "_comment": "This file is auto-generated by write-translations.js",
+ "localized-strings": {
+ "next": "Next",
+ "previous": "Previous",
+ "tagline": "The scalable web crawling and scraping library for JavaScript.",
+ "docs": {
+ "api/apify": {
+ "title": "Apify"
+ },
+ "api/autoscaledpool": {
+ "title": "AutoscaledPool"
+ },
+ "api/basiccrawler": {
+ "title": "BasicCrawler"
+ },
+ "api/cheeriocrawler": {
+ "title": "CheerioCrawler"
+ },
+ "api/dataset": {
+ "title": "Dataset"
+ },
+ "api/keyvaluestore": {
+ "title": "KeyValueStore"
+ },
+ "api/pseudourl": {
+ "title": "PseudoUrl"
+ },
+ "api/puppeteer": {
+ "title": "utils.puppeteer"
+ },
+ "api/puppeteercrawler": {
+ "title": "PuppeteerCrawler"
+ },
+ "api/puppeteerpool": {
+ "title": "PuppeteerPool"
+ },
+ "api/request": {
+ "title": "Request"
+ },
+ "api/requestlist": {
+ "title": "RequestList"
+ },
+ "api/requestqueue": {
+ "title": "RequestQueue"
+ },
+ "api/snapshotter": {
+ "title": "Snapshotter"
+ },
+ "api/systemstatus": {
+ "title": "SystemStatus"
+ },
+ "api/utils": {
+ "title": "utils"
+ },
+ "examples/basiccrawler": {
+ "title": "Basic Crawler"
+ },
+ "examples/callactor": {
+ "title": "Call Actor"
+ },
+ "examples/cheeriocrawler": {
+ "title": "Cheerio Crawler"
+ },
+ "examples/helloworld": {
+ "title": "Hello World"
+ },
+ "examples/puppeteercrawler": {
+ "title": "Puppeteer Crawler"
+ },
+ "examples/puppeteersitemap": {
+ "title": "Puppeteer Sitemap"
+ },
+ "examples/puppeteerwithproxy": {
+ "title": "Puppeteer With Proxy"
+ },
+ "examples/screenshots": {
+ "title": "Screenshots"
+ },
+ "examples/synchronousrun": {
+ "title": "Synchronous Run"
+ },
+ "guides/datastorage": {
+ "title": "Data Storage"
+ },
+ "guides/environmentvariables": {
+ "title": "Environment Variables"
+ },
+ "guides/gettingstarted": {
+ "title": "Getting Started"
+ },
+ "guides/motivation": {
+ "title": "Motivation"
+ },
+ "guides/puppeteerliveview": {
+ "title": "Puppeteer Live View"
+ },
+ "guides/whatisanactor": {
+ "title": "What is an Actor"
+ },
+ "typedefs/actorrun": {
+ "title": "ActorRun"
+ },
+ "typedefs/apifycallerror": {
+ "title": "ApifyCallError"
+ },
+ "typedefs/launchpuppeteeroptions": {
+ "title": "LaunchPuppeteerOptions"
+ },
+ "typedefs/paginationlist": {
+ "title": "PaginationList"
+ },
+ "typedefs/requestoperationinfo": {
+ "title": "RequestOperationInfo"
+ }
+ },
+ "links": {
+ "Guide": "Guide",
+ "Examples": "Examples",
+ "Reference": "Reference",
+ "GitHub": "GitHub"
+ },
+ "categories": {
+ "Guide": "Guide",
+ "Reference": "Reference",
+ "Examples": "Examples",
+ "Autoscaling Tools": "Autoscaling Tools",
+ "Type Definitions": "Type Definitions"
+ }
+ },
+ "pages-strings": {
+ "Help Translate|recruit community translators for your project": "Help Translate",
+ "Edit this Doc|recruitment message asking to edit the doc source": "Edit",
+ "Translate this Doc|recruitment message asking to translate the docs": "Translate"
+ }
+}
diff --git a/website/package.json b/website/package.json
new file mode 100644
index 000000000..81c12c17d
--- /dev/null
+++ b/website/package.json
@@ -0,0 +1,16 @@
+{
+ "scripts": {
+ "examples": "docusaurus-examples",
+ "start": "docusaurus-start",
+ "build": "docusaurus-build",
+ "publish-gh-pages": "docusaurus-publish",
+ "write-translations": "docusaurus-write-translations",
+ "version": "docusaurus-version",
+ "rename-version": "docusaurus-rename-version",
+ "generate": "node ./tools/build_docs.js"
+ },
+ "devDependencies": {
+ "docusaurus": "^1.5.0",
+ "jsdoc-to-markdown": "^4.0.1"
+ }
+}
diff --git a/website/pages/en/help.js b/website/pages/en/help.js
new file mode 100755
index 000000000..f50dbbb94
--- /dev/null
+++ b/website/pages/en/help.js
@@ -0,0 +1,58 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+const React = require('react');
+
+const CompLibrary = require('../../core/CompLibrary.js');
+
+const Container = CompLibrary.Container;
+const GridBlock = CompLibrary.GridBlock;
+
+const siteConfig = require(`${process.cwd()}/siteConfig.js`);
+
+function docUrl(doc, language) {
+ return `${siteConfig.baseUrl}docs/${language ? `${language}/` : ''}${doc}`;
+}
+
+class Help extends React.Component {
+ render() {
+ const language = this.props.language || '';
+ const supportLinks = [
+ {
+ content: `Learn more using the [documentation on this site.](${docUrl(
+ 'doc1.html',
+ language
+ )})`,
+ title: 'Browse Docs',
+ },
+ {
+ content: 'Ask questions about the documentation and project',
+ title: 'Join the community',
+ },
+ {
+ content: "Find out what's new with this project",
+ title: 'Stay up to date',
+ },
+ ];
+
+ return (
+ <div className="docMainWrapper wrapper">
+ <Container className="mainContainer documentContainer postContainer">
+ <div className="post">
+ <header className="postHeader">
+ <h1>Need help?</h1>
+ </header>
+ <p>This project is maintained by a dedicated group of people.</p>
+ <GridBlock contents={supportLinks} layout="threeColumn" />
+ </div>
+ </Container>
+ </div>
+ );
+ }
+}
+
+module.exports = Help;
diff --git a/website/pages/en/index.js b/website/pages/en/index.js
new file mode 100755
index 000000000..c62879926
--- /dev/null
+++ b/website/pages/en/index.js
@@ -0,0 +1,287 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+const React = require('react');
+
+const CompLibrary = require('../../core/CompLibrary.js');
+
+const MarkdownBlock = CompLibrary.MarkdownBlock;
+/* Used to read markdown */
+const Container = CompLibrary.Container;
+const GridBlock = CompLibrary.GridBlock;
+
+const siteConfig = require(`${process.cwd()}/siteConfig.js`);
+
+function imgUrl(img) {
+ return `${siteConfig.baseUrl}img/${img}`;
+}
+
+function docUrl(doc, language) {
+ return `${siteConfig.baseUrl}docs/${language ? `${language}/` : ''}${doc}`;
+}
+
+function pageUrl(page, language) {
+ return siteConfig.baseUrl + (language ? `${language}/` : '') + page;
+}
+
+class Button extends React.Component {
+ render() {
+ return (
+ <div className="pluginWrapper buttonWrapper">
+ <a className="button" href={this.props.href} target={this.props.target}>
+ {this.props.children}
+ </a>
+ </div>
+ );
+ }
+}
+
+Button.defaultProps = {
+ target: '_self',
+};
+
+const SplashContainer = props => (
+ <div className="homeContainer">
+ <div className="homeSplashFade">
+ <div className="wrapper homeWrapper">{props.children}</div>
+ </div>
+ </div>
+);
+
+const Logo = props => (
+ <div className="projectLogo">
+ <img src={props.img_src} alt="Project Logo"/>
+ </div>
+);
+
+const ProjectTitle = () => (
+ <h2 className="projectTitle">
+ {siteConfig.title}
+ <small>{siteConfig.tagline}</small>
+ </h2>
+);
+
+const PromoSection = props => (
+ <div className="section promoSection">
+ <div className="promoRow">
+ <div className="pluginRowBlock">{props.children}</div>
+ </div>
+ </div>
+);
+
+class HomeSplash extends React.Component {
+ render() {
+ const language = this.props.language || '';
+ return (
+ <SplashContainer>
+ {/*<Logo img_src={imgUrl('apify_logo.svg')}/>*/}
+ <div className="inner">
+ <ProjectTitle/>
+ <PromoSection>
+ <Button href={docUrl('api/apify', language)}>Read the Docs</Button>
+ <Button href={docUrl('examples/basiccrawler', language)}>See Examples</Button>
+ <Button href="#try">Try It Out</Button>
+ </PromoSection>
+ </div>
+ <a
+ className="github-button"
+ href={this.props.config.repoUrl}
+ data-icon="octicon-star"
+ data-count-href="/apifytech/apify-js/stargazers"
+ data-show-count="true"
+ data-count-aria-label="# stargazers on GitHub"
+ aria-label="Star Apify SDK on GitHub">
+ Star
+ </a>
+ </SplashContainer>
+ );
+ }
+}
+
+const Block = props => (
+ <Container
+ padding={props.paddingBottomOnly ? ['bottom'] : ['bottom', 'top']}
+ id={props.id}
+ background={props.background}>
+ <GridBlock align={props.gridBlockAlign || 'center'} contents={props.children} layout={props.layout}/>
+ </Container>
+);
+
+const Features = () => (
+ <Block layout="fourColumn" paddingBottomOnly>
+ {[
+ {
+ content: '**JavaScript** is the language of the web. Although there are JavaScript tools like `puppeteer` and `cheerio`, ' +
+ 'there was no universal framework that would enable **large-scale high-performance** web scraping and crawling of any website. **Until now!**',
+ image: imgUrl('javascript_logo.svg'),
+ imageAlign: 'top',
+ title: 'Runs on JavaScript',
+ },
+ {
+ content: 'Run **headless Chrome** or Selenium, manage **lists and queues** of URLs to crawl, run crawlers in **parallel** at maximum system capacity. ' +
+ 'Handle **storage and export** of results and rotate **proxies**.',
+ image: imgUrl('robot.png'),
+ imageAlign: 'top',
+ title: 'Automates any web workflow',
+ },
+ {
+ content: '`Apify SDK` can be used **stand-alone** in your Node.js projects or it can run as a **serverless microservice on the Apify Cloud Platform**. ' +
+ '[Get started.](https://my.apify.com/actors)',
+ image: imgUrl('cloud_icon.svg'),
+ imageAlign: 'top',
+ title: 'Works locally and in the cloud',
+ }
+ ]}
+ </Block>
+);
+
+// const FeatureCallout = () => (
+// <div
+// className="productShowcaseSection paddingBottom"
+// style={{ textAlign: 'center' }}>
+// <h2>All the features you need are already included</h2>
+// <MarkdownBlock>
+// We have built three different crawler classes for you so you can be up and running in no time.
+// Do you need to crawl just plain HTML? Use our **blazing fast** [`CheerioCrawler`](examples/cheeriocrawler).
+//
+// For complex websites that use **React, Vue** and other front-end javascript libraries and require real-time manipulation,
+// spawn a headless browser with our [`PuppeteerCrawler`](examples/puppeteercrawler).
+//
+// And if you need **control of all aspects** of your crawling, just use the bare bones [`BasicCrawler`](examples/basiccrawler)
+//
+// All of your crawlers will be automatically **scaled** based on available system resources with our [`AutoscaledPool`](api/AutoscaledPool).
+// And if you use the [Apify Cloud Platform](https://my.apify.com/actors), we will also provide you with a pool of **Proxies** to avoid detection.
+//
+// For your persistence needs, check out the [`Dataset`](api/dataset) and [`Key-Value Store`](api/keyvaluestore) storages.
+// </MarkdownBlock>
+// </div>
+// );
+
+const EasyCrawling = () => (
+ <Block background="light" gridBlockAlign="left">
+ {[
+ {
+ content: 'We have built three different crawler classes for you so you can be up and running in no time. ' +
+ 'Do you need to crawl just plain HTML? Use our **blazing fast** [`CheerioCrawler`](examples/cheeriocrawler).\n' +
+ 'For complex websites that use **React, Vue** and other front-end javascript libraries and require real-time manipulation, ' +
+ 'spawn a headless browser with [`PuppeteerCrawler`](examples/puppeteercrawler).\n' +
+ 'And if you need **control of all aspects** of your crawling, just use the bare bones [`BasicCrawler`](examples/basiccrawler)',
+ image: imgUrl('chrome_scrape.gif'),
+ imageAlign: 'right',
+ title: 'Easy Crawling',
+ },
+ ]}
+ </Block>
+);
+
+const PowerfulTools = () => (
+ <Block gridBlockAlign="left">
+ {[
+ {
+ content: 'All of your crawlers will be automatically **scaled** based on available system resources with our [`AutoscaledPool`](api/AutoscaledPool), ' +
+ 'and if you use the [Apify Cloud Platform](https://my.apify.com/actors), we will also provide you with a pool of **Proxies** to avoid detection. ' +
+ 'For your persistence needs, check out the [`Dataset`](api/dataset) and [`Key-Value Store`](api/keyvaluestore) storages.',
+ image: imgUrl('source_code.png'),
+ imageAlign: 'left',
+ title: 'Powerful Tools',
+ },
+ ]}
+ </Block>
+);
+
+
+const TryOut = () => (
+ <Block id="try" background="light" gridBlockAlign="left">
+ {[
+ {
+ content: 'Install **Apify SDK** into a Node.js project. You must have Node.js 8 or higher installed.\n' +
+ '```\n' +
+ 'npm install apify\n' +
+ '```\n' +
+ 'Copy the following code into a file in the project, for example `main.js`:\n' +
+ '```\n' +
+ 'const Apify = require(\'apify\');\n' +
+ '\n' +
+ 'Apify.main(async () => {\n' +
+ ' const requestQueue = await Apify.openRequestQueue();\n' +
+ ' await requestQueue.addRequest({ url: \'https://www.iana.org/\' });\n' +
+ ' const pseudoUrls = [new Apify.PseudoUrl(\'https://www.iana.org/[.*]\')];\n' +
+ '\n' +
+ ' const crawler = new Apify.PuppeteerCrawler({\n' +
+ ' requestQueue,\n' +
+ ' handlePageFunction: async ({ request, page }) => {\n' +
+ ' const title = await page.title();\n' +
+ ' console.log(`Title of ${request.url}: ${title}`);\n' +
+ ' await Apify.utils.puppeteer.enqueueLinks(page, \'a\', pseudoUrls, requestQueue);\n' +
+ ' },\n' +
+ ' maxRequestsPerCrawl: 100,\n' +
+ ' maxConcurrency: 10,\n' +
+ ' });\n' +
+ '\n' +
+ ' await crawler.run();\n' +
+ '});\n' +
+ '```\n' +
+ 'Execute the following command in the project\'s folder and watch it recursively crawl ' +
+ '[IANA](https://www.iana.org) with Puppeteer and Chromium.\n' +
+ '```\n' +
+ 'node main.js\n' +
+ '```\n',
+ // image: imgUrl('apify_logo.svg'),
+ // imageAlign: 'right',
+ title: 'Try it Out',
+ },
+ ]}
+ </Block>
+);
+
+// const Showcase = props => {
+// if ((siteConfig.users || []).length === 0) {
+// return null;
+// }
+//
+// const showcase = siteConfig.users.filter(user => user.pinned)
+// .map(user => (
+// <a href={user.infoLink} key={user.infoLink}>
+// <img src={user.image} alt={user.caption} title={user.caption}/>
+// </a>
+// ));
+//
+// return (
+// <div className="productShowcaseSection paddingBottom">
+// <h2>Who is Using This?</h2>
+// <p>This project is used by all these people</p>
+// <div className="logos">{showcase}</div>
+// <div className="more-users">
+// <a className="button" href={pageUrl('users.html', props.language)}>
+// More {siteConfig.title} Users
+// </a>
+// </div>
+// </div>
+// );
+// };
+
+class Index extends React.Component {
+ render() {
+ const language = this.props.language || '';
+
+ return (
+ <div>
+ <HomeSplash language={language} config={siteConfig}/>
+ <div className="mainContainer">
+ <Features/>
+ {/*<FeatureCallout/>*/}
+ <EasyCrawling/>
+ <PowerfulTools/>
+ <TryOut/>
+ {/*<Showcase language={language}/>*/}
+ </div>
+ </div>
+ );
+ }
+}
+
+module.exports = Index;
diff --git a/website/pages/en/users.js b/website/pages/en/users.js
new file mode 100644
index 000000000..b03fb812a
--- /dev/null
+++ b/website/pages/en/users.js
@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+const React = require('react');
+
+const CompLibrary = require('../../core/CompLibrary.js');
+
+const Container = CompLibrary.Container;
+
+const siteConfig = require(`${process.cwd()}/siteConfig.js`);
+
+class Users extends React.Component {
+ render() {
+ if ((siteConfig.users || []).length === 0) {
+ return null;
+ }
+
+ const editUrl = `${siteConfig.repoUrl}/edit/master/website/siteConfig.js`;
+ const showcase = siteConfig.users.map(user => (
+ <a href={user.infoLink} key={user.infoLink}>
+ <img src={user.image} alt={user.caption} title={user.caption} />
+ </a>
+ ));
+
+ return (
+ <div className="mainContainer">
+ <Container padding={['bottom', 'top']}>
+ <div className="showcaseSection">
+ <div className="prose">
+ <h1>Who is Using This?</h1>
+ <p>This project is used by many folks</p>
+ </div>
+ <div className="logos">{showcase}</div>
+ <p>Are you using this project?</p>
+ <a href={editUrl} className="button">
+ Add your company
+ </a>
+ </div>
+ </Container>
+ </div>
+ );
+ }
+}
+
+module.exports = Users;
diff --git a/website/publish_docs.sh b/website/publish_docs.sh
new file mode 100755
index 000000000..e4fe107ff
--- /dev/null
+++ b/website/publish_docs.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export ALGOLIA_API_KEY=$1
+export GIT_USER=$2
+export USE_SSH=true
+npm run publish-gh-pages
diff --git a/website/sidebars.json b/website/sidebars.json
new file mode 100644
index 000000000..f1bb14815
--- /dev/null
+++ b/website/sidebars.json
@@ -0,0 +1,55 @@
+{
+ "docs": {
+ "Guide": [
+ "guides/motivation",
+ "guides/gettingstarted",
+ "guides/whatisanactor",
+ "guides/environmentvariables",
+ "guides/datastorage",
+ "guides/puppeteerliveview"
+ ],
+ "Reference": [
+ "api/apify",
+ "api/autoscaledpool",
+ "api/basiccrawler",
+ "api/cheeriocrawler",
+ "api/dataset",
+ "api/keyvaluestore",
+ "api/pseudourl",
+ "api/puppeteercrawler",
+ "api/puppeteerpool",
+ "api/request",
+ "api/requestlist",
+ "api/requestqueue",
+ "api/utils",
+ "api/puppeteer"
+ ]
+ },
+ "examples": {
+ "Examples": [
+ "examples/basiccrawler",
+ "examples/cheeriocrawler",
+ "examples/puppeteercrawler",
+ "examples/puppeteerwithproxy",
+ "examples/puppeteersitemap",
+ "examples/screenshots",
+ "examples/callactor",
+ "examples/synchronousrun"
+ ]
+ },
+ "autoscaling": {
+ "Autoscaling Tools": [
+ "api/snapshotter",
+ "api/systemstatus"
+ ]
+ },
+ "typedefs": {
+ "Type Definitions": [
+ "typedefs/actorrun",
+ "typedefs/apifycallerror",
+ "typedefs/launchpuppeteeroptions",
+ "typedefs/paginationlist",
+ "typedefs/requestoperationinfo"
+ ]
+ }
+}
diff --git a/website/siteConfig.js b/website/siteConfig.js
new file mode 100644
index 000000000..062c9c90f
--- /dev/null
+++ b/website/siteConfig.js
@@ -0,0 +1,116 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// See https://docusaurus.io/docs/site-config for all the possible
+// site configuration options.
+
+// List of projects/orgs using your project for the users page.
+// const users = [
+// {
+// caption: 'User1',
+// // You will need to prepend the image path with your baseUrl
+// // if it is not '/', like: '/test-site/img/docusaurus.svg'.
+// image: '/img/docusaurus.svg',
+// infoLink: 'https://www.facebook.com',
+// pinned: true,
+// },
+// ];
+
+const repoUrl = 'https://github.com/apifytech/apify-js';
+
+const siteConfig = {
+ title: 'Apify SDK', // Title for your website.
+ tagline: 'The scalable web crawling and scraping library for JavaScript.',
+ url: 'https://sdk.apify.com', // Your website URL
+ cname: 'sdk.apify.com',
+ baseUrl: '/', // Base URL for your project */
+ // For github.io type URLs, you would set the url and baseUrl like:
+ // url: 'https://facebook.github.io',
+ // baseUrl: '/test-site/',
+
+ // Used for publishing and more
+ projectName: 'apify-js',
+ organizationName: 'apifytech',
+ // For top-level user or org sites, the organization is still the same.
+ // e.g., for the https://JoelMarcey.github.io site, it would be set like...
+ // organizationName: 'JoelMarcey'
+
+ // For no header links in the top nav bar -> headerLinks: [],
+ headerLinks: [
+ { search: true },
+ { doc: 'guides/motivation', label: 'Guide' },
+ { doc: 'examples/basiccrawler', label: 'Examples' },
+ { doc: 'api/apify', label: 'Reference' },
+ { href: repoUrl, label: 'GitHub' },
+ // { page: 'help', label: 'Help' },
+ // { blog: true, label: 'Blog' },
+ ],
+
+ // If you have users set above, you add it here:
+ // users,
+
+ /* path to images for header/footer */
+ headerIcon: 'img/apify_logo.svg',
+ footerIcon: 'img/apify_logo.svg',
+ favicon: 'img/favicon.ico',
+
+ /* Colors for website */
+ colors: {
+ primaryColor: '#001F5B',
+ secondaryColor: '#FF9012',
+ },
+
+ algolia: {
+ apiKey: process.env.ALGOLIA_API_KEY,
+ indexName: 'apify_sdk',
+ algoliaOptions: {}, // Optional, if provided by Algolia
+ },
+
+ /* Custom fonts for website */
+ /*
+ fonts: {
+ myFont: [
+ "Times New Roman",
+ "Serif"
+ ],
+ myOtherFont: [
+ "-apple-system",
+ "system-ui"
+ ]
+ },
+ */
+
+ // This copyright info is used in /core/Footer.js and blog RSS/Atom feeds.
+ copyright: `Copyright © ${new Date().getFullYear()} Apify Technologies s.r.o.`,
+
+ highlight: {
+ // Highlight.js theme to use for syntax highlighting in code blocks.
+ theme: 'monokai-sublime',
+ defaultLang: 'javascript',
+ },
+
+ // Add custom scripts here that would be placed in <script> tags.
+ scripts: [
+ 'https://buttons.github.io/buttons.js',
+ ],
+
+ // On page navigation for the current documentation page.
+ onPageNav: 'separate',
+ // No .html extensions for paths.
+ cleanUrl: true,
+
+ // Open Graph and Twitter card images.
+ ogImage: 'img/apify_logo.png',
+ twitterImage: 'img/apify_logo.png',
+
+ // You may provide arbitrary config keys to be used as needed by your
+ // template. For example, if you need your repo's URL...
+ // repoUrl: 'https://github.com/facebook/test-site',
+ repoUrl,
+};
+
+module.exports = siteConfig;
diff --git a/website/static/css/custom.css b/website/static/css/custom.css
new file mode 100644
index 000000000..d7887584b
--- /dev/null
+++ b/website/static/css/custom.css
@@ -0,0 +1,38 @@
+/* your custom css */
+
+@media only screen and (min-device-width: 360px) and (max-device-width: 736px) {
+}
+
+@media only screen and (min-width: 1024px) {
+}
+
+@media only screen and (max-width: 1023px) {
+}
+
+@media only screen and (min-width: 1400px) {
+}
+
+@media only screen and (min-width: 1500px) {
+}
+
+a {
+ color: $secondaryColor;
+}
+/*Inverting input coloring because the default is too dark on our dark blue.*/
+input#search_input_react {
+ background-color: #fff !important;
+ color: #000 !important;
+}
+
+input#search_input_react::placeholder {
+ color: #000;
+ font-weight: bolder;
+}
+
+.navSearchWrapper::after {
+ background-color: rgb(129, 129, 129);
+}
+
+.navSearchWrapper::before {
+ border-color: rgb(129, 129, 129);
+}
diff --git a/website/static/img/apify_logo.svg b/website/static/img/apify_logo.svg
new file mode 100644
index 000000000..7a548fd78
--- /dev/null
+++ b/website/static/img/apify_logo.svg
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 19.2.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+ viewBox="0 0 1080 1080" style="enable-background:new 0 0 1080 1080;" xml:space="preserve">
+<style type="text/css">
+ .st0{fill:#6BB435;}
+ .st1{fill:#0099C5;}
+ .st2{fill:#FF7800;}
+</style>
+<g>
+ <g>
+ <path class="st0" d="M189.4,148.8C114.7,159.3,62.7,228.3,73.1,303l80.6,576.8l339.2-773.3L189.4,148.8z"/>
+ <path class="st1" d="M1007.9,629.4L976.2,186c-5.6-78.8-76.8-136.4-155.1-125.5L690.5,78.8l287.5,646
+ C999.2,698.5,1010.4,664.6,1007.9,629.4z"/>
+ <path class="st2" d="M277.1,1020c22.3,2.5,45.6-0.4,68.2-10L839,801.2L604.3,273.9L277.1,1020z"/>
+ </g>
+</g>
+</svg>
diff --git a/website/static/img/chrome_scrape.gif b/website/static/img/chrome_scrape.gif
new file mode 100644
index 000000000..08a455e70
Binary files /dev/null and b/website/static/img/chrome_scrape.gif differ
diff --git a/website/static/img/cloud_icon.svg b/website/static/img/cloud_icon.svg
new file mode 100644
index 000000000..23226f2cc
--- /dev/null
+++ b/website/static/img/cloud_icon.svg
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 22.1.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+ viewBox="0 0 24 24" style="enable-background:new 0 0 24 24;" xml:space="preserve">
+<style type="text/css">
+ .st0{fill:none;}
+ .st1{fill:#00A7CE;}
+</style>
+<path class="st0" d="M0,0h24v24H0V0z"/>
+<path class="st1" d="M19.4,10c-0.7-3.4-3.7-6-7.4-6C9.1,4,6.6,5.6,5.3,8C2.3,8.4,0,10.9,0,14c0,3.3,2.7,6,6,6h13c2.8,0,5-2.2,5-5
+ C24,12.4,22,10.2,19.4,10z"/>
+</svg>
diff --git a/website/static/img/docusaurus.svg b/website/static/img/docusaurus.svg
new file mode 100644
index 000000000..9db6d0d06
--- /dev/null
+++ b/website/static/img/docusaurus.svg
@@ -0,0 +1 @@
+<svg width="200" height="200" viewBox="0 0 200 200" xmlns="http://www.w3.org/2000/svg"><g fill="none" fill-rule="evenodd"><path fill="#FFF" d="M99 52h84v34H99z"/><path d="M23 163c-7.398 0-13.843-4.027-17.303-10A19.886 19.886 0 0 0 3 163c0 11.046 8.954 20 20 20h20v-20H23z" fill="#3ECC5F"/><path d="M112.98 57.376L183 53V43c0-11.046-8.954-20-20-20H73l-2.5-4.33c-1.112-1.925-3.889-1.925-5 0L63 23l-2.5-4.33c-1.111-1.925-3.889-1.925-5 0L53 23l-2.5-4.33c-1.111-1.925-3.889-1.925-5 0L43 23c-.022 0-.042.003-.065.003l-4.142-4.141c-1.57-1.571-4.252-.853-4.828 1.294l-1.369 5.104-5.192-1.392c-2.148-.575-4.111 1.389-3.535 3.536l1.39 5.193-5.102 1.367c-2.148.576-2.867 3.259-1.296 4.83l4.142 4.142c0 .021-.003.042-.003.064l-4.33 2.5c-1.925 1.111-1.925 3.889 0 5L23 53l-4.33 2.5c-1.925 1.111-1.925 3.889 0 5L23 63l-4.33 2.5c-1.925 1.111-1.925 3.889 0 5L23 73l-4.33 2.5c-1.925 1.111-1.925 3.889 0 5L23 83l-4.33 2.5c-1.925 1.111-1.925 3.889 0 5L23 93l-4.33 2.5c-1.925 1.111-1.925 3.889 0 5L23 103l-4.33 2.5c-1.925 1.111-1.925 3.889 0 5L23 113l-4.33 2.5c-1.925 1.111-1.925 3.889 0 5L23 123l-4.33 2.5c-1.925 1.111-1.925 3.889 0 5L23 133l-4.33 2.5c-1.925 1.111-1.925 3.889 0 5L23 143l-4.33 2.5c-1.925 1.111-1.925 3.889 0 5L23 153l-4.33 2.5c-1.925 1.111-1.925 3.889 0 5L23 163c0 11.046 8.954 20 20 20h120c11.046 0 20-8.954 20-20V83l-70.02-4.376A10.645 10.645 0 0 1 103 68c0-5.621 4.37-10.273 9.98-10.624" fill="#3ECC5F"/><path fill="#3ECC5F" d="M143 183h30v-40h-30z"/><path d="M193 158c-.219 0-.428.037-.639.064-.038-.15-.074-.301-.116-.451A5 5 0 0 0 190.32 148a4.96 4.96 0 0 0-3.016 1.036 26.531 26.531 0 0 0-.335-.336 4.955 4.955 0 0 0 1.011-2.987 5 5 0 0 0-9.599-1.959c-.148-.042-.297-.077-.445-.115.027-.211.064-.42.064-.639a5 5 0 0 0-5-5 5 5 0 0 0-5 5c0 .219.037.428.064.639-.148.038-.297.073-.445.115a4.998 4.998 0 0 0-9.599 1.959c0 1.125.384 2.151 1.011 2.987-3.717 3.632-6.031 8.693-6.031 14.3 0 11.046 8.954 20 20 20 9.339 0 17.16-6.41 19.361-15.064.211.027.42.064.639.064a5 5 0 0 0 5-5 5 5 0 0 0-5-5" fill="#44D860"/><path fill="#3ECC5F" d="M153 123h30v-20h-30z"/><path d="M193 115.5a2.5 2.5 0 1 0 0-5c-.109 0-.214.019-.319.032-.02-.075-.037-.15-.058-.225a2.501 2.501 0 0 0-.963-4.807c-.569 0-1.088.197-1.508.518a6.653 6.653 0 0 0-.168-.168c.314-.417.506-.931.506-1.494a2.5 2.5 0 0 0-4.8-.979A9.987 9.987 0 0 0 183 103c-5.522 0-10 4.478-10 10s4.478 10 10 10c.934 0 1.833-.138 2.69-.377a2.5 2.5 0 0 0 4.8-.979c0-.563-.192-1.077-.506-1.494.057-.055.113-.111.168-.168.42.321.939.518 1.508.518a2.5 2.5 0 0 0 .963-4.807c.021-.074.038-.15.058-.225.105.013.21.032.319.032" fill="#44D860"/><path d="M63 55.5a2.5 2.5 0 0 1-2.5-2.5c0-4.136-3.364-7.5-7.5-7.5s-7.5 3.364-7.5 7.5a2.5 2.5 0 1 1-5 0c0-6.893 5.607-12.5 12.5-12.5S65.5 46.107 65.5 53a2.5 2.5 0 0 1-2.5 2.5" fill="#000"/><path d="M103 183h60c11.046 0 20-8.954 20-20V93h-60c-11.046 0-20 8.954-20 20v70z" fill="#FFFF50"/><path d="M168.02 124h-50.04a1 1 0 1 1 0-2h50.04a1 1 0 1 1 0 2m0 20h-50.04a1 1 0 1 1 0-2h50.04a1 1 0 1 1 0 2m0 20h-50.04a1 1 0 1 1 0-2h50.04a1 1 0 1 1 0 2m0-49.814h-50.04a1 1 0 1 1 0-2h50.04a1 1 0 1 1 0 2m0 19.814h-50.04a1 1 0 1 1 0-2h50.04a1 1 0 1 1 0 2m0 20h-50.04a1 1 0 1 1 0-2h50.04a1 1 0 1 1 0 2M183 61.611c-.012 0-.022-.006-.034-.005-3.09.105-4.552 3.196-5.842 5.923-1.346 2.85-2.387 4.703-4.093 4.647-1.889-.068-2.969-2.202-4.113-4.46-1.314-2.594-2.814-5.536-5.963-5.426-3.046.104-4.513 2.794-5.807 5.167-1.377 2.528-2.314 4.065-4.121 3.994-1.927-.07-2.951-1.805-4.136-3.813-1.321-2.236-2.848-4.75-5.936-4.664-2.994.103-4.465 2.385-5.763 4.4-1.373 2.13-2.335 3.428-4.165 3.351-1.973-.07-2.992-1.51-4.171-3.177-1.324-1.873-2.816-3.993-5.895-3.89-2.928.1-4.399 1.97-5.696 3.618-1.232 1.564-2.194 2.802-4.229 2.724a1 1 0 0 0-.072 2c3.017.101 4.545-1.8 5.872-3.487 1.177-1.496 2.193-2.787 4.193-2.855 1.926-.082 2.829 1.115 4.195 3.045 1.297 1.834 2.769 3.914 5.731 4.021 3.103.104 4.596-2.215 5.918-4.267 1.182-1.834 2.202-3.417 4.15-3.484 1.793-.067 2.769 1.35 4.145 3.681 1.297 2.197 2.766 4.686 5.787 4.796 3.125.108 4.634-2.62 5.949-5.035 1.139-2.088 2.214-4.06 4.119-4.126 1.793-.042 2.728 1.595 4.111 4.33 1.292 2.553 2.757 5.445 5.825 5.556l.169.003c3.064 0 4.518-3.075 5.805-5.794 1.139-2.41 2.217-4.68 4.067-4.773v-2z" fill="#000"/><path fill="#3ECC5F" d="M83 183h40v-40H83z"/><path d="M143 158c-.219 0-.428.037-.639.064-.038-.15-.074-.301-.116-.451A5 5 0 0 0 140.32 148a4.96 4.96 0 0 0-3.016 1.036 26.531 26.531 0 0 0-.335-.336 4.955 4.955 0 0 0 1.011-2.987 5 5 0 0 0-9.599-1.959c-.148-.042-.297-.077-.445-.115.027-.211.064-.42.064-.639a5 5 0 0 0-5-5 5 5 0 0 0-5 5c0 .219.037.428.064.639-.148.038-.297.073-.445.115a4.998 4.998 0 0 0-9.599 1.959c0 1.125.384 2.151 1.011 2.987-3.717 3.632-6.031 8.693-6.031 14.3 0 11.046 8.954 20 20 20 9.339 0 17.16-6.41 19.361-15.064.211.027.42.064.639.064a5 5 0 0 0 5-5 5 5 0 0 0-5-5" fill="#44D860"/><path fill="#3ECC5F" d="M83 123h40v-20H83z"/><path d="M133 115.5a2.5 2.5 0 1 0 0-5c-.109 0-.214.019-.319.032-.02-.075-.037-.15-.058-.225a2.501 2.501 0 0 0-.963-4.807c-.569 0-1.088.197-1.508.518a6.653 6.653 0 0 0-.168-.168c.314-.417.506-.931.506-1.494a2.5 2.5 0 0 0-4.8-.979A9.987 9.987 0 0 0 123 103c-5.522 0-10 4.478-10 10s4.478 10 10 10c.934 0 1.833-.138 2.69-.377a2.5 2.5 0 0 0 4.8-.979c0-.563-.192-1.077-.506-1.494.057-.055.113-.111.168-.168.42.321.939.518 1.508.518a2.5 2.5 0 0 0 .963-4.807c.021-.074.038-.15.058-.225.105.013.21.032.319.032" fill="#44D860"/><path d="M143 41.75c-.16 0-.33-.02-.49-.05a2.52 2.52 0 0 1-.47-.14c-.15-.06-.29-.14-.431-.23-.13-.09-.259-.2-.38-.31-.109-.12-.219-.24-.309-.38s-.17-.28-.231-.43a2.619 2.619 0 0 1-.189-.96c0-.16.02-.33.05-.49.03-.16.08-.31.139-.47.061-.15.141-.29.231-.43.09-.13.2-.26.309-.38.121-.11.25-.22.38-.31.141-.09.281-.17.431-.23.149-.06.31-.11.47-.14.32-.07.65-.07.98 0 .159.03.32.08.47.14.149.06.29.14.43.23.13.09.259.2.38.31.11.12.22.25.31.38.09.14.17.28.23.43.06.16.11.31.14.47.029.16.05.33.05.49 0 .66-.271 1.31-.73 1.77-.121.11-.25.22-.38.31-.14.09-.281.17-.43.23a2.565 2.565 0 0 1-.96.19m20-1.25c-.66 0-1.3-.27-1.771-.73a3.802 3.802 0 0 1-.309-.38c-.09-.14-.17-.28-.231-.43a2.619 2.619 0 0 1-.189-.96c0-.66.27-1.3.729-1.77.121-.11.25-.22.38-.31.141-.09.281-.17.431-.23.149-.06.31-.11.47-.14.32-.07.66-.07.98 0 .159.03.32.08.47.14.149.06.29.14.43.23.13.09.259.2.38.31.459.47.73 1.11.73 1.77 0 .16-.021.33-.05.49-.03.16-.08.32-.14.47-.07.15-.14.29-.23.43-.09.13-.2.26-.31.38-.121.11-.25.22-.38.31-.14.09-.281.17-.43.23a2.565 2.565 0 0 1-.96.19" fill="#000"/></g></svg>
\ No newline at end of file
diff --git a/website/static/img/favicon.ico b/website/static/img/favicon.ico
new file mode 100644
index 000000000..1062669a8
Binary files /dev/null and b/website/static/img/favicon.ico differ
diff --git a/website/static/img/javascript_logo.svg b/website/static/img/javascript_logo.svg
new file mode 100644
index 000000000..9650ca78e
--- /dev/null
+++ b/website/static/img/javascript_logo.svg
@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 630 630">
+<rect width="630" height="630" fill="#f7df1e"/>
+<path d="m423.2 492.19c12.69 20.72 29.2 35.95 58.4 35.95 24.53 0 40.2-12.26 40.2-29.2 0-20.3-16.1-27.49-43.1-39.3l-14.8-6.35c-42.72-18.2-71.1-41-71.1-89.2 0-44.4 33.83-78.2 86.7-78.2 37.64 0 64.7 13.1 84.2 47.4l-46.1 29.6c-10.15-18.2-21.1-25.37-38.1-25.37-17.34 0-28.33 11-28.33 25.37 0 17.76 11 24.95 36.4 35.95l14.8 6.34c50.3 21.57 78.7 43.56 78.7 93 0 53.3-41.87 82.5-98.1 82.5-54.98 0-90.5-26.2-107.88-60.54zm-209.13 5.13c9.3 16.5 17.76 30.45 38.1 30.45 19.45 0 31.72-7.61 31.72-37.2v-201.3h59.2v202.1c0 61.3-35.94 89.2-88.4 89.2-47.4 0-74.85-24.53-88.81-54.075z"/>
+</svg>
\ No newline at end of file
diff --git a/website/static/img/oss_logo.png b/website/static/img/oss_logo.png
new file mode 100644
index 000000000..8183e289b
Binary files /dev/null and b/website/static/img/oss_logo.png differ
diff --git a/docs/puppeteer-live-view-dashboard.png b/website/static/img/puppeteer-live-view-dashboard.png
similarity index 100%
rename from docs/puppeteer-live-view-dashboard.png
rename to website/static/img/puppeteer-live-view-dashboard.png
diff --git a/docs/puppeteer-live-view-detail.png b/website/static/img/puppeteer-live-view-detail.png
similarity index 100%
rename from docs/puppeteer-live-view-detail.png
rename to website/static/img/puppeteer-live-view-detail.png
diff --git a/website/static/img/robot.png b/website/static/img/robot.png
new file mode 100644
index 000000000..9be65e62c
Binary files /dev/null and b/website/static/img/robot.png differ
diff --git a/website/static/img/source_code.png b/website/static/img/source_code.png
new file mode 100644
index 000000000..5eb9fc06c
Binary files /dev/null and b/website/static/img/source_code.png differ
diff --git a/website/tools/build_docs.js b/website/tools/build_docs.js
new file mode 100644
index 000000000..a182a29c6
--- /dev/null
+++ b/website/tools/build_docs.js
@@ -0,0 +1,160 @@
+const jsdoc2md = require('jsdoc-to-markdown'); // eslint-disable-line
+const readline = require('readline');
+const fs = require('fs');
+const path = require('path');
+const { promisify } = require('util');
+
+const writeFile = promisify(fs.writeFile);
+
+/* eslint-disable no-shadow */
+
+const classNames = [];
+const namespaces = [];
+const typedefs = [];
+
+const getHeader = (title) => {
+ const prefix = title === 'puppeteer' ? 'utils.' : '';
+ const id = title.replace(/\s/g, '').toLowerCase();
+ return `---\nid: ${id}\ntitle: ${prefix}${title}\n---\n`;
+};
+
+const getRenderOptions = (template, data) => ({
+ template,
+ data,
+ 'name-format': true,
+ 'param-list-format': 'table',
+ 'heading-depth': 1,
+ helper: [path.join(__dirname, 'helpers.js')],
+ partial: [
+ path.join(__dirname, 'partials', 'params-table.hbs'),
+ path.join(__dirname, 'partials', 'properties-table.hbs'),
+ ],
+});
+
+const readFileFromLine = async (path, lineNumber = 1) => {
+ return new Promise((resolve, reject) => {
+ const output = [];
+ const rl = readline.createInterface({
+ input: fs.createReadStream(path),
+ crlfDelay: Infinity,
+ });
+ let lineCounter = 0;
+ rl.on('line', (line) => {
+ lineCounter++;
+ if (lineCounter >= lineNumber) output.push(line);
+ });
+ rl.on('close', () => resolve(output.join('\n')));
+ rl.on('error', err => reject(err));
+ });
+};
+
+const generateFinalMarkdown = (title, text) => {
+ const header = getHeader(title);
+ // Remove Class titles so we don't have double page titles with Docusaurus.
+ const rx = new RegExp(`# \`?${title}\`?.*?\n`);
+ text = text.replace(rx, '');
+ // Remove 'Kind' annotations.
+ text = text.replace(/\*\*Kind\*\*.*\n/g, '');
+ // Remove dots in type annotations
+ const dotsRx = /([A-Z][a-z]+)\.(&lt;.+&gt;)/g;
+ const replacer = (match, p1, p2) => {
+ return p1 + p2.replace(dotsRx, replacer);
+ };
+ text = text.replace(dotsRx, replacer);
+ // Fix class links
+ const linksRx = new RegExp(`([("])#(module_)?(${classNames.join('|')})([)"])`, 'gi');
+ text = text.replace(linksRx, (match, p1, p2, p3, p4) => p1 + p3.toLowerCase() + p4);
+ // Fix typedef links
+ const typeLinkRx = new RegExp(`([("])#(module_)?(${typedefs.join('|')})([)"])`, 'gi');
+ text = text.replace(typeLinkRx, (match, p1, p2, p3, p4) => `${p1}../typedefs/${p3.toLowerCase()}${p4}`);
+
+ return header + text;
+};
+
+const main = async () => {
+ /* input and output paths */
+ const sourceFiles = path.join(__dirname, '..', '..', 'src', '**', '*.js');
+ const exampleFiles = path.join(__dirname, '..', '..', 'examples', '**', '*.js');
+ const sourceFilesOutputDir = path.join(__dirname, '..', '..', 'docs', 'api');
+ const typeFilesOutputDir = path.join(__dirname, '..', '..', 'docs', 'typedefs');
+ const exampleFilesOutputDir = path.join(__dirname, '..', '..', 'docs', 'examples');
+
+ /* get template data */
+ const templateData = jsdoc2md.getTemplateDataSync({ files: sourceFiles });
+ const exampleData = jsdoc2md.getTemplateDataSync({ files: exampleFiles });
+
+ // handle examples
+ const examplePromises = exampleData.map(async (example) => {
+ const { description, meta: { filename, path: filepath, lineno } } = example;
+ const code = await readFileFromLine(path.join(filepath, filename), lineno);
+ const sep = '```';
+ const codeblock = `${sep}javascript\n${code}\n${sep}`;
+
+ const title = filename.split('.')[0].split('_').map(word => `${word[0].toUpperCase()}${word.substr(1)}`).join(' ');
+ const header = getHeader(title);
+ const markdown = `${header}\n${description}\n${codeblock}`;
+ await writeFile(path.join(exampleFilesOutputDir, `${title.replace(/\s/g, '')}.md`), markdown);
+ });
+
+ await Promise.all(examplePromises);
+
+ /* reduce templateData to an array of class names */
+ templateData.forEach((identifier) => {
+ if (identifier.kind === 'class' && !identifier.ignore) classNames.push(identifier.name);
+ if (identifier.kind === 'namespace' && !identifier.ignore) namespaces.push(identifier.name);
+ if (identifier.kind === 'typedef' && !identifier.ignore) typedefs.push(identifier.name);
+ });
+
+
+ // create a doc file for Apify
+ const mainModule = 'Apify';
+ const template = `{{#module name="${mainModule}"}}{{>docs}}{{/module}}`;
+ console.log(`Rendering ${mainModule}, template: ${template}`); // eslint-disable-line no-console
+ const moduleData = [];
+ const otherData = [];
+ templateData.forEach((item) => {
+ if (item.id.startsWith('module:Apify')) moduleData.push(item);
+ else otherData.push(item);
+ });
+ const comparator = (a, b) => {
+ const nameA = a.name.toLowerCase();
+ const nameB = b.name.toLowerCase();
+ if (nameA < nameB) return -1;
+ if (nameA > nameB) return 1;
+ return 0;
+ };
+ const finalData = moduleData.sort(comparator).concat(otherData);
+
+ const output = jsdoc2md.renderSync(getRenderOptions(template, finalData));
+ const markdown = generateFinalMarkdown(mainModule, output);
+ fs.writeFileSync(path.resolve(sourceFilesOutputDir, `${mainModule}.md`), markdown);
+
+ // create a doc file file for each class
+ classNames.forEach((className) => {
+ const template = `{{#class name="${className}"}}{{>docs}}{{/class}}`;
+ console.log(`Rendering ${className}, template: ${template}`); // eslint-disable-line no-console
+ const output = jsdoc2md.renderSync(getRenderOptions(template, templateData));
+ const markdown = generateFinalMarkdown(className, output);
+ fs.writeFileSync(path.resolve(sourceFilesOutputDir, `${className}.md`), markdown);
+ });
+
+ // create a doc file file for each namespace
+ namespaces.forEach((name) => {
+ const template = `{{#namespace name="${name}"}}{{>docs}}{{/namespace}}`;
+ console.log(`Rendering ${name}, template: ${template}`); // eslint-disable-line no-console
+ const output = jsdoc2md.renderSync(getRenderOptions(template, templateData));
+ const markdown = generateFinalMarkdown(name, output);
+ fs.writeFileSync(path.resolve(sourceFilesOutputDir, `${name}.md`), markdown);
+ });
+
+ // create a doc file for each type
+ typedefs.forEach((name) => {
+ const template = `{{#identifier name="${name}"}}{{>docs}}{{/identifier}}`;
+ console.log(`Rendering ${name}, template: ${template}`); // eslint-disable-line no-console
+ const output = jsdoc2md.renderSync(getRenderOptions(template, templateData));
+ const markdown = generateFinalMarkdown(name, output);
+ fs.writeFileSync(path.resolve(typeFilesOutputDir, `${name}.md`), markdown);
+ });
+};
+
+main();
diff --git a/website/tools/helpers.js b/website/tools/helpers.js
new file mode 100644
index 000000000..919d61dd4
--- /dev/null
+++ b/website/tools/helpers.js
@@ -0,0 +1,16 @@
+const { inlineLinks } = require('dmd/helpers/helpers'); // eslint-disable-line
+const ddata = require('dmd/helpers/ddata'); // eslint-disable-line
+
+exports.inlineLinks = (text, options) => {
+ if (text) {
+ const links = ddata.parseLink(text);
+ links.forEach((link) => {
+ const linked = ddata._link(link.url, options); // eslint-disable-line no-underscore-dangle
+ if (link.caption === link.url) link.caption = linked.name;
+ if (linked.url) link.url = linked.url;
+ const url = link.url.includes('+') ? link.url : link.url.toLowerCase();
+ text = text.replace(link.original, `[\`${link.caption}\`](${url})`);
+ });
+ }
+ return text;
+};
diff --git a/website/tools/partials/params-table.hbs b/website/tools/partials/params-table.hbs
new file mode 100644
index 000000000..68a24c675
--- /dev/null
+++ b/website/tools/partials/params-table.hbs
@@ -0,0 +1,21 @@
+{{#if params ~}}
+<table>
+<thead>
+<tr>
+{{#each (tableHeadHtml params "name|Param" "type|Type" "defaultvalue|Default")}}<th>{{this}}</th>{{/each}}
+</tr>
+</thead>
+<tbody>
+{{#tableRow params "name" "type" "defaultvalue" "description" ~}}
+<tr>
+{{#if @col1}}<td><code>{{>param-table-name}}</code></td>{{/if~}}
+{{#if @col2}}<td>{{>linked-type-list types=type.names delimiter=" | " html=true}}</td>{{/if~}}
+{{#if @col3}}<td>{{>defaultvalue}}</td>{{/if}}
+</tr>
+<tr>
+{{#if @col4}}<td colspan="3">{{{md (inlineLinks description)}}}</td>{{/if~}}
+</tr>
+{{~/tableRow}}
+</tbody>
+</table>
+{{/if~}}
diff --git a/website/tools/partials/properties-table.hbs b/website/tools/partials/properties-table.hbs
new file mode 100644
index 000000000..1a91368a9
--- /dev/null
+++ b/website/tools/partials/properties-table.hbs
@@ -0,0 +1,21 @@
+{{#if properties ~}}**Properties**
+<table>
+<thead>
+<tr>
+{{#each (tableHeadHtml properties "name|Param" "type|Type" "defaultvalue|Default")}}<th>{{this}}</th>{{/each}}
+</tr>
+</thead>
+<tbody>
+{{#tableRow properties "name" "type" "defaultvalue" "description" ~}}
+<tr>
+{{#if @col1}}<td><code>{{>param-table-name}}</code></td>{{/if~}}
+{{#if @col2}}<td>{{>linked-type-list types=type.names delimiter=" | " html=true}}</td>{{/if~}}
+{{#if @col3}}<td>{{>defaultvalue}}</td>{{/if}}
+</tr>
+<tr>
+{{#if @col4}}<td colspan="3">{{{md (inlineLinks description)}}}</td>{{/if~}}
+</tr>
+{{~/tableRow}}
+</tbody>
+</table>
+{{/if~}}