Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
docs: tag v2.3 docs
Martin Adámek3348175464e3
diff --git a/website/i18n/en.json b/website/i18n/en.jsonindex 798c9fd15..1d9ccbd7a 100644--- a/website/i18n/en.json+++ b/website/i18n/en.json@@ -5,6 +5,144 @@"previous": "Previous","tagline": "The scalable web crawling, scraping and automation library for JavaScript/Node.js.","docs": {+ "api/apify": {+ "title": "Apify"+ },+ "api/apify-call-error": {+ "title": "ApifyCallError"+ },+ "api/autoscaled-pool": {+ "title": "AutoscaledPool"+ },+ "api/basic-crawler": {+ "title": "BasicCrawler"+ },+ "api/browser-plugin": {+ "title": "BrowserPlugin"+ },+ "api/cheerio-crawler": {+ "title": "CheerioCrawler"+ },+ "api/configuration": {+ "title": "Configuration"+ },+ "api/dataset": {+ "title": "Dataset"+ },+ "api/key-value-store": {+ "title": "KeyValueStore"+ },+ "api/log": {+ "title": "utils.log"+ },+ "api/playwright": {+ "title": "utils.playwright"+ },+ "api/playwright-crawler": {+ "title": "PlaywrightCrawler"+ },+ "api/proxy-configuration": {+ "title": "ProxyConfiguration"+ },+ "api/pseudo-url": {+ "title": "PseudoUrl"+ },+ "api/puppeteer": {+ "title": "utils.puppeteer"+ },+ "api/puppeteer-crawler": {+ "title": "PuppeteerCrawler"+ },+ "api/request": {+ "title": "Request"+ },+ "api/request-list": {+ "title": "RequestList"+ },+ "api/request-queue": {+ "title": "RequestQueue"+ },+ "api/session": {+ "title": "Session"+ },+ "api/session-pool": {+ "title": "SessionPool"+ },+ "api/snapshotter": {+ "title": "Snapshotter"+ },+ "api/social": {+ "title": "utils.social"+ },+ "api/statistics": {+ "title": "Statistics"+ },+ "api/system-status": {+ "title": "SystemStatus"+ },+ "api/utils": {+ "title": "utils"+ },+ "examples/accept-user-input": {+ "title": "Accept user input"+ },+ "examples/add-data-to-dataset": {+ "title": "Add data to dataset"+ },+ "examples/basic-crawler": {+ "title": "Basic crawler"+ },+ "examples/call-actor": {+ "title": "Call actor"+ },+ "examples/capture-screenshot": {+ "title": "Capture a screenshot"+ },+ "examples/cheerio-crawler": {+ "title": "Cheerio crawler"+ },+ "examples/crawl-all-links": {+ "title": "Crawl all links on a website"+ },+ "examples/crawl-multiple-urls": {+ "title": "Crawl multiple URLs"+ },+ "examples/crawl-relative-links": {+ "title": "Crawl a website with relative links"+ },+ "examples/crawl-single-url": {+ "title": "Crawl a single URL"+ },+ "examples/crawl-sitemap": {+ "title": "Crawl a sitemap"+ },+ "examples/crawl-some-links": {+ "title": "Crawl some links on a website"+ },+ "examples/forms": {+ "title": "Forms"+ },+ "examples/map-and-reduce": {+ "title": "Dataset Map and Reduce methods"+ },+ "examples/playwright-crawler": {+ "title": "Playwright crawler"+ },+ "examples/puppeteer-crawler": {+ "title": "Puppeteer crawler"+ },+ "examples/puppeteer-recursive-crawl": {+ "title": "Puppeteer recursive crawl"+ },+ "examples/puppeteer-with-proxy": {+ "title": "Puppeteer with proxy"+ },+ "examples/synchronous-run": {+ "title": "Synchronous run"+ },+ "examples/use-stealth-mode": {+ "title": "Use stealth mode"+ },"guides/apify-platform": {"title": "Apify Platform"},@@ -53,6 +191,225 @@"readme/support": {"title": "readme/support"},+ "typedefs/abort-function": {+ "title": "AbortFunction"+ },+ "typedefs/actor-run": {+ "title": "ActorRun"+ },+ "typedefs/adhoc-webhook": {+ "title": "AdhocWebhook"+ },+ "typedefs/apify-env": {+ "title": "ApifyEnv"+ },+ "typedefs/autoscaled-pool-options": {+ "title": "AutoscaledPoolOptions"+ },+ "typedefs/basic-crawler-options": {+ "title": "BasicCrawlerOptions"+ },+ "typedefs/browser-crawling-context": {+ "title": "BrowserCrawlingContext"+ },+ "typedefs/browser-handle-page-function": {+ "title": "BrowserHandlePageFunction"+ },+ "typedefs/browser-launch-context": {+ "title": "BrowserLaunchContext"+ },+ "typedefs/cheerio-crawler-options": {+ "title": "CheerioCrawlerOptions"+ },+ "typedefs/cheerio-handle-page": {+ "title": "CheerioHandlePage"+ },+ "typedefs/cheerio-handle-page-inputs": {+ "title": "CheerioHandlePageInputs"+ },+ "typedefs/compiled-script-function": {+ "title": "CompiledScriptFunction"+ },+ "typedefs/compiled-script-params": {+ "title": "CompiledScriptParams"+ },+ "typedefs/crawling-context": {+ "title": "CrawlingContext"+ },+ "typedefs/create-session": {+ "title": "CreateSession"+ },+ "typedefs/dataset-consumer": {+ "title": "DatasetConsumer"+ },+ "typedefs/dataset-content": {+ "title": "DatasetContent"+ },+ "typedefs/dataset-mapper": {+ "title": "DatasetMapper"+ },+ "typedefs/dataset-reducer": {+ "title": "DatasetReducer"+ },+ "typedefs/direct-navigation-options": {+ "title": "DirectNavigationOptions"+ },+ "typedefs/event-types": {+ "title": "EventTypes"+ },+ "typedefs/goto-function": {+ "title": "GotoFunction"+ },+ "typedefs/handle-failed-request": {+ "title": "HandleFailedRequest"+ },+ "typedefs/handle-failed-request-input": {+ "title": "HandleFailedRequestInput"+ },+ "typedefs/handle-request": {+ "title": "HandleRequest"+ },+ "typedefs/handle-request-inputs": {+ "title": "HandleRequestInputs"+ },+ "typedefs/hook": {+ "title": "Hook"+ },+ "typedefs/intercept-handler": {+ "title": "InterceptHandler"+ },+ "typedefs/key-consumer": {+ "title": "KeyConsumer"+ },+ "typedefs/key-value-store-value-types": {+ "title": "KeyValueStoreValueTypes"+ },+ "typedefs/logger-options": {+ "title": "LoggerOptions"+ },+ "typedefs/memory-info": {+ "title": "MemoryInfo"+ },+ "typedefs/playwright-crawler-options": {+ "title": "PlaywrightCrawlerOptions"+ },+ "typedefs/playwright-goto-options": {+ "title": "PlaywrightGotoOptions"+ },+ "typedefs/playwright-handle-page-function": {+ "title": "PlaywrightHandlePageFunction"+ },+ "typedefs/playwright-handle-page-function-param": {+ "title": "PlaywrightHandlePageFunctionParam"+ },+ "typedefs/playwright-hook": {+ "title": "PlaywrightHook"+ },+ "typedefs/playwright-launch-context": {+ "title": "PlaywrightLaunchContext"+ },+ "typedefs/post-response": {+ "title": "PostResponse"+ },+ "typedefs/post-response-inputs": {+ "title": "PostResponseInputs"+ },+ "typedefs/prepare-request": {+ "title": "PrepareRequest"+ },+ "typedefs/prepare-request-inputs": {+ "title": "PrepareRequestInputs"+ },+ "typedefs/proxy-configuration-function": {+ "title": "ProxyConfigurationFunction"+ },+ "typedefs/proxy-configuration-options": {+ "title": "ProxyConfigurationOptions"+ },+ "typedefs/proxy-info": {+ "title": "ProxyInfo"+ },+ "typedefs/puppeteer-cookie": {+ "title": "PuppeteerCookie"+ },+ "typedefs/puppeteer-crawler-options": {+ "title": "PuppeteerCrawlerOptions"+ },+ "typedefs/puppeteer-handle-page": {+ "title": "PuppeteerHandlePage"+ },+ "typedefs/puppeteer-handle-page-function-param": {+ "title": "PuppeteerHandlePageFunctionParam"+ },+ "typedefs/puppeteer-hook": {+ "title": "PuppeteerHook"+ },+ "typedefs/puppeteer-launch-context": {+ "title": "PuppeteerLaunchContext"+ },+ "typedefs/queue-operation-info": {+ "title": "QueueOperationInfo"+ },+ "typedefs/request-as-browser-options": {+ "title": "RequestAsBrowserOptions"+ },+ "typedefs/request-as-browser-result": {+ "title": "RequestAsBrowserResult"+ },+ "typedefs/request-list-options": {+ "title": "RequestListOptions"+ },+ "typedefs/request-list-sources-function": {+ "title": "RequestListSourcesFunction"+ },+ "typedefs/request-list-state": {+ "title": "RequestListState"+ },+ "typedefs/request-options": {+ "title": "RequestOptions"+ },+ "typedefs/request-queue-info": {+ "title": "RequestQueueInfo"+ },+ "typedefs/request-transform": {+ "title": "RequestTransform"+ },+ "typedefs/session-options": {+ "title": "SessionOptions"+ },+ "typedefs/session-pool-options": {+ "title": "SessionPoolOptions"+ },+ "typedefs/session-state": {+ "title": "SessionState"+ },+ "typedefs/snapshotter-options": {+ "title": "SnapshotterOptions"+ },+ "typedefs/social-handles": {+ "title": "SocialHandles"+ },+ "typedefs/statistic-persisted-state": {+ "title": "StatisticPersistedState"+ },+ "typedefs/statistic-state": {+ "title": "StatisticState"+ },+ "typedefs/stealth-options": {+ "title": "StealthOptions"+ },+ "typedefs/system-info": {+ "title": "SystemInfo"+ },+ "typedefs/system-status-options": {+ "title": "SystemStatusOptions"+ },+ "typedefs/user-func": {+ "title": "UserFunc"+ },+ "typedefs/webhook-run": {+ "title": "WebhookRun"+ },"version-0.22.4/api/version-0.22.4-apify": {"title": "Apify"},diff --git a/website/versioned_docs/version-2.3.0/api/Apify.md b/website/versioned_docs/version-2.3.0/api/Apify.mdnew file mode 100644index 000000000..07d8b9063--- /dev/null+++ b/website/versioned_docs/version-2.3.0/api/Apify.md@@ -0,0 +1,812 @@+---+id: version-2.3.0-apify+title: Apify+original_id: apify+---++<a name="apify"></a>++The following section describes all functions and properties provided by the `apify` package, except individual classes and namespaces that have their+separate, detailed, documentation pages accessible from the left sidebar. To learn how Apify SDK works, we suggest following the+[Getting Started](../guides/getting-started) tutorial.++**Important:**++> The following functions: `addWebhook`, `call`, `callTask` and `newClient` invoke features of the [Apify platform](../guides/apify-platform) and+> require your scripts to be authenticated. See the [authentication guide](../guides/apify-platform#logging-into-apify-platform-from-apify-sdk) for+> instructions.++## `Apify` Class++As opposed to those helper functions, there is an alternative approach using `Apify` class (a named export). It has mostly the same API, but the+methods on `Apify` instance will use the configuration provided in the constructor. Environment variables will have precedence over this+configuration.++```js+const { Apify } = require('apify'); // use named export to get the class++const sdk = new Apify({ token: '123' });+console.log(sdk.config.get('token')); // '123'++// the token will be passed to the `call` method automatically+const run = await sdk.call('apify/hello-world', { myInput: 123 });+console.log(`Received message: ${run.output.body.message}`);+```++Another example shows how the default dataset name can be changed:++```js+const { Apify } = require('apify'); // use named export to get the class++const sdk = new Apify({ defaultDatasetId: 'custom-name' });+await sdk.pushData({ myValue: 123 });+```++is equivalent to:++```js+const Apify = require('apify'); // use default export to get the helper functions++const dataset = await Apify.openDataset('custom-name');+await dataset.pushData({ myValue: 123 });+```++See [`Configuration`](../api/configuration) for details about what can be configured and what are the default values.++---++<a name="addwebhook"></a>++## `Apify.addWebhook(options)`++Creates an ad-hoc webhook for the current actor run, which lets you receive a notification when the actor run finished or failed. For more information+about Apify actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).++Note that webhooks are only supported for actors running on the Apify platform. In local environment, the function will print a warning and have no+effect.++**Parameters**:++- **`options`**: `object`+ - **`eventTypes`**: [`EventTypes`](../typedefs/event-types) - Array of event types, which you can set for actor run, see the+ [actor run events](https://docs.apify.com/webhooks/events#actor-run) in the Apify doc.+ - **`requestUrl`**: `string` - URL which will be requested using HTTP POST request, when actor run will reach the set event type.+ - **`[payloadTemplate]`**: `string` - Payload template is a JSON-like string that describes the structure of the webhook POST request payload.+ It uses JSON syntax, extended with a double curly braces syntax for injecting variables `{{variable}}`. Those variables are resolved at the+ time of the webhook's dispatch, and a list of available variables with their descriptions is available in the+ [Apify webhook documentation](https://docs.apify.com/webhooks). If `payloadTemplate` is omitted, the default payload template is used+ ([view docs](https://docs.apify.com/webhooks/actions#payload-template)).+ - **`[idempotencyKey]`**: `string` - Idempotency key enables you to ensure that a webhook will not be added multiple times in case of an actor+ restart or other situation that would cause the `addWebhook()` function to be called again. We suggest using the actor run ID as the+ idempotency key. You can get the run ID by calling [`Apify.getEnv()`](../api/apify#getenv) function.++**Returns**:++[`Promise<(WebhookRun|undefined)>`](../typedefs/webhook-run) - The return value is the Webhook object. For more information, see the+[Get webhook](https://apify.com/docs/api/v2#/reference/webhooks/webhook-object/get-webhook) API endpoint.++---++<a name="call"></a>++## `Apify.call(actId, [input], [options])`++Runs an actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable), waits for the actor to+finish and fetches its output.++By passing the `waitSecs` option you can reduce the maximum amount of time to wait for the run to finish. If the value is less than or equal to zero,+the function returns immediately after the run is started.++The result of the function is an [`ActorRun`](../typedefs/actor-run) object that contains details about the actor run and its output (if any). If the+actor run fails, the function throws the [`ApifyCallError`](../api/apify-call-error) exception.++If you want to run an actor task rather than an actor, please use the [`Apify.callTask()`](../api/apify#calltask) function instead.++For more information about actors, read the [documentation](https://docs.apify.com/actor).++**Example usage:**++```javascript+const run = await Apify.call('apify/hello-world', { myInput: 123 });+console.log(`Received message: ${run.output.body.message}`);+```++Internally, the `call()` function invokes the [Run actor](https://apify.com/docs/api/v2#/reference/actors/run-collection/run-actor) and several other+API endpoints to obtain the output.++**Throws**:++- [`ApifyCallError`](../api/apify-call-error) If the run did not succeed, e.g. if it failed or timed out.++**Parameters**:++- **`actId`**: `string` - Allowed formats are `username/actor-name`, `userId/actor-name` or actor ID.+- **`[input]`**: `Object<string, *>` - Input for the actor. If it is an object, it will be stringified to JSON and its content type set to+ `application/json; charset=utf-8`. Otherwise the `options.contentType` parameter must be provided.+- **`[options]`**: `object` - Object with the settings below:+ - **`[contentType]`**: `string` - Content type for the `input`. If not specified, `input` is expected to be an object that will be stringified+ to JSON and content type set to `application/json; charset=utf-8`. If `options.contentType` is specified, then `input` must be a `String` or+ `Buffer`.+ - **`[token]`**: `string` - User API token that is used to run the actor. By default, it is taken from the `APIFY_TOKEN` environment variable.+ - **`[memoryMbytes]`**: `number` - Memory in megabytes which will be allocated for the new actor run. If not provided, the run uses memory of+ the default actor run configuration.+ - **`[timeoutSecs]`**: `number` - Timeout for the actor run in seconds. Zero value means there is no timeout. If not provided, the run uses+ timeout of the default actor run configuration.+ - **`[build]`**: `string` - Tag or number of the actor build to run (e.g. `beta` or `1.2.345`). If not provided, the run uses build tag or+ number from the default actor run configuration (typically `latest`).+ - **`[waitSecs]`**: `number` - Maximum time to wait for the actor run to finish, in seconds. If the limit is reached, the returned promise is+ resolved to a run object that will have status `READY` or `RUNNING` and it will not contain the actor run output. If `waitSecs` is null or+ undefined, the function waits for the actor to finish (default behavior).+ - **`[fetchOutput]`**: `boolean` <code> = true</code> - If `false` then the function does not fetch output of the actor.+ - **`[disableBodyParser]`**: `boolean` <code> = false</code> - If `true` then the function will not attempt to parse the actor's output and will+ return it in a raw `Buffer`.+ - **`[webhooks]`**: [`Array<AdhocWebhook>`](../typedefs/adhoc-webhook) - Specifies optional webhooks associated with the actor run, which can be+ used to receive a notification e.g. when the actor finished or failed, see+ [ad hook webhooks documentation](https://docs.apify.com/webhooks/ad-hoc-webhooks) for detailed description.++**Returns**:++[`Promise<ActorRun>`](../typedefs/actor-run)++---++<a name="calltask"></a>++## `Apify.callTask(taskId, [input], [options])`++Runs an actor task on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable), waits for the task to+finish and fetches its output.++By passing the `waitSecs` option you can reduce the maximum amount of time to wait for the run to finish. If the value is less than or equal to zero,+the function returns immediately after the run is started.++The result of the function is an [`ActorRun`](../typedefs/actor-run) object that contains details about the actor run and its output (if any). If the+actor run failed, the function fails with [`ApifyCallError`](../api/apify-call-error) exception.++Note that an actor task is a saved input configuration and options for an actor. If you want to run an actor directly rather than an actor task,+please use the [`Apify.call()`](../api/apify#call) function instead.++For more information about actor tasks, read the [documentation](https://docs.apify.com/tasks).++**Example usage:**++```javascript+const run = await Apify.callTask('bob/some-task');+console.log(`Received message: ${run.output.body.message}`);+```++Internally, the `callTask()` function calls the [Run task](https://apify.com/docs/api/v2#/reference/actor-tasks/run-collection/run-task) and several+other API endpoints to obtain the output.++**Throws**:++- [`ApifyCallError`](../api/apify-call-error) If the run did not succeed, e.g. if it failed or timed out.++**Parameters**:++- **`taskId`**: `string` - Allowed formats are `username/task-name`, `userId/task-name` or task ID.+- **`[input]`**: `Object<string, *>` - Input overrides for the actor task. If it is an object, it will be stringified to JSON and its content type+ set to `application/json; charset=utf-8`. Provided input will be merged with actor task input.+- **`[options]`**: `object` - Object with the settings below:+ - **`[token]`**: `string` - User API token that is used to run the actor. By default, it is taken from the `APIFY_TOKEN` environment variable.+ - **`[memoryMbytes]`**: `number` - Memory in megabytes which will be allocated for the new actor task run. If not provided, the run uses memory+ of the default actor run configuration.+ - **`[timeoutSecs]`**: `number` - Timeout for the actor task run in seconds. Zero value means there is no timeout. If not provided, the run uses+ timeout of the default actor run configuration.+ - **`[build]`**: `string` - Tag or number of the actor build to run (e.g. `beta` or `1.2.345`). If not provided, the run uses build tag or+ number from the default actor run configuration (typically `latest`).+ - **`[waitSecs]`**: `number` - Maximum time to wait for the actor task run to finish, in seconds. If the limit is reached, the returned promise+ is resolved to a run object that will have status `READY` or `RUNNING` and it will not contain the actor run output. If `waitSecs` is null or+ undefined, the function waits for the actor task to finish (default behavior).+ - **`[webhooks]`**: [`Array<AdhocWebhook>`](../typedefs/adhoc-webhook) - Specifies optional webhooks associated with the actor run, which can be+ used to receive a notification e.g. when the actor finished or failed, see+ [ad hook webhooks documentation](https://docs.apify.com/webhooks/ad-hoc-webhooks) for detailed description.++**Returns**:++[`Promise<ActorRun>`](../typedefs/actor-run)++---++<a name="createproxyconfiguration"></a>++## `Apify.createProxyConfiguration([proxyConfigurationOptions])`++Creates a proxy configuration and returns a promise resolving to an instance of the [`ProxyConfiguration`](../api/proxy-configuration) class that is+already initialized.++Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking your crawlers based+on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures them to use the selected proxies for+all connections.++For more details and code examples, see the [`ProxyConfiguration`](../api/proxy-configuration) class.++```javascript++// Returns initialized proxy configuration class+const proxyConfiguration = await Apify.createProxyConfiguration({+ groups: ['GROUP1', 'GROUP2'] // List of Apify proxy groups+ countryCode: 'US'+});++const crawler = new Apify.CheerioCrawler({+ // ...+ proxyConfiguration,+ handlePageFunction: ({ proxyInfo }) => {+ const usedProxyUrl = proxyInfo.url; // Getting the proxy URL+ }+})++```++For compatibility with existing Actor Input UI (Input Schema), this function returns `undefined` when the following object is passed as+`proxyConfigurationOptions`.++```+{ useApifyProxy: false }+```++**Parameters**:++- **`[proxyConfigurationOptions]`**: [`ProxyConfigurationOptions`](../typedefs/proxy-configuration-options)++**Returns**:++[`Promise<(ProxyConfiguration|undefined)>`](../api/proxy-configuration)++---++<a name="events"></a>++## `Apify.events`++Gets an instance of a Node.js' [EventEmitter](https://nodejs.org/api/events.html#events_class_eventemitter) class that emits various events from the+SDK or the Apify platform. The event emitter is initialized by calling the [`Apify.main()`](../api/apify#main) function.++**Example usage:**++```javascript+Apify.events.on('cpuInfo', data => {+ if (data.isCpuOverloaded) console.log('Oh no, the CPU is overloaded!');+});+```++The following events are emitted:++- `cpuInfo`: `{ "isCpuOverloaded": Boolean }` The event is emitted approximately every second and it indicates whether the actor is using the+ maximum of available CPU resources. If that's the case, the actor should not add more workload. For example, this event is used by the+ [`AutoscaledPool`](../api/autoscaled-pool) class.+- `migrating`: `void` Emitted when the actor running on the Apify platform is going to be migrated to another worker server soon. You can use it to+ persist the state of the actor and abort the run, to speed up migration. For example, this is used by the [`RequestList`](../api/request-list)+ class.+- `aborting`: `void` When a user aborts an actor run on the Apify platform, they can choose to abort gracefully to allow the actor some time before+ getting killed. This graceful abort emits the `aborting` event which the SDK uses to gracefully stop running crawls and you can use it to do your+ own cleanup as well.+- `persistState`: `{ "isMigrating": Boolean }` Emitted in regular intervals (by default 60 seconds) to notify all components of Apify SDK that it is+ time to persist their state, in order to avoid repeating all work when the actor restarts. This event is automatically emitted together with the+ `migrating` event, in which case the `isMigrating` flag is set to `true`. Otherwise the flag is `false`. Note that the `persistState` event is+ provided merely for user convenience, you can achieve the same effect using `setInterval()` and listening for the `migrating` event.++---++<a name="getenv"></a>++## `Apify.getEnv()`++Returns a new [`ApifyEnv`](../typedefs/apify-env) object which contains information parsed from all the `APIFY_XXX` environment variables.++For the list of the `APIFY_XXX` environment variables, see [Actor documentation](https://docs.apify.com/actor/run#environment-variables). If some of+the variables are not defined or are invalid, the corresponding value in the resulting object will be null.++**Returns**:++[`ApifyEnv`](../typedefs/apify-env)++---++<a name="getinput"></a>++## `Apify.getInput()`++Gets the actor input value from the default [`KeyValueStore`](../api/key-value-store) associated with the current actor run.++This is just a convenient shortcut for [`keyValueStore.getValue('INPUT')`](key-value-store#getvalue). For example, calling the following code:++```javascript+const input = await Apify.getInput();+```++is equivalent to:++```javascript+const store = await Apify.openKeyValueStore();+await store.getValue('INPUT');+```++Note that the `getInput()` function does not cache the value read from the key-value store. If you need to use the input multiple times in your actor,+it is far more efficient to read it once and store it locally.++For more information, see [`Apify.openKeyValueStore()`](../api/apify#openkeyvaluestore) and+[`KeyValueStore.getValue()`](../api/key-value-store#getvalue).++**Returns**:++`Promise<(Object<string, *>|string|Buffer|null)>` - Returns a promise that resolves to an object, string or+[`Buffer`](https://nodejs.org/api/buffer.html), depending on the MIME content type of the record, or `null` if the record is missing.++---++<a name="getmemoryinfo"></a>++## `Apify.getMemoryInfo()`++Returns memory statistics of the process and the system, see [`MemoryInfo`](../typedefs/memory-info).++If the process runs inside of Docker, the `getMemoryInfo` gets container memory limits, otherwise it gets system memory limits.++Beware that the function is quite inefficient because it spawns a new process. Therefore you shouldn't call it too often, like more than once per+second.++**Returns**:++[`Promise<MemoryInfo>`](../typedefs/memory-info)++---++<a name="getvalue"></a>++## `Apify.getValue(key)`++Gets a value from the default [`KeyValueStore`](../api/key-value-store) associated with the current actor run.++This is just a convenient shortcut for [`KeyValueStore.getValue()`](../api/key-value-store#getvalue). For example, calling the following code:++```javascript+const value = await Apify.getValue('my-key');+```++is equivalent to:++```javascript+const store = await Apify.openKeyValueStore();+const value = await store.getValue('my-key');+```++To store the value to the default key-value store, you can use the [`Apify.setValue()`](../api/apify#setvalue) function.++For more information, see [`Apify.openKeyValueStore()`](../api/apify#openkeyvaluestore) and+[`KeyValueStore.getValue()`](../api/key-value-store#getvalue).++**Parameters**:++- **`key`**: `string` - Unique record key.++**Returns**:++`Promise<(Object<string, *>|string|Buffer|null)>` - Returns a promise that resolves to an object, string or+[`Buffer`](https://nodejs.org/api/buffer.html), depending on the MIME content type of the record, or `null` if the record is missing.++---++<a name="isathome"></a>++## `Apify.isAtHome()`++Returns `true` when code is running on Apify platform and `false` otherwise (for example locally).++**Returns**:++`boolean`++---++<a name="launchplaywright"></a>++## `Apify.launchPlaywright([launchContext])`++Launches headless browsers using Playwright pre-configured to work within the Apify platform. The function has the same return value as+`browserType.launch()`. See <a href="https://playwright.dev/docs/api/class-browsertype" target="_blank"> Playwright documentation</a> for more+details.++The `launchPlaywright()` function alters the following Playwright options:++- Passes the setting from the `APIFY_HEADLESS` environment variable to the `headless` option, unless it was already defined by the caller or+ `APIFY_XVFB` environment variable is set to `1`. Note that Apify Actor cloud platform automatically sets `APIFY_HEADLESS=1` to all running actors.+- Takes the `proxyUrl` option, validates it and adds it to `launchOptions` in a proper format. The proxy URL must define a port number and have one+ of the following schemes: `http://`, `https://`, `socks4://` or `socks5://`. If the proxy is HTTP (i.e. has the `http://` scheme) and contains+ username or password, the `launchPlaywright` functions sets up an anonymous proxy HTTP to make the proxy work with headless Chrome. For more+ information, read the+ <a href="https://blog.apify.com/how-to-make-headless-chrome-and-puppeteer-use-a-proxy-server-with-authentication-249a21a79212"+ target="_blank">blog post about proxy-chain library</a>.++To use this function, you need to have the [Playwright](https://www.npmjs.com/package/playwright) NPM package installed in your project. When running+on the Apify Platform, you can achieve that simply by using the `apify/actor-node-playwright-*` base Docker image for your actor - see+[Apify Actor documentation](https://docs.apify.com/actor/build#base-images) for details.++**Parameters**:++- **`[launchContext]`**: [`PlaywrightLaunchContext`](../typedefs/playwright-launch-context) - Optional settings passed to `browserType.launch()`. In+ addition to [Playwright's options](https://playwright.dev/docs/api/class-browsertype?_highlight=launch#browsertypelaunchoptions) the object may+ contain our own [`PlaywrightLaunchContext`](../typedefs/playwright-launch-context) that enable additional features.++**Returns**:++`Promise<*>` - Promise that resolves to Playwright's `Browser` instance.++---++<a name="launchpuppeteer"></a>++## `Apify.launchPuppeteer([launchContext])`++Launches headless Chrome using Puppeteer pre-configured to work within the Apify platform. The function has the same argument and the return value as+`puppeteer.launch()`. See <a href="https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#puppeteerlaunchoptions" target="_blank"> Puppeteer+documentation</a> for more details.++The `launchPuppeteer()` function alters the following Puppeteer options:++- Passes the setting from the `APIFY_HEADLESS` environment variable to the `headless` option, unless it was already defined by the caller or+ `APIFY_XVFB` environment variable is set to `1`. Note that Apify Actor cloud platform automatically sets `APIFY_HEADLESS=1` to all running actors.+- Takes the `proxyUrl` option, validates it and adds it to `args` as `--proxy-server=XXX`. The proxy URL must define a port number and have one of+ the following schemes: `http://`, `https://`, `socks4://` or `socks5://`. If the proxy is HTTP (i.e. has the `http://` scheme) and contains+ username or password, the `launchPuppeteer` functions sets up an anonymous proxy HTTP to make the proxy work with headless Chrome. For more+ information, read the+ <a href="https://blog.apify.com/how-to-make-headless-chrome-and-puppeteer-use-a-proxy-server-with-authentication-249a21a79212"+ target="_blank">blog post about proxy-chain library</a>.++To use this function, you need to have the [puppeteer](https://www.npmjs.com/package/puppeteer) NPM package installed in your project. When running on+the Apify cloud, you can achieve that simply by using the `apify/actor-node-chrome` base Docker image for your actor - see+[Apify Actor documentation](https://docs.apify.com/actor/build#base-images) for details.++For an example of usage, see the [Synchronous run Example](../examples/synchronous-run) or the+[Puppeteer proxy Example](../examples/puppeteer-with-proxy)++**Parameters**:++- **`[launchContext]`**: [`PuppeteerLaunchContext`](../typedefs/puppeteer-launch-context) - All `PuppeteerLauncher` parameters are passed via an+ launchContext object. If you want to pass custom `puppeteer.launch(options)` options you can use the `PuppeteerLaunchContext.launchOptions`+ property.++**Returns**:++`Promise<*>` - Promise that resolves to Puppeteer's `Browser` instance.++---++<a name="main"></a>++## `Apify.main(userFunc)`++Runs the main user function that performs the job of the actor and terminates the process when the user function finishes.++**The `Apify.main()` function is optional** and is provided merely for your convenience. It is mainly useful when you're running your code as an actor+on the [Apify platform](https://apify.com/actors). However, if you want to use Apify SDK tools directly inside your existing projects, e.g. running in+an [Express](https://expressjs.com/) server, on [Google Cloud functions](https://cloud.google.com/functions) or+[AWS Lambda](https://aws.amazon.com/lambda/), it's better to avoid it since the function terminates the main process when it finishes!++The `Apify.main()` function performs the following actions:++- When running on the Apify platform (i.e. <code>APIFY_IS_AT_HOME</code> environment variable is set), it sets up a connection to listen for+ platform events. For example, to get a notification about an imminent migration to another server. See [`Apify.events`](../api/apify#events) for+ details.+- It checks that either <code>APIFY_TOKEN</code> or <code>APIFY_LOCAL_STORAGE_DIR</code> environment variable is defined. If not, the functions sets+ <code>APIFY_LOCAL_STORAGE_DIR</code> to <code>./apify_storage</code> inside the current working directory. This is to simplify running code+ examples.+- It invokes the user function passed as the <code>userFunc</code> parameter.+- If the user function returned a promise, waits for it to resolve.+- If the user function throws an exception or some other error is encountered, prints error details to console so that they are stored to the log.+- Exits the Node.js process, with zero exit code on success and non-zero on errors.++The user function can be synchronous:++```javascript+Apify.main(() => {+ // My synchronous function that returns immediately+ console.log('Hello world from actor!');+});+```++If the user function returns a promise, it is considered asynchronous:++```javascript+const { requestAsBrowser } = require('some-request-library');++Apify.main(() => {+ // My asynchronous function that returns a promise+ return request('http://www.example.com').then(html => {+ console.log(html);+ });+});+```++To simplify your code, you can take advantage of the `async`/`await` keywords:++```javascript+const request = require('some-request-library');++Apify.main(async () => {+ // My asynchronous function+ const html = await request('http://www.example.com');+ console.log(html);+});+```++**Parameters**:++- **`userFunc`**: [`UserFunc`](../typedefs/user-func) - User function to be executed. If it returns a promise, the promise will be awaited. The user+ function is called with no arguments.++---++<a name="metamorph"></a>++## `Apify.metamorph(targetActorId, [input], [options])`++Transforms this actor run to an actor run of a given actor. The system stops the current container and starts the new container instead. All the+default storages are preserved and the new input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store.++**Parameters**:++- **`targetActorId`**: `string` - Either `username/actor-name` or actor ID of an actor to which we want to metamorph.+- **`[input]`**: `Object<string, *>` - Input for the actor. If it is an object, it will be stringified to JSON and its content type set to+ `application/json; charset=utf-8`. Otherwise the `options.contentType` parameter must be provided.+- **`[options]`**: `object` - Object with the settings below:+ - **`[contentType]`**: `string` - Content type for the `input`. If not specified, `input` is expected to be an object that will be stringified+ to JSON and content type set to `application/json; charset=utf-8`. If `options.contentType` is specified, then `input` must be a `String` or+ `Buffer`.+ - **`[build]`**: `string` - Tag or number of the target actor build to metamorph into (e.g. `beta` or `1.2.345`). If not provided, the run uses+ build tag or number from the default actor run configuration (typically `latest`).++**Returns**:++`Promise<void>`++---++<a name="newclient"></a>++## `Apify.newClient([options])`++Returns a new instance of the Apify API client. The `ApifyClient` class is provided by the+<a href="https://www.npmjs.com/package/apify-client" target="_blank">apify-client</a> NPM package, and it is automatically configured using the+`APIFY_API_BASE_URL`, and `APIFY_TOKEN` environment variables. You can override the token via the available options. That's useful if you want to use+the client as a different Apify user than the SDK internals are using.++**Parameters**:++- **`[options]`**: `object`+ - **`[token]`**: `string`+ - **`[maxRetries]`**: `string`+ - **`[minDelayBetweenRetriesMillis]`**: `string`++**Returns**:++[`ApifyClient`](../api/apify)++---++<a name="opendataset"></a>++## `Apify.openDataset([datasetIdOrName], [options])`++Opens a dataset and returns a promise resolving to an instance of the [`Dataset`](../api/dataset) class.++Datasets are used to store structured data where each object stored has the same attributes, such as online store products or real estate offers. The+actual data is stored either on the local filesystem or in the cloud.++For more details and code examples, see the [`Dataset`](../api/dataset) class.++**Parameters**:++- **`[datasetIdOrName]`**: `string` - ID or name of the dataset to be opened. If `null` or `undefined`, the function returns the default dataset+ associated with the actor run.+- **`[options]`**: `Object`+ - **`[forceCloud]`**: `boolean` <code> = false</code> - If set to `true` then the function uses cloud storage usage even if the+ `APIFY_LOCAL_STORAGE_DIR` environment variable is set. This way it is possible to combine local and cloud storage.+ - **`[config]`**: [`Configuration`](../api/configuration) - SDK configuration instance, defaults to the static register++**Returns**:++[`Promise<Dataset>`](../api/dataset)++---++<a name="openkeyvaluestore"></a>++## `Apify.openKeyValueStore([storeIdOrName], [options])`++Opens a key-value store and returns a promise resolving to an instance of the [`KeyValueStore`](../api/key-value-store) class.++Key-value stores are used to store records or files, along with their MIME content type. The records are stored and retrieved using a unique key. The+actual data is stored either on a local filesystem or in the Apify cloud.++For more details and code examples, see the [`KeyValueStore`](../api/key-value-store) class.++**Parameters**:++- **`[storeIdOrName]`**: `string` - ID or name of the key-value store to be opened. If `null` or `undefined`, the function returns the default+ key-value store associated with the actor run.+- **`[options]`**: `object`+ - **`[forceCloud]`**: `boolean` <code> = false</code> - If set to `true` then the function uses cloud storage usage even if the+ `APIFY_LOCAL_STORAGE_DIR` environment variable is set. This way it is possible to combine local and cloud storage.+ - **`[config]`**: [`Configuration`](../api/configuration) - SDK configuration instance, defaults to the static register++**Returns**:++[`Promise<KeyValueStore>`](../api/key-value-store)++---++<a name="openrequestlist"></a>++## `Apify.openRequestList(listName, sources, [options])`++Opens a request list and returns a promise resolving to an instance of the [`RequestList`](../api/request-list) class that is already initialized.++[`RequestList`](../api/request-list) represents a list of URLs to crawl, which is always stored in memory. To enable picking up where left off after a+process restart, the request list sources are persisted to the key-value store at initialization of the list. Then, while crawling, a small state+object is regularly persisted to keep track of the crawling status.++For more details and code examples, see the [`RequestList`](../api/request-list) class.++**Example usage:**++```javascript+const sources = ['https://www.example.com', 'https://www.google.com', 'https://www.bing.com'];++const requestList = await Apify.openRequestList('my-name', sources);+```++**Parameters**:++- **`listName`**: `string` | `null` - Name of the request list to be opened. Setting a name enables the `RequestList`'s state to be persisted in the+ key-value store. This is useful in case of a restart or migration. Since `RequestList` is only stored in memory, a restart or migration wipes it+ clean. Setting a name will enable the `RequestList`'s state to survive those situations and continue where it left off.++ The name will be used as a prefix in key-value store, producing keys such as `NAME-REQUEST_LIST_STATE` and `NAME-REQUEST_LIST_SOURCES`.++ If `null`, the list will not be persisted and will only be stored in memory. Process restart will then cause the list to be crawled again from the+ beginning. We suggest always using a name.++- **`sources`**: [`Array<(RequestOptions|Request|string)>`](../typedefs/request-options) - An array of sources of URLs for the+ [`RequestList`](../api/request-list). It can be either an array of strings, plain objects that define at least the `url` property, or an array of+ [`Request`](../api/request) instances.++ **IMPORTANT:** The `sources` array will be consumed (left empty) after [`RequestList`](../api/request-list) initializes. This is a measure to+ prevent memory leaks in situations when millions of sources are added.++Additionally, the `requestsFromUrl` property may be used instead of `url`, which will instruct [`RequestList`](../api/request-list) to download the+source URLs from a given remote location. The URLs will be parsed from the received response. In this case you can limit the URLs using `regex`+parameter containing regular expression pattern for URLs to be included.++For details, see the [`RequestListOptions.sources`](../typedefs/request-list-options#sources)++- **`[options]`**: [`RequestListOptions`](../typedefs/request-list-options) - The [`RequestList`](../api/request-list) options. Note that the+ `listName` parameter supersedes the [`RequestListOptions.persistStateKey`](../typedefs/request-list-options#persiststatekey) and+ [`RequestListOptions.persistRequestsKey`](../typedefs/request-list-options#persistrequestskey) options and the `sources` parameter supersedes the+ [`RequestListOptions.sources`](../typedefs/request-list-options#sources) option.++**Returns**:++[`Promise<RequestList>`](../api/request-list)++---++<a name="openrequestqueue"></a>++## `Apify.openRequestQueue([queueIdOrName], [options])`++Opens a request queue and returns a promise resolving to an instance of the [`RequestQueue`](../api/request-queue) class.++[`RequestQueue`](../api/request-queue) represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud. The queue is+used for deep crawling of websites, where you start with several URLs and then recursively follow links to other pages. The data structure supports+both breadth-first and depth-first crawling orders.++For more details and code examples, see the [`RequestQueue`](../api/request-queue) class.++**Parameters**:++- **`[queueIdOrName]`**: `string` - ID or name of the request queue to be opened. If `null` or `undefined`, the function returns the default request+ queue associated with the actor run.+- **`[options]`**: `object`+ - **`[forceCloud]`**: `boolean` <code> = false</code> - If set to `true` then the function uses cloud storage usage even if the+ `APIFY_LOCAL_STORAGE_DIR` environment variable is set. This way it is possible to combine local and cloud storage.++**Returns**:++[`Promise<RequestQueue>`](../api/request-queue)++---++<a name="opensessionpool"></a>++## `Apify.openSessionPool(sessionPoolOptions)`++Opens a SessionPool and returns a promise resolving to an instance of the [`SessionPool`](../api/session-pool) class that is already initialized.++For more details and code examples, see the [`SessionPool`](../api/session-pool) class.++**Parameters**:++- **`sessionPoolOptions`**: [`SessionPoolOptions`](../typedefs/session-pool-options)++**Returns**:++[`Promise<SessionPool>`](../api/session-pool)++---++<a name="pushdata"></a>++## `Apify.pushData(item)`++Stores an object or an array of objects to the default [`Dataset`](../api/dataset) of the current actor run.++This is just a convenient shortcut for [`Dataset.pushData()`](../api/dataset#pushdata). For example, calling the following code:++```javascript+await Apify.pushData({ myValue: 123 });+```++is equivalent to:++```javascript+const dataset = await Apify.openDataset();+await dataset.pushData({ myValue: 123 });+```++For more information, see [`Apify.openDataset()`](../api/apify#opendataset) and [`Dataset.pushData()`](../api/dataset#pushdata)++**IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`, otherwise the actor process might finish before the data are stored!++**Parameters**:++- **`item`**: `object` - Object or array of objects containing data to be stored in the default dataset. The objects must be serializable to JSON+ and the JSON representation of each object must be smaller than 9MB.++**Returns**:++`Promise<void>`++---++<a name="setvalue"></a>++## `Apify.setValue(key, value, [options])`++Stores or deletes a value in the default [`KeyValueStore`](../api/key-value-store) associated with the current actor run.++This is just a convenient shortcut for [`KeyValueStore.setValue()`](../api/key-value-store#setvalue). For example, calling the following code:++```javascript+await Apify.setValue('OUTPUT', { foo: 'bar' });+```++is equivalent to:++```javascript+const store = await Apify.openKeyValueStore();+await store.setValue('OUTPUT', { foo: 'bar' });+```++To get a value from the default key-value store, you can use the [`Apify.getValue()`](../api/apify#getvalue) function.++For more information, see [`Apify.openKeyValueStore()`](../api/apify#openkeyvaluestore) and+[`KeyValueStore.getValue()`](../api/key-value-store#getvalue).++**Parameters**:++- **`key`**: `string` - Unique record key.+- **`value`**: `*` - Record data, which can be one of the following values:+ - If `null`, the record in the key-value store is deleted.+ - If no `options.contentType` is specified, `value` can be any JavaScript object and it will be stringified to JSON.+ - If `options.contentType` is set, `value` is taken as is and it must be a `String` or [`Buffer`](https://nodejs.org/api/buffer.html). For any+ other value an error will be thrown.+- **`[options]`**: `object`+ - **`[contentType]`**: `string` - Specifies a custom MIME content type of the record.++**Returns**:++`Promise<void>`++---diff --git a/website/versioned_docs/version-2.3.0/api/BasicCrawler.md b/website/versioned_docs/version-2.3.0/api/BasicCrawler.mdnew file mode 100644index 000000000..8d68ee2b1--- /dev/null+++ b/website/versioned_docs/version-2.3.0/api/BasicCrawler.md@@ -0,0 +1,148 @@+---+id: version-2.3.0-basic-crawler+title: BasicCrawler+original_id: basic-crawler+---++<a name="basiccrawler"></a>++Provides a simple framework for parallel crawling of web pages. The URLs to crawl are fed either from a static list of URLs or from a dynamic queue of+URLs enabling recursive crawling of websites.++`BasicCrawler` is a low-level tool that requires the user to implement the page download and data extraction functionality themselves. If you want a+crawler that already facilitates this functionality, please consider using [`CheerioCrawler`](../api/cheerio-crawler),+[`PuppeteerCrawler`](../api/puppeteer-crawler) or [`PlaywrightCrawler`](../api/playwright-crawler).++`BasicCrawler` invokes the user-provided [`BasicCrawlerOptions.handleRequestFunction`](../typedefs/basic-crawler-options#handlerequestfunction) for+each [`Request`](../api/request) object, which represents a single URL to crawl. The [`Request`](../api/request) objects are fed from the+[`RequestList`](../api/request-list) or the [`RequestQueue`](../api/request-queue) instances provided by the+[`BasicCrawlerOptions.requestList`](../typedefs/basic-crawler-options#requestlist) or+[`BasicCrawlerOptions.requestQueue`](../typedefs/basic-crawler-options#requestqueue) constructor options, respectively.++If both [`BasicCrawlerOptions.requestList`](../typedefs/basic-crawler-options#requestlist) and+[`BasicCrawlerOptions.requestQueue`](../typedefs/basic-crawler-options#requestqueue) options are used, the instance first processes URLs from the+[`RequestList`](../api/request-list) and automatically enqueues all of them to [`RequestQueue`](../api/request-queue) before it starts their+processing. This ensures that a single URL is not crawled multiple times.++The crawler finishes if there are no more [`Request`](../api/request) objects to crawl.++New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the+[`AutoscaledPool`](../api/autoscaled-pool) class. All [`AutoscaledPool`](../api/autoscaled-pool) configuration options can be passed to the+`autoscaledPoolOptions` parameter of the `BasicCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`+[`AutoscaledPool`](../api/autoscaled-pool) options are available directly in the `BasicCrawler` constructor.++**Example usage:**++```javascript+// Prepare a list of URLs to crawl+const requestList = new Apify.RequestList({+ sources: [{ url: 'http://www.example.com/page-1' }, { url: 'http://www.example.com/page-2' }],+});+await requestList.initialize();++// Crawl the URLs+const crawler = new Apify.BasicCrawler({+ requestList,+ handleRequestFunction: async ({ request }) => {+ // 'request' contains an instance of the Request class+ // Here we simply fetch the HTML of the page and store it to a dataset+ const { body } = await Apify.utils.requestAsBrowser(request);+ await Apify.pushData({+ url: request.url,+ html: body,+ });+ },+});++await crawler.run();+```++## Properties++### `stats`++**Type**: [`Statistics`](../api/statistics)++Contains statistics about the current run.++---++### `requestList`++**Type**: [`RequestList`](../api/request-list)++A reference to the underlying [`RequestList`](../api/request-list) class that manages the crawler's [`Request`](../api/request)s. Only available if+used by the crawler.++---++### `requestQueue`++**Type**: [`RequestQueue`](../api/request-queue)++A reference to the underlying [`RequestQueue`](../api/request-queue) class that manages the crawler's [`Request`](../api/request)s. Only available if+used by the crawler.++---++### `sessionPool`++**Type**: [`SessionPool`](../api/session-pool)++A reference to the underlying [`SessionPool`](../api/session-pool) class that manages the crawler's [`Session`](../api/session)s. Only available if+used by the crawler.++---++### `autoscaledPool`++**Type**: [`AutoscaledPool`](../api/autoscaled-pool)++A reference to the underlying [`AutoscaledPool`](../api/autoscaled-pool) class that manages the concurrency of the crawler. Note that this property is+only initialized after calling the [`BasicCrawler.run()`](../api/basic-crawler#run) function. You can use it to change the concurrency settings on the+fly, to pause the crawler by calling [`AutoscaledPool.pause()`](../api/autoscaled-pool#pause) or to abort it by calling+[`AutoscaledPool.abort()`](../api/autoscaled-pool#abort).++---++<a name="exports.basiccrawler"></a>++## `new BasicCrawler(options)`++**Parameters**:++- **`options`**: [`BasicCrawlerOptions`](../typedefs/basic-crawler-options) - All `BasicCrawler` parameters are passed via an options object.++---++<a name="optionsshape"></a>++## `basicCrawler.optionsShape`++**Internal**:++---++<a name="log"></a>++## `basicCrawler.log`++---++<a name="sessionpooloptions"></a>++## `basicCrawler.sessionPoolOptions`++---++<a name="run"></a>++## `basicCrawler.run()`++Runs the crawler. Returns a promise that gets resolved once all the requests are processed.++**Returns**:++`Promise<void>`++---diff --git a/website/versioned_docs/version-2.3.0/api/CheerioCrawler.md b/website/versioned_docs/version-2.3.0/api/CheerioCrawler.mdnew file mode 100644index 000000000..1fc033080--- /dev/null+++ b/website/versioned_docs/version-2.3.0/api/CheerioCrawler.md@@ -0,0 +1,181 @@+---+id: version-2.3.0-cheerio-crawler+title: CheerioCrawler+original_id: cheerio-crawler+---++<a name="cheeriocrawler"></a>++Provides a framework for the parallel crawling of web pages using plain HTTP requests and [cheerio](https://www.npmjs.com/package/cheerio) HTML+parser. The URLs to crawl are fed either from a static list of URLs or from a dynamic queue of URLs enabling recursive crawling of websites.++Since `CheerioCrawler` uses raw HTTP requests to download web pages, it is very fast and efficient on data bandwidth. However, if the target website+requires JavaScript to display the content, you might need to use [`PuppeteerCrawler`](../api/puppeteer-crawler) or+[`PlaywrightCrawler`](../api/playwright-crawler) instead, because it loads the pages using full-featured headless Chrome browser.++`CheerioCrawler` downloads each URL using a plain HTTP request, parses the HTML content using [Cheerio](https://www.npmjs.com/package/cheerio) and+then invokes the user-provided [`CheerioCrawlerOptions.handlePageFunction`](../typedefs/cheerio-crawler-options#handlepagefunction) to extract page+data using a [jQuery](https://jquery.com/)-like interface to the parsed HTML DOM.++The source URLs are represented using [`Request`](../api/request) objects that are fed from [`RequestList`](../api/request-list) or+[`RequestQueue`](../api/request-queue) instances provided by the+[`CheerioCrawlerOptions.requestList`](../typedefs/cheerio-crawler-options#requestlist) or+[`CheerioCrawlerOptions.requestQueue`](../typedefs/cheerio-crawler-options#requestqueue) constructor options, respectively.++If both [`CheerioCrawlerOptions.requestList`](../typedefs/cheerio-crawler-options#requestlist) and+[`CheerioCrawlerOptions.requestQueue`](../typedefs/cheerio-crawler-options#requestqueue) are used, the instance first processes URLs from the+[`RequestList`](../api/request-list) and automatically enqueues all of them to [`RequestQueue`](../api/request-queue) before it starts their+processing. This ensures that a single URL is not crawled multiple times.++The crawler finishes when there are no more [`Request`](../api/request) objects to crawl.++`CheerioCrawler` downloads the web pages using the `[`utils.requestAsBrowser()`](../api/utils#requestasbrowser)` utility function. As opposed to the+browser based crawlers that are automatically encoding the URLs, the `[`utils.requestAsBrowser()`](../api/utils#requestasbrowser)` function will not+do so. We either need to manually encode the URLs via `encodeURI()` function, or set `forceUrlEncoding: true` in the `requestAsBrowserOptions`, which+will automatically encode all the URLs before accessing them.++> We can either use `forceUrlEncoding` or encode manually, but not both - it would result in double encoding and therefore lead to invalid URLs.++We can use the `preNavigationHooks` to adjust `requestAsBrowserOptions`:++```+preNavigationHooks: [+ (crawlingContext, requestAsBrowserOptions) => {+ requestAsBrowserOptions.forceUrlEncoding = true;+ },+]+```++By default, `CheerioCrawler` only processes web pages with the `text/html` and `application/xhtml+xml` MIME content types (as reported by the+`Content-Type` HTTP header), and skips pages with other content types. If you want the crawler to process other content types, use the+[`CheerioCrawlerOptions.additionalMimeTypes`](../typedefs/cheerio-crawler-options#additionalmimetypes) constructor option. Beware that the parsing+behavior differs for HTML, XML, JSON and other types of content. For details, see+[`CheerioCrawlerOptions.handlePageFunction`](../typedefs/cheerio-crawler-options#handlepagefunction).++New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the+[`AutoscaledPool`](../api/autoscaled-pool) class. All [`AutoscaledPool`](../api/autoscaled-pool) configuration options can be passed to the+`autoscaledPoolOptions` parameter of the `CheerioCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`+[`AutoscaledPool`](../api/autoscaled-pool) options are available directly in the `CheerioCrawler` constructor.++**Example usage:**++```javascript+// Prepare a list of URLs to crawl+const requestList = new Apify.RequestList({+ sources: [{ url: 'http://www.example.com/page-1' }, { url: 'http://www.example.com/page-2' }],+});+await requestList.initialize();++// Crawl the URLs+const crawler = new Apify.CheerioCrawler({+ requestList,+ handlePageFunction: async ({ request, response, body, contentType, $ }) => {+ const data = [];++ // Do some data extraction from the page with Cheerio.+ $('.some-collection').each((index, el) => {+ data.push({+ title: $(el)+ .find('.some-title')+ .text(),+ });+ });++ // Save the data to dataset.+ await Apify.pushData({+ url: request.url,+ html: body,+ data,+ });+ },+});++await crawler.run();+```++## Properties++### `stats`++**Type**: [`Statistics`](../api/statistics)++Contains statistics about the current run.++---++### `requestList`++**Type**: [`RequestList`](../api/request-list)++A reference to the underlying [`RequestList`](../api/request-list) class that manages the crawler's [`Request`](../api/request)s. Only available if+used by the crawler.++---++### `requestQueue`++**Type**: [`RequestQueue`](../api/request-queue)++A reference to the underlying [`RequestQueue`](../api/request-queue) class that manages the crawler's [`Request`](../api/request)s. Only available if+used by the crawler.++---++### `sessionPool`++**Type**: [`SessionPool`](../api/session-pool)++A reference to the underlying [`SessionPool`](../api/session-pool) class that manages the crawler's [`Session`](../api/session)s. Only available if+used by the crawler.++---++### `proxyConfiguration`++**Type**: [`ProxyConfiguration`](../api/proxy-configuration)++A reference to the underlying [`ProxyConfiguration`](../api/proxy-configuration) class that manages the crawler's proxies. Only available if used by+the crawler.++---++### `autoscaledPool`++**Type**: [`AutoscaledPool`](../api/autoscaled-pool)++A reference to the underlying [`AutoscaledPool`](../api/autoscaled-pool) class that manages the concurrency of the crawler. Note that this property is+only initialized after calling the [`CheerioCrawler.run()`](../api/cheerio-crawler#run) function. You can use it to change the concurrency settings on+the fly, to pause the crawler by calling [`AutoscaledPool.pause()`](../api/autoscaled-pool#pause) or to abort it by calling+[`AutoscaledPool.abort()`](../api/autoscaled-pool#abort).++---++<a name="cheeriocrawler"></a>++## `new CheerioCrawler(options)`++**Parameters**:++- **`options`**: [`CheerioCrawlerOptions`](../typedefs/cheerio-crawler-options) - All `CheerioCrawler` parameters are passed via an options object.++---++<a name="optionsshape"></a>++## `cheerioCrawler.optionsShape`++**Internal**:++---++<a name="use"></a>++## `cheerioCrawler.use(extension)`++**EXPERIMENTAL** Function for attaching CrawlerExtensions such as the Unblockers.++**Parameters**:++- **`extension`**: `CrawlerExtension` - Crawler extension that overrides the crawler configuration.++---diff --git a/website/versioned_docs/version-2.3.0/api/PlaywrightCrawler.md b/website/versioned_docs/version-2.3.0/api/PlaywrightCrawler.mdnew file mode 100644index 000000000..6933961b1--- /dev/null+++ b/website/versioned_docs/version-2.3.0/api/PlaywrightCrawler.md@@ -0,0 +1,152 @@+---+id: version-2.3.0-playwright-crawler+title: PlaywrightCrawler+original_id: playwright-crawler+---++<a name="playwrightcrawler"></a>++Provides a simple framework for parallel crawling of web pages using headless Chromium, Firefox and Webkit browsers with+[Playwright](https://github.com/microsoft/playwright). The URLs to crawl are fed either from a static list of URLs or from a dynamic queue of URLs+enabling recursive crawling of websites.++Since `Playwright` uses headless browser to download web pages and extract data, it is useful for crawling of websites that require to execute+JavaScript. If the target website doesn't need JavaScript, consider using [`CheerioCrawler`](../api/cheerio-crawler), which downloads the pages using+raw HTTP requests and is about 10x faster.++The source URLs are represented using [`Request`](../api/request) objects that are fed from [`RequestList`](../api/request-list) or+[`RequestQueue`](../api/request-queue) instances provided by the+[`PlaywrightCrawlerOptions.requestList`](../typedefs/playwright-crawler-options#requestlist) or+[`PlaywrightCrawlerOptions.requestQueue`](../typedefs/playwright-crawler-options#requestqueue) constructor options, respectively.++If both [`PlaywrightCrawlerOptions.requestList`](../typedefs/playwright-crawler-options#requestlist) and+[`PlaywrightCrawlerOptions.requestQueue`](../typedefs/playwright-crawler-options#requestqueue) are used, the instance first processes URLs from the+[`RequestList`](../api/request-list) and automatically enqueues all of them to [`RequestQueue`](../api/request-queue) before it starts their+processing. This ensures that a single URL is not crawled multiple times.++The crawler finishes when there are no more [`Request`](../api/request) objects to crawl.++`PlaywrightCrawler` opens a new Chrome page (i.e. tab) for each [`Request`](../api/request) object to crawl and then calls the function provided by+user as the [`PlaywrightCrawlerOptions.handlePageFunction`](../typedefs/playwright-crawler-options#handlepagefunction) option.++New pages are only opened when there is enough free CPU and memory available, using the functionality provided by the+[`AutoscaledPool`](../api/autoscaled-pool) class. All [`AutoscaledPool`](../api/autoscaled-pool) configuration options can be passed to the+[`PlaywrightCrawlerOptions.autoscaledPoolOptions`](../typedefs/playwright-crawler-options#autoscaledpooloptions) parameter of the `PlaywrightCrawler`+constructor. For user convenience, the `minConcurrency` and `maxConcurrency` [`AutoscaledPoolOptions`](../typedefs/autoscaled-pool-options) are+available directly in the `PlaywrightCrawler` constructor.++Note that the pool of Playwright instances is internally managed by the `BrowserPool` class. Many constructor options such as+[`PlaywrightCrawlerOptions.maxOpenPagesPerInstance`](../typedefs/playwright-crawler-options#maxopenpagesperinstance) or++**Example usage:**++```javascript+const crawler = new Apify.PlaywrightCrawler({+ requestList,+ handlePageFunction: async ({ page, request }) => {+ // This function is called to extract data from a single web page+ // 'page' is an instance of Playwright.Page with page.goto(request.url) already called+ // 'request' is an instance of Request class with information about the page to load+ await Apify.pushData({+ title: await page.title(),+ url: request.url,+ succeeded: true,+ });+ },+ handleFailedRequestFunction: async ({ request }) => {+ // This function is called when the crawling of a request failed too many times+ await Apify.pushData({+ url: request.url,+ succeeded: false,+ errors: request.errorMessages,+ });+ },+});++await crawler.run();+```++## Properties++### `stats`++**Type**: [`Statistics`](../api/statistics)++Contains statistics about the current run.++---++### `requestList`++**Type**: [`RequestList`](../api/request-list)++A reference to the underlying [`RequestList`](../api/request-list) class that manages the crawler's [`Request`](../api/request)s. Only available if+used by the crawler.++---++### `requestQueue`++**Type**: [`RequestQueue`](../api/request-queue)++A reference to the underlying [`RequestQueue`](../api/request-queue) class that manages the crawler's [`Request`](../api/request)s. Only available if+used by the crawler.++---++### `sessionPool`++**Type**: [`SessionPool`](../api/session-pool)++A reference to the underlying [`SessionPool`](../api/session-pool) class that manages the crawler's [`Session`](../api/session)s. Only available if+used by the crawler.++---++### `proxyConfiguration`++**Type**: [`ProxyConfiguration`](../api/proxy-configuration)++A reference to the underlying [`ProxyConfiguration`](../api/proxy-configuration) class that manages the crawler's proxies. Only available if used by+the crawler.++---++### `browserPool`++**Type**: `BrowserPool`++A reference to the underlying `BrowserPool` class that manages the crawler's browsers. For more information about it, see the+[`browser-pool` module](https://github.com/apify/browser-pool).++---++### `autoscaledPool`++**Type**: [`AutoscaledPool`](../api/autoscaled-pool)++A reference to the underlying [`AutoscaledPool`](../api/autoscaled-pool) class that manages the concurrency of the crawler. Note that this property is+only initialized after calling the [`CheerioCrawler.run()`](../api/cheerio-crawler#run) function. You can use it to change the concurrency settings on+the fly, to pause the crawler by calling [`AutoscaledPool.pause()`](../api/autoscaled-pool#pause) or to abort it by calling+[`AutoscaledPool.abort()`](../api/autoscaled-pool#abort).++---++<a name="playwrightcrawler"></a>++## `new PlaywrightCrawler(options)`++**Parameters**:++- **`options`**: [`PlaywrightCrawlerOptions`](../typedefs/playwright-crawler-options) - All `PlaywrightCrawler` parameters are passed via an options+ object.++---++<a name="optionsshape"></a>++## `playwrightCrawler.optionsShape`++**Internal**:++---diff --git a/website/versioned_docs/version-2.3.0/api/PuppeteerCrawler.md b/website/versioned_docs/version-2.3.0/api/PuppeteerCrawler.mdnew file mode 100644index 000000000..19b6cd7f7--- /dev/null+++ b/website/versioned_docs/version-2.3.0/api/PuppeteerCrawler.md@@ -0,0 +1,150 @@+---+id: version-2.3.0-puppeteer-crawler+title: PuppeteerCrawler+original_id: puppeteer-crawler+---++<a name="puppeteercrawler"></a>++Provides a simple framework for parallel crawling of web pages using headless Chrome with [Puppeteer](https://github.com/puppeteer/puppeteer). The+URLs to crawl are fed either from a static list of URLs or from a dynamic queue of URLs enabling recursive crawling of websites.++Since `PuppeteerCrawler` uses headless Chrome to download web pages and extract data, it is useful for crawling of websites that require to execute+JavaScript. If the target website doesn't need JavaScript, consider using [`CheerioCrawler`](../api/cheerio-crawler), which downloads the pages using+raw HTTP requests and is about 10x faster.++The source URLs are represented using [`Request`](../api/request) objects that are fed from [`RequestList`](../api/request-list) or+[`RequestQueue`](../api/request-queue) instances provided by the+[`PuppeteerCrawlerOptions.requestList`](../typedefs/puppeteer-crawler-options#requestlist) or+[`PuppeteerCrawlerOptions.requestQueue`](../typedefs/puppeteer-crawler-options#requestqueue) constructor options, respectively.++If both [`PuppeteerCrawlerOptions.requestList`](../typedefs/puppeteer-crawler-options#requestlist) and+[`PuppeteerCrawlerOptions.requestQueue`](../typedefs/puppeteer-crawler-options#requestqueue) are used, the instance first processes URLs from the+[`RequestList`](../api/request-list) and automatically enqueues all of them to [`RequestQueue`](../api/request-queue) before it starts their+processing. This ensures that a single URL is not crawled multiple times.++The crawler finishes when there are no more [`Request`](../api/request) objects to crawl.++`PuppeteerCrawler` opens a new Chrome page (i.e. tab) for each [`Request`](../api/request) object to crawl and then calls the function provided by+user as the [`PuppeteerCrawlerOptions.handlePageFunction`](../typedefs/puppeteer-crawler-options#handlepagefunction) option.++New pages are only opened when there is enough free CPU and memory available, using the functionality provided by the+[`AutoscaledPool`](../api/autoscaled-pool) class. All [`AutoscaledPool`](../api/autoscaled-pool) configuration options can be passed to the+[`PuppeteerCrawlerOptions.autoscaledPoolOptions`](../typedefs/puppeteer-crawler-options#autoscaledpooloptions) parameter of the `PuppeteerCrawler`+constructor. For user convenience, the `minConcurrency` and `maxConcurrency` [`AutoscaledPoolOptions`](../typedefs/autoscaled-pool-options) are+available directly in the `PuppeteerCrawler` constructor.++Note that the pool of Puppeteer instances is internally managed by the `BrowserPool` class.++**Example usage:**++```javascript+const crawler = new Apify.PuppeteerCrawler({+ requestList,+ handlePageFunction: async ({ page, request }) => {+ // This function is called to extract data from a single web page+ // 'page' is an instance of Puppeteer.Page with page.goto(request.url) already called+ // 'request' is an instance of Request class with information about the page to load+ await Apify.pushData({+ title: await page.title(),+ url: request.url,+ succeeded: true,+ });+ },+ handleFailedRequestFunction: async ({ request }) => {+ // This function is called when the crawling of a request failed too many times+ await Apify.pushData({+ url: request.url,+ succeeded: false,+ errors: request.errorMessages,+ });+ },+});++await crawler.run();+```++## Properties++### `stats`++**Type**: [`Statistics`](../api/statistics)++Contains statistics about the current run.++---++### `requestList`++**Type**: [`RequestList`](../api/request-list)++A reference to the underlying [`RequestList`](../api/request-list) class that manages the crawler's [`Request`](../api/request)s. Only available if+used by the crawler.++---++### `requestQueue`++**Type**: [`RequestQueue`](../api/request-queue)++A reference to the underlying [`RequestQueue`](../api/request-queue) class that manages the crawler's [`Request`](../api/request)s. Only available if+used by the crawler.++---++### `sessionPool`++**Type**: [`SessionPool`](../api/session-pool)++A reference to the underlying [`SessionPool`](../api/session-pool) class that manages the crawler's [`Session`](../api/session)s. Only available if+used by the crawler.++---++### `proxyConfiguration`++**Type**: [`ProxyConfiguration`](../api/proxy-configuration)++A reference to the underlying [`ProxyConfiguration`](../api/proxy-configuration) class that manages the crawler's proxies. Only available if used by+the crawler.++---++### `browserPool`++**Type**: `BrowserPool`++A reference to the underlying `BrowserPool` class that manages the crawler's browsers. For more information about it, see the+[`browser-pool` module](https://github.com/apify/browser-pool).++---++### `autoscaledPool`++**Type**: [`AutoscaledPool`](../api/autoscaled-pool)++A reference to the underlying [`AutoscaledPool`](../api/autoscaled-pool) class that manages the concurrency of the crawler. Note that this property is+only initialized after calling the [`CheerioCrawler.run()`](../api/cheerio-crawler#run) function. You can use it to change the concurrency settings on+the fly, to pause the crawler by calling [`AutoscaledPool.pause()`](../api/autoscaled-pool#pause) or to abort it by calling+[`AutoscaledPool.abort()`](../api/autoscaled-pool#abort).++---++<a name="puppeteercrawler"></a>++## `new PuppeteerCrawler(options)`++**Parameters**:++- **`options`**: [`PuppeteerCrawlerOptions`](../typedefs/puppeteer-crawler-options) - All `PuppeteerCrawler` parameters are passed via an options+ object.++---++<a name="optionsshape"></a>++## `puppeteerCrawler.optionsShape`++**Internal**:++---diff --git a/website/versioned_docs/version-2.3.0/api/Session.md b/website/versioned_docs/version-2.3.0/api/Session.mdnew file mode 100644index 000000000..2059e47dd--- /dev/null+++ b/website/versioned_docs/version-2.3.0/api/Session.md@@ -0,0 +1,207 @@+---+id: version-2.3.0-session+title: Session+original_id: session+---++<a name="session"></a>++Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions. You can imagine each session as+a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint. Session internal state can be enriched with custom+user data for example some authorization tokens and specific headers in general.++---++<a name="exports.session"></a>++## `new Session(options)`++Session configuration.++**Parameters**:++- **`options`**: [`SessionOptions`](../typedefs/session-options)++---++<a name="userdata"></a>++## `session.userData`++---++<a name="isblocked"></a>++## `session.isBlocked()`++indicates whether the session is blocked. Session is blocked once it reaches the `maxErrorScore`.++**Returns**:++`boolean`++---++<a name="isexpired"></a>++## `session.isExpired()`++Indicates whether the session is expired. Session expiration is determined by the `maxAgeSecs`. Once the session is older than+`createdAt + maxAgeSecs` the session is considered expired.++**Returns**:++`boolean`++---++<a name="ismaxusagecountreached"></a>++## `session.isMaxUsageCountReached()`++Indicates whether the session is used maximum number of times. Session maximum usage count can be changed by `maxUsageCount` parameter.++**Returns**:++`boolean`++---++<a name="isusable"></a>++## `session.isUsable()`++Indicates whether the session can be used for next requests. Session is usable when it is not expired, not blocked and the maximum usage count has not+be reached.++**Returns**:++`boolean`++---++<a name="markgood"></a>++## `session.markGood()`++This method should be called after a successful session usage. It increases `usageCount` and potentially lowers the `errorScore` by the+`errorScoreDecrement`.++---++<a name="getstate"></a>++## `session.getState()`++Gets session state for persistence in KeyValueStore.++**Returns**:++[`SessionState`](../typedefs/session-state) - represents session internal state.++---++<a name="retire"></a>++## `session.retire()`++Marks session as blocked and emits event on the `SessionPool` This method should be used if the session usage was unsuccessful and you are sure that+it is because of the session configuration and not any external matters. For example when server returns 403 status code. If the session does not work+due to some external factors as server error such as 5XX you probably want to use `markBad` method.++---++<a name="markbad"></a>++## `session.markBad()`++Increases usage and error count. Should be used when the session has been used unsuccessfully. For example because of timeouts.++---++<a name="retireonblockedstatuscodes"></a>++## `session.retireOnBlockedStatusCodes(statusCode, [blockedStatusCodes])`++With certain status codes: `401`, `403` or `429` we can be certain that the target website is blocking us. This function helps to do this conveniently+by retiring the session when such code is received. Optionally the default status codes can be extended in the second parameter.++**Parameters**:++- **`statusCode`**: `number` - HTTP status code+- **`[blockedStatusCodes]`**: `Array<number>` - Custom HTTP status codes that means blocking on particular website.++**Returns**:++`boolean` - whether the session was retired.++---++<a name="setcookiesfromresponse"></a>++## `session.setCookiesFromResponse(response)`++Saves cookies from an HTTP response to be used with the session. It expects an object with a `headers` property that's either an `Object` (typical+Node.js responses) or a `Function` (Puppeteer Response).++It then parses and saves the cookies from the `set-cookie` header, if available.++**Parameters**:++- **`response`**: `PuppeteerResponse` | `IncomingMessage`++---++<a name="setpuppeteercookies"></a>++## `session.setPuppeteerCookies(cookies, url)`++Saves an array with cookie objects to be used with the session. The objects should be in the format that+[Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls), but you can also use this function to set cookies+manually:++```+[+ { name: 'cookie1', value: 'my-cookie' },+ { name: 'cookie2', value: 'your-cookie' }+]+```++**Parameters**:++- **`cookies`**: [`Array<PuppeteerCookie>`](../typedefs/puppeteer-cookie)+- **`url`**: `string`++---++<a name="getpuppeteercookies"></a>++## `session.getPuppeteerCookies(url)`++Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.++**Parameters**:++- **`url`**: `string` - website url. Only cookies stored for this url will be returned++**Returns**:++[`Array<PuppeteerCookie>`](../typedefs/puppeteer-cookie)++---++<a name="getcookiestring"></a>++## `session.getCookieString(url)`++Returns cookies saved with the session in the typical key1=value1; key2=value2 format, ready to be used in a cookie header or elsewhere.++**Parameters**:++- **`url`**: `string`++**Returns**:++`string` - represents `Cookie` header.++---diff --git a/website/versioned_docs/version-2.3.0/api/SessionPool.md b/website/versioned_docs/version-2.3.0/api/SessionPool.mdnew file mode 100644index 000000000..20f9adede--- /dev/null+++ b/website/versioned_docs/version-2.3.0/api/SessionPool.md@@ -0,0 +1,170 @@+---+id: version-2.3.0-session-pool+title: SessionPool+original_id: session-pool+---++<a name="sessionpool"></a>++Handles the rotation, creation and persistence of user-like sessions. Creates a pool of [`Session`](../api/session) instances, that are randomly+rotated. When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session). Learn more+in the [`Session management guide`](../guides/session-management).++You can create one by calling the [`Apify.openSessionPool`](../api/apify#opensessionpool) function.++Session pool is already integrated into crawlers, and it can significantly improve your scraper performance with just 2 lines of code.++**Example usage:**++```javascript+const crawler = new Apify.CheerioCrawler({+ useSessionPool: true,+ persistCookiesPerSession: true,+ // ...+});+```++You can configure the pool with many options. See the [`SessionPoolOptions`](../typedefs/session-pool-options). Session pool is by default persisted+in default [`KeyValueStore`](../api/key-value-store). If you want to have one pool for all runs you have to specify+[`SessionPoolOptions.persistStateKeyValueStoreId`](../typedefs/session-pool-options#persiststatekeyvaluestoreid).++**Advanced usage:**++```javascript+const sessionPool = await Apify.openSessionPool({+ maxPoolSize: 25,+ sessionOptions: {+ maxAgeSecs: 10,+ maxUsageCount: 150, // for example when you know that the site blocks after 150 requests.+ },+ persistStateKeyValueStoreId: 'my-key-value-store-for-sessions',+ persistStateKey: 'my-session-pool',+});++// Get random session from the pool+const session1 = await sessionPool.getSession();+const session2 = await sessionPool.getSession();+const session3 = await sessionPool.getSession();++// Now you can mark the session either failed or successful++// Marks session as bad after unsuccessful usage -> it increases error count (soft retire)+session1.markBad();++// Marks as successful.+session2.markGood();++// Retires session -> session is removed from the pool+session3.retire();+```++---++<a name="sessions"></a>++## `sessionPool.sessions`++---++<a name="forcecloud"></a>++## `sessionPool.forceCloud`++---++<a name="usablesessionscount"></a>++## `sessionPool.usableSessionsCount`++Gets count of usable sessions in the pool.++**Returns**:++`number`++---++<a name="retiredsessionscount"></a>++## `sessionPool.retiredSessionsCount`++Gets count of retired sessions in the pool.++**Returns**:++`number`++---++<a name="initialize"></a>++## `sessionPool.initialize()`++Starts periodic state persistence and potentially loads SessionPool state from [`KeyValueStore`](../api/key-value-store). It is called automatically+by the [`Apify.openSessionPool`](../api/apify#opensessionpool) function.++**Returns**:++`Promise<void>`++---++<a name="addsession"></a>++## `sessionPool.addSession([options])`++Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool, but this allows you to add more+sessions once the max pool size is reached. This also allows you to add session with overridden session options (e.g. with specific session id).++**Parameters**:++- **`[options]`**: [`Session`](../api/session) | [`SessionOptions`](../typedefs/session-options) - The configuration options for the session being+ added to the session pool.++---++<a name="getsession"></a>++## `sessionPool.getSession([sessionId])`++Gets session. If there is space for new session, it creates and returns new session. If the session pool is full, it picks a session from the pool, If+the picked session is usable it is returned, otherwise it creates and returns a new one.++**Parameters**:++- **`[sessionId]`**: `String` - If provided, it returns the usable session with this id, `undefined` otherwise.++**Returns**:++[`Promise<Session>`](../api/session)++---++<a name="getstate"></a>++## `sessionPool.getState()`++Returns an object representing the internal state of the `SessionPool` instance. Note that the object's fields can change in future releases.++---++<a name="persiststate"></a>++## `sessionPool.persistState()`++Persists the current state of the `SessionPool` into the default [`KeyValueStore`](../api/key-value-store). The state is persisted automatically in+regular intervals.++**Returns**:++`Promise<void>`++---++<a name="teardown"></a>++## `sessionPool.teardown()`++Removes listener from `persistState` event. This function should be called after you are done with using the `SessionPool` instance.++---diff --git a/website/versioned_docs/version-2.3.0/api/puppeteer.md b/website/versioned_docs/version-2.3.0/api/puppeteer.mdnew file mode 100644index 000000000..9f5ab44aa--- /dev/null+++ b/website/versioned_docs/version-2.3.0/api/puppeteer.md@@ -0,0 +1,473 @@+---+id: version-2.3.0-puppeteer+title: utils.puppeteer+original_id: puppeteer+---++<a name="puppeteer"></a>++A namespace that contains various utilities for [Puppeteer](https://github.com/puppeteer/puppeteer) - the headless Chrome Node API.++**Example usage:**++```javascript+const Apify = require('apify');+const { puppeteer } = Apify.utils;++// Open https://www.example.com in Puppeteer+const browser = await Apify.launchPuppeteer();+const page = await browser.newPage();+await page.goto('https://www.example.com');++// Inject jQuery into a page+await puppeteer.injectJQuery(page);+```++---++<a name="addinterceptrequesthandler"></a>++## `puppeteer.addInterceptRequestHandler`++Adds request interception handler in similar to `page.on('request', handler);` but in addition to that supports multiple parallel handlers.++All the handlers are executed sequentially in the order as they were added. Each of the handlers must call one of `request.continue()`,+`request.abort()` and `request.respond()`. In addition to that any of the handlers may modify the request object (method, postData, headers) by+passing its overrides to `request.continue()`. If multiple handlers modify same property then the last one wins. Headers are merged separately so you+can override only a value of specific header.++If one the handlers calls `request.abort()` or `request.respond()` then request is not propagated further to any of the remaining handlers.++**Example usage:**++```javascript+// Replace images with placeholder.+await addInterceptRequestHandler(page, request => {+ if (request.resourceType() === 'image') {+ return request.respond({+ statusCode: 200,+ contentType: 'image/jpeg',+ body: placeholderImageBuffer,+ });+ }+ return request.continue();+});++// Abort all the scripts.+await addInterceptRequestHandler(page, request => {+ if (request.resourceType() === 'script') return request.abort();+ return request.continue();+});++// Change requests to post.+await addInterceptRequestHandler(page, request => {+ return request.continue({+ method: 'POST',+ });+});++await page.goto('http://example.com');+```++**Parameters**:++- **`page`**: `Page` - Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.+- **`handler`**: [`InterceptHandler`](../typedefs/intercept-handler) - Request interception handler.++**Returns**:++`Promise<void>`++---++<a name="removeinterceptrequesthandler"></a>++## `puppeteer.removeInterceptRequestHandler`++Removes request interception handler for given page.++**Parameters**:++- **`page`**: `Page` - Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.+- **`handler`**: [`InterceptHandler`](../typedefs/intercept-handler) - Request interception handler.++**Returns**:++`Promise<void>`++---++<a name="gotoextended"></a>++## `puppeteer.gotoExtended`++Extended version of Puppeteer's `page.goto()` allowing to perform requests with HTTP method other than GET, with custom headers and POST payload. URL,+method, headers and payload are taken from request parameter that must be an instance of Apify.Request class.++_NOTE:_ In recent versions of Puppeteer using requests other than GET, overriding headers and adding payloads disables browser cache which degrades+performance.++**Parameters**:++- **`page`**: `Page` - Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.+- **`request`**: [`Request`](../api/request)+- **`[gotoOptions]`**: [`DirectNavigationOptions`](../typedefs/direct-navigation-options) - Custom options for `page.goto()`.++**Returns**:++`Promise<(Response|null)>`++---++<a name="infinitescroll"></a>++## `puppeteer.infiniteScroll`++Scrolls to the bottom of a page, or until it times out. Loads dynamic content when it hits the bottom of a page, and then continues scrolling.++**Parameters**:++- **`page`**: `Page` - Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.+- **`[options]`**: `object`+ - **`[timeoutSecs]`**: `number` <code> = 0</code> - How many seconds to scroll for. If 0, will scroll until bottom of page.+ - **`[waitForSecs]`**: `number` <code> = 4</code> - How many seconds to wait for no new content to load before exit.+ - **`[scrollDownAndUp]`**: `boolean` <code> = false</code> - If true, it will scroll up a bit after each scroll down. This is required on some+ websites for the scroll to work.+ - **`[buttonSelector]`**: `string` - Optionally checks and clicks a button if it appears while scrolling. This is required on some websites for+ the scroll to work.+ - **`[stopScrollCallback]`**: `function` - This function is called after every scroll and stops the scrolling process if it returns `true`. The+ function can be `async`.++**Returns**:++`Promise<void>`++---++<a name="savesnapshot"></a>++## `puppeteer.saveSnapshot`++Saves a full screenshot and HTML of the current page into a Key-Value store.++**Parameters**:++- **`page`**: `Page` - Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.+- **`[options]`**: `object`+ - **`[key]`**: `string` <code> = "SNAPSHOT"</code> - Key under which the screenshot and HTML will be saved. `.jpg` will be appended+ for screenshot and `.html` for HTML.+ - **`[screenshotQuality]`**: `number` <code> = 50</code> - The quality of the image, between 0-100. Higher quality images have bigger size and+ require more storage.+ - **`[saveScreenshot]`**: `boolean` <code> = true</code> - If true, it will save a full screenshot of the current page as a record with `key`+ appended by `.jpg`.+ - **`[saveHtml]`**: `boolean` <code> = true</code> - If true, it will save a full HTML of the current page as a record with `key` appended by+ `.html`.+ - **`[keyValueStoreName]`**: `string` | `null` <code> = null</code> - Name or id of the Key-Value store where snapshot is saved. By default it+ is saved to default Key-Value store.++**Returns**:++`Promise<void>`++---++<a name="enqueuelinksbyclickingelements"></a>++## `puppeteer.enqueueLinksByClickingElements(options)`++The function finds elements matching a specific CSS selector in a Puppeteer page, clicks all those elements using a mouse move and a left mouse button+click and intercepts all the navigation requests that are subsequently produced by the page. The intercepted requests, including their methods,+headers and payloads are then enqueued to a provided [`RequestQueue`](../api/request-queue). This is useful to crawl JavaScript heavy pages where+links are not available in `href` elements, but rather navigations are triggered in click handlers. If you're looking to find URLs in `href`+attributes of the page, see [`utils.enqueueLinks()`](../api/utils#enqueuelinks).++Optionally, the function allows you to filter the target links' URLs using an array of [`PseudoUrl`](../api/pseudo-url) objects and override settings+of the enqueued [`Request`](../api/request) objects.++**IMPORTANT**: To be able to do this, this function uses various mutations on the page, such as changing the Z-index of elements being clicked and+their visibility. Therefore, it is recommended to only use this function as the last operation in the page.++**USING HEADFUL BROWSER**: When using a headful browser, this function will only be able to click elements in the focused tab, effectively limiting+concurrency to 1. In headless mode, full concurrency can be achieved.++**PERFORMANCE**: Clicking elements with a mouse and intercepting requests is not a low level operation that takes nanoseconds. It's not very CPU+intensive, but it takes time. We strongly recommend limiting the scope of the clicking as much as possible by using a specific selector that targets+only the elements that you assume or know will produce a navigation. You can certainly click everything by using the `*` selector, but be prepared to+wait minutes to get results on a large and complex page.++**Example usage**++```javascript+await Apify.utils.puppeteer.enqueueLinksByClickingElements({+ page,+ requestQueue,+ selector: 'a.product-detail',+ pseudoUrls: [+ 'https://www.example.com/handbags/[.*]'+ 'https://www.example.com/purses/[.*]'+ ],+});+```++**Parameters**:++- **`options`**: `object` - All `enqueueLinksByClickingElements()` parameters are passed via an options object with the following keys:++ - **`page`**: `Page` - Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.+ - **`requestQueue`**: [`RequestQueue`](../api/request-queue) - A request queue to which the URLs will be enqueued.+ - **`selector`**: `string` - A CSS selector matching elements to be clicked on. Unlike in [`utils.enqueueLinks()`](../api/utils#enqueuelinks),+ there is no default value. This is to prevent suboptimal use of this function by using it too broadly.+ - **`[pseudoUrls]`**: `Array<(string|RegExp|Object<string, *>)>` - An array of [`PseudoUrl`](../api/pseudo-url)s matching the URLs to be+ enqueued, or an array of strings or RegExps or plain Objects from which the [`PseudoUrl`](../api/pseudo-url)s can be constructed.++ The plain objects must include at least the `purl` property, which holds the pseudo-URL string or RegExp. All remaining keys will be used as the+ `requestTemplate` argument of the [`PseudoUrl`](../api/pseudo-url) constructor, which lets you specify special properties for the enqueued+ [`Request`](../api/request) objects.++ If `pseudoUrls` is an empty array, `null` or `undefined`, then the function enqueues all links found on the page.++ - **`[transformRequestFunction]`**: [`RequestTransform`](../typedefs/request-transform) - Just before a new [`Request`](../api/request) is+ constructed and enqueued to the [`RequestQueue`](../api/request-queue), this function can be used to remove it or modify its contents such as+ `userData`, `payload` or, most importantly `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share+ the same URL, but differ in methods or payloads, or to dynamically update or create `userData`.++ For example: by adding `useExtendedUniqueKey: true` to the `request` object, `uniqueKey` will be computed from a combination of `url`, `method`+ and `payload` which enables crawling of websites that navigate using form submits (POST requests).++ **Example:**++ ```javascript+ {+ transformRequestFunction: request => {+ request.userData.foo = 'bar';+ request.useExtendedUniqueKey = true;+ return request;+ };+ }+ ```++ - **`[waitForPageIdleSecs]`**: `number` <code> = 1</code> - Clicking in the page triggers various asynchronous operations that lead to new URLs+ being shown by the browser. It could be a simple JavaScript redirect or opening of a new tab in the browser. These events often happen only+ some time after the actual click. Requests typically take milliseconds while new tabs open in hundreds of milliseconds.++ To be able to capture all those events, the `enqueueLinksByClickingElements()` function repeatedly waits for the `waitForPageIdleSecs`. By+ repeatedly we mean that whenever a relevant event is triggered, the timer is restarted. As long as new events keep coming, the function will not+ return, unless the below `maxWaitForPageIdleSecs` timeout is reached.++ You may want to reduce this for example when you're sure that your clicks do not open new tabs, or increase when you're not getting all the+ expected URLs.++ - **`[maxWaitForPageIdleSecs]`**: `number` <code> = 5</code> - This is the maximum period for which the function will keep tracking events, even+ if more events keep coming. Its purpose is to prevent a deadlock in the page by periodic events, often unrelated to the clicking itself. See+ `waitForPageIdleSecs` above for an explanation.++**Returns**:++[`Promise<Array<QueueOperationInfo>>`](../typedefs/queue-operation-info) - Promise that resolves to an array of+[`QueueOperationInfo`](../typedefs/queue-operation-info) objects.++---++<a name="injectfile"></a>++## `puppeteer.injectFile(page, filePath, [options])`++Injects a JavaScript file into a Puppeteer page. Unlike Puppeteer's `addScriptTag` function, this function works on pages with arbitrary Cross-Origin+Resource Sharing (CORS) policies.++File contents are cached for up to 10 files to limit file system access.++**Parameters**:++- **`page`**: `Page` - Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.+- **`filePath`**: `string` - File path+- **`[options]`**: `object`+ - **`[surviveNavigations]`**: `boolean` - Enables the injected script to survive page navigations and reloads without need to be re-injected+ manually. This does not mean, however, that internal state will be preserved. Just that it will be automatically re-injected on each+ navigation before any other scripts get the chance to execute.++**Returns**:++`Promise<*>`++---++<a name="injectjquery"></a>++## `puppeteer.injectJQuery(page)`++Injects the [jQuery](https://jquery.com/) library into a Puppeteer page. jQuery is often useful for various web scraping and crawling tasks. For+example, it can help extract text from HTML elements using CSS selectors.++Beware that the injected jQuery object will be set to the `window.$` variable and thus it might cause conflicts with other libraries included by the+page that use the same variable name (e.g. another version of jQuery). This can affect functionality of page's scripts.++The injected jQuery will survive page navigations and reloads.++**Example usage:**++```javascript+await Apify.utils.puppeteer.injectJQuery(page);+const title = await page.evaluate(() => {+ return $('head title').text();+});+```++Note that `injectJQuery()` does not affect the Puppeteer's [`page.$()`](https://pptr.dev/#?product=Puppeteer&show=api-pageselector) function in any+way.++**Parameters**:++- **`page`**: `Page` - Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.++**Returns**:++`Promise<*>`++---++<a name="injectunderscore"></a>++## `puppeteer.injectUnderscore(page)`++Injects the [Underscore](https://underscorejs.org/) library into a Puppeteer page.++Beware that the injected Underscore object will be set to the `window._` variable and thus it might cause conflicts with libraries included by the+page that use the same variable name. This can affect functionality of page's scripts.++The injected Underscore will survive page navigations and reloads.++**Example usage:**++```javascript+await Apify.utils.puppeteer.injectUnderscore(page);+const escapedHtml = await page.evaluate(() => {+ return _.escape('<h1>Hello</h1>');+});+```++**Parameters**:++- **`page`**: `Page` - Puppeteer [Page](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#class-page) object.++**Returns**:++`Promise<*>`++---++<a name="blockrequests"></a>++## `puppeteer.blockRequests(page, [options])`++Forces the Puppeteer browser tab to block loading URLs that match a provided pattern. This is useful to speed up crawling of websites, since it+reduces the amount of data that needs to be downloaded from the web, but it may break some websites or unexpectedly prevent loading of resources.++By default, the function will block all URLs including the following patterns:++```json+[".css", ".jpg", ".jpeg", ".png", ".svg", ".gif", ".woff", ".pdf", ".zip"]+```++If you want to extend this list further, use the `extraUrlPatterns` option, which will keep blocking the default patterns, as well as add your custom+ones. If you would like to block only specific patterns, use the `urlPatterns` option, which will override the defaults and block only URLs with your+custom patterns.++This function does not use Puppeteer's request interception and therefore does not interfere with browser cache. It's also faster than blocking+requests using interception, because the blocking happens directly in the browser without the round-trip to Node.js, but it does not provide the extra+benefits of request interception.++The function will never block main document loads and their respective redirects.++**Example usage**++```javascript+const Apify = require('apify');++const browser = await Apify.launchPuppeteer();+const page = await browser.newPage();++// Block all requests to URLs that include `adsbygoogle.js` and also all defaults.+await Apify.utils.puppeteer.blockRequests(page, {+ extraUrlPatterns: ['adsbygoogle.js'],+});++await page.goto('https://cnn.com');+```++**Parameters**:++- **`page`**: `Page` - Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.+- **`[options]`**: `object`+ - **`[urlPatterns]`**: `Array<string>` - The patterns of URLs to block from being loaded by the browser. Only `*` can be used as a wildcard. It+ is also automatically added to the beginning and end of the pattern. This limitation is enforced by the DevTools protocol. `.png` is the same+ as `*.png*`.+ - **`[extraUrlPatterns]`**: `Array<string>` - If you just want to append to the default blocked patterns, use this property.++**Returns**:++`Promise<void>`++---++<a name="cacheresponses"></a>++## ~~`puppeteer.cacheResponses(page, cache, responseUrlRules)`~~++**_Deprecated_**++_NOTE:_ In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal performance. Until this+resolves, we suggest just relying on the in-browser cache unless absolutely necessary.++Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer. _IMPORTANT_: Caching+responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers. This issue should be resolved or atleast+mitigated in future iterations of this feature.++**Parameters**:++- **`page`**: `Page` - Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.+- **`cache`**: `Object<string, *>` - Object in which responses are stored+- **`responseUrlRules`**: `Array<(string|RegExp)>` - List of rules that are used to check if the response should be cached. String rules are+ compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).++**Returns**:++`Promise<void>`++---++<a name="compilescript"></a>++## `puppeteer.compileScript(scriptString, context)`++Compiles a Puppeteer script into an async function that may be executed at any time by providing it with the following object:++```+{+ page: Page,+ request: Request,+}+```++Where `page` is a Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) and `request` is a [`Request`](../api/request).++The function is compiled by using the `scriptString` parameter as the function's body, so any limitations to function bodies apply. Return value of+the compiled function is the return value of the function body = the `scriptString` parameter.++As a security measure, no globals such as `process` or `require` are accessible from within the function body. Note that the function does not provide+a safe sandbox and even though globals are not easily accessible, malicious code may still execute in the main process via prototype manipulation.+Therefore you should only use this function to execute sanitized or safe code.++Custom context may also be provided using the `context` parameter. To improve security, make sure to only pass the really necessary objects to the+context. Preferably making secured copies beforehand.++**Parameters**:++- **`scriptString`**: `string`+- **`context`**: `Object<string, *>`++**Returns**:++[`CompiledScriptFunction`](../typedefs/compiled-script-function)++---diff --git a/website/versioned_docs/version-2.3.0/api/social.md b/website/versioned_docs/version-2.3.0/api/social.mdnew file mode 100644index 000000000..d18ceb87f--- /dev/null+++ b/website/versioned_docs/version-2.3.0/api/social.md@@ -0,0 +1,439 @@+---+id: version-2.3.0-social+title: utils.social+original_id: social+---++<a name="social"></a>++A namespace that contains various utilities to help you extract social handles from text, URLs and and HTML documents.++**Example usage:**++```javascript+const Apify = require('apify');++const emails = Apify.utils.social.emailsFromText('[email protected] [email protected]');+```++---++<a name="linkedin_regex"></a>++## `social.LINKEDIN_REGEX`++Regular expression to exactly match a single LinkedIn profile URL. It has the following form: `/^...$/i` and matches URLs such as:++```+https://www.linkedin.com/in/alan-turing+en.linkedin.com/in/alan-turing+linkedin.com/in/alan-turing+```++The regular expression does NOT match URLs with additional subdirectories or query parameters, such as:++```+https://www.linkedin.com/in/linus-torvalds/latest-activity+```++Example usage:++```+if (Apify.utils.social.LINKEDIN_REGEX.test('https://www.linkedin.com/in/alan-turing')) {+ console.log('Match!');+}+```++---++<a name="linkedin_regex_global"></a>++## `social.LINKEDIN_REGEX_GLOBAL`++Regular expression to find multiple LinkedIn profile URLs in a text or HTML. It has the following form: `/.../ig` and matches URLs such as:++```+https://www.linkedin.com/in/alan-turing+en.linkedin.com/in/alan-turing+linkedin.com/in/alan-turing+```++If the profile URL contains subdirectories or query parameters, the regular expression extracts just the base part of the profile URL. For example,+from text such as:++```+https://www.linkedin.com/in/linus-torvalds/latest-activity+```++the expression extracts just the following base URL:++```+https://www.linkedin.com/in/linus-torvalds+```++Example usage:++```+const matches = text.match(Apify.utils.social.LINKEDIN_REGEX_GLOBAL);+if (matches) console.log(`${matches.length} LinkedIn profiles found!`);+```++---++<a name="instagram_regex"></a>++## `social.INSTAGRAM_REGEX`++Regular expression to exactly match a single Instagram profile URL. It has the following form: `/^...$/i` and matches URLs such as:++```+https://www.instagram.com/old_prague+www.instagram.com/old_prague/+instagr.am/old_prague+```++The regular expression does NOT match URLs with additional subdirectories or query parameters, such as:++```+https://www.instagram.com/cristiano/followers+```++Example usage:++```+if (Apify.utils.social.INSTAGRAM_REGEX.test('https://www.instagram.com/old_prague')) {+ console.log('Match!');+}+```++---++<a name="instagram_regex_global"></a>++## `social.INSTAGRAM_REGEX_GLOBAL`++Regular expression to find multiple Instagram profile URLs in a text or HTML. It has the following form: `/.../ig` and matches URLs such as:++```+https://www.instagram.com/old_prague+www.instagram.com/old_prague/+instagr.am/old_prague+```++If the profile URL contains subdirectories or query parameters, the regular expression extracts just the base part of the profile URL. For example,+from text such as:++```+https://www.instagram.com/cristiano/followers+```++the expression extracts just the following base URL:++```+https://www.instagram.com/cristiano+```++Example usage:++```+const matches = text.match(Apify.utils.social.INSTAGRAM_REGEX_GLOBAL);+if (matches) console.log(`${matches.length} Instagram profiles found!`);+```++---++<a name="twitter_regex"></a>++## `social.TWITTER_REGEX`++Regular expression to exactly match a single Twitter profile URL. It has the following form: `/^...$/i` and matches URLs such as:++```+https://www.twitter.com/apify+twitter.com/apify+```++The regular expression does NOT match URLs with additional subdirectories or query parameters, such as:++```+https://www.twitter.com/realdonaldtrump/following+```++Example usage:++```+if (Apify.utils.social.TWITTER_REGEX.test('https://www.twitter.com/apify')) {+ console.log('Match!');+}+```++---++<a name="twitter_regex_global"></a>++## `social.TWITTER_REGEX_GLOBAL`++Regular expression to find multiple Twitter profile URLs in a text or HTML. It has the following form: `/.../ig` and matches URLs such as:++```+https://www.twitter.com/apify+twitter.com/apify+```++If the profile URL contains subdirectories or query parameters, the regular expression extracts just the base part of the profile URL. For example,+from text such as:++```+https://www.twitter.com/realdonaldtrump/following+```++the expression extracts only the following base URL:++```+https://www.twitter.com/realdonaldtrump+```++Example usage:++```+const matches = text.match(Apify.utils.social.TWITTER_REGEX_STRING);+if (matches) console.log(`${matches.length} Twitter profiles found!`);+```++---++<a name="facebook_regex"></a>++## `social.FACEBOOK_REGEX`++Regular expression to exactly match a single Facebook profile URL. It has the following form: `/^...$/i` and matches URLs such as:++```+https://www.facebook.com/apifytech+facebook.com/apifytech+fb.com/apifytech+https://www.facebook.com/profile.php?id=123456789+```++The regular expression does NOT match URLs with additional subdirectories or query parameters, such as:++```+https://www.facebook.com/apifytech/photos+```++Example usage:++```+if (Apify.utils.social.FACEBOOK_REGEX.test('https://www.facebook.com/apifytech')) {+ console.log('Match!');+}+```++---++<a name="facebook_regex_global"></a>++## `social.FACEBOOK_REGEX_GLOBAL`++Regular expression to find multiple Facebook profile URLs in a text or HTML. It has the following form: `/.../ig` and matches URLs such as:++```+https://www.facebook.com/apifytech+facebook.com/apifytech+fb.com/apifytech+```++If the profile URL contains subdirectories or query parameters, the regular expression extracts just the base part of the profile URL. For example,+from text such as:++```+https://www.facebook.com/apifytech/photos+```++the expression extracts only the following base URL:++```+https://www.facebook.com/apifytech+```++Example usage:++```+const matches = text.match(Apify.utils.social.FACEBOOK_REGEX_GLOBAL);+if (matches) console.log(`${matches.length} Facebook profiles found!`);+```++---++<a name="youtube_regex"></a>++## `social.YOUTUBE_REGEX`++Regular expression to exactly match a single Youtube channel, user or video URL. It has the following form: `/^...$/i` and matches URLs such as:++```+https://www.youtube.com/watch?v=kM7YfhfkiEE+https://youtu.be/kM7YfhfkiEE+https://www.youtube.com/c/TrapNation+https://www.youtube.com/channel/UCklie6BM0fhFvzWYqQVoCTA+https://www.youtube.com/user/pewdiepie+```++Please note that this won't match URLs like https://www.youtube.com/pewdiepie that redirect to /user or /channel.++Example usage:++```+if (Apify.utils.social.YOUTUBE_REGEX.test('https://www.youtube.com/watch?v=kM7YfhfkiEE')) {+ console.log('Match!');+}+```++---++<a name="youtube_regex_global"></a>++## `social.YOUTUBE_REGEX_GLOBAL`++Regular expression to find multiple Youtube channel, user or video URLs in a text or HTML. It has the following form: `/.../ig` and matches URLs such+as:++```+https://www.youtube.com/watch?v=kM7YfhfkiEE+https://youtu.be/kM7YfhfkiEE+https://www.youtube.com/c/TrapNation+https://www.youtube.com/channel/UCklie6BM0fhFvzWYqQVoCTA+https://www.youtube.com/user/pewdiepie+```++Please note that this won't match URLs like https://www.youtube.com/pewdiepie that redirect to /user or /channel.++Example usage:++```+const matches = text.match(Apify.utils.social.YOUTUBE_REGEX_GLOBAL);+if (matches) console.log(`${matches.length} Youtube videos found!`);+```++---++<a name="email_regex"></a>++## `social.EMAIL_REGEX`++Regular expression to exactly match a single email address. It has the following form: `/^...$/i`.++---++<a name="email_regex_global"></a>++## `social.EMAIL_REGEX_GLOBAL`++Regular expression to find multiple email addresses in a text. It has the following form: `/.../ig`.++---++<a name="emailsfromtext"></a>++## `social.emailsFromText(text)`++The function extracts email addresses from a plain text. Note that the function preserves the order of emails and keep duplicates.++**Parameters**:++- **`text`**: `string` - Text to search in.++**Returns**:++`Array<string>` - Array of emails addresses found. If no emails are found, the function returns an empty array.++---++<a name="emailsfromurls"></a>++## `social.emailsFromUrls(urls)`++The function extracts email addresses from a list of URLs. Basically it looks for all `mailto:` URLs and returns valid email addresses from them. Note+that the function preserves the order of emails and keep duplicates.++**Parameters**:++- **`urls`**: `Array<string>` - Array of URLs.++**Returns**:++`Array<string>` - Array of emails addresses found. If no emails are found, the function returns an empty array.++---++<a name="phonesfromtext"></a>++## `social.phonesFromText(text)`++The function attempts to extract phone numbers from a text. Please note that the results might not be accurate, since phone numbers appear in a large+variety of formats and conventions. If you encounter some problems, please [file an issue](https://github.com/apify/apify-js/issues).++**Parameters**:++- **`text`**: `string` - Text to search the phone numbers in.++**Returns**:++`Array<string>` - Array of phone numbers found. If no phone numbers are found, the function returns an empty array.++---++<a name="phonesfromurls"></a>++## `social.phonesFromUrls(urls)`++Finds phone number links in an array of URLs and extracts the phone numbers from them. Note that the phone number links look like `tel://123456789`,+`tel:/123456789` or `tel:123456789`.++**Parameters**:++- **`urls`**: `Array<string>` - Array of URLs.++**Returns**:++`Array<string>` - Array of phone numbers found. If no phone numbers are found, the function returns an empty array.++---++<a name="parsehandlesfromhtml"></a>++## `social.parseHandlesFromHtml(html, [data])`++The function attempts to extract emails, phone numbers and social profile URLs from a HTML document, specifically LinkedIn, Twitter, Instagram and+Facebook profile URLs. The function removes duplicates from the resulting arrays and sorts the items alphabetically.++Note that the `phones` field contains phone numbers extracted from the special phone links such as `[call us](tel:+1234556789)` (see+[`social.phonesFromUrls()`](../api/social#phonesfromurls)) and potentially other sources with high certainty, while `phonesUncertain` contains phone+numbers extracted from the plain text, which might be very inaccurate.++**Example usage:**++```javascript+const Apify = require('apify');++const browser = await Apify.launchPuppeteer();+const page = await browser.newPage();+await page.goto('http://www.example.com');+const html = await page.content();++const result = Apify.utils.social.parseHandlesFromHtml(html);+console.log('Social handles:');+console.dir(result);+```++**Parameters**:++- **`html`**: `string` - HTML text+- **`[data]`**: `*` | `null` <code> = </code> - Optional object which will receive the `text` and `$` properties that contain text content of the+ HTML and `cheerio` object, respectively. This is an optimization so that the caller doesn't need to parse the HTML document again, if needed.++**Returns**:++[`SocialHandles`](../typedefs/social-handles) - An object with the social handles.++---diff --git a/website/versioned_docs/version-2.3.0/api/utils.md b/website/versioned_docs/version-2.3.0/api/utils.mdnew file mode 100644index 000000000..5bc16966d--- /dev/null+++ b/website/versioned_docs/version-2.3.0/api/utils.md@@ -0,0 +1,299 @@+---+id: version-2.3.0-utils+title: utils+original_id: utils+---++<a name="utils"></a>++A namespace that contains various utilities.++**Example usage:**++```javascript+const Apify = require('apify');++...++// Sleep 1.5 seconds+await Apify.utils.sleep(1500);+```++---++<a name="url_no_commas_regex"></a>++## `utils.URL_NO_COMMAS_REGEX`++Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters and does not+support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).++---++<a name="url_with_commas_regex"></a>++## `utils.URL_WITH_COMMAS_REGEX`++Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query. Note,+however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.++---++<a name="enqueuelinks"></a>++## `utils.enqueueLinks(options)`++The function finds elements matching a specific CSS selector (HTML anchor (`<a>`) by default) either in a Puppeteer page, or in a Cheerio object+(parsed HTML), and enqueues the URLs in their `href` attributes to the provided [`RequestQueue`](../api/request-queue). If you're looking to find URLs+in JavaScript heavy pages where links are not available in `href` elements, but rather navigations are triggered in click handlers see+[`puppeteer.enqueueLinksByClickingElements()`](../api/puppeteer#enqueuelinksbyclickingelements).++Optionally, the function allows you to filter the target links' URLs using an array of [`PseudoUrl`](../api/pseudo-url) objects and override settings+of the enqueued [`Request`](../api/request) objects.++**Example usage**++```javascript+await Apify.utils.enqueueLinks({+ page,+ requestQueue,+ selector: 'a.product-detail',+ pseudoUrls: ['https://www.example.com/handbags/[.*]', 'https://www.example.com/purses/[.*]'],+});+```++**Parameters**:++- **`options`**: `object` - All `enqueueLinks()` parameters are passed via an options object with the following keys:++ - **`[page]`**: `PuppeteerPage` | `PlaywrightPage` - Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object. Either+ `page` or `$` option must be provided.+ - **`[limit]`**: `Number` - Limit the count of actually enqueued URLs to this number. Useful for testing across the entire crawling scope.+ - **`[$]`**: `CheerioAPI` - [`Cheerio`](https://github.com/cheeriojs/cheerio) function with loaded HTML. Either `page` or `$` option must be+ provided.+ - **`requestQueue`**: [`RequestQueue`](../api/request-queue) - A request queue to which the URLs will be enqueued.+ - **`[selector]`**: `string` <code> = "'a'"</code> - A CSS selector matching links to be enqueued.+ - **`[baseUrl]`**: `string` - A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer, since the+ relative URL resolution is done inside the browser automatically.+ - **`[pseudoUrls]`**: `Array<Object<string, *>>` | `Array.<string>` - An array of [`PseudoUrl`](../api/pseudo-url)s matching the URLs to+ be enqueued, or an array of strings or RegExps or plain Objects from which the [`PseudoUrl`](../api/pseudo-url)s can be constructed.++ The plain objects must include at least the `purl` property, which holds the pseudo-URL string or RegExp. All remaining keys will be used as the+ `requestTemplate` argument of the [`PseudoUrl`](../api/pseudo-url) constructor, which lets you specify special properties for the enqueued+ [`Request`](../api/request) objects.++ If `pseudoUrls` is an empty array, `null` or `undefined`, then the function enqueues all links found on the page.++ - **`[transformRequestFunction]`**: [`RequestTransform`](../typedefs/request-transform) - Just before a new [`Request`](../api/request) is+ constructed and enqueued to the [`RequestQueue`](../api/request-queue), this function can be used to remove it or modify its contents such as+ `userData`, `payload` or, most importantly `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share+ the same URL, but differ in methods or payloads, or to dynamically update or create `userData`.++ For example: by adding `keepUrlFragment: true` to the `request` object, URL fragments will not be removed when `uniqueKey` is computed.++ **Example:**++ ```javascript+ {+ transformRequestFunction: request => {+ request.userData.foo = 'bar';+ request.keepUrlFragment = true;+ return request;+ };+ }+ ```++**Returns**:++[`Promise<Array<QueueOperationInfo>>`](../typedefs/queue-operation-info) - Promise that resolves to an array of+[`QueueOperationInfo`](../typedefs/queue-operation-info) objects.++---++<a name="requestasbrowser"></a>++## `utils.requestAsBrowser(options)`++**IMPORTANT:** This function uses an insecure version of HTTP parser by default and also ignores SSL/TLS errors. This is very useful in scraping,+because it allows bypassing certain anti-scraping walls, but it also exposes some vulnerability. For other than scraping scenarios, please set+`useInsecureHttpParser: false` and `ignoreSslErrors: false`.++Sends a HTTP request that looks like a request sent by a web browser, fully emulating browser's HTTP headers. It uses HTTP2 by default for Node 12+.++This function is useful for web scraping of websites that send the full HTML in the first response. Thanks to this function, the target web server has+no simple way to find out the request hasn't been sent by a human's web browser. Using a headless browser for such requests is an order of magnitude+more resource-intensive than this function.++The function emulates the Chrome and Firefox web browsers. If you want more control over the browsers and their versions, use the+`headerGeneratorOptions` property. You can find more info in the readme of the [`header-generator`](https://github.com/apify/header-generator)+library.++Internally, the function uses the [`got-scraping`](https://github.com/apify/got-scraping) library to perform the request. All `options` not recognized+by this function are passed to it so see it for more details.++**Example usage:**++```js+const Apify = require('apify');++const { utils: { requestAsBrowser } } = Apify;++...++const response = await requestAsBrowser({ url: 'https://www.example.com/' });++const html = response.body;+const status = response.statusCode;+const contentType = response.headers['content-type'];+```++**Parameters**:++- **`options`**: [`RequestAsBrowserOptions`](../typedefs/request-as-browser-options) - All `requestAsBrowser` configuration options.++**Returns**:++[`Promise<RequestAsBrowserResult>`](../typedefs/request-as-browser-result) - The result can be various objects, but it will always be like a+[Node.js HTTP response stream](https://nodejs.org/api/http.html#http_class_http_incomingmessage) with a 'body' property for the parsed response body,+unless the 'stream' option is used.++---++<a name="isdocker"></a>++## `utils.isDocker(forceReset)`++Returns a `Promise` that resolves to true if the code is running in a Docker container.++**Parameters**:++- **`forceReset`**: `boolean`++**Returns**:++`Promise<boolean>`++---++<a name="sleep"></a>++## `utils.sleep(millis)`++Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting in your code, e.g. to prevent overloading of+target website or to avoid bot detection.++**Example usage:**++```+const Apify = require('apify');++...++// Sleep 1.5 seconds+await Apify.utils.sleep(1500);+```++**Parameters**:++- **`millis`**: `number` - Period of time to sleep, in milliseconds. If not a positive number, the returned promise resolves immediately.++**Returns**:++`Promise<void>`++---++<a name="downloadlistofurls"></a>++## `utils.downloadListOfUrls(options)`++Returns a promise that resolves to an array of urls parsed from the resource available at the provided url. Optionally, custom regular expression and+encoding may be provided.++**Parameters**:++- **`options`**: `object`+ - **`url`**: `string` - URL to the file+ - **`[encoding]`**: `string` <code> = "'utf8'"</code> - The encoding of the file.+ - **`[urlRegExp]`**: `RegExp` <code> = URL_NO_COMMAS_REGEX</code> - Custom regular expression to identify the URLs in the file to extract. The+ regular expression should be case-insensitive and have global flag set (i.e. `/something/gi`).++**Returns**:++`Promise<Array<string>>`++---++<a name="extracturls"></a>++## `utils.extractUrls(options)`++Collects all URLs in an arbitrary string to an array, optionally using a custom regular expression.++**Parameters**:++- **`options`**: `object`+ - **`string`**: `string`+ - **`[urlRegExp]`**: `RegExp` <code> = Apify.utils.URL_NO_COMMAS_REGEX</code>++**Returns**:++`Array<string>`++---++<a name="htmltotext"></a>++## `utils.htmlToText(html)`++The function converts a HTML document to a plain text.++The plain text generated by the function is similar to a text captured by pressing Ctrl+A and Ctrl+C on a page when loaded in a web browser. The+function doesn't aspire to preserve the formatting or to be perfectly correct with respect to HTML specifications. However, it attempts to generate+newlines and whitespaces in and around HTML elements to avoid merging distinct parts of text and thus enable extraction of data from the text (e.g.+phone numbers).++**Example usage**++```javascript+const text = htmlToText('<html><body>Some text</body></html>');+console.log(text);+```++Note that the function uses [cheerio](https://www.npmjs.com/package/cheerio) to parse the HTML. Optionally, to avoid duplicate parsing of HTML and+thus improve performance, you can pass an existing Cheerio object to the function instead of the HTML text. The HTML should be parsed with the+`decodeEntities` option set to `true`. For example:++```javascript+const cheerio = require('cheerio');+const html = '<html><body>Some text</body></html>';+const text = htmlToText(cheerio.load(html, { decodeEntities: true }));+```++**Parameters**:++- **`html`**: `string` | `CheerioAPI` - HTML text or parsed HTML represented using a [cheerio](https://www.npmjs.com/package/cheerio) function.++**Returns**:++`string` - Plain text++---++<a name="purgelocalstorage"></a>++## `utils.purgeLocalStorage([folder])`++Cleans up the local storage folder created when testing locally. This is useful in the event you are debugging your code locally.++Be careful as this will remove the folder you provide and everything in it!++**Parameters**:++- **`[folder]`**: `string` - The folder to clean up++**Returns**:++`Promise<void>`++---diff --git a/website/versioned_docs/version-2.3.0/examples/basic_crawler.md b/website/versioned_docs/version-2.3.0/examples/basic_crawler.mdnew file mode 100644index 000000000..f155b5867--- /dev/null+++ b/website/versioned_docs/version-2.3.0/examples/basic_crawler.md@@ -0,0 +1,57 @@+---+id: version-2.3.0-basic-crawler+title: Basic crawler+original_id: basic-crawler+---++This is the most bare-bones example of the Apify SDK, which demonstrates some of its building blocks such as the+[`BasicCrawler`](/docs/api/basic-crawler). You probably don't need to go this deep though, and it would be better to start with one of the full+featured crawlers like [`CheerioCrawler`](https://sdk.apify.com/docs/examples/cheerio-crawler) or+[`PlaywrightCrawler`](https://sdk.apify.com/docs/examples/playwright-crawler).++The script simply downloads several web pages with plain HTTP requests using the [`Apify.utils.requestAsBrowser()`](/docs/api/utils#requestasbrowser)+convenience function and stores their raw HTML and URL in the default dataset. In local configuration, the data will be stored as JSON files in+`./apify_storage/datasets/default`.++```javascript+const Apify = require('apify');++// Apify.main() function wraps the crawler logic (it is optional).+Apify.main(async () => {+ // Create and initialize an instance of the RequestList class that contains+ // a list of URLs to crawl. Here we use just a few hard-coded URLs.+ const requestList = await Apify.openRequestList('start-urls', [+ { url: 'http://www.google.com/' },+ { url: 'http://www.example.com/' },+ { url: 'http://www.bing.com/' },+ { url: 'http://www.wikipedia.com/' },+ ]);++ // Create a BasicCrawler - the simplest crawler that enables+ // users to implement the crawling logic themselves.+ const crawler = new Apify.BasicCrawler({+ // Let the crawler fetch URLs from our list.+ requestList,++ // This function will be called for each URL to crawl.+ handleRequestFunction: async ({ request }) => {+ const { url } = request;+ console.log(`Processing ${url}...`);++ // Fetch the page HTML via Apify utils requestAsBrowser+ const { body } = await Apify.utils.requestAsBrowser({ url });++ // Store the HTML and URL to the default dataset.+ await Apify.pushData({+ url: request.url,+ html: body,+ });+ },+ });++ // Run the crawler and wait for it to finish.+ await crawler.run();++ console.log('Crawler finished.');+});+```diff --git a/website/versioned_docs/version-2.3.0/guides/avoid_blocking.md b/website/versioned_docs/version-2.3.0/guides/avoid_blocking.mdnew file mode 100644index 000000000..cec305798--- /dev/null+++ b/website/versioned_docs/version-2.3.0/guides/avoid_blocking.md@@ -0,0 +1,48 @@+---+id: version-2.3.0-avoid-blocking+title: Avoid getting blocked+original_id: avoid-blocking+---++A Scraper might get blocked for numerous reasons. Let's narrow it down to two main reasons. The first one is a bad or blocked IP address. This topic is covered in the [proxy management guide](proxy_management.md). The second reason we will explore more is [browser fingerprints](https://pixelprivacy.com/resources/browser-fingerprinting/) or signatures.+Browser fingerprint is a collection of browser attributes and significant features that can show if your browser is a bot or a real user. Moreover, even most browsers have these unique features that allow the website to track the browser even within different IP addresses. This is the main reason why scrapers should change browser fingerprints while doing browser-based scraping. In addition, it should reduce blocking significantly.++Changing browser fingerprints can be a tedious job. Luckily, Apify SDK provides this feature out of the box with zero configuration necessary. Let's take a look at how it is done.++ Changing browser fingerprints is available in `PuppeteerCrawler` and `PlaywrightCrawler`. You have to pass the `useFingerprints` option to the `browserPoolOptions`.++ ```javascript+const crawler = new Apify.PlaywrightCrawler({+ browserPoolOptions: {+ useFingerprints: true,+ },+})++ ```+Now, it is all set. The fingerprints are going to be generated for the default browser and the operating system. The Crawler can have the generation alghoritm customized to reflect particular browser version and many more. Let's take a look at the example bellow:++ ```javascript+const crawler = new Apify.PlaywrightCrawler({+ browserPoolOptions: {+ useFingerprints: true,+ fingerprintOptions: {+ fingerprintGeneratorOptions: {+ browsers: [+ { name: 'firefox', minVersion: 80 },+ { name: 'chrome', minVersion: 87 },+ 'safari',+ ],+ devices: [+ 'desktop',+ ],+ operatingSystems: [+ 'windows',+ ],+ },+ },+ },+})++ ```+ Fingerprint generator has more options available check out the [Fingerprint generator docs](https://github.com/apify/fingerprint-generator#HeaderGeneratorOptions).+\ No newline at end of filediff --git a/website/versioned_docs/version-2.3.0/guides/docker_images.md b/website/versioned_docs/version-2.3.0/guides/docker_images.mdnew file mode 100644index 000000000..2c64bd813--- /dev/null+++ b/website/versioned_docs/version-2.3.0/guides/docker_images.md@@ -0,0 +1,181 @@+---+id: version-2.3.0-docker-images+title: Running in Docker+original_id: docker-images+---++Running headless browsers in Docker requires a lot of setup to do it right. But you don't need to worry about that, because we already did it for you and created base images that you can freely use. We use them every day on the [Apify Platform](../guides/apify_platform.md).++All images can be found in their [GitHub repo](https://github.com/apify/apify-actor-docker) and in our [DockerHub](https://hub.docker.com/orgs/apify).++## Overview+Browsers are pretty big, so we try to provide a wide variety of images to suit your needs. Here's a full list of our Docker images.++- [`apify/actor-node`](#actor-node)+- [`apify/actor-node-puppeteer-chrome`](#actor-node-puppeteer-chrome)+- [`apify/actor-node-playwright`](#actor-node-playwright)+- [`apify/actor-node-playwright-chrome`](#actor-node-playwright-chrome)+- [`apify/actor-node-playwright-firefox`](#actor-node-playwright-firefox)+- [`apify/actor-node-playwright-webkit`](#actor-node-playwright-webkit)++## Example Dockerfile+To use our images, you need a [`Dockerfile`](https://docs.docker.com/engine/reference/builder/). You can either use this example, or bootstrap your projects with the [Apify CLI](../guides/getting_started.md#creating-a-new-project) which automatically copies the correct Dockerfile into your project folder.++```dockerfile+# First, specify the base Docker image. You can read more about+# the available images at https://sdk.apify.com/docs/guides/docker-images+# You can also use any other image from Docker Hub.+# The 16 represents the version of Node.js you want to use.+FROM apify/actor-node:16++# Second, copy just package.json and package-lock.json since it should be+# the only file that affects "npm install" in the next step, to speed up the build+COPY package*.json ./++# Install NPM packages, skip optional and development dependencies to+# keep the image small. Avoid logging too much and print the dependency+# tree for debugging+RUN npm --quiet set progress=false \+ && npm install --only=prod --no-optional \+ && echo "Installed NPM packages:" \+ && (npm list || true) \+ && echo "Node.js version:" \+ && node --version \+ && echo "NPM version:" \+ && npm --version++# Next, copy the remaining files and directories with the source code.+# Since we do this after NPM install, quick build will be really fast+# for most source file changes.+COPY . ./++# Optionally, specify how to launch the source code of your actor.+# By default, Apify's base Docker images define the CMD instruction+# that runs the Node.js source code using the command specified+# in the "scripts.start" section of the package.json file.+# In short, the instruction looks something like this:+#+# CMD npm start+```++## Versioning+Each image is tagged with up to 2 version tags, depending on the type of the image. One for Node.js version and second for pre-installed web automation library version. If you use the image name without a version tag, you'll always get the latest available version.++> We recommend always using at least the Node.js version tag in your production Dockerfiles. It will ensure that a future update of Node.js will not break your automations.++### Node.js versioning+Our images are built with multiple Node.js versions to ensure backwards compatibility. Currently, Node.js versions 14, 15 and 16 are supported. To select the preferred version, use the appropriate number as the image tag.++```dockerfile+# Use Node.js 14+FROM apify/actor-node:14+# Use Node.js 16+FROM apify/actor-node-playwright:16+```++### Automation library versioning+Images that include a pre-installed automation library, which means all images that include `puppeteer` or `playwright` in their name, are also tagged with the pre-installed version of the library. For example, `apify/actor-node-puppeteer-chrome:16-8.0.0` comes with Node.js 16 and Puppeteer v8.0.0. If you try to install a different version of Puppeteer into this image, you may run into compatibility issues, because the Chromium version bundled with `puppeteer` will not match the version of Chrome we pre-installed.++Similarly `apify/actor-node-playwright-firefox:14-1.10.0` runs on Node.js 14 and is pre-installed with the Firefox version that comes with v1.10.0.++Installing `apify/actor-node-puppeteer-chrome` (without a tag) will install the latest available version of Node.js and `puppeteer`.++### Pre-release tags+We also build pre-release versions of the images to test the changes we make. Those are typically denoted by a `beta` suffix, but it can vary depending on our needs. If you need to try a pre-release version, you can do it like this:++```dockerfile+# Without library version.+FROM apify/actor-node:16-beta+# With library version.+FROM apify/actor-node-playwright-chrome:16-1.10.0-beta+```++## Best practices+- **Always** use Node.js version tag.+- For **added security**, use the automation library version tag.+- Use asterisk `*` as the automation library version in your `package.json` files.++It makes sure the pre-installed version of Puppeteer or Playwright is not reinstalled on build. This is important, because those libraries are only guaranteed to work with specific versions of browsers, and those browsers come pre-installed in the image.++```dockerfile+FROM apify/actor-node-playwright-chrome:16+```++```json+{+ "dependencies": {+ "apify": "^1.2.0",+ "playwright": "*"+ }+}+```++### Warning about image size+Browsers are huge. If you don't need them all in your image, it's better to use a smaller image with only the one browser you need.++Be careful when installing new dependencies. Nothing prevents you from installing Playwright into the`actor-node-puppeteer-chrome` image, but the resulting image will be about 3 times larger and extremely slow to download and build.++Use only what you need, and you'll be rewarded with reasonable build and start times.++## actor-node+This is the smallest image we have based on Alpine Linux. It does not include any browsers, and it's therefore+best used with [`CheerioCrawler`](../api/cheerio-crawler). It benefits from lightning fast builds and container startups.++[`PuppeteerCrawler`](../api/puppeteer-crawler), [`PlaywrightCrawler`](../api/playwright-crawler)+and other browser based features will **NOT** work with this image.++```dockerfile+FROM apify/actor-node:16+```++## actor-node-puppeteer-chrome+This image includes Puppeteer (Chromium) and the Chrome browser. It can be used with+[`CheerioCrawler`](../api/cheerio-crawler) and [`PuppeteerCrawler`](../api/puppeteer-crawler), but **NOT** with+[`PlaywrightCrawler`](../api/playwright-crawler).++The image supports XVFB by default, so you can run both `headless` and `headful` browsers with it.++```dockerfile+FROM apify/actor-node-puppeteer-chrome:16+```++## actor-node-playwright+A very large and slow image that can run all Playwright browsers: Chromium, Chrome, Firefox,+WebKit. Everything is installed. If you need to develop or test with multiple browsers, this is the image to choose,+but in most cases, we suggest using the specialized images below.++```dockerfile+FROM apify/actor-node-playwright:16+```++## actor-node-playwright-chrome+Similar to [`actor-node-puppeteer-chrome`](#actor-node-puppeteer-chrome), but for Playwright. You can run+[`CheerioCrawler`](../api/cheerio-crawler) and [`PlaywrightCrawler`](../api/playwright-crawler),+but **NOT** [`PuppeteerCrawler`](../api/puppeteer-crawler).++It uses the [`PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD`](https://playwright.dev/docs/api/environment-variables/)+environment variable to block installation of more browsers into your images (to keep them small).+If you want more browsers, either choose the [`actor-node-playwright`](#actor-node-playwright) image+or override this env var.++The image supports XVFB by default, so you can run both `headless` and `headful` browsers with it.++```dockerfile+FROM apify/actor-node-playwright-chrome:16+```++## actor-node-playwright-firefox+Same idea as [`actor-node-playwright-chrome`](#actor-node-playwright-chrome), but with Firefox+pre-installed.++```dockerfile+FROM apify/actor-node-playwright-firefox:16+```++## actor-node-playwright-webkit+Same idea as [`actor-node-playwright-chrome`](#actor-node-playwright-chrome), but with WebKit+pre-installed.++```dockerfile+FROM apify/actor-node-playwright-webkit:16+```diff --git a/website/versioned_docs/version-2.3.0/guides/getting_started.md b/website/versioned_docs/version-2.3.0/guides/getting_started.mdnew file mode 100644index 000000000..17fe993b6--- /dev/null+++ b/website/versioned_docs/version-2.3.0/guides/getting_started.md@@ -0,0 +1,1749 @@+---+id: version-2.3.0-getting-started+title: Getting Started+original_id: getting-started+---++Without the right tools, crawling and scraping the web can be difficult. At the very least, you need an HTTP client to make the necessary+requests, but that only gets you raw HTML and sometimes not even that. Then you have to read this HTML and extract the data you're interested in. Once+extracted, it must be stored in a machine-readable format and easily accessible for further processing, because it is the processed data that holds+value.++Apify SDK covers the process end-to-end. From crawling the web for links and scraping the raw data to storing it in various machine readable formats,+ready for processing. With this guide in hand, you should have your own data extraction solutions up and running in a few hours.++## Intro++The goal of this getting started guide is to provide a step-by-step introduction to all the features of the Apify SDK. It will walk you through+creating the simplest of crawlers that only prints text to console, all the way up to complex systems that crawl pages, interact with them as if a real+user were sitting in front of a real browser and output structured data.++Since Apify SDK is usable both locally on any computer and on the [Apify platform](../guides/apify-platform), you will be able+to use the source code in both environments interchangeably. Nevertheless, some initial setup is still required, so choose your preferred starting+environment and let's get into it.++## Setting up locally++To run Apify SDK on your own computer, you need to meet the following pre-requisites first:++1. Have Node.js version 15.10 or higher installed.+ - Visit [Node.js website](https://nodejs.org/en/download/) to download or use+ [fnm](https://github.com/Schniz/fnm)+2. Have NPM installed.+ - NPM comes bundled with Node.js so you should already have it. If not, reinstall Node.js.++If you're not certain, confirm the prerequisites by running:++```bash+node -v+```++```bash+npm -v+```++### Creating a new project++The fastest and best way to create new projects with the Apify SDK is to use our own+[Apify CLI](https://www.npmjs.com/package/apify-cli). This command line tool allows you to create, run and manage Apify+projects with ease, including their deployment to the [Apify platform](../guides/apify-platform) if you wish to run them in the+cloud after developing them locally.++Let's install the Apify CLI with the following command:++```bash+npm install -g apify-cli+```++Once the installation finishes, all you need to do to set up an Apify SDK project is to run:++```bash+apify create my-new-project+```++A prompt will be shown, asking you to choose a template. Disregard the different options for now and choose the template labeled `Hello world`. The+command will now create a new directory in your current working directory, called `my-new-project`, create a `package.json` in this folder and install+all the necessary dependencies. It will also add example source code that you can immediately run.++Let's try that!++```bash+cd my-new-project+```++```bash+apify run -p+```++> The `-p` flag is great to remember, because it stands for `--purge` and it clears out your persistent storages before starting the actor.+> `INPUT.json` and named storages are kept. Whenever you're just restarting your actor and you're not interested in the data of the previous run, you+> should use `apify run -p` to prevent the old state from messing with your current run. If this is confusing, don't worry. You'll learn about+> storages and `INPUT.json` soon.++You should start seeing log messages in the terminal as the system boots up and after a second, a Chromium browser window should pop up. In the+window, you'll see quickly changing pages and back in the terminal, you should see the titles (contents of the `<title>` HTML tags) of the pages+printed.++You can always terminate the crawl with a keypress in the terminal:++```bash+CTRL+C+```++Did you see all that? If you did, congratulations! You're ready to go!++## Setting up on the Apify Platform++Maybe you don't have Node.js installed and don't want the hassle. Or you can't install anything on your computer because you're using one provided by your company. Or perhaps you'd just prefer to start working in the cloud right away. Well, no worries, we've got you covered.++The [Apify platform](../guides/apify-platform) is the foundational product of+[Apify](https://apify.com). It's a serverless cloud computing platform, specifically designed for any web automation jobs,+that may include crawling and scraping, but really works amazingly for any batch jobs and long-running tasks.++It comes with a free account, so let's go to our [sign-up page](https://console.apify.com/sign-up) and create one, if you+haven't already. Don't forget to verify your email. Without it, you won't be able to run any projects.++Once you're in, you might be prompted by our in-app help to walk through a step-by-step guide to some of our new features. Feel free to finish that,+if you'd like, but once you're done, click on the **Actors** tab in the left menu. To read more about **Actors**, see:+[What is an actor](../guides/apify-platform#what-is-an-actor)++### Creating a new project++In the page that shows after clicking on Actors in the left menu, choose **Create new**. Give it a name in the form that opens, let's say,+`my-new-actor`. Disregard all the available options for now and save your changes.++Now click on the **Sources** tab at the top. Disregard the version and environment variables inputs for now and proceed directly to **Source code**.+This is where you develop the actor, if you choose not to do it locally. Just press **Run** below the **Source code** panel. It will automatically+build and run the example source code. You should start seeing log messages that represent the build and after the build is complete, the log messages of+the running actor. Feel free to check out the other **Run** tabs, such as **Info**, where you can find useful information about the run, or+**Key-value-store**, where the actor's **INPUT** and **OUTPUT** are stored.++Good job. You're now ready to run your own source code on the Apify Platform. For more information, visit the+[Actor documentation page](https://docs.apify.com/actor), where you'll find everything about the platform's various+options.++## First crawler++Whether you've chosen to develop locally or in the cloud, it's time to start writing some actual source code. But before we do, let's just briefly+introduce all the Apify SDK classes necessary to make it happen.++### The general idea++There are 4 crawler classes available for use in the Apify SDK. [`BasicCrawler`](../api/basic-crawler), [`CheerioCrawler`](../api/cheerio-crawler), [`PuppeteerCrawler`](../api/puppeteer-crawler) and [`PlaywrightCrawler`](../api/playwright-crawler). We'll talk about their differences later. Now, let's talk about what they have in common.++The general idea of each crawler is to go to a web page, open it, do some stuff there, save some results and continue to the next page, until it's done+its job. So the crawler always needs to find answers to two questions: **Where should I go?** and **What should I do there?** Answering those two+questions is the only setup mandatory for running the crawlers.++### The Where - `Request`, `RequestList` and `RequestQueue`++All crawlers use instances of the [`Request`](../api/request) class to determine where they need to go. Each request may hold a lot of information,+but at the very least, it must hold a URL - a web page to open. But having only one URL would not make sense for crawling. We need to either have a+pre-existing list of our own URLs that we wish to visit, perhaps a thousand, or a million, or we need to build this list dynamically as we crawl,+adding more and more URLs to the list as we progress.++A representation of the pre-existing list is an instance of the [`RequestList`](../api/request-list) class. It is a static, immutable list of URLs and+other metadata (see the [`Request`](../api/request) object) that the crawler will visit, one by one, retrying whenever an error occurs, until there+are no more `Requests` to process.++[`RequestQueue`](../api/request-queue) on the other hand, represents a dynamic queue of `Requests`. One that can be updated at runtime by adding more+pages - `Requests` to process. This allows the crawler to open one page, extract interesting URLs, such as links to other pages on the same domain,+add them to the queue (called _enqueuing_) and repeat this process to build a queue of tens of thousands or more URLs while knowing only a single one+at the beginning.++`RequestList` and `RequestQueue` are essential for the crawler's operation. There is no other way to supply `Requests` = "pages to crawl" to the+crawlers. At least one of them always needs to be provided while setting up. You can also use both at the same time, if you wish.++### The What - `handlePageFunction`++The `handlePageFunction` is the brain of the crawler. It tells it what to do at each and every page it visits. Generally it handles extraction of data+from the page, processing the data, saving it, calling APIs, doing calculations and whatever else you need it to do.++The `handlePageFunction` is provided by you, the user, and invoked automatically by the crawler for each `Request` from either the `RequestList` or+`RequestQueue`. It always receives a single argument and that is a plain `Object`. Its properties change depending on the crawler class used, but it+always includes at least the `request` property, which represents the currently crawled `Request` instance (i.e. the URL the crawler is visiting and+related metadata) and the `autoscaledPool` property, which is an instance of the [`AutoscaledPool`](../api/autoscaled-pool) class and we'll talk about+it in detail later.++```js+// The object received as a single argument by the handlePageFunction+{+ request: Request,+ autoscaledPool: AutoscaledPool+}+```++### Putting it all together++Enough theory! Let's put some of those hard-learned facts into practice. We learned above that we need `Requests` and a `handlePageFunction` to setup+a crawler. We will also use the [`Apify.main()`](../api/apify#main) function. It's not mandatory, but it makes our life easier. We'll+learn about it in detail later on.++Let's start with something super easy. Visit a page, get its title and close. First of all we need to require Apify, to make all of its features available to us:++```js+const Apify = require('apify');+```++Easy, right? It really doesn't get much more difficult than that. For the purposes of this tutorial, we'll be scraping our own webpage+[https://apify.com](https://apify.com). Now, to get there, we need a `Request` with the page's URL in one of our sources,+`RequestList` or `RequestQueue`. Let's go with `RequestQueue` for now.++```js+const Apify = require('apify');++// This is how you use the Apify.main() function.+Apify.main(async () => {+ // First we create the request queue instance.+ const requestQueue = await Apify.openRequestQueue();+ // And then we add a request to it.+ await requestQueue.addRequest({ url: 'https://apify.com' });+});+```++> If you're not familiar with the `async` and `await` keywords used in the example, you should know that these are native syntax in modern JavaScript. You can+> [learn more about them here](https://nikgrozev.com/2017/10/01/async-await/).++The [`requestQueue.addRequest()`](../api/request-queue#addrequest) function automatically converts the plain object we passed to it to a+`Request` instance, so now we have a `requestQueue` that holds one `request` which points to `https://apify.com`. Now we need the+`handlePageFunction`.++```js+// We'll define the function separately so it's more obvious.+const handlePageFunction = async ({ request, $ }) => {+ // This should look familiar if you ever worked with jQuery.+ // We're just getting the text content of the <title> HTML element.+ const title = $('title').text();++ console.log(`The title of "${request.url}" is: ${title}.`);+};+```++Wait, where did the `$` come from? Remember what we learned about the `handlePageFunction` earlier. It expects a plain `Object` as an argument that+will always have a `request` property, but it will also have other properties, depending on the chosen crawler class. Well, `$` is a property provided+by the `CheerioCrawler` class, which we'll set up right now.++```js+const Apify = require('apify');++Apify.main(async () => {+ const requestQueue = await Apify.openRequestQueue();+ await requestQueue.addRequest({ url: 'https://apify.com' });++ const handlePageFunction = async ({ request, $ }) => {+ const title = $('title').text();++ console.log(`The title of "${request.url}" is: ${title}.`);+ };++ // Set up the crawler, passing a single options object as an argument.+ const crawler = new Apify.CheerioCrawler({+ requestQueue,+ handlePageFunction,+ });++ await crawler.run();+});+```++And we're done! You just created your first crawler from scratch. It will download the HTML of `https://apify.com`, find the `<title>` element, get+its text content and print it to console. Good job!++To run the code locally, copy and paste the code, if you haven't already typed it in yourself, to the `main.js` file in the `my-new-project` we+created earlier and run `apify run` from that project's directory.++To run the code on Apify Platform, just replace the original example with your new code and hit Run.++Whichever environment you choose, you should see the message+`The title of "https://apify.com" is: Web Scraping, Data Extraction and Automation - Apify.` printed to the screen. If you do, congratulations and+let's move onto some bigger challenges! And if you feel like you don't really know what just happened there, no worries, it will all become clear when+you learn more about `CheerioCrawler`.++## CheerioCrawler aka jQuery crawler++This is the crawler that we used in our earlier example. Our simplest and also the fastest crawling solution. If you're familiar with `jQuery`, you'll+understand [`CheerioCrawler`](../api/cheerio-crawler) in minutes. [Cheerio](https://www.npmjs.com/package/cheerio) is+essentially `jQuery` for Node.js. It offers the same API, including the familiar `$` object. You can use it, as you would `jQuery`, for manipulating+the DOM of a HTML page. In crawling, you'll mostly use it to select the right elements and extract their text values - the data you're interested in.+But `jQuery` runs in a browser and attaches directly to the browser's DOM. Where does `cheerio` get its HTML? This is where the `Crawler` part of+[`CheerioCrawler`](../api/cheerio-crawler) comes in.++### Overview++[`CheerioCrawler`](../api/cheerio-crawler) crawls by making plain HTTP requests to the provided URLs. As you remember from the previous section, the+URLs are fed to the crawler using either the [`RequestList`](../api/cheerio-crawler) or the [`RequestQueue`](../api/cheerio-crawler). The HTTP responses+it gets back are HTML pages, the same pages you would get in your browser when you first load a URL.++> Note, however, that modern web pages often do not serve all of their content in the first HTML response, but rather the first HTML contains links to+> other resources such as CSS and JavaScript that get downloaded afterwards and together they create the final page. See our+> [`PuppeteerCrawler`](../api/puppeteer-crawler) to crawl those.++Once the page's HTML is retrieved, the crawler will pass it to [Cheerio](https://www.npmjs.com/package/cheerio) for+parsing. The result is the typical `$` function, which should be familiar to `jQuery` users. You can use this `$` to do all sorts of lookups and+manipulation of the page's HTML, but in scraping, we will mostly use it to find specific HTML elements and extract their data.++Example use of Cheerio and its `$` function in comparison to browser JavaScript:++```js+// Return the text content of the <title> element.+document.querySelector('title').textContent; // plain JS+$('title').text(); // Cheerio++// Return an array of all 'href' links on the page.+Array.from(document.querySelectorAll('[href]')).map(el => el.href); // plain JS+$('[href]')+ .map((i, el) => $(el).attr('href'))+ .get(); // Cheerio+```++> This is not to show that Cheerio is better than plain browser JavaScript. Some might actually prefer the more expressive way plain JS provides.+> Unfortunately, the browser JavaScript methods are not available in Node.js, so Cheerio is our best bet to do the parsing.++### When to use `CheerioCrawler`++Even though using `CheerioCrawler` is extremely easy, it probably will not be your first choice for most kinds of crawling or scraping in production+environments. Since most websites nowadays use modern JavaScript to create rich, responsive and data-driven user experiences, the plain HTTP requests+the crawler uses may just fall short of your needs.++But [`CheerioCrawler`](../api/cheerio-crawler) is far from useless! It really shines when you need to cope with extremely high workloads. With just 4 GBs of memory and a single CPU+core, you can scrape 500 or more pages a minute! _(assuming each page contains approximately 400KB of HTML)_ To scrape this fast with a full browser+scraper, such as the [`PuppeteerCrawler`](../api/puppeteer-crawler), you'd need significantly more computing power.++**Advantages:**++- Extremely fast+- Easy to set up+- Familiar for jQuery users+- Super cheap to run+- Each request can go through a different proxy++**Disadvantages:**++- Does not work for all websites+- May easily overload the target website with requests+- Does not enable any manipulation of the website before scraping++### Basic use of `CheerioCrawler`++Now that we have an idea of the crawler's inner workings, let's build one. We'll use the example from the previous section and improve on it by+letting it truly crawl the page, finding new links as it goes, enqueuing them into the `RequestQueue` and then scraping them.++#### Refresher++Just to refresh your memory, in the previous section we built a very simple crawler that downloads the HTML of a single page, reads its title and prints+it to the console. This is the original source code:++```js+const Apify = require('apify');++Apify.main(async () => {+ const requestQueue = await Apify.openRequestQueue();+ await requestQueue.addRequest({ url: 'https://apify.com' });++ const handlePageFunction = async ({ request, $ }) => {+ const title = $('title').text();++ console.log(`The title of "${request.url}" is: ${title}.`);+ };++ // Set up the crawler, passing a single options object as an argument.+ const crawler = new Apify.CheerioCrawler({+ requestQueue,+ handlePageFunction,+ });++ await crawler.run();+});+```++Earlier we said that we would let the crawler:++1. Find new links on the page+2. Filter only those pointing to the same hostname, in this case `apify.com`+3. Enqueue them to the `RequestQueue`+4. Scrape the newly enqueued links++So let's get to it!++#### Finding new links++There are numerous approaches to finding links to follow when crawling the web. For our purposes, we will be looking for `<a>` elements that contain+the `href` attribute. For example `<a href="https://apify.com/store">This is a link to Apify Store</a>`. To do this, we need to update our Cheerio+function.++```js+const links = $('a[href]')+ .map((i, el) => $(el).attr('href'))+ .get();+```++Our new function finds all the `<a>` elements that contain the `href` attribute and extracts the attributes into an array of strings. But there's a problem. There could be relative links in the list and those can't be used on their own. We need to resolve them using our domain as base URL and+we will use one of Node.js' standard libraries to do this.++Apify exposes two URL properties: [`request.url`](../api/request#url) and [`request.loadedUrl`](../api/request#loadedurl). What's the difference? Well, `request.url` is the URL of the first request. In case of redirects, the URL changes. The final one is stored as `request.loadedUrl`.++```js+// At the top of the file:+const { URL } = require('url');++// ...++const { hostname } = new URL(request.loadedUrl);+const absoluteUrls = links.map(link => new URL(link, request.loadedUrl));+```++#### Filtering links to same domain++Websites typically contain a lot of links that lead away from the original page. This is normal, but when crawling a website, we usually want to crawl+that one site and not let our crawler wander away to Google, Facebook and Twitter. Therefore, we need to filter out the off-domain links and only+keep the ones that lead to the same domain.++> Don't worry, we'll learn how to do this with a single function call using Apify in a few moments.++```js+// At the top of the file:+const { URL } = require('url');++// ...++const links = $('a[href]')+ .map((i, el) => $(el).attr('href'))+ .get();++const { hostname } = new URL(request.loadedUrl);+const absoluteUrls = links.map(link => new URL(link, request.loadedUrl));++const sameDomainLinks = absoluteUrls.filter(url => url.hostname.endsWith(hostname));++// ...+```++This includes subdomains. In order to filter the same origin, simply compare the `url.origin` property instead:++```js+const { origin } = new URL(request.loadedUrl);+const absoluteUrls = links.map(link => new URL(link, request.loadedUrl));++const sameDomainLinks = absoluteUrls.filter(url => url.origin === origin);+```++> The `URL` class contains many other useful properties. You can read more about `url.origin` [here](https://developer.mozilla.org/en-US/docs/Web/API/URL/origin).++#### Enqueueing links to `RequestQueue`++This should be easy, because we already did that [earlier](#putting-it-all-together), remember? Just call `requestQueue.addRequest()` for all the new+links. This will add them to the end of the queue for processing.++```js+// At the top of the file:+const { URL } = require('url');++// ...++const links = $('a[href]')+ .map((i, el) => $(el).attr('href'))+ .get();++const { origin } = new URL(request.loadedUrl);+const absoluteUrls = links.map(link => new URL(link, request.loadedUrl));++const sameDomainLinks = absoluteUrls.filter(url => url.origin === origin);++// Add the requests in series. There's of course room for speed+// improvement by parallelization. Try to implement it, if you wish.+console.log(`Enqueueing ${sameDomainLinks.length} URLs.`);+for (const url of sameDomainLinks) {+ await requestQueue.addRequest({ url: url.href });+}++// ...+```++#### Scrape the newly enqueued links++And we're approching the finishing line. All we need to do now is integrate the new code into our original crawler. It will be easy, because+almost everything needs to go into the `handlePageFunction`. But just before we do that, let's introduce the first crawler configuration option that+is not a `handlePageFunction` or `requestQueue`. It's called `maxRequestsPerCrawl`.++##### The `maxRequestsPerCrawl` limit++This configuration option is available in all crawler classes and you can use it to limit the number of `Requests` the crawler should process. It's+very useful when you're just testing your code or when your crawler could potentially crawl millions of pages and you want to save resources. You can+add it to the crawler options like this:++```js+const crawler = new Apify.CheerioCrawler({+ maxRequestsPerCrawl: 20,+ requestQueue,+ handlePageFunction,+});+```++This limits the number of successfully handled `Requests` to 20. Bear in mind that the actual number of processed requests might be a little higher+and that's because usually there are multiple `Requests` processed at the same time and once the 20th `Request` finishes, the other running `Requests`+will be allowed to finish too.++#### Putting it all together++```js+const { URL } = require('url'); // <------ This is new.+const Apify = require('apify');++Apify.main(async () => {+ const requestQueue = await Apify.openRequestQueue();+ await requestQueue.addRequest({ url: 'https://apify.com' });++ const handlePageFunction = async ({ request, $ }) => {+ const title = $('title').text();+ console.log(`The title of "${request.url}" is: ${title}.`);++ // Here starts the new part of handlePageFunction.+ const links = $('a[href]')+ .map((i, el) => $(el).attr('href'))+ .get();++ const { origin } = new URL(request.loadedUrl);+ const absoluteUrls = links.map(link => new URL(link, request.loadedUrl));++ const sameDomainLinks = absoluteUrls.filter(url => url.origin === origin);++ console.log(`Enqueueing ${sameDomainLinks.length} URLs.`);+ for (const url of sameDomainLinks) {+ await requestQueue.addRequest({ url: url.href });+ }+ };++ const crawler = new Apify.CheerioCrawler({+ maxRequestsPerCrawl: 20, // <------ This is new too.+ requestQueue,+ handlePageFunction,+ });++ await crawler.run();+});+```++No matter if you followed along with our coding or just copy-pasted the resulting source, try running it now, perhaps even in both environments. You+should see the crawler log the **title** of the first page, then the **enqueueing** message showing number of URLs, followed by the **title** of the+first enqueued page and so on and so on.++> If you need help with running the code, refer back to the chapters on environment setup: [Setting up locally](#setting-up-locally) and+> [Setting up on the Apify platform](#setting-up-on-the-apify-platform).++## Using Apify SDK to enqueue links like a boss++If you were paying attention carefully in the previous chapter, we said that we would show you a way to enqueue new `Requests` with a single function+call. You might be wondering why we had to go through the whole process of getting the individual links, filtering the same domain ones and then+manually enqueuing them into the `RequestQueue`, when there is a simpler way.++Well, the obvious reason is practice. This is a tutorial after all. The other reason is to make you think about all the bits and pieces that come+together, so that in the end, a new page, not previously entered in by you, can be scraped. We think that by seeing the bigger picture, you will be+able to get the most out of Apify SDK.++### Meet `Apify.utils`++We will talk at length about them later, but in short, `Apify.utils` is a namespace where you can find various helpful functions and constants that+make your life easier. One of the available functions is `Apify.utils.enqueueLinks()` which encapsulates the whole enqueueing process and even adds+some extra functionality.++### Introduction to `Apify.utils.enqueueLinks()`++Since enqueuing new links to crawl is such an integral part of web crawling, we created a function that attempts to simplify this process as much as+possible. With a single function call, it allows you to find all the links on a page that match specified criteria and add them to a `RequestQueue`.+It also allows you to modify the resulting `Requests` to match your crawling needs.++`enqueueLinks` is quite a powerful function so, like crawlers, it gets its arguments from an options object. This is useful, because you don't have to+remember their order! But also because we can easily extend its API and add new features. You can+[find the full reference here](../api/utils#enqueuelinks).++We suggest using ES6 destructuring to grab the `enqueueLinks()` function off of the `utils` object, so you don't have to type `Apify.utils` all the+time.++```js+const Apify = require('apify');+const {+ utils: { enqueueLinks },+} = Apify;++// Now you can use enqueueLinks like this:+await enqueueLinks({+ /* options */+});+```++### Basic use of `enqueueLinks()` with `CheerioCrawler`++We already implemented logic that takes care of enqueueing new links to a `RequestQueue` in the previous chapter on `CheerioCrawler`. Let's look at+that logic and implement the same functionality using `enqueueLinks()`.++We found that the crawler needed to do these 4 things to crawl `apify.com`:++1. Find new links on the page+2. Filter only those pointing to `apify.com`+3. Enqueue them to the `RequestQueue`+4. Scrape the newly enqueued links++Using `enqueueLinks()` we can squash the first 3 into a single function call, if we set the options correctly. For now, let's just stick to the+basics. At the very least, we need a source where to find the links and the queue to enqueue them to. The `$` Cheerio object is one of the sources the+function accepts and we already know how to work with it in the `handlePageFunction`. We also know how to get a `requestQueue` instance.++```js+// Assuming previous existence of the '$' and 'requestQueue' variables.+await enqueueLinks({ $, requestQueue });+```++That's all we need to do to enqueue all `<a href="...">` links from the given page to the given queue. Easy, right? Scratch number 1 and 3 off the+list. Only number 2 remains and to tackle this one, we need to talk about yet another new concept, the pseudo-URL.++#### Introduction to pseudo-URLs++Pseudo-URLs are represented by our `PseudoUrl` class and even though the name sounds discouraging, they're a pretty simple concept. They're just URLs+with some parts replaced by wildcards (read <a href="https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions">regular+expressions</a>). They are matched against URLs to find specific links, domains, patterns, file extensions and so on.++In scraping, there are usually patterns to be found in website URLs that can be leveraged to scrape only the pages we're interested in. Imagine+a typical online store. It has different categories which list different items The URL for might looks something like this:++```+https://www.online-store.com/categories+```++A category would then have a different URL:++```+https://www.online-store.com/categories/computers+```++Going to this page would produce a list of offered computers. Then, clicking on one of the computers might take us to a detail URL:++```+https://www.online-store.com/items/613804+```++As you can see, there's a structure to the links. In the real world, the structure might not always be perfectly obvious, but it's very often there.+Pseudo-URLs help to use this structure to select only the relevant links from a given page.++#### Structure of a pseudo-URL++A pseudo-URL is a URL with [regular expressions](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions)) enclosed+in `[]` brackets. Since we're running Node.js, the regular expressions should follow the JavaScript style.++For example, a pseudo-URL++```+https://www.online-store.com/categories/[(\w|-)+]+```++will match all of the following URLs:++```+https://www.online-store.com/categories/computers+https://www.online-store.com/categories/mobile-phones+https://www.online-store.com/categories/black-friday+```++but it will not match++```+https://www.online-store.com/categories+https://www.online-store.com/items/613804+```++This way, you can easily find just the URLs that you're looking for while ignoring the rest.++A pseudo-URL may include any number of bracketed regular expressions, so you can compose much more complex matching logic. The following Pseudo URL+will match the items in the store even if the links use the non-secure `http` protocol, omit the `www` from the hostname or use a different TLD.++```+http[s?]://[(www)?\.]online-store.[com|net|org]/items/[\d+]+```++will match any combination of:++```+http://www.online-store.org/items/12345+https://online-store.com/items/633423+http://online-store.net/items/7003+```++but it will not match:++```+http://shop.online-store.org/items/12345+https://www.online-store.com/items/calculator+www.online-store.org/items/7003+```++> Pssst! Don't tell anyone, but you can create `PseudoUrls` with plain old `RegExp` instances instead of this brackety madness as well.++#### Using `enqueueLinks()` to filter links++That's been quite a lot of theory and examples. We might as well put it to practice. Going back to our `CheerioCrawler` exercise, we still have number+2 left to cross off the list - filter links pointing to `apify.com`. We've already shown that at the very least, the `enqueueLinks()` function needs+two arguments. The source, in our case the `$` object, and the destination - the `requestQueue`. To filter links, we need to add a third argument:+`pseudoUrls`.++The `options.pseudoUrls` argument is always an `Array`, but its contents can take on many forms. [See the reference](../api/utils#enqueueLinks)+for all of them. Since we just need to filter out same domain links, we'll keep it simple and use a pseudo-URL `string`.++```js+// Assuming previous existence of the '$' and 'requestQueue' variables.+const options = {+ $,+ requestQueue,+ pseudoUrls: ['http[s?]://apify.com[.*]'],+};++await enqueueLinks(options);+```++> To break the pseudo-URL string down, we're looking for both `http` and `https` protocols and the links may only lead to `apify.com` domain. The+> final brackets `[.*]` allow everything, so `apify.com/contact` as well as `apify.com/store` will match. If this sounds complex to you, we suggest+> [reading a tutorial](https://www.regular-expressions.info/tutorial.html) or two on regular expression syntax.++#### Resolving relative URLs with `enqueueLinks()`++**TLDR;** Just use `baseUrl: request.loadedUrl` when working with `CheerioCrawler`.++This is probably the weirdest and most complicated addition to the list. This is not the place to talk at length about+[absolute and relative paths](https://stackoverflow.com/questions/2005079/absolute-vs-relative-urls), but in short, the+links we encounter in a page can either be absolute, such as:++```+https://apify.com/john-doe/my-actor+```++or relative:++```+./john-doe/my-actor+```++Browsers handle this automatically, but since we're only using plain HTTP requests, we need to tell the `enqueueLinks()` function how to resolve the+relative links to the absolute ones, so we can use them for scraping. This is where the `request.loadedUrl` comes into play, because it returns the+correct URL to use as `baseUrl`.++```js+// Assuming previous existence of the '$', 'requestQueue' and 'request' variables.+const options = {+ $,+ requestQueue,+ pseudoUrls: ['http[s?]://apify.com[.*]'],+ baseUrl: request.loadedUrl,+};++await enqueueLinks(options);+```++> Even though it seems possible, we can't use the `request.url` of our `Request` instances, because the page could have been redirected and the final+> URL would be different from the one we requested.++#### Integrating `enqueueLinks()` into our crawler++That was fairly easy, wasn't it. That ticks number 2 off our list and we're done! Let's take a look at the original crawler code, where we+enqueued all the links manually.++```js+const { URL } = require('url'); // <------ This is new.+const Apify = require('apify');++Apify.main(async () => {+ const requestQueue = await Apify.openRequestQueue();+ await requestQueue.addRequest({ url: 'https://apify.com' });++ const handlePageFunction = async ({ request, $ }) => {+ const title = $('title').text();+ console.log(`The title of "${request.url}" is: ${title}.`);++ // Here starts the new part of handlePageFunction.+ const links = $('a[href]')+ .map((i, el) => $(el).attr('href'))+ .get();++ const ourDomain = 'https://apify.com';+ const absoluteUrls = links.map(link => new URL(link, ourDomain));++ const sameDomainLinks = absoluteUrls.filter(url => url.href.startsWith(ourDomain));++ console.log(`Enqueueing ${sameDomainLinks.length} URLs.`);+ for (const url of sameDomainLinks) {+ await requestQueue.addRequest({ url: url.href });+ }+ };++ const crawler = new Apify.CheerioCrawler({+ maxRequestsPerCrawl: 20, // <------ This is new too.+ requestQueue,+ handlePageFunction,+ });++ await crawler.run();+});+```++Since we've already prepared the `enqueueLinks()` options, we can just replace all the above enqueuing logic with a single function call, as promised.++```js+const Apify = require('apify');+const {+ utils: { enqueueLinks },+} = Apify;++Apify.main(async () => {+ const requestQueue = await Apify.openRequestQueue();+ await requestQueue.addRequest({ url: 'https://apify.com' });++ const handlePageFunction = async ({ request, $ }) => {+ const title = $('title').text();+ console.log(`The title of "${request.url}" is: ${title}.`);++ // Enqueue links+ const enqueued = await enqueueLinks({+ $,+ requestQueue,+ pseudoUrls: ['http[s?]://apify.com[.*]'],+ baseUrl: request.loadedUrl,+ });+ console.log(`Enqueued ${enqueued.length} URLs.`);+ };++ const crawler = new Apify.CheerioCrawler({+ maxRequestsPerCrawl: 20,+ requestQueue,+ handlePageFunction,+ });++ await crawler.run();+});+```++And that's it! No more parsing the links from HTML using Cheerio, filtering them and enqueueing them one by one It all gets done automatically!+`enqueueLinks()` is just one example of Apify SDK's powerful helper functions. They're all designed to make your life easier so you can focus on+getting your data, while leaving the mundane crawling management to your tools.++`Apify.utils.enqueueLinks()` has a lot more tricks up its sleeve. Make sure to check out the+[reference documentation](../api/utils#enqueueLinks) to see what else it can do for you. Namely the feature to prepopulate the `Request`+instances it creates with `userData` of your choice is extremely useful!++## Getting some real-world data++> Hey, guys, you know, it's cool that we can scrape the `<title>` elements of web pages, but that's not very useful. Can we finally scrape some real+> data and save it somewhere in a machine readable format? Because that's why you started reading this tutorial in the first place!++We hear you, young padawan! First, learn how to crawl, you must. Only then, walk through data, you can!++### Making a store crawler++Fortunately, we don't have to travel to a galaxy far far away to find a good candidate for learning how to scrape+structured data. The [Apify Store](https://apify.com/store) is a library of public actors that anyone can grab and use. You+can find ready-made solutions for crawling [Google Places](https://apify.com/drobnikj/crawler-google-places),+[Amazon](https://apify.com/vaclavrut/amazon-crawler),+[Google Search](https://apify.com/apify/google-search-scraper),+[Booking](https://apify.com/dtrungtin/booking-scraper),+[Instagram](https://apify.com/jaroslavhejlek/instagram-scraper),+[Tripadvisor](https://apify.com/maxcopell/tripadvisor) and many other websites. Feel free to check them out! It's also a great place to practice our Jedi scraping skills since it has categories, lists and details. That's almost like our imaginary+`online-store.com` from the previous chapter.++### The importance of having a plan++Sometimes scraping is really straightforward, but most of the times, it really pays to do a little bit of research first. How is the website+structured? Can I scrape it only with HTTP requests (read "with `CheerioCrawler`") or would I need a full browser solution? Are there any+anti-scraping protections in place? Do I need to parse the HTML or can I get the data otherwise, such as directly from the website's API. Jakub,+one of Apify's founders, wrote a+[great article about all the different techniques](https://blog.apify.com/web-scraping-in-2018-forget-html-use-xhrs-metadata-or-javascript-variables-8167f252439c)+and tips and tricks, so make sure to check that out!++For the purposes of this tutorial, let's just go ahead with HTTP requests and HTML parsing using `CheerioCrawler`. The number one reason being: We+already know how to use it and we want to build on that knowledge to learn specific crawling and scraping techniques.++#### Choosing the data we need++A good first step is always to figure out what it is we want to scrape and where to find it. For the time being, let's just agree that we want to+scrape all actors (see the `Show` dropdown) in all categories (which can be found on the left side of the page) and for each actor we want to get its++1. URL+2. Owner+3. Unique identifier (such as `apify/web-scraper`)+4. Title+5. Description+6. Last modification date+7. Number of runs++We can see that some of the information is available directly on the list page, but for details such as "Last modification date" or "Number of runs" we'll also need+to open the actor detail pages.++++#### Analyzing the target++Knowing that we will use plain HTTP requests, we immediately know that we won't be able to manipulate the website in any way. We will only be able to+go through the HTML it gives us and parse our data from there. This might sound like a huge limitation, but you might be surprised in how effective it+can be. Let's get to it!++#### The start URL(s)++This is where we start our crawl. It's convenient to start as close to our data as possible. For example, it wouldn't make much sense to start at+`apify.com` and look for a `store` link there, when we already know that everything we want to extract can be found at the `apify.com/store` page.++Once we look at the `apify.com/store` page more carefully, we see that the categories themselves produce URLs that we can use to access those+individual categories.++```+https://apify.com/store?category=ENTERTAINMENT+```++Should we write down all the category URLs down and use all of them as start URLs? It's definitely possible, but what if a new category appears on the+page later? We would not learn about it unless we manually visit the page and inspect it again. So scraping the category links off the store page+definitely makes sense. This way we always get an up to date list of categories.++But is it really that straightforward? By digging further into the store page's HTML we find that it does not actually contain the category links. The+menu on the left uses JavaScript to display the items from a given category and, as we've learned earlier, `CheerioCrawler` cannot execute JavaScript.++> We've deliberately chosen this scenario to show an example of the number one weakness of `CheerioCrawler`. We will overcome this difficulty in our+> `PuppeteerCrawler` tutorial, but at the cost of compute resources and speed. Always remember that no tool is best for everything!++So we're back to the pre-selected list of URLs. Since we cannot scrape the list dynamically, we have to manually collect the links and then use them+in our crawler. We lose the ability to scrape new categories, but we keep the low resource consumption and speed advantages of `CheerioCrawler`.++Therefore, after careful consideration, we've determined that we should use multiple start URLs and that they should look as follows:++```+https://apify.com/store?category=TRAVEL+https://apify.com/store?category=ECOMMERCE+https://apify.com/store?category=ENTERTAINMENT+```++### The crawling strategy++Now that we know where to start, we need to figure out where to go next. Since we've eliminated one level of crawling by selecting the categories+manually, we now only need to crawl the actor detail pages. The algorithm therefore follows:++1. Visit the category list page (one of our start URLs).+2. Enqueue all links to actor details.+3. Visit all actor details and extract data.+4. Repeat 1 - 3 for all categories.++> Technically, this is a depth first crawl and the crawler will perform a breadth first crawl by default, but that's an implementation detail. We've+> chosen this notation since a breadth first crawl would be less readable.++`CheerioCrawler` will make sure to visit the pages for us, if we provide the correct `Requests` and we already know how to enqueue pages, so this+should be fairly easy. Nevertheless, there are two more tricks that we'd like to show you.++#### Using a `RequestList`++`RequestList` is a perfect tool for scraping a pre-existing list of URLs and if you think about our start URLs, this is exactly what we have! A list+of links to the different categories of the store. Let's see how we'd get them into a `RequestList`.++```js+const sources = [+ 'https://apify.com/store?category=TRAVEL',+ 'https://apify.com/store?category=ECOMMERCE',+ 'https://apify.com/store?category=ENTERTAINMENT',+];++const requestList = await Apify.openRequestList('categories', sources);+```++As you can see, similarly to the `Apify.openRequestQueue()` function, there is an `Apify.openRequestList()` function that will create a `RequestList`+instance for you. The first argument is the name of the `RequestList`. It is used to persist the crawling state of the list. This is useful when you+want to continue where you left off after an error or a process restart. The second argument is the `sources` array, which is nothing more than a list+of URLs you wish to crawl.++> `RequestQueue` is a persistent store by default, so no name is needed, while the `RequestList` only lives in memory and giving it a name enables it+> to become persistent.++You might now want to ask one of these questions:++- Can I enqueue into `RequestList` too?+- How do I make `RequestList` work together with `RequestQueue` since I need the queue to enqueue new `Requests`.++The answer to the first one is a definitive no. `RequestList` is immutable and once you create it, you cannot add or remove `Requests` from it. The+answer to the second one is simple. `RequestList` and `RequestQueue` are made to work together out of the box in crawlers, so all you need to do is+use them both and the crawlers will do the rest.++```js+const crawler = new Apify.CheerioCrawler({+ requestList,+ requestQueue,+ handlePageFunction,+});+```++> For those wondering how this works, the `RequestList` `Requests` are enqueued into the `RequestQueue` right before their execution and only+> processed by the `RequestQueue` afterwards. You can, of course, enqueue the `Requests` to the queue manually, but that would take some boilerplate+> code and perhaps quite a long time, if we were talking about tens of thousands or more `Requests`. The crawlers do it while running, so the time to+> enqueue is spread out and you won't even notice it.++#### Sanity check++It's always useful to create some simple boilerplate code to see that we've got everything set up correctly before we start to write+the scraping logic itself. We might realize that something in our previous analysis doesn't quite add up, or the website might not behave exactly as we expected.++Let's use our newly acquired `RequestList` knowledge and everything we know from the previous chapters to create a new crawler that'll just visit all+the category URLs we selected and print the text content of all the actors in the category. Try running the code below in your selected environment.+You should see, albeit very badly formatted, the text of the individual actor cards that are displayed in the selected categories.++```js+const Apify = require('apify');++Apify.main(async () => {+ const sources = [+ 'https://apify.com/store?category=TRAVEL',+ 'https://apify.com/store?category=ECOMMERCE',+ 'https://apify.com/store?category=ENTERTAINMENT',+ ];++ const requestList = await Apify.openRequestList('categories', sources);++ const crawler = new Apify.CheerioCrawler({+ requestList,+ handlePageFunction: async ({ $, request }) => {+ // Select all the actor cards.+ $('.item').each((i, el) => {+ const text = $(el).text();+ console.log(`ITEM: ${text}\n`);+ });+ },+ });++ await crawler.run();+});+```++> If there's anything you don't understand, refer to the previous chapters on setting up your environment, building your first crawler and+> `CheerioCrawler`.++You might be wondering how we got that `.item` selector. After analyzing the category pages using a browser's DevTools, we've determined that it's a+good selector to select all the currently displayed actor cards. DevTools and CSS selectors are quite a large topic, so we can't go into too much+detail now, but here are a few general pointers.++#### DevTools crash course++> We'll use Chrome DevTools here, since it's the most common browser, but feel free to use any other, it's all very similar.++We could pick any category, but let's just go with Travel because it includes some interesting actors. Go to++```+https://apify.com/store?category=TRAVEL+```++and open DevTools either by right-clicking anywhere in the page and selecting `Inspect`, or by pressing `F12` or by any other means relevant to your+system. Once you're there, you'll see a bunch of DevToolsy stuff and a view of the category page with the individual actor cards.++Now, find the `Select an element` tool and use it to select one of the actor cards. Make sure to select the whole card, not just some of its contents, such+as its title or description.++In the resulting HTML display, it will put your cursor somewhere. Inspect the HTML around it. You'll see that there are CSS classes attached to the+different HTML elements.++By hovering over the individual elements, you will see their placement in the page's view. It's easy to see the page's structure around the actor+cards now. All the cards are displayed in a `<div>` with a classname that starts with `ItemsGrid__StyledDiv`, which holds another `<div>` with some+computer-generated class names and finally, inside this `<div>`, the individual cards are represented by other `<div>` elements with the class of+`item`.++> Yes, there are other HTML elements and other classes too. We can safely ignore them.++It should now make sense how we got that `.item` selector. It's just a selector that finds all elements that are annotated with the `item` class and+those just happen to be the actor cards only.++It's always a good idea to double check that though, so go into the DevTools Console and run++```js+document.querySelectorAll('.item');+```++You will see that only the actor cards will be returned, and nothing else.++#### Enqueueing the detail links using a custom selector++In the previous chapter, we used the `Apify.utils.enqueueLinks()` function like this:++```js+await enqueueLinks({+ $,+ requestQueue,+ pseudoUrls: ['http[s?]://apify.com[.*]'],+ baseUrl: request.loadedUrl,+});+```++While very useful in that scenario, we need something different now. Instead of finding all the `<a href="..">` links that match the `pseudoUrl`, we+need to find only the specific ones that will take us to the actor detail pages. Otherwise, we'd be visiting a lot of other pages that we're not+interested in. Using the power of DevTools and yet another `enqueueLinks()` parameter, this becomes fairly easy.++```js+const handlePageFunction = async ({ $, request }) => {+ console.log(`Processing ${request.url}`);++ // Only enqueue new links from the category pages.+ if (!request.userData.detailPage) {+ await Apify.utils.enqueueLinks({+ $,+ requestQueue,+ selector: 'div.item > a',+ baseUrl: request.loadedUrl,+ transformRequestFunction: req => {+ req.userData.detailPage = true;+ return req;+ },+ });+ }+};+```++The code should look pretty familiar to you. It's a very simple `handlePageFunction` where we log the currently processed URL to the console and+enqueue more links. But there are also a few new, interesting additions. Let's break it down.++##### The `selector` parameter of `enqueueLinks()`++When we previously used `enqueueLinks()`, we were not providing any `selector` parameter and it was fine, because we wanted to use the default+setting, which is `a` - finds all `<a>` elements. But now, we need to be more specific. There are multiple `<a>` links on the given category page, but+we're only interested in those that will take us to item (actor) details. Using the DevTools, we found out that we can select the links we wanted+using the `div.item > a` selector, which selects all the `<a>` elements that have a `<div class="item ...">` parent. And those are exactly the ones+we're interested in.++##### The missing `pseudoUrls`++Earlier we learned that `pseudoUrls` are not required and if omitted, all links matching the given `selector` will be enqueued. This is exactly+what we need, so we're skipping `pseudoUrls` this time. That does not mean that you can't use `pseudoUrls` together with a custom `selector` though,+because you absolutely can!++##### Finally, the `userData` of `enqueueLinks()`++You will see `userData` used often throughout Apify SDK and it's nothing more than a place to store your own data on a `Request` instance. You can+access it with `request.userData` and it's a plain `Object` that can be used to store anything that needs to survive the full life-cycle of the+`Request`.++We can use the `transformRequestFunction` option of `enqueueLinks()` to modify all the `Request` instances it creates and enqueues. In our case, we+use it to set a `detailPage` property to the enqueued `Requests` to be able to easily differentiate between the category pages and the detail pages.++#### Another sanity check++It's always good to work step by step. We have this new enqueueing logic in place and since the previous [Sanity check](#sanity-check) worked only+with a `RequestList`, because we were not enqueueing anything, don't forget to add back the `RequestQueue` and `maxRequestsPerCrawl` limit. Let's+test it out!++```js+const Apify = require('apify');++Apify.main(async () => {+ const sources = [+ 'https://apify.com/store?category=TRAVEL',+ 'https://apify.com/store?category=ECOMMERCE',+ 'https://apify.com/store?category=ENTERTAINMENT',+ ];++ const requestList = await Apify.openRequestList('categories', sources);+ const requestQueue = await Apify.openRequestQueue(); // <----------------++ const crawler = new Apify.CheerioCrawler({+ maxRequestsPerCrawl: 50, // <----------------------------------------+ requestList,+ requestQueue, // <---------------------------------------------------+ handlePageFunction: async ({ $, request }) => {+ console.log(`Processing ${request.url}`);++ // Only enqueue new links from the category pages.+ if (!request.userData.detailPage) {+ await Apify.utils.enqueueLinks({+ $,+ requestQueue,+ selector: 'div.item > a',+ baseUrl: request.loadedUrl,+ transformRequestFunction: req => {+ req.userData.detailPage = true;+ return req;+ },+ });+ }+ },+ });++ await crawler.run();+});+```++We've added the `handlePageFunction()` with the `enqueueLinks()` logic from the previous section to the code we wrote earlier. As always, try+running it in the environment of your choice. You should see the crawler output a number of links to the console, as it crawls the category pages first+and then all the links to the actor detail pages it found.++This concludes our Crawling strategy section, because we have taught the crawler to visit all the pages we need. Let's continue with scraping the+tasty data.++### Scraping data++At the beginning of this chapter, we created a list of the information we wanted to collect about the actors in the store. Let's review that and figure+out ways to access it.++1. URL+2. Owner+3. Unique identifier (such as `apify/web-scraper`)+4. Title+5. Description+6. Last modification date+7. Number of runs++++#### Scraping the URL, Owner and Unique identifier++Some information is lying right there in front of us without even having to touch the actor detail pages. The `URL` we already have - the+`request.url`. And by looking at it carefully, we realize that it already includes the `owner` and the `unique identifier` too. We can just split the+`string` and be on our way then!++```js+// request.url = https://apify.com/apify/web-scraper++const urlArr = request.url.split('/').slice(-2); // ['apify', 'web-scraper']+const uniqueIdentifier = urlArr.join('/'); // 'apify/web-scraper'+const owner = urlArr[0]; // 'apify'+```++> It's always a matter of preference, whether to store this information separately in the resulting dataset, or not. Whoever uses the dataset can+> easily parse the `owner` from the `URL`, so should we duplicate the data unnecessarily? Our opinion is that unless the increased data consumption+> would be too large to bear, it's always better to make the dataset as readable as possible. Someone might want to filter by `owner` for example and+> keeping only the `URL` in the dataset would make this complicated without using additional tools.++#### Scraping Title, Description, Last modification date and Number of runs++Now it's time to add more data to the results. Let's open one of the actor detail pages in the Store, for example the+[`apify/web-scraper`](https://apify.com/apify/web-scraper) page and use our DevTools-Fu to figure out how to get the title of the actor.++##### Title++++By using the element selector tool, we find out that the title is there under an `<h1>` tag, as titles should be.+Maybe surprisingly, we find that there are actually two `<h1>` tags on the detail page. This should get us thinking.+Is there any parent element that includes our `<h1>` tag, but not the other ones? Yes, there is! There is a `<header>`+element that we can use to select only the heading we're interested in.++> Remember that you can press CTRL+F (CMD+F) in the Elements tab of DevTools to open the search bar where you can quickly search for elements using+> their selectors. And always make sure to use the DevTools to verify your scraping process and assumptions. It's faster than changing the crawler+> code all the time.++To get the title we just need to find it using `Cheerio` and a `header h1` selector, which selects all `<h1>` elements that have a `<header>` ancestor.+And as we already know, there's only one.++```js+return {+ title: $('header h1').text(),+};+```++##### Description++Getting the actor's description is a little more involved, but still pretty straightforward. We can't just simply search for a `<p>` tag, because+there's a lot of them in the page. We need to narrow our search down a little. Using the DevTools we find that the actor description is nested within+the `<header>` element too, same as the title. Moreover, the actual description is nested inside a `<span>` tag with a class `actor-description`.++++```js+return {+ title: $('header h1').text(),+ description: $('header span.actor-description').text(),+};+```++##### Last modification date++The DevTools tell us that the `modifiedDate` can be found in the `<time>` element inside `<ul class="ActorHeader-stats">`.++++```js+return {+ title: $('header h1').text(),+ description: $('header span.actor-description').text(),+ modifiedDate: new Date(+ Number(+ $('ul.ActorHeader-stats time').attr('datetime'),+ ),+ ),+};+```++It might look a little too complex at first glance, but let's walk through it. We find the right `<time>` element,+and then we read its `datetime` attribute, because that's where a unix timestamp is stored as a `string`.++But we would much rather see a readable date in our results, not a unix timestamp, so we need to convert it. Unfortunately the `new Date()`+constructor will not accept a `string`, so we cast the `string` to a `number` using the `Number()` function before actually calling `new Date()`.+Phew!++##### Run count++And so we're finishing up with the `runCount`. There's no specific element like `<time>`, so we need to create a complex selector and then do a+transformation on the result.++```js+return {+ title: $('header h1').text(),+ description: $('header span.actor-description').text(),+ modifiedDate: new Date(+ Number(+ $('ul.ActorHeader-stats time').attr('datetime'),+ ),+ ),+ runCount: Number(+ $('ul.ActorHeader-stats > li:nth-of-type(3)')+ .text()+ .match(/[\d,]+/)[0]+ .replace(',', ''),+ ),+};+```++The `ul.ActorHeader-stats > li:nth-of-type(3)` looks complicated, but it only reads that we're looking for a `<ul class="ActorHeader-stats ...">` element and within that+element we're looking for the third `<li>` element. We grab its text, but we're only interested in the number of runs. So we parse the number out+using a regular expression, but its type is still a `string`, so we finally convert the result to a `number` by wrapping it with a `Number()` call.++> The numbers are formatted with commas as thousands separators (e.g. `'1,234,567'`), so to extract it, we+> first use regular expression `/[\d,]+/` - it will search for consecutive number or comma characters.+> Then we extract the match via `.match(/[\d,]+/)[0]` and finally remove the commas by calling `.replace(',', '')`.+> This will give us a string (e.g. `'1234567'`) that can be converted via `Number` function.++And there we have it! All the data we needed in a single object. For the sake of completeness, let's add the properties we parsed from the URL earlier+and we're good to go.++```js+const urlArr = request.url.split('/').slice(-2);++const results = {+ url: request.url,+ uniqueIdentifier: urlArr.join('/'),+ owner: urlArr[0],+ title: $('header h1').text(),+ description: $('header span.actor-description').text(),+ modifiedDate: new Date(+ Number(+ $('ul.ActorHeader-stats time').attr('datetime'),+ ),+ ),+ runCount: Number(+ $('ul.ActorHeader-stats > li:nth-of-type(3)')+ .text()+ .match(/[\d,]+/)[0]+ .replace(',', ''),+ ),+};++console.log('RESULTS: ', results);+```++#### Trying it out (sanity check #3)++We have everything we need so just grab our newly created scraping logic, dump it into our original `handlePageFunction()` and see the magic happen!++```js+const Apify = require('apify');++Apify.main(async () => {+ const sources = [+ 'https://apify.com/store?category=TRAVEL',+ 'https://apify.com/store?category=ECOMMERCE',+ 'https://apify.com/store?category=ENTERTAINMENT',+ ];++ const requestList = await Apify.openRequestList('categories', sources);+ const requestQueue = await Apify.openRequestQueue();++ const crawler = new Apify.CheerioCrawler({+ maxRequestsPerCrawl: 50,+ requestList,+ requestQueue,+ handlePageFunction: async ({ $, request }) => {+ console.log(`Processing ${request.url}`);++ // This is our new scraping logic.+ if (request.userData.detailPage) {+ const urlArr = request.url.split('/').slice(-2);++ const results = {+ url: request.url,+ uniqueIdentifier: urlArr.join('/'),+ owner: urlArr[0],+ title: $('header h1').text(),+ description: $('header span.actor-description').text(),+ modifiedDate: new Date(+ Number(+ $('ul.ActorHeader-stats time').attr('datetime'),+ ),+ ),+ runCount: Number(+ $('ul.ActorHeader-stats > li:nth-of-type(3)')+ .text()+ .match(/[\d,]+/)[0]+ .replace(',', ''),+ ),+ };+ console.log('RESULTS', results);+ }++ // Only enqueue new links from the category pages.+ if (!request.userData.detailPage) {+ await Apify.utils.enqueueLinks({+ $,+ requestQueue,+ selector: 'div.item > a',+ baseUrl: request.loadedUrl,+ transformRequestFunction: req => {+ req.userData.detailPage = true;+ return req;+ },+ });+ }+ },+ });++ await crawler.run();+});+```++> Notice again that we're scraping on the detail pages `request.userData.detailPage === true`, but we're only enqueueing on the category pages+> `request.userData.detailPage === undefined`.++When running the actor in the environment of your choice, you should see the crawled URLs and their scraped data printed to the console.++### Saving the scraped data++A data extraction job would not be complete without saving the data for later use and processing. We've come to the final and most difficult part of+this chapter so make sure to pay attention very carefully!++First, replace the `console.log('RESULTS', results)` call with++```js+await Apify.pushData(results);+```++and that's it. Unlike in the previous paragraph, I'm being serious now. That's it, we're done. The final code therefore looks exactly like this:++```js+const Apify = require('apify');++Apify.main(async () => {+ const sources = [+ 'https://apify.com/store?category=TRAVEL',+ 'https://apify.com/store?category=ECOMMERCE',+ 'https://apify.com/store?category=ENTERTAINMENT',+ ];++ const requestList = await Apify.openRequestList('categories', sources);+ const requestQueue = await Apify.openRequestQueue();++ const crawler = new Apify.CheerioCrawler({+ maxRequestsPerCrawl: 50,+ requestList,+ requestQueue,+ handlePageFunction: async ({ $, request }) => {+ console.log(`Processing ${request.url}`);++ // This is our new scraping logic.+ if (request.userData.detailPage) {+ const urlArr = request.url.split('/').slice(-2);++ const results = {+ url: request.url,+ uniqueIdentifier: urlArr.join('/'),+ owner: urlArr[0],+ title: $('header h1').text(),+ description: $('header span.actor-description').text(),+ modifiedDate: new Date(+ Number(+ $('ul.ActorHeader-stats time').attr('datetime'),+ ),+ ),+ runCount: Number(+ $('ul.ActorHeader-stats > li:nth-of-type(3)')+ .text()+ .match(/[\d,]+/)[0]+ .replace(',', ''),+ ),+ };+ await Apify.pushData(results);+ }++ // Only enqueue new links from the category pages.+ if (!request.userData.detailPage) {+ await Apify.utils.enqueueLinks({+ $,+ requestQueue,+ selector: 'div.item > a',+ baseUrl: request.loadedUrl,+ transformRequestFunction: req => {+ req.userData.detailPage = true;+ return req;+ },+ });+ }+ },+ });++ await crawler.run();+});+```++#### What's `Apify.pushData()`++[`Apify.pushData()`](../api/apify#pushdata) is a helper function that saves data to the default [`Dataset`](../api/dataset). `Dataset` is a+storage designed to hold virtually unlimited amount of data in a format similar to a table. Each time you call `Apify.pushData()` a new row in the+table is created, with the property names serving as column titles.++> Each actor run has one default `Dataset` so no need to initialize it or create an instance first. It just gets done automatically for you. You can+> also create named datasets at will.++#### Finding my saved data++It might not be perfectly obvious where the data we saved using the previous command went, so let's break it down by environment:++##### Dataset on the Apify platform++Open any Run of your actor on the Platform and you will see a Dataset as one of the available tabs. Clicking on it will reveal basic information about+the Dataset and a list of options that you can use to download your data. There are various formats such as JSON, XLSX or CSV available and there's+also the possibility of downloading only clean items, i.e. a filtered dataset with empty rows and hidden fields removed.++##### Local Dataset++Unless you changed the environment variables that Apify SDK uses locally, which would suggest that you knew what you were doing and you didn't need+this tutorial anyway, you'll find your data in your local Apify Storage.++```+{PROJECT_FOLDER}/apify_storage/datasets/default/+```++The above folder will hold all your saved data in numbered files, as they were pushed into the dataset. Each file represents one invocation of+`Apify.pushData()` or one table row.++> Unfortunately, the local datasets don't yet support the export in various formats functionality that the Platform Dataset page offers, so for the+> time being, we're stuck with JSON.++### Final touch++It may seem that the data are extracted and the actor is done, but honestly, this is just the beginning. For the sake of brevity, we've completely+omitted error handling, proxies, debug logging, tests, documentation and other stuff that a reliable software should have. The good thing is, **error+handling is mostly done by Apify SDK itself**, so no worries on that front, unless you need some custom magic.++Anyway, to spark some ideas, let's look at two more things. First, passing an input to the actor, which will enable us to change the categories we+want to scrape without changing the source code itself! And then some refactoring, to show you how we reckon is preferable to structure and annotate+actor code.++#### Meet the `INPUT`++`INPUT` is just a convention on how we call the actor's input. Because there's no magic in actors, just features, the `INPUT` is actually nothing more+than a key in the default [`KeyValueStore`](../api/key-value-store) that's, by convention, used as input on the Apify platform. Also by convention, the+`INPUT` is mostly expected to be of `Content-Type: application/json`.++We will not go into `KeyValueStore` details here, but for the sake of `INPUT` you need to remember that there is a function that helps you get it.++```js+const input = await Apify.getInput();+```++On the Apify Platform, the actor's input that you can set in the Console is automatically saved to the default `KeyValueStore` under the key `INPUT`+and by calling [`Apify.getInput()`](../api/apify#getvalue) you retrieve the value from the `KeyValueStore`.++Running locally, you need to place an `INPUT.json` file in your default key value store for this to work.++```+{PROJECT_FOLDER}/apify_storage/key_value_stores/default/INPUT.json+```++#### Use `INPUT` to seed our actor with categories++Currently we're using the full URLs of categories as sources, but it's quite obvious that we only need the final parameters, the rest of the URL is+always the same. Knowing that, we can pass an array of those parameters on `INPUT` and build the URLs dynamically, which would allow us to scrape+different categories without changing the source code. Let's get to it!++First, we set up our `INPUT`, either in the `INPUT` form of the actor on the Apify platform, or by creating an `INPUT.json` in our default key-value store+locally.++```json+["TRAVEL", "ECOMMERCE", "ENTERTAINMENT"]+```++Once we have that, we can load it in the actor and populate the crawler's sources with it. In the following example, we're using the categories in the+input to construct the category URLs and we're also passing custom `userData` to the sources. This means that the `Requests` that get created will+automatically contain this `userData`.++```js+// ...+const input = await Apify.getInput();++const sources = input.map(category => ({+ url: `https://apify.com/store?category=${category}`,+ userData: {+ label: 'CATEGORY',+ },+}));++const requestList = await Apify.openRequestList('categories', sources);+// ...+```++The `userData.label` is also a convention that we've been using for quite some time to label different `Requests`. We know that this is a category URL+so we `label` it `CATEGORY`. This way, we can easily make decisions in the `handlePageFunction` without having to inspect the URL itself.++We can then refactor the `if` clauses in the `handlePageFunction` to use the `label` for decision-making. This does not make much sense for a crawler+with only two different pages, because a simple `boolean` would suffice, but for pages with multiple different views, it becomes very useful.++#### Structuring the code better++But perhaps we should not stop at refactoring the `if` clauses. There are several ways we can make the actor look more elegant and - most+importantly - easier to reason about and make changes to.++In the following code we've made several changes.++- Split the code into multiple files.+- Added the `Apify.utils.log` and replaced `console.log` with it.+- Added a `getSources()` function to encapsulate `INPUT` consumption.+- Added a `createRouter()` function to make our routing cleaner, without nested `if` clauses.+- Removed the `maxRequestsPerCrawl` limit.++> To create a multi-file actor on the Apify Platform, select **Multiple source files** in the **Type** dropdown on the **Source** screen.++In our `main.js` file, we place the general structure of the crawler:++```js+// main.js+const Apify = require('apify');+const tools = require('./tools');+const {+ utils: { log },+} = Apify;++Apify.main(async () => {+ log.info('Starting actor.');+ const requestList = await Apify.openRequestList('categories', await tools.getSources());+ const requestQueue = await Apify.openRequestQueue();+ const router = tools.createRouter({ requestQueue });++ log.debug('Setting up crawler.');+ const crawler = new Apify.CheerioCrawler({+ requestList,+ requestQueue,+ handlePageFunction: async context => {+ const { request } = context;+ log.info(`Processing ${request.url}`);+ await router(request.userData.label, context);+ },+ });++ log.info('Starting the crawl.');+ await crawler.run();+ log.info('Actor finished.');+});+```++Then in a separate `tools.js`, we add our helper functions:++```js+// tools.js+const Apify = require('apify');+const routes = require('./routes');+const {+ utils: { log },+} = Apify;++exports.getSources = async () => {+ log.debug('Getting sources.');+ const input = await Apify.getInput();+ return input.map(category => ({+ url: `https://apify.com/store?category=${category}`,+ userData: {+ label: 'CATEGORY',+ },+ }));+};++exports.createRouter = globalContext => {+ return async function(routeName, requestContext) {+ const route = routes[routeName];+ if (!route) throw new Error(`No route for name: ${routeName}`);+ log.debug(`Invoking route: ${routeName}`);+ return route(requestContext, globalContext);+ };+};+```++And finally our routes in a separate `routes.js` file:++```js+// routes.js+const Apify = require('apify');+const {+ utils: { log },+} = Apify;++exports.CATEGORY = async ({ $, request }, { requestQueue }) => {+ return Apify.utils.enqueueLinks({+ $,+ requestQueue,+ selector: 'div.item > a',+ baseUrl: request.loadedUrl,+ transformRequestFunction: req => {+ req.userData.label = 'DETAIL';+ return req;+ },+ });+};++exports.DETAIL = async ({ $, request }) => {+ const urlArr = request.url.split('/').slice(-2);++ log.debug('Scraping results.');+ const results = {+ url: request.url,+ uniqueIdentifier: urlArr.join('/'),+ owner: urlArr[0],+ title: $('header h1').text(),+ description: $('header span.actor-description').text(),+ modifiedDate: new Date(+ Number(+ $('ul.ActorHeader-stats time').attr('datetime'),+ ),+ ),+ runCount: Number(+ $('ul.ActorHeader-stats > li:nth-of-type(3)')+ .text()+ .match(/[\d,]+/)[0]+ .replace(',', ''),+ ),+ };++ log.debug('Pushing data to dataset.');+ await Apify.pushData(results);+};+```++Let us tell you a little bit more about the changes. We hope that in the end, you'll agree that this structure makes the actor more readable and+manageable.++#### Splitting your code into multiple files++This was not always the case, but now that the Apify platform has a multifile editor, there's no reason not to split your code into multiple files and keep+your logic separate. Less code in a single file means less code you need to think about at any time, and that's a great thing!++#### Using `Apify.utils.log` instead of `console.log`++We wont go to great lengths here to talk about `utils.log`, because you can read [it all in the documentation](../api/log), but there's just+one thing that we need to stress: **log levels**.++`utils.log` enables you to use different log levels, such as `log.debug`, `log.info` or `log.warning`. It not only makes your log more readable, but+it also allows selective turning off of some levels by either calling the `utils.log.setLevel()` function or by setting an `APIFY_LOG_LEVEL` variable.+This is huge! Because you can now add a lot of debug logs in your actor, which will help you when something goes wrong and turn them on or off with a+simple `INPUT` change, or by setting an environment variable.++The punch line? Use `Apify.utils.log` instead of `console.log` now and thank us later when something goes wrong!++#### Using a router to structure your crawling++At first, it might seem more readable using just a simple `if / else` statement to select different logic based on the crawled pages, but trust me, it+becomes far less impressive when working with more than two different pages and it definitely starts to fall apart when the logic to handle each page+spans tens or hundreds of lines of code.++It's good practice in any programming to split your logic into bite-sized chunks that are easy to read and reason about. Scrolling through a+thousand line long `handlePageFunction()` where everything interacts with everything and variables can be used everywhere is not a beautiful thing to+do and a pain to debug. That's why we prefer the separation of routes into a special file and with large routes, we would even suggest having one file+per route.++> TO BE CONTINUED with details on `PuppeteerCrawler` and other features...diff --git a/website/versioned_docs/version-2.3.0/typedefs/CheerioCrawlerOptions.md b/website/versioned_docs/version-2.3.0/typedefs/CheerioCrawlerOptions.mdnew file mode 100644index 000000000..5c570f553--- /dev/null+++ b/website/versioned_docs/version-2.3.0/typedefs/CheerioCrawlerOptions.md@@ -0,0 +1,365 @@+---+id: version-2.3.0-cheerio-crawler-options+title: CheerioCrawlerOptions+original_id: cheerio-crawler-options+---++<a name="cheeriocrawleroptions"></a>++## Properties++### `handlePageFunction`++**Type**: [`CheerioHandlePage`](../typedefs/cheerio-handle-page)++User-provided function that performs the logic of the crawler. It is called for each page loaded and parsed by the crawler.++The function receives the following object as an argument:++```+{+ // The Cheerio object's function with the parsed HTML.+ $: Cheerio,++ // The request body of the web page, whose type depends on the content type.+ body: String|Buffer,++ // The parsed object from JSON for responses with the "application/json" content types.+ // For other content types it's null.+ json: Object,++ // Apify.Request object with details of the requested web page+ request: Request,++ // Parsed Content-Type HTTP header: { type, encoding }+ contentType: Object,++ // An instance of Node's http.IncomingMessage object,+ response: Object,++ // Session object, useful to work around anti-scraping protections+ session: Session++ // ProxyInfo object with information about currently used proxy+ proxyInfo: ProxyInfo++ // The running cheerio crawler instance.+ crawler: CheerioCrawler+}+```++Type of `body` depends on the `Content-Type` header of the web page:++- String for `text/html`, `application/xhtml+xml`, `application/xml` MIME content types+- Buffer for others MIME content types++Parsed `Content-Type` header using [content-type package](https://www.npmjs.com/package/content-type) is stored in `contentType`.++Cheerio is available only for HTML and XML content types.++With the [`Request`](../api/request) object representing the URL to crawl.++If the function returns, the returned promise is awaited by the crawler.++If the function throws an exception, the crawler will try to re-crawl the request later, up to `option.maxRequestRetries` times. If all the retries+fail, the crawler calls the function provided to the `handleFailedRequestFunction` parameter. To make this work, you should **always** let your+function throw exceptions rather than catch them. The exceptions are logged to the request using the+[`Request.pushErrorMessage()`](../api/request#pusherrormessage) function.++---++### `requestList`++**Type**: [`RequestList`](../api/request-list)++Static list of URLs to be processed. Either `requestList` or `requestQueue` option must be provided (or both).++---++### `requestQueue`++**Type**: [`RequestQueue`](../api/request-queue)++Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites. Either `requestList` or `requestQueue` option must be+provided (or both).++---++### `prepareRequestFunction`++**Type**: [`PrepareRequest`](../typedefs/prepare-request)++> This option is deprecated, use `preNavigationHooks` instead.++A function that executes before the HTTP request is made to the target resource. This function is suitable for setting dynamic properties such as+cookies to the [`Request`](../api/request).++The function receives the following object as an argument:++```+{+ request: Request,+ session: Session,+ proxyInfo: ProxyInfo,+ crawler: CheerioCrawler,+}+```++where the [`Request`](../api/request) instance corresponds to the initialized request and the [`Session`](../api/session) instance corresponds to used+session.++The function should modify the properties of the passed [`Request`](../api/request) instance in place because there are already earlier references to+it. Making a copy and returning it from this function is therefore not supported, because it would create inconsistencies where different parts of SDK+would have access to a different [`Request`](../api/request) instance.++---++### `postResponseFunction`++**Type**: [`PostResponse`](../typedefs/post-response)++> This option is deprecated, use `postNavigationHooks` instead.++A function that executes right after the HTTP request is made to the target resource and response is returned. This function is suitable for+overriding custom properties of response e.g. setting headers because of response parsing.++**Example usage:**++```javascript+const cheerioCrawlerOptions = {+ // ...+ postResponseFunction: ({ request, response }) => {+ if (request.userData.parseAsJSON) {+ response.headers['content-type'] = 'application/json; charset=utf-8';+ }+ },+};+```++The function receives the following object as an argument:++```+{+ response: Object,+ request: Request,+ session: Session,+ proxyInfo: ProxyInfo,+ crawler: CheerioCrawler,+}+```++The response is an instance of Node's http.IncomingMessage object.++---++### `handlePageTimeoutSecs`++**Type**: `number` <code> = 60</code>++Timeout in which the function passed as `handlePageFunction` needs to finish, given in seconds.++---++### `requestTimeoutSecs`++**Type**: `number` <code> = 30</code>++Timeout in which the HTTP request to the resource needs to finish, given in seconds.++---++### `ignoreSslErrors`++**Type**: `boolean` <code> = true</code>++If set to true, SSL certificate errors will be ignored.++---++### `proxyConfiguration`++**Type**: [`ProxyConfiguration`](../api/proxy-configuration)++If set, `CheerioCrawler` will be configured for all connections to use [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided+and rotated according to the configuration. For more information, see the [documentation](https://docs.apify.com/proxy).++---++### `handleFailedRequestFunction`++**Type**: [`HandleFailedRequest`](../typedefs/handle-failed-request)++A function to handle requests that failed more than `option.maxRequestRetries` times. The function receives the following object as an argument:++```+{+ error: Error,+ request: Request,+ session: Session,+ $: Cheerio,+ body: String|Buffer,+ json: Object,+ contentType: Object,+ response: Object,+ proxyInfo: ProxyInfo,+ crawler: CheerioCrawler,+}+```++where the [`Request`](../api/request) instance corresponds to the failed request, and the `Error` instance represents the last error thrown during+processing of the request.++See [source code](https://github.com/apify/apify-js/blob/master/src/crawlers/cheerio_crawler.js#L13) for the default implementation of this function.++---++### `preNavigationHooks`++**Type**: [`Array<Hook>`](../typedefs/hook)++Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies or browser properties before navigation.+The function accepts two parameters, `crawlingContext` and `requestAsBrowserOptions`, which are passed to the `requestAsBrowser()` function the+crawler calls to navigate. Example:++```+preNavigationHooks: [+ async (crawlingContext, requestAsBrowserOptions) => {+ requestAsBrowserOptions.forceUrlEncoding = true;+ },+]+```++---++### `postNavigationHooks`++**Type**: [`Array<Hook>`](../typedefs/hook)++Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful. The function accepts+`crawlingContext` as the only parameter. Example:++```+postNavigationHooks: [+ async (crawlingContext) => {+ // ...+ },+]+```++---++### `additionalMimeTypes`++**Type**: `Array<string>`++An array of <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types"+ target="_blank">MIME types</a> you want the crawler to load and process. By default, only `text/html` and `application/xhtml+xml` MIME types are+supported.++---++### `suggestResponseEncoding`++**Type**: `string`++By default `CheerioCrawler` will extract correct encoding from the HTTP response headers. Sadly, there are some websites which use invalid headers.+Those are encoded using the UTF-8 encoding. If those sites actually use a different encoding, the response will be corrupted. You can use+`suggestResponseEncoding` to fall back to a certain encoding, if you know that your target website uses it. To force a certain encoding, disregarding+the response headers, use [`CheerioCrawlerOptions.forceResponseEncoding`](../typedefs/cheerio-crawler-options#forceresponseencoding)++```+// Will fall back to windows-1250 encoding if none found+suggestResponseEncoding: 'windows-1250'+```++---++### `forceResponseEncoding`++**Type**: `string`++By default `CheerioCrawler` will extract correct encoding from the HTTP response headers. Use `forceResponseEncoding` to force a certain encoding,+disregarding the response headers. To only provide a default for missing encodings, use+[`CheerioCrawlerOptions.suggestResponseEncoding`](../typedefs/cheerio-crawler-options#suggestresponseencoding)++```+// Will force windows-1250 encoding even if headers say otherwise+forceResponseEncoding: 'windows-1250'+```++---++### `maxRequestRetries`++**Type**: `number` <code> = 3</code>++Indicates how many times the request is retried if either `requestFunction` or `handlePageFunction` fails.++---++### `maxRequestsPerCrawl`++**Type**: `number`++Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached. Always set this value in order to prevent infinite+loops in misconfigured crawlers. Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.++---++### `autoscaledPoolOptions`++**Type**: [`AutoscaledPoolOptions`](../typedefs/autoscaled-pool-options)++Custom options passed to the underlying [`AutoscaledPool`](../api/autoscaled-pool) constructor. Note that the `runTaskFunction`, `isTaskReadyFunction`+and `isFinishedFunction` options are provided by `CheerioCrawler` and cannot be overridden. Reasonable [`Snapshotter`](../api/snapshotter) and+[`SystemStatus`](../api/system-status) defaults are provided to account for the fact that `cheerio` parses HTML synchronously and therefore blocks the+event loop.++---++### `minConcurrency`++**Type**: `number` <code> = 1</code>++Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding [`AutoscaledPool`](../api/autoscaled-pool) option.++_WARNING:_ If you set this value too high with respect to the available system memory and CPU, your crawler will run extremely slow or crash. If+you're not sure, just keep the default value and the concurrency will scale up automatically.++---++### `maxConcurrency`++**Type**: `number` <code> = 1000</code>++Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding [`AutoscaledPool`](../api/autoscaled-pool) option.++---++### `useSessionPool`++**Type**: `boolean` <code> = true</code>++If set to true Crawler will automatically use Session Pool. It will automatically retire sessions on 403, 401 and 429 status codes. It also marks+Session as bad after a request timeout.++---++### `sessionPoolOptions`++**Type**: [`SessionPoolOptions`](../typedefs/session-pool-options)++Custom options passed to the underlying [`SessionPool`](../api/session-pool) constructor.++---++### `persistCookiesPerSession`++**Type**: `boolean`++Automatically saves cookies to Session. Works only if Session Pool is used.++It parses cookie from response "set-cookie" header saves or updates cookies for session and once the session is used for next request. It passes the+"Cookie" header to the request with the session cookies.++---diff --git a/website/versioned_docs/version-2.3.0/typedefs/CheerioHandlePageInputs.md b/website/versioned_docs/version-2.3.0/typedefs/CheerioHandlePageInputs.mdnew file mode 100644index 000000000..7cf55be60--- /dev/null+++ b/website/versioned_docs/version-2.3.0/typedefs/CheerioHandlePageInputs.md@@ -0,0 +1,77 @@+---+id: version-2.3.0-cheerio-handle-page-inputs+title: CheerioHandlePageInputs+original_id: cheerio-handle-page-inputs+---++<a name="cheeriohandlepageinputs"></a>++## Properties++### `$`++**Type**: `CheerioAPI`++The [Cheerio](https://cheerio.js.org/) object with parsed HTML.++---++### `body`++**Type**: `string` | `Buffer`++The request body of the web page.++---++### `json`++**Type**: `*`++The parsed object from JSON string if the response contains the content type application/json.++---++### `request`++**Type**: [`Request`](../api/request)++The original [`Request`](../api/request) object.++---++### `contentType`++**Type**: `Object`++Parsed `Content-Type header: { type, encoding }`.++---++### `response`++**Type**: `IncomingMessage`++An instance of Node's [http.IncomingMessage](https://nodejs.org/api/http.html#http_class_http_incomingmessage) object,++---++### `session`++**Type**: [`Session`](../api/session)++---++### `proxyInfo`++**Type**: [`ProxyInfo`](../typedefs/proxy-info)++An object with information about currently used proxy by the crawler and configured by the [`ProxyConfiguration`](../api/proxy-configuration) class.++---++### `crawler`++**Type**: [`CheerioCrawler`](../api/cheerio-crawler)++---diff --git a/website/versioned_docs/version-2.3.0/typedefs/PlaywrightCrawlerOptions.md b/website/versioned_docs/version-2.3.0/typedefs/PlaywrightCrawlerOptions.mdnew file mode 100644index 000000000..cdd63ff6d--- /dev/null+++ b/website/versioned_docs/version-2.3.0/typedefs/PlaywrightCrawlerOptions.md@@ -0,0 +1,234 @@+---+id: version-2.3.0-playwright-crawler-options+title: PlaywrightCrawlerOptions+original_id: playwright-crawler-options+---++<a name="playwrightcrawleroptions"></a>++## Properties++### `handlePageFunction`++**Type**: [`PlaywrightHandlePageFunction`](../typedefs/playwright-handle-page-function)++Function that is called to process each request. It is passed an object with the following fields:++```+{+ request: Request,+ response: Response,+ page: Page,+ session: Session,+ browserController: BrowserController,+ proxyInfo: ProxyInfo,+ crawler: PlaywrightCrawler,+}+```++`request` is an instance of the [`Request`](../api/request) object with details about the URL to open, HTTP method etc. `page` is an instance of the+`Playwright` [`Page`](https://playwright.dev/docs/api/class-page) `browserController` is an instance of the+[`BrowserController`](https://github.com/apify/browser-pool#browsercontroller), `response` is an instance of the `Playwright`+[`Response`](https://playwright.dev/docs/api/class-response), which is the main resource response as returned by `page.goto(request.url)`. The+function must return a promise, which is then awaited by the crawler.++If the function throws an exception, the crawler will try to re-crawl the request later, up to `option.maxRequestRetries` times. If all the retries+fail, the crawler calls the function provided to the `handleFailedRequestFunction` parameter. To make this work, you should **always** let your+function throw exceptions rather than catch them. The exceptions are logged to the request using the+[`Request.pushErrorMessage()`](../api/request#pusherrormessage) function.++---++### `navigationTimeoutSecs`++**Type**: `number` <code> = 60</code>++Timeout in which page navigation needs to finish, in seconds.++---++### `handleFailedRequestFunction`++**Type**: [`HandleFailedRequest`](../typedefs/handle-failed-request)++A function to handle requests that failed more than `option.maxRequestRetries` times.++The function receives the following object as an argument:++```+{+ request: Request,+ response: Response,+ page: Page,+ session: Session,+ browserController: BrowserController,+ proxyInfo: ProxyInfo,+ crawler: PlaywrightCrawler,+}+```++Where the [`Request`](../api/request) instance corresponds to the failed request, and the `Error` instance represents the last error thrown during+processing of the request.++---++### `preNavigationHooks`++**Type**: [`Array<PlaywrightHook>`](../typedefs/playwright-hook)++Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies or browser properties before navigation.+The function accepts two parameters, `crawlingContext` and `gotoOptions`, which are passed to the `page.goto()` function the crawler calls to+navigate. Example:++```+preNavigationHooks: [+ async (crawlingContext, gotoOptions) => {+ await page.evaluate((attr) => { window.foo = attr; }, 'bar');+ },+]+```++---++### `postNavigationHooks`++**Type**: [`Array<PlaywrightHook>`](../typedefs/playwright-hook)++Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful. The function accepts+`crawlingContext` as the only parameter. Example:++```+postNavigationHooks: [+ async (crawlingContext) => {+ const { page } = crawlingContext;+ if (hasCaptcha(page)) {+ await solveCaptcha (page);+ }+ },+]+```++---++### `launchContext`++**Type**: [`PlaywrightLaunchContext`](../typedefs/playwright-launch-context)++The same options as used by [`Apify.launchPlaywright()`](../api/apify#launchplaywright).++---++### `handlePageTimeoutSecs`++**Type**: `number` <code> = 60</code>++Timeout in which the function passed as `handlePageFunction` needs to finish, in seconds.++---++### `browserPoolOptions`++**Type**: `BrowserPoolOptions`++Custom options passed to the underlying [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool) constructor. You can tweak those to+fine-tune browser management.++---++### `persistCookiesPerSession`++**Type**: `boolean` <code> = true</code>++Automatically saves cookies to Session. Works only if Session Pool is used.++---++### `proxyConfiguration`++**Type**: [`ProxyConfiguration`](../api/proxy-configuration)++If set, `PlaywrightCrawler` will be configured for all connections to use [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs+provided and rotated according to the configuration. For more information, see the [documentation](https://docs.apify.com/proxy).++---++### `requestList`++**Type**: [`RequestList`](../api/request-list)++Static list of URLs to be processed. Either `requestList` or `requestQueue` option must be provided (or both).++---++### `requestQueue`++**Type**: [`RequestQueue`](../api/request-queue)++Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites. Either `requestList` or `requestQueue` option must be+provided (or both).++---++### `maxRequestRetries`++**Type**: `number` <code> = 3</code>++Indicates how many times the request is retried if+[`PlaywrightCrawlerOptions.handlePageFunction`](../typedefs/playwright-crawler-options#handlepagefunction) fails.++---++### `maxRequestsPerCrawl`++**Type**: `number`++Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached. Always set this value in order to prevent infinite+loops in misconfigured crawlers. Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.++---++### `autoscaledPoolOptions`++**Type**: [`AutoscaledPoolOptions`](../typedefs/autoscaled-pool-options)++Custom options passed to the underlying [`AutoscaledPool`](../api/autoscaled-pool) constructor. Note that the `runTaskFunction` and+`isTaskReadyFunction` options are provided by `BasicCrawler` and cannot be overridden. However, you can provide a custom implementation of+`isFinishedFunction`.++---++### `minConcurrency`++**Type**: `number` <code> = 1</code>++Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding [`AutoscaledPool`](../api/autoscaled-pool) option.++_WARNING:_ If you set this value too high with respect to the available system memory and CPU, your crawler will run extremely slow or crash. If+you're not sure, just keep the default value and the concurrency will scale up automatically.++---++### `maxConcurrency`++**Type**: `number` <code> = 1000</code>++Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding [`AutoscaledPool`](../api/autoscaled-pool) option.++---++### `useSessionPool`++**Type**: `boolean` <code> = true</code>++Playwright crawler will initialize the [`SessionPool`](../api/session-pool) with the corresponding `sessionPoolOptions`. The session instance will be+than available in the `handleRequestFunction`.++---++### `sessionPoolOptions`++**Type**: [`SessionPoolOptions`](../typedefs/session-pool-options)++The configuration options for [`SessionPool`](../api/session-pool) to use.++---diff --git a/website/versioned_docs/version-2.3.0/typedefs/PuppeteerCookie.md b/website/versioned_docs/version-2.3.0/typedefs/PuppeteerCookie.mdnew file mode 100644index 000000000..981ff34fb--- /dev/null+++ b/website/versioned_docs/version-2.3.0/typedefs/PuppeteerCookie.md@@ -0,0 +1,9 @@+---+id: version-2.3.0-puppeteer-cookie+title: PuppeteerCookie+original_id: puppeteer-cookie+---++<a name="puppeteercookie"></a>++---diff --git a/website/versioned_docs/version-2.3.0/typedefs/PuppeteerCrawlerOptions.md b/website/versioned_docs/version-2.3.0/typedefs/PuppeteerCrawlerOptions.mdnew file mode 100644index 000000000..50cf503e9--- /dev/null+++ b/website/versioned_docs/version-2.3.0/typedefs/PuppeteerCrawlerOptions.md@@ -0,0 +1,236 @@+---+id: version-2.3.0-puppeteer-crawler-options+title: PuppeteerCrawlerOptions+original_id: puppeteer-crawler-options+---++<a name="puppeteercrawleroptions"></a>++## Properties++### `handlePageFunction`++**Type**: [`PuppeteerHandlePage`](../typedefs/puppeteer-handle-page)++Function that is called to process each request. It is passed an object with the following fields:++```+{+ request: Request,+ response: Response,+ page: Page,+ session: Session,+ browserController: BrowserController,+ proxyInfo: ProxyInfo,+ crawler: PuppeteerCrawler,+}+```++`request` is an instance of the [`Request`](../api/request) object with details about the URL to open, HTTP method etc. `page` is an instance of the+`Puppeteer` [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) `browserPool` is an instance of the+[`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool), `browserController` is an instance of the+[`BrowserController`](https://github.com/apify/browser-pool#browsercontroller), `response` is an instance of the `Puppeteer`+[`Response`](https://pptr.dev/#?product=Puppeteer&show=api-class-response), which is the main resource response as returned by+`page.goto(request.url)`. The function must return a promise, which is then awaited by the crawler.++If the function throws an exception, the crawler will try to re-crawl the request later, up to `option.maxRequestRetries` times. If all the retries+fail, the crawler calls the function provided to the `handleFailedRequestFunction` parameter. To make this work, you should **always** let your+function throw exceptions rather than catch them. The exceptions are logged to the request using the+[`Request.pushErrorMessage()`](../api/request#pusherrormessage) function.++---++### `navigationTimeoutSecs`++**Type**: `number` <code> = 60</code>++Timeout in which page navigation needs to finish, in seconds.++---++### `handleFailedRequestFunction`++**Type**: [`HandleFailedRequest`](../typedefs/handle-failed-request)++A function to handle requests that failed more than `option.maxRequestRetries` times.++The function receives the following object as an argument:++```+{+ request: Request,+ response: Response,+ page: Page,+ session: Session,+ browserController: BrowserController,+ proxyInfo: ProxyInfo,+ crawler: PuppeteerCrawler,+}+```++Where the [`Request`](../api/request) instance corresponds to the failed request, and the `Error` instance represents the last error thrown during+processing of the request.++---++### `launchContext`++**Type**: [`PuppeteerLaunchContext`](../typedefs/puppeteer-launch-context)++Options used by [`Apify.launchPuppeteer()`](../api/apify#launchpuppeteer) to start new Puppeteer instances.++---++### `handlePageTimeoutSecs`++**Type**: `number` <code> = 60</code>++Timeout in which the function passed as `handlePageFunction` needs to finish, in seconds.++---++### `browserPoolOptions`++**Type**: `BrowserPoolOptions`++Custom options passed to the underlying [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool) constructor. You can tweak those to+fine-tune browser management.++---++### `persistCookiesPerSession`++**Type**: `boolean` <code> = true</code>++Automatically saves cookies to Session. Works only if Session Pool is used.++---++### `proxyConfiguration`++**Type**: [`ProxyConfiguration`](../api/proxy-configuration)++If set, `PuppeteerCrawler` will be configured for all connections to use [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs+provided and rotated according to the configuration. For more information, see the [documentation](https://docs.apify.com/proxy).++---++### `preNavigationHooks`++**Type**: [`Array<PuppeteerHook>`](../typedefs/puppeteer-hook)++Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies or browser properties before navigation.+The function accepts two parameters, `crawlingContext` and `gotoOptions`, which are passed to the `page.goto()` function the crawler calls to+navigate. Example:++```+preNavigationHooks: [+ async (crawlingContext, gotoOptions) => {+ const { page } = crawlingContext;+ await page.evaluate((attr) => { window.foo = attr; }, 'bar');+ },+]+```++---++### `postNavigationHooks`++**Type**: [`Array<PuppeteerHook>`](../typedefs/puppeteer-hook)++Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful. The function accepts+`crawlingContext` as the only parameter. Example:++```+postNavigationHooks: [+ async (crawlingContext) => {+ const { page } = crawlingContext;+ if (hasCaptcha(page)) {+ await solveCaptcha (page);+ }+ },+]+```++---++### `requestList`++**Type**: [`RequestList`](../api/request-list)++Static list of URLs to be processed. Either `requestList` or `requestQueue` option must be provided (or both).++---++### `requestQueue`++**Type**: [`RequestQueue`](../api/request-queue)++Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites. Either `requestList` or `requestQueue` option must be+provided (or both).++---++### `maxRequestRetries`++**Type**: `number` <code> = 3</code>++Indicates how many times the request is retried if+[`PuppeteerCrawlerOptions.handlePageFunction`](../typedefs/puppeteer-crawler-options#handlepagefunction) fails.++---++### `maxRequestsPerCrawl`++**Type**: `number`++Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached. Always set this value in order to prevent infinite+loops in misconfigured crawlers. Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.++---++### `autoscaledPoolOptions`++**Type**: [`AutoscaledPoolOptions`](../typedefs/autoscaled-pool-options)++Custom options passed to the underlying [`AutoscaledPool`](../api/autoscaled-pool) constructor. Note that the `runTaskFunction` and+`isTaskReadyFunction` options are provided by the crawler and cannot be overridden. However, you can provide a custom implementation of+`isFinishedFunction`.++---++### `minConcurrency`++**Type**: `number` <code> = 1</code>++Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding [`AutoscaledPool`](../api/autoscaled-pool) option.++_WARNING:_ If you set this value too high with respect to the available system memory and CPU, your crawler will run extremely slow or crash. If+you're not sure, just keep the default value and the concurrency will scale up automatically.++---++### `maxConcurrency`++**Type**: `number` <code> = 1000</code>++Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding [`AutoscaledPool`](../api/autoscaled-pool) option.++---++### `useSessionPool`++**Type**: `boolean` <code> = true</code>++Puppeteer crawler will initialize the [`SessionPool`](../api/session-pool) with the corresponding `sessionPoolOptions`. The session instance will be+than available in the `handleRequestFunction`.++---++### `sessionPoolOptions`++**Type**: [`SessionPoolOptions`](../typedefs/session-pool-options)++The configuration options for [`SessionPool`](../api/session-pool) to use.++---diff --git a/website/versioned_docs/version-2.3.0/typedefs/PuppeteerHook.md b/website/versioned_docs/version-2.3.0/typedefs/PuppeteerHook.mdnew file mode 100644index 000000000..a15117aca--- /dev/null+++ b/website/versioned_docs/version-2.3.0/typedefs/PuppeteerHook.md@@ -0,0 +1,19 @@+---+id: version-2.3.0-puppeteer-hook+title: PuppeteerHook+original_id: puppeteer-hook+---++<a name="puppeteerhook"></a>++**Parameters**:++- **`crawlingContext`**: `Object` | [`BrowserCrawlingContext`](../typedefs/browser-crawling-context) |+ [`CrawlingContext`](../typedefs/crawling-context)+- **`gotoOptions`**: `any`++**Returns**:++`Promise<void>`++---diff --git a/website/versioned_docs/version-2.3.0/typedefs/RequestListOptions.md b/website/versioned_docs/version-2.3.0/typedefs/RequestListOptions.mdnew file mode 100644index 000000000..cdad23736--- /dev/null+++ b/website/versioned_docs/version-2.3.0/typedefs/RequestListOptions.md@@ -0,0 +1,152 @@+---+id: version-2.3.0-request-list-options+title: RequestListOptions+original_id: request-list-options+---++<a name="requestlistoptions"></a>++## Properties++### `sources`++**Type**: [`Array<(RequestOptions|Request|{requestsFromUrl: string}|string)>`](../typedefs/request-options)++An array of sources of URLs for the [`RequestList`](../api/request-list). It can be either an array of strings, plain objects that define at least the+`url` property, or an array of [`Request`](../api/request) instances.++**IMPORTANT:** The `sources` array will be consumed (left empty) after `RequestList` initializes. This is a measure to prevent memory leaks in+situations when millions of sources are added.++Additionally, the `requestsFromUrl` property may be used instead of `url`, which will instruct `RequestList` to download the source URLs from a given+remote location. The URLs will be parsed from the received response.++```+[+ // A single URL+ 'http://example.com/a/b',++ // Modify Request options+ { method: PUT, 'https://example.com/put, payload: { foo: 'bar' }}++ // Batch import of URLs from a file hosted on the web,+ // where the URLs should be requested using the HTTP POST request+ { method: 'POST', requestsFromUrl: 'http://example.com/urls.txt' },++ // Batch import from remote file, using a specific regular expression to extract the URLs.+ { requestsFromUrl: 'http://example.com/urls.txt', regex: /https:\/\/example.com\/.+/ },++ // Get list of URLs from a Google Sheets document. Just add "/gviz/tq?tqx=out:csv" to the Google Sheet URL.+ // For details, see https://help.apify.com/en/articles/2906022-scraping-a-list-of-urls-from-a-google-sheets-document+ { requestsFromUrl: 'https://docs.google.com/spreadsheets/d/1GA5sSQhQjB_REes8I5IKg31S-TuRcznWOPjcpNqtxmU/gviz/tq?tqx=out:csv' }+]+```++---++### `sourcesFunction`++**Type**: [`RequestListSourcesFunction`](../typedefs/request-list-sources-function)++A function that will be called to get the sources for the `RequestList`, but only if `RequestList` was not able to fetch their persisted version (see+[`RequestListOptions.persistRequestsKey`](../typedefs/request-list-options#persistrequestskey)). It must return an `Array` of+[`Request`](../api/request) or [`RequestOptions`](../typedefs/request-options).++This is very useful in a scenario when getting the sources is a resource intensive or time consuming task, such as fetching URLs from multiple+sitemaps or parsing URLs from large datasets. Using the `sourcesFunction` in combination with `persistStateKey` and `persistRequestsKey` will allow+you to fetch and parse those URLs only once, saving valuable time when your actor migrates or restarts.++If both [`RequestListOptions.sources`](../typedefs/request-list-options#sources) and+[`RequestListOptions.sourcesFunction`](../typedefs/request-list-options#sourcesfunction) are provided, the sources returned by the function will be+added after the `sources`.++**Example:**++```javascript+// Let's say we want to scrape URLs extracted from sitemaps.++const sourcesFunction = async () => {+ // With super large sitemaps, this operation could take very long+ // and big websites typically have multiple sitemaps.+ const sitemaps = await downloadHugeSitemaps();+ return parseUrlsFromSitemaps(sitemaps);+};++// Sitemaps can change in real-time, so it's important to persist+// the URLs we collected. Otherwise we might lose our scraping+// state in case of an actor migration / failure / time-out.+const requestList = new RequestList({+ sourcesFunction,+ persistStateKey: 'state-key',+ persistRequestsKey: 'requests-key',+});++// The sourcesFunction is called now and the Requests are persisted.+// If something goes wrong and we need to start again, RequestList+// will load the persisted Requests from storage and will NOT+// call the sourcesFunction again, saving time and resources.+await requestList.initialize();+```++---++### `persistStateKey`++**Type**: `string`++Identifies the key in the default key-value store under which `RequestList` periodically stores its state (i.e. which URLs were crawled and which+not). If the actor is restarted, `RequestList` will read the state and continue where it left off.++If `persistStateKey` is not set, `RequestList` will always start from the beginning, and all the source URLs will be crawled again.++---++### `persistRequestsKey`++**Type**: `string`++Identifies the key in the default key-value store under which the `RequestList` persists its Requests during the+[`RequestList.initialize()`](../api/request-list#initialize) call. This is necessary if `persistStateKey` is set and the source URLs might potentially+change, to ensure consistency of the source URLs and state object. However, it comes with some storage and performance overheads.++If `persistRequestsKey` is not set, [`RequestList.initialize()`](../api/request-list#initialize) will always fetch the sources from their origin,+check that they are consistent with the restored state (if any) and throw an error if they are not.++---++### `state`++**Type**: [`RequestListState`](../typedefs/request-list-state)++The state object that the `RequestList` will be initialized from. It is in the form as returned by `RequestList.getState()`, such as follows:++```+{+ nextIndex: 5,+ nextUniqueKey: 'unique-key-5'+ inProgress: {+ 'unique-key-1': true,+ 'unique-key-4': true,+ },+}+```++Note that the preferred (and simpler) way to persist the state of crawling of the `RequestList` is to use the `stateKeyPrefix` parameter instead.++---++### `keepDuplicateUrls`++**Type**: `boolean` <code> = false</code>++By default, `RequestList` will deduplicate the provided URLs. Default deduplication is based on the `uniqueKey` property of passed source+[`Request`](../api/request) objects.++If the property is not present, it is generated by normalizing the URL. If present, it is kept intact. In any case, only one request per `uniqueKey`+is added to the `RequestList` resulting in removal of duplicate URLs / unique keys.++Setting `keepDuplicateUrls` to `true` will append an additional identifier to the `uniqueKey` of each request that does not already include a+`uniqueKey`. Therefore, duplicate URLs will be kept in the list. It does not protect the user from having duplicates in user set `uniqueKey`s however.+It is the user's responsibility to ensure uniqueness of their unique keys if they wish to keep more than just a single copy in the `RequestList`.++---diff --git a/website/versioned_docs/version-2.3.0/typedefs/SessionPoolOptions.md b/website/versioned_docs/version-2.3.0/typedefs/SessionPoolOptions.mdnew file mode 100644index 000000000..009735375--- /dev/null+++ b/website/versioned_docs/version-2.3.0/typedefs/SessionPoolOptions.md@@ -0,0 +1,62 @@+---+id: version-2.3.0-session-pool-options+title: SessionPoolOptions+original_id: session-pool-options+---++<a name="sessionpooloptions"></a>++## Properties++### `maxPoolSize`++**Type**: `number` <code> = 1000</code>++Maximum size of the pool. Indicates how many sessions are rotated.++---++### `sessionOptions`++**Type**: [`SessionOptions`](../typedefs/session-options)++The configuration options for [`Session`](../api/session) instances.++---++### `persistStateKeyValueStoreId`++**Type**: `string`++Name or Id of `KeyValueStore` where is the `SessionPool` state stored.++---++### `persistStateKey`++**Type**: `string` <code> = "\"SESSION_POOL_STATE\""</code>++Session pool persists it's state under this key in Key value store.++---++### `createSessionFunction`++**Type**: [`CreateSession`](../typedefs/create-session)++Custom function that should return `Session` instance. Any error thrown from this function will terminate the process. Function receives `SessionPool`+instance as a parameter++---++### `forceCloud`++**Type**: `boolean` <code> = false</code>++If set to `true` then the function uses cloud storage usage even if the `APIFY_LOCAL_STORAGE_DIR` environment variable is set. This way it is possible+to combine local and cloud storage.++**Note:** If you use `forceCloud`, it is recommended to also set the `persistStateKeyValueStoreId` option, as otherwise the `KeyValueStore` will be+unnamed!++---diff --git a/website/versioned_sidebars/version-2.3.0-sidebars.json b/website/versioned_sidebars/version-2.3.0-sidebars.jsonnew file mode 100644index 000000000..08eb0e715--- /dev/null+++ b/website/versioned_sidebars/version-2.3.0-sidebars.json@@ -0,0 +1,187 @@+{+ "version-2.3.0-docs": {+ "Guide": [+ "version-2.3.0-guides/motivation",+ "version-2.3.0-guides/quick-start",+ "version-2.3.0-guides/apify-platform",+ "version-2.3.0-guides/getting-started",+ "version-2.3.0-guides/request-storage",+ "version-2.3.0-guides/result-storage",+ "version-2.3.0-guides/environment-variables",+ "version-2.3.0-guides/proxy-management",+ "version-2.3.0-guides/session-management",+ "version-2.3.0-guides/type-script-actor",+ "version-2.3.0-guides/docker-images",+ "version-2.3.0-guides/migration-to-v1",+ "version-2.3.0-guides/avoid-blocking"+ ],+ "Examples": [+ "version-2.3.0-examples/accept-user-input",+ "version-2.3.0-examples/add-data-to-dataset",+ "version-2.3.0-examples/basic-crawler",+ "version-2.3.0-examples/call-actor",+ "version-2.3.0-examples/capture-screenshot",+ "version-2.3.0-examples/cheerio-crawler",+ "version-2.3.0-examples/crawl-all-links",+ "version-2.3.0-examples/crawl-multiple-urls",+ "version-2.3.0-examples/crawl-relative-links",+ "version-2.3.0-examples/crawl-single-url",+ "version-2.3.0-examples/crawl-sitemap",+ "version-2.3.0-examples/crawl-some-links",+ "version-2.3.0-examples/forms",+ "version-2.3.0-examples/map-and-reduce",+ "version-2.3.0-examples/playwright-crawler",+ "version-2.3.0-examples/puppeteer-crawler",+ "version-2.3.0-examples/puppeteer-recursive-crawl",+ "version-2.3.0-examples/puppeteer-with-proxy",+ "version-2.3.0-examples/synchronous-run",+ "version-2.3.0-examples/use-stealth-mode"+ ],+ "API Reference": [+ "version-2.3.0-api/apify",+ "version-2.3.0-api/configuration",+ {+ "type": "subcategory",+ "label": "Crawlers",+ "ids": [+ "version-2.3.0-api/basic-crawler",+ "version-2.3.0-api/cheerio-crawler",+ "version-2.3.0-api/playwright-crawler",+ "version-2.3.0-api/puppeteer-crawler",+ "version-2.3.0-api/statistics"+ ]+ },+ {+ "type": "subcategory",+ "label": "Result Stores",+ "ids": [+ "version-2.3.0-api/dataset",+ "version-2.3.0-api/key-value-store"+ ]+ },+ {+ "type": "subcategory",+ "label": "Scaling",+ "ids": [+ "version-2.3.0-api/autoscaled-pool",+ "version-2.3.0-api/session",+ "version-2.3.0-api/session-pool",+ "version-2.3.0-api/proxy-configuration",+ "version-2.3.0-api/snapshotter",+ "version-2.3.0-api/system-status"+ ]+ },+ {+ "type": "subcategory",+ "label": "Sources",+ "ids": [+ "version-2.3.0-api/request",+ "version-2.3.0-api/request-list",+ "version-2.3.0-api/request-queue",+ "version-2.3.0-api/pseudo-url"+ ]+ },+ {+ "type": "subcategory",+ "label": "Utilities",+ "ids": [+ "version-2.3.0-api/utils",+ "version-2.3.0-api/log",+ "version-2.3.0-api/playwright",+ "version-2.3.0-api/puppeteer",+ "version-2.3.0-api/social"+ ]+ }+ ],+ "Type Definitions": [+ {+ "type": "subcategory",+ "label": "Constructor Options",+ "ids": [+ "version-2.3.0-typedefs/autoscaled-pool-options",+ "version-2.3.0-typedefs/basic-crawler-options",+ "version-2.3.0-typedefs/cheerio-crawler-options",+ "version-2.3.0-typedefs/playwright-crawler-options",+ "version-2.3.0-typedefs/playwright-launch-context",+ "version-2.3.0-typedefs/puppeteer-crawler-options",+ "version-2.3.0-typedefs/puppeteer-launch-context",+ "version-2.3.0-typedefs/proxy-configuration-options",+ "version-2.3.0-typedefs/proxy-info",+ "version-2.3.0-typedefs/logger-options",+ "version-2.3.0-typedefs/request-options",+ "version-2.3.0-typedefs/request-list-options",+ "version-2.3.0-typedefs/request-as-browser-options",+ "version-2.3.0-typedefs/session-options",+ "version-2.3.0-typedefs/session-pool-options",+ "version-2.3.0-typedefs/snapshotter-options",+ "version-2.3.0-typedefs/stealth-options",+ "version-2.3.0-typedefs/system-status-options"+ ]+ },+ {+ "type": "subcategory",+ "label": "Functions - Crawlers",+ "ids": [+ "version-2.3.0-typedefs/cheerio-handle-page",+ "version-2.3.0-typedefs/cheerio-handle-page-inputs",+ "version-2.3.0-typedefs/handle-failed-request",+ "version-2.3.0-typedefs/handle-failed-request-input",+ "version-2.3.0-typedefs/handle-request",+ "version-2.3.0-typedefs/handle-request-inputs",+ "version-2.3.0-typedefs/prepare-request",+ "version-2.3.0-typedefs/prepare-request-inputs",+ "version-2.3.0-typedefs/playwright-handle-page",+ "version-2.3.0-typedefs/playwright-handle-page-inputs",+ "version-2.3.0-typedefs/puppeteer-handle-page",+ "version-2.3.0-typedefs/puppeteer-handle-page-inputs"+ ]+ },+ {+ "type": "subcategory",+ "label": "Functions - Dataset",+ "ids": [+ "version-2.3.0-typedefs/dataset-consumer",+ "version-2.3.0-typedefs/dataset-mapper",+ "version-2.3.0-typedefs/dataset-reducer"+ ]+ },+ {+ "type": "subcategory",+ "label": "Functions - KeyValueStore",+ "ids": [+ "version-2.3.0-typedefs/key-consumer"+ ]+ },+ {+ "type": "subcategory",+ "label": "Functions - Sessions",+ "ids": [+ "version-2.3.0-typedefs/create-session"+ ]+ },+ {+ "type": "subcategory",+ "label": "Functions - Utilities",+ "ids": [+ "version-2.3.0-typedefs/request-transform"+ ]+ },+ {+ "type": "subcategory",+ "label": "Return Types",+ "ids": [+ "version-2.3.0-typedefs/actor-run",+ "version-2.3.0-api/apify-call-error",+ "version-2.3.0-typedefs/apify-env",+ "version-2.3.0-typedefs/dataset-content",+ "version-2.3.0-typedefs/memory-info",+ "version-2.3.0-typedefs/queue-operation-info",+ "version-2.3.0-typedefs/request-list-state",+ "version-2.3.0-typedefs/session-state",+ "version-2.3.0-typedefs/social-handles",+ "version-2.3.0-typedefs/system-info"+ ]+ }+ ]+ }+}diff --git a/website/versions.json b/website/versions.jsonindex cdcea9882..f1642a74b 100644--- a/website/versions.json+++ b/website/versions.json@@ -1,4 +1,5 @@[+ "2.3.0","2.2.0","2.1.0","2.0.6",