Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Build docs with new examples
Ondra Urbanfe3490d92f19
diff --git a/website/i18n/en.json b/website/i18n/en.jsonindex 2dea7bb5f..f50076424 100644--- a/website/i18n/en.json+++ b/website/i18n/en.json@@ -116,30 +116,27 @@"examples/forms": {"title": "Forms"},- "examples/handle-broken-links": {- "title": "Handle broken links"- },"examples/map-and-reduce": {"title": "Dataset Map and Reduce methods"},+ "examples/playwright-crawler": {+ "title": "Playwright crawler"+ },"examples/puppeteer-crawler": {"title": "Puppeteer crawler"},"examples/puppeteer-recursive-crawl": {"title": "Puppeteer recursive crawl"},- "examples/puppeteer-sitemap": {- "title": "Puppeteer sitemap"- },"examples/puppeteer-with-proxy": {"title": "Puppeteer with proxy"},- "examples/screenshots": {- "title": "Screenshots"- },"examples/synchronous-run": {"title": "Synchronous run"},+ "examples/use-stealth-mode": {+ "title": "use-stealth-mode"+ },"guides/apify-platform": {"title": "Apify Platform"},diff --git a/website/sidebars.json b/website/sidebars.jsonindex b25751d14..ee599fdea 100644--- a/website/sidebars.json+++ b/website/sidebars.json@@ -27,14 +27,13 @@"examples/crawl-sitemap","examples/crawl-some-links","examples/forms",- "examples/handle-broken-links","examples/map-and-reduce",+ "examples/playwright-crawler","examples/puppeteer-crawler","examples/puppeteer-recursive-crawl",- "examples/puppeteer-sitemap","examples/puppeteer-with-proxy",- "examples/screenshots",- "examples/synchronous-run"+ "examples/synchronous-run",+ "examples/use-stealth-mode"],"API Reference": ["api/apify",diff --git a/website/versioned_docs/version-1.0.1/examples/add_data_to_dataset.md b/website/versioned_docs/version-1.0.1/examples/add_data_to_dataset.mdnew file mode 100644index 000000000..719b5b859--- /dev/null+++ b/website/versioned_docs/version-1.0.1/examples/add_data_to_dataset.md@@ -0,0 +1,43 @@+---+id: version-1.0.1-add-data-to-dataset+title: Add data to dataset+original_id: add-data-to-dataset+---++This example saves data to the default dataset. If the dataset doesn't exist, it will be created. You can save data to custom datasets by using+[`Apify.openDataset()`](../api/apify#opendataset)++```javascript+const Apify = require('apify');++Apify.main(async () => {+ const requestList = await Apify.openRequestList('start-urls', [+ { url: 'http://www.example.com/page-1' },+ { url: 'http://www.example.com/page-2' },+ { url: 'http://www.example.com/page-3' },+ ]);++ // Function called for each URL+ const handlePageFunction = async ({ request, body }) => {+ // Save data to default dataset+ await Apify.pushData({+ url: request.url,+ html: body,+ });+ };++ const crawler = new Apify.CheerioCrawler({+ requestList,+ handlePageFunction,+ });++ // Run the crawler+ await crawler.run();+});+```++Each item in this dataset will be saved to its own file in the following directory:++```bash+{PROJECT_FOLDER}/apify_storage/datasets/default/+```diff --git a/website/versioned_docs/version-1.0.1/examples/basic_crawler.md b/website/versioned_docs/version-1.0.1/examples/basic_crawler.mdnew file mode 100644index 000000000..8767d5154--- /dev/null+++ b/website/versioned_docs/version-1.0.1/examples/basic_crawler.md@@ -0,0 +1,56 @@+---+id: version-1.0.1-basic-crawler+title: Basic crawler+original_id: basic-crawler+---++This is the most bare-bones example of the Apify SDK, which demonstrates some of its building blocks such as the+[`BasicCrawler`](/docs/api/basic-crawler). You probably don't need to go this deep though, and it would be better to start with one of the full+featured crawlers like [`CheerioCrawler`](https://sdk.apify.com/docs/examples/cheerio-crawler) or+[`PlaywrightCrawler`](https://sdk.apify.com/docs/examples/playwright-crawler).++The script simply downloads several web pages with plain HTTP requests using the [`Apify.utils.requestAsBrowser()`](/docs/api/utils#requestasbrowser)+convenience function and stores their raw HTML and URL in the default dataset. In local configuration, the data will be stored as JSON files in+`./apify_storage/datasets/default`.++```javascript+const Apify = require('apify');++// Apify.main() function wraps the crawler logic (it is optional).+Apify.main(async () => {+ // Create and initialize an instance of the RequestList class that contains+ // a list of URLs to crawl. Here we use just a few hard-coded URLs.+ const requestList = await Apify.openRequestList('start-urls', [+ { url: 'http://www.google.com/' },+ { url: 'http://www.example.com/' },+ { url: 'http://www.bing.com/' },+ { url: 'http://www.wikipedia.com/' },+ ]);++ // Create a BasicCrawler - the simplest crawler that enables+ // users to implement the crawling logic themselves.+ const crawler = new Apify.BasicCrawler({+ // Let the crawler fetch URLs from our list.+ requestList,++ // This function will be called for each URL to crawl.+ handleRequestFunction: async ({ request }) => {+ console.log(`Processing ${request.url}...`);++ // Fetch the page HTML via Apify utils requestAsBrowser+ const { body } = await Apify.utils.requestAsBrowser(request);++ // Store the HTML and URL to the default dataset.+ await Apify.pushData({+ url: request.url,+ html: body,+ });+ },+ });++ // Run the crawler and wait for it to finish.+ await crawler.run();++ console.log('Crawler finished.');+});+```diff --git a/website/versioned_docs/version-1.0.1/examples/call_actor.md b/website/versioned_docs/version-1.0.1/examples/call_actor.mdnew file mode 100644index 000000000..b3981f424--- /dev/null+++ b/website/versioned_docs/version-1.0.1/examples/call_actor.md@@ -0,0 +1,54 @@+---+id: version-1.0.1-call-actor+title: Call actor+original_id: call-actor+---++This example demonstrates how to start an Apify actor using [`Apify.call()`](/docs/api/apify#call) and how to call the Apify API using+[`Apify.client`](/docs/api/apify#client). The script gets a random weird word and its explanation from [randomword.com](https://randomword.com/) and+sends it to your email using the [`apify/send-mail`](https://apify.com/apify/send-mail) actor.++To make the example work, you'll need an [Apify account](https://my.apify.com/). Go to the+[Account - Integrations](https://my.apify.com/account#/integrations) page to obtain your API token and set it to the+[`APIFY_TOKEN`](/docs/guides/environment-variables#APIFY_TOKEN) environment variable, or run the script using the Apify CLI. If you deploy this actor+to the Apify Cloud, you can do things like set up a scheduler to run your actor early in the morning.++To see what other actors are available, visit the [Apify Store](https://apify.com/store).++> To run this example on the Apify Platform, select the `Node.js 12 + Chrome on Debian (apify/actor-node-chrome)` base image on the **Source** tab+> when configuring the actor.++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Launch the web browser.+ const browser = await Apify.launchPuppeteer();++ console.log('Obtaining own email address...');+ const apifyClient = Apify.newClient();+ const user = await apifyClient.user().get();++ // Load randomword.com and get a random word+ console.log('Fetching a random word.');+ const page = await browser.newPage();+ await page.goto('https://randomword.com/');+ const randomWord = await page.$eval('#shared_section', el => el.outerHTML);++ // Send random word to your email. For that, you can use an actor we already+ // have available on the platform under the name: apify/send-mail.+ // The second parameter to the Apify.call() invocation is the actor's+ // desired input. You can find the required input parameters by checking+ // the actor's documentation page: https://apify.com/apify/send-mail+ console.log(`Sending email to ${user.email}...`);+ await Apify.call('apify/send-mail', {+ to: user.email,+ subject: 'Random Word',+ html: `<h1>Random Word</h1>${randomWord}`,+ });+ console.log('Email sent. Good luck!');++ // Close Browser+ await browser.close();+});+```diff --git a/website/versioned_docs/version-1.0.1/examples/capture_screenshot.md b/website/versioned_docs/version-1.0.1/examples/capture_screenshot.mdnew file mode 100644index 000000000..5733e6caf--- /dev/null+++ b/website/versioned_docs/version-1.0.1/examples/capture_screenshot.md@@ -0,0 +1,136 @@+---+id: version-1.0.1-capture-screenshot+title: Capture a screenshot+original_id: capture-screenshot+---++> To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.++This example captures a screenshot of a web page using `Puppeteer`. It would look almost exactly the same with `Playwright`.++<!--DOCUSAURUS_CODE_TABS-->++<!-- PageScreenshot -->++\+Using `page.screenshot()`:++```javascript+const Apify = require('apify');++Apify.main(async () => {+ const url = 'http://www.example.com/';+ // Start a browser+ const browser = await Apify.launchPuppeteer();+ // Open new tab in the browser+ const page = await browser.newPage();+ // Navigate to the URL+ await page.goto(url);+ // Capture the screenshot+ const screenshot = await page.screenshot();+ // Save the screenshot to the default key-value store+ await Apify.setValue('my-key', screenshot, { contentType: 'image/png' });+ // Close Puppeteer+ await browser.close();+});+```++<!-- ApifySnapshot -->++\+Using `Apify.utils.puppeteer.saveSnapshot()`:++```javascript+const Apify = require('apify');++Apify.main(async () => {+ const url = 'http://www.example.com/';+ // Start a browser+ const browser = await Apify.launchPuppeteer();+ // Open new tab in the browser+ const page = await browser.newPage();+ // Navigate to the URL+ await page.goto(url);+ // Capture the screenshot+ await Apify.utils.puppeteer.saveSnapshot(page, { key: 'my-key', saveHtml: false });+ // Close Puppeteer+ await browser.close();+});+```++<!--END_DOCUSAURUS_CODE_TABS-->++This example captures a screenshot of multiple web pages when using `PuppeteerCrawler`:++<!--DOCUSAURUS_CODE_TABS-->++<!-- PageScreenshot -->++\+Using `page.screenshot()`:++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Add URLs to a RequestList+ const requestList = await Apify.openRequestList('start-urls', [+ { url: 'http://www.example.com/page-1' },+ { url: 'http://www.example.com/page-2' },+ { url: 'http://www.example.com/page-3' },+ ]);+ // Function called for each URL+ const handlePageFunction = async ({ request, page }) => {+ // Capture the screenshot with Puppeteer+ const screenshot = await page.screenshot();+ // Convert the URL into a valid key+ const key = request.url.replace(/[:/]/g, '_');+ // Save the screenshot to the default key-value store+ await Apify.setValue(key, screenshot, { contentType: 'image/png' });+ };+ // Create a PuppeteerCrawler+ const crawler = new Apify.PuppeteerCrawler({+ requestList,+ handlePageFunction,+ });+ // Run the crawler+ await crawler.run();+});+```++<!-- ApifySnapshot -->++\+Using `Apify.utils.puppeteer.saveSnapshot()`:++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Add URLs to a RequestList+ const requestList = await Apify.openRequestList('start-urls', [+ { url: 'http://www.example.com/page-1' },+ { url: 'http://www.example.com/page-2' },+ { url: 'http://www.example.com/page-3' },+ ]);+ // Function called for each URL+ const handlePageFunction = async ({ request, page }) => {+ // Convert the URL into a valid key+ const key = request.url.replace(/[:/]/g, '_');+ // Capture the screenshot+ await Apify.utils.puppeteer.saveSnapshot(page, { key, saveHtml: false });+ };+ // Create a PuppeteerCrawler+ const crawler = new Apify.PuppeteerCrawler({+ requestList,+ handlePageFunction,+ });+ // Run the crawler+ await crawler.run();+});+```++<!--END_DOCUSAURUS_CODE_TABS-->++In both examples using `page.screenshot()`, a `key` variable is created based on the URL of the web page. This variable is used as the key when saving+each screenshot into a key-value store.diff --git a/website/versioned_docs/version-1.0.1/examples/cheerio_crawler.md b/website/versioned_docs/version-1.0.1/examples/cheerio_crawler.mdnew file mode 100644index 000000000..e4910163b--- /dev/null+++ b/website/versioned_docs/version-1.0.1/examples/cheerio_crawler.md@@ -0,0 +1,88 @@+---+id: version-1.0.1-cheerio-crawler+title: Cheerio crawler+original_id: cheerio-crawler+---++This example demonstrates how to use [`CheerioCrawler`](/docs/api/cheerio-crawler) to crawl a list of URLs from an external file, load each URL using+a plain HTTP request, parse the HTML using the [Cheerio library](https://www.npmjs.com/package/cheerio) and extract some data from it: the page title+and all `h1` tags.++```javascript+const Apify = require('apify');++// Apify.utils contains various utilities, e.g. for logging.+// Here we use debug level of logging to improve the debugging experience.+// This functionality is optional!+const { log } = Apify.utils;+log.setLevel(log.LEVELS.DEBUG);++// Apify.main() function wraps the crawler logic (it is optional).+Apify.main(async () => {+ // Create an instance of the RequestList class that contains a list of URLs to crawl.+ // Add URLs to a RequestList+ const requestList = await Apify.openRequestList('start-urls', [+ { url: 'http://www.example.com/page-1' },+ { url: 'http://www.example.com/page-2' },+ { url: 'http://www.example.com/page-3' },+ ]);+ // Create an instance of the CheerioCrawler class - a crawler+ // that automatically loads the URLs and parses their HTML using the cheerio library.+ const crawler = new Apify.CheerioCrawler({+ // Let the crawler fetch URLs from our list.+ requestList,++ // The crawler downloads and processes the web pages in parallel, with a concurrency+ // automatically managed based on the available system memory and CPU (see AutoscaledPool class).+ // Here we define some hard limits for the concurrency.+ minConcurrency: 10,+ maxConcurrency: 50,++ // On error, retry each page at most once.+ maxRequestRetries: 1,++ // Increase the timeout for processing of each page.+ handlePageTimeoutSecs: 30,++ // Limit to 10 requests per one crawl+ maxRequestsPerCrawl: 10,++ // This function will be called for each URL to crawl.+ // It accepts a single parameter, which is an object with options as:+ // https://sdk.apify.com/docs/typedefs/cheerio-crawler-options#handlepagefunction+ // We use for demonstration only 2 of them:+ // - request: an instance of the Request class with information such as URL and HTTP method+ // - $: the cheerio object containing parsed HTML+ handlePageFunction: async ({ request, $ }) => {+ log.debug(`Processing ${request.url}...`);++ // Extract data from the page using cheerio.+ const title = $('title').text();+ const h1texts = [];+ $('h1').each((index, el) => {+ h1texts.push({+ text: $(el).text(),+ });+ });++ // Store the results to the default dataset. In local configuration,+ // the data will be stored as JSON files in ./apify_storage/datasets/default+ await Apify.pushData({+ url: request.url,+ title,+ h1texts,+ });+ },++ // This function is called if the page processing failed more than maxRequestRetries+1 times.+ handleFailedRequestFunction: async ({ request }) => {+ log.debug(`Request ${request.url} failed twice.`);+ },+ });++ // Run the crawler and wait for it to finish.+ await crawler.run();++ log.debug('Crawler finished.');+});+```diff --git a/website/versioned_docs/version-1.0.1/examples/crawl_all_links.md b/website/versioned_docs/version-1.0.1/examples/crawl_all_links.mdnew file mode 100644index 000000000..a8ee5ebc3--- /dev/null+++ b/website/versioned_docs/version-1.0.1/examples/crawl_all_links.md@@ -0,0 +1,116 @@+---+id: version-1.0.1-crawl-all-links+title: Crawl all links on a website+original_id: crawl-all-links+---++This example uses the `Apify.enqueueLinks()` method to add new links to the `RequestQueue` as the crawler navigates from page to page. If only the+required parameters are defined, all links will be crawled.++<!--DOCUSAURUS_CODE_TABS-->++<!-- CheerioCrawler -->++\+Using `CheerioCrawler`:++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Create a RequestQueue+ const requestQueue = await Apify.openRequestQueue();+ // Define the starting URL+ await requestQueue.addRequest({ url: 'https://apify.com/' });+ // Function called for each URL+ const handlePageFunction = async ({ request, $ }) => {+ console.log(request.url);+ // Add all links from page to RequestQueue+ await Apify.utils.enqueueLinks({+ $,+ requestQueue,+ baseUrl: request.loadedUrl,+ });+ };+ // Create a CheerioCrawler+ const crawler = new Apify.CheerioCrawler({+ requestQueue,+ handlePageFunction,+ maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)+ });+ // Run the crawler+ await crawler.run();+});+```++<!-- PuppeteerCrawler -->++\+Using `PuppeteerCrawler`:++> To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Create a RequestQueue+ const requestQueue = await Apify.openRequestQueue();+ // Define the starting URL+ await requestQueue.addRequest({ url: 'https://apify.com/' });+ // Function called for each URL+ const handlePageFunction = async ({ request, page }) => {+ console.log(request.url);+ // Add all links from page to RequestQueue+ await Apify.utils.enqueueLinks({+ page,+ requestQueue,+ });+ };+ // Create a PuppeteerCrawler+ const crawler = new Apify.PuppeteerCrawler({+ requestQueue,+ handlePageFunction,+ maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)+ });+ // Run the crawler+ await crawler.run();+});+```++<!-- PlaywrightCrawler -->++\+Using `PlaywrightCrawler`:++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Create a RequestQueue+ const requestQueue = await Apify.openRequestQueue();+ // Define the starting URL+ await requestQueue.addRequest({ url: 'https://apify.com/' });+ // Function called for each URL+ const handlePageFunction = async ({ request, page }) => {+ console.log(request.url);+ // Add all links from page to RequestQueue+ await Apify.utils.enqueueLinks({+ page,+ requestQueue,+ });+ };+ // Create a PuppeteerCrawler+ const crawler = new Apify.PlaywrightCrawler({+ requestQueue,+ handlePageFunction,+ maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)+ });+ // Run the crawler+ await crawler.run();+});+```++> To run this example on the Apify Platform, select the `apify/actor-node-playwright-chrome` image for your Dockerfile.++<!--END_DOCUSAURUS_CODE_TABS-->diff --git a/website/versioned_docs/version-1.0.1/examples/crawl_multiple_urls.md b/website/versioned_docs/version-1.0.1/examples/crawl_multiple_urls.mdnew file mode 100644index 000000000..baac7947c--- /dev/null+++ b/website/versioned_docs/version-1.0.1/examples/crawl_multiple_urls.md@@ -0,0 +1,105 @@+---+id: version-1.0.1-crawl-multiple-urls+title: Crawl multiple URLs+original_id: crawl-multiple-urls+---++This example crawls the specified list of URLs.++<!--DOCUSAURUS_CODE_TABS-->++<!-- CheerioCrawler -->++\+Using `CheerioCrawler`:++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Create a RequestList+ const requestList = await Apify.openRequestList('start-urls', [+ { url: 'http://www.example.com/page-1' },+ { url: 'http://www.example.com/page-2' },+ { url: 'http://www.example.com/page-3' },+ ]);+ // Function called for each URL+ const handlePageFunction = async ({ request, $ }) => {+ const title = $('title').text();+ console.log(`URL: ${request.url}\nTITLE: ${title}`);+ };+ // Create a CheerioCrawler+ const crawler = new Apify.CheerioCrawler({+ requestList,+ handlePageFunction,+ });+ // Run the crawler+ await crawler.run();+});+```++<!-- PuppeteerCrawler -->++\+Using `PuppeteerCrawler`:++> To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Create a RequestList+ const requestList = await Apify.openRequestList('start-urls', [+ { url: 'http://www.example.com/page-1' },+ { url: 'http://www.example.com/page-2' },+ { url: 'http://www.example.com/page-3' },+ ]);+ // Function called for each URL+ const handlePageFunction = async ({ request, page }) => {+ const title = await page.title();+ console.log(`URL: ${request.url}\nTITLE: ${title}`);+ };+ // Create a PuppeteerCrawler+ const crawler = new Apify.PuppeteerCrawler({+ requestList,+ handlePageFunction,+ });+ // Run the crawler+ await crawler.run();+});+```++<!-- PlaywrightCrawler -->++\+Using `PlaywrightCrawler`:++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Create a RequestList+ const requestList = await Apify.openRequestList('start-urls', [+ { url: 'http://www.example.com/page-1' },+ { url: 'http://www.example.com/page-2' },+ { url: 'http://www.example.com/page-3' },+ ]);+ // Function called for each URL+ const handlePageFunction = async ({ request, page }) => {+ const title = await page.title();+ console.log(`URL: ${request.url}\nTITLE: ${title}`);+ };+ // Create a PuppeteerCrawler+ const crawler = new Apify.PlaywrightCrawler({+ requestList,+ handlePageFunction,+ });+ // Run the crawler+ await crawler.run();+});+```++> To run this example on the Apify Platform, select the `apify/actor-node-playwright-chrome` image for your Dockerfile.++<!--END_DOCUSAURUS_CODE_TABS-->diff --git a/website/versioned_docs/version-1.0.1/examples/crawl_sitemap.md b/website/versioned_docs/version-1.0.1/examples/crawl_sitemap.mdnew file mode 100644index 000000000..9da10c021--- /dev/null+++ b/website/versioned_docs/version-1.0.1/examples/crawl_sitemap.md@@ -0,0 +1,105 @@+---+id: version-1.0.1-crawl-sitemap+title: Crawl a sitemap+original_id: crawl-sitemap+---++This example downloads and crawls the URLs from a sitemap.++<!--DOCUSAURUS_CODE_TABS-->++<!-- CheerioCrawler -->++\+Using `CheerioCrawler`:++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Add URLs to a RequestList from a sitemap+ const sources = [{ requestsFromUrl: 'https://apify.com/sitemap.xml' }];+ const requestList = await Apify.openRequestList('start-urls', sources);++ // Function called for each URL+ const handlePageFunction = async ({ request }) => {+ console.log(request.url);+ };++ // Create a crawler that uses Cheerio+ const crawler = new Apify.CheerioCrawler({+ requestList,+ handlePageFunction,+ maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl a sitemap)+ });++ // Run the crawler+ await crawler.run();+});+```++<!-- PuppeteerCrawler -->++\+Using `PuppeteerCrawler`:++> To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Add URLs to a RequestList from a sitemap+ const sources = [{ requestsFromUrl: 'https://apify.com/sitemap.xml' }];+ const requestList = await Apify.openRequestList('start-urls', sources);++ // Function called for each URL+ const handlePageFunction = async ({ request }) => {+ console.log(request.url);+ };++ // Create a crawler that runs Puppeteer+ const crawler = new Apify.PuppeteerCrawler({+ requestList,+ handlePageFunction,+ maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl a sitemap)+ });++ // Run the crawler+ await crawler.run();+});+```++<!-- PlaywrightCrawler -->++\+Using `PlaywrightCrawler`:++> To run this example on the Apify Platform, select the `apify/actor-node-playwright-chrome` image for your Dockerfile.++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Add URLs to a RequestList from a sitemap+ const sources = [{ requestsFromUrl: 'https://apify.com/sitemap.xml' }];+ const requestList = await Apify.openRequestList('start-urls', sources);++ // Function called for each URL+ const handlePageFunction = async ({ request }) => {+ console.log(request.url);+ };++ // Create a crawler that runs Playwright+ const crawler = new Apify.PlaywrightCrawler({+ requestList,+ handlePageFunction,+ maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl a sitemap)+ });++ // Run the crawler+ await crawler.run();+});+```++<!--END_DOCUSAURUS_CODE_TABS-->diff --git a/website/versioned_docs/version-1.0.1/examples/forms.md b/website/versioned_docs/version-1.0.1/examples/forms.mdnew file mode 100644index 000000000..cdf1d147f--- /dev/null+++ b/website/versioned_docs/version-1.0.1/examples/forms.md@@ -0,0 +1,52 @@+---+id: version-1.0.1-forms+title: Forms+original_id: forms+---++This example demonstrates how to use [`PuppeteerCrawler`](/docs/api/puppeteer-crawler) to automatically fill and submit a search form to look up+repositories on [GitHub](https://github.com) using headless Chrome / Puppeteer. The actor first fills in the search term, repository owner, start date+and language of the repository, then submits the form and prints out the results. Finally, the results are saved either on the Apify platform to the+default [`dataset`](/docs/api/dataset) or on the local machine as JSON files in `./apify_storage/datasets/default`.++> To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Launch the web browser.+ const browser = await Apify.launchPuppeteer();++ // Create and navigate new page+ console.log('Open target page');+ const page = await browser.newPage();+ await page.goto('https://github.com/search/advanced');++ // Fill form fields and select desired search options+ console.log('Fill in search form');+ await page.type('#adv_code_search input.js-advanced-search-input', 'apify-js');+ await page.type('#search_from', 'apify');+ await page.type('#search_date', '>2015');+ await page.select('select#search_language', 'JavaScript');++ // Submit the form and wait for full load of next page+ console.log('Submit search form');+ await Promise.all([page.waitForNavigation(), page.click('#adv_code_search button[type="submit"]')]);++ // Obtain and print list of search results+ const results = await page.$$eval('div.f4.text-normal a', nodes =>+ nodes.map(node => ({+ url: node.href,+ name: node.innerText,+ })),+ );+ console.log('Results:', results);++ // Store data in default dataset+ await Apify.pushData(results);++ // Close browser+ await browser.close();+});+```diff --git a/website/versioned_docs/version-1.0.1/examples/playwright_crawler.md b/website/versioned_docs/version-1.0.1/examples/playwright_crawler.mdnew file mode 100644index 000000000..8b4996b51--- /dev/null+++ b/website/versioned_docs/version-1.0.1/examples/playwright_crawler.md@@ -0,0 +1,88 @@+---+id: version-1.0.1-playwright-crawler+title: Playwright crawler+original_id: playwright-crawler+---++This example demonstrates how to use [`PlaywrightCrawler`](../api/playwright-crawler) in combination with [`RequestQueue`](../api/request-queue) to+recursively scrape the [Hacker News website](https://news.ycombinator.com) using headless Chrome / Playwright.++The crawler starts with a single URL, finds links to next pages, enqueues them and continues until no more desired links are available. The results+are stored to the default dataset. In local configuration, the results are stored as JSON files in `./apify_storage/datasets/default`++> To run this example on the Apify Platform, select the `apify/actor-node-playwright-chrome` image for your Dockerfile.++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Apify.openRequestQueue() creates a preconfigured RequestQueue instance.+ // We add our first request to it - the initial page the crawler will visit.+ const requestQueue = await Apify.openRequestQueue();+ await requestQueue.addRequest({ url: 'https://news.ycombinator.com/' });++ // Create an instance of the PlaywrightCrawler class - a crawler+ // that automatically loads the URLs in headless Chrome / Playwright.+ const crawler = new Apify.PlaywrightCrawler({+ requestQueue,+ launchContext: {+ // Here you can set options that are passed to the playwright .launch() function.+ launchOptions: {+ headless: true,+ },+ },++ // Stop crawling after several pages+ maxRequestsPerCrawl: 50,++ // This function will be called for each URL to crawl.+ // Here you can write the Playwright scripts you are familiar with,+ // with the exception that browsers and pages are automatically managed by the Apify SDK.+ // The function accepts a single parameter, which is an object with a lot of properties,+ // the most important being:+ // - request: an instance of the Request class with information such as URL and HTTP method+ // - page: Playwright's Page object (see https://playwright.dev/docs/api/class-page)+ handlePageFunction: async ({ request, page }) => {+ console.log(`Processing ${request.url}...`);++ // A function to be evaluated by Playwright within the browser context.+ const data = await page.$$eval('.athing', $posts => {+ const scrapedData = [];++ // We're getting the title, rank and URL of each post on Hacker News.+ $posts.forEach($post => {+ scrapedData.push({+ title: $post.querySelector('.title a').innerText,+ rank: $post.querySelector('.rank').innerText,+ href: $post.querySelector('.title a').href,+ });+ });++ return scrapedData;+ });++ // Store the results to the default dataset.+ await Apify.pushData(data);++ // Find a link to the next page and enqueue it if it exists.+ const infos = await Apify.utils.enqueueLinks({+ page,+ requestQueue,+ selector: '.morelink',+ });++ if (infos.length === 0) console.log(`${request.url} is the last page!`);+ },++ // This function is called if the page processing failed more than maxRequestRetries+1 times.+ handleFailedRequestFunction: async ({ request }) => {+ console.log(`Request ${request.url} failed too many times.`);+ },+ });++ // Run the crawler and wait for it to finish.+ await crawler.run();++ console.log('Crawler finished.');+});+```diff --git a/website/versioned_docs/version-1.0.1/examples/puppeteer_crawler.md b/website/versioned_docs/version-1.0.1/examples/puppeteer_crawler.mdnew file mode 100644index 000000000..89fc1026f--- /dev/null+++ b/website/versioned_docs/version-1.0.1/examples/puppeteer_crawler.md@@ -0,0 +1,88 @@+---+id: version-1.0.1-puppeteer-crawler+title: Puppeteer crawler+original_id: puppeteer-crawler+---++This example demonstrates how to use [`PuppeteerCrawler`](/docs/api/puppeteer-crawler) in combination with [`RequestQueue`](/docs/api/request-queue)+to recursively scrape the [Hacker News website](https://news.ycombinator.com) using headless Chrome / Puppeteer. The crawler starts with a single URL,+finds links to next pages, enqueues them and continues until no more desired links are available. The results are stored to the default dataset. In+local configuration, the results are stored as JSON files in `./apify_storage/datasets/default`++> To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Apify.openRequestQueue() creates a preconfigured RequestQueue instance.+ // We add our first request to it - the initial page the crawler will visit.+ const requestQueue = await Apify.openRequestQueue();+ await requestQueue.addRequest({ url: 'https://news.ycombinator.com/' });++ // Create an instance of the PuppeteerCrawler class - a crawler+ // that automatically loads the URLs in headless Chrome / Puppeteer.+ const crawler = new Apify.PuppeteerCrawler({+ requestQueue,++ // Here you can set options that are passed to the Apify.launchPuppeteer() function.+ launchContext: {+ launchOptions: {+ headless: true,+ // Other Puppeteer options+ },+ },++ // Stop crawling after several pages+ maxRequestsPerCrawl: 50,++ // This function will be called for each URL to crawl.+ // Here you can write the Puppeteer scripts you are familiar with,+ // with the exception that browsers and pages are automatically managed by the Apify SDK.+ // The function accepts a single parameter, which is an object with the following fields:+ // - request: an instance of the Request class with information such as URL and HTTP method+ // - page: Puppeteer's Page object (see https://pptr.dev/#show=api-class-page)+ handlePageFunction: async ({ request, page }) => {+ console.log(`Processing ${request.url}...`);++ // A function to be evaluated by Puppeteer within the browser context.+ const data = await page.$$eval('.athing', $posts => {+ const scrapedData = [];++ // We're getting the title, rank and URL of each post on Hacker News.+ $posts.forEach($post => {+ scrapedData.push({+ title: $post.querySelector('.title a').innerText,+ rank: $post.querySelector('.rank').innerText,+ href: $post.querySelector('.title a').href,+ });+ });++ return scrapedData;+ });++ // Store the results to the default dataset.+ await Apify.pushData(data);++ // Find a link to the next page and enqueue it if it exists.+ const infos = await Apify.utils.enqueueLinks({+ page,+ requestQueue,+ selector: '.morelink',+ });++ if (infos.length === 0) console.log(`${request.url} is the last page!`);+ },++ // This function is called if the page processing failed more than maxRequestRetries+1 times.+ handleFailedRequestFunction: async ({ request }) => {+ console.log(`Request ${request.url} failed too many times.`);+ },+ });++ // Run the crawler and wait for it to finish.+ await crawler.run();++ console.log('Crawler finished.');+});+```diff --git a/website/versioned_docs/version-1.0.1/examples/puppeteer_recursive_crawl.md b/website/versioned_docs/version-1.0.1/examples/puppeteer_recursive_crawl.mdnew file mode 100644index 000000000..f379a6a90--- /dev/null+++ b/website/versioned_docs/version-1.0.1/examples/puppeteer_recursive_crawl.md@@ -0,0 +1,34 @@+---+id: version-1.0.1-puppeteer-recursive-crawl+title: Puppeteer recursive crawl+original_id: puppeteer-recursive-crawl+---++Run the following example to perform a recursive crawl of a website using [`PuppeteerCrawler`](/docs/api/puppeteer-crawler).++> To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.++```javascript+const Apify = require('apify');++Apify.main(async () => {+ const requestQueue = await Apify.openRequestQueue();+ await requestQueue.addRequest({ url: 'https://www.iana.org/' });++ const crawler = new Apify.PuppeteerCrawler({+ requestQueue,+ handlePageFunction: async ({ request, page }) => {+ const title = await page.title();+ console.log(`Title of ${request.url}: ${title}`);+ await Apify.utils.enqueueLinks({+ page,+ requestQueue,+ pseudoUrls: ['https://www.iana.org/[.*]'],+ });+ },+ maxRequestsPerCrawl: 10,+ });++ await crawler.run();+});+```diff --git a/website/versioned_docs/version-1.0.1/examples/puppeteer_with_proxy.md b/website/versioned_docs/version-1.0.1/examples/puppeteer_with_proxy.mdnew file mode 100644index 000000000..92843b496--- /dev/null+++ b/website/versioned_docs/version-1.0.1/examples/puppeteer_with_proxy.md@@ -0,0 +1,35 @@+---+id: version-1.0.1-puppeteer-with-proxy+title: Puppeteer with proxy+original_id: puppeteer-with-proxy+---++This example demonstrates how to load pages in headless Chrome / Puppeteer over [Apify Proxy](https://docs.apify.com/proxy). To make it work, you'll+need an Apify account with access to the proxy. Visit the [Apify platform introduction](/docs/guides/apify-platform) to find how to log into your+account from the SDK.++> To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.++```javascript+const Apify = require('apify');++Apify.main(async () => {+ const requestList = await Apify.openRequestList('start-urls', ['http://proxy.apify.com']);++ // Proxy connection is automatically established in the Crawler+ const proxyConfiguration = await Apify.createProxyConfiguration();++ const crawler = new Apify.PuppeteerCrawler({+ requestList,+ proxyConfiguration,+ handlePageFunction: async ({ page }) => {+ const status = await page.$eval('td.status', el => el.textContent);+ console.log(`Proxy Status: ${status}`);+ },+ });++ console.log('Running Puppeteer script...');+ await crawler.run();+ console.log('Puppeteer closed.');+});+```diff --git a/website/versioned_docs/version-1.0.1/examples/synchronous_run.md b/website/versioned_docs/version-1.0.1/examples/synchronous_run.mdnew file mode 100644index 000000000..dc8b81f15--- /dev/null+++ b/website/versioned_docs/version-1.0.1/examples/synchronous_run.md@@ -0,0 +1,38 @@+---+id: version-1.0.1-synchronous-run+title: Synchronous run+original_id: synchronous-run+---++This example shows a quick actor that has a run time of just a few seconds. It opens a [web page](https://en.wikipedia.org) (the Wikipedia home page),+which contains a list of "Did you know" texts that change daily. The actor scrapes all the "Did you know" items and saves them to the default dataset.++This actor can be invoked synchronously using a single HTTP request to directly obtain its output as a response, using the+[Run actor synchronously](https://apify.com/docs/api/v2#/reference/actors/run-actor-synchronously/without-input) Apify API endpoint.++> To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.++```javascript+const Apify = require('apify');++Apify.main(async () => {+ // Launch web browser.+ const browser = await Apify.launchPuppeteer({ launchOptions: { headless: true } });+ // Load https://en.wikipedia.org and get all "Did you know" texts.+ console.log('Opening web page...');+ const page = await browser.newPage();+ await page.goto('https://en.wikipedia.org');++ // Get all "Did you know" items from the page.+ console.log('Getting "Did you know" items from the page.');+ const results = await page.$$eval('div#mp-dyk > ul li', nodes => nodes.map(node => node.innerText.replace('...', 'Did you know')));+ console.log(results);++ // Save all the items to the Apify dataSet.+ await Apify.pushData(results);+ console.log('Actor finished.');++ // Close browser+ await browser.close();+});+```diff --git a/website/versioned_docs/version-1.0.1/examples/use_stealth_mode.md b/website/versioned_docs/version-1.0.1/examples/use_stealth_mode.mdnew file mode 100644index 000000000..8742e7e96--- /dev/null+++ b/website/versioned_docs/version-1.0.1/examples/use_stealth_mode.md@@ -0,0 +1,72 @@+---+id: version-1.0.1-use-stealth-mode+title: use-stealth-mode+original_id: use-stealth-mode+---++Stealth mode allows you to bypass anti-scraping techniques which use+[browser fingerprinting](https://pixelprivacy.com/resources/browser-fingerprinting/). It overrides the attributes specified for+[headless](https://developers.google.com/web/updates/2017/04/headless-chrome) browser mode, making your headless browser harder to distinguish from+the full Chrome browser.++To activate stealth mode, you need to `useChrome`, run `headless` and turn `stealth` on in your+[`launchContext`](https://sdk.apify.com/docs/typedefs/puppeteer-crawler-options#launchcontext).++```js+const launchContext = {+ useChrome: true,+ stealth: true,+ launchOptions: {+ headless: true,+ },+};+```++```javascript+const Apify = require('apify');++Apify.main(async () => {+ const requestList = await Apify.openRequestList('start-urls', ['https://news.ycombinator.com/']);++ const crawler = new Apify.PuppeteerCrawler({+ requestList,+ launchContext: {+ useChrome: true,+ stealth: true,+ launchOptions: {+ headless: true,+ },+ // You can override default stealth options+ // stealthOptions: {+ // addLanguage: false,+ // },+ },+ handlePageFunction: async ({ page }) => {+ const data = await page.$$eval('.athing', $posts => {+ const scrapedData = [];+ // Get the title of each post on Hacker News+ $posts.forEach($post => {+ const title = $post.querySelector('.title a').innerText;+ scrapedData.push({+ title: `The title is: ${title}`,+ });+ });+ return scrapedData;+ });+ // Save the data array to the Apify dataSet+ await Apify.pushData(data);+ },+ });+ await crawler.run();+});+```++You can then specify the [`stealthOptions`](https://sdk.apify.com/docs/typedefs/stealth-options), which allow you to adapt to different anti-scraping+techniques. All the options are set to `true` by default. The number of options does not affect performance.++While the default configuration will be fine in many cases, you can adapt the options to your use case.++### Single-browser instances++You can also use stealth mode in single-browser instances when using [`Apify.launchPuppeteer`](https://sdk.apify.com/docs/api/apify#launchpuppeteer).+The `launchContext` is the same.diff --git a/website/versioned_sidebars/version-1.0.1-sidebars.json b/website/versioned_sidebars/version-1.0.1-sidebars.jsonnew file mode 100644index 000000000..6e3881ddf--- /dev/null+++ b/website/versioned_sidebars/version-1.0.1-sidebars.json@@ -0,0 +1,183 @@+{+ "version-1.0.1-docs": {+ "Guide": [+ "version-1.0.1-guides/motivation",+ "version-1.0.1-guides/quick-start",+ "version-1.0.1-guides/apify-platform",+ "version-1.0.1-guides/getting-started",+ "version-1.0.1-guides/request-storage",+ "version-1.0.1-guides/result-storage",+ "version-1.0.1-guides/environment-variables",+ "version-1.0.1-guides/proxy-management",+ "version-1.0.1-guides/session-management",+ "version-1.0.1-guides/type-script-actor",+ "version-1.0.1-guides/docker-images"+ ],+ "Examples": [+ "version-1.0.1-examples/accept-user-input",+ "version-1.0.1-examples/add-data-to-dataset",+ "version-1.0.1-examples/basic-crawler",+ "version-1.0.1-examples/call-actor",+ "version-1.0.1-examples/capture-screenshot",+ "version-1.0.1-examples/cheerio-crawler",+ "version-1.0.1-examples/crawl-all-links",+ "version-1.0.1-examples/crawl-multiple-urls",+ "version-1.0.1-examples/crawl-relative-links",+ "version-1.0.1-examples/crawl-single-url",+ "version-1.0.1-examples/crawl-sitemap",+ "version-1.0.1-examples/crawl-some-links",+ "version-1.0.1-examples/forms",+ "version-1.0.1-examples/map-and-reduce",+ "version-1.0.1-examples/playwright-crawler",+ "version-1.0.1-examples/puppeteer-crawler",+ "version-1.0.1-examples/puppeteer-recursive-crawl",+ "version-1.0.1-examples/puppeteer-with-proxy",+ "version-1.0.1-examples/synchronous-run",+ "version-1.0.1-examples/use-stealth-mode"+ ],+ "API Reference": [+ "version-1.0.1-api/apify",+ {+ "type": "subcategory",+ "label": "Crawlers",+ "ids": [+ "version-1.0.1-api/basic-crawler",+ "version-1.0.1-api/cheerio-crawler",+ "version-1.0.1-api/playwright-crawler",+ "version-1.0.1-api/puppeteer-crawler"+ ]+ },+ {+ "type": "subcategory",+ "label": "Result Stores",+ "ids": [+ "version-1.0.1-api/dataset",+ "version-1.0.1-api/key-value-store"+ ]+ },+ {+ "type": "subcategory",+ "label": "Scaling",+ "ids": [+ "version-1.0.1-api/autoscaled-pool",+ "version-1.0.1-api/session",+ "version-1.0.1-api/session-pool",+ "version-1.0.1-api/proxy-configuration",+ "version-1.0.1-api/snapshotter",+ "version-1.0.1-api/system-status"+ ]+ },+ {+ "type": "subcategory",+ "label": "Sources",+ "ids": [+ "version-1.0.1-api/request",+ "version-1.0.1-api/request-list",+ "version-1.0.1-api/request-queue",+ "version-1.0.1-api/pseudo-url"+ ]+ },+ {+ "type": "subcategory",+ "label": "Utilities",+ "ids": [+ "version-1.0.1-api/utils",+ "version-1.0.1-api/log",+ "version-1.0.1-api/playwright",+ "version-1.0.1-api/puppeteer",+ "version-1.0.1-api/social"+ ]+ }+ ],+ "Type Definitions": [+ {+ "type": "subcategory",+ "label": "Constructor Options",+ "ids": [+ "version-1.0.1-typedefs/autoscaled-pool-options",+ "version-1.0.1-typedefs/basic-crawler-options",+ "version-1.0.1-typedefs/cheerio-crawler-options",+ "version-1.0.1-typedefs/playwright-crawler-options",+ "version-1.0.1-typedefs/playwright-launch-context",+ "version-1.0.1-typedefs/puppeteer-crawler-options",+ "version-1.0.1-typedefs/puppeteer-launch-context",+ "version-1.0.1-typedefs/proxy-configuration-options",+ "version-1.0.1-typedefs/proxy-info",+ "version-1.0.1-typedefs/logger-options",+ "version-1.0.1-typedefs/request-options",+ "version-1.0.1-typedefs/request-list-options",+ "version-1.0.1-typedefs/request-as-browser-options",+ "version-1.0.1-typedefs/session-options",+ "version-1.0.1-typedefs/session-pool-options",+ "version-1.0.1-typedefs/snapshotter-options",+ "version-1.0.1-typedefs/stealth-options",+ "version-1.0.1-typedefs/system-status-options"+ ]+ },+ {+ "type": "subcategory",+ "label": "Functions - Crawlers",+ "ids": [+ "version-1.0.1-typedefs/cheerio-handle-page",+ "version-1.0.1-typedefs/cheerio-handle-page-inputs",+ "version-1.0.1-typedefs/handle-failed-request",+ "version-1.0.1-typedefs/handle-failed-request-input",+ "version-1.0.1-typedefs/handle-request",+ "version-1.0.1-typedefs/handle-request-inputs",+ "version-1.0.1-typedefs/prepare-request",+ "version-1.0.1-typedefs/prepare-request-inputs",+ "version-1.0.1-typedefs/playwright-handle-page",+ "version-1.0.1-typedefs/playwright-handle-page-inputs",+ "version-1.0.1-typedefs/puppeteer-handle-page",+ "version-1.0.1-typedefs/puppeteer-handle-page-inputs"+ ]+ },+ {+ "type": "subcategory",+ "label": "Functions - Dataset",+ "ids": [+ "version-1.0.1-typedefs/dataset-consumer",+ "version-1.0.1-typedefs/dataset-mapper",+ "version-1.0.1-typedefs/dataset-reducer"+ ]+ },+ {+ "type": "subcategory",+ "label": "Functions - KeyValueStore",+ "ids": [+ "version-1.0.1-typedefs/key-consumer"+ ]+ },+ {+ "type": "subcategory",+ "label": "Functions - Sessions",+ "ids": [+ "version-1.0.1-typedefs/create-session"+ ]+ },+ {+ "type": "subcategory",+ "label": "Functions - Utilities",+ "ids": [+ "version-1.0.1-typedefs/request-transform"+ ]+ },+ {+ "type": "subcategory",+ "label": "Return Types",+ "ids": [+ "version-1.0.1-typedefs/actor-run",+ "version-1.0.1-api/apify-call-error",+ "version-1.0.1-typedefs/apify-env",+ "version-1.0.1-typedefs/dataset-content",+ "version-1.0.1-typedefs/memory-info",+ "version-1.0.1-typedefs/queue-operation-info",+ "version-1.0.1-typedefs/request-list-state",+ "version-1.0.1-typedefs/session-state",+ "version-1.0.1-typedefs/social-handles",+ "version-1.0.1-typedefs/system-info"+ ]+ }+ ]+ }+}