Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
docs: update examples (#1406)
Martin Adámek0c3e025f66e7
diff --git a/docs/examples/.eslintrc.json b/docs/examples/.eslintrc.jsonnew file mode 100644index 000000000..98504f200--- /dev/null+++ b/docs/examples/.eslintrc.json@@ -0,0 +1,12 @@+{+ "root": true,+ "extends": "@apify",+ "parserOptions": {+ "ecmaVersion": 2022+ },+ "rules": {+ "import/extensions": 0,+ "import/no-extraneous-dependencies": 0,+ "no-console": "off"+ }+}diff --git a/docs/examples/accept_user_input.ts b/docs/examples/accept_user_input.tsindex cedb6e938..25d4b48dd 100644--- a/docs/examples/accept_user_input.ts+++ b/docs/examples/accept_user_input.ts@@ -1,6 +1,4 @@import { KeyValueStore } from 'crawlee';-const keyValueStore = await KeyValueStore.open();-const input = await keyValueStore.getValue('INPUT');-+const input = await KeyValueStore.getInput();console.log(input);diff --git a/docs/examples/add_data_to_dataset.ts b/docs/examples/add_data_to_dataset.tsindex 27e978013..d57a0186e 100644--- a/docs/examples/add_data_to_dataset.ts+++ b/docs/examples/add_data_to_dataset.ts@@ -1,13 +1,10 @@import { Dataset, CheerioCrawler } from 'crawlee';-// Create a dataset where we will store the results.-const dataset = await Dataset.open();-const crawler = new CheerioCrawler({// Function called for each URLasync requestHandler({ request, body }) {// Save data to default dataset- await dataset.pushData({+ await Dataset.pushData({url: request.url,html: body,});@@ -15,9 +12,9 @@ const crawler = new CheerioCrawler({});await crawler.addRequests([- { url: 'http://www.example.com/page-1' },- { url: 'http://www.example.com/page-2' },- { url: 'http://www.example.com/page-3' },+ 'http://www.example.com/page-1',+ 'http://www.example.com/page-2',+ 'http://www.example.com/page-3',]);// Run the crawlerdiff --git a/docs/examples/basic_crawler.ts b/docs/examples/basic_crawler.tsindex e3c01383b..f1e4584a4 100644--- a/docs/examples/basic_crawler.ts+++ b/docs/examples/basic_crawler.ts@@ -1,15 +1,12 @@import { BasicCrawler, Dataset } from 'crawlee';-// Create a dataset where we will store the results.-const dataset = await Dataset.open();-// Create a BasicCrawler - the simplest crawler that enables// users to implement the crawling logic themselves.const crawler = new BasicCrawler({// This function will be called for each URL to crawl.- async requestHandler({ request, sendRequest }) {+ async requestHandler({ request, sendRequest, log }) {const { url } = request;- console.log(`Processing ${url}...`);+ log.info(`Processing ${url}...`);// Fetch the page HTML via the crawlee sendRequest utility method// By default, the method will use the current request that is being handled, so you don't have to@@ -17,7 +14,7 @@ const crawler = new BasicCrawler({const { body } = await sendRequest();// Store the HTML and URL to the default dataset.- await dataset.pushData({+ await Dataset.pushData({url,html: body,});@@ -26,10 +23,10 @@ const crawler = new BasicCrawler({// The initial list of URLs to crawl. Here we use just a few hard-coded URLs.await crawler.addRequests([- { url: 'http://www.google.com/' },- { url: 'http://www.example.com/' },- { url: 'http://www.bing.com/' },- { url: 'http://www.wikipedia.com/' },+ 'https://www.google.com',+ 'https://www.example.com',+ 'https://www.bing.com',+ 'https://www.wikipedia.com',]);// Run the crawler and wait for it to finish.diff --git a/docs/examples/cheerio_crawler.ts b/docs/examples/cheerio_crawler.tsindex 84b98e065..21fa67acf 100644--- a/docs/examples/cheerio_crawler.ts+++ b/docs/examples/cheerio_crawler.ts@@ -1,7 +1,5 @@import { Dataset, CheerioCrawler, log, LogLevel } from 'crawlee';-const dataset = await Dataset.open();-// Crawlers come with various utilities, e.g. for logging.// Here we use debug level of logging to improve the debugging experience.// This functionality is optional!@@ -27,7 +25,7 @@ const crawler = new CheerioCrawler({// This function will be called for each URL to crawl.// It accepts a single parameter, which is an object with options as:- // https://sdk.apify.com/docs/typedefs/cheerio-crawler-options#handlepagefunction+ // https://crawlee.dev/api/cheerio-crawler/interface/CheerioCrawlerOptions#requestHandler// We use for demonstration only 2 of them:// - request: an instance of the Request class with information such as the URL that is being crawled and HTTP method// - $: the cheerio object containing parsed HTML@@ -36,7 +34,7 @@ const crawler = new CheerioCrawler({// Extract data from the page using cheerio.const title = $('title').text();- const h1texts = [];+ const h1texts: { text: string }[] = [];$('h1').each((index, el) => {h1texts.push({text: $(el).text(),@@ -45,7 +43,7 @@ const crawler = new CheerioCrawler({// Store the results to the dataset. In local configuration,// the data will be stored as JSON files in ./storage/datasets/default- await dataset.pushData({+ await Dataset.pushData({url: request.url,title,h1texts,diff --git a/docs/examples/crawl_all_links_cheerio.ts b/docs/examples/crawl_all_links_cheerio.tsindex 9e62d1a35..e3c930bc2 100644--- a/docs/examples/crawl_all_links_cheerio.ts+++ b/docs/examples/crawl_all_links_cheerio.ts@@ -1,15 +1,13 @@import { CheerioCrawler } from 'crawlee';const crawler = new CheerioCrawler({- async requestHandler({ request, enqueueLinks }) {- console.log(request.url);+ async requestHandler({ request, enqueueLinks, log }) {+ log.info(request.url);// Add all links from page to RequestQueueawait enqueueLinks();},maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)});-await crawler.addRequests(['https://apify.com/']);--// Run the crawler-await crawler.run();+// Run the crawler with initial request+await crawler.run(['https://crawlee.dev']);diff --git a/docs/examples/crawl_all_links_playwright.ts b/docs/examples/crawl_all_links_playwright.tsindex 778c67640..bf0440dee 100644--- a/docs/examples/crawl_all_links_playwright.ts+++ b/docs/examples/crawl_all_links_playwright.ts@@ -1,15 +1,13 @@import { PlaywrightCrawler } from 'crawlee';const crawler = new PlaywrightCrawler({- async requestHandler({ request, enqueueLinks }) {- console.log(request.url);+ async requestHandler({ request, enqueueLinks, log }) {+ log.info(request.url);// Add all links from page to RequestQueueawait enqueueLinks();},maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)});-await crawler.addRequests(['https://apify.com/']);--// Run the crawler-await crawler.run();+// Run the crawler with initial request+await crawler.run(['https://crawlee.dev']);diff --git a/docs/examples/crawl_all_links_puppeteer.ts b/docs/examples/crawl_all_links_puppeteer.tsindex e3ac8dec2..cf6fcb55c 100644--- a/docs/examples/crawl_all_links_puppeteer.ts+++ b/docs/examples/crawl_all_links_puppeteer.ts@@ -1,15 +1,13 @@import { PuppeteerCrawler } from 'crawlee';const crawler = new PuppeteerCrawler({- async requestHandler({ request, enqueueLinks }) {- console.log(request.url);+ async requestHandler({ request, enqueueLinks, log }) {+ log.info(request.url);// Add all links from page to RequestQueueawait enqueueLinks();},maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)});-await crawler.addRequests(['https://apify.com/']);--// Run the crawler-await crawler.run();+// Run the crawler with initial request+await crawler.run(['https://crawlee.dev']);diff --git a/docs/examples/crawl_multiple_urls_cheerio.ts b/docs/examples/crawl_multiple_urls_cheerio.tsindex 0241e040e..011d8b7d6 100644--- a/docs/examples/crawl_multiple_urls_cheerio.ts+++ b/docs/examples/crawl_multiple_urls_cheerio.ts@@ -2,17 +2,15 @@ import { CheerioCrawler } from 'crawlee';const crawler = new CheerioCrawler({// Function called for each URL- async requestHandler({ request, $ }) {+ async requestHandler({ request, $, log }) {const title = $('title').text();- console.log(`URL: ${request.url}\nTITLE: ${title}`);+ log.info(`URL: ${request.url}\nTITLE: ${title}`);},});-await crawler.addRequests([+// Run the crawler with initial request+await crawler.run(['http://www.example.com/page-1','http://www.example.com/page-2','http://www.example.com/page-3',]);--// Run the crawler-await crawler.run();diff --git a/docs/examples/crawl_multiple_urls_playwright.ts b/docs/examples/crawl_multiple_urls_playwright.tsindex cd34b5b52..9581bea3f 100644--- a/docs/examples/crawl_multiple_urls_playwright.ts+++ b/docs/examples/crawl_multiple_urls_playwright.ts@@ -2,17 +2,15 @@ import { PlaywrightCrawler } from 'crawlee';const crawler = new PlaywrightCrawler({// Function called for each URL- async requestHandler({ request, page }) {+ async requestHandler({ request, page, log }) {const title = await page.title();- console.log(`URL: ${request.url}\nTITLE: ${title}`);+ log.info(`URL: ${request.url}\nTITLE: ${title}`);},});-await crawler.addRequests([+// Run the crawler with initial request+await crawler.run(['http://www.example.com/page-1','http://www.example.com/page-2','http://www.example.com/page-3',]);--// Run the crawler-await crawler.run();diff --git a/docs/examples/crawl_multiple_urls_puppeteer.ts b/docs/examples/crawl_multiple_urls_puppeteer.tsindex cf027fef5..d7c6a99a5 100644--- a/docs/examples/crawl_multiple_urls_puppeteer.ts+++ b/docs/examples/crawl_multiple_urls_puppeteer.ts@@ -2,17 +2,15 @@ import { PuppeteerCrawler } from 'crawlee';const crawler = new PuppeteerCrawler({// Function called for each URL- async requestHandler({ request, page }) {+ async requestHandler({ request, page, log }) {const title = await page.title();- console.log(`URL: ${request.url}\nTITLE: ${title}`);+ log.info(`URL: ${request.url}\nTITLE: ${title}`);},});-await crawler.addRequests([+// Run the crawler with initial request+await crawler.run(['http://www.example.com/page-1','http://www.example.com/page-2','http://www.example.com/page-3',]);--// Run the crawler-await crawler.run();diff --git a/docs/examples/crawl_relative_links_all.ts b/docs/examples/crawl_relative_links_all.tsindex c9265c20a..e4401cb9f 100644--- a/docs/examples/crawl_relative_links_all.ts+++ b/docs/examples/crawl_relative_links_all.ts@@ -2,8 +2,8 @@ import { CheerioCrawler, EnqueueStrategy } from 'crawlee';const crawler = new CheerioCrawler({maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)- async requestHandler({ request, enqueueLinks }) {- console.log(request.url);+ async requestHandler({ request, enqueueLinks, log }) {+ log.info(request.url);await enqueueLinks({// Setting the strategy to 'all' will enqueue all links found// highlight-next-line@@ -14,7 +14,5 @@ const crawler = new CheerioCrawler({},});-await crawler.addRequests(['https://apify.com/']);--// Run the crawler-await crawler.run();+// Run the crawler with initial request+await crawler.run(['https://crawlee.dev']);diff --git a/docs/examples/crawl_relative_links_same_domain.ts b/docs/examples/crawl_relative_links_same_domain.tsindex be4b9ec61..59c833c05 100644--- a/docs/examples/crawl_relative_links_same_domain.ts+++ b/docs/examples/crawl_relative_links_same_domain.ts@@ -2,8 +2,8 @@ import { CheerioCrawler, EnqueueStrategy } from 'crawlee';const crawler = new CheerioCrawler({maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)- async requestHandler({ request, enqueueLinks }) {- console.log(request.url);+ async requestHandler({ request, enqueueLinks, log }) {+ log.info(request.url);await enqueueLinks({// Setting the strategy to 'same-domain' will enqueue all links found that are on the// same hostname as request.loadedUrl or request.url@@ -15,7 +15,5 @@ const crawler = new CheerioCrawler({},});-await crawler.addRequests(['https://apify.com/']);--// Run the crawler-await crawler.run();+// Run the crawler with initial request+await crawler.run(['https://crawlee.dev']);diff --git a/docs/examples/crawl_relative_links_same_hostname.ts b/docs/examples/crawl_relative_links_same_hostname.tsindex 2fb2bcc8d..052f24dcf 100644--- a/docs/examples/crawl_relative_links_same_hostname.ts+++ b/docs/examples/crawl_relative_links_same_hostname.ts@@ -2,8 +2,8 @@ import { CheerioCrawler, EnqueueStrategy } from 'crawlee';const crawler = new CheerioCrawler({maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)- async requestHandler({ request, enqueueLinks }) {- console.log(request.url);+ async requestHandler({ request, enqueueLinks, log }) {+ log.info(request.url);await enqueueLinks({// Setting the strategy to 'same-hostname' will enqueue all links found that are on the// same hostname (including subdomain) as request.loadedUrl or request.url@@ -15,7 +15,5 @@ const crawler = new CheerioCrawler({},});-await crawler.addRequests(['https://apify.com/']);--// Run the crawler-await crawler.run();+// Run the crawler with initial request+await crawler.run(['https://crawlee.dev']);diff --git a/docs/examples/crawl_sitemap_cheerio.ts b/docs/examples/crawl_sitemap_cheerio.tsindex afde7a6c5..cad832f39 100644--- a/docs/examples/crawl_sitemap_cheerio.ts+++ b/docs/examples/crawl_sitemap_cheerio.ts@@ -2,13 +2,13 @@ import { CheerioCrawler, downloadListOfUrls } from 'crawlee';const crawler = new CheerioCrawler({// Function called for each URL- async requestHandler({ request }) {- console.log(request.url);+ async requestHandler({ request, log }) {+ log.info(request.url);},maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl a sitemap)});-const listOfUrls = await downloadListOfUrls({ url: 'https://apify.com/sitemap.xml' });+const listOfUrls = await downloadListOfUrls({ url: 'https://crawlee.dev/sitemap.xml' });await crawler.addRequests(listOfUrls);diff --git a/docs/examples/crawl_sitemap_playwright.ts b/docs/examples/crawl_sitemap_playwright.tsindex d4948aa08..d829451d1 100644--- a/docs/examples/crawl_sitemap_playwright.ts+++ b/docs/examples/crawl_sitemap_playwright.ts@@ -2,13 +2,13 @@ import { PlaywrightCrawler, downloadListOfUrls } from 'crawlee';const crawler = new PlaywrightCrawler({// Function called for each URL- async requestHandler({ request }) {- console.log(request.url);+ async requestHandler({ request, log }) {+ log.info(request.url);},maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl a sitemap)});-const listOfUrls = await downloadListOfUrls({ url: 'https://apify.com/sitemap.xml' });+const listOfUrls = await downloadListOfUrls({ url: 'https://crawlee.dev/sitemap.xml' });await crawler.addRequests(listOfUrls);diff --git a/docs/examples/crawl_sitemap_puppeteer.ts b/docs/examples/crawl_sitemap_puppeteer.tsindex 488e4e2ae..96122d590 100644--- a/docs/examples/crawl_sitemap_puppeteer.ts+++ b/docs/examples/crawl_sitemap_puppeteer.ts@@ -2,13 +2,13 @@ import { PuppeteerCrawler, downloadListOfUrls } from 'crawlee';const crawler = new PuppeteerCrawler({// Function called for each URL- async requestHandler({ request }) {- console.log(request.url);+ async requestHandler({ request, log }) {+ log.info(request.url);},maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl a sitemap)});-const listOfUrls = await downloadListOfUrls({ url: 'https://apify.com/sitemap.xml' });+const listOfUrls = await downloadListOfUrls({ url: 'https://crawlee.dev/sitemap.xml' });await crawler.addRequests(listOfUrls);diff --git a/docs/examples/crawl_some_links.ts b/docs/examples/crawl_some_links.tsindex ff14af8e0..51912bb10 100644--- a/docs/examples/crawl_some_links.ts+++ b/docs/examples/crawl_some_links.ts@@ -5,17 +5,17 @@ const crawler = new CheerioCrawler({// Limits the crawler to only 10 requests (do not use if you want to crawl all links)maxRequestsPerCrawl: 10,// Function called for each URL- async requestHandler({ request, enqueueLinks }) {- console.log(request.url);+ async requestHandler({ request, enqueueLinks, log }) {+ log.info(request.url);// Add some links from page to the crawler's RequestQueueawait enqueueLinks({- globs: ['http?(s)://apify.com/*/*'],+ globs: ['http?(s)://crawlee.dev/*/*'],});},});// Define the starting URL-await crawler.addRequests(['https://apify.com/store']);+await crawler.addRequests(['https://crawlee.dev']);// Run the crawlerawait crawler.run();diff --git a/docs/examples/forms.ts b/docs/examples/forms.tsindex 4c1ffe4da..c311857fa 100644--- a/docs/examples/forms.ts+++ b/docs/examples/forms.ts@@ -1,7 +1,5 @@import { Dataset, launchPuppeteer } from 'crawlee';-const dataset = await Dataset.open();-// Launch the web browser.const browser = await launchPuppeteer();@@ -33,7 +31,7 @@ const results = await page.$$eval('div.f4.text-normal a', (nodes) => nodes.map((console.log('Results:', results);// Store data in default dataset-await dataset.pushData(results);+await Dataset.pushData(results);// Close browserawait browser.close();diff --git a/docs/examples/map.ts b/docs/examples/map.tsindex c8955eb8a..67d67d7f0 100644--- a/docs/examples/map.ts+++ b/docs/examples/map.ts@@ -1,10 +1,9 @@import { Dataset, KeyValueStore } from 'crawlee';const dataset = await Dataset.open<{ headingCount: number }>();-const keyValueStore = await KeyValueStore.open();// calling map function and filtering through mapped itemsconst moreThan5headers = (await dataset.map((item) => item.headingCount)).filter((count) => count > 5);// saving result of map to default Key-value store-await keyValueStore.setValue('pages_with_more_than_5_headers', moreThan5headers);+await KeyValueStore.setValue('pages_with_more_than_5_headers', moreThan5headers);diff --git a/docs/examples/map_and_reduce.mdx b/docs/examples/map_and_reduce.mdxindex 4f57795b0..da66bc00c 100644--- a/docs/examples/map_and_reduce.mdx+++ b/docs/examples/map_and_reduce.mdx@@ -25,15 +25,15 @@ method to save the example `JSON array` to your dataset.```json[{- "url": "https://apify.com/",+ "url": "https://crawlee.dev/","headingCount": 11},{- "url": "https://apify.com/storage",+ "url": "https://crawlee.dev/storage","headingCount": 8},{- "url": "https://apify.com/proxy",+ "url": "https://crawlee.dev/proxy","headingCount": 4}]diff --git a/docs/examples/playwright_crawler.ts b/docs/examples/playwright_crawler.tsindex 367cda99a..d27830b76 100644--- a/docs/examples/playwright_crawler.ts+++ b/docs/examples/playwright_crawler.ts@@ -1,7 +1,5 @@import { Dataset, PlaywrightCrawler } from 'crawlee';-const dataset = await Dataset.open();-// Create an instance of the PlaywrightCrawler class - a crawler// that automatically loads the URLs in headless Chrome / Playwright.const crawler = new PlaywrightCrawler({@@ -22,12 +20,12 @@ const crawler = new PlaywrightCrawler({// the most important being:// - request: an instance of the Request class with information such as URL and HTTP method// - page: Playwright's Page object (see https://playwright.dev/docs/api/class-page)- async requestHandler({ request, page, enqueueLinks }) {- console.log(`Processing ${request.url}...`);+ async requestHandler({ request, page, enqueueLinks, log }) {+ log.info(`Processing ${request.url}...`);// A function to be evaluated by Playwright within the browser context.const data = await page.$$eval('.athing', ($posts) => {- const scrapedData = [];+ const scrapedData: { title: string; rank: string; href: string }[] = [];// We're getting the title, rank and URL of each post on Hacker News.$posts.forEach(($post) => {@@ -42,19 +40,19 @@ const crawler = new PlaywrightCrawler({});// Store the results to the default dataset.- await dataset.pushData(data);+ await Dataset.pushData(data);// Find a link to the next page and enqueue it if it exists.const infos = await enqueueLinks({selector: '.morelink',});- if (infos.length === 0) console.log(`${request.url} is the last page!`);+ if (infos.processedRequests.length === 0) log.info(`${request.url} is the last page!`);},// This function is called if the page processing failed more than maxRequestRetries+1 times.- failedRequestHandler({ request }) {- console.log(`Request ${request.url} failed too many times.`);+ failedRequestHandler({ request, log }) {+ log.info(`Request ${request.url} failed too many times.`);},});diff --git a/docs/examples/puppeteer_crawler.ts b/docs/examples/puppeteer_crawler.tsindex a029a5114..17d74e1b4 100644--- a/docs/examples/puppeteer_crawler.ts+++ b/docs/examples/puppeteer_crawler.ts@@ -1,7 +1,5 @@import { Dataset, PuppeteerCrawler } from 'crawlee';-const dataset = await Dataset.open();-// Create an instance of the PuppeteerCrawler class - a crawler// that automatically loads the URLs in headless Chrome / Puppeteer.const crawler = new PuppeteerCrawler({@@ -22,12 +20,12 @@ const crawler = new PuppeteerCrawler({// The function accepts a single parameter, which is an object with the following fields:// - request: an instance of the Request class with information such as URL and HTTP method// - page: Puppeteer's Page object (see https://pptr.dev/#show=api-class-page)- async requestHandler({ request, page, enqueueLinks }) {- console.log(`Processing ${request.url}...`);+ async requestHandler({ request, page, enqueueLinks, log }) {+ log.info(`Processing ${request.url}...`);// A function to be evaluated by Puppeteer within the browser context.const data = await page.$$eval('.athing', ($posts) => {- const scrapedData = [];+ const scrapedData: { title: string; rank: string; href: string }[] = [];// We're getting the title, rank and URL of each post on Hacker News.$posts.forEach(($post) => {@@ -42,19 +40,19 @@ const crawler = new PuppeteerCrawler({});// Store the results to the default dataset.- await dataset.pushData(data);+ await Dataset.pushData(data);// Find a link to the next page and enqueue it if it exists.const infos = await enqueueLinks({selector: '.morelink',});- if (infos.length === 0) console.log(`${request.url} is the last page!`);+ if (infos.processedRequests.length === 0) log.info(`${request.url} is the last page!`);},// This function is called if the page processing failed more than maxRequestRetries+1 times.- failedRequestHandler({ request }) {- console.log(`Request ${request.url} failed too many times.`);+ failedRequestHandler({ request, log }) {+ log.error(`Request ${request.url} failed too many times.`);},});diff --git a/docs/examples/puppeteer_crawler_page_screenshot.ts b/docs/examples/puppeteer_crawler_page_screenshot.tsindex a01f85688..7cc4095c4 100644--- a/docs/examples/puppeteer_crawler_page_screenshot.ts+++ b/docs/examples/puppeteer_crawler_page_screenshot.ts@@ -1,7 +1,5 @@import { PuppeteerCrawler, KeyValueStore } from 'crawlee';-const keyValueStore = await KeyValueStore.open();-// Create a PuppeteerCrawlerconst crawler = new PuppeteerCrawler({async requestHandler({ request, page }) {@@ -10,7 +8,7 @@ const crawler = new PuppeteerCrawler({// Convert the URL into a valid keyconst key = request.url.replace(/[:/]/g, '_');// Save the screenshot to the default key-value store- await keyValueStore.setValue(key, screenshot, { contentType: 'image/png' });+ await KeyValueStore.setValue(key, screenshot, { contentType: 'image/png' });},});diff --git a/docs/examples/puppeteer_page_screenshot.ts b/docs/examples/puppeteer_page_screenshot.tsindex 079467439..fd4e99c5f 100644--- a/docs/examples/puppeteer_page_screenshot.ts+++ b/docs/examples/puppeteer_page_screenshot.ts@@ -2,7 +2,7 @@ import { KeyValueStore, launchPuppeteer } from 'crawlee';const keyValueStore = await KeyValueStore.open();-const url = 'http://www.example.com/';+const url = 'https://crawlee.dev';// Start a browserconst browser = await launchPuppeteer();diff --git a/docs/examples/puppeteer_recursive_crawl.ts b/docs/examples/puppeteer_recursive_crawl.tsindex 7deeb80bb..ad48b3247 100644--- a/docs/examples/puppeteer_recursive_crawl.ts+++ b/docs/examples/puppeteer_recursive_crawl.ts@@ -1,9 +1,9 @@import { PuppeteerCrawler } from 'crawlee';const crawler = new PuppeteerCrawler({- async requestHandler({ request, page, enqueueLinks }) {+ async requestHandler({ request, page, enqueueLinks, log }) {const title = await page.title();- console.log(`Title of ${request.url}: ${title}`);+ log.info(`Title of ${request.url}: ${title}`);await enqueueLinks({globs: ['http?(s)://www.iana.org/**'],diff --git a/docs/examples/puppeteer_with_proxy.mdx b/docs/examples/puppeteer_with_proxy.mdxindex c2556ad52..4b88b1d1a 100644--- a/docs/examples/puppeteer_with_proxy.mdx+++ b/docs/examples/puppeteer_with_proxy.mdx@@ -13,7 +13,7 @@ FIXME: is this staying?:::-This example demonstrates how to load pages in headless Chrome / Puppeteer over [Apify Proxy](https://docs.apify.com/proxy).+This example demonstrates how to load pages in headless Chrome / Puppeteer over [Apify Proxy](https://docs.crawlee.dev/proxy).To make it work, you'll need an Apify account with access to the proxy. Visit the [Apify platform introduction](/docs/guides/apify-platform) to findhow to log into your account from the SDK.diff --git a/docs/examples/puppeteer_with_proxy.ts b/docs/examples/puppeteer_with_proxy.tsindex eb7d6aeff..90e0dabb7 100644--- a/docs/examples/puppeteer_with_proxy.ts+++ b/docs/examples/puppeteer_with_proxy.ts@@ -5,13 +5,13 @@ const proxyConfiguration = new ProxyConfiguration();const crawler = new PuppeteerCrawler({proxyConfiguration,- async requestHandler({ page }) {+ async requestHandler({ page, log }) {const status = await page.$eval('td.status', (el) => el.textContent);- console.log(`Proxy Status: ${status}`);+ log.info(`Proxy Status: ${status}`);},});-await crawler.addRequests(['http://proxy.apify.com']);+await crawler.addRequests(['http://proxy.crawlee.dev']);console.log('Running Puppeteer script...');diff --git a/docs/examples/reduce.ts b/docs/examples/reduce.tsindex f637cbfed..106d80d2b 100644--- a/docs/examples/reduce.ts+++ b/docs/examples/reduce.ts@@ -1,7 +1,6 @@import { Dataset, KeyValueStore } from 'crawlee';const dataset = await Dataset.open();-const keyValueStore = await KeyValueStore.open();// calling reduce function and using memo to calculate number of headersconst pagesHeadingCount = await dataset.reduce((memo, value) => {@@ -9,4 +8,4 @@ const pagesHeadingCount = await dataset.reduce((memo, value) => {}, 0);// saving result of map to default Key-value store-await keyValueStore.setValue('pages_heading_count', pagesHeadingCount);+await KeyValueStore.setValue('pages_heading_count', pagesHeadingCount);diff --git a/docs/examples/skip-navigation.mdx b/docs/examples/skip-navigation.mdxindex 79b1c2abe..4f52521d8 100644--- a/docs/examples/skip-navigation.mdx+++ b/docs/examples/skip-navigation.mdx@@ -14,7 +14,7 @@ By combining the <ApiLink to="core/class/Request#skipNavigation">`Request#skipNa:::info-For this example, we are using the <ApiLink to="puppeteer-crawler/class/PuppeteerCrawler">`PuppeteerCrawler`</ApiLink> to showcase this, but this is available on all the crawlers we provide.+For this example, we are using the <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> to showcase this, but this is available on all the crawlers we provide.:::diff --git a/docs/examples/skip-navigation.ts b/docs/examples/skip-navigation.tsindex 0204d9568..cfeddce76 100644--- a/docs/examples/skip-navigation.ts+++ b/docs/examples/skip-navigation.ts@@ -1,9 +1,9 @@-import { PuppeteerCrawler, KeyValueStore } from 'crawlee';+import { PlaywrightCrawler, KeyValueStore } from 'crawlee';// Create a key value store for all images we findconst imageStore = await KeyValueStore.open('images');-const crawler = new PuppeteerCrawler({+const crawler = new PlaywrightCrawler({async requestHandler({ request, page, sendRequest }) {// The request should have the navigation skippedif (request.skipNavigation) {@@ -25,7 +25,7 @@ const crawler = new PuppeteerCrawler({},});-await crawler.addRequests(['https://apify.com/']);+await crawler.addRequests(['https://crawlee.dev']);// Run the crawlerawait crawler.run();diff --git a/docs/examples/tsconfig.json b/docs/examples/tsconfig.jsonindex 735857f05..21a046453 100644--- a/docs/examples/tsconfig.json+++ b/docs/examples/tsconfig.json@@ -1,7 +1,18 @@{"extends": "@apify/tsconfig","compilerOptions": {- "target": "ES2020",- "module": "ESNext"+ "baseUrl": "../..",+ "paths": {+ "crawlee": ["packages/crawlee/src"],+ "@crawlee/basic": ["packages/basic-crawler/src"],+ "@crawlee/browser": ["packages/browser-crawler/src"],+ "@crawlee/cheerio": ["packages/cheerio-crawler/src"],+ "@crawlee/playwright": ["packages/playwright-crawler/src"],+ "@crawlee/puppeteer": ["packages/puppeteer-crawler/src"],+ "@crawlee/*": ["packages/*/src"]+ },+ "target": "ES2022",+ "module": "ES2022",+ "noUnusedParameters": false}}diff --git a/website/src/pages/index.js b/website/src/pages/index.jsindex 01bdd973a..a9cb41b6b 100644--- a/website/src/pages/index.js+++ b/website/src/pages/index.js@@ -59,20 +59,20 @@ function Features() {const example = `import { PlaywrightCrawler, Dataset } from 'crawlee';-const crawler = new PlaywrightCrawler();+const crawler = new PlaywrightCrawler({+ async requestHandler({ request, page, enqueueLinks }) {+ const title = await page.title();+ console.log(\`Title of $\{request.loadedUrl} is '$\{title}'\`);-crawler.router.addDefaultHandler(async ({ request, page, enqueueLinks }) => {- const title = await page.title();- console.log(\`Title of $\{request.loadedUrl} is '$\{title}'\`);+ // save some results+ await Dataset.pushData({ title, url: request.loadedUrl });- // save some results- await Dataset.pushData({ title, url: request.loadedUrl });-- // enqueue all links targeting the same hostname- await enqueueLinks();+ // enqueue all links targeting the same hostname+ await enqueueLinks();+ }});-await crawler.run(['https://www.iana.org/']);+await crawler.run(['https://crawlee.dev']);`;function ActorExample() {