Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

docs: update examples (#1406)

Martin Adámek0c3e025f66e7
diff --git a/docs/examples/.eslintrc.json b/docs/examples/.eslintrc.json
new file mode 100644
index 000000000..98504f200
--- /dev/null
+++ b/docs/examples/.eslintrc.json
@@ -0,0 +1,12 @@
+{
+ "root": true,
+ "extends": "@apify",
+ "parserOptions": {
+ "ecmaVersion": 2022
+ },
+ "rules": {
+ "import/extensions": 0,
+ "import/no-extraneous-dependencies": 0,
+ "no-console": "off"
+ }
+}
diff --git a/docs/examples/accept_user_input.ts b/docs/examples/accept_user_input.ts
index cedb6e938..25d4b48dd 100644
--- a/docs/examples/accept_user_input.ts
+++ b/docs/examples/accept_user_input.ts
@@ -1,6 +1,4 @@
import { KeyValueStore } from 'crawlee';
-const keyValueStore = await KeyValueStore.open();
-const input = await keyValueStore.getValue('INPUT');
-
+const input = await KeyValueStore.getInput();
console.log(input);
diff --git a/docs/examples/add_data_to_dataset.ts b/docs/examples/add_data_to_dataset.ts
index 27e978013..d57a0186e 100644
--- a/docs/examples/add_data_to_dataset.ts
+++ b/docs/examples/add_data_to_dataset.ts
@@ -1,13 +1,10 @@
import { Dataset, CheerioCrawler } from 'crawlee';
-// Create a dataset where we will store the results.
-const dataset = await Dataset.open();
-
const crawler = new CheerioCrawler({
// Function called for each URL
async requestHandler({ request, body }) {
// Save data to default dataset
- await dataset.pushData({
+ await Dataset.pushData({
url: request.url,
html: body,
});
@@ -15,9 +12,9 @@ const crawler = new CheerioCrawler({
});
await crawler.addRequests([
- { url: 'http://www.example.com/page-1' },
- { url: 'http://www.example.com/page-2' },
- { url: 'http://www.example.com/page-3' },
+ 'http://www.example.com/page-1',
+ 'http://www.example.com/page-2',
+ 'http://www.example.com/page-3',
]);
// Run the crawler
diff --git a/docs/examples/basic_crawler.ts b/docs/examples/basic_crawler.ts
index e3c01383b..f1e4584a4 100644
--- a/docs/examples/basic_crawler.ts
+++ b/docs/examples/basic_crawler.ts
@@ -1,15 +1,12 @@
import { BasicCrawler, Dataset } from 'crawlee';
-// Create a dataset where we will store the results.
-const dataset = await Dataset.open();
-
// Create a BasicCrawler - the simplest crawler that enables
// users to implement the crawling logic themselves.
const crawler = new BasicCrawler({
// This function will be called for each URL to crawl.
- async requestHandler({ request, sendRequest }) {
+ async requestHandler({ request, sendRequest, log }) {
const { url } = request;
- console.log(`Processing ${url}...`);
+ log.info(`Processing ${url}...`);
// Fetch the page HTML via the crawlee sendRequest utility method
// By default, the method will use the current request that is being handled, so you don't have to
@@ -17,7 +14,7 @@ const crawler = new BasicCrawler({
const { body } = await sendRequest();
// Store the HTML and URL to the default dataset.
- await dataset.pushData({
+ await Dataset.pushData({
url,
html: body,
});
@@ -26,10 +23,10 @@ const crawler = new BasicCrawler({
// The initial list of URLs to crawl. Here we use just a few hard-coded URLs.
await crawler.addRequests([
- { url: 'http://www.google.com/' },
- { url: 'http://www.example.com/' },
- { url: 'http://www.bing.com/' },
- { url: 'http://www.wikipedia.com/' },
+ 'https://www.google.com',
+ 'https://www.example.com',
+ 'https://www.bing.com',
+ 'https://www.wikipedia.com',
]);
// Run the crawler and wait for it to finish.
diff --git a/docs/examples/cheerio_crawler.ts b/docs/examples/cheerio_crawler.ts
index 84b98e065..21fa67acf 100644
--- a/docs/examples/cheerio_crawler.ts
+++ b/docs/examples/cheerio_crawler.ts
@@ -1,7 +1,5 @@
import { Dataset, CheerioCrawler, log, LogLevel } from 'crawlee';
-const dataset = await Dataset.open();
-
// Crawlers come with various utilities, e.g. for logging.
// Here we use debug level of logging to improve the debugging experience.
// This functionality is optional!
@@ -27,7 +25,7 @@ const crawler = new CheerioCrawler({
// This function will be called for each URL to crawl.
// It accepts a single parameter, which is an object with options as:
- // https://sdk.apify.com/docs/typedefs/cheerio-crawler-options#handlepagefunction
+ // https://crawlee.dev/api/cheerio-crawler/interface/CheerioCrawlerOptions#requestHandler
// We use for demonstration only 2 of them:
// - request: an instance of the Request class with information such as the URL that is being crawled and HTTP method
// - $: the cheerio object containing parsed HTML
@@ -36,7 +34,7 @@ const crawler = new CheerioCrawler({
// Extract data from the page using cheerio.
const title = $('title').text();
- const h1texts = [];
+ const h1texts: { text: string }[] = [];
$('h1').each((index, el) => {
h1texts.push({
text: $(el).text(),
@@ -45,7 +43,7 @@ const crawler = new CheerioCrawler({
// Store the results to the dataset. In local configuration,
// the data will be stored as JSON files in ./storage/datasets/default
- await dataset.pushData({
+ await Dataset.pushData({
url: request.url,
title,
h1texts,
diff --git a/docs/examples/crawl_all_links_cheerio.ts b/docs/examples/crawl_all_links_cheerio.ts
index 9e62d1a35..e3c930bc2 100644
--- a/docs/examples/crawl_all_links_cheerio.ts
+++ b/docs/examples/crawl_all_links_cheerio.ts
@@ -1,15 +1,13 @@
import { CheerioCrawler } from 'crawlee';
const crawler = new CheerioCrawler({
- async requestHandler({ request, enqueueLinks }) {
- console.log(request.url);
+ async requestHandler({ request, enqueueLinks, log }) {
+ log.info(request.url);
// Add all links from page to RequestQueue
await enqueueLinks();
},
maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)
});
-await crawler.addRequests(['https://apify.com/']);
-
-// Run the crawler
-await crawler.run();
+// Run the crawler with initial request
+await crawler.run(['https://crawlee.dev']);
diff --git a/docs/examples/crawl_all_links_playwright.ts b/docs/examples/crawl_all_links_playwright.ts
index 778c67640..bf0440dee 100644
--- a/docs/examples/crawl_all_links_playwright.ts
+++ b/docs/examples/crawl_all_links_playwright.ts
@@ -1,15 +1,13 @@
import { PlaywrightCrawler } from 'crawlee';
const crawler = new PlaywrightCrawler({
- async requestHandler({ request, enqueueLinks }) {
- console.log(request.url);
+ async requestHandler({ request, enqueueLinks, log }) {
+ log.info(request.url);
// Add all links from page to RequestQueue
await enqueueLinks();
},
maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)
});
-await crawler.addRequests(['https://apify.com/']);
-
-// Run the crawler
-await crawler.run();
+// Run the crawler with initial request
+await crawler.run(['https://crawlee.dev']);
diff --git a/docs/examples/crawl_all_links_puppeteer.ts b/docs/examples/crawl_all_links_puppeteer.ts
index e3ac8dec2..cf6fcb55c 100644
--- a/docs/examples/crawl_all_links_puppeteer.ts
+++ b/docs/examples/crawl_all_links_puppeteer.ts
@@ -1,15 +1,13 @@
import { PuppeteerCrawler } from 'crawlee';
const crawler = new PuppeteerCrawler({
- async requestHandler({ request, enqueueLinks }) {
- console.log(request.url);
+ async requestHandler({ request, enqueueLinks, log }) {
+ log.info(request.url);
// Add all links from page to RequestQueue
await enqueueLinks();
},
maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)
});
-await crawler.addRequests(['https://apify.com/']);
-
-// Run the crawler
-await crawler.run();
+// Run the crawler with initial request
+await crawler.run(['https://crawlee.dev']);
diff --git a/docs/examples/crawl_multiple_urls_cheerio.ts b/docs/examples/crawl_multiple_urls_cheerio.ts
index 0241e040e..011d8b7d6 100644
--- a/docs/examples/crawl_multiple_urls_cheerio.ts
+++ b/docs/examples/crawl_multiple_urls_cheerio.ts
@@ -2,17 +2,15 @@ import { CheerioCrawler } from 'crawlee';
const crawler = new CheerioCrawler({
// Function called for each URL
- async requestHandler({ request, $ }) {
+ async requestHandler({ request, $, log }) {
const title = $('title').text();
- console.log(`URL: ${request.url}\nTITLE: ${title}`);
+ log.info(`URL: ${request.url}\nTITLE: ${title}`);
},
});
-await crawler.addRequests([
+// Run the crawler with initial request
+await crawler.run([
'http://www.example.com/page-1',
'http://www.example.com/page-2',
'http://www.example.com/page-3',
]);
-
-// Run the crawler
-await crawler.run();
diff --git a/docs/examples/crawl_multiple_urls_playwright.ts b/docs/examples/crawl_multiple_urls_playwright.ts
index cd34b5b52..9581bea3f 100644
--- a/docs/examples/crawl_multiple_urls_playwright.ts
+++ b/docs/examples/crawl_multiple_urls_playwright.ts
@@ -2,17 +2,15 @@ import { PlaywrightCrawler } from 'crawlee';
const crawler = new PlaywrightCrawler({
// Function called for each URL
- async requestHandler({ request, page }) {
+ async requestHandler({ request, page, log }) {
const title = await page.title();
- console.log(`URL: ${request.url}\nTITLE: ${title}`);
+ log.info(`URL: ${request.url}\nTITLE: ${title}`);
},
});
-await crawler.addRequests([
+// Run the crawler with initial request
+await crawler.run([
'http://www.example.com/page-1',
'http://www.example.com/page-2',
'http://www.example.com/page-3',
]);
-
-// Run the crawler
-await crawler.run();
diff --git a/docs/examples/crawl_multiple_urls_puppeteer.ts b/docs/examples/crawl_multiple_urls_puppeteer.ts
index cf027fef5..d7c6a99a5 100644
--- a/docs/examples/crawl_multiple_urls_puppeteer.ts
+++ b/docs/examples/crawl_multiple_urls_puppeteer.ts
@@ -2,17 +2,15 @@ import { PuppeteerCrawler } from 'crawlee';
const crawler = new PuppeteerCrawler({
// Function called for each URL
- async requestHandler({ request, page }) {
+ async requestHandler({ request, page, log }) {
const title = await page.title();
- console.log(`URL: ${request.url}\nTITLE: ${title}`);
+ log.info(`URL: ${request.url}\nTITLE: ${title}`);
},
});
-await crawler.addRequests([
+// Run the crawler with initial request
+await crawler.run([
'http://www.example.com/page-1',
'http://www.example.com/page-2',
'http://www.example.com/page-3',
]);
-
-// Run the crawler
-await crawler.run();
diff --git a/docs/examples/crawl_relative_links_all.ts b/docs/examples/crawl_relative_links_all.ts
index c9265c20a..e4401cb9f 100644
--- a/docs/examples/crawl_relative_links_all.ts
+++ b/docs/examples/crawl_relative_links_all.ts
@@ -2,8 +2,8 @@ import { CheerioCrawler, EnqueueStrategy } from 'crawlee';
const crawler = new CheerioCrawler({
maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)
- async requestHandler({ request, enqueueLinks }) {
- console.log(request.url);
+ async requestHandler({ request, enqueueLinks, log }) {
+ log.info(request.url);
await enqueueLinks({
// Setting the strategy to 'all' will enqueue all links found
// highlight-next-line
@@ -14,7 +14,5 @@ const crawler = new CheerioCrawler({
},
});
-await crawler.addRequests(['https://apify.com/']);
-
-// Run the crawler
-await crawler.run();
+// Run the crawler with initial request
+await crawler.run(['https://crawlee.dev']);
diff --git a/docs/examples/crawl_relative_links_same_domain.ts b/docs/examples/crawl_relative_links_same_domain.ts
index be4b9ec61..59c833c05 100644
--- a/docs/examples/crawl_relative_links_same_domain.ts
+++ b/docs/examples/crawl_relative_links_same_domain.ts
@@ -2,8 +2,8 @@ import { CheerioCrawler, EnqueueStrategy } from 'crawlee';
const crawler = new CheerioCrawler({
maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)
- async requestHandler({ request, enqueueLinks }) {
- console.log(request.url);
+ async requestHandler({ request, enqueueLinks, log }) {
+ log.info(request.url);
await enqueueLinks({
// Setting the strategy to 'same-domain' will enqueue all links found that are on the
// same hostname as request.loadedUrl or request.url
@@ -15,7 +15,5 @@ const crawler = new CheerioCrawler({
},
});
-await crawler.addRequests(['https://apify.com/']);
-
-// Run the crawler
-await crawler.run();
+// Run the crawler with initial request
+await crawler.run(['https://crawlee.dev']);
diff --git a/docs/examples/crawl_relative_links_same_hostname.ts b/docs/examples/crawl_relative_links_same_hostname.ts
index 2fb2bcc8d..052f24dcf 100644
--- a/docs/examples/crawl_relative_links_same_hostname.ts
+++ b/docs/examples/crawl_relative_links_same_hostname.ts
@@ -2,8 +2,8 @@ import { CheerioCrawler, EnqueueStrategy } from 'crawlee';
const crawler = new CheerioCrawler({
maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)
- async requestHandler({ request, enqueueLinks }) {
- console.log(request.url);
+ async requestHandler({ request, enqueueLinks, log }) {
+ log.info(request.url);
await enqueueLinks({
// Setting the strategy to 'same-hostname' will enqueue all links found that are on the
// same hostname (including subdomain) as request.loadedUrl or request.url
@@ -15,7 +15,5 @@ const crawler = new CheerioCrawler({
},
});
-await crawler.addRequests(['https://apify.com/']);
-
-// Run the crawler
-await crawler.run();
+// Run the crawler with initial request
+await crawler.run(['https://crawlee.dev']);
diff --git a/docs/examples/crawl_sitemap_cheerio.ts b/docs/examples/crawl_sitemap_cheerio.ts
index afde7a6c5..cad832f39 100644
--- a/docs/examples/crawl_sitemap_cheerio.ts
+++ b/docs/examples/crawl_sitemap_cheerio.ts
@@ -2,13 +2,13 @@ import { CheerioCrawler, downloadListOfUrls } from 'crawlee';
const crawler = new CheerioCrawler({
// Function called for each URL
- async requestHandler({ request }) {
- console.log(request.url);
+ async requestHandler({ request, log }) {
+ log.info(request.url);
},
maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl a sitemap)
});
-const listOfUrls = await downloadListOfUrls({ url: 'https://apify.com/sitemap.xml' });
+const listOfUrls = await downloadListOfUrls({ url: 'https://crawlee.dev/sitemap.xml' });
await crawler.addRequests(listOfUrls);
diff --git a/docs/examples/crawl_sitemap_playwright.ts b/docs/examples/crawl_sitemap_playwright.ts
index d4948aa08..d829451d1 100644
--- a/docs/examples/crawl_sitemap_playwright.ts
+++ b/docs/examples/crawl_sitemap_playwright.ts
@@ -2,13 +2,13 @@ import { PlaywrightCrawler, downloadListOfUrls } from 'crawlee';
const crawler = new PlaywrightCrawler({
// Function called for each URL
- async requestHandler({ request }) {
- console.log(request.url);
+ async requestHandler({ request, log }) {
+ log.info(request.url);
},
maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl a sitemap)
});
-const listOfUrls = await downloadListOfUrls({ url: 'https://apify.com/sitemap.xml' });
+const listOfUrls = await downloadListOfUrls({ url: 'https://crawlee.dev/sitemap.xml' });
await crawler.addRequests(listOfUrls);
diff --git a/docs/examples/crawl_sitemap_puppeteer.ts b/docs/examples/crawl_sitemap_puppeteer.ts
index 488e4e2ae..96122d590 100644
--- a/docs/examples/crawl_sitemap_puppeteer.ts
+++ b/docs/examples/crawl_sitemap_puppeteer.ts
@@ -2,13 +2,13 @@ import { PuppeteerCrawler, downloadListOfUrls } from 'crawlee';
const crawler = new PuppeteerCrawler({
// Function called for each URL
- async requestHandler({ request }) {
- console.log(request.url);
+ async requestHandler({ request, log }) {
+ log.info(request.url);
},
maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl a sitemap)
});
-const listOfUrls = await downloadListOfUrls({ url: 'https://apify.com/sitemap.xml' });
+const listOfUrls = await downloadListOfUrls({ url: 'https://crawlee.dev/sitemap.xml' });
await crawler.addRequests(listOfUrls);
diff --git a/docs/examples/crawl_some_links.ts b/docs/examples/crawl_some_links.ts
index ff14af8e0..51912bb10 100644
--- a/docs/examples/crawl_some_links.ts
+++ b/docs/examples/crawl_some_links.ts
@@ -5,17 +5,17 @@ const crawler = new CheerioCrawler({
// Limits the crawler to only 10 requests (do not use if you want to crawl all links)
maxRequestsPerCrawl: 10,
// Function called for each URL
- async requestHandler({ request, enqueueLinks }) {
- console.log(request.url);
+ async requestHandler({ request, enqueueLinks, log }) {
+ log.info(request.url);
// Add some links from page to the crawler's RequestQueue
await enqueueLinks({
- globs: ['http?(s)://apify.com/*/*'],
+ globs: ['http?(s)://crawlee.dev/*/*'],
});
},
});
// Define the starting URL
-await crawler.addRequests(['https://apify.com/store']);
+await crawler.addRequests(['https://crawlee.dev']);
// Run the crawler
await crawler.run();
diff --git a/docs/examples/forms.ts b/docs/examples/forms.ts
index 4c1ffe4da..c311857fa 100644
--- a/docs/examples/forms.ts
+++ b/docs/examples/forms.ts
@@ -1,7 +1,5 @@
import { Dataset, launchPuppeteer } from 'crawlee';
-const dataset = await Dataset.open();
-
// Launch the web browser.
const browser = await launchPuppeteer();
@@ -33,7 +31,7 @@ const results = await page.$$eval('div.f4.text-normal a', (nodes) => nodes.map((
console.log('Results:', results);
// Store data in default dataset
-await dataset.pushData(results);
+await Dataset.pushData(results);
// Close browser
await browser.close();
diff --git a/docs/examples/map.ts b/docs/examples/map.ts
index c8955eb8a..67d67d7f0 100644
--- a/docs/examples/map.ts
+++ b/docs/examples/map.ts
@@ -1,10 +1,9 @@
import { Dataset, KeyValueStore } from 'crawlee';
const dataset = await Dataset.open<{ headingCount: number }>();
-const keyValueStore = await KeyValueStore.open();
// calling map function and filtering through mapped items
const moreThan5headers = (await dataset.map((item) => item.headingCount)).filter((count) => count > 5);
// saving result of map to default Key-value store
-await keyValueStore.setValue('pages_with_more_than_5_headers', moreThan5headers);
+await KeyValueStore.setValue('pages_with_more_than_5_headers', moreThan5headers);
diff --git a/docs/examples/map_and_reduce.mdx b/docs/examples/map_and_reduce.mdx
index 4f57795b0..da66bc00c 100644
--- a/docs/examples/map_and_reduce.mdx
+++ b/docs/examples/map_and_reduce.mdx
@@ -25,15 +25,15 @@ method to save the example `JSON array` to your dataset.
```json
[
{
- "url": "https://apify.com/",
+ "url": "https://crawlee.dev/",
"headingCount": 11
},
{
- "url": "https://apify.com/storage",
+ "url": "https://crawlee.dev/storage",
"headingCount": 8
},
{
- "url": "https://apify.com/proxy",
+ "url": "https://crawlee.dev/proxy",
"headingCount": 4
}
]
diff --git a/docs/examples/playwright_crawler.ts b/docs/examples/playwright_crawler.ts
index 367cda99a..d27830b76 100644
--- a/docs/examples/playwright_crawler.ts
+++ b/docs/examples/playwright_crawler.ts
@@ -1,7 +1,5 @@
import { Dataset, PlaywrightCrawler } from 'crawlee';
-const dataset = await Dataset.open();
-
// Create an instance of the PlaywrightCrawler class - a crawler
// that automatically loads the URLs in headless Chrome / Playwright.
const crawler = new PlaywrightCrawler({
@@ -22,12 +20,12 @@ const crawler = new PlaywrightCrawler({
// the most important being:
// - request: an instance of the Request class with information such as URL and HTTP method
// - page: Playwright's Page object (see https://playwright.dev/docs/api/class-page)
- async requestHandler({ request, page, enqueueLinks }) {
- console.log(`Processing ${request.url}...`);
+ async requestHandler({ request, page, enqueueLinks, log }) {
+ log.info(`Processing ${request.url}...`);
// A function to be evaluated by Playwright within the browser context.
const data = await page.$$eval('.athing', ($posts) => {
- const scrapedData = [];
+ const scrapedData: { title: string; rank: string; href: string }[] = [];
// We're getting the title, rank and URL of each post on Hacker News.
$posts.forEach(($post) => {
@@ -42,19 +40,19 @@ const crawler = new PlaywrightCrawler({
});
// Store the results to the default dataset.
- await dataset.pushData(data);
+ await Dataset.pushData(data);
// Find a link to the next page and enqueue it if it exists.
const infos = await enqueueLinks({
selector: '.morelink',
});
- if (infos.length === 0) console.log(`${request.url} is the last page!`);
+ if (infos.processedRequests.length === 0) log.info(`${request.url} is the last page!`);
},
// This function is called if the page processing failed more than maxRequestRetries+1 times.
- failedRequestHandler({ request }) {
- console.log(`Request ${request.url} failed too many times.`);
+ failedRequestHandler({ request, log }) {
+ log.info(`Request ${request.url} failed too many times.`);
},
});
diff --git a/docs/examples/puppeteer_crawler.ts b/docs/examples/puppeteer_crawler.ts
index a029a5114..17d74e1b4 100644
--- a/docs/examples/puppeteer_crawler.ts
+++ b/docs/examples/puppeteer_crawler.ts
@@ -1,7 +1,5 @@
import { Dataset, PuppeteerCrawler } from 'crawlee';
-const dataset = await Dataset.open();
-
// Create an instance of the PuppeteerCrawler class - a crawler
// that automatically loads the URLs in headless Chrome / Puppeteer.
const crawler = new PuppeteerCrawler({
@@ -22,12 +20,12 @@ const crawler = new PuppeteerCrawler({
// The function accepts a single parameter, which is an object with the following fields:
// - request: an instance of the Request class with information such as URL and HTTP method
// - page: Puppeteer's Page object (see https://pptr.dev/#show=api-class-page)
- async requestHandler({ request, page, enqueueLinks }) {
- console.log(`Processing ${request.url}...`);
+ async requestHandler({ request, page, enqueueLinks, log }) {
+ log.info(`Processing ${request.url}...`);
// A function to be evaluated by Puppeteer within the browser context.
const data = await page.$$eval('.athing', ($posts) => {
- const scrapedData = [];
+ const scrapedData: { title: string; rank: string; href: string }[] = [];
// We're getting the title, rank and URL of each post on Hacker News.
$posts.forEach(($post) => {
@@ -42,19 +40,19 @@ const crawler = new PuppeteerCrawler({
});
// Store the results to the default dataset.
- await dataset.pushData(data);
+ await Dataset.pushData(data);
// Find a link to the next page and enqueue it if it exists.
const infos = await enqueueLinks({
selector: '.morelink',
});
- if (infos.length === 0) console.log(`${request.url} is the last page!`);
+ if (infos.processedRequests.length === 0) log.info(`${request.url} is the last page!`);
},
// This function is called if the page processing failed more than maxRequestRetries+1 times.
- failedRequestHandler({ request }) {
- console.log(`Request ${request.url} failed too many times.`);
+ failedRequestHandler({ request, log }) {
+ log.error(`Request ${request.url} failed too many times.`);
},
});
diff --git a/docs/examples/puppeteer_crawler_page_screenshot.ts b/docs/examples/puppeteer_crawler_page_screenshot.ts
index a01f85688..7cc4095c4 100644
--- a/docs/examples/puppeteer_crawler_page_screenshot.ts
+++ b/docs/examples/puppeteer_crawler_page_screenshot.ts
@@ -1,7 +1,5 @@
import { PuppeteerCrawler, KeyValueStore } from 'crawlee';
-const keyValueStore = await KeyValueStore.open();
-
// Create a PuppeteerCrawler
const crawler = new PuppeteerCrawler({
async requestHandler({ request, page }) {
@@ -10,7 +8,7 @@ const crawler = new PuppeteerCrawler({
// Convert the URL into a valid key
const key = request.url.replace(/[:/]/g, '_');
// Save the screenshot to the default key-value store
- await keyValueStore.setValue(key, screenshot, { contentType: 'image/png' });
+ await KeyValueStore.setValue(key, screenshot, { contentType: 'image/png' });
},
});
diff --git a/docs/examples/puppeteer_page_screenshot.ts b/docs/examples/puppeteer_page_screenshot.ts
index 079467439..fd4e99c5f 100644
--- a/docs/examples/puppeteer_page_screenshot.ts
+++ b/docs/examples/puppeteer_page_screenshot.ts
@@ -2,7 +2,7 @@ import { KeyValueStore, launchPuppeteer } from 'crawlee';
const keyValueStore = await KeyValueStore.open();
-const url = 'http://www.example.com/';
+const url = 'https://crawlee.dev';
// Start a browser
const browser = await launchPuppeteer();
diff --git a/docs/examples/puppeteer_recursive_crawl.ts b/docs/examples/puppeteer_recursive_crawl.ts
index 7deeb80bb..ad48b3247 100644
--- a/docs/examples/puppeteer_recursive_crawl.ts
+++ b/docs/examples/puppeteer_recursive_crawl.ts
@@ -1,9 +1,9 @@
import { PuppeteerCrawler } from 'crawlee';
const crawler = new PuppeteerCrawler({
- async requestHandler({ request, page, enqueueLinks }) {
+ async requestHandler({ request, page, enqueueLinks, log }) {
const title = await page.title();
- console.log(`Title of ${request.url}: ${title}`);
+ log.info(`Title of ${request.url}: ${title}`);
await enqueueLinks({
globs: ['http?(s)://www.iana.org/**'],
diff --git a/docs/examples/puppeteer_with_proxy.mdx b/docs/examples/puppeteer_with_proxy.mdx
index c2556ad52..4b88b1d1a 100644
--- a/docs/examples/puppeteer_with_proxy.mdx
+++ b/docs/examples/puppeteer_with_proxy.mdx
@@ -13,7 +13,7 @@ FIXME: is this staying?
:::
-This example demonstrates how to load pages in headless Chrome / Puppeteer over [Apify Proxy](https://docs.apify.com/proxy).
+This example demonstrates how to load pages in headless Chrome / Puppeteer over [Apify Proxy](https://docs.crawlee.dev/proxy).
To make it work, you'll need an Apify account with access to the proxy. Visit the [Apify platform introduction](/docs/guides/apify-platform) to find
how to log into your account from the SDK.
diff --git a/docs/examples/puppeteer_with_proxy.ts b/docs/examples/puppeteer_with_proxy.ts
index eb7d6aeff..90e0dabb7 100644
--- a/docs/examples/puppeteer_with_proxy.ts
+++ b/docs/examples/puppeteer_with_proxy.ts
@@ -5,13 +5,13 @@ const proxyConfiguration = new ProxyConfiguration();
const crawler = new PuppeteerCrawler({
proxyConfiguration,
- async requestHandler({ page }) {
+ async requestHandler({ page, log }) {
const status = await page.$eval('td.status', (el) => el.textContent);
- console.log(`Proxy Status: ${status}`);
+ log.info(`Proxy Status: ${status}`);
},
});
-await crawler.addRequests(['http://proxy.apify.com']);
+await crawler.addRequests(['http://proxy.crawlee.dev']);
console.log('Running Puppeteer script...');
diff --git a/docs/examples/reduce.ts b/docs/examples/reduce.ts
index f637cbfed..106d80d2b 100644
--- a/docs/examples/reduce.ts
+++ b/docs/examples/reduce.ts
@@ -1,7 +1,6 @@
import { Dataset, KeyValueStore } from 'crawlee';
const dataset = await Dataset.open();
-const keyValueStore = await KeyValueStore.open();
// calling reduce function and using memo to calculate number of headers
const pagesHeadingCount = await dataset.reduce((memo, value) => {
@@ -9,4 +8,4 @@ const pagesHeadingCount = await dataset.reduce((memo, value) => {
}, 0);
// saving result of map to default Key-value store
-await keyValueStore.setValue('pages_heading_count', pagesHeadingCount);
+await KeyValueStore.setValue('pages_heading_count', pagesHeadingCount);
diff --git a/docs/examples/skip-navigation.mdx b/docs/examples/skip-navigation.mdx
index 79b1c2abe..4f52521d8 100644
--- a/docs/examples/skip-navigation.mdx
+++ b/docs/examples/skip-navigation.mdx
@@ -14,7 +14,7 @@ By combining the <ApiLink to="core/class/Request#skipNavigation">`Request#skipNa
:::info
-For this example, we are using the <ApiLink to="puppeteer-crawler/class/PuppeteerCrawler">`PuppeteerCrawler`</ApiLink> to showcase this, but this is available on all the crawlers we provide.
+For this example, we are using the <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> to showcase this, but this is available on all the crawlers we provide.
:::
diff --git a/docs/examples/skip-navigation.ts b/docs/examples/skip-navigation.ts
index 0204d9568..cfeddce76 100644
--- a/docs/examples/skip-navigation.ts
+++ b/docs/examples/skip-navigation.ts
@@ -1,9 +1,9 @@
-import { PuppeteerCrawler, KeyValueStore } from 'crawlee';
+import { PlaywrightCrawler, KeyValueStore } from 'crawlee';
// Create a key value store for all images we find
const imageStore = await KeyValueStore.open('images');
-const crawler = new PuppeteerCrawler({
+const crawler = new PlaywrightCrawler({
async requestHandler({ request, page, sendRequest }) {
// The request should have the navigation skipped
if (request.skipNavigation) {
@@ -25,7 +25,7 @@ const crawler = new PuppeteerCrawler({
},
});
-await crawler.addRequests(['https://apify.com/']);
+await crawler.addRequests(['https://crawlee.dev']);
// Run the crawler
await crawler.run();
diff --git a/docs/examples/tsconfig.json b/docs/examples/tsconfig.json
index 735857f05..21a046453 100644
--- a/docs/examples/tsconfig.json
+++ b/docs/examples/tsconfig.json
@@ -1,7 +1,18 @@
{
"extends": "@apify/tsconfig",
"compilerOptions": {
- "target": "ES2020",
- "module": "ESNext"
+ "baseUrl": "../..",
+ "paths": {
+ "crawlee": ["packages/crawlee/src"],
+ "@crawlee/basic": ["packages/basic-crawler/src"],
+ "@crawlee/browser": ["packages/browser-crawler/src"],
+ "@crawlee/cheerio": ["packages/cheerio-crawler/src"],
+ "@crawlee/playwright": ["packages/playwright-crawler/src"],
+ "@crawlee/puppeteer": ["packages/puppeteer-crawler/src"],
+ "@crawlee/*": ["packages/*/src"]
+ },
+ "target": "ES2022",
+ "module": "ES2022",
+ "noUnusedParameters": false
}
}
diff --git a/website/src/pages/index.js b/website/src/pages/index.js
index 01bdd973a..a9cb41b6b 100644
--- a/website/src/pages/index.js
+++ b/website/src/pages/index.js
@@ -59,20 +59,20 @@ function Features() {
const example = `import { PlaywrightCrawler, Dataset } from 'crawlee';
-const crawler = new PlaywrightCrawler();
+const crawler = new PlaywrightCrawler({
+ async requestHandler({ request, page, enqueueLinks }) {
+ const title = await page.title();
+ console.log(\`Title of $\{request.loadedUrl} is '$\{title}'\`);
-crawler.router.addDefaultHandler(async ({ request, page, enqueueLinks }) => {
- const title = await page.title();
- console.log(\`Title of $\{request.loadedUrl} is '$\{title}'\`);
+ // save some results
+ await Dataset.pushData({ title, url: request.loadedUrl });
- // save some results
- await Dataset.pushData({ title, url: request.loadedUrl });
-
- // enqueue all links targeting the same hostname
- await enqueueLinks();
+ // enqueue all links targeting the same hostname
+ await enqueueLinks();
+ }
});
-await crawler.run(['https://www.iana.org/']);
+await crawler.run(['https://crawlee.dev']);
`;
function ActorExample() {