Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

docs: add documentation for playwright and puppeteer-extra (#1823)

Vlad Franguc96ea113efc1
diff --git a/docs/examples/.eslintrc.json b/docs/examples/.eslintrc.json
index 98504f200..d41f6dbc8 100644
--- a/docs/examples/.eslintrc.json
+++ b/docs/examples/.eslintrc.json
@@ -1,7 +1,8 @@
{
"root": true,
- "extends": "@apify",
+ "extends": "@apify/ts",
"parserOptions": {
+ "project": "./tsconfig.eslint.json",
"ecmaVersion": 2022
},
"rules": {
diff --git a/docs/examples/crawler-plugins/index.mdx b/docs/examples/crawler-plugins/index.mdx
new file mode 100644
index 000000000..625478061
--- /dev/null
+++ b/docs/examples/crawler-plugins/index.mdx
@@ -0,0 +1,78 @@
+---
+id: playwright-puppeteer-extra
+title: Using puppeteer-extra and playwright-extra
+---
+
+import CodeBlock from '@theme/CodeBlock';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import ApiLink from '@site/src/components/ApiLink';
+import PuppeteerExtraSource from '!!raw-loader!./puppeteer-extra.ts';
+import PlaywrightExtraSource from '!!raw-loader!./playwright-extra.ts';
+
+[`puppeteer-extra`](https://www.npmjs.com/package/puppeteer-extra) and [`playwright-extra`](https://www.npmjs.com/package/playwright-extra) are community-built
+libraries that bring in a plugin system to enhance the usage of [`puppeteer`](https://www.npmjs.com/package/puppeteer) and
+[`playwright`](https://www.npmjs.com/package/playwright) respectively (bringing in extra functionality, like improving stealth for
+example by using the [`puppeteer-extra-plugin-stealth`](https://www.npmjs.com/package/puppeteer-extra-plugin-stealth) plugin).
+
+:::tip Available plugins
+
+You can see a list of available plugins on the [`puppeteer-extra`s plugin list](https://www.npmjs.com/package/puppeteer-extra#plugins).
+
+For [`playwright`](https://www.npmjs.com/package/playwright), please see [`playwright-extra`s plugin list](https://www.npmjs.com/package/playwright-extra#plugins) instead.
+
+:::
+
+In this example, we'll show you how to use the [`puppeteer-extra-plugin-stealth`](https://www.npmjs.com/package/puppeteer-extra-plugin-stealth) plugin
+to help you avoid bot detections when crawling your target website.
+
+<Tabs>
+<TabItem value="puppeteer" label="Puppeteer & puppeteer-extra" default>
+
+:::info Before you begin
+
+Make sure you've installed the `puppeteer-extra` and `puppeteer-extra-plugin-stealth` packages via your preferred package manager
+
+```bash
+npm install puppeteer-extra puppeteer-extra-plugin-stealth
+```
+
+:::
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.
+
+:::
+
+<CodeBlock className="language-js" title="src/crawler.ts">
+ {PuppeteerExtraSource}
+</CodeBlock>
+
+</TabItem>
+
+<TabItem value="playwright" label="Playwright & playwright-extra">
+
+:::info Before you begin
+
+Make sure you've installed the `playwright-extra` and `puppeteer-extra-plugin-stealth` packages via your preferred package manager
+
+```bash
+npm install playwright-extra puppeteer-extra-plugin-stealth
+```
+
+:::
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.
+
+:::
+
+<CodeBlock className="language-js" title="src/crawler.ts">
+ {PlaywrightExtraSource}
+</CodeBlock>
+
+</TabItem>
+
+</Tabs>
diff --git a/docs/examples/crawler-plugins/playwright-extra.ts b/docs/examples/crawler-plugins/playwright-extra.ts
new file mode 100644
index 000000000..c2b733f47
--- /dev/null
+++ b/docs/examples/crawler-plugins/playwright-extra.ts
@@ -0,0 +1,74 @@
+import { Dataset, PlaywrightCrawler } from 'crawlee';
+
+// For playwright-extra you will need to import the browser type itself that you want to use!
+// By default, PlaywrightCrawler uses chromium, but you can also use firefox or webkit.
+import { chromium } from 'playwright-extra';
+import stealthPlugin from 'puppeteer-extra-plugin-stealth';
+
+// First, we tell playwright-extra to use the plugin (or plugins) we want.
+// Certain plugins might have options you can pass in - read up on their documentation!
+chromium.use(stealthPlugin());
+
+// Create an instance of the PlaywrightCrawler class - a crawler
+// that automatically loads the URLs in headless Chrome / Playwright.
+const crawler = new PlaywrightCrawler({
+ launchContext: {
+ // !!! You need to specify this option to tell Crawlee to use playwright-extra as the launcher !!!
+ launcher: chromium,
+ launchOptions: {
+ // Other playwright options work as usual
+ headless: true,
+ },
+ },
+
+ // Stop crawling after several pages
+ maxRequestsPerCrawl: 50,
+
+ // This function will be called for each URL to crawl.
+ // Here you can write the Puppeteer scripts you are familiar with,
+ // with the exception that browsers and pages are automatically managed by Crawlee.
+ // The function accepts a single parameter, which is an object with the following fields:
+ // - request: an instance of the Request class with information such as URL and HTTP method
+ // - page: Puppeteer's Page object (see https://pptr.dev/#show=api-class-page)
+ async requestHandler({ request, page, enqueueLinks, log }) {
+ log.info(`Processing ${request.url}...`);
+
+ // A function to be evaluated by Puppeteer within the browser context.
+ const data = await page.$$eval('.athing', ($posts) => {
+ const scrapedData: { title: string; rank: string; href: string }[] = [];
+
+ // We're getting the title, rank and URL of each post on Hacker News.
+ $posts.forEach(($post) => {
+ scrapedData.push({
+ title: $post.querySelector('.title a').innerText,
+ rank: $post.querySelector('.rank').innerText,
+ href: $post.querySelector('.title a').href,
+ });
+ });
+
+ return scrapedData;
+ });
+
+ // Store the results to the default dataset.
+ await Dataset.pushData(data);
+
+ // Find a link to the next page and enqueue it if it exists.
+ const infos = await enqueueLinks({
+ selector: '.morelink',
+ });
+
+ if (infos.processedRequests.length === 0) log.info(`${request.url} is the last page!`);
+ },
+
+ // This function is called if the page processing failed more than maxRequestRetries+1 times.
+ failedRequestHandler({ request, log }) {
+ log.error(`Request ${request.url} failed too many times.`);
+ },
+});
+
+await crawler.addRequests(['https://news.ycombinator.com/']);
+
+// Run the crawler and wait for it to finish.
+await crawler.run();
+
+console.log('Crawler finished.');
diff --git a/docs/examples/crawler-plugins/puppeteer-extra.ts b/docs/examples/crawler-plugins/puppeteer-extra.ts
new file mode 100644
index 000000000..6da7bf8a8
--- /dev/null
+++ b/docs/examples/crawler-plugins/puppeteer-extra.ts
@@ -0,0 +1,71 @@
+import { Dataset, PuppeteerCrawler } from 'crawlee';
+import puppeteerExtra from 'puppeteer-extra';
+import stealthPlugin from 'puppeteer-extra-plugin-stealth';
+
+// First, we tell puppeteer-extra to use the plugin (or plugins) we want.
+// Certain plugins might have options you can pass in - read up on their documentation!
+puppeteerExtra.use(stealthPlugin());
+
+// Create an instance of the PuppeteerCrawler class - a crawler
+// that automatically loads the URLs in headless Chrome / Puppeteer.
+const crawler = new PuppeteerCrawler({
+ launchContext: {
+ // !!! You need to specify this option to tell Crawlee to use puppeteer-extra as the launcher !!!
+ launcher: puppeteerExtra,
+ launchOptions: {
+ // Other puppeteer options work as usual
+ headless: true,
+ },
+ },
+
+ // Stop crawling after several pages
+ maxRequestsPerCrawl: 50,
+
+ // This function will be called for each URL to crawl.
+ // Here you can write the Puppeteer scripts you are familiar with,
+ // with the exception that browsers and pages are automatically managed by Crawlee.
+ // The function accepts a single parameter, which is an object with the following fields:
+ // - request: an instance of the Request class with information such as URL and HTTP method
+ // - page: Puppeteer's Page object (see https://pptr.dev/#show=api-class-page)
+ async requestHandler({ request, page, enqueueLinks, log }) {
+ log.info(`Processing ${request.url}...`);
+
+ // A function to be evaluated by Puppeteer within the browser context.
+ const data = await page.$$eval('.athing', ($posts) => {
+ const scrapedData: { title: string; rank: string; href: string }[] = [];
+
+ // We're getting the title, rank and URL of each post on Hacker News.
+ $posts.forEach(($post) => {
+ scrapedData.push({
+ title: $post.querySelector('.title a').innerText,
+ rank: $post.querySelector('.rank').innerText,
+ href: $post.querySelector('.title a').href,
+ });
+ });
+
+ return scrapedData;
+ });
+
+ // Store the results to the default dataset.
+ await Dataset.pushData(data);
+
+ // Find a link to the next page and enqueue it if it exists.
+ const infos = await enqueueLinks({
+ selector: '.morelink',
+ });
+
+ if (infos.processedRequests.length === 0) log.info(`${request.url} is the last page!`);
+ },
+
+ // This function is called if the page processing failed more than maxRequestRetries+1 times.
+ failedRequestHandler({ request, log }) {
+ log.error(`Request ${request.url} failed too many times.`);
+ },
+});
+
+await crawler.addRequests(['https://news.ycombinator.com/']);
+
+// Run the crawler and wait for it to finish.
+await crawler.run();
+
+console.log('Crawler finished.');
diff --git a/tsconfig.eslint.json b/tsconfig.eslint.json
index 07b0c63bf..01f0f9c8f 100644
--- a/tsconfig.eslint.json
+++ b/tsconfig.eslint.json
@@ -10,6 +10,6 @@
"test/**/*.ts",
"packages/**/test/**/*.js",
"packages/**/test/**/*.ts",
- "docs/**/*"
+ "docs/**/*.ts"
]
}
diff --git a/website/versioned_docs/version-3.3/examples/crawler-plugins/index.mdx b/website/versioned_docs/version-3.3/examples/crawler-plugins/index.mdx
new file mode 100644
index 000000000..625478061
--- /dev/null
+++ b/website/versioned_docs/version-3.3/examples/crawler-plugins/index.mdx
@@ -0,0 +1,78 @@
+---
+id: playwright-puppeteer-extra
+title: Using puppeteer-extra and playwright-extra
+---
+
+import CodeBlock from '@theme/CodeBlock';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import ApiLink from '@site/src/components/ApiLink';
+import PuppeteerExtraSource from '!!raw-loader!./puppeteer-extra.ts';
+import PlaywrightExtraSource from '!!raw-loader!./playwright-extra.ts';
+
+[`puppeteer-extra`](https://www.npmjs.com/package/puppeteer-extra) and [`playwright-extra`](https://www.npmjs.com/package/playwright-extra) are community-built
+libraries that bring in a plugin system to enhance the usage of [`puppeteer`](https://www.npmjs.com/package/puppeteer) and
+[`playwright`](https://www.npmjs.com/package/playwright) respectively (bringing in extra functionality, like improving stealth for
+example by using the [`puppeteer-extra-plugin-stealth`](https://www.npmjs.com/package/puppeteer-extra-plugin-stealth) plugin).
+
+:::tip Available plugins
+
+You can see a list of available plugins on the [`puppeteer-extra`s plugin list](https://www.npmjs.com/package/puppeteer-extra#plugins).
+
+For [`playwright`](https://www.npmjs.com/package/playwright), please see [`playwright-extra`s plugin list](https://www.npmjs.com/package/playwright-extra#plugins) instead.
+
+:::
+
+In this example, we'll show you how to use the [`puppeteer-extra-plugin-stealth`](https://www.npmjs.com/package/puppeteer-extra-plugin-stealth) plugin
+to help you avoid bot detections when crawling your target website.
+
+<Tabs>
+<TabItem value="puppeteer" label="Puppeteer & puppeteer-extra" default>
+
+:::info Before you begin
+
+Make sure you've installed the `puppeteer-extra` and `puppeteer-extra-plugin-stealth` packages via your preferred package manager
+
+```bash
+npm install puppeteer-extra puppeteer-extra-plugin-stealth
+```
+
+:::
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.
+
+:::
+
+<CodeBlock className="language-js" title="src/crawler.ts">
+ {PuppeteerExtraSource}
+</CodeBlock>
+
+</TabItem>
+
+<TabItem value="playwright" label="Playwright & playwright-extra">
+
+:::info Before you begin
+
+Make sure you've installed the `playwright-extra` and `puppeteer-extra-plugin-stealth` packages via your preferred package manager
+
+```bash
+npm install playwright-extra puppeteer-extra-plugin-stealth
+```
+
+:::
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.
+
+:::
+
+<CodeBlock className="language-js" title="src/crawler.ts">
+ {PlaywrightExtraSource}
+</CodeBlock>
+
+</TabItem>
+
+</Tabs>
diff --git a/website/versioned_docs/version-3.3/examples/crawler-plugins/playwright-extra.ts b/website/versioned_docs/version-3.3/examples/crawler-plugins/playwright-extra.ts
new file mode 100644
index 000000000..c2b733f47
--- /dev/null
+++ b/website/versioned_docs/version-3.3/examples/crawler-plugins/playwright-extra.ts
@@ -0,0 +1,74 @@
+import { Dataset, PlaywrightCrawler } from 'crawlee';
+
+// For playwright-extra you will need to import the browser type itself that you want to use!
+// By default, PlaywrightCrawler uses chromium, but you can also use firefox or webkit.
+import { chromium } from 'playwright-extra';
+import stealthPlugin from 'puppeteer-extra-plugin-stealth';
+
+// First, we tell playwright-extra to use the plugin (or plugins) we want.
+// Certain plugins might have options you can pass in - read up on their documentation!
+chromium.use(stealthPlugin());
+
+// Create an instance of the PlaywrightCrawler class - a crawler
+// that automatically loads the URLs in headless Chrome / Playwright.
+const crawler = new PlaywrightCrawler({
+ launchContext: {
+ // !!! You need to specify this option to tell Crawlee to use playwright-extra as the launcher !!!
+ launcher: chromium,
+ launchOptions: {
+ // Other playwright options work as usual
+ headless: true,
+ },
+ },
+
+ // Stop crawling after several pages
+ maxRequestsPerCrawl: 50,
+
+ // This function will be called for each URL to crawl.
+ // Here you can write the Puppeteer scripts you are familiar with,
+ // with the exception that browsers and pages are automatically managed by Crawlee.
+ // The function accepts a single parameter, which is an object with the following fields:
+ // - request: an instance of the Request class with information such as URL and HTTP method
+ // - page: Puppeteer's Page object (see https://pptr.dev/#show=api-class-page)
+ async requestHandler({ request, page, enqueueLinks, log }) {
+ log.info(`Processing ${request.url}...`);
+
+ // A function to be evaluated by Puppeteer within the browser context.
+ const data = await page.$$eval('.athing', ($posts) => {
+ const scrapedData: { title: string; rank: string; href: string }[] = [];
+
+ // We're getting the title, rank and URL of each post on Hacker News.
+ $posts.forEach(($post) => {
+ scrapedData.push({
+ title: $post.querySelector('.title a').innerText,
+ rank: $post.querySelector('.rank').innerText,
+ href: $post.querySelector('.title a').href,
+ });
+ });
+
+ return scrapedData;
+ });
+
+ // Store the results to the default dataset.
+ await Dataset.pushData(data);
+
+ // Find a link to the next page and enqueue it if it exists.
+ const infos = await enqueueLinks({
+ selector: '.morelink',
+ });
+
+ if (infos.processedRequests.length === 0) log.info(`${request.url} is the last page!`);
+ },
+
+ // This function is called if the page processing failed more than maxRequestRetries+1 times.
+ failedRequestHandler({ request, log }) {
+ log.error(`Request ${request.url} failed too many times.`);
+ },
+});
+
+await crawler.addRequests(['https://news.ycombinator.com/']);
+
+// Run the crawler and wait for it to finish.
+await crawler.run();
+
+console.log('Crawler finished.');
diff --git a/website/versioned_docs/version-3.3/examples/crawler-plugins/puppeteer-extra.ts b/website/versioned_docs/version-3.3/examples/crawler-plugins/puppeteer-extra.ts
new file mode 100644
index 000000000..6da7bf8a8
--- /dev/null
+++ b/website/versioned_docs/version-3.3/examples/crawler-plugins/puppeteer-extra.ts
@@ -0,0 +1,71 @@
+import { Dataset, PuppeteerCrawler } from 'crawlee';
+import puppeteerExtra from 'puppeteer-extra';
+import stealthPlugin from 'puppeteer-extra-plugin-stealth';
+
+// First, we tell puppeteer-extra to use the plugin (or plugins) we want.
+// Certain plugins might have options you can pass in - read up on their documentation!
+puppeteerExtra.use(stealthPlugin());
+
+// Create an instance of the PuppeteerCrawler class - a crawler
+// that automatically loads the URLs in headless Chrome / Puppeteer.
+const crawler = new PuppeteerCrawler({
+ launchContext: {
+ // !!! You need to specify this option to tell Crawlee to use puppeteer-extra as the launcher !!!
+ launcher: puppeteerExtra,
+ launchOptions: {
+ // Other puppeteer options work as usual
+ headless: true,
+ },
+ },
+
+ // Stop crawling after several pages
+ maxRequestsPerCrawl: 50,
+
+ // This function will be called for each URL to crawl.
+ // Here you can write the Puppeteer scripts you are familiar with,
+ // with the exception that browsers and pages are automatically managed by Crawlee.
+ // The function accepts a single parameter, which is an object with the following fields:
+ // - request: an instance of the Request class with information such as URL and HTTP method
+ // - page: Puppeteer's Page object (see https://pptr.dev/#show=api-class-page)
+ async requestHandler({ request, page, enqueueLinks, log }) {
+ log.info(`Processing ${request.url}...`);
+
+ // A function to be evaluated by Puppeteer within the browser context.
+ const data = await page.$$eval('.athing', ($posts) => {
+ const scrapedData: { title: string; rank: string; href: string }[] = [];
+
+ // We're getting the title, rank and URL of each post on Hacker News.
+ $posts.forEach(($post) => {
+ scrapedData.push({
+ title: $post.querySelector('.title a').innerText,
+ rank: $post.querySelector('.rank').innerText,
+ href: $post.querySelector('.title a').href,
+ });
+ });
+
+ return scrapedData;
+ });
+
+ // Store the results to the default dataset.
+ await Dataset.pushData(data);
+
+ // Find a link to the next page and enqueue it if it exists.
+ const infos = await enqueueLinks({
+ selector: '.morelink',
+ });
+
+ if (infos.processedRequests.length === 0) log.info(`${request.url} is the last page!`);
+ },
+
+ // This function is called if the page processing failed more than maxRequestRetries+1 times.
+ failedRequestHandler({ request, log }) {
+ log.error(`Request ${request.url} failed too many times.`);
+ },
+});
+
+await crawler.addRequests(['https://news.ycombinator.com/']);
+
+// Run the crawler and wait for it to finish.
+await crawler.run();
+
+console.log('Crawler finished.');