Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
docs: fix JSDOM examples (#1693)
Jindřich Bär5ec089d5628c
diff --git a/docs/examples/jsdom_crawler.mdx b/docs/examples/jsdom_crawler.mdxindex 6f7b27df5..6c5d1764d 100644--- a/docs/examples/jsdom_crawler.mdx+++ b/docs/examples/jsdom_crawler.mdx@@ -6,10 +6,17 @@ title: JSDOM crawlerimport CodeBlock from '@theme/CodeBlock';import ApiLink from '@site/src/components/ApiLink';import JSDOMCrawlerSource from '!!raw-loader!./jsdom_crawler.ts';+import JSDOMCrawlerRunScriptSource from '!!raw-loader!./jsdom_crawler_react.ts';This example demonstrates how to use <ApiLink to="jsdom-crawler/class/JSDOMCrawler">`JSDOMCrawler`</ApiLink> to interact with a website using [jsdom](https://www.npmjs.com/package/jsdom) DOM implementation.Here the script will open a calculator app from the [React examples](https://reactjs.org/community/examples.html), click `1` `+` `1` `=` and extract the result.+<CodeBlock className="language-ts">+ {JSDOMCrawlerRunScriptSource}+</CodeBlock>++In the following example, we use <ApiLink to="jsdom-crawler/class/JSDOMCrawler">`JSDOMCrawler`</ApiLink> to crawl a list of URLs from an external file, load each URL using a plain HTTP request, parse the HTML using the [jsdom](https://www.npmjs.com/package/jsdom) DOM implementation and extract some data from it: the page title and all `h1` tags.+<CodeBlock className="language-ts">{JSDOMCrawlerSource}</CodeBlock>diff --git a/docs/examples/jsdom_crawler.ts b/docs/examples/jsdom_crawler.tsindex 09d2180e8..e72034113 100644--- a/docs/examples/jsdom_crawler.ts+++ b/docs/examples/jsdom_crawler.ts@@ -1,20 +1,64 @@-import { JSDOMCrawler } from '@crawlee/jsdom';+import { Dataset, JSDOMCrawler, log, LogLevel } from 'crawlee';+// Crawlers come with various utilities, e.g. for logging.+// Here we use debug level of logging to improve the debugging experience.+// This functionality is optional!+log.setLevel(LogLevel.DEBUG);++// Create an instance of the JSDOMCrawler class - a crawler+// that automatically loads the URLs and parses their HTML using the jsdom library.const crawler = new JSDOMCrawler({- runScripts: true,- requestHandler: async ({ window }) => {- const { document } = window;- document.querySelectorAll('button')[12].click(); // 1- document.querySelectorAll('button')[15].click(); // +- document.querySelectorAll('button')[12].click(); // 1- document.querySelectorAll('button')[18].click(); // =+ // The crawler downloads and processes the web pages in parallel, with a concurrency+ // automatically managed based on the available system memory and CPU (see AutoscaledPool class).+ // Here we define some hard limits for the concurrency.+ minConcurrency: 10,+ maxConcurrency: 50,++ // On error, retry each page at most once.+ maxRequestRetries: 1,++ // Increase the timeout for processing of each page.+ requestHandlerTimeoutSecs: 30,++ // Limit to 10 requests per one crawl+ maxRequestsPerCrawl: 10,- const result = document.querySelectorAll('.component-display')[0].childNodes[0] as Element;+ // This function will be called for each URL to crawl.+ // It accepts a single parameter, which is an object with options as:+ // https://crawlee.dev/api/jsdom-crawler/interface/JSDOMCrawlerOptions#requestHandler+ // We use for demonstration only 2 of them:+ // - request: an instance of the Request class with information such as the URL that is being crawled and HTTP method+ // - window: the JSDOM window object+ async requestHandler({ request, window }) {+ log.debug(`Processing ${request.url}...`);- console.log(result.innerHTML); // 2+ // Extract data from the page+ const title = window.document.title;+ const h1texts: { text: string }[] = [];+ window.document.querySelectorAll('h1').forEach((element) => {+ h1texts.push({+ text: element.textContent!,+ });+ });++ // Store the results to the dataset. In local configuration,+ // the data will be stored as JSON files in ./storage/datasets/default+ await Dataset.pushData({+ url: request.url,+ title,+ h1texts,+ });+ },++ // This function is called if the page processing failed more than maxRequestRetries + 1 times.+ failedRequestHandler({ request }) {+ log.debug(`Request ${request.url} failed twice.`);},});+// Run the crawler and wait for it to finish.await crawler.run([- 'https://ahfarmer.github.io/calculator/',+ 'https://crawlee.dev',]);++log.debug('Crawler finished.');diff --git a/docs/examples/jsdom_crawler_react.ts b/docs/examples/jsdom_crawler_react.tsnew file mode 100644index 000000000..8b3ec163b--- /dev/null+++ b/docs/examples/jsdom_crawler_react.ts@@ -0,0 +1,32 @@+import { JSDOMCrawler, log } from 'crawlee';++// Create an instance of the JSDOMCrawler class - crawler that automatically+// loads the URLs and parses their HTML using the jsdom library.+const crawler = new JSDOMCrawler({+ // Setting the `runScripts` option to `true` allows the crawler to execute client-side+ // JavaScript code on the page. This is required for some websites (such as the React application in this example), but may pose a security risk.+ runScripts: true,+ // This function will be called for each crawled URL.+ // Here we extract the window object from the options and use it to extract data from the page.+ requestHandler: async ({ window }) => {+ const { document } = window;+ // The `document` object is analogous to the `window.document` object you know from your favourite web browsers.+ // Thanks to this, you can use the regular browser-side APIs here.+ document.querySelectorAll('button')[12].click(); // 1+ document.querySelectorAll('button')[15].click(); // ++ document.querySelectorAll('button')[12].click(); // 1+ document.querySelectorAll('button')[18].click(); // =++ const result = document.querySelectorAll('.component-display')[0].childNodes[0] as Element;+ // The result is passed to the console. Unlike with Playwright or Puppeteer crawlers,+ // this console call goes to the Node.js console, not the browser console. All the code here runs right in Node.js!+ log.info(result.innerHTML); // 2+ },+});++// Run the crawler and wait for it to finish.+await crawler.run([+ 'https://ahfarmer.github.io/calculator/',+]);++log.debug('Crawler finished.');\ No newline at end of filediff --git a/website/versioned_docs/version-3.0/examples/jsdom_crawler.mdx b/website/versioned_docs/version-3.0/examples/jsdom_crawler.mdxindex 72decf749..fb0d4c297 100644--- a/website/versioned_docs/version-3.0/examples/jsdom_crawler.mdx+++ b/website/versioned_docs/version-3.0/examples/jsdom_crawler.mdx@@ -11,4 +11,4 @@ This example demonstrates how to use <ApiLink to="jsdom-crawler/class/JSDOMCrawl<CodeBlock className="language-js">{JSDOMCrawlerSource}-</CodeBlock>+</CodeBlock>\ No newline at end of filediff --git a/website/versioned_docs/version-3.0/examples/jsdom_crawler.ts b/website/versioned_docs/version-3.0/examples/jsdom_crawler.tsindex 13b22fc74..e72034113 100644--- a/website/versioned_docs/version-3.0/examples/jsdom_crawler.ts+++ b/website/versioned_docs/version-3.0/examples/jsdom_crawler.ts@@ -5,8 +5,8 @@ import { Dataset, JSDOMCrawler, log, LogLevel } from 'crawlee';// This functionality is optional!log.setLevel(LogLevel.DEBUG);-// Create an instance of the CheerioCrawler class - a crawler-// that automatically loads the URLs and parses their HTML using the cheerio library.+// Create an instance of the JSDOMCrawler class - a crawler+// that automatically loads the URLs and parses their HTML using the jsdom library.const crawler = new JSDOMCrawler({// The crawler downloads and processes the web pages in parallel, with a concurrency// automatically managed based on the available system memory and CPU (see AutoscaledPool class).@@ -25,17 +25,17 @@ const crawler = new JSDOMCrawler({// This function will be called for each URL to crawl.// It accepts a single parameter, which is an object with options as:- // https://crawlee.dev/api/cheerio-crawler/interface/CheerioCrawlerOptions#requestHandler+ // https://crawlee.dev/api/jsdom-crawler/interface/JSDOMCrawlerOptions#requestHandler// We use for demonstration only 2 of them:// - request: an instance of the Request class with information such as the URL that is being crawled and HTTP method- // - $: the cheerio object containing parsed HTML+ // - window: the JSDOM window objectasync requestHandler({ request, window }) {log.debug(`Processing ${request.url}...`);// Extract data from the pageconst title = window.document.title;const h1texts: { text: string }[] = [];- document.querySelectorAll('h1').forEach((element) => {+ window.document.querySelectorAll('h1').forEach((element) => {h1texts.push({text: element.textContent!,});diff --git a/website/versioned_docs/version-3.1/examples/jsdom_crawler.mdx b/website/versioned_docs/version-3.1/examples/jsdom_crawler.mdxindex 72decf749..9be862195 100644--- a/website/versioned_docs/version-3.1/examples/jsdom_crawler.mdx+++ b/website/versioned_docs/version-3.1/examples/jsdom_crawler.mdx@@ -6,8 +6,16 @@ title: JSDOM crawlerimport CodeBlock from '@theme/CodeBlock';import ApiLink from '@site/src/components/ApiLink';import JSDOMCrawlerSource from '!!raw-loader!./jsdom_crawler.ts';+import JSDOMCrawlerRunScriptSource from '!!raw-loader!./jsdom_crawler_react.ts';-This example demonstrates how to use <ApiLink to="jsdom-crawler/class/JSDOMCrawler">`JSDOMCrawler`</ApiLink> to crawl a list of URLs from an external file, load each URL using a plain HTTP request, parse the HTML using the [jsdom](https://www.npmjs.com/package/jsdom) DOM implementation and extract some data from it: the page title and all `h1` tags.+This example demonstrates how to use <ApiLink to="jsdom-crawler/class/JSDOMCrawler">`JSDOMCrawler`</ApiLink> to interact with a website using [jsdom](https://www.npmjs.com/package/jsdom) DOM implementation.+Here the script will open a calculator app from the [React examples](https://reactjs.org/community/examples.html), click `1` `+` `1` `=` and extract the result.++<CodeBlock className="language-ts">+ {JSDOMCrawlerRunScriptSource}+</CodeBlock>++In the following example, we use <ApiLink to="jsdom-crawler/class/JSDOMCrawler">`JSDOMCrawler`</ApiLink> to crawl a list of URLs from an external file, load each URL using a plain HTTP request, parse the HTML using the [jsdom](https://www.npmjs.com/package/jsdom) DOM implementation and extract some data from it: the page title and all `h1` tags.<CodeBlock className="language-js">{JSDOMCrawlerSource}diff --git a/website/versioned_docs/version-3.1/examples/jsdom_crawler.ts b/website/versioned_docs/version-3.1/examples/jsdom_crawler.tsindex 13b22fc74..e72034113 100644--- a/website/versioned_docs/version-3.1/examples/jsdom_crawler.ts+++ b/website/versioned_docs/version-3.1/examples/jsdom_crawler.ts@@ -5,8 +5,8 @@ import { Dataset, JSDOMCrawler, log, LogLevel } from 'crawlee';// This functionality is optional!log.setLevel(LogLevel.DEBUG);-// Create an instance of the CheerioCrawler class - a crawler-// that automatically loads the URLs and parses their HTML using the cheerio library.+// Create an instance of the JSDOMCrawler class - a crawler+// that automatically loads the URLs and parses their HTML using the jsdom library.const crawler = new JSDOMCrawler({// The crawler downloads and processes the web pages in parallel, with a concurrency// automatically managed based on the available system memory and CPU (see AutoscaledPool class).@@ -25,17 +25,17 @@ const crawler = new JSDOMCrawler({// This function will be called for each URL to crawl.// It accepts a single parameter, which is an object with options as:- // https://crawlee.dev/api/cheerio-crawler/interface/CheerioCrawlerOptions#requestHandler+ // https://crawlee.dev/api/jsdom-crawler/interface/JSDOMCrawlerOptions#requestHandler// We use for demonstration only 2 of them:// - request: an instance of the Request class with information such as the URL that is being crawled and HTTP method- // - $: the cheerio object containing parsed HTML+ // - window: the JSDOM window objectasync requestHandler({ request, window }) {log.debug(`Processing ${request.url}...`);// Extract data from the pageconst title = window.document.title;const h1texts: { text: string }[] = [];- document.querySelectorAll('h1').forEach((element) => {+ window.document.querySelectorAll('h1').forEach((element) => {h1texts.push({text: element.textContent!,});diff --git a/website/versioned_docs/version-3.1/examples/jsdom_crawler_react.ts b/website/versioned_docs/version-3.1/examples/jsdom_crawler_react.tsnew file mode 100644index 000000000..8b3ec163b--- /dev/null+++ b/website/versioned_docs/version-3.1/examples/jsdom_crawler_react.ts@@ -0,0 +1,32 @@+import { JSDOMCrawler, log } from 'crawlee';++// Create an instance of the JSDOMCrawler class - crawler that automatically+// loads the URLs and parses their HTML using the jsdom library.+const crawler = new JSDOMCrawler({+ // Setting the `runScripts` option to `true` allows the crawler to execute client-side+ // JavaScript code on the page. This is required for some websites (such as the React application in this example), but may pose a security risk.+ runScripts: true,+ // This function will be called for each crawled URL.+ // Here we extract the window object from the options and use it to extract data from the page.+ requestHandler: async ({ window }) => {+ const { document } = window;+ // The `document` object is analogous to the `window.document` object you know from your favourite web browsers.+ // Thanks to this, you can use the regular browser-side APIs here.+ document.querySelectorAll('button')[12].click(); // 1+ document.querySelectorAll('button')[15].click(); // ++ document.querySelectorAll('button')[12].click(); // 1+ document.querySelectorAll('button')[18].click(); // =++ const result = document.querySelectorAll('.component-display')[0].childNodes[0] as Element;+ // The result is passed to the console. Unlike with Playwright or Puppeteer crawlers,+ // this console call goes to the Node.js console, not the browser console. All the code here runs right in Node.js!+ log.info(result.innerHTML); // 2+ },+});++// Run the crawler and wait for it to finish.+await crawler.run([+ 'https://ahfarmer.github.io/calculator/',+]);++log.debug('Crawler finished.');\ No newline at end of file