Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: dom-crawler (#1451)
Szymon Marczakddb6cf050fc0
diff --git a/docs/examples/dom_crawler.mdx b/docs/examples/dom_crawler.mdxnew file mode 100644index 000000000..9bda5246d--- /dev/null+++ b/docs/examples/dom_crawler.mdx@@ -0,0 +1,14 @@+---+id: dom-crawler+title: DOM crawler+---++import CodeBlock from '@theme/CodeBlock';+import ApiLink from '@site/src/components/ApiLink';+import DOMCrawlerSource from '!!raw-loader!./dom_crawler.ts';++This example demonstrates how to use <ApiLink to="dom-crawler/class/DOMCrawler">`DOMCrawler`</ApiLink> to crawl a list of URLs from an external file, load each URL using a plain HTTP request, parse the HTML using the [jsdom](https://www.npmjs.com/package/jsdom) DOM implementation and extract some data from it: the page title and all `h1` tags.++<CodeBlock className="language-js">+ {DOMCrawlerSource}+</CodeBlock>diff --git a/docs/examples/dom_crawler.ts b/docs/examples/dom_crawler.tsnew file mode 100644index 000000000..d54102fcd--- /dev/null+++ b/docs/examples/dom_crawler.ts@@ -0,0 +1,64 @@+import { Dataset, DOMCrawler, log, LogLevel } from 'crawlee';++// Crawlers come with various utilities, e.g. for logging.+// Here we use debug level of logging to improve the debugging experience.+// This functionality is optional!+log.setLevel(LogLevel.DEBUG);++// Create an instance of the CheerioCrawler class - a crawler+// that automatically loads the URLs and parses their HTML using the cheerio library.+const crawler = new DOMCrawler({+ // The crawler downloads and processes the web pages in parallel, with a concurrency+ // automatically managed based on the available system memory and CPU (see AutoscaledPool class).+ // Here we define some hard limits for the concurrency.+ minConcurrency: 10,+ maxConcurrency: 50,++ // On error, retry each page at most once.+ maxRequestRetries: 1,++ // Increase the timeout for processing of each page.+ requestHandlerTimeoutSecs: 30,++ // Limit to 10 requests per one crawl+ maxRequestsPerCrawl: 10,++ // This function will be called for each URL to crawl.+ // It accepts a single parameter, which is an object with options as:+ // https://crawlee.dev/api/cheerio-crawler/interface/CheerioCrawlerOptions#requestHandler+ // We use for demonstration only 2 of them:+ // - request: an instance of the Request class with information such as the URL that is being crawled and HTTP method+ // - $: the cheerio object containing parsed HTML+ async requestHandler({ request, window }) {+ log.debug(`Processing ${request.url}...`);++ // Extract data from the page+ const title = window.document.title;+ const h1texts: { text: string }[] = [];+ document.querySelectorAll('h1').forEach((element) => {+ h1texts.push({+ text: element.textContent!,+ });+ });++ // Store the results to the dataset. In local configuration,+ // the data will be stored as JSON files in ./storage/datasets/default+ await Dataset.pushData({+ url: request.url,+ title,+ h1texts,+ });+ },++ // This function is called if the page processing failed more than maxRequestRetries + 1 times.+ failedRequestHandler({ request }) {+ log.debug(`Request ${request.url} failed twice.`);+ },+});++// Run the crawler and wait for it to finish.+await crawler.run([+ 'https://crawlee.dev',+]);++log.debug('Crawler finished.');diff --git a/docs/examples/tsconfig.json b/docs/examples/tsconfig.jsonindex 397bead8b..7758dc0ba 100644--- a/docs/examples/tsconfig.json+++ b/docs/examples/tsconfig.json@@ -7,6 +7,7 @@"@crawlee/basic": ["packages/basic-crawler/src"],"@crawlee/browser": ["packages/browser-crawler/src"],"@crawlee/http": ["packages/http-crawler/src"],+ "@crawlee/dom": ["packages/dom-crawler/src"],"@crawlee/cheerio": ["packages/cheerio-crawler/src"],"@crawlee/playwright": ["packages/playwright-crawler/src"],"@crawlee/puppeteer": ["packages/puppeteer-crawler/src"],diff --git a/docs/guides/dom_crawler.mdx b/docs/guides/dom_crawler.mdxnew file mode 100644index 000000000..618e26ffb--- /dev/null+++ b/docs/guides/dom_crawler.mdx@@ -0,0 +1,61 @@+---+id: dom-crawler-guide+title: "DOMCrawler guide"+sidebar_label: "DOMCrawler"+description: Your first steps into the world of scraping with Crawlee+---++import ApiLink from '@site/src/components/ApiLink';++​<ApiLink to="dom-crawler/class/DOMCrawler">`DOMCrawler`</ApiLink> is very useful for scraping with the Window API.++## How the crawler works++​<ApiLink to="dom-crawler/class/DOMCrawler">`DOMCrawler`</ApiLink> crawls by making plain HTTP requests to the provided URLs using the specialized [got-scraping](https://github.com/apify/got-scraping) HTTP client. The URLs are fed to the crawler using <ApiLink to="core/class/RequestQueue">`RequestQueue`</ApiLink>. The HTTP responses it gets back are usually HTML pages. The same pages you would get in your browser when you first load a URL. But it can handle any content types with the help of the <ApiLink to="dom-crawler/interface/DOMCrawlerOptions#additionalMimeTypes">`additionalMimeTypes`</ApiLink> option.++:::info++Modern web pages often do not serve all of their content in the first HTML response, but rather the first HTML contains links to other resources such as CSS and JavaScript that get downloaded afterwards, and together they create the final page. To crawl those, see <ApiLink to="puppeteer-crawler/class/PuppeteerCrawler">`PuppeteerCrawler`</ApiLink> and <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink>.++:::++Once the page's HTML is retrieved, the crawler will pass it to [JSDOM](https://www.npmjs.com/package/jsdom) for parsing. The result is a `window` property, which should be familiar to frontend developers. You can use the Window API to do all sorts of lookups and manipulation of the page's HTML, but in scraping, you will mostly use it to find specific HTML elements and extract their data.++Example use of browser JavaScript:++```ts+// Return the page title+document.title; // browsers+window.document.title; // JSDOM+```++## When to use `DOMCrawler`++`DOMCrawler` really shines when `CheerioCrawler` is just not enough. There is an entire set of [APIs](https://developer.mozilla.org/en-US/docs/Web/API/HTML_DOM_API) available!++**Advantages:**++- Easy to set up+- Familiar for frontend developers+- Content can be manipulated+- Automatically avoids some anti-scraping bans++**Disadvantages:**++- Slower than `CheerioCrawler`+- Does not work for websites that require JavaScript rendering+- May easily overload the target website with requests++## Example use of Element API++### Find all links on a page++This snippet finds all `<a>` elements which have the `href` attribute and extracts the hrefs into an array.++```js+Array.from(document.querySelectorAll('a[href]')).map((a) => a.href);+```++### Other examples++Visit the [Examples](../examples) section to browse examples of `DOMCrawler` usage. Almost all examples show `DOMCrawler` code in their code tabs.diff --git a/docs/guides/proxy_management.mdx b/docs/guides/proxy_management.mdxindex a2a2e779a..2c1d7ab8a 100644--- a/docs/guides/proxy_management.mdx+++ b/docs/guides/proxy_management.mdx@@ -11,15 +11,18 @@ import TabItem from '@theme/TabItem';import CodeBlock from '@theme/CodeBlock';import HttpSource from '!!raw-loader!./proxy_management_integration_http.ts';+import DOMSource from '!!raw-loader!./proxy_management_integration_dom.ts';import CheerioSource from '!!raw-loader!./proxy_management_integration_cheerio.ts';import PlaywrightSource from '!!raw-loader!./proxy_management_integration_playwright.ts';import PuppeteerSource from '!!raw-loader!./proxy_management_integration_puppeteer.ts';import SessionStandaloneSource from '!!raw-loader!./proxy_management_session_standalone.ts';import SessionHttpSource from '!!raw-loader!./proxy_management_session_http.ts';+import SessionDOMSource from '!!raw-loader!./proxy_management_session_dom.ts';import SessionCheerioSource from '!!raw-loader!./proxy_management_session_cheerio.ts';import SessionPlaywrightSource from '!!raw-loader!./proxy_management_session_playwright.ts';import SessionPuppeteerSource from '!!raw-loader!./proxy_management_session_puppeteer.ts';import InspectionHttpSource from '!!raw-loader!./proxy_management_inspection_http.ts';+import InspectionDOMSource from '!!raw-loader!./proxy_management_inspection_dom.ts';import InspectionCheerioSource from '!!raw-loader!./proxy_management_inspection_cheerio.ts';import InspectionPlaywrightSource from '!!raw-loader!./proxy_management_inspection_playwright.ts';import InspectionPuppeteerSource from '!!raw-loader!./proxy_management_inspection_puppeteer.ts';@@ -58,7 +61,7 @@ All our proxy needs are managed by the <ApiLink to="core/class/ProxyConfiguratio### Crawler integration-`ProxyConfiguration` integrates seamlessly into <ApiLink to="http-crawler/class/HttpCrawler">`HttpCrawler`</ApiLink>, <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink>, <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> and <ApiLink to="puppeteer-crawler/class/PuppeteerCrawler">`PuppeteerCrawler`</ApiLink>.+`ProxyConfiguration` integrates seamlessly into <ApiLink to="http-crawler/class/HttpCrawler">`HttpCrawler`</ApiLink>, <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink>, <ApiLink to="dom-crawler/class/DOMCrawler">`DOMCrawler`</ApiLink>, <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> and <ApiLink to="puppeteer-crawler/class/PuppeteerCrawler">`PuppeteerCrawler`</ApiLink>.<Tabs groupId="proxy_session_management"><TabItem value="http" label="HttpCrawler">@@ -71,6 +74,11 @@ All our proxy needs are managed by the <ApiLink to="core/class/ProxyConfiguratio{CheerioSource}</CodeBlock></TabItem>+ <TabItem value="dom" label="DOMCrawler">+ <CodeBlock language="js">+ {DOMSource}+ </CodeBlock>+ </TabItem><TabItem value="playwright" label="PlaywrightCrawler"><CodeBlock language="js">{PlaywrightSource}@@ -102,6 +110,11 @@ When no `sessionId` is provided, our proxy URLs are rotated round-robin.{SessionCheerioSource}</CodeBlock></TabItem>+ <TabItem value="dom" label="DOMCrawler">+ <CodeBlock language="js">+ {SessionDOMSource}+ </CodeBlock>+ </TabItem><TabItem value="playwright" label="PlaywrightCrawler"><CodeBlock language="js">{SessionPlaywrightSource}@@ -121,7 +134,7 @@ When no `sessionId` is provided, our proxy URLs are rotated round-robin.## Inspecting current proxy in Crawlers-`HttpCrawler`, `CheerioCrawler`, `PlaywrightCrawler` and `PuppeteerCrawler` grant access to information about the currently used proxy+`HttpCrawler`, `CheerioCrawler`, `DOMCrawler`, `PlaywrightCrawler` and `PuppeteerCrawler` grant access to information about the currently used proxyin their `requestHandler` using a <ApiLink to="core/interface/ProxyInfo">`proxyInfo`</ApiLink> object.With the `proxyInfo` object, we can easily access the proxy URL.@@ -136,6 +149,11 @@ With the `proxyInfo` object, we can easily access the proxy URL.{InspectionCheerioSource}</CodeBlock></TabItem>+ <TabItem value="dom" label="DOMCrawler">+ <CodeBlock language="js">+ {InspectionDOMSource}+ </CodeBlock>+ </TabItem><TabItem value="playwright" label="PlaywrightCrawler"><CodeBlock language="js">{InspectionPlaywrightSource}diff --git a/docs/guides/proxy_management_inspection_dom.ts b/docs/guides/proxy_management_inspection_dom.tsnew file mode 100644index 000000000..88260eefb--- /dev/null+++ b/docs/guides/proxy_management_inspection_dom.ts@@ -0,0 +1,11 @@+import { DOMCrawler, ProxyConfiguration } from 'crawlee';++const proxyConfiguration = new ProxyConfiguration({ /* opts */ });++const crawler = new DOMCrawler({+ proxyConfiguration,+ async requestHandler({ proxyInfo }) {+ console.log(proxyInfo);+ },+ // ...+});diff --git a/docs/guides/proxy_management_integration_dom.ts b/docs/guides/proxy_management_integration_dom.tsnew file mode 100644index 000000000..f2e4d4203--- /dev/null+++ b/docs/guides/proxy_management_integration_dom.ts@@ -0,0 +1,13 @@+import { DOMCrawler, ProxyConfiguration } from 'crawlee';++const proxyConfiguration = new ProxyConfiguration({+ proxyUrls: [+ 'http://proxy-1.com',+ 'http://proxy-2.com',+ ],+});++const crawler = new DOMCrawler({+ proxyConfiguration,+ // ...+});diff --git a/docs/guides/proxy_management_session_dom.ts b/docs/guides/proxy_management_session_dom.tsnew file mode 100644index 000000000..5e792919e--- /dev/null+++ b/docs/guides/proxy_management_session_dom.ts@@ -0,0 +1,10 @@+import { DOMCrawler, ProxyConfiguration } from 'crawlee';++const proxyConfiguration = new ProxyConfiguration({ /* opts */ });++const crawler = new DOMCrawler({+ useSessionPool: true,+ persistCookiesPerSession: true,+ proxyConfiguration,+ // ...+});diff --git a/docs/guides/session_management.mdx b/docs/guides/session_management.mdxindex a7dc34e16..9429be3d9 100644--- a/docs/guides/session_management.mdx+++ b/docs/guides/session_management.mdx@@ -13,6 +13,7 @@ import CodeBlock from '@theme/CodeBlock';import BasicSource from '!!raw-loader!./session_management_basic.ts';import HttpSource from '!!raw-loader!./session_management_http.ts';import CheerioSource from '!!raw-loader!./session_management_cheerio.ts';+import DOMSource from '!!raw-loader!./session_management_dom.ts';import PlaywrightSource from '!!raw-loader!./session_management_playwright.ts';import PuppeteerSource from '!!raw-loader!./session_management_puppeteer.ts';import StandaloneSource from '!!raw-loader!./session_management_standalone.ts';@@ -30,6 +31,7 @@ Now let's take a look at the examples of how to use Session pool:- with <ApiLink to="basic-crawler/class/BasicCrawler">`BasicCrawler`</ApiLink>;- with <ApiLink to="http-crawler/class/HttpCrawler">`HttpCrawler`</ApiLink>;- with <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink>;+- with <ApiLink to="dom-crawler/class/DOMCrawler">`DOMCrawler`</ApiLink>;- with <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink>;- with <ApiLink to="puppeteer-crawler/class/PuppeteerCrawler">`PuppeteerCrawler`</ApiLink>;- without a crawler (standalone usage to manage sessions manually).@@ -50,6 +52,11 @@ Now let's take a look at the examples of how to use Session pool:{CheerioSource}</CodeBlock></TabItem>+ <TabItem value="dom" label="DOMCrawler">+ <CodeBlock language="js">+ {DOMSource}+ </CodeBlock>+ </TabItem><TabItem value="playwright" label="PlaywrightCrawler"><CodeBlock language="js">{PlaywrightSource}diff --git a/docs/guides/session_management_dom.ts b/docs/guides/session_management_dom.tsnew file mode 100644index 000000000..1e754581b--- /dev/null+++ b/docs/guides/session_management_dom.ts@@ -0,0 +1,26 @@+import { DOMCrawler, ProxyConfiguration } from 'crawlee';++const proxyConfiguration = new ProxyConfiguration({ /* opts */ });++const crawler = new DOMCrawler({+ // To use the proxy IP session rotation logic, you must turn the proxy usage on.+ proxyConfiguration,+ // Activates the Session pool (default is true).+ useSessionPool: true,+ // Overrides default Session pool configuration.+ sessionPoolOptions: { maxPoolSize: 100 },+ // Set to true if you want the crawler to save cookies per session,+ // and set the cookie header to request automatically (default is true).+ persistCookiesPerSession: true,+ async requestHandler({ session, window }) {+ const title = window.document.title;++ if (title === 'Blocked') {+ session.retire();+ } else if (title === 'Not sure if blocked, might also be a connection error') {+ session.markBad();+ } else {+ // session.markGood() - this step is done automatically in BasicCrawler.+ }+ },+});diff --git a/docs/guides/session_management_http.ts b/docs/guides/session_management_http.tsindex cfb83cd70..b6d40d3ec 100644--- a/docs/guides/session_management_http.ts+++ b/docs/guides/session_management_http.ts@@ -13,7 +13,7 @@ const crawler = new HttpCrawler({// and set the cookie header to request automatically (default is true).persistCookiesPerSession: true,async requestHandler({ session, body }) {- const title = body.match(/<title>(.*?)<\/title>/)?.[1];+ const title = body.match(/<title(?:.*?)>(.*?)<\/title>/)?.[1];if (title === 'Blocked') {session.retire();diff --git a/docs/upgrading/upgrading_v3.md b/docs/upgrading/upgrading_v3.mdindex 79ae4b284..3d671e4aa 100644--- a/docs/upgrading/upgrading_v3.md+++ b/docs/upgrading/upgrading_v3.md@@ -22,7 +22,8 @@ The [`crawlee`](https://www.npmjs.com/package/crawlee) package consists of sever- `@crawlee/core`: the base for all the crawler implementations, also contains things like `Request`, `RequestQueue`, `RequestList` or `Dataset` classes- `@crawlee/basic`: exports `BasicCrawler`-- `@crawlee/http`: exports `HttpCrawler` (which is used for creating `@crawlee/cheerio`)+- `@crawlee/http`: exports `HttpCrawler` (which is used for creating `@crawlee/dom` and `@crawlee/cheerio`)+- `@crawlee/dom`: exports `DOMCrawler`- `@crawlee/cheerio`: exports `CheerioCrawler`- `@crawlee/browser`: exports `BrowserCrawler` (which is used for creating `@crawlee/playwright` and `@crawlee/puppeteer`)- `@crawlee/playwright`: exports `PlaywrightCrawler`diff --git a/jest.config.js b/jest.config.jsindex 22b2f89f2..ded5b9d50 100644--- a/jest.config.js+++ b/jest.config.js@@ -16,6 +16,7 @@ module.exports = {'^@crawlee/basic$': '<rootDir>/packages/basic-crawler/src','^@crawlee/browser$': '<rootDir>/packages/browser-crawler/src','^@crawlee/http$': '<rootDir>/packages/http-crawler/src',+ '^@crawlee/dom$': '<rootDir>/packages/dom-crawler/src','^@crawlee/cheerio$': '<rootDir>/packages/cheerio-crawler/src','^@crawlee/playwright$': '<rootDir>/packages/playwright-crawler/src','^@crawlee/puppeteer$': '<rootDir>/packages/puppeteer-crawler/src',diff --git a/package-lock.json b/package-lock.jsonindex ddf5d66b2..7e9e02346 100644--- a/package-lock.json+++ b/package-lock.json@@ -1030,6 +1030,10 @@"resolved": "packages/core","link": true},+ "node_modules/@crawlee/dom": {+ "resolved": "packages/dom-crawler",+ "link": true+ },"node_modules/@crawlee/http": {"resolved": "packages/http-crawler","link": true@@ -3500,7 +3504,6 @@"version": "2.0.0","resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz","integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==",- "dev": true,"engines": {"node": ">= 10"}@@ -3737,6 +3740,16 @@"pretty-format": "^28.0.0"}},+ "node_modules/@types/jsdom": {+ "version": "20.0.0",+ "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-20.0.0.tgz",+ "integrity": "sha512-YfAchFs0yM1QPDrLm2VHe+WHGtqms3NXnXAMolrgrVP6fgBHHXy1ozAbo/dFtPNtZC/m66bPiCTWYmqp1F14gA==",+ "dependencies": {+ "@types/node": "*",+ "@types/tough-cookie": "*",+ "parse5": "^7.0.0"+ }+ },"node_modules/@types/json-buffer": {"version": "3.0.0","resolved": "https://registry.npmjs.org/@types/json-buffer/-/json-buffer-3.0.0.tgz",@@ -4186,6 +4199,11 @@"npm": ">=7.0.0"}},+ "node_modules/abab": {+ "version": "2.0.6",+ "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",+ "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA=="+ },"node_modules/abbrev": {"version": "1.1.1","resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",@@ -4209,7 +4227,6 @@"version": "8.8.0","resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz","integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==",- "dev": true,"bin": {"acorn": "bin/acorn"},@@ -4217,6 +4234,34 @@"node": ">=0.4.0"}},+ "node_modules/acorn-globals": {+ "version": "6.0.0",+ "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz",+ "integrity": "sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg==",+ "dependencies": {+ "acorn": "^7.1.1",+ "acorn-walk": "^7.1.1"+ }+ },+ "node_modules/acorn-globals/node_modules/acorn": {+ "version": "7.4.1",+ "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz",+ "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==",+ "bin": {+ "acorn": "bin/acorn"+ },+ "engines": {+ "node": ">=0.4.0"+ }+ },+ "node_modules/acorn-globals/node_modules/acorn-walk": {+ "version": "7.2.0",+ "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz",+ "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==",+ "engines": {+ "node": ">=0.4.0"+ }+ },"node_modules/acorn-jsx": {"version": "5.3.2","resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz",@@ -4253,7 +4298,6 @@"version": "6.0.2","resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz","integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",- "devOptional": true,"dependencies": {"debug": "4"},@@ -4743,6 +4787,11 @@"retry": "0.13.1"}},+ "node_modules/asynckit": {+ "version": "0.4.0",+ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",+ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="+ },"node_modules/at-least-node": {"version": "1.0.0","resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz",@@ -5041,6 +5090,11 @@"node": ">=8"}},+ "node_modules/browser-process-hrtime": {+ "version": "1.0.0",+ "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz",+ "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow=="+ },"node_modules/browserslist": {"version": "4.21.3","resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.3.tgz",@@ -5711,6 +5765,17 @@"node": ">=8.0.0"}},+ "node_modules/combined-stream": {+ "version": "1.0.8",+ "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",+ "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",+ "dependencies": {+ "delayed-stream": "~1.0.0"+ },+ "engines": {+ "node": ">= 0.8"+ }+ },"node_modules/commander": {"version": "9.4.0","resolved": "https://registry.npmjs.org/commander/-/commander-9.4.0.tgz",@@ -6130,6 +6195,27 @@"url": "https://github.com/sponsors/fb55"}},+ "node_modules/cssom": {+ "version": "0.5.0",+ "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",+ "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="+ },+ "node_modules/cssstyle": {+ "version": "2.3.0",+ "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz",+ "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==",+ "dependencies": {+ "cssom": "~0.3.6"+ },+ "engines": {+ "node": ">=8"+ }+ },+ "node_modules/cssstyle/node_modules/cssom": {+ "version": "0.3.8",+ "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz",+ "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg=="+ },"node_modules/damerau-levenshtein": {"version": "1.0.8","resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz",@@ -6145,6 +6231,50 @@"node": ">=8"}},+ "node_modules/data-urls": {+ "version": "3.0.2",+ "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz",+ "integrity": "sha512-Jy/tj3ldjZJo63sVAvg6LHt2mHvl4V6AgRAmNDtLdm7faqtsx+aJG42rsyCo9JCoRVKwPFzKlIPx3DIibwSIaQ==",+ "dependencies": {+ "abab": "^2.0.6",+ "whatwg-mimetype": "^3.0.0",+ "whatwg-url": "^11.0.0"+ },+ "engines": {+ "node": ">=12"+ }+ },+ "node_modules/data-urls/node_modules/tr46": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz",+ "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==",+ "dependencies": {+ "punycode": "^2.1.1"+ },+ "engines": {+ "node": ">=12"+ }+ },+ "node_modules/data-urls/node_modules/webidl-conversions": {+ "version": "7.0.0",+ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",+ "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",+ "engines": {+ "node": ">=12"+ }+ },+ "node_modules/data-urls/node_modules/whatwg-url": {+ "version": "11.0.0",+ "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz",+ "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==",+ "dependencies": {+ "tr46": "^3.0.0",+ "webidl-conversions": "^7.0.0"+ },+ "engines": {+ "node": ">=12"+ }+ },"node_modules/dateformat": {"version": "3.0.3","resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz",@@ -6158,7 +6288,6 @@"version": "4.3.4","resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz","integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",- "devOptional": true,"dependencies": {"ms": "2.1.2"},@@ -6211,6 +6340,11 @@"node": ">=0.10.0"}},+ "node_modules/decimal.js": {+ "version": "10.3.1",+ "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz",+ "integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ=="+ },"node_modules/decompress-response": {"version": "6.0.0","resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",@@ -6260,8 +6394,7 @@"node_modules/deep-is": {"version": "0.1.4","resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",- "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",- "dev": true+ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ=="},"node_modules/deepmerge": {"version": "4.2.2",@@ -6313,6 +6446,14 @@"url": "https://github.com/sponsors/ljharb"}},+ "node_modules/delayed-stream": {+ "version": "1.0.0",+ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",+ "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",+ "engines": {+ "node": ">=0.4.0"+ }+ },"node_modules/delegates": {"version": "1.0.0","resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",@@ -6458,6 +6599,25 @@}]},+ "node_modules/domexception": {+ "version": "4.0.0",+ "resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz",+ "integrity": "sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==",+ "dependencies": {+ "webidl-conversions": "^7.0.0"+ },+ "engines": {+ "node": ">=12"+ }+ },+ "node_modules/domexception/node_modules/webidl-conversions": {+ "version": "7.0.0",+ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",+ "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",+ "engines": {+ "node": ">=12"+ }+ },"node_modules/domhandler": {"version": "2.4.2","resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",@@ -6766,6 +6926,74 @@"url": "https://github.com/sponsors/sindresorhus"}},+ "node_modules/escodegen": {+ "version": "2.0.0",+ "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.0.0.tgz",+ "integrity": "sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw==",+ "dependencies": {+ "esprima": "^4.0.1",+ "estraverse": "^5.2.0",+ "esutils": "^2.0.2",+ "optionator": "^0.8.1"+ },+ "bin": {+ "escodegen": "bin/escodegen.js",+ "esgenerate": "bin/esgenerate.js"+ },+ "engines": {+ "node": ">=6.0"+ },+ "optionalDependencies": {+ "source-map": "~0.6.1"+ }+ },+ "node_modules/escodegen/node_modules/levn": {+ "version": "0.3.0",+ "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz",+ "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==",+ "dependencies": {+ "prelude-ls": "~1.1.2",+ "type-check": "~0.3.2"+ },+ "engines": {+ "node": ">= 0.8.0"+ }+ },+ "node_modules/escodegen/node_modules/optionator": {+ "version": "0.8.3",+ "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz",+ "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==",+ "dependencies": {+ "deep-is": "~0.1.3",+ "fast-levenshtein": "~2.0.6",+ "levn": "~0.3.0",+ "prelude-ls": "~1.1.2",+ "type-check": "~0.3.2",+ "word-wrap": "~1.2.3"+ },+ "engines": {+ "node": ">= 0.8.0"+ }+ },+ "node_modules/escodegen/node_modules/prelude-ls": {+ "version": "1.1.2",+ "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",+ "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==",+ "engines": {+ "node": ">= 0.8.0"+ }+ },+ "node_modules/escodegen/node_modules/type-check": {+ "version": "0.3.2",+ "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz",+ "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==",+ "dependencies": {+ "prelude-ls": "~1.1.2"+ },+ "engines": {+ "node": ">= 0.8.0"+ }+ },"node_modules/eslint": {"version": "8.20.0","resolved": "https://registry.npmjs.org/eslint/-/eslint-8.20.0.tgz",@@ -7274,7 +7502,6 @@"version": "4.0.1","resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz","integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",- "dev": true,"bin": {"esparse": "bin/esparse.js","esvalidate": "bin/esvalidate.js"@@ -7311,7 +7538,6 @@"version": "5.3.0","resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz","integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",- "dev": true,"engines": {"node": ">=4.0"}@@ -7320,7 +7546,6 @@"version": "2.0.3","resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz","integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",- "dev": true,"engines": {"node": ">=0.10.0"}@@ -7569,8 +7794,7 @@"node_modules/fast-levenshtein": {"version": "2.0.6","resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",- "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==",- "dev": true+ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw=="},"node_modules/fastq": {"version": "1.13.0",@@ -7809,6 +8033,19 @@"is-callable": "^1.1.3"}},+ "node_modules/form-data": {+ "version": "4.0.0",+ "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",+ "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",+ "dependencies": {+ "asynckit": "^0.4.0",+ "combined-stream": "^1.0.8",+ "mime-types": "^2.1.12"+ },+ "engines": {+ "node": ">= 6"+ }+ },"node_modules/form-data-encoder": {"version": "1.7.1","resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.1.tgz",@@ -8584,6 +8821,17 @@"node": ">=10"}},+ "node_modules/html-encoding-sniffer": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz",+ "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==",+ "dependencies": {+ "whatwg-encoding": "^2.0.0"+ },+ "engines": {+ "node": ">=12"+ }+ },"node_modules/html-escaper": {"version": "2.0.2","resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz",@@ -8656,7 +8904,6 @@"version": "5.0.0","resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz","integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==",- "dev": true,"dependencies": {"@tootallnate/once": "2","agent-base": "6",@@ -8693,7 +8940,6 @@"version": "5.0.1","resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz","integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",- "devOptional": true,"dependencies": {"agent-base": "6","debug": "4"@@ -9255,6 +9501,11 @@"node": ">=0.10.0"}},+ "node_modules/is-potential-custom-element-name": {+ "version": "1.0.1",+ "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",+ "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ=="+ },"node_modules/is-regex": {"version": "1.1.4","resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz",@@ -10165,6 +10416,102 @@"js-yaml": "bin/js-yaml.js"}},+ "node_modules/jsdom": {+ "version": "20.0.0",+ "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-20.0.0.tgz",+ "integrity": "sha512-x4a6CKCgx00uCmP+QakBDFXwjAJ69IkkIWHmtmjd3wvXPcdOS44hfX2vqkOQrVrq8l9DhNNADZRXaCEWvgXtVA==",+ "dependencies": {+ "abab": "^2.0.6",+ "acorn": "^8.7.1",+ "acorn-globals": "^6.0.0",+ "cssom": "^0.5.0",+ "cssstyle": "^2.3.0",+ "data-urls": "^3.0.2",+ "decimal.js": "^10.3.1",+ "domexception": "^4.0.0",+ "escodegen": "^2.0.0",+ "form-data": "^4.0.0",+ "html-encoding-sniffer": "^3.0.0",+ "http-proxy-agent": "^5.0.0",+ "https-proxy-agent": "^5.0.1",+ "is-potential-custom-element-name": "^1.0.1",+ "nwsapi": "^2.2.0",+ "parse5": "^7.0.0",+ "saxes": "^6.0.0",+ "symbol-tree": "^3.2.4",+ "tough-cookie": "^4.0.0",+ "w3c-hr-time": "^1.0.2",+ "w3c-xmlserializer": "^3.0.0",+ "webidl-conversions": "^7.0.0",+ "whatwg-encoding": "^2.0.0",+ "whatwg-mimetype": "^3.0.0",+ "whatwg-url": "^11.0.0",+ "ws": "^8.8.0",+ "xml-name-validator": "^4.0.0"+ },+ "engines": {+ "node": ">=14"+ },+ "peerDependencies": {+ "canvas": "^2.5.0"+ },+ "peerDependenciesMeta": {+ "canvas": {+ "optional": true+ }+ }+ },+ "node_modules/jsdom/node_modules/tr46": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz",+ "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==",+ "dependencies": {+ "punycode": "^2.1.1"+ },+ "engines": {+ "node": ">=12"+ }+ },+ "node_modules/jsdom/node_modules/webidl-conversions": {+ "version": "7.0.0",+ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",+ "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",+ "engines": {+ "node": ">=12"+ }+ },+ "node_modules/jsdom/node_modules/whatwg-url": {+ "version": "11.0.0",+ "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz",+ "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==",+ "dependencies": {+ "tr46": "^3.0.0",+ "webidl-conversions": "^7.0.0"+ },+ "engines": {+ "node": ">=12"+ }+ },+ "node_modules/jsdom/node_modules/ws": {+ "version": "8.8.1",+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.8.1.tgz",+ "integrity": "sha512-bGy2JzvzkPowEJV++hF07hAD6niYSr0JzBNo/J29WsB57A2r7Wlc1UFcTR9IzrPvuNVO4B8LGqF8qcpsVOhJCA==",+ "engines": {+ "node": ">=10.0.0"+ },+ "peerDependencies": {+ "bufferutil": "^4.0.1",+ "utf-8-validate": "^5.0.2"+ },+ "peerDependenciesMeta": {+ "bufferutil": {+ "optional": true+ },+ "utf-8-validate": {+ "optional": true+ }+ }+ },"node_modules/jsesc": {"version": "2.5.2","resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz",@@ -11445,8 +11792,7 @@"node_modules/ms": {"version": "2.1.2","resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",- "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",- "devOptional": true+ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="},"node_modules/multimatch": {"version": "5.0.0",@@ -11920,6 +12266,11 @@"url": "https://github.com/fb55/nth-check?sponsor=1"}},+ "node_modules/nwsapi": {+ "version": "2.2.1",+ "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.1.tgz",+ "integrity": "sha512-JYOWTeFoS0Z93587vRJgASD5Ut11fYl5NyihP3KrYBvMe1FRRs6RN7m20SA/16GM4P6hTnZjT+UmDOt38UeXNg=="+ },"node_modules/nx": {"version": "14.5.1","resolved": "https://registry.npmjs.org/nx/-/nx-14.5.1.tgz",@@ -13870,6 +14221,17 @@"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz","integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="},+ "node_modules/saxes": {+ "version": "6.0.0",+ "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz",+ "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==",+ "dependencies": {+ "xmlchars": "^2.2.0"+ },+ "engines": {+ "node": ">=v12.22.7"+ }+ },"node_modules/semver": {"version": "7.3.7","resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz",@@ -14142,7 +14504,7 @@"version": "0.6.1","resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz","integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",- "dev": true,+ "devOptional": true,"engines": {"node": ">=0.10.0"}@@ -14488,6 +14850,11 @@"url": "https://github.com/sponsors/ljharb"}},+ "node_modules/symbol-tree": {+ "version": "3.2.4",+ "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",+ "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="+ },"node_modules/tar": {"version": "6.1.11","resolved": "https://registry.npmjs.org/tar/-/tar-6.1.11.tgz",@@ -15388,6 +15755,25 @@"node": ">= 0.8"}},+ "node_modules/w3c-hr-time": {+ "version": "1.0.2",+ "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz",+ "integrity": "sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ==",+ "dependencies": {+ "browser-process-hrtime": "^1.0.0"+ }+ },+ "node_modules/w3c-xmlserializer": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-3.0.0.tgz",+ "integrity": "sha512-3WFqGEgSXIyGhOmAFtlicJNMjEps8b1MG31NCA0/vOF9+nKMUW1ckhi9cnNHmf88Rzw5V+dwIwsm2C7X8k9aQg==",+ "dependencies": {+ "xml-name-validator": "^4.0.0"+ },+ "engines": {+ "node": ">=12"+ }+ },"node_modules/walk-up-path": {"version": "1.0.0","resolved": "https://registry.npmjs.org/walk-up-path/-/walk-up-path-1.0.0.tgz",@@ -15420,6 +15806,36 @@"node": ">=10.4"}},+ "node_modules/whatwg-encoding": {+ "version": "2.0.0",+ "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz",+ "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==",+ "dependencies": {+ "iconv-lite": "0.6.3"+ },+ "engines": {+ "node": ">=12"+ }+ },+ "node_modules/whatwg-encoding/node_modules/iconv-lite": {+ "version": "0.6.3",+ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",+ "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",+ "dependencies": {+ "safer-buffer": ">= 2.1.2 < 3.0.0"+ },+ "engines": {+ "node": ">=0.10.0"+ }+ },+ "node_modules/whatwg-mimetype": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz",+ "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==",+ "engines": {+ "node": ">=12"+ }+ },"node_modules/whatwg-url": {"version": "8.7.0","resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.7.0.tgz",@@ -15513,7 +15929,6 @@"version": "1.2.3","resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz","integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",- "dev": true,"engines": {"node": ">=0.10.0"}@@ -15723,6 +16138,19 @@}}},+ "node_modules/xml-name-validator": {+ "version": "4.0.0",+ "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz",+ "integrity": "sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==",+ "engines": {+ "node": ">=12"+ }+ },+ "node_modules/xmlchars": {+ "version": "2.2.0",+ "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",+ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="+ },"node_modules/xtend": {"version": "4.0.2","resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",@@ -16057,6 +16485,8 @@"@crawlee/cheerio": "^3.0.3","@crawlee/cli": "^3.0.3","@crawlee/core": "^3.0.3",+ "@crawlee/dom": "^3.0.3",+ "@crawlee/http": "^3.0.3","@crawlee/playwright": "^3.0.3","@crawlee/puppeteer": "^3.0.3","@crawlee/utils": "^3.0.3",@@ -16081,6 +16511,21 @@}}},+ "packages/dom-crawler": {+ "name": "@crawlee/dom",+ "version": "3.0.3",+ "license": "Apache-2.0",+ "dependencies": {+ "@apify/utilities": "^2.1.4",+ "@crawlee/http": "^3.0.3",+ "@crawlee/types": "^3.0.3",+ "@types/jsdom": "^20.0.0",+ "jsdom": "^20.0.0"+ },+ "engines": {+ "node": ">=16.0.0"+ }+ },"packages/http-crawler": {"name": "@crawlee/http","version": "3.0.3",@@ -17353,6 +17798,16 @@}}},+ "@crawlee/dom": {+ "version": "file:packages/dom-crawler",+ "requires": {+ "@apify/utilities": "^2.1.4",+ "@crawlee/http": "^3.0.3",+ "@crawlee/types": "^3.0.3",+ "@types/jsdom": "^20.0.0",+ "jsdom": "^20.0.0"+ }+ },"@crawlee/http": {"version": "file:packages/http-crawler","requires": {@@ -19614,8 +20069,7 @@"@tootallnate/once": {"version": "2.0.0","resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",- "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==",- "dev": true+ "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A=="},"@tsconfig/node10": {"version": "1.0.9",@@ -19849,6 +20303,16 @@"pretty-format": "^28.0.0"}},+ "@types/jsdom": {+ "version": "20.0.0",+ "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-20.0.0.tgz",+ "integrity": "sha512-YfAchFs0yM1QPDrLm2VHe+WHGtqms3NXnXAMolrgrVP6fgBHHXy1ozAbo/dFtPNtZC/m66bPiCTWYmqp1F14gA==",+ "requires": {+ "@types/node": "*",+ "@types/tough-cookie": "*",+ "parse5": "^7.0.0"+ }+ },"@types/json-buffer": {"version": "3.0.0","resolved": "https://registry.npmjs.org/@types/json-buffer/-/json-buffer-3.0.0.tgz",@@ -20198,6 +20662,11 @@"resolved": "https://registry.npmjs.org/@vladfrangu/async_event_emitter/-/async_event_emitter-2.0.1.tgz","integrity": "sha512-zCOcPuMcCHAqTLqYvltYBGRUrYiDZ/RLpih4G/AsSlMgoSdv8EvtnAtyiM3gbRIzrdc42e55VM2UUyLRcR8ufg=="},+ "abab": {+ "version": "2.0.6",+ "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",+ "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA=="+ },"abbrev": {"version": "1.1.1","resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",@@ -20217,8 +20686,28 @@"acorn": {"version": "8.8.0","resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz",- "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==",- "dev": true+ "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w=="+ },+ "acorn-globals": {+ "version": "6.0.0",+ "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz",+ "integrity": "sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg==",+ "requires": {+ "acorn": "^7.1.1",+ "acorn-walk": "^7.1.1"+ },+ "dependencies": {+ "acorn": {+ "version": "7.4.1",+ "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz",+ "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A=="+ },+ "acorn-walk": {+ "version": "7.2.0",+ "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz",+ "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA=="+ }+ }},"acorn-jsx": {"version": "5.3.2",@@ -20248,7 +20737,6 @@"version": "6.0.2","resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz","integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",- "devOptional": true,"requires": {"debug": "4"}@@ -20623,6 +21111,11 @@"retry": "0.13.1"}},+ "asynckit": {+ "version": "0.4.0",+ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",+ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="+ },"at-least-node": {"version": "1.0.0","resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz",@@ -20859,6 +21352,11 @@"fill-range": "^7.0.1"}},+ "browser-process-hrtime": {+ "version": "1.0.0",+ "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz",+ "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow=="+ },"browserslist": {"version": "4.21.3","resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.3.tgz",@@ -21337,6 +21835,14 @@"wcwidth": "^1.0.0"}},+ "combined-stream": {+ "version": "1.0.8",+ "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",+ "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",+ "requires": {+ "delayed-stream": "~1.0.0"+ }+ },"commander": {"version": "9.4.0","resolved": "https://registry.npmjs.org/commander/-/commander-9.4.0.tgz",@@ -21618,6 +22124,8 @@"@crawlee/cheerio": "^3.0.3","@crawlee/cli": "^3.0.3","@crawlee/core": "^3.0.3",+ "@crawlee/dom": "^3.0.3",+ "@crawlee/http": "^3.0.3","@crawlee/playwright": "^3.0.3","@crawlee/puppeteer": "^3.0.3","@crawlee/utils": "^3.0.3",@@ -21677,6 +22185,26 @@"resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz","integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw=="},+ "cssom": {+ "version": "0.5.0",+ "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",+ "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="+ },+ "cssstyle": {+ "version": "2.3.0",+ "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz",+ "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==",+ "requires": {+ "cssom": "~0.3.6"+ },+ "dependencies": {+ "cssom": {+ "version": "0.3.8",+ "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz",+ "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg=="+ }+ }+ },"damerau-levenshtein": {"version": "1.0.8","resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz",@@ -21689,6 +22217,40 @@"integrity": "sha512-2iy1EkLdlBzQGvbweYRFxmFath8+K7+AKB0TlhHWkNuH+TmovaMH/Wp7V7R4u7f4SnX3OgLsU9t1NI9ioDnUpg==","dev": true},+ "data-urls": {+ "version": "3.0.2",+ "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz",+ "integrity": "sha512-Jy/tj3ldjZJo63sVAvg6LHt2mHvl4V6AgRAmNDtLdm7faqtsx+aJG42rsyCo9JCoRVKwPFzKlIPx3DIibwSIaQ==",+ "requires": {+ "abab": "^2.0.6",+ "whatwg-mimetype": "^3.0.0",+ "whatwg-url": "^11.0.0"+ },+ "dependencies": {+ "tr46": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz",+ "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==",+ "requires": {+ "punycode": "^2.1.1"+ }+ },+ "webidl-conversions": {+ "version": "7.0.0",+ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",+ "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g=="+ },+ "whatwg-url": {+ "version": "11.0.0",+ "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz",+ "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==",+ "requires": {+ "tr46": "^3.0.0",+ "webidl-conversions": "^7.0.0"+ }+ }+ }+ },"dateformat": {"version": "3.0.3","resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz",@@ -21699,7 +22261,6 @@"version": "4.3.4","resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz","integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",- "devOptional": true,"requires": {"ms": "2.1.2"}@@ -21734,6 +22295,11 @@}}},+ "decimal.js": {+ "version": "10.3.1",+ "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz",+ "integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ=="+ },"decompress-response": {"version": "6.0.0","resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",@@ -21774,8 +22340,7 @@"deep-is": {"version": "0.1.4","resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",- "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",- "dev": true+ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ=="},"deepmerge": {"version": "4.2.2",@@ -21812,6 +22377,11 @@"object-keys": "^1.1.1"}},+ "delayed-stream": {+ "version": "1.0.0",+ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",+ "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="+ },"delegates": {"version": "1.0.0","resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",@@ -21919,6 +22489,21 @@"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz","integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="},+ "domexception": {+ "version": "4.0.0",+ "resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz",+ "integrity": "sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==",+ "requires": {+ "webidl-conversions": "^7.0.0"+ },+ "dependencies": {+ "webidl-conversions": {+ "version": "7.0.0",+ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",+ "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g=="+ }+ }+ },"domhandler": {"version": "2.4.2","resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",@@ -22164,6 +22749,55 @@"integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==","dev": true},+ "escodegen": {+ "version": "2.0.0",+ "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.0.0.tgz",+ "integrity": "sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw==",+ "requires": {+ "esprima": "^4.0.1",+ "estraverse": "^5.2.0",+ "esutils": "^2.0.2",+ "optionator": "^0.8.1",+ "source-map": "~0.6.1"+ },+ "dependencies": {+ "levn": {+ "version": "0.3.0",+ "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz",+ "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==",+ "requires": {+ "prelude-ls": "~1.1.2",+ "type-check": "~0.3.2"+ }+ },+ "optionator": {+ "version": "0.8.3",+ "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz",+ "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==",+ "requires": {+ "deep-is": "~0.1.3",+ "fast-levenshtein": "~2.0.6",+ "levn": "~0.3.0",+ "prelude-ls": "~1.1.2",+ "type-check": "~0.3.2",+ "word-wrap": "~1.2.3"+ }+ },+ "prelude-ls": {+ "version": "1.1.2",+ "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",+ "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w=="+ },+ "type-check": {+ "version": "0.3.2",+ "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz",+ "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==",+ "requires": {+ "prelude-ls": "~1.1.2"+ }+ }+ }+ },"eslint": {"version": "8.20.0","resolved": "https://registry.npmjs.org/eslint/-/eslint-8.20.0.tgz",@@ -22560,8 +23194,7 @@"esprima": {"version": "4.0.1","resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",- "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",- "dev": true+ "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A=="},"esquery": {"version": "1.4.0",@@ -22584,14 +23217,12 @@"estraverse": {"version": "5.3.0","resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",- "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",- "dev": true+ "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA=="},"esutils": {"version": "2.0.3","resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",- "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",- "dev": true+ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g=="},"etag": {"version": "1.8.1",@@ -22798,8 +23429,7 @@"fast-levenshtein": {"version": "2.0.6","resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",- "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==",- "dev": true+ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw=="},"fastq": {"version": "1.13.0",@@ -22979,6 +23609,16 @@"is-callable": "^1.1.3"}},+ "form-data": {+ "version": "4.0.0",+ "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",+ "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",+ "requires": {+ "asynckit": "^0.4.0",+ "combined-stream": "^1.0.8",+ "mime-types": "^2.1.12"+ }+ },"form-data-encoder": {"version": "1.7.1","resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.1.tgz",@@ -23562,6 +24202,14 @@}}},+ "html-encoding-sniffer": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz",+ "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==",+ "requires": {+ "whatwg-encoding": "^2.0.0"+ }+ },"html-escaper": {"version": "2.0.2","resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz",@@ -23619,7 +24267,6 @@"version": "5.0.0","resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz","integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==",- "dev": true,"requires": {"@tootallnate/once": "2","agent-base": "6",@@ -23646,7 +24293,6 @@"version": "5.0.1","resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz","integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",- "devOptional": true,"requires": {"agent-base": "6","debug": "4"@@ -24043,6 +24689,11 @@"integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==","dev": true},+ "is-potential-custom-element-name": {+ "version": "1.0.1",+ "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",+ "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ=="+ },"is-regex": {"version": "1.1.4","resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz",@@ -24740,6 +25391,70 @@"argparse": "^2.0.1"}},+ "jsdom": {+ "version": "20.0.0",+ "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-20.0.0.tgz",+ "integrity": "sha512-x4a6CKCgx00uCmP+QakBDFXwjAJ69IkkIWHmtmjd3wvXPcdOS44hfX2vqkOQrVrq8l9DhNNADZRXaCEWvgXtVA==",+ "requires": {+ "abab": "^2.0.6",+ "acorn": "^8.7.1",+ "acorn-globals": "^6.0.0",+ "cssom": "^0.5.0",+ "cssstyle": "^2.3.0",+ "data-urls": "^3.0.2",+ "decimal.js": "^10.3.1",+ "domexception": "^4.0.0",+ "escodegen": "^2.0.0",+ "form-data": "^4.0.0",+ "html-encoding-sniffer": "^3.0.0",+ "http-proxy-agent": "^5.0.0",+ "https-proxy-agent": "^5.0.1",+ "is-potential-custom-element-name": "^1.0.1",+ "nwsapi": "^2.2.0",+ "parse5": "^7.0.0",+ "saxes": "^6.0.0",+ "symbol-tree": "^3.2.4",+ "tough-cookie": "^4.0.0",+ "w3c-hr-time": "^1.0.2",+ "w3c-xmlserializer": "^3.0.0",+ "webidl-conversions": "^7.0.0",+ "whatwg-encoding": "^2.0.0",+ "whatwg-mimetype": "^3.0.0",+ "whatwg-url": "^11.0.0",+ "ws": "^8.8.0",+ "xml-name-validator": "^4.0.0"+ },+ "dependencies": {+ "tr46": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz",+ "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==",+ "requires": {+ "punycode": "^2.1.1"+ }+ },+ "webidl-conversions": {+ "version": "7.0.0",+ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",+ "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g=="+ },+ "whatwg-url": {+ "version": "11.0.0",+ "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz",+ "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==",+ "requires": {+ "tr46": "^3.0.0",+ "webidl-conversions": "^7.0.0"+ }+ },+ "ws": {+ "version": "8.8.1",+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.8.1.tgz",+ "integrity": "sha512-bGy2JzvzkPowEJV++hF07hAD6niYSr0JzBNo/J29WsB57A2r7Wlc1UFcTR9IzrPvuNVO4B8LGqF8qcpsVOhJCA==",+ "requires": {}+ }+ }+ },"jsesc": {"version": "2.5.2","resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz",@@ -25701,8 +26416,7 @@"ms": {"version": "2.1.2","resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",- "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",- "devOptional": true+ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="},"multimatch": {"version": "5.0.0",@@ -26076,6 +26790,11 @@"boolbase": "^1.0.0"}},+ "nwsapi": {+ "version": "2.2.1",+ "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.1.tgz",+ "integrity": "sha512-JYOWTeFoS0Z93587vRJgASD5Ut11fYl5NyihP3KrYBvMe1FRRs6RN7m20SA/16GM4P6hTnZjT+UmDOt38UeXNg=="+ },"nx": {"version": "14.5.1","resolved": "https://registry.npmjs.org/nx/-/nx-14.5.1.tgz",@@ -27495,6 +28214,14 @@"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz","integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="},+ "saxes": {+ "version": "6.0.0",+ "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz",+ "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==",+ "requires": {+ "xmlchars": "^2.2.0"+ }+ },"semver": {"version": "7.3.7","resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz",@@ -27709,7 +28436,7 @@"version": "0.6.1","resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz","integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",- "dev": true+ "devOptional": true},"source-map-support": {"version": "0.5.13",@@ -27984,6 +28711,11 @@"integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==","dev": true},+ "symbol-tree": {+ "version": "3.2.4",+ "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",+ "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="+ },"tar": {"version": "6.1.11","resolved": "https://registry.npmjs.org/tar/-/tar-6.1.11.tgz",@@ -28618,6 +29350,22 @@"integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==","dev": true},+ "w3c-hr-time": {+ "version": "1.0.2",+ "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz",+ "integrity": "sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ==",+ "requires": {+ "browser-process-hrtime": "^1.0.0"+ }+ },+ "w3c-xmlserializer": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-3.0.0.tgz",+ "integrity": "sha512-3WFqGEgSXIyGhOmAFtlicJNMjEps8b1MG31NCA0/vOF9+nKMUW1ckhi9cnNHmf88Rzw5V+dwIwsm2C7X8k9aQg==",+ "requires": {+ "xml-name-validator": "^4.0.0"+ }+ },"walk-up-path": {"version": "1.0.0","resolved": "https://registry.npmjs.org/walk-up-path/-/walk-up-path-1.0.0.tgz",@@ -28647,6 +29395,29 @@"integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==","dev": true},+ "whatwg-encoding": {+ "version": "2.0.0",+ "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz",+ "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==",+ "requires": {+ "iconv-lite": "0.6.3"+ },+ "dependencies": {+ "iconv-lite": {+ "version": "0.6.3",+ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",+ "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",+ "requires": {+ "safer-buffer": ">= 2.1.2 < 3.0.0"+ }+ }+ }+ },+ "whatwg-mimetype": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz",+ "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q=="+ },"whatwg-url": {"version": "8.7.0","resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.7.0.tgz",@@ -28718,8 +29489,7 @@"word-wrap": {"version": "1.2.3","resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",- "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",- "dev": true+ "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ=="},"wordwrap": {"version": "1.0.0",@@ -28874,6 +29644,16 @@"dev": true,"requires": {}},+ "xml-name-validator": {+ "version": "4.0.0",+ "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz",+ "integrity": "sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw=="+ },+ "xmlchars": {+ "version": "2.2.0",+ "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",+ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="+ },"xtend": {"version": "4.0.2","resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",diff --git a/package.json b/package.jsonindex bd1e32b17..720ebbbd3 100644--- a/package.json+++ b/package.json@@ -63,10 +63,10 @@"@types/htmlparser2": "^3.10.3","@types/inquirer": "^8.2.1","@types/jest": "^28.1.1",- "@types/proper-lockfile": "^4.1.2","@types/lodash.merge": "^4.6.7","@types/mime-types": "^2.1.1","@types/node": "^18.0.3",+ "@types/proper-lockfile": "^4.1.2","@types/ps-tree": "^1.1.2","@types/rimraf": "^3.0.2","@types/semver": "^7.3.9",diff --git a/packages/core/README.md b/packages/core/README.mdindex 9c945738b..068b7482a 100644--- a/packages/core/README.md+++ b/packages/core/README.md@@ -6,7 +6,8 @@ The [`crawlee`](https://www.npmjs.com/package/crawlee) package consists of sever- `@crawlee/core`: the base for all the crawler implementations, also contains things like `Request`, `RequestQueue`, `RequestList` or `Dataset` classes- `@crawlee/basic`: exports `BasicCrawler`-- `@crawlee/http`: exports `HttpCrawler` (which is used for creating `@crawlee/cheerio`)+- `@crawlee/http`: exports `HttpCrawler` (which is used for creating `@crawlee/dom` and `@crawlee/cheerio`)+- `@crawlee/dom`: exports `DOMCrawler`- `@crawlee/cheerio`: exports `CheerioCrawler`- `@crawlee/browser`: exports `BrowserCrawler` (which is used for creating `@crawlee/playwright` and `@crawlee/puppeteer`)- `@crawlee/playwright`: exports `PlaywrightCrawler`diff --git a/packages/crawlee/package.json b/packages/crawlee/package.jsonindex 80f447631..81dfe385b 100644--- a/packages/crawlee/package.json+++ b/packages/crawlee/package.json@@ -60,7 +60,9 @@"@crawlee/core": "^3.0.3","@crawlee/basic": "^3.0.3","@crawlee/browser": "^3.0.3",+ "@crawlee/http": "^3.0.3","@crawlee/cheerio": "^3.0.3",+ "@crawlee/dom": "^3.0.3","@crawlee/playwright": "^3.0.3","@crawlee/puppeteer": "^3.0.3","@crawlee/utils": "^3.0.3",diff --git a/packages/crawlee/src/index.ts b/packages/crawlee/src/index.tsindex a5d1e6ef3..4c0c05dfe 100644--- a/packages/crawlee/src/index.ts+++ b/packages/crawlee/src/index.ts@@ -10,6 +10,7 @@ export * from '@crawlee/utils';export * from '@crawlee/basic';export * from '@crawlee/browser';export * from '@crawlee/http';+export * from '@crawlee/dom';export * from '@crawlee/cheerio';export * from '@crawlee/puppeteer';export * from '@crawlee/playwright';diff --git a/packages/dom-crawler/.npmignore b/packages/dom-crawler/.npmignorenew file mode 100644index 000000000..ada367d72--- /dev/null+++ b/packages/dom-crawler/.npmignore@@ -0,0 +1,6 @@+node_modules+src+test+coverage+apify_storage+tsconfig.*diff --git a/packages/dom-crawler/README.md b/packages/dom-crawler/README.mdnew file mode 100644index 000000000..3d4cf1d81--- /dev/null+++ b/packages/dom-crawler/README.md@@ -0,0 +1,44 @@+# `@crawlee/dom`++Provides a framework for the parallel crawling of web pages using plain HTTP requests and [jsdom](https://www.npmjs.com/package/jsdom) DOM implementation. The URLs to crawl are fed either from a static list of URLs or from a dynamic queue of URLs enabling recursive crawling of websites.++Since `DOMCrawler` uses raw HTTP requests to download web pages, it is very fast and efficient on data bandwidth. However, if the target website requires JavaScript to display the content, you might need to use [PuppeteerCrawler](https://crawlee.dev/api/puppeteer-crawler/class/PuppeteerCrawler) or [PlaywrightCrawler](https://crawlee.dev/api/playwright-crawler/class/PlaywrightCrawler) instead, because it loads the pages using full-featured headless Chrome browser.++`DOMCrawler` downloads each URL using a plain HTTP request, parses the HTML content using [JSDOM](https://www.npmjs.com/package/jsdom) and then invokes the user-provided [DOMCrawlerOptions.requestHandler](https://crawlee.dev/api/dom-crawler/interface/DOMCrawlerOptions#requestHandler) to extract page data using the `window` object.++The source URLs are represented using [Request](https://crawlee.dev/api/core/class/Request) objects that are fed from [RequestList](https://crawlee.dev/api/core/class/RequestList) or [RequestQueue](https://crawlee.dev/api/core/class/RequestQueue) instances provided by the [DOMCrawlerOptions.requestList](https://crawlee.dev/api/dom-crawler/interface/DOMCrawlerOptions#requestList) or [DOMCrawlerOptions.requestQueue](https://crawlee.dev/api/dom-crawler/interface/DOMCrawlerOptions#requestQueue) constructor options, respectively.++If both [DOMCrawlerOptions.requestList](https://crawlee.dev/api/dom-crawler/interface/DOMCrawlerOptions#requestList) and [DOMCrawlerOptions.requestQueue](https://crawlee.dev/api/dom-crawler/interface/DOMCrawlerOptions#requestQueue) are used, the instance first processes URLs from the [RequestList](https://crawlee.dev/api/core/class/RequestList) and automatically enqueues all of them to [RequestQueue](https://crawlee.dev/api/core/class/RequestQueue) before it starts their processing. This ensures that a single URL is not crawled multiple times.++The crawler finishes when there are no more [Request](https://crawlee.dev/api/core/class/Request) objects to crawl.++We can use the `preNavigationHooks` to adjust `gotOptions`:++```+preNavigationHooks: [+ (crawlingContext, gotOptions) => {+ // ...+ },+]+```++By default, `DOMCrawler` only processes web pages with the `text/html` and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header), and skips pages with other content types. If you want the crawler to process other content types, use the [DOMCrawlerOptions.additionalMimeTypes](https://crawlee.dev/api/dom-crawler/interface/DOMCrawlerOptions#additionalMimeTypes) constructor option. Beware that the parsing behavior differs for HTML, XML, JSON and other types of content. For more details, see [DOMCrawlerOptions.requestHandler](https://crawlee.dev/api/dom-crawler/interface/DOMCrawlerOptions#requestHandler).++New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the [AutoscaledPool](https://crawlee.dev/api/core/class/AutoscaledPool) class. All [AutoscaledPool](https://crawlee.dev/api/core/class/AutoscaledPool) configuration options can be passed to the `autoscaledPoolOptions` parameter of the `DOMCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency` [AutoscaledPool](https://crawlee.dev/api/core/class/AutoscaledPool) options are available directly in the `DOMCrawler` constructor.++## Example usage++```javascript+const crawler = new DOMCrawler({+ async requestHandler({ request, window }) {+ await Dataset.pushData({+ url: request.url,+ title: window.document.title,+ });+ },+});++await crawler.run([+ 'http://crawlee.dev',+]);+```diff --git a/packages/dom-crawler/package.json b/packages/dom-crawler/package.jsonnew file mode 100644index 000000000..8243750a7--- /dev/null+++ b/packages/dom-crawler/package.json@@ -0,0 +1,62 @@+{+ "name": "@crawlee/dom",+ "version": "3.0.3",+ "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",+ "engines": {+ "node": ">=16.0.0"+ },+ "main": "./dist/index.js",+ "module": "./dist/index.mjs",+ "types": "./dist/index.d.ts",+ "exports": {+ ".": {+ "import": "./dist/index.mjs",+ "require": "./dist/index.js",+ "types": "./dist/index.d.ts"+ },+ "./package.json": "./package.json"+ },+ "keywords": [+ "apify",+ "headless",+ "chrome",+ "puppeteer",+ "crawler",+ "scraper"+ ],+ "author": {+ "name": "Apify",+ "email": "[email protected]",+ "url": "https://apify.com"+ },+ "contributors": [+ "Jan Curn <[email protected]>",+ "Marek Trunkat <[email protected]>",+ "Ondra Urban <[email protected]>"+ ],+ "license": "Apache-2.0",+ "repository": {+ "type": "git",+ "url": "git+https://github.com/apify/crawlee"+ },+ "bugs": {+ "url": "https://github.com/apify/crawlee/issues"+ },+ "homepage": "https://crawlee.dev",+ "scripts": {+ "build": "npm run clean && npm run compile && npm run copy",+ "clean": "rimraf ./dist",+ "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs",+ "copy": "ts-node -T ../../scripts/copy.ts"+ },+ "publishConfig": {+ "access": "public"+ },+ "dependencies": {+ "@apify/utilities": "^2.1.4",+ "@crawlee/http": "^3.0.3",+ "@crawlee/types": "^3.0.3",+ "@types/jsdom": "^20.0.0",+ "jsdom": "^20.0.0"+ }+}diff --git a/packages/dom-crawler/src/index.ts b/packages/dom-crawler/src/index.tsnew file mode 100644index 000000000..8c89e6e07--- /dev/null+++ b/packages/dom-crawler/src/index.ts@@ -0,0 +1,2 @@+export * from '@crawlee/http';+export * from './internals/dom-crawler';diff --git a/packages/dom-crawler/src/internals/dom-crawler.ts b/packages/dom-crawler/src/internals/dom-crawler.tsnew file mode 100644index 000000000..99f198ab5--- /dev/null+++ b/packages/dom-crawler/src/internals/dom-crawler.ts@@ -0,0 +1,220 @@+import type {+ HttpCrawlerOptions,+ InternalHttpCrawlingContext,+ InternalHttpHook,+ ErrorHandler,+ RequestHandler,+ EnqueueLinksOptions,+ RequestQueue,+} from '@crawlee/http';+import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering } from '@crawlee/http';+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';+import { concatStreamToBuffer } from '@apify/utilities';+import type { DOMWindow } from 'jsdom';+import { JSDOM } from 'jsdom';+import type { IncomingMessage } from 'http';++export type DOMErrorHandler<+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler+ JSONData extends Dictionary = Dictionary,+ > = ErrorHandler<DOMCrawlingContext<UserData, JSONData>>;++export interface DOMCrawlerOptions<+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler+ JSONData extends Dictionary = Dictionary,+ > extends HttpCrawlerOptions<DOMCrawlingContext<UserData, JSONData>> {}++export type DOMHook<+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler+ JSONData extends Dictionary = Dictionary,+ > = InternalHttpHook<DOMCrawlingContext<UserData, JSONData>>;++export interface DOMCrawlingContext<+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler+ JSONData extends Dictionary = Dictionary,+ > extends InternalHttpCrawlingContext<UserData, JSONData, DOMCrawler> {+ window: DOMWindow;++ enqueueLinks: (options?: DOMCrawlerEnqueueLinksOptions) => Promise<BatchAddRequestsResult>;+}++export type DOMRequestHandler<+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler+ JSONData extends Dictionary = Dictionary,+ > = RequestHandler<DOMCrawlingContext<UserData, JSONData>>;+export interface DOMCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestQueue'> {}++/**+ * Provides a framework for the parallel crawling of web pages using plain HTTP requests and+ * [jsdom](https://www.npmjs.com/package/jsdom) DOM implementation.+ * The URLs to crawl are fed either from a static list of URLs+ * or from a dynamic queue of URLs enabling recursive crawling of websites.+ *+ * Since `DOMCrawler` uses raw HTTP requests to download web pages,+ * it is very fast and efficient on data bandwidth. However, if the target website requires JavaScript+ * to display the content, you might need to use {@apilink PuppeteerCrawler} or {@apilink PlaywrightCrawler} instead,+ * because it loads the pages using full-featured headless Chrome browser.+ *+ * `DOMCrawler` downloads each URL using a plain HTTP request,+ * parses the HTML content using [JSDOM](https://www.npmjs.com/package/jsdom)+ * and then invokes the user-provided {@apilink DOMCrawlerOptions.requestHandler} to extract page data+ * using the `window` object.+ *+ * The source URLs are represented using {@apilink Request} objects that are fed from+ * {@apilink RequestList} or {@apilink RequestQueue} instances provided by the {@apilink DOMCrawlerOptions.requestList}+ * or {@apilink DOMCrawlerOptions.requestQueue} constructor options, respectively.+ *+ * If both {@apilink DOMCrawlerOptions.requestList} and {@apilink DOMCrawlerOptions.requestQueue} are used,+ * the instance first processes URLs from the {@apilink RequestList} and automatically enqueues all of them+ * to {@apilink RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.+ *+ * The crawler finishes when there are no more {@apilink Request} objects to crawl.+ *+ * We can use the `preNavigationHooks` to adjust `gotOptions`:+ *+ * ```+ * preNavigationHooks: [+ * (crawlingContext, gotOptions) => {+ * // ...+ * },+ * ]+ * ```+ *+ * By default, `DOMCrawler` only processes web pages with the `text/html`+ * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),+ * and skips pages with other content types. If you want the crawler to process other content types,+ * use the {@apilink DOMCrawlerOptions.additionalMimeTypes} constructor option.+ * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.+ * For more details, see {@apilink DOMCrawlerOptions.requestHandler}.+ *+ * New requests are only dispatched when there is enough free CPU and memory available,+ * using the functionality provided by the {@apilink AutoscaledPool} class.+ * All {@apilink AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`+ * parameter of the `CheerioCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`+ * {@apilink AutoscaledPool} options are available directly in the `CheerioCrawler` constructor.+ *+ * **Example usage:**+ *+ * ```javascript+ * const crawler = new DOMCrawler({+ * async requestHandler({ request, window }) {+ * await Dataset.pushData({+ * url: request.url,+ * title: window.document.title,+ * });+ * },+ * });+ *+ * await crawler.run([+ * 'http://crawlee.dev',+ * ]);+ * ```+ * @category Crawlers+ */+export class DOMCrawler extends HttpCrawler<DOMCrawlingContext> {+ protected override async _parseHTML(response: IncomingMessage, isXml: boolean, crawlingContext: DOMCrawlingContext) {+ const body = await concatStreamToBuffer(response);++ const { window } = new JSDOM(body, {+ url: response.url,+ contentType: isXml ? 'text/xml' : 'text/html',+ });++ return {+ window,+ get body() {+ return window.document.documentElement.outerHTML;+ },+ enqueueLinks: async (enqueueOptions?: DOMCrawlerEnqueueLinksOptions) => {+ return domCrawlerEnqueueLinks({+ options: enqueueOptions,+ window,+ requestQueue: await this.getRequestQueue(),+ originalRequestUrl: crawlingContext.request.url,+ finalRequestUrl: crawlingContext.request.loadedUrl,+ });+ },+ };+ }+}++interface EnqueueLinksInternalOptions {+ options?: DOMCrawlerEnqueueLinksOptions;+ window: DOMWindow | null;+ requestQueue: RequestQueue;+ originalRequestUrl: string;+ finalRequestUrl?: string;+}++/** @internal */+export async function domCrawlerEnqueueLinks({ options, window, requestQueue, originalRequestUrl, finalRequestUrl }: EnqueueLinksInternalOptions) {+ if (!window) {+ throw new Error('Cannot enqueue links because the DOM is not available.');+ }++ const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({+ enqueueStrategy: options?.strategy,+ finalRequestUrl,+ originalRequestUrl,+ userProvidedBaseUrl: options?.baseUrl,+ });++ const urls = extractUrlsFromWindow(window, options?.selector ?? 'a', options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);++ return enqueueLinks({+ requestQueue,+ urls,+ baseUrl,+ ...options,+ });+}++/**+ * Extracts URLs from a given Window object.+ * @ignore+ */+function extractUrlsFromWindow(window: DOMWindow, selector: string, baseUrl: string): string[] {+ return Array.from(window.document.querySelectorAll(selector))+ .map((e: any) => e.href)+ .filter((href) => href !== undefined && href !== '')+ .map((href: string | undefined) => {+ if (href === undefined) {+ return undefined;+ }++ try {+ return (new URL(href, baseUrl)).href;+ } catch {+ return undefined;+ }+ })+ .filter((href) => href !== undefined && href !== '') as string[];+}++/**+ * Creates new {@apilink Router} instance that works based on request labels.+ * This instance can then serve as a `requestHandler` of your {@apilink DOMCrawler}.+ * Defaults to the {@apilink DOMCrawlingContext}.+ *+ * > Serves as a shortcut for using `Router.create<DOMCrawlingContext>()`.+ *+ * ```ts+ * import { DOMCrawler, createDOMRouter } from 'crawlee';+ *+ * const router = createDOMRouter();+ * router.addHandler('label-a', async (ctx) => {+ * ctx.log.info('...');+ * });+ * router.addDefaultHandler(async (ctx) => {+ * ctx.log.info('...');+ * });+ *+ * const crawler = new DOMCrawler({+ * requestHandler: router,+ * });+ * await crawler.run();+ * ```+ */+export function createDOMRouter<Context extends DOMCrawlingContext = DOMCrawlingContext>() {+ return Router.create<Context>();+}diff --git a/packages/dom-crawler/tsconfig.build.json b/packages/dom-crawler/tsconfig.build.jsonnew file mode 100644index 000000000..856db0f21--- /dev/null+++ b/packages/dom-crawler/tsconfig.build.json@@ -0,0 +1,7 @@+{+ "extends": "../../tsconfig.build.json",+ "compilerOptions": {+ "outDir": "./dist"+ },+ "include": ["src/**/*"]+}diff --git a/packages/dom-crawler/tsconfig.json b/packages/dom-crawler/tsconfig.jsonnew file mode 100644index 000000000..52d43eaaa--- /dev/null+++ b/packages/dom-crawler/tsconfig.json@@ -0,0 +1,4 @@+{+ "extends": "../../tsconfig.json",+ "include": ["src/**/*"]+}diff --git a/test/core/crawlers/dom_crawler.test.ts b/test/core/crawlers/dom_crawler.test.tsnew file mode 100644index 000000000..d083b353a--- /dev/null+++ b/test/core/crawlers/dom_crawler.test.ts@@ -0,0 +1,61 @@+import { DOMCrawler } from '@crawlee/dom';+import type { AddressInfo } from 'node:net';+import http from 'node:http';+import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';++const router = new Map<string, http.RequestListener>();+router.set('/', (req, res) => {+ res.setHeader('content-type', 'text/html; charset=utf-8');+ res.end(`<!DOCTYPE html><html><head><title>Example Domain</title></head><body><p>Hello, world!</p></body></html>`);+});++let server: http.Server;+let url: string;++beforeAll((cb) => {+ server = http.createServer((request, response) => {+ try {+ const requestUrl = new URL(request.url, 'http://localhost');+ router.get(requestUrl.pathname)(request, response);+ } catch (error) {+ response.destroy();+ }+ });++ server.listen(() => {+ url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;+ cb();+ });+});++afterAll((cb) => {+ server.close(cb);+});++const localStorageEmulator = new MemoryStorageEmulator();++beforeEach(async () => {+ await localStorageEmulator.init();+});++afterAll(async () => {+ await localStorageEmulator.destroy();+});++test('works', async () => {+ const results: string[] = [];++ const crawler = new DOMCrawler({+ maxRequestRetries: 0,+ requestHandler: ({ window }) => {+ results.push(window.document.title, window.document.querySelector('p').textContent);+ },+ });++ await crawler.run([url]);++ expect(results).toStrictEqual([+ 'Example Domain',+ 'Hello, world!',+ ]);+});diff --git a/test/e2e/dom-default-ts/actor/.gitignore b/test/e2e/dom-default-ts/actor/.gitignorenew file mode 100644index 000000000..f2fc11c72--- /dev/null+++ b/test/e2e/dom-default-ts/actor/.gitignore@@ -0,0 +1,11 @@+.idea+.DS_Store+node_modules+package-lock.json+apify_storage+crawlee_storage+storage+main.d.ts+main.d.ts.map+main.js+main.js.mapdiff --git a/test/e2e/dom-default-ts/actor/Dockerfile b/test/e2e/dom-default-ts/actor/Dockerfilenew file mode 100644index 000000000..0e59eb921--- /dev/null+++ b/test/e2e/dom-default-ts/actor/Dockerfile@@ -0,0 +1,27 @@+# using multistage build, as we need dev deps to build the TS source code+FROM apify/actor-node:16 AS builder++# copy all files, install all dependencies (including dev deps) and build the project+COPY . ./+RUN npm install --include=dev \+ && npm run build++# create final image+FROM apify/actor-node:16+# copy only necessary files+COPY --from=builder /usr/src/app/packages ./packages+COPY --from=builder /usr/src/app/package.json ./+COPY --from=builder /usr/src/app/main.js ./++# install only prod deps+RUN npm --quiet set progress=false \+ && npm install --only=prod --no-optional \+ && echo "Installed NPM packages:" \+ && (npm list --only=prod --no-optional --all || true) \+ && echo "Node.js version:" \+ && node --version \+ && echo "NPM version:" \+ && npm --version++# run compiled code+CMD npm run start:proddiff --git a/test/e2e/dom-default-ts/actor/apify.json b/test/e2e/dom-default-ts/actor/apify.jsonnew file mode 100644index 000000000..f9b4f3874--- /dev/null+++ b/test/e2e/dom-default-ts/actor/apify.json@@ -0,0 +1,6 @@+{+ "name": "test-dom-default-ts",+ "version": "0.0",+ "buildTag": "latest",+ "env": null+}diff --git a/test/e2e/dom-default-ts/actor/main.ts b/test/e2e/dom-default-ts/actor/main.tsnew file mode 100644index 000000000..2d36c2c47--- /dev/null+++ b/test/e2e/dom-default-ts/actor/main.ts@@ -0,0 +1,27 @@+import { Actor } from 'apify';+import { DOMCrawler, Dataset } from '@crawlee/dom';+import { ApifyStorageLocal } from '@apify/storage-local';++if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {+ await Actor.init({ storage: new ApifyStorageLocal() });+} else {+ await Actor.init();+}++const crawler = new DOMCrawler();++crawler.router.addDefaultHandler(async ({ window, enqueueLinks, request, log }) => {+ const { url } = request;+ await enqueueLinks({+ globs: ['https://crawlee.dev/docs/**'],+ });++ const pageTitle = window.document.title;+ log.info(`URL: ${url} TITLE: ${pageTitle}`);++ await Dataset.pushData({ url, pageTitle });+});++await crawler.run(['https://crawlee.dev/docs/quick-start']);++await Actor.exit({ exit: Actor.isAtHome() });diff --git a/test/e2e/dom-default-ts/actor/package.json b/test/e2e/dom-default-ts/actor/package.jsonnew file mode 100644index 000000000..404012934--- /dev/null+++ b/test/e2e/dom-default-ts/actor/package.json@@ -0,0 +1,28 @@+{+ "name": "test-dom-default-ts",+ "version": "0.0.1",+ "description": "DOM Crawler Test - TypeScript",+ "dependencies": {+ "apify": "^3.0.0-beta.75",+ "@apify/storage-local": "^2.1.0",+ "@crawlee/basic": "file:./packages/basic-crawler",+ "@crawlee/browser-pool": "file:./packages/browser-pool",+ "@crawlee/http": "file:./packages/http-crawler",+ "@crawlee/dom": "file:./packages/dom-crawler",+ "@crawlee/core": "file:./packages/core",+ "@crawlee/memory-storage": "file:./packages/memory-storage",+ "@crawlee/types": "file:./packages/types",+ "@crawlee/utils": "file:./packages/utils"+ },+ "devDependencies": {+ "@apify/tsconfig": "^0.1.0",+ "typescript": "4.7.4"+ },+ "scripts": {+ "start": "tsc && node main.js",+ "start:prod": "node main.js",+ "build": "tsc"+ },+ "type": "module",+ "license": "ISC"+}diff --git a/test/e2e/dom-default-ts/actor/tsconfig.json b/test/e2e/dom-default-ts/actor/tsconfig.jsonnew file mode 100644index 000000000..741845092--- /dev/null+++ b/test/e2e/dom-default-ts/actor/tsconfig.json@@ -0,0 +1,11 @@+{+ "extends": "@apify/tsconfig",+ "compilerOptions": {+ "module": "ES2022",+ "target": "ES2022",+ "lib": ["DOM"]+ },+ "include": [+ "./**/*.ts"+ ]+}diff --git a/test/e2e/dom-default-ts/test.mjs b/test/e2e/dom-default-ts/test.mjsnew file mode 100644index 000000000..bf2015b4e--- /dev/null+++ b/test/e2e/dom-default-ts/test.mjs@@ -0,0 +1,10 @@+import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs';++const testActorDirname = getActorTestDir(import.meta.url);+await initialize(testActorDirname);++const { stats, datasetItems } = await runActor(testActorDirname);++await expect(stats.requestsFinished > 40, 'All requests finished');+await expect(datasetItems.length > 40, 'Number of dataset items');+await expect(validateDataset(datasetItems, ['url', 'pageTitle']), 'Dataset items validation');diff --git a/tsconfig.json b/tsconfig.jsonindex 9482010cb..aadee6a9f 100644--- a/tsconfig.json+++ b/tsconfig.json@@ -7,6 +7,7 @@"@crawlee/basic": ["packages/basic-crawler/src"],"@crawlee/browser": ["packages/browser-crawler/src"],"@crawlee/http": ["packages/http-crawler/src"],+ "@crawlee/dom": ["packages/dom-crawler/src"],"@crawlee/cheerio": ["packages/cheerio-crawler/src"],"@crawlee/playwright": ["packages/playwright-crawler/src"],"@crawlee/puppeteer": ["packages/puppeteer-crawler/src"],diff --git a/website/docusaurus.config.js b/website/docusaurus.config.jsindex bd78632e9..2df6ce95d 100644--- a/website/docusaurus.config.js+++ b/website/docusaurus.config.js@@ -14,6 +14,7 @@ const packages = ['cheerio-crawler','puppeteer-crawler','playwright-crawler',+ 'dom-crawler','memory-storage','utils','types',@@ -25,6 +26,7 @@ const packagesOrder = ['@crawlee/playwright','@crawlee/basic','@crawlee/http',+ '@crawlee/dom','@crawlee/browser','@crawlee/memory-storage','@crawlee/browser-pool',diff --git a/website/sidebars.js b/website/sidebars.jsindex daf192590..5c6d229d2 100644--- a/website/sidebars.js+++ b/website/sidebars.js@@ -39,6 +39,7 @@ module.exports = {'guides/session-management','guides/scaling-crawlers','guides/avoid-blocking',+ 'guides/dom-crawler-guide','guides/got-scraping','guides/typescript-project','guides/docker-images',