Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat!: remove `@crawlee/jsdom` and `@crawlee/linkedom` from the monorepo (#4119)
Jindřich Bärbc11f33e7195
diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.mdindex f5556a263..c9d91a3db 100644--- a/.claude/CLAUDE.md+++ b/.claude/CLAUDE.md@@ -67,10 +67,9 @@ Crawlee is a **Yarn workspaces monorepo** with Turbo build orchestration. All pa↓@crawlee/http # HttpCrawler↓-┌──────┴──────┬─────────────┐-↓ ↓ ↓-@crawlee/cheerio @crawlee/jsdom @crawlee/linkedom-(HTML parsing variants)+↓+@crawlee/cheerio+(@crawlee/jsdom and @crawlee/linkedom moved to their own repositories)@crawlee/browser-pool # Browser instance management↓diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.ymlindex 462e87c7e..2425b491f 100644--- a/.github/ISSUE_TEMPLATE/bug_report.yml+++ b/.github/ISSUE_TEMPLATE/bug_report.yml@@ -15,7 +15,6 @@ body:options:- "@crawlee/core"- "@crawlee/cheerio (CheerioCrawler)"- - "@crawlee/jsdom (JSDOMCrawler)"- "@crawlee/playwright (PlaywrightCrawler)"- "@crawlee/puppeteer (PuppeteerCrawler)"- "@crawlee/basic (BasicCrawler)"diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.ymlindex fa5728388..2dee14786 100644--- a/.github/ISSUE_TEMPLATE/feature_request.yml+++ b/.github/ISSUE_TEMPLATE/feature_request.yml@@ -10,7 +10,6 @@ body:options:- "@crawlee/core"- "@crawlee/cheerio (CheerioCrawler)"- - "@crawlee/jsdom (JSDOMCrawler)"- "@crawlee/playwright (PlaywrightCrawler)"- "@crawlee/puppeteer (PuppeteerCrawler)"- "@crawlee/basic (BasicCrawler)"diff --git a/docs/examples/jsdom_crawler.mdx b/docs/examples/jsdom_crawler.mdxdeleted file mode 100644index 32732a0d8..000000000--- a/docs/examples/jsdom_crawler.mdx+++ /dev/null@@ -1,22 +0,0 @@-----id: jsdom-crawler-title: JSDOM crawler------import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';-import ApiLink from '@site/src/components/ApiLink';-import JSDOMCrawlerSource from '!!raw-loader!roa-loader!./jsdom_crawler.ts';-import JSDOMCrawlerRunScriptSource from '!!raw-loader!roa-loader!./jsdom_crawler_react.ts';--This example demonstrates how to use <ApiLink to="jsdom-crawler/class/JSDOMCrawler">`JSDOMCrawler`</ApiLink> to interact with a website using [jsdom](https://www.npmjs.com/package/jsdom) DOM implementation.-Here the script will open a calculator app from the [React examples](https://reactjs.org/community/examples.html), click `1` `+` `1` `=` and extract the result.--<RunnableCodeBlock className="language-ts" type="cheerio">- {JSDOMCrawlerRunScriptSource}-</RunnableCodeBlock>--In the following example, we use <ApiLink to="jsdom-crawler/class/JSDOMCrawler">`JSDOMCrawler`</ApiLink> to crawl a list of URLs from an external file, load each URL using a plain HTTP request, parse the HTML using the [jsdom](https://www.npmjs.com/package/jsdom) DOM implementation and extract some data from it: the page title and all `h1` tags.--<RunnableCodeBlock className="language-ts" type="cheerio">- {JSDOMCrawlerSource}-</RunnableCodeBlock>diff --git a/docs/examples/jsdom_crawler.ts b/docs/examples/jsdom_crawler.tsdeleted file mode 100644index ed5b8a8cf..000000000--- a/docs/examples/jsdom_crawler.ts+++ /dev/null@@ -1,62 +0,0 @@-import { JSDOMCrawler, log, LogLevel } from 'crawlee';--// Crawlers come with various utilities, e.g. for logging.-// Here we use debug level of logging to improve the debugging experience.-// This functionality is optional!-log.setLevel(LogLevel.DEBUG);--// Create an instance of the JSDOMCrawler class - a crawler-// that automatically loads the URLs and parses their HTML using the jsdom library.-const crawler = new JSDOMCrawler({- // The crawler downloads and processes the web pages in parallel, with a concurrency- // automatically managed based on the available system memory and CPU (see ConcurrencySystem class).- // Here we define some hard limits for the concurrency.- minConcurrency: 10,- maxConcurrency: 50,-- // On error, retry each page at most once.- maxRequestRetries: 1,-- // Increase the timeout for processing of each page.- requestHandlerTimeoutSecs: 30,-- // Limit to 10 requests per one crawl- maxRequestsPerCrawl: 10,-- // This function will be called for each URL to crawl.- // It accepts a single parameter, which is an object with options as:- // https://crawlee.dev/js/api/jsdom-crawler/interface/JSDOMCrawlerOptions#requestHandler- // We use for demonstration only 2 of them:- // - request: an instance of the Request class with information such as the URL that is being crawled and HTTP method- // - window: the JSDOM window object- async requestHandler({ pushData, request, window }) {- log.debug(`Processing ${request.url}...`);-- // Extract data from the page- const title = window.document.title;- const h1texts: { text: string }[] = [];- window.document.querySelectorAll('h1').forEach((element) => {- h1texts.push({- text: element.textContent!,- });- });-- // Store the results to the dataset. In local configuration,- // the data will be stored as JSON files in ./storage/datasets/default- await pushData({- url: request.url,- title,- h1texts,- });- },-- // This function is called if the page processing failed more than maxRequestRetries + 1 times.- failedRequestHandler({ request }) {- log.debug(`Request ${request.url} failed twice.`);- },-});--// Run the crawler and wait for it to finish.-await crawler.run(['https://crawlee.dev']);--log.debug('Crawler finished.');diff --git a/docs/examples/jsdom_crawler_react.ts b/docs/examples/jsdom_crawler_react.tsdeleted file mode 100644index 67e55e175..000000000--- a/docs/examples/jsdom_crawler_react.ts+++ /dev/null@@ -1,30 +0,0 @@-import { JSDOMCrawler, log } from 'crawlee';--// Create an instance of the JSDOMCrawler class - crawler that automatically-// loads the URLs and parses their HTML using the jsdom library.-const crawler = new JSDOMCrawler({- // Setting the `runScripts` option to `true` allows the crawler to execute client-side- // JavaScript code on the page. This is required for some websites (such as the React application in this example), but may pose a security risk.- runScripts: true,- // This function will be called for each crawled URL.- // Here we extract the window object from the options and use it to extract data from the page.- requestHandler: async ({ window }) => {- const { document } = window;- // The `document` object is analogous to the `window.document` object you know from your favourite web browsers.- // Thanks to this, you can use the regular browser-side APIs here.- document.querySelectorAll('button')[12].click(); // 1- document.querySelectorAll('button')[15].click(); // +- document.querySelectorAll('button')[12].click(); // 1- document.querySelectorAll('button')[18].click(); // =-- const result = document.querySelectorAll('.component-display')[0].childNodes[0] as Element;- // The result is passed to the console. Unlike with Playwright or Puppeteer crawlers,- // this console call goes to the Node.js console, not the browser console. All the code here runs right in Node.js!- log.info(result.innerHTML); // 2- },-});--// Run the crawler and wait for it to finish.-await crawler.run(['https://ahfarmer.github.io/calculator/']);--log.debug('Crawler finished.');diff --git a/docs/guides/jsdom_crawler.mdx b/docs/guides/jsdom_crawler.mdxdeleted file mode 100644index 86c402d6d..000000000--- a/docs/guides/jsdom_crawler.mdx+++ /dev/null@@ -1,61 +0,0 @@-----id: jsdom-crawler-guide-title: "JSDOMCrawler guide"-sidebar_label: "JSDOMCrawler"-description: Your first steps into the world of scraping with Crawlee------import ApiLink from '@site/src/components/ApiLink';--​<ApiLink to="jsdom-crawler/class/JSDOMCrawler">`JSDOMCrawler`</ApiLink> is very useful for scraping with the Window API.--## How the crawler works--​<ApiLink to="jsdom-crawler/class/JSDOMCrawler">`JSDOMCrawler`</ApiLink> crawls by making plain HTTP requests to the provided URLs using the specialized [got-scraping](https://github.com/apify/got-scraping) HTTP client. The URLs are fed to the crawler using <ApiLink to="core/class/RequestQueue">`RequestQueue`</ApiLink>. The HTTP responses it gets back are usually HTML pages. The same pages you would get in your browser when you first load a URL. But it can handle any content types with the help of the <ApiLink to="jsdom-crawler/interface/JSDOMCrawlerOptions#additionalMimeTypes">`additionalMimeTypes`</ApiLink> option.--:::info--Modern web pages often do not serve all of their content in the first HTML response, but rather the first HTML contains links to other resources such as CSS and JavaScript that get downloaded afterwards, and together they create the final page. To crawl those, see <ApiLink to="puppeteer-crawler/class/PuppeteerCrawler">`PuppeteerCrawler`</ApiLink> and <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink>.--:::--Once the page's HTML is retrieved, the crawler will pass it to [JSDOM](https://www.npmjs.com/package/jsdom) for parsing. The result is a `window` property, which should be familiar to frontend developers. You can use the Window API to do all sorts of lookups and manipulation of the page's HTML, but in scraping, you will mostly use it to find specific HTML elements and extract their data.--Example use of browser JavaScript:--```ts-// Return the page title-document.title; // browsers-window.document.title; // JSDOM-```--## When to use `JSDOMCrawler`--`JSDOMCrawler` really shines when `CheerioCrawler` is just not enough. There is an entire set of [APIs](https://developer.mozilla.org/en-US/docs/Web/API/HTML_DOM_API) available!--**Advantages:**--- Easy to set up-- Familiar for frontend developers-- Content can be manipulated-- Automatically avoids some anti-scraping bans--**Disadvantages:**--- Slower than `CheerioCrawler`-- Does not work for websites that require JavaScript rendering-- May easily overload the target website with requests--## Example use of Element API--### Find all links on a page--This snippet finds all `<a>` elements which have the `href` attribute and extracts the hrefs into an array.--```js-Array.from(document.querySelectorAll('a[href]')).map((a) => a.href);-```--### Other examples--Visit the [Examples](../examples) section to browse examples of `JSDOMCrawler` usage. Almost all examples show `JSDOMCrawler` code in their code tabs.diff --git a/docs/guides/proxy_management.mdx b/docs/guides/proxy_management.mdxindex b27f2ab32..a3178d7ba 100644--- a/docs/guides/proxy_management.mdx+++ b/docs/guides/proxy_management.mdx@@ -11,18 +11,15 @@ import TabItem from '@theme/TabItem';import CodeBlock from '@theme/CodeBlock';import HttpSource from '!!raw-loader!./proxy_management_integration_http.ts';-import JSDOMSource from '!!raw-loader!./proxy_management_integration_jsdom.ts';import CheerioSource from '!!raw-loader!./proxy_management_integration_cheerio.ts';import PlaywrightSource from '!!raw-loader!./proxy_management_integration_playwright.ts';import PuppeteerSource from '!!raw-loader!./proxy_management_integration_puppeteer.ts';import SessionStandaloneSource from '!!raw-loader!./proxy_management_session_standalone.ts';import SessionHttpSource from '!!raw-loader!./proxy_management_session_http.ts';-import SessionJSDOMSource from '!!raw-loader!./proxy_management_session_jsdom.ts';import SessionCheerioSource from '!!raw-loader!./proxy_management_session_cheerio.ts';import SessionPlaywrightSource from '!!raw-loader!./proxy_management_session_playwright.ts';import SessionPuppeteerSource from '!!raw-loader!./proxy_management_session_puppeteer.ts';import InspectionHttpSource from '!!raw-loader!./proxy_management_inspection_http.ts';-import InspectionJSDOMSource from '!!raw-loader!./proxy_management_inspection_jsdom.ts';import InspectionCheerioSource from '!!raw-loader!./proxy_management_inspection_cheerio.ts';import InspectionPlaywrightSource from '!!raw-loader!./proxy_management_inspection_playwright.ts';import InspectionPuppeteerSource from '!!raw-loader!./proxy_management_inspection_puppeteer.ts';@@ -99,7 +96,7 @@ The `options` parameter is an object containing a <ApiLink to="core/class/Reques## Crawler integration-`ProxyConfiguration` integrates seamlessly into <ApiLink to="http-crawler/class/HttpCrawler">`HttpCrawler`</ApiLink>, <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink>, <ApiLink to="jsdom-crawler/class/JSDOMCrawler">`JSDOMCrawler`</ApiLink>, <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> and <ApiLink to="puppeteer-crawler/class/PuppeteerCrawler">`PuppeteerCrawler`</ApiLink>.+`ProxyConfiguration` integrates seamlessly into <ApiLink to="http-crawler/class/HttpCrawler">`HttpCrawler`</ApiLink>, <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink>, <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> and <ApiLink to="puppeteer-crawler/class/PuppeteerCrawler">`PuppeteerCrawler`</ApiLink>.<Tabs groupId="proxy_session_management"><TabItem value="http" label="HttpCrawler">@@ -112,11 +109,6 @@ The `options` parameter is an object containing a <ApiLink to="core/class/Reques{CheerioSource}</CodeBlock></TabItem>- <TabItem value="jsdom" label="JSDOMCrawler">- <CodeBlock language="js">- {JSDOMSource}- </CodeBlock>- </TabItem><TabItem value="playwright" label="PlaywrightCrawler"><CodeBlock language="js">{PlaywrightSource}@@ -146,11 +138,6 @@ Each call to <ApiLink to="core/class/ProxyConfiguration#newUrl">`proxyConfigurat{SessionCheerioSource}</CodeBlock></TabItem>- <TabItem value="jsdom" label="JSDOMCrawler">- <CodeBlock language="js">- {SessionJSDOMSource}- </CodeBlock>- </TabItem><TabItem value="playwright" label="PlaywrightCrawler"><CodeBlock language="js">{SessionPlaywrightSource}@@ -170,7 +157,7 @@ Each call to <ApiLink to="core/class/ProxyConfiguration#newUrl">`proxyConfigurat## Inspecting current proxy in Crawlers-`HttpCrawler`, `CheerioCrawler`, `JSDOMCrawler`, `PlaywrightCrawler` and `PuppeteerCrawler` grant access to information about the currently used proxy+`HttpCrawler`, `CheerioCrawler`, `PlaywrightCrawler` and `PuppeteerCrawler` grant access to information about the currently used proxyin their `requestHandler` using a <ApiLink to="types/interface/ProxyInfo">`proxyInfo`</ApiLink> object.With the `proxyInfo` object, we can easily access the proxy URL.@@ -185,11 +172,6 @@ With the `proxyInfo` object, we can easily access the proxy URL.{InspectionCheerioSource}</CodeBlock></TabItem>- <TabItem value="jsdom" label="JSDOMCrawler">- <CodeBlock language="js">- {InspectionJSDOMSource}- </CodeBlock>- </TabItem><TabItem value="playwright" label="PlaywrightCrawler"><CodeBlock language="js">{InspectionPlaywrightSource}diff --git a/docs/guides/proxy_management_inspection_jsdom.ts b/docs/guides/proxy_management_inspection_jsdom.tsdeleted file mode 100644index 3ab09968a..000000000--- a/docs/guides/proxy_management_inspection_jsdom.ts+++ /dev/null@@ -1,13 +0,0 @@-import { JSDOMCrawler, ProxyConfiguration } from 'crawlee';--const proxyConfiguration = new ProxyConfiguration({- /* opts */-});--const crawler = new JSDOMCrawler({- proxyConfiguration,- async requestHandler({ proxyInfo }) {- console.log(proxyInfo);- },- // ...-});diff --git a/docs/guides/proxy_management_integration_jsdom.ts b/docs/guides/proxy_management_integration_jsdom.tsdeleted file mode 100644index f8a60ddab..000000000--- a/docs/guides/proxy_management_integration_jsdom.ts+++ /dev/null@@ -1,10 +0,0 @@-import { JSDOMCrawler, ProxyConfiguration } from 'crawlee';--const proxyConfiguration = new ProxyConfiguration({- proxyUrls: ['http://proxy-1.com', 'http://proxy-2.com'],-});--const crawler = new JSDOMCrawler({- proxyConfiguration,- // ...-});diff --git a/docs/guides/proxy_management_session_jsdom.ts b/docs/guides/proxy_management_session_jsdom.tsdeleted file mode 100644index 8162643bd..000000000--- a/docs/guides/proxy_management_session_jsdom.ts+++ /dev/null@@ -1,11 +0,0 @@-import { JSDOMCrawler, ProxyConfiguration } from 'crawlee';--const proxyConfiguration = new ProxyConfiguration({- /* opts */-});--const crawler = new JSDOMCrawler({- saveResponseCookies: true,- proxyConfiguration,- // ...-});diff --git a/docs/guides/session_management.mdx b/docs/guides/session_management.mdxindex 197ba9f1c..8c11e35a9 100644--- a/docs/guides/session_management.mdx+++ b/docs/guides/session_management.mdx@@ -13,7 +13,6 @@ import CodeBlock from '@theme/CodeBlock';import BasicSource from '!!raw-loader!./session_management_basic.ts';import HttpSource from '!!raw-loader!./session_management_http.ts';import CheerioSource from '!!raw-loader!./session_management_cheerio.ts';-import JSDOMSource from '!!raw-loader!./session_management_jsdom.ts';import PlaywrightSource from '!!raw-loader!./session_management_playwright.ts';import PuppeteerSource from '!!raw-loader!./session_management_puppeteer.ts';import StandaloneSource from '!!raw-loader!./session_management_standalone.ts';@@ -30,7 +29,6 @@ Now let's take a look at the examples of how to use the session pool:- with <ApiLink to="basic-crawler/class/BasicCrawler">`BasicCrawler`</ApiLink>;- with <ApiLink to="http-crawler/class/HttpCrawler">`HttpCrawler`</ApiLink>;- with <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink>;-- with <ApiLink to="jsdom-crawler/class/JSDOMCrawler">`JSDOMCrawler`</ApiLink>;- with <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink>;- with <ApiLink to="puppeteer-crawler/class/PuppeteerCrawler">`PuppeteerCrawler`</ApiLink>;- without a crawler (standalone usage to manage sessions manually).@@ -51,11 +49,6 @@ Now let's take a look at the examples of how to use the session pool:{CheerioSource}</CodeBlock></TabItem>- <TabItem value="jsdom" label="JSDOMCrawler">- <CodeBlock language="js">- {JSDOMSource}- </CodeBlock>- </TabItem><TabItem value="playwright" label="PlaywrightCrawler"><CodeBlock language="js">{PlaywrightSource}diff --git a/docs/guides/session_management_jsdom.ts b/docs/guides/session_management_jsdom.tsdeleted file mode 100644index ee8e7cfff..000000000--- a/docs/guides/session_management_jsdom.ts+++ /dev/null@@ -1,26 +0,0 @@-import { JSDOMCrawler, ProxyConfiguration, SessionPool } from 'crawlee';--const proxyConfiguration = new ProxyConfiguration({- /* opts */-});--const crawler = new JSDOMCrawler({- // To use the proxy IP session rotation logic, you must turn the proxy usage on.- proxyConfiguration,- // Overrides default Session pool configuration.- sessionPool: new SessionPool({ maxPoolSize: 100 }),- // Set to true if you want the crawler to save cookies per session,- // and set the cookie header to request automatically (default is true).- saveResponseCookies: true,- async requestHandler({ session, window }) {- const title = window.document.title;-- if (title === 'Blocked') {- session?.retire();- } else if (title === 'Not sure if blocked, might also be a connection error') {- session?.markBad();- } else {- // session.markGood() - this step is done automatically in BasicCrawler.- }- },-});diff --git a/docs/public-api/crawlee-jsdom.api.md b/docs/public-api/crawlee-jsdom.api.mddeleted file mode 100644index 0cf817e6e..000000000--- a/docs/public-api/crawlee-jsdom.api.md+++ /dev/null@@ -1,79 +0,0 @@-## Public API Report File for "@crawlee/jsdom"--> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).--```ts--import type { CrawlingContext } from '@crawlee/http';-import type { Dictionary } from '@crawlee/types';-import { DOMCrawler } from '@crawlee/http';-import type { DOMCrawlingContext } from '@crawlee/http';-import type { DOMWindow } from 'jsdom';-import type { ErrorHandler } from '@crawlee/http';-import type { GetUserDataFromRequest } from '@crawlee/http';-import type { HttpCrawlerOptions } from '@crawlee/http';-import type { InternalHttpHook } from '@crawlee/http';-import type { RequestHandler } from '@crawlee/http';-import type { RouterHandler } from '@crawlee/http';-import type { RouterRoutes } from '@crawlee/http';-import type { RouteSchemas } from '@crawlee/http';-import type { RoutesFromSchemas } from '@crawlee/http';-import { VirtualConsole } from 'jsdom';--// @public-export function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;--// @public (undocumented)-export function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;--// @public (undocumented)-export function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;--// @public-export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<JSDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends DOMCrawler<JSDOMParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {- constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);- getVirtualConsole(): VirtualConsole;-}--// @public (undocumented)-export interface JSDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>, StatisticStateExtension extends object = {}> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {- hideInternalConsole?: boolean;- runScripts?: boolean;-}--// @public (undocumented)-export interface JSDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-JSONData extends Dictionary = any> extends DOMCrawlingContext<JSDOMParseResult, UserData, JSONData> {-}--// @public (undocumented)-export type JSDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, JSDOMCrawlingContext<UserData, JSONData> & ContextExtension>;--// @public (undocumented)-export type JSDOMHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-JSONData extends Dictionary = any> = InternalHttpHook<JSDOMCrawlingContext<UserData, JSONData>>;--// @public (undocumented)-export interface JSDOMParseResult {- // (undocumented)- body: string;- // (undocumented)- document: Document;- // (undocumented)- window: DOMWindow;-}--// @public (undocumented)-export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-JSONData extends Dictionary = any> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;---export * from "@crawlee/http";--// (No @packageDocumentation comment for this package)--```diff --git a/docs/public-api/crawlee-linkedom.api.md b/docs/public-api/crawlee-linkedom.api.mddeleted file mode 100644index e8e3e3e22..000000000--- a/docs/public-api/crawlee-linkedom.api.md+++ /dev/null@@ -1,74 +0,0 @@-## Public API Report File for "@crawlee/linkedom"--> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).--```ts--import type { CrawlingContext } from '@crawlee/http';-import type { Dictionary } from '@crawlee/types';-import { DOMCrawler } from '@crawlee/http';-import type { DOMCrawlingContext } from '@crawlee/http';-import type { ErrorHandler } from '@crawlee/http';-import type { GetUserDataFromRequest } from '@crawlee/http';-import type { HttpCrawlerOptions } from '@crawlee/http';-import type { InternalHttpHook } from '@crawlee/http';-import type { RequestHandler } from '@crawlee/http';-import type { RouterHandler } from '@crawlee/http';-import type { RouterRoutes } from '@crawlee/http';-import type { RouteSchemas } from '@crawlee/http';-import type { RoutesFromSchemas } from '@crawlee/http';--// @public-export function createLinkeDOMRouter<Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;--// @public (undocumented)-export function createLinkeDOMRouter<Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;--// @public (undocumented)-export function createLinkeDOMRouter<Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;--// @public-export class LinkeDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends DOMCrawler<LinkeDOMParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {- constructor(options?: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);-}--// @public (undocumented)-export interface LinkeDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>, StatisticStateExtension extends object = {}> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {-}--// @public (undocumented)-export interface LinkeDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-JSONData extends Dictionary = any> extends DOMCrawlingContext<LinkeDOMParseResult, UserData, JSONData> {-}--// @public (undocumented)-export type LinkeDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, LinkeDOMCrawlingContext<UserData, JSONData> & ContextExtension>;--// @public (undocumented)-export type LinkeDOMHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-JSONData extends Dictionary = any> = InternalHttpHook<LinkeDOMCrawlingContext<UserData, JSONData>>;--// @public (undocumented)-export interface LinkeDOMParseResult {- // (undocumented)- body: string;- // (undocumented)- document: Document;- // (undocumented)- window: Window;-}--// @public (undocumented)-export type LinkeDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-JSONData extends Dictionary = any> = RequestHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;---export * from "@crawlee/http";--// (No @packageDocumentation comment for this package)--```diff --git a/docs/public-api/crawlee.api.md b/docs/public-api/crawlee.api.mdindex 598fbca12..e6b955a29 100644--- a/docs/public-api/crawlee.api.md+++ b/docs/public-api/crawlee.api.md@@ -33,8 +33,6 @@ export * from "@crawlee/cheerio";export * from "@crawlee/core";export * from "@crawlee/fs-storage";export * from "@crawlee/http";-export * from "@crawlee/jsdom";-export * from "@crawlee/linkedom";export * from "@crawlee/playwright";export * from "@crawlee/puppeteer";export * from "@crawlee/utils";diff --git a/docs/tsconfig.json b/docs/tsconfig.jsonindex 9df76ee8e..fda6debaf 100644--- a/docs/tsconfig.json+++ b/docs/tsconfig.json@@ -2,7 +2,7 @@"extends": "../tsconfig.build.json","include": ["./**/*.ts"],"compilerOptions": {- "lib": ["ES2022", "DOM.AsyncIterable", "ESNext.Disposable"],+ "lib": ["ES2022", "DOM", "DOM.AsyncIterable", "ESNext.Disposable"],"noUnusedLocals": false}}diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex 1be517df4..2e2309119 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -2158,6 +2158,16 @@ Relatedly, `RobotsTxtFile.getSitemaps()`, `parseSitemaps()`, and `parseUrlsFromSThe HTML-parsing helper functions `htmlToText`, `parseHandlesFromHtml` and `parseOpenGraph` are now asynchronous and return promises.+## Only if you use `JSDOMCrawler` or `LinkeDOMCrawler`++They moved out of this repository into their own packages, so install them explicitly:++```bash+npm install @crawlee/jsdom @crawlee/linkedom+```++The `crawlee` meta-package no longer re-exports them.+## Only if you import from `@crawlee/core` directlyThe crawler-only parts of `@crawlee/core` moved to `@crawlee/basic`, so that `@crawlee/core` carries just the storage, request and configuration layer. The moved exports are:diff --git a/package.json b/package.jsonindex 4444dbfb8..b86e72018 100644--- a/package.json+++ b/package.json@@ -67,8 +67,6 @@"@crawlee/core": "workspace:*","@crawlee/http-client": "workspace:*","@crawlee/impit-client": "workspace:*",- "@crawlee/jsdom": "workspace:*",- "@crawlee/linkedom": "workspace:*","@crawlee/otel": "workspace:*","@crawlee/playwright": "workspace:*","@crawlee/puppeteer": "workspace:*",diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.tsindex b565005cd..0fb47e975 100644--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts@@ -15,8 +15,6 @@ import type { BrowserController } from './browser-controller.js';* - without using a fingerprint,* - without specifying a user agent.* Last updated on 2022-05-05.- *- * After you update it here, please update it also in jsdom-crawler.ts*/export const DEFAULT_USER_AGENT ='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36';diff --git a/packages/core/README.md b/packages/core/README.mdindex daa808eb1..651185cd6 100644--- a/packages/core/README.md+++ b/packages/core/README.md@@ -8,10 +8,8 @@ The [`crawlee`](https://www.npmjs.com/package/crawlee) package consists of sever- [`@crawlee/cheerio`](https://crawlee.dev/js/api/cheerio-crawler): exports `CheerioCrawler`- [`@crawlee/playwright`](https://crawlee.dev/js/api/playwright-crawler): exports `PlaywrightCrawler`- [`@crawlee/puppeteer`](https://crawlee.dev/js/api/puppeteer-crawler): exports `PuppeteerCrawler`-- [`@crawlee/linkedom`](https://crawlee.dev/js/api/linkedom-crawler): exports `LinkeDOMCrawler`-- [`@crawlee/jsdom`](https://crawlee.dev/js/api/jsdom-crawler): exports `JSDOMCrawler`- [`@crawlee/basic`](https://crawlee.dev/js/api/basic-crawler): exports `BasicCrawler`-- [`@crawlee/http`](https://crawlee.dev/js/api/http-crawler): exports `HttpCrawler` (which is used for creating [`@crawlee/jsdom`](https://crawlee.dev/js/api/jsdom-crawler) and [`@crawlee/cheerio`](https://crawlee.dev/js/api/cheerio-crawler))+- [`@crawlee/http`](https://crawlee.dev/js/api/http-crawler): exports `HttpCrawler` (which is used for creating [`@crawlee/cheerio`](https://crawlee.dev/js/api/cheerio-crawler))- [`@crawlee/browser`](https://crawlee.dev/js/api/browser-crawler): exports `BrowserCrawler` (which is used for creating [`@crawlee/playwright`](https://crawlee.dev/js/api/playwright-crawler) and [`@crawlee/puppeteer`](https://crawlee.dev/js/api/puppeteer-crawler))- [`@crawlee/memory-storage`](https://crawlee.dev/js/api/memory-storage): [`@apify/storage-local`](https://npmjs.com/package/@apify/storage-local) alternative- [`@crawlee/browser-pool`](https://crawlee.dev/js/api/browser-pool): previously [`browser-pool`](https://npmjs.com/package/browser-pool) packagediff --git a/packages/crawlee/package.json b/packages/crawlee/package.jsonindex 90aeebaf1..7524ee1f6 100644--- a/packages/crawlee/package.json+++ b/packages/crawlee/package.json@@ -57,8 +57,6 @@"@crawlee/core": "workspace:*","@crawlee/fs-storage": "workspace:*","@crawlee/http": "workspace:*",- "@crawlee/jsdom": "workspace:*",- "@crawlee/linkedom": "workspace:*","@crawlee/playwright": "workspace:*","@crawlee/puppeteer": "workspace:*","@crawlee/utils": "workspace:*",diff --git a/packages/crawlee/src/index.ts b/packages/crawlee/src/index.tsindex 54c490ccf..bfd1a4a5f 100644--- a/packages/crawlee/src/index.ts+++ b/packages/crawlee/src/index.ts@@ -8,8 +8,6 @@ export * from '@crawlee/utils';export * from '@crawlee/basic';export * from '@crawlee/browser';export * from '@crawlee/http';-export * from '@crawlee/jsdom';-export * from '@crawlee/linkedom';export * from '@crawlee/cheerio';export * from '@crawlee/puppeteer';export * from '@crawlee/playwright';diff --git a/packages/jsdom-crawler/.npmignore b/packages/jsdom-crawler/.npmignoredeleted file mode 100644index ada367d72..000000000--- a/packages/jsdom-crawler/.npmignore+++ /dev/null@@ -1,6 +0,0 @@-node_modules-src-test-coverage-apify_storage-tsconfig.*diff --git a/packages/jsdom-crawler/CHANGELOG.md b/packages/jsdom-crawler/CHANGELOG.mddeleted file mode 100644index e394f5c6e..000000000--- a/packages/jsdom-crawler/CHANGELOG.md+++ /dev/null@@ -1,653 +0,0 @@-# Change Log--All notable changes to this project will be documented in this file.-See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.--## [3.18.1](https://github.com/apify/crawlee/compare/v3.18.0...v3.18.1) (2026-08-12)--### Bug Fixes--* respect `maxCrawlDepth` in `JSDOM/LinkeDOM` context `enqueueLinks` ([#4021](https://github.com/apify/crawlee/issues/4021)) ([4d34aea](https://github.com/apify/crawlee/commit/4d34aeab6d78d1bd0bbe27418daf621f28a3524c)), closes [#3927](https://github.com/apify/crawlee/issues/3927) [#4020](https://github.com/apify/crawlee/issues/4020)---# [3.18.0](https://github.com/apify/crawlee/compare/v3.17.0...v3.18.0) (2026-08-04)--### Bug Fixes--* **core:** ignore explicitly undefined baseUrl in enqueueLinks helpers ([#3924](https://github.com/apify/crawlee/issues/3924)) ([a58a9c1](https://github.com/apify/crawlee/commit/a58a9c1417c34c2a45a10f3855c4bee721a480ae))--### Features--* opt-in schema validation of request userData per router label ([#3851](https://github.com/apify/crawlee/issues/3851)) ([c7347f9](https://github.com/apify/crawlee/commit/c7347f94f72980b7c424fb4fc7dc8f990f360d70)), closes [#3747](https://github.com/apify/crawlee/issues/3747) [#3748](https://github.com/apify/crawlee/issues/3748) [#3747](https://github.com/apify/crawlee/issues/3747) [#3082](https://github.com/apify/crawlee/issues/3082)-* type-safe router labels via per-label userData map ([#3747](https://github.com/apify/crawlee/issues/3747)) ([1124aca](https://github.com/apify/crawlee/commit/1124aca3c5eb527c097decd7263a6816c411369e)), closes [#3082](https://github.com/apify/crawlee/issues/3082)---# [3.17.0](https://github.com/apify/crawlee/compare/v3.16.0...v3.17.0) (2026-06-04)--**Note:** Version bump only for package @crawlee/jsdom------# [3.16.0](https://github.com/apify/crawlee/compare/v3.15.3...v3.16.0) (2026-02-06)---### Performance Improvements--* drop `tsbuildinfo` from published packages ([#3243](https://github.com/apify/crawlee/issues/3243)) ([3450f27](https://github.com/apify/crawlee/commit/3450f27880afb9e9d857a54d9212b54c397aed91)), closes [#3239](https://github.com/apify/crawlee/issues/3239)------## [3.15.3](https://github.com/apify/crawlee/compare/v3.15.2...v3.15.3) (2025-11-10)--**Note:** Version bump only for package @crawlee/jsdom------## [3.15.2](https://github.com/apify/crawlee/compare/v3.15.1...v3.15.2) (2025-10-23)--**Note:** Version bump only for package @crawlee/jsdom------## [3.15.1](https://github.com/apify/crawlee/compare/v3.15.0...v3.15.1) (2025-09-26)--**Note:** Version bump only for package @crawlee/jsdom------# [3.15.0](https://github.com/apify/crawlee/compare/v3.14.1...v3.15.0) (2025-09-17)--**Note:** Version bump only for package @crawlee/jsdom------## [3.14.1](https://github.com/apify/crawlee/compare/v3.14.0...v3.14.1) (2025-08-05)--**Note:** Version bump only for package @crawlee/jsdom------# [3.14.0](https://github.com/apify/crawlee/compare/v3.13.10...v3.14.0) (2025-07-25)---### Features--* add `maxCrawlDepth` crawler option ([#3045](https://github.com/apify/crawlee/issues/3045)) ([0090df9](https://github.com/apify/crawlee/commit/0090df93a12df9918d016cf2f1378f1f7d40557d)), closes [#2633](https://github.com/apify/crawlee/issues/2633)------## [3.13.10](https://github.com/apify/crawlee/compare/v3.13.9...v3.13.10) (2025-07-09)--**Note:** Version bump only for package @crawlee/jsdom------## [3.13.9](https://github.com/apify/crawlee/compare/v3.13.8...v3.13.9) (2025-06-27)--**Note:** Version bump only for package @crawlee/jsdom------## [3.13.8](https://github.com/apify/crawlee/compare/v3.13.7...v3.13.8) (2025-06-16)---### Bug Fixes--* Do not enqueue more links than what the crawler is capable of processing ([#2990](https://github.com/apify/crawlee/issues/2990)) ([ea094c8](https://github.com/apify/crawlee/commit/ea094c819232e0b30bc550270836d10506eb9454)), closes [#2728](https://github.com/apify/crawlee/issues/2728)------## [3.13.7](https://github.com/apify/crawlee/compare/v3.13.6...v3.13.7) (2025-06-06)--**Note:** Version bump only for package @crawlee/jsdom------## [3.13.6](https://github.com/apify/crawlee/compare/v3.13.5...v3.13.6) (2025-06-05)--**Note:** Version bump only for package @crawlee/jsdom------## [3.13.5](https://github.com/apify/crawlee/compare/v3.13.4...v3.13.5) (2025-05-20)--**Note:** Version bump only for package @crawlee/jsdom------## [3.13.4](https://github.com/apify/crawlee/compare/v3.13.3...v3.13.4) (2025-05-14)--**Note:** Version bump only for package @crawlee/jsdom------## [3.13.3](https://github.com/apify/crawlee/compare/v3.13.2...v3.13.3) (2025-05-05)--**Note:** Version bump only for package @crawlee/jsdom------## [3.13.2](https://github.com/apify/crawlee/compare/v3.13.1...v3.13.2) (2025-04-08)---### Features--* add `onSkippedRequest` option ([#2916](https://github.com/apify/crawlee/issues/2916)) ([764f992](https://github.com/apify/crawlee/commit/764f99203627b6a44d2ee90d623b8b0e6ecbffb5)), closes [#2910](https://github.com/apify/crawlee/issues/2910)------## [3.13.1](https://github.com/apify/crawlee/compare/v3.13.0...v3.13.1) (2025-04-07)---### Bug Fixes--* rename `RobotsFile` to `RobotsTxtFile` ([#2913](https://github.com/apify/crawlee/issues/2913)) ([3160f71](https://github.com/apify/crawlee/commit/3160f717e865326476d78089d778cbc7d35aa58d)), closes [#2910](https://github.com/apify/crawlee/issues/2910)---### Features--* add `respectRobotsTxtFile` crawler option ([#2910](https://github.com/apify/crawlee/issues/2910)) ([0eabed1](https://github.com/apify/crawlee/commit/0eabed1f13070d902c2c67b340621830a7f64464))------# [3.13.0](https://github.com/apify/crawlee/compare/v3.12.2...v3.13.0) (2025-03-04)--**Note:** Version bump only for package @crawlee/jsdom------## [3.12.2](https://github.com/apify/crawlee/compare/v3.12.1...v3.12.2) (2025-01-27)--**Note:** Version bump only for package @crawlee/jsdom------## [3.12.1](https://github.com/apify/crawlee/compare/v3.12.0...v3.12.1) (2024-12-04)--**Note:** Version bump only for package @crawlee/jsdom------# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)--**Note:** Version bump only for package @crawlee/jsdom------## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)--**Note:** Version bump only for package @crawlee/jsdom------## [3.11.4](https://github.com/apify/crawlee/compare/v3.11.3...v3.11.4) (2024-09-23)--**Note:** Version bump only for package @crawlee/jsdom------## [3.11.3](https://github.com/apify/crawlee/compare/v3.11.2...v3.11.3) (2024-09-03)--**Note:** Version bump only for package @crawlee/jsdom------## [3.11.2](https://github.com/apify/crawlee/compare/v3.11.1...v3.11.2) (2024-08-28)--**Note:** Version bump only for package @crawlee/jsdom------## [3.11.1](https://github.com/apify/crawlee/compare/v3.11.0...v3.11.1) (2024-07-24)--**Note:** Version bump only for package @crawlee/jsdom------# [3.11.0](https://github.com/apify/crawlee/compare/v3.10.5...v3.11.0) (2024-07-09)--**Note:** Version bump only for package @crawlee/jsdom------## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)---### Bug Fixes--* declare missing peer dependencies in `@crawlee/browser` package ([#2532](https://github.com/apify/crawlee/issues/2532)) ([3357c7f](https://github.com/apify/crawlee/commit/3357c7fc5ab071b12f72097c190dbee9990e3751))------## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)--**Note:** Version bump only for package @crawlee/jsdom------## [3.10.3](https://github.com/apify/crawlee/compare/v3.10.2...v3.10.3) (2024-06-07)---### Features--* add `waitForSelector` context helper + `parseWithCheerio` in adaptive crawler ([#2522](https://github.com/apify/crawlee/issues/2522)) ([6f88e73](https://github.com/apify/crawlee/commit/6f88e738d43ab4774dc4ef3f78775a5d88728e0d))------## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)--**Note:** Version bump only for package @crawlee/jsdom------## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)--**Note:** Version bump only for package @crawlee/jsdom------# [3.10.0](https://github.com/apify/crawlee/compare/v3.9.2...v3.10.0) (2024-05-16)--**Note:** Version bump only for package @crawlee/jsdom------## [3.9.2](https://github.com/apify/crawlee/compare/v3.9.1...v3.9.2) (2024-04-17)--**Note:** Version bump only for package @crawlee/jsdom------## [3.9.1](https://github.com/apify/crawlee/compare/v3.9.0...v3.9.1) (2024-04-11)--**Note:** Version bump only for package @crawlee/jsdom------# [3.9.0](https://github.com/apify/crawlee/compare/v3.8.2...v3.9.0) (2024-04-10)--**Note:** Version bump only for package @crawlee/jsdom------## [3.8.2](https://github.com/apify/crawlee/compare/v3.8.1...v3.8.2) (2024-03-21)--**Note:** Version bump only for package @crawlee/jsdom------## [3.8.1](https://github.com/apify/crawlee/compare/v3.8.0...v3.8.1) (2024-02-22)--**Note:** Version bump only for package @crawlee/jsdom------# [3.8.0](https://github.com/apify/crawlee/compare/v3.7.3...v3.8.0) (2024-02-21)--**Note:** Version bump only for package @crawlee/jsdom------## [3.7.3](https://github.com/apify/crawlee/compare/v3.7.2...v3.7.3) (2024-01-30)--**Note:** Version bump only for package @crawlee/jsdom------## [3.7.2](https://github.com/apify/crawlee/compare/v3.7.1...v3.7.2) (2024-01-09)--**Note:** Version bump only for package @crawlee/jsdom------## [3.7.1](https://github.com/apify/crawlee/compare/v3.7.0...v3.7.1) (2024-01-02)--**Note:** Version bump only for package @crawlee/jsdom------# [3.7.0](https://github.com/apify/crawlee/compare/v3.6.2...v3.7.0) (2023-12-21)--**Note:** Version bump only for package @crawlee/jsdom------## [3.6.2](https://github.com/apify/crawlee/compare/v3.6.1...v3.6.2) (2023-11-26)--**Note:** Version bump only for package @crawlee/jsdom------## [3.6.1](https://github.com/apify/crawlee/compare/v3.6.0...v3.6.1) (2023-11-15)--**Note:** Version bump only for package @crawlee/jsdom------# [3.6.0](https://github.com/apify/crawlee/compare/v3.5.8...v3.6.0) (2023-11-15)--**Note:** Version bump only for package @crawlee/jsdom------## [3.5.8](https://github.com/apify/crawlee/compare/v3.5.7...v3.5.8) (2023-10-17)--**Note:** Version bump only for package @crawlee/jsdom------## [3.5.7](https://github.com/apify/crawlee/compare/v3.5.6...v3.5.7) (2023-10-05)--**Note:** Version bump only for package @crawlee/jsdom------## [3.5.6](https://github.com/apify/crawlee/compare/v3.5.5...v3.5.6) (2023-10-04)--**Note:** Version bump only for package @crawlee/jsdom------## [3.5.5](https://github.com/apify/crawlee/compare/v3.5.4...v3.5.5) (2023-10-02)---### Features--* Request Queue v2 ([#1975](https://github.com/apify/crawlee/issues/1975)) ([70a77ee](https://github.com/apify/crawlee/commit/70a77ee15f984e9ae67cd584fc58ace7e55346db)), closes [#1365](https://github.com/apify/crawlee/issues/1365)------## [3.5.4](https://github.com/apify/crawlee/compare/v3.5.3...v3.5.4) (2023-09-11)--**Note:** Version bump only for package @crawlee/jsdom------## [3.5.3](https://github.com/apify/crawlee/compare/v3.5.2...v3.5.3) (2023-08-31)---### Bug Fixes--* pin all internal dependencies ([#2041](https://github.com/apify/crawlee/issues/2041)) ([d6f2b17](https://github.com/apify/crawlee/commit/d6f2b172d4a6776137c7893ca798d5b4a9408e79)), closes [#2040](https://github.com/apify/crawlee/issues/2040)------## [3.5.2](https://github.com/apify/crawlee/compare/v3.5.1...v3.5.2) (2023-08-21)--**Note:** Version bump only for package @crawlee/jsdom------## [3.5.1](https://github.com/apify/crawlee/compare/v3.5.0...v3.5.1) (2023-08-16)--**Note:** Version bump only for package @crawlee/jsdom------# [3.5.0](https://github.com/apify/crawlee/compare/v3.4.2...v3.5.0) (2023-07-31)--**Note:** Version bump only for package @crawlee/jsdom------## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)--**Note:** Version bump only for package @crawlee/jsdom------## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)---### Features--* **jsdom,linkedom:** Expose document to crawler router context ([#1950](https://github.com/apify/crawlee/issues/1950)) ([4536dc2](https://github.com/apify/crawlee/commit/4536dc2900ee6d0acb562583ed8fca183df28e39))------# [3.4.0](https://github.com/apify/crawlee/compare/v3.3.3...v3.4.0) (2023-06-12)--**Note:** Version bump only for package @crawlee/jsdom------## [3.3.3](https://github.com/apify/crawlee/compare/v3.3.2...v3.3.3) (2023-05-31)--**Note:** Version bump only for package @crawlee/jsdom------## [3.3.2](https://github.com/apify/crawlee/compare/v3.3.1...v3.3.2) (2023-05-11)---### Features--* **HttpCrawler:** add `parseWithCheerio` helper to `HttpCrawler` ([#1906](https://github.com/apify/crawlee/issues/1906)) ([ff5f76f](https://github.com/apify/crawlee/commit/ff5f76f9336c47c555c28038cdc72dc650bb5065))-* **router:** allow inline router definition ([#1877](https://github.com/apify/crawlee/issues/1877)) ([2d241c9](https://github.com/apify/crawlee/commit/2d241c9f88964ebd41a181069c378b6b7b5bf262))------## [3.3.1](https://github.com/apify/crawlee/compare/v3.3.0...v3.3.1) (2023-04-11)---### Bug Fixes--* **jsdom:** add timeout to the window.load wait when `runScripts` are enabled ([806de31](https://github.com/apify/crawlee/commit/806de31222e138ef2d8e2706536a7288423be3d4))-* **jsdom:** delay closing of the window and add some polyfills ([2e81618](https://github.com/apify/crawlee/commit/2e81618afb5f3890495e3e5fcfa037eb3319edc9))-* **jsdom:** use no-op `enqueueLinks` in http crawlers when parsing fails ([fd35270](https://github.com/apify/crawlee/commit/fd35270e7da67a77eb60108e19294f0fd2016706))---### Features--* **jsdom:** add `parseWithCheerio` context helper ([c8f0796](https://github.com/apify/crawlee/commit/c8f0796aebc0dfa6e6d04740a0bb7d8ddd5b2d96))------# [3.3.0](https://github.com/apify/crawlee/compare/v3.2.2...v3.3.0) (2023-03-09)---### Bug Fixes--* ignore invalid URLs in `enqueueLinks` in browser crawlers ([#1803](https://github.com/apify/crawlee/issues/1803)) ([5ac336c](https://github.com/apify/crawlee/commit/5ac336c5b83b212fd6281659b8ceee091e259ff1))------## [3.2.2](https://github.com/apify/crawlee/compare/v3.2.1...v3.2.2) (2023-02-08)--**Note:** Version bump only for package @crawlee/jsdom------## [3.2.1](https://github.com/apify/crawlee/compare/v3.2.0...v3.2.1) (2023-02-07)--**Note:** Version bump only for package @crawlee/jsdom------# [3.2.0](https://github.com/apify/crawlee/compare/v3.1.4...v3.2.0) (2023-02-07)---### Bug Fixes--* declare missing dependency on `tslib` ([27e96c8](https://github.com/apify/crawlee/commit/27e96c80c26e7fc31809a4b518d699573cb8c662)), closes [#1747](https://github.com/apify/crawlee/issues/1747)------## [3.1.4](https://github.com/apify/crawlee/compare/v3.1.3...v3.1.4) (2022-12-14)--**Note:** Version bump only for package @crawlee/jsdom------## [3.1.3](https://github.com/apify/crawlee/compare/v3.1.2...v3.1.3) (2022-12-07)---### Features--* hideInternalConsole in JSDOMCrawler ([#1707](https://github.com/apify/crawlee/issues/1707)) ([8975f90](https://github.com/apify/crawlee/commit/8975f9088cf4dd38629c21e21061616fc1e7b003))------## 3.1.2 (2022-11-15)--**Note:** Version bump only for package @crawlee/jsdom------## 3.1.1 (2022-11-07)--**Note:** Version bump only for package @crawlee/jsdom------# 3.1.0 (2022-10-13)--**Note:** Version bump only for package @crawlee/jsdom------## [3.0.4](https://github.com/apify/crawlee/compare/v3.0.3...v3.0.4) (2022-08-22)--**Note:** Version bump only for package @crawlee/jsdomdiff --git a/packages/jsdom-crawler/README.md b/packages/jsdom-crawler/README.mddeleted file mode 100644index 818d84fa4..000000000--- a/packages/jsdom-crawler/README.md+++ /dev/null@@ -1,44 +0,0 @@-# `@crawlee/jsdom`--Provides a framework for the parallel crawling of web pages using plain HTTP requests and [jsdom](https://www.npmjs.com/package/jsdom) DOM implementation. The URLs to crawl are fed either from a static list of URLs or from a dynamic queue of URLs enabling recursive crawling of websites.--Since `JSDOMCrawler` uses raw HTTP requests to download web pages, it is very fast and efficient on data bandwidth. However, if the target website requires JavaScript to display the content, you might need to use [PuppeteerCrawler](https://crawlee.dev/js/api/puppeteer-crawler/class/PuppeteerCrawler) or [PlaywrightCrawler](https://crawlee.dev/js/api/playwright-crawler/class/PlaywrightCrawler) instead, because it loads the pages using full-featured headless Chrome browser.--`JSDOMCrawler` downloads each URL using a plain HTTP request, parses the HTML content using [JSDOM](https://www.npmjs.com/package/jsdom) and then invokes the user-provided [JSDOMCrawlerOptions.requestHandler](https://crawlee.dev/js/api/jsdom-crawler/interface/JSDOMCrawlerOptions#requestHandler) to extract page data using the `window` object.--The source URLs are represented using [Request](https://crawlee.dev/js/api/core/class/Request) objects that are fed from [RequestList](https://crawlee.dev/js/api/core/class/RequestList) or [RequestQueue](https://crawlee.dev/js/api/core/class/RequestQueue) instances provided by the [JSDOMCrawlerOptions.requestList](https://crawlee.dev/js/api/jsdom-crawler/interface/JSDOMCrawlerOptions#requestList) or [JSDOMCrawlerOptions.requestQueue](https://crawlee.dev/js/api/jsdom-crawler/interface/JSDOMCrawlerOptions#requestQueue) constructor options, respectively.--If both [JSDOMCrawlerOptions.requestList](https://crawlee.dev/js/api/jsdom-crawler/interface/JSDOMCrawlerOptions#requestList) and [JSDOMCrawlerOptions.requestQueue](https://crawlee.dev/js/api/jsdom-crawler/interface/JSDOMCrawlerOptions#requestQueue) are used, the instance first processes URLs from the [RequestList](https://crawlee.dev/js/api/core/class/RequestList) and automatically enqueues all of them to [RequestQueue](https://crawlee.dev/js/api/core/class/RequestQueue) before it starts their processing. This ensures that a single URL is not crawled multiple times.--The crawler finishes when there are no more [Request](https://crawlee.dev/js/api/core/class/Request) objects to crawl.--We can use the `preNavigationHooks` to adjust `gotOptions`:--```-preNavigationHooks: [- (crawlingContext, gotOptions) => {- // ...- },-]-```--By default, `JSDOMCrawler` only processes web pages with the `text/html` and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header), and skips pages with other content types. If you want the crawler to process other content types, use the [JSDOMCrawlerOptions.additionalMimeTypes](https://crawlee.dev/js/api/jsdom-crawler/interface/JSDOMCrawlerOptions#additionalMimeTypes) constructor option. Beware that the parsing behavior differs for HTML, XML, JSON and other types of content. For more details, see [JSDOMCrawlerOptions.requestHandler](https://crawlee.dev/js/api/jsdom-crawler/interface/JSDOMCrawlerOptions#requestHandler).--New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's `ConcurrencySystem`. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `JSDOMCrawler` constructor, or, for finer control, by injecting a pre-configured `concurrencySystem`.--## Example usage--```javascript-const crawler = new JSDOMCrawler({- async requestHandler({ request, window }) {- await Dataset.pushData({- url: request.url,- title: window.document.title,- });- },-});--await crawler.run([- 'http://crawlee.dev',-]);-```diff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.jsondeleted file mode 100644index ac0d8fdc8..000000000--- a/packages/jsdom-crawler/package.json+++ /dev/null@@ -1,61 +0,0 @@-{- "name": "@crawlee/jsdom",- "version": "4.0.0",- "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",- "engines": {- "node": ">=22.0.0"- },- "type": "module",- "exports": {- ".": "./dist/index.js",- "./package.json": "./package.json"- },- "keywords": [- "apify",- "headless",- "chrome",- "puppeteer",- "crawler",- "scraper"- ],- "author": {- "name": "Apify",- "email": "[email protected]",- "url": "https://apify.com"- },- "contributors": [- "Jan Curn <[email protected]>",- "Marek Trunkat <[email protected]>",- "Ondra Urban <[email protected]>"- ],- "license": "Apache-2.0",- "repository": {- "type": "git",- "url": "git+https://github.com/apify/crawlee"- },- "bugs": {- "url": "https://github.com/apify/crawlee/issues"- },- "homepage": "https://crawlee.dev",- "scripts": {- "build": "pnpm clean && pnpm compile && pnpm copy",- "clean": "rimraf ./dist",- "compile": "tsc -p tsconfig.build.json",- "copy": "tsx ../../scripts/copy.ts"- },- "publishConfig": {- "access": "public"- },- "dependencies": {- "@apify/timeout": "^1.0.1",- "@apify/utilities": "^3.0.1",- "@crawlee/http": "workspace:*",- "@crawlee/types": "workspace:*",- "@crawlee/utils": "workspace:*",- "@types/jsdom": "^21.1.7",- "cheerio": "^1.0.0",- "jsdom": "^26.1.0",- "tslib": "^2.8.1",- "zod": "catalog:"- }-}diff --git a/packages/jsdom-crawler/src/index.ts b/packages/jsdom-crawler/src/index.tsdeleted file mode 100644index 383b91e58..000000000--- a/packages/jsdom-crawler/src/index.ts+++ /dev/null@@ -1,3 +0,0 @@-export * from '@crawlee/http';-export * from './internals/jsdom-crawler.js';-export type { JSDOMParseResult } from './internals/jsdom-parser.js';diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsdeleted file mode 100644index 0f208d080..000000000--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ /dev/null@@ -1,261 +0,0 @@-import type {- CrawlingContext,- DOMCrawlingContext,- ErrorHandler,- GetUserDataFromRequest,- HttpCrawlerOptions,- InternalHttpHook,- RequestHandler,- RouterHandler,- RouterRoutes,- RouteSchemas,- RoutesFromSchemas,-} from '@crawlee/http';-import { DOMCrawler, HttpCrawler, Router } from '@crawlee/http';-import type { Dictionary } from '@crawlee/types';-import { parseArgument } from '@crawlee/utils/internal';-import { VirtualConsole } from 'jsdom';-import { z } from 'zod';--import type { JSDOMParseResult } from './jsdom-parser.js';-import { jsdomParser } from './jsdom-parser.js';--export type JSDOMErrorHandler<- UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- ContextExtension = Dictionary<never>,-> = ErrorHandler<CrawlingContext, JSDOMCrawlingContext<UserData, JSONData> & ContextExtension>;--export interface JSDOMCrawlerOptions<- ContextExtension = Dictionary<never>,- ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension,- UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,- StatisticStateExtension extends object = {},-> extends HttpCrawlerOptions<- JSDOMCrawlingContext<UserData, JSONData>,- ContextExtension,- ExtendedContext,- Routes,- StatisticStateExtension-> {- /**- * Download and run scripts.- */- runScripts?: boolean;- /**- * Suppress the logs from JSDOM internal console.- */- hideInternalConsole?: boolean;-}--export type JSDOMHook<- UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> = InternalHttpHook<JSDOMCrawlingContext<UserData, JSONData>>;--export interface JSDOMCrawlingContext<- UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> extends DOMCrawlingContext<JSDOMParseResult, UserData, JSONData> {}--export type JSDOMRequestHandler<- UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;--/**- * Provides a framework for the parallel crawling of web pages using plain HTTP requests and- * [jsdom](https://www.npmjs.com/package/jsdom) JSDOM implementation.- * The URLs to crawl are fed either from a static list of URLs- * or from a dynamic queue of URLs enabling recursive crawling of websites.- *- * Since `JSDOMCrawler` uses raw HTTP requests to download web pages,- * it is very fast and efficient on data bandwidth. However, if the target website requires JavaScript- * to display the content, you might need to use {@apilink PuppeteerCrawler} or {@apilink PlaywrightCrawler} instead,- * because it loads the pages using full-featured headless Chrome browser.- *- * Alternatively, you can use {@apilink JSDOMCrawlerOptions.runScripts} to run website scripts in Node.- * JSDOM does not implement all the standards, so websites can break.- *- * **Limitation**:- * This crawler does not support proxies and cookies yet (each open starts with empty cookie store), and the user agent is always set to `Chrome`.- *- * `JSDOMCrawler` downloads each URL using a plain HTTP request,- * parses the HTML content using [JSDOM](https://www.npmjs.com/package/jsdom)- * and then invokes the user-provided {@apilink JSDOMCrawlerOptions.requestHandler} to extract page data- * using the `window` object.- *- * The source URLs are represented using {@apilink Request} objects that are fed from the- * {@apilink IRequestManager|request manager} provided via the {@apilink JSDOMCrawlerOptions.requestManager|`requestManager`}- * constructor option (a {@apilink RequestQueue} is itself a request manager). To read from a read-only source such- * as a {@apilink RequestList} while still being able to enqueue new requests, combine it with a queue into a- * {@apilink RequestManagerTandem} via {@apilink IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the- * result as `requestManager`.- *- * > The {@apilink JSDOMCrawlerOptions.requestList|`requestList`} and {@apilink JSDOMCrawlerOptions.requestQueue|`requestQueue`}- * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.- *- * The crawler finishes when there are no more {@apilink Request} objects to crawl.- *- * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:- *- * ```- * preNavigationHooks: [- * (crawlingContext) => {- * // ...- * },- * ]- * ```- *- * By default, `JSDOMCrawler` only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,- * and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),- * and skips pages with other content types. If you want the crawler to process other content types,- * use the {@apilink JSDOMCrawlerOptions.additionalMimeTypes} constructor option.- * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.- * For more details, see {@apilink JSDOMCrawlerOptions.requestHandler}.- *- * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's- * {@apilink ConcurrencySystem}.- * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the- * `JSDOMCrawler` constructor, or, for finer control, by injecting a pre-configured- * {@apilink ConcurrencySystem|`concurrencySystem`}.- *- * **Example usage:**- *- * ```javascript- * const crawler = new JSDOMCrawler({- * async requestHandler({ request, window }) {- * await Dataset.pushData({- * url: request.url,- * title: window.document.title,- * });- * },- * });- *- * await crawler.run([- * 'http://crawlee.dev',- * ]);- * ```- * @category Crawlers- */-export class JSDOMCrawler<- ContextExtension = Dictionary<never>,- ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension,- Routes extends Record<keyof Routes, Dictionary> = Record<- string,- GetUserDataFromRequest<JSDOMCrawlingContext['request']>- >,- StatisticStateExtension extends object = {},-> extends DOMCrawler<JSDOMParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {- /**- * @internal- */- protected static override optionsShape = {- ...HttpCrawler.optionsShape,- runScripts: z.boolean().optional(),- hideInternalConsole: z.boolean().optional(),- };-- /** @internal */- protected static override optionsSchema = z.strictObject(JSDOMCrawler.optionsShape);-- #hideInternalConsole: boolean;- #virtualConsole: VirtualConsole | null = null;-- constructor(- options: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension> = {},- ) {- const {- runScripts = false,- hideInternalConsole = false,- contextPipelineBuilder,- ...httpOptions- } = parseArgument(options, JSDOMCrawler.optionsSchema, 'JSDOMCrawlerOptions');-- super({- ...httpOptions,- contextPipelineBuilder,- parser: jsdomParser({- runScripts,- virtualConsole: () => this.getVirtualConsole(),- log: () => this.log,- }),- });-- this.#hideInternalConsole = hideInternalConsole;- }-- /**- * Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.- *- * If the `hideInternalConsole` option is set to `true`, the messages aren't logged to the console by default,- * but the virtual console can still be listened to.- *- * **Example usage:**- * ```javascript- * const console = crawler.getVirtualConsole();- * console.on('error', (e) => {- * log.error(e);- * });- * ```- */- getVirtualConsole() {- if (this.#virtualConsole) {- return this.#virtualConsole;- }-- this.#virtualConsole = new VirtualConsole();-- if (!this.#hideInternalConsole) {- this.#virtualConsole.sendTo(console, { omitJSDOMErrors: true });- }-- this.#virtualConsole.on('jsdomError', this.jsdomErrorHandler);-- return this.#virtualConsole;- }-- private readonly jsdomErrorHandler = (error: Error) => this.log.debug('JSDOM error from console', { error });-}--/**- * Creates new {@apilink Router} instance that works based on request labels.- * This instance can then serve as a `requestHandler` of your {@apilink JSDOMCrawler}.- * Defaults to the {@apilink JSDOMCrawlingContext}.- *- * > Serves as a shortcut for using `Router.create<JSDOMCrawlingContext>()`.- *- * ```ts- * import { JSDOMCrawler, createJSDOMRouter } from 'crawlee';- *- * const router = createJSDOMRouter();- * router.addHandler('label-a', async (ctx) => {- * ctx.log.info('...');- * });- * router.addDefaultHandler(async (ctx) => {- * ctx.log.info('...');- * });- *- * const crawler = new JSDOMCrawler({- * requestHandler: router,- * });- * await crawler.run();- * ```- */-export function createJSDOMRouter<- Context extends JSDOMCrawlingContext = JSDOMCrawlingContext,- Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,->(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;-export function createJSDOMRouter<- Context extends JSDOMCrawlingContext = JSDOMCrawlingContext,- UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,->(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;-export function createJSDOMRouter<- Context extends JSDOMCrawlingContext = JSDOMCrawlingContext,- const Schemas extends RouteSchemas = RouteSchemas,->(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;-export function createJSDOMRouter(routesOrSchemas?: any): any {- return Router.create(routesOrSchemas);-}diff --git a/packages/jsdom-crawler/src/internals/jsdom-parser.ts b/packages/jsdom-crawler/src/internals/jsdom-parser.tsdeleted file mode 100644index e96eb48b7..000000000--- a/packages/jsdom-crawler/src/internals/jsdom-parser.ts+++ /dev/null@@ -1,138 +0,0 @@-import type { DOMParser, InternalHttpCrawlingContext } from '@crawlee/http';-import type { CrawleeLogger } from '@crawlee/types';-import { tryAbsoluteURL } from '@crawlee/utils/internal';-import type { DOMWindow, VirtualConsole } from 'jsdom';-import { JSDOM, ResourceLoader } from 'jsdom';--import { addTimeoutToPromise } from '@apify/timeout';--const resources = new ResourceLoader({- // Copy from /packages/browser-pool/src/abstract-classes/browser-plugin.ts:17- // in order not to include the entire package here- userAgent:- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',-});--export interface JSDOMParseResult {- window: DOMWindow;- document: Document;- body: string;-}--export interface JsdomParserOptions {- /**- * Download and run scripts.- */- runScripts?: boolean;-- /**- * Resolves the `VirtualConsole` to hand to JSDOM. Called for every parse, so a crawler can create its console- * lazily. Defaults to JSDOM's own console handling.- */- virtualConsole?: () => VirtualConsole;-- /**- * Resolves the logger for JSDOM diagnostics. Called only when there is something to log, so a crawler can hand- * over its own logger.- */- log?: () => CrawleeLogger;-}--/**- * A {@apilink DOMParser} backed by [jsdom](https://www.npmjs.com/package/jsdom). Pass it to a- * {@apilink DOMCrawler} to get the crawling context {@apilink JSDOMCrawler} provides.- */-export function jsdomParser(options: JsdomParserOptions = {}): DOMParser<JSDOMParseResult> {- const { runScripts = false, virtualConsole, log } = options;-- return {- placeholderMembers: { window: true, document: true, body: true },- mutable: runScripts,- async parse(context: InternalHttpCrawlingContext) {- const isXml = context.contentType.type.includes('xml');-- // TODO handle non-string- const { window } = new JSDOM(context.body.toString(), {- url: context.response.url,- contentType: isXml ? 'text/xml' : 'text/html',- runScripts: runScripts ? 'dangerously' : undefined,- resources,- virtualConsole: virtualConsole?.(),- pretendToBeVisual: true,- });-- // add some stubs in place of missing API so processing won't fail- Object.defineProperty(window, 'matchMedia', {- writable: true,- value: (query: unknown): any => ({- matches: false,- media: query,- onchange: null,- addListener: () => {},- removeListener: () => {},- addEventListener: () => {},- removeEventListener: () => {},- dispatchEvent: () => {},- }),- });- window.document.createRange = () => {- const range = new window.Range();- range.getBoundingClientRect = () => ({}) as any;- range.getClientRects = () => ({ item: () => null as any, length: 0 }) as any;- return range;- };-- if (runScripts) {- try {- await addTimeoutToPromise(- async () => {- return new Promise<void>((resolve) => {- window.addEventListener(- 'load',- () => {- resolve();- },- false,- );- }).catch();- },- 10_000,- 'Window.load event not fired after 10 seconds.',- ).catch();- } catch (e) {- log?.().debug((e as Error).message);- }- }-- return {- window,- get body() {- return window.document.documentElement.outerHTML;- },- get document() {- return window.document;- },- };- },- extractLinks: ({ window }, selector, baseUrl) => extractUrlsFromWindow(window, selector, baseUrl),- select: ({ document }, selector) => document.querySelectorAll(selector),- cleanup: ({ window }) => window.close(),- };-}--/**- * Extracts URLs from a given Window object.- * @ignore- */-function extractUrlsFromWindow(window: DOMWindow, selector: string, baseUrl: string): string[] {- return Array.from(window.document.querySelectorAll(selector))- .map((e: any) => e.href)- .filter((href) => href !== undefined && href !== '')- .map((href: string | undefined) => {- if (href === undefined) {- return undefined;- }- return tryAbsoluteURL(href, baseUrl);- })- .filter((href) => href !== undefined && href !== '') as string[];-}diff --git a/packages/jsdom-crawler/tsconfig.build.json b/packages/jsdom-crawler/tsconfig.build.jsondeleted file mode 100644index 5f63b6d3d..000000000--- a/packages/jsdom-crawler/tsconfig.build.json+++ /dev/null@@ -1,8 +0,0 @@-{- "extends": "../../tsconfig.build.json",- "compilerOptions": {- "outDir": "./dist",- "rootDir": "./src"- },- "include": ["src/**/*"]-}diff --git a/packages/jsdom-crawler/tsconfig.json b/packages/jsdom-crawler/tsconfig.jsondeleted file mode 100644index 66bb87a91..000000000--- a/packages/jsdom-crawler/tsconfig.json+++ /dev/null@@ -1,4 +0,0 @@-{- "extends": "../../tsconfig.json",- "include": ["src/**/*"]-}diff --git a/packages/linkedom-crawler/.npmignore b/packages/linkedom-crawler/.npmignoredeleted file mode 100644index ada367d72..000000000--- a/packages/linkedom-crawler/.npmignore+++ /dev/null@@ -1,6 +0,0 @@-node_modules-src-test-coverage-apify_storage-tsconfig.*diff --git a/packages/linkedom-crawler/CHANGELOG.md b/packages/linkedom-crawler/CHANGELOG.mddeleted file mode 100644index 7c39881a7..000000000--- a/packages/linkedom-crawler/CHANGELOG.md+++ /dev/null@@ -1,530 +0,0 @@-# Change Log--All notable changes to this project will be documented in this file.-See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.--## [3.18.1](https://github.com/apify/crawlee/compare/v3.18.0...v3.18.1) (2026-08-12)--### Bug Fixes--* respect `maxCrawlDepth` in `JSDOM/LinkeDOM` context `enqueueLinks` ([#4021](https://github.com/apify/crawlee/issues/4021)) ([4d34aea](https://github.com/apify/crawlee/commit/4d34aeab6d78d1bd0bbe27418daf621f28a3524c)), closes [#3927](https://github.com/apify/crawlee/issues/3927) [#4020](https://github.com/apify/crawlee/issues/4020)---# [3.18.0](https://github.com/apify/crawlee/compare/v3.17.0...v3.18.0) (2026-08-04)--### Bug Fixes--* **core:** ignore explicitly undefined baseUrl in enqueueLinks helpers ([#3924](https://github.com/apify/crawlee/issues/3924)) ([a58a9c1](https://github.com/apify/crawlee/commit/a58a9c1417c34c2a45a10f3855c4bee721a480ae))-* declare missing dependencies ([#3817](https://github.com/apify/crawlee/issues/3817)) ([5a9a661](https://github.com/apify/crawlee/commit/5a9a661401a3aa895738c79a8310d9cc18e28826))--### Features--* opt-in schema validation of request userData per router label ([#3851](https://github.com/apify/crawlee/issues/3851)) ([c7347f9](https://github.com/apify/crawlee/commit/c7347f94f72980b7c424fb4fc7dc8f990f360d70)), closes [#3747](https://github.com/apify/crawlee/issues/3747) [#3748](https://github.com/apify/crawlee/issues/3748) [#3747](https://github.com/apify/crawlee/issues/3747) [#3082](https://github.com/apify/crawlee/issues/3082)-* type-safe router labels via per-label userData map ([#3747](https://github.com/apify/crawlee/issues/3747)) ([1124aca](https://github.com/apify/crawlee/commit/1124aca3c5eb527c097decd7263a6816c411369e)), closes [#3082](https://github.com/apify/crawlee/issues/3082)---# [3.17.0](https://github.com/apify/crawlee/compare/v3.16.0...v3.17.0) (2026-06-04)--**Note:** Version bump only for package @crawlee/linkedom------# [3.16.0](https://github.com/apify/crawlee/compare/v3.15.3...v3.16.0) (2026-02-06)---### Performance Improvements--* drop `tsbuildinfo` from published packages ([#3243](https://github.com/apify/crawlee/issues/3243)) ([3450f27](https://github.com/apify/crawlee/commit/3450f27880afb9e9d857a54d9212b54c397aed91)), closes [#3239](https://github.com/apify/crawlee/issues/3239)------## [3.15.3](https://github.com/apify/crawlee/compare/v3.15.2...v3.15.3) (2025-11-10)--**Note:** Version bump only for package @crawlee/linkedom------## [3.15.2](https://github.com/apify/crawlee/compare/v3.15.1...v3.15.2) (2025-10-23)--**Note:** Version bump only for package @crawlee/linkedom------## [3.15.1](https://github.com/apify/crawlee/compare/v3.15.0...v3.15.1) (2025-09-26)--**Note:** Version bump only for package @crawlee/linkedom------# [3.15.0](https://github.com/apify/crawlee/compare/v3.14.1...v3.15.0) (2025-09-17)--**Note:** Version bump only for package @crawlee/linkedom------## [3.14.1](https://github.com/apify/crawlee/compare/v3.14.0...v3.14.1) (2025-08-05)--**Note:** Version bump only for package @crawlee/linkedom------# [3.14.0](https://github.com/apify/crawlee/compare/v3.13.10...v3.14.0) (2025-07-25)---### Features--* add `maxCrawlDepth` crawler option ([#3045](https://github.com/apify/crawlee/issues/3045)) ([0090df9](https://github.com/apify/crawlee/commit/0090df93a12df9918d016cf2f1378f1f7d40557d)), closes [#2633](https://github.com/apify/crawlee/issues/2633)------## [3.13.10](https://github.com/apify/crawlee/compare/v3.13.9...v3.13.10) (2025-07-09)--**Note:** Version bump only for package @crawlee/linkedom------## [3.13.9](https://github.com/apify/crawlee/compare/v3.13.8...v3.13.9) (2025-06-27)--**Note:** Version bump only for package @crawlee/linkedom------## [3.13.8](https://github.com/apify/crawlee/compare/v3.13.7...v3.13.8) (2025-06-16)---### Bug Fixes--* Do not enqueue more links than what the crawler is capable of processing ([#2990](https://github.com/apify/crawlee/issues/2990)) ([ea094c8](https://github.com/apify/crawlee/commit/ea094c819232e0b30bc550270836d10506eb9454)), closes [#2728](https://github.com/apify/crawlee/issues/2728)------## [3.13.7](https://github.com/apify/crawlee/compare/v3.13.6...v3.13.7) (2025-06-06)--**Note:** Version bump only for package @crawlee/linkedom------## [3.13.6](https://github.com/apify/crawlee/compare/v3.13.5...v3.13.6) (2025-06-05)--**Note:** Version bump only for package @crawlee/linkedom------## [3.13.5](https://github.com/apify/crawlee/compare/v3.13.4...v3.13.5) (2025-05-20)--**Note:** Version bump only for package @crawlee/linkedom------## [3.13.4](https://github.com/apify/crawlee/compare/v3.13.3...v3.13.4) (2025-05-14)--**Note:** Version bump only for package @crawlee/linkedom------## [3.13.3](https://github.com/apify/crawlee/compare/v3.13.2...v3.13.3) (2025-05-05)--**Note:** Version bump only for package @crawlee/linkedom------## [3.13.2](https://github.com/apify/crawlee/compare/v3.13.1...v3.13.2) (2025-04-08)---### Features--* add `onSkippedRequest` option ([#2916](https://github.com/apify/crawlee/issues/2916)) ([764f992](https://github.com/apify/crawlee/commit/764f99203627b6a44d2ee90d623b8b0e6ecbffb5)), closes [#2910](https://github.com/apify/crawlee/issues/2910)------## [3.13.1](https://github.com/apify/crawlee/compare/v3.13.0...v3.13.1) (2025-04-07)---### Bug Fixes--* rename `RobotsFile` to `RobotsTxtFile` ([#2913](https://github.com/apify/crawlee/issues/2913)) ([3160f71](https://github.com/apify/crawlee/commit/3160f717e865326476d78089d778cbc7d35aa58d)), closes [#2910](https://github.com/apify/crawlee/issues/2910)---### Features--* add `respectRobotsTxtFile` crawler option ([#2910](https://github.com/apify/crawlee/issues/2910)) ([0eabed1](https://github.com/apify/crawlee/commit/0eabed1f13070d902c2c67b340621830a7f64464))------# [3.13.0](https://github.com/apify/crawlee/compare/v3.12.2...v3.13.0) (2025-03-04)--**Note:** Version bump only for package @crawlee/linkedom------## [3.12.2](https://github.com/apify/crawlee/compare/v3.12.1...v3.12.2) (2025-01-27)--**Note:** Version bump only for package @crawlee/linkedom------## [3.12.1](https://github.com/apify/crawlee/compare/v3.12.0...v3.12.1) (2024-12-04)--**Note:** Version bump only for package @crawlee/linkedom------# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)--**Note:** Version bump only for package @crawlee/linkedom------## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)--**Note:** Version bump only for package @crawlee/linkedom------## [3.11.4](https://github.com/apify/crawlee/compare/v3.11.3...v3.11.4) (2024-09-23)--**Note:** Version bump only for package @crawlee/linkedom------## [3.11.3](https://github.com/apify/crawlee/compare/v3.11.2...v3.11.3) (2024-09-03)--**Note:** Version bump only for package @crawlee/linkedom------## [3.11.2](https://github.com/apify/crawlee/compare/v3.11.1...v3.11.2) (2024-08-28)--**Note:** Version bump only for package @crawlee/linkedom------## [3.11.1](https://github.com/apify/crawlee/compare/v3.11.0...v3.11.1) (2024-07-24)--**Note:** Version bump only for package @crawlee/linkedom------# [3.11.0](https://github.com/apify/crawlee/compare/v3.10.5...v3.11.0) (2024-07-09)--**Note:** Version bump only for package @crawlee/linkedom------## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)--**Note:** Version bump only for package @crawlee/linkedom------## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)--**Note:** Version bump only for package @crawlee/linkedom------## [3.10.3](https://github.com/apify/crawlee/compare/v3.10.2...v3.10.3) (2024-06-07)---### Features--* add `waitForSelector` context helper + `parseWithCheerio` in adaptive crawler ([#2522](https://github.com/apify/crawlee/issues/2522)) ([6f88e73](https://github.com/apify/crawlee/commit/6f88e738d43ab4774dc4ef3f78775a5d88728e0d))------## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)--**Note:** Version bump only for package @crawlee/linkedom------## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)--**Note:** Version bump only for package @crawlee/linkedom------# [3.10.0](https://github.com/apify/crawlee/compare/v3.9.2...v3.10.0) (2024-05-16)--**Note:** Version bump only for package @crawlee/linkedom------## [3.9.2](https://github.com/apify/crawlee/compare/v3.9.1...v3.9.2) (2024-04-17)--**Note:** Version bump only for package @crawlee/linkedom------## [3.9.1](https://github.com/apify/crawlee/compare/v3.9.0...v3.9.1) (2024-04-11)--**Note:** Version bump only for package @crawlee/linkedom------# [3.9.0](https://github.com/apify/crawlee/compare/v3.8.2...v3.9.0) (2024-04-10)--**Note:** Version bump only for package @crawlee/linkedom------## [3.8.2](https://github.com/apify/crawlee/compare/v3.8.1...v3.8.2) (2024-03-21)--**Note:** Version bump only for package @crawlee/linkedom------## [3.8.1](https://github.com/apify/crawlee/compare/v3.8.0...v3.8.1) (2024-02-22)--**Note:** Version bump only for package @crawlee/linkedom------# [3.8.0](https://github.com/apify/crawlee/compare/v3.7.3...v3.8.0) (2024-02-21)--**Note:** Version bump only for package @crawlee/linkedom------## [3.7.3](https://github.com/apify/crawlee/compare/v3.7.2...v3.7.3) (2024-01-30)--**Note:** Version bump only for package @crawlee/linkedom------## [3.7.2](https://github.com/apify/crawlee/compare/v3.7.1...v3.7.2) (2024-01-09)--**Note:** Version bump only for package @crawlee/linkedom------## [3.7.1](https://github.com/apify/crawlee/compare/v3.7.0...v3.7.1) (2024-01-02)--**Note:** Version bump only for package @crawlee/linkedom------# [3.7.0](https://github.com/apify/crawlee/compare/v3.6.2...v3.7.0) (2023-12-21)--**Note:** Version bump only for package @crawlee/linkedom------## [3.6.2](https://github.com/apify/crawlee/compare/v3.6.1...v3.6.2) (2023-11-26)--**Note:** Version bump only for package @crawlee/linkedom------## [3.6.1](https://github.com/apify/crawlee/compare/v3.6.0...v3.6.1) (2023-11-15)--**Note:** Version bump only for package @crawlee/linkedom------# [3.6.0](https://github.com/apify/crawlee/compare/v3.5.8...v3.6.0) (2023-11-15)---### Features--* got-scraping v4 ([#2110](https://github.com/apify/crawlee/issues/2110)) ([2f05ed2](https://github.com/apify/crawlee/commit/2f05ed22b203f688095300400bb0e6d03a03283c))------## [3.5.8](https://github.com/apify/crawlee/compare/v3.5.7...v3.5.8) (2023-10-17)--**Note:** Version bump only for package @crawlee/linkedom------## [3.5.7](https://github.com/apify/crawlee/compare/v3.5.6...v3.5.7) (2023-10-05)--**Note:** Version bump only for package @crawlee/linkedom------## [3.5.6](https://github.com/apify/crawlee/compare/v3.5.5...v3.5.6) (2023-10-04)--**Note:** Version bump only for package @crawlee/linkedom------## [3.5.5](https://github.com/apify/crawlee/compare/v3.5.4...v3.5.5) (2023-10-02)---### Features--* Request Queue v2 ([#1975](https://github.com/apify/crawlee/issues/1975)) ([70a77ee](https://github.com/apify/crawlee/commit/70a77ee15f984e9ae67cd584fc58ace7e55346db)), closes [#1365](https://github.com/apify/crawlee/issues/1365)------## [3.5.4](https://github.com/apify/crawlee/compare/v3.5.3...v3.5.4) (2023-09-11)--**Note:** Version bump only for package @crawlee/linkedom------## [3.5.3](https://github.com/apify/crawlee/compare/v3.5.2...v3.5.3) (2023-08-31)---### Bug Fixes--* pin all internal dependencies ([#2041](https://github.com/apify/crawlee/issues/2041)) ([d6f2b17](https://github.com/apify/crawlee/commit/d6f2b172d4a6776137c7893ca798d5b4a9408e79)), closes [#2040](https://github.com/apify/crawlee/issues/2040)------## [3.5.2](https://github.com/apify/crawlee/compare/v3.5.1...v3.5.2) (2023-08-21)--**Note:** Version bump only for package @crawlee/linkedom------## [3.5.1](https://github.com/apify/crawlee/compare/v3.5.0...v3.5.1) (2023-08-16)--**Note:** Version bump only for package @crawlee/linkedom------# [3.5.0](https://github.com/apify/crawlee/compare/v3.4.2...v3.5.0) (2023-07-31)--**Note:** Version bump only for package @crawlee/linkedom------## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)--**Note:** Version bump only for package @crawlee/linkedom------## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)---### Features--* **jsdom,linkedom:** Expose document to crawler router context ([#1950](https://github.com/apify/crawlee/issues/1950)) ([4536dc2](https://github.com/apify/crawlee/commit/4536dc2900ee6d0acb562583ed8fca183df28e39))------# [3.4.0](https://github.com/apify/crawlee/compare/v3.3.3...v3.4.0) (2023-06-12)---### Features--* add LinkeDOMCrawler ([#1907](https://github.com/apify/crawlee/issues/1907)) ([1c69560](https://github.com/apify/crawlee/commit/1c69560fe7ef45097e6be1037b79a84eb9a06337)), closes [/github.com/apify/crawlee/pull/1890#issuecomment-1533271694](https://github.com//github.com/apify/crawlee/pull/1890/issues/issuecomment-1533271694)diff --git a/packages/linkedom-crawler/README.md b/packages/linkedom-crawler/README.mddeleted file mode 100644index a75a6cd0c..000000000--- a/packages/linkedom-crawler/README.md+++ /dev/null@@ -1,46 +0,0 @@-# `@crawlee/linkedom`--<!-- Note for editors: This README should be kept consistent with JSDOMCrawler one -->--Provides a framework for the parallel crawling of web pages using plain HTTP requests and [linkedom](https://www.npmjs.com/package/linkedom) DOM implementation. The URLs to crawl are fed either from a static list of URLs or from a dynamic queue of URLs enabling recursive crawling of websites.--Since `LinkeDOMCrawler` uses raw HTTP requests to download web pages, it is very fast and efficient on data bandwidth. However, if the target website requires JavaScript to display the content, you might need to use [PuppeteerCrawler](https://crawlee.dev/js/api/puppeteer-crawler/class/PuppeteerCrawler) or [PlaywrightCrawler](https://crawlee.dev/js/api/playwright-crawler/class/PlaywrightCrawler) instead, because it loads the pages using full-featured headless Chrome browser.--`LinkeDOMCrawler` downloads each URL using a plain HTTP request, parses the HTML content using [LinkeDOM](https://www.npmjs.com/package/linkedom) and then invokes the user-provided [LinkeDOMCrawlerOptions.requestHandler](https://crawlee.dev/js/api/linkedom-crawler/interface/LinkeDOMCrawlerOptions#requestHandler) to extract page data using the `window` object.--The source URLs are represented using [Request](https://crawlee.dev/js/api/core/class/Request) objects that are fed from [RequestList](https://crawlee.dev/js/api/core/class/RequestList) or [RequestQueue](https://crawlee.dev/js/api/core/class/RequestQueue) instances provided by the [LinkeDOMCrawlerOptions.requestList](https://crawlee.dev/js/api/linkedom-crawler/interface/LinkeDOMCrawlerOptions#requestList) or [LinkeDOMCrawlerOptions.requestQueue](https://crawlee.dev/js/api/linkedom-crawler/interface/LinkeDOMCrawlerOptions#requestQueue) constructor options, respectively.--If both [LinkeDOMCrawlerOptions.requestList](https://crawlee.dev/js/api/linkedom-crawler/interface/LinkeDOMCrawlerOptions#requestList) and [LinkeDOMCrawlerOptions.requestQueue](https://crawlee.dev/js/api/linkedom-crawler/interface/LinkeDOMCrawlerOptions#requestQueue) are used, the instance first processes URLs from the [RequestList](https://crawlee.dev/js/api/core/class/RequestList) and automatically enqueues all of them to [RequestQueue](https://crawlee.dev/js/api/core/class/RequestQueue) before it starts their processing. This ensures that a single URL is not crawled multiple times.--The crawler finishes when there are no more [Request](https://crawlee.dev/js/api/core/class/Request) objects to crawl.--We can use the `preNavigationHooks` to adjust `gotOptions`:--```-preNavigationHooks: [- (crawlingContext, gotOptions) => {- // ...- },-]-```--By default, `LinkeDOMCrawler` only processes web pages with the `text/html` and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header), and skips pages with other content types. If you want the crawler to process other content types, use the [LinkeDOMCrawlerOptions.additionalMimeTypes](https://crawlee.dev/js/api/linkedom-crawler/interface/LinkeDOMCrawlerOptions#additionalMimeTypes) constructor option. Beware that the parsing behavior differs for HTML, XML, JSON and other types of content. For more details, see [LinkeDOMCrawlerOptions.requestHandler](https://crawlee.dev/js/api/linkedom-crawler/interface/LinkeDOMCrawlerOptions#requestHandler).--New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's `ConcurrencySystem`. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `LinkeDOMCrawler` constructor, or, for finer control, by injecting a pre-configured `concurrencySystem`.--## Example usage--```javascript-const crawler = new LinkeDOMCrawler({- async requestHandler({ request, window }) {- await Dataset.pushData({- url: request.url,- title: window.document.title,- });- },-});--await crawler.run([- 'http://crawlee.dev',-]);-```diff --git a/packages/linkedom-crawler/package.json b/packages/linkedom-crawler/package.jsondeleted file mode 100644index 4b7e3d04f..000000000--- a/packages/linkedom-crawler/package.json+++ /dev/null@@ -1,59 +0,0 @@-{- "name": "@crawlee/linkedom",- "version": "4.0.0",- "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",- "engines": {- "node": ">=22.0.0"- },- "type": "module",- "exports": {- ".": "./dist/index.js",- "./package.json": "./package.json"- },- "keywords": [- "apify",- "headless",- "chrome",- "puppeteer",- "crawler",- "scraper"- ],- "author": {- "name": "Apify",- "email": "[email protected]",- "url": "https://apify.com"- },- "contributors": [- "Jan Curn <[email protected]>",- "Marek Trunkat <[email protected]>",- "Ondra Urban <[email protected]>"- ],- "license": "Apache-2.0",- "repository": {- "type": "git",- "url": "git+https://github.com/apify/crawlee"- },- "bugs": {- "url": "https://github.com/apify/crawlee/issues"- },- "homepage": "https://crawlee.dev",- "scripts": {- "build": "pnpm clean && pnpm compile && pnpm copy",- "clean": "rimraf ./dist",- "compile": "tsc -p tsconfig.build.json",- "copy": "tsx ../../scripts/copy.ts"- },- "publishConfig": {- "access": "public"- },- "dependencies": {- "@apify/timeout": "^1.0.1",- "@apify/utilities": "^3.0.1",- "@crawlee/http": "workspace:*",- "@crawlee/types": "workspace:*",- "@crawlee/utils": "workspace:*",- "cheerio": "^1.0.0",- "linkedom": "^0.18.10",- "tslib": "^2.8.1"- }-}diff --git a/packages/linkedom-crawler/src/index.ts b/packages/linkedom-crawler/src/index.tsdeleted file mode 100644index ce2e96b04..000000000--- a/packages/linkedom-crawler/src/index.ts+++ /dev/null@@ -1,3 +0,0 @@-export * from '@crawlee/http';-export * from './internals/linkedom-crawler.js';-export type { LinkeDOMParseResult } from './internals/linkedom-parser.js';diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.tsdeleted file mode 100644index cd1b1e627..000000000--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts+++ /dev/null@@ -1,190 +0,0 @@-import type {- CrawlingContext,- DOMCrawlingContext,- ErrorHandler,- GetUserDataFromRequest,- HttpCrawlerOptions,- InternalHttpHook,- RequestHandler,- RouterHandler,- RouterRoutes,- RouteSchemas,- RoutesFromSchemas,-} from '@crawlee/http';-import { DOMCrawler, Router } from '@crawlee/http';-import type { Dictionary } from '@crawlee/types';--import type { LinkeDOMParseResult } from './linkedom-parser.js';-import { linkedomParser } from './linkedom-parser.js';--export type LinkeDOMErrorHandler<- UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- ContextExtension = Dictionary<never>,-> = ErrorHandler<CrawlingContext, LinkeDOMCrawlingContext<UserData, JSONData> & ContextExtension>;--export interface LinkeDOMCrawlerOptions<- ContextExtension = Dictionary<never>,- ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension,- UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,- StatisticStateExtension extends object = {},-> extends HttpCrawlerOptions<- LinkeDOMCrawlingContext<UserData, JSONData>,- ContextExtension,- ExtendedContext,- Routes,- StatisticStateExtension-> {}--export type LinkeDOMHook<- UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> = InternalHttpHook<LinkeDOMCrawlingContext<UserData, JSONData>>;--export interface LinkeDOMCrawlingContext<- UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> extends DOMCrawlingContext<LinkeDOMParseResult, UserData, JSONData> {}--export type LinkeDOMRequestHandler<- UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> = RequestHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;--/**- * Provides a framework for the parallel crawling of web pages using plain HTTP requests and- * [linkedom](https://www.npmjs.com/package/linkedom) LinkeDOM implementation.- * The URLs to crawl are fed either from a static list of URLs- * or from a dynamic queue of URLs enabling recursive crawling of websites.- *- * Since `LinkeDOMCrawler` uses raw HTTP requests to download web pages,- * it is very fast and efficient on data bandwidth. However, if the target website requires JavaScript- * to display the content, you might need to use {@apilink PuppeteerCrawler} or {@apilink PlaywrightCrawler} instead,- * because it loads the pages using full-featured headless Chrome browser.- *- * **Limitation**:- * This crawler does not support proxies and cookies yet (each open starts with empty cookie store), and the user agent is always set to `Chrome`.- *- * `LinkeDOMCrawler` downloads each URL using a plain HTTP request,- * parses the HTML content using [LinkeDOM](https://www.npmjs.com/package/linkedom)- * and then invokes the user-provided {@apilink LinkeDOMCrawlerOptions.requestHandler} to extract page data- * using the `window` object.- *- * The source URLs are represented using {@apilink Request} objects that are fed from the- * {@apilink IRequestManager|request manager} provided via the {@apilink LinkeDOMCrawlerOptions.requestManager|`requestManager`}- * constructor option (a {@apilink RequestQueue} is itself a request manager). To read from a read-only source such- * as a {@apilink RequestList} while still being able to enqueue new requests, combine it with a queue into a- * {@apilink RequestManagerTandem} via {@apilink IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the- * result as `requestManager`.- *- * > The {@apilink LinkeDOMCrawlerOptions.requestList|`requestList`} and {@apilink LinkeDOMCrawlerOptions.requestQueue|`requestQueue`}- * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.- *- * The crawler finishes when there are no more {@apilink Request} objects to crawl.- *- * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:- *- * ```- * preNavigationHooks: [- * (crawlingContext) => {- * // ...- * },- * ]- * ```- *- * By default, `LinkeDOMCrawler` only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,- * and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),- * and skips pages with other content types. If you want the crawler to process other content types,- * use the {@apilink LinkeDOMCrawlerOptions.additionalMimeTypes} constructor option.- * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.- * For more details, see {@apilink LinkeDOMCrawlerOptions.requestHandler}.- *- * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's- * {@apilink ConcurrencySystem}.- * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the- * `LinkeDOMCrawler` constructor, or, for finer control, by injecting a pre-configured- * {@apilink ConcurrencySystem|`concurrencySystem`}.- *- * **Example usage:**- *- * ```javascript- * const crawler = new LinkeDOMCrawler({- * async requestHandler({ request, window }) {- * await Dataset.pushData({- * url: request.url,- * title: window.document.title,- * });- * },- * });- *- * await crawler.run([- * 'http://crawlee.dev',- * ]);- * ```- * @category Crawlers- */--export class LinkeDOMCrawler<- ContextExtension = Dictionary<never>,- ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension,- Routes extends Record<keyof Routes, Dictionary> = Record<- string,- GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>- >,- StatisticStateExtension extends object = {},-> extends DOMCrawler<LinkeDOMParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {- constructor(- options: LinkeDOMCrawlerOptions<- ContextExtension,- ExtendedContext,- any,- any,- Routes,- StatisticStateExtension- > = {},- ) {- super({ ...options, parser: linkedomParser() });- }-}--/**- * Creates new {@apilink Router} instance that works based on request labels.- * This instance can then serve as a `requestHandler` of your {@apilink LinkeDOMCrawler}.- * Defaults to the {@apilink LinkeDOMCrawlingContext}.- *- * > Serves as a shortcut for using `Router.create<LinkeDOMCrawlingContext>()`.- *- * ```ts- * import { LinkeDOMCrawler, createLinkeDOMRouter } from 'crawlee';- *- * const router = createLinkeDOMRouter();- * router.addHandler('label-a', async (ctx) => {- * ctx.log.info('...');- * });- * router.addDefaultHandler(async (ctx) => {- * ctx.log.info('...');- * });- *- * const crawler = new LinkeDOMCrawler({- * requestHandler: router,- * });- * await crawler.run();- * ```- */-export function createLinkeDOMRouter<- Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext,- Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,->(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;-export function createLinkeDOMRouter<- Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext,- UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,->(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;-export function createLinkeDOMRouter<- Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext,- const Schemas extends RouteSchemas = RouteSchemas,->(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;-export function createLinkeDOMRouter(routesOrSchemas?: any): any {- return Router.create(routesOrSchemas);-}diff --git a/packages/linkedom-crawler/src/internals/linkedom-parser.ts b/packages/linkedom-crawler/src/internals/linkedom-parser.tsdeleted file mode 100644index 4bdf23c24..000000000--- a/packages/linkedom-crawler/src/internals/linkedom-parser.ts+++ /dev/null@@ -1,60 +0,0 @@-import type { DOMParser, InternalHttpCrawlingContext } from '@crawlee/http';-import { tryAbsoluteURL } from '@crawlee/utils/internal';-import { DOMParser as LinkeDOMParser } from 'linkedom/cached';--export interface LinkeDOMParseResult {- window: Window;- // Technically the document is not of type Document but of type either HTMLDocument or XMLDocument- // from linkedom/types/{html/xml}/document, depending on the content type of the response- // Using union of the real types would make writing the crawlers inconvenient,- // so we specify the type as the native Document type from lib.dom.d.ts- // even though it's not technically 100% correct- document: Document;- body: string;-}--/**- * A {@apilink DOMParser} backed by [linkedom](https://www.npmjs.com/package/linkedom). Pass it to a- * {@apilink DOMCrawler} to get the crawling context {@apilink LinkeDOMCrawler} provides.- */-export function linkedomParser(): DOMParser<LinkeDOMParseResult> {- const parser = new LinkeDOMParser();-- return {- placeholderMembers: { window: true, document: true, body: true },- parse(context: InternalHttpCrawlingContext) {- const isXml = context.contentType.type.includes('xml');- const document = parser.parseFromString(context.body.toString(), isXml ? 'text/xml' : 'text/html');-- return {- window: document.defaultView as unknown as Window,- get body() {- return document.documentElement.outerHTML;- },- get document() {- // See comment about typing in LinkeDOMParseResult definition- return document as unknown as Document;- },- };- },- extractLinks: ({ window }, selector, baseUrl) => extractUrlsFromWindow(window, selector, baseUrl),- select: ({ document }, selector) => document.querySelectorAll(selector),- };-}--/**- * Extracts URLs from a given Window object.- * @ignore- */-function extractUrlsFromWindow(window: Window, selector: string, baseUrl: string): string[] {- return Array.from(window.document.querySelectorAll(selector))- .map((e: any) => e.href)- .filter((href) => href !== undefined && href !== '')- .map((href: string | undefined) => {- if (href === undefined) {- return undefined;- }- return tryAbsoluteURL(href, baseUrl);- })- .filter((href) => href !== undefined && href !== '') as string[];-}diff --git a/packages/linkedom-crawler/tsconfig.build.json b/packages/linkedom-crawler/tsconfig.build.jsondeleted file mode 100644index 5f63b6d3d..000000000--- a/packages/linkedom-crawler/tsconfig.build.json+++ /dev/null@@ -1,8 +0,0 @@-{- "extends": "../../tsconfig.build.json",- "compilerOptions": {- "outDir": "./dist",- "rootDir": "./src"- },- "include": ["src/**/*"]-}diff --git a/packages/linkedom-crawler/tsconfig.json b/packages/linkedom-crawler/tsconfig.jsondeleted file mode 100644index 66bb87a91..000000000--- a/packages/linkedom-crawler/tsconfig.json+++ /dev/null@@ -1,4 +0,0 @@-{- "extends": "../../tsconfig.json",- "include": ["src/**/*"]-}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 991368649..97ed23e42 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -256,12 +256,6 @@ importers:'@crawlee/impit-client':specifier: workspace:*version: link:packages/impit-client- '@crawlee/jsdom':- specifier: workspace:*- version: link:packages/jsdom-crawler- '@crawlee/linkedom':- specifier: workspace:*- version: link:packages/linkedom-crawler'@crawlee/otel':specifier: workspace:*version: link:packages/otel@@ -816,12 +810,6 @@ importers:'@crawlee/impit-client':specifier: workspace:*version: link:../impit-client- '@crawlee/jsdom':- specifier: workspace:*- version: link:../jsdom-crawler- '@crawlee/linkedom':- specifier: workspace:*- version: link:../linkedom-crawler'@crawlee/playwright':specifier: workspace:*version: link:../playwright-crawler@@ -943,66 +931,6 @@ importers:specifier: ^6.0.0version: 6.0.1- packages/jsdom-crawler:- dependencies:- '@apify/timeout':- specifier: ^1.0.1- version: 1.0.1- '@apify/utilities':- specifier: ^3.0.1- version: 3.0.1- '@crawlee/http':- specifier: workspace:*- version: link:../http-crawler- '@crawlee/types':- specifier: workspace:*- version: link:../types- '@crawlee/utils':- specifier: workspace:*- version: link:../utils- '@types/jsdom':- specifier: ^21.1.7- version: 21.1.7- cheerio:- specifier: ^1.0.0- version: 1.2.0- jsdom:- specifier: ^26.1.0- version: 26.1.0([email protected])([email protected])- tslib:- specifier: ^2.8.1- version: 2.8.1- zod:- specifier: 'catalog:'- version: 4.5.4-- packages/linkedom-crawler:- dependencies:- '@apify/timeout':- specifier: ^1.0.1- version: 1.0.1- '@apify/utilities':- specifier: ^3.0.1- version: 3.0.1- '@crawlee/http':- specifier: workspace:*- version: link:../http-crawler- '@crawlee/types':- specifier: workspace:*- version: link:../types- '@crawlee/utils':- specifier: workspace:*- version: link:../utils- cheerio:- specifier: ^1.0.0- version: 1.2.0- linkedom:- specifier: ^0.18.10- version: 0.18.12- tslib:- specifier: ^2.8.1- version: 2.8.1-packages/otel:dependencies:'@opentelemetry/instrumentation':@@ -5272,9 +5200,6 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==}- '@types/[email protected]':- resolution: {integrity: sha512-yOriVnggzrnQ3a9OKOCxaVuSug3w3/SbOj5i7VwXWZEyUNl3bLF9V3MfxGbZKuwqJOQyRfqXyROBB1CoZLFWzA==}-'@types/[email protected]':resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==}@@ -5399,9 +5324,6 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-HsJ+z3QuETzP3cswwtzt2vEIiHBk/dCcHGhbmG5X3ecnwFD/lPrMpliGXxSCg03L9AhrdwA4Oz/qfspkDW+xGQ==}- '@types/[email protected]':- resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==}-'@types/[email protected]':resolution: {integrity: sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==}@@ -7029,9 +6951,6 @@ packages:resolution: {integrity: sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==}engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0, npm: '>=7.0.0'}- resolution: {integrity: sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==}-resolution: {integrity: sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==}engines: {node: '>=18'}@@ -8564,9 +8483,6 @@ packages:resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==}- resolution: {integrity: sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==}-resolution: {integrity: sha512-YXxSlJBZTP7RS3tWnQw74ooKa6L9b9i9QYXY21eUEvhZ3u9XLfv6OnFsQq6RxkhHygsaUMvYsZRV5rU/OVNZxw==}engines: {node: '>=12'}@@ -9510,15 +9426,6 @@ packages:resolution: {integrity: sha512-cNOjgCnLB+FnvWWtyRTzmB3POJ+cXxTA81LoW7u8JdmhfXzriropYwpjShnz1QLLWsQwY7nIxoDmcPTwphDK9w==}engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}- resolution: {integrity: sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q==}- engines: {node: '>=16'}- peerDependencies:- canvas: '>= 2'- peerDependenciesMeta:- canvas:- optional: true-resolution: {integrity: sha512-ONTm2jCMAVZjgQa/Fy1kScXsuOoF5NPTsoFBdE1KVIZ2vAh/r9+Bqo+0jINCBYnavTPQZz38QzFTme79ENoN3Q==}@@ -12724,9 +12631,6 @@ packages:engines: {node: '>=0.8.0'}hasBin: true- resolution: {integrity: sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==}-resolution: {integrity: sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==}engines: {node: '>=18'}@@ -13722,6 +13626,7 @@ snapshots:'@csstools/css-parser-algorithms': 3.0.5(@csstools/[email protected])'@csstools/css-tokenizer': 3.0.4lru-cache: 10.4.3+ optional: true'@babel/[email protected]':dependencies:@@ -18302,12 +18207,6 @@ snapshots:dependencies:'@types/istanbul-lib-report': 3.0.3- '@types/[email protected]':- dependencies:- '@types/node': 24.12.2- '@types/tough-cookie': 4.0.5- parse5: 7.3.0-'@types/[email protected]': {}'@types/[email protected]': {}@@ -18447,8 +18346,6 @@ snapshots:dependencies:'@types/node': 24.12.2- '@types/[email protected]': {}-'@types/[email protected]': {}'@types/[email protected]': {}@@ -20324,12 +20221,11 @@ snapshots:dependencies:css-tree: 2.2.1- [email protected]: {}-dependencies:'@asamuzakjp/css-color': 3.2.0rrweb-cssom: 0.8.0+ optional: true@@ -20531,6 +20427,7 @@ snapshots:dependencies:whatwg-mimetype: 4.0.0whatwg-url: 14.2.0+ optional: truedependencies:@@ -20595,7 +20492,8 @@ snapshots:- [email protected]: {}+ optional: truedependencies:@@ -22437,13 +22335,12 @@ snapshots:dependencies:whatwg-encoding: 3.1.1+ optional: true- [email protected]: {}-dependencies:camel-case: 4.1.2@@ -22912,7 +22809,8 @@ snapshots:dependencies:isobject: 3.0.1- [email protected]: {}+ optional: true@@ -23118,33 +23016,6 @@ snapshots:- utf-8-validateoptional: true- dependencies:- cssstyle: 4.6.0- data-urls: 5.0.0- decimal.js: 10.6.0- html-encoding-sniffer: 4.0.0- http-proxy-agent: 7.0.2([email protected])- https-proxy-agent: 7.0.6([email protected])- is-potential-custom-element-name: 1.0.1- nwsapi: 2.2.23- parse5: 7.3.0- rrweb-cssom: 0.8.0- saxes: 6.0.0- symbol-tree: 3.2.4- tough-cookie: 5.1.2- w3c-xmlserializer: 5.0.0- webidl-conversions: 7.0.0- whatwg-encoding: 3.1.1- whatwg-mimetype: 4.0.0- whatwg-url: 14.2.0- ws: 8.21.3([email protected])- xml-name-validator: 5.0.0- transitivePeerDependencies:- - bufferutil- - supports-color- - utf-8-validate-@@ -23449,14 +23320,6 @@ snapshots:- dependencies:- css-select: 5.2.2- cssom: 0.5.0- html-escaper: 3.0.3- htmlparser2: 10.1.0- uhyphen: 0.2.0-dependencies:uc.micro: 2.1.0@@ -24612,7 +24475,8 @@ snapshots:schema-utils: 3.3.0- [email protected]: {}+ optional: truedependencies:@@ -26668,7 +26532,8 @@ snapshots:transitivePeerDependencies:- supports-color- [email protected]: {}+ optional: truedependencies:@@ -26723,6 +26588,7 @@ snapshots:dependencies:xmlchars: 2.2.0+ optional: true@@ -27345,7 +27211,8 @@ snapshots:'@swc/counter': 0.1.3- [email protected]: {}+ optional: true@@ -27446,13 +27313,15 @@ snapshots:- [email protected]: {}+ optional: truedependencies:tldts-core: 6.1.86+ optional: truedependencies:@@ -27483,6 +27352,7 @@ snapshots:dependencies:tldts: 6.1.86+ optional: truedependencies:@@ -27493,6 +27363,7 @@ snapshots:dependencies:punycode: 2.3.1+ optional: truedependencies:@@ -27694,8 +27565,6 @@ snapshots:optional: true- [email protected]: {}-@@ -27951,6 +27820,7 @@ snapshots:dependencies:xml-name-validator: 5.0.0+ optional: true@@ -27979,7 +27849,8 @@ snapshots:- [email protected]: {}+ optional: truedependencies:@@ -28125,6 +27996,7 @@ snapshots:dependencies:tr46: 5.1.1webidl-conversions: 7.0.0+ optional: truedependencies:@@ -28284,7 +28156,8 @@ snapshots:dependencies:sax: 1.6.0- [email protected]: {}+ optional: truedependencies:@@ -28293,7 +28166,8 @@ snapshots:- [email protected]: {}+ optional: truediff --git a/test/core/crawlers/dom_crawler.test.ts b/test/core/crawlers/dom_crawler.test.tsindex 279530be8..c6602618b 100644--- a/test/core/crawlers/dom_crawler.test.ts+++ b/test/core/crawlers/dom_crawler.test.ts@@ -4,8 +4,6 @@ import type { AddressInfo } from 'node:net';import { MemoryStorageBackend, serviceLocator } from '@crawlee/core';import type { DOMParser } from '@crawlee/http';import { DOMCrawler } from '@crawlee/http';-import { JSDOMCrawler } from '@crawlee/jsdom';-import { LinkeDOMCrawler } from '@crawlee/linkedom';interface FakeParseResult {title: string;@@ -69,55 +67,6 @@ beforeEach(async () => {serviceLocator.setStorageBackend(new MemoryStorageBackend());});-test('works', async () => {- const results: string[] = [];-- const crawler = new JSDOMCrawler({- maxRequestRetries: 0,- requestHandler: ({ window }) => {- results.push(window.document.title, window.document.querySelector('p')!.textContent!);- },- });-- await crawler.run([url]);-- expect(results).toStrictEqual(['Example Domain', 'Hello, world!']);-});--test('JSDOMCrawler enqueueLinks should respect maxCrawlDepth', async () => {- const titles: string[] = [];-- const crawler = new JSDOMCrawler({- maxCrawlDepth: 1,- maxRequestsPerCrawl: 10, // to avoid accidental runaway- requestHandler: async ({ window, enqueueLinks }) => {- titles.push(window.document.title);- await enqueueLinks();- },- });-- await crawler.run([`${url}/depth-0`]);-- expect(titles).toEqual(['Depth 0', 'Depth 1']);-});--test('LinkeDOMCrawler enqueueLinks should respect maxCrawlDepth', async () => {- const titles: string[] = [];-- const crawler = new LinkeDOMCrawler({- maxCrawlDepth: 1,- maxRequestsPerCrawl: 10, // to avoid accidental runaway- requestHandler: async ({ document, enqueueLinks }) => {- titles.push(document.title);- await enqueueLinks();- },- });-- await crawler.run([`${url}/depth-0`]);-- expect(titles).toEqual(['Depth 0', 'Depth 1']);-});-test('DOMCrawler works with skipNavigation', async () => {const errors: string[] = [];const failed: string[] = [];diff --git a/test/e2e/jsdom-default-ts/actor/.actor/actor.json b/test/e2e/jsdom-default-ts/actor/.actor/actor.jsondeleted file mode 100644index c2269107d..000000000--- a/test/e2e/jsdom-default-ts/actor/.actor/actor.json+++ /dev/null@@ -1,7 +0,0 @@-{- "actorSpecification": 1,- "name": "test-jsdom-default-ts",- "version": "0.0",- "buildTag": "latest",- "env": null-}diff --git a/test/e2e/jsdom-default-ts/actor/.eslintrc.json b/test/e2e/jsdom-default-ts/actor/.eslintrc.jsondeleted file mode 100644index fb55b4b7f..000000000--- a/test/e2e/jsdom-default-ts/actor/.eslintrc.json+++ /dev/null@@ -1,8 +0,0 @@-{- "root": true,- "extends": "../../.eslintrc.json",- "parserOptions": {- "project": "./test/e2e/jsdom-default-ts/actor/tsconfig.json",- "ecmaVersion": 2022- }-}diff --git a/test/e2e/jsdom-default-ts/actor/.gitignore b/test/e2e/jsdom-default-ts/actor/.gitignoredeleted file mode 100644index f2fc11c72..000000000--- a/test/e2e/jsdom-default-ts/actor/.gitignore+++ /dev/null@@ -1,11 +0,0 @@-.idea-.DS_Store-node_modules-package-lock.json-apify_storage-crawlee_storage-storage-main.d.ts-main.d.ts.map-main.js-main.js.mapdiff --git a/test/e2e/jsdom-default-ts/actor/Dockerfile b/test/e2e/jsdom-default-ts/actor/Dockerfiledeleted file mode 100644index ba62560de..000000000--- a/test/e2e/jsdom-default-ts/actor/Dockerfile+++ /dev/null@@ -1,30 +0,0 @@-# using multistage build, as we need dev deps to build the TS source code-FROM apify/actor-node:24-beta AS builder--# copy all files, install all dependencies (including dev deps) and build the project-COPY . ./-RUN rm -rf node_modules package-lock.json-RUN npm install --include=dev \- && npm run build--# create final image-FROM apify/actor-node:24-beta-# copy only necessary files-COPY --from=builder /usr/src/app/packages ./packages-COPY --from=builder /usr/src/app/package.json ./-COPY --from=builder /usr/src/app/main.js ./--# install only prod deps-RUN rm -rf node_modules package-lock.json-RUN npm --quiet set progress=false \- && npm install --only=prod --no-audit \- && npm update --no-audit \- && echo "Installed NPM packages:" \- && (npm list --only=prod --all || true) \- && echo "Node.js version:" \- && node --version \- && echo "NPM version:" \- && npm --version--# run compiled code-CMD npm run start:proddiff --git a/test/e2e/jsdom-default-ts/actor/main.ts b/test/e2e/jsdom-default-ts/actor/main.tsdeleted file mode 100644index 384a76682..000000000--- a/test/e2e/jsdom-default-ts/actor/main.ts+++ /dev/null@@ -1,31 +0,0 @@-import assert from 'node:assert';--import { Dataset, JSDOMCrawler } from '@crawlee/jsdom';-import { Actor } from 'apify';--if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {- // @ts-ignore- await Actor.init({ storage: new (await import('@apify/storage-local')).ApifyStorageLocal() });-} else {- await Actor.init();-}--const crawler = new JSDOMCrawler();--crawler.router.addDefaultHandler(async ({ window, document, enqueueLinks, request, log }) => {- const { url } = request;- await enqueueLinks({- include: ['https://crawlee.dev/js/docs/**'],- });-- const pageTitle = window.document.title;- const { title } = document;- assert.strictEqual(pageTitle, title);- log.info(`URL: ${url} TITLE: ${pageTitle}`);-- await Dataset.pushData({ url, pageTitle });-});--await crawler.run(['https://crawlee.dev/js/docs/quick-start']);--await Actor.exit({ exit: Actor.isAtHome() });diff --git a/test/e2e/jsdom-default-ts/actor/package.json b/test/e2e/jsdom-default-ts/actor/package.jsondeleted file mode 100644index 895e4b9be..000000000--- a/test/e2e/jsdom-default-ts/actor/package.json+++ /dev/null@@ -1,34 +0,0 @@-{- "name": "test-jsdom-default-ts",- "version": "0.0.1",- "description": "JSDOM Crawler Test - TypeScript",- "dependencies": {- "apify": "^4.0.0-beta.24",- "@crawlee/basic": "file:./packages/basic-crawler",- "@crawlee/browser-pool": "file:./packages/browser-pool",- "@crawlee/http": "file:./packages/http-crawler",- "@crawlee/jsdom": "file:./packages/jsdom-crawler",- "@crawlee/core": "file:./packages/core",- "@crawlee/types": "file:./packages/types",- "@crawlee/utils": "file:./packages/utils"- },- "overrides": {- "apify": {- "@crawlee/core": "file:./packages/core",- "@crawlee/types": "file:./packages/types",- "@crawlee/utils": "file:./packages/utils"- }- },- "devDependencies": {- "@apify/tsconfig": "^0.1.0",- "typescript": "^5.0.0",- "@types/node": "^24.0.0"- },- "scripts": {- "start": "tsc && node main.js",- "start:prod": "node main.js",- "build": "tsc"- },- "type": "module",- "license": "ISC"-}diff --git a/test/e2e/jsdom-default-ts/actor/tsconfig.json b/test/e2e/jsdom-default-ts/actor/tsconfig.jsondeleted file mode 100644index dc194464d..000000000--- a/test/e2e/jsdom-default-ts/actor/tsconfig.json+++ /dev/null@@ -1,11 +0,0 @@-{- "extends": "@apify/tsconfig",- "compilerOptions": {- "module": "NodeNext",- "moduleResolution": "NodeNext",- "target": "ES2022",- "lib": ["ES2022", "DOM"],- "skipLibCheck": true- },- "include": ["./**/*.ts"]-}diff --git a/test/e2e/jsdom-default-ts/test.mjs b/test/e2e/jsdom-default-ts/test.mjsdeleted file mode 100644index 7de081929..000000000--- a/test/e2e/jsdom-default-ts/test.mjs+++ /dev/null@@ -1,10 +0,0 @@-import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs';--const testActorDirname = getActorTestDir(import.meta.url);-await initialize(testActorDirname);--const { stats, datasetItems } = await runActor(testActorDirname);--await expect(stats.requestsSucceeded > 40, 'All requests finished');-await expect(datasetItems.length > 40, 'Number of dataset items');-await expect(validateDataset(datasetItems, ['url', 'pageTitle']), 'Dataset items validation');diff --git a/test/e2e/jsdom-react-ts/actor/.actor/actor.json b/test/e2e/jsdom-react-ts/actor/.actor/actor.jsondeleted file mode 100644index c34385d2e..000000000--- a/test/e2e/jsdom-react-ts/actor/.actor/actor.json+++ /dev/null@@ -1,7 +0,0 @@-{- "actorSpecification": 1,- "name": "test-jsdom-react-ts",- "version": "0.0",- "buildTag": "latest",- "env": null-}diff --git a/test/e2e/jsdom-react-ts/actor/.eslintrc.json b/test/e2e/jsdom-react-ts/actor/.eslintrc.jsondeleted file mode 100644index 3e1e87a6e..000000000--- a/test/e2e/jsdom-react-ts/actor/.eslintrc.json+++ /dev/null@@ -1,8 +0,0 @@-{- "root": true,- "extends": "../../.eslintrc.json",- "parserOptions": {- "project": "./test/e2e/jsdom-react-ts/actor/tsconfig.json",- "ecmaVersion": 2022- }-}diff --git a/test/e2e/jsdom-react-ts/actor/.gitignore b/test/e2e/jsdom-react-ts/actor/.gitignoredeleted file mode 100644index f2fc11c72..000000000--- a/test/e2e/jsdom-react-ts/actor/.gitignore+++ /dev/null@@ -1,11 +0,0 @@-.idea-.DS_Store-node_modules-package-lock.json-apify_storage-crawlee_storage-storage-main.d.ts-main.d.ts.map-main.js-main.js.mapdiff --git a/test/e2e/jsdom-react-ts/actor/Dockerfile b/test/e2e/jsdom-react-ts/actor/Dockerfiledeleted file mode 100644index ba62560de..000000000--- a/test/e2e/jsdom-react-ts/actor/Dockerfile+++ /dev/null@@ -1,30 +0,0 @@-# using multistage build, as we need dev deps to build the TS source code-FROM apify/actor-node:24-beta AS builder--# copy all files, install all dependencies (including dev deps) and build the project-COPY . ./-RUN rm -rf node_modules package-lock.json-RUN npm install --include=dev \- && npm run build--# create final image-FROM apify/actor-node:24-beta-# copy only necessary files-COPY --from=builder /usr/src/app/packages ./packages-COPY --from=builder /usr/src/app/package.json ./-COPY --from=builder /usr/src/app/main.js ./--# install only prod deps-RUN rm -rf node_modules package-lock.json-RUN npm --quiet set progress=false \- && npm install --only=prod --no-audit \- && npm update --no-audit \- && echo "Installed NPM packages:" \- && (npm list --only=prod --all || true) \- && echo "Node.js version:" \- && node --version \- && echo "NPM version:" \- && npm --version--# run compiled code-CMD npm run start:proddiff --git a/test/e2e/jsdom-react-ts/actor/main.ts b/test/e2e/jsdom-react-ts/actor/main.tsdeleted file mode 100644index 7d6be675a..000000000--- a/test/e2e/jsdom-react-ts/actor/main.ts+++ /dev/null@@ -1,29 +0,0 @@-import { Dataset, JSDOMCrawler } from '@crawlee/jsdom';-import { Actor } from 'apify';--if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {- // @ts-ignore- await Actor.init({ storage: new (await import('@apify/storage-local')).ApifyStorageLocal() });-} else {- await Actor.init();-}--const crawler = new JSDOMCrawler({- runScripts: true,- requestHandler: async ({ window }) => {- const { document } = window;- document.querySelectorAll('button')[12].click(); // 1- document.querySelectorAll('button')[15].click(); // +- document.querySelectorAll('button')[12].click(); // 1- document.querySelectorAll('button')[18].click(); // =-- // 2- const { innerHTML } = document.querySelectorAll('.component-display')[0].childNodes[0] as Element;-- await Dataset.pushData({ result: innerHTML });- },-});--await crawler.run(['https://ahfarmer.github.io/calculator/']);--await Actor.exit({ exit: Actor.isAtHome() });diff --git a/test/e2e/jsdom-react-ts/actor/package.json b/test/e2e/jsdom-react-ts/actor/package.jsondeleted file mode 100644index 4d5286746..000000000--- a/test/e2e/jsdom-react-ts/actor/package.json+++ /dev/null@@ -1,34 +0,0 @@-{- "name": "test-jsdom-react-ts",- "version": "0.0.1",- "description": "JSDOM Crawler Test - React - TypeScript",- "dependencies": {- "apify": "^4.0.0-beta.24",- "@crawlee/basic": "file:./packages/basic-crawler",- "@crawlee/browser-pool": "file:./packages/browser-pool",- "@crawlee/http": "file:./packages/http-crawler",- "@crawlee/jsdom": "file:./packages/jsdom-crawler",- "@crawlee/core": "file:./packages/core",- "@crawlee/types": "file:./packages/types",- "@crawlee/utils": "file:./packages/utils"- },- "overrides": {- "apify": {- "@crawlee/core": "file:./packages/core",- "@crawlee/types": "file:./packages/types",- "@crawlee/utils": "file:./packages/utils"- }- },- "devDependencies": {- "@apify/tsconfig": "^0.1.0",- "typescript": "^5.0.0",- "@types/node": "^24.0.0"- },- "scripts": {- "start": "tsc && node main.js",- "start:prod": "node main.js",- "build": "tsc"- },- "type": "module",- "license": "ISC"-}diff --git a/test/e2e/jsdom-react-ts/actor/tsconfig.json b/test/e2e/jsdom-react-ts/actor/tsconfig.jsondeleted file mode 100644index dc194464d..000000000--- a/test/e2e/jsdom-react-ts/actor/tsconfig.json+++ /dev/null@@ -1,11 +0,0 @@-{- "extends": "@apify/tsconfig",- "compilerOptions": {- "module": "NodeNext",- "moduleResolution": "NodeNext",- "target": "ES2022",- "lib": ["ES2022", "DOM"],- "skipLibCheck": true- },- "include": ["./**/*.ts"]-}diff --git a/test/e2e/jsdom-react-ts/test.mjs b/test/e2e/jsdom-react-ts/test.mjsdeleted file mode 100644index 3917fa1bb..000000000--- a/test/e2e/jsdom-react-ts/test.mjs+++ /dev/null@@ -1,13 +0,0 @@-import { expect, getActorTestDir, initialize, runActor, skipTest, validateDataset } from '../tools.mjs';--await skipTest('target site no longer exists');--const testActorDirname = getActorTestDir(import.meta.url);-await initialize(testActorDirname);--const { stats, datasetItems } = await runActor(testActorDirname);--await expect(stats.requestsSucceeded === 1, 'All requests finished');-await expect(datasetItems.length === 1, 'Number of dataset items');-await expect(validateDataset(datasetItems, ['result']), 'Dataset items validation');-await expect(datasetItems[0].result, 'Dataset items');diff --git a/test/e2e/linkedom-default-ts/actor/.actor/actor.json b/test/e2e/linkedom-default-ts/actor/.actor/actor.jsondeleted file mode 100644index a7cd6f005..000000000--- a/test/e2e/linkedom-default-ts/actor/.actor/actor.json+++ /dev/null@@ -1,7 +0,0 @@-{- "actorSpecification": 1,- "name": "test-linkedom-default-ts",- "version": "0.0",- "buildTag": "latest",- "env": null-}diff --git a/test/e2e/linkedom-default-ts/actor/.eslintrc.json b/test/e2e/linkedom-default-ts/actor/.eslintrc.jsondeleted file mode 100644index 7bbe18228..000000000--- a/test/e2e/linkedom-default-ts/actor/.eslintrc.json+++ /dev/null@@ -1,8 +0,0 @@-{- "root": true,- "extends": "../../.eslintrc.json",- "parserOptions": {- "project": "./test/e2e/linkedom-default-ts/actor/tsconfig.json",- "ecmaVersion": 2022- }-}diff --git a/test/e2e/linkedom-default-ts/actor/.gitignore b/test/e2e/linkedom-default-ts/actor/.gitignoredeleted file mode 100644index f2fc11c72..000000000--- a/test/e2e/linkedom-default-ts/actor/.gitignore+++ /dev/null@@ -1,11 +0,0 @@-.idea-.DS_Store-node_modules-package-lock.json-apify_storage-crawlee_storage-storage-main.d.ts-main.d.ts.map-main.js-main.js.mapdiff --git a/test/e2e/linkedom-default-ts/actor/Dockerfile b/test/e2e/linkedom-default-ts/actor/Dockerfiledeleted file mode 100644index ba62560de..000000000--- a/test/e2e/linkedom-default-ts/actor/Dockerfile+++ /dev/null@@ -1,30 +0,0 @@-# using multistage build, as we need dev deps to build the TS source code-FROM apify/actor-node:24-beta AS builder--# copy all files, install all dependencies (including dev deps) and build the project-COPY . ./-RUN rm -rf node_modules package-lock.json-RUN npm install --include=dev \- && npm run build--# create final image-FROM apify/actor-node:24-beta-# copy only necessary files-COPY --from=builder /usr/src/app/packages ./packages-COPY --from=builder /usr/src/app/package.json ./-COPY --from=builder /usr/src/app/main.js ./--# install only prod deps-RUN rm -rf node_modules package-lock.json-RUN npm --quiet set progress=false \- && npm install --only=prod --no-audit \- && npm update --no-audit \- && echo "Installed NPM packages:" \- && (npm list --only=prod --all || true) \- && echo "Node.js version:" \- && node --version \- && echo "NPM version:" \- && npm --version--# run compiled code-CMD npm run start:proddiff --git a/test/e2e/linkedom-default-ts/actor/main.ts b/test/e2e/linkedom-default-ts/actor/main.tsdeleted file mode 100644index ac5b87821..000000000--- a/test/e2e/linkedom-default-ts/actor/main.ts+++ /dev/null@@ -1,30 +0,0 @@-import assert from 'node:assert';--import { Dataset, LinkeDOMCrawler } from '@crawlee/linkedom';-import { Actor } from 'apify';--if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {- // @ts-ignore- await Actor.init({ storage: new (await import('@apify/storage-local')).ApifyStorageLocal() });-} else {- await Actor.init();-}--const crawler = new LinkeDOMCrawler();--crawler.router.addDefaultHandler(async ({ document, enqueueLinks, request, log }) => {- const { url } = request;- await enqueueLinks({- include: ['https://crawlee.dev/js/docs/**'],- });-- const pageTitle = document.querySelector('title')?.textContent ?? '';- assert.notEqual(pageTitle, '');- log.info(`URL: ${url} TITLE: ${pageTitle}`);-- await Dataset.pushData({ url, pageTitle });-});--await crawler.run(['https://crawlee.dev/js/docs/quick-start']);--await Actor.exit({ exit: Actor.isAtHome() });diff --git a/test/e2e/linkedom-default-ts/actor/package.json b/test/e2e/linkedom-default-ts/actor/package.jsondeleted file mode 100644index 5311cea3b..000000000--- a/test/e2e/linkedom-default-ts/actor/package.json+++ /dev/null@@ -1,34 +0,0 @@-{- "name": "test-linkedom-default-ts",- "version": "0.0.1",- "description": "LinkeDOM Crawler Test - TypeScript",- "dependencies": {- "apify": "^4.0.0-beta.24",- "@crawlee/basic": "file:./packages/basic-crawler",- "@crawlee/browser-pool": "file:./packages/browser-pool",- "@crawlee/http": "file:./packages/http-crawler",- "@crawlee/linkedom": "file:./packages/linkedom-crawler",- "@crawlee/core": "file:./packages/core",- "@crawlee/types": "file:./packages/types",- "@crawlee/utils": "file:./packages/utils"- },- "overrides": {- "apify": {- "@crawlee/core": "file:./packages/core",- "@crawlee/types": "file:./packages/types",- "@crawlee/utils": "file:./packages/utils"- }- },- "devDependencies": {- "@apify/tsconfig": "^0.1.0",- "typescript": "^5.0.0",- "@types/node": "^24.0.0"- },- "scripts": {- "start": "tsc && node main.js",- "start:prod": "node main.js",- "build": "tsc"- },- "type": "module",- "license": "ISC"-}diff --git a/test/e2e/linkedom-default-ts/actor/tsconfig.json b/test/e2e/linkedom-default-ts/actor/tsconfig.jsondeleted file mode 100644index dc194464d..000000000--- a/test/e2e/linkedom-default-ts/actor/tsconfig.json+++ /dev/null@@ -1,11 +0,0 @@-{- "extends": "@apify/tsconfig",- "compilerOptions": {- "module": "NodeNext",- "moduleResolution": "NodeNext",- "target": "ES2022",- "lib": ["ES2022", "DOM"],- "skipLibCheck": true- },- "include": ["./**/*.ts"]-}diff --git a/test/e2e/linkedom-default-ts/test.mjs b/test/e2e/linkedom-default-ts/test.mjsdeleted file mode 100644index 7de081929..000000000--- a/test/e2e/linkedom-default-ts/test.mjs+++ /dev/null@@ -1,10 +0,0 @@-import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs';--const testActorDirname = getActorTestDir(import.meta.url);-await initialize(testActorDirname);--const { stats, datasetItems } = await runActor(testActorDirname);--await expect(stats.requestsSucceeded > 40, 'All requests finished');-await expect(datasetItems.length > 40, 'Number of dataset items');-await expect(validateDataset(datasetItems, ['url', 'pageTitle']), 'Dataset items validation');diff --git a/test/tsconfig.json b/test/tsconfig.jsonindex 628878c2b..95ff8d4b6 100644--- a/test/tsconfig.json+++ b/test/tsconfig.json@@ -14,8 +14,6 @@"@crawlee/basic": ["../packages/basic-crawler/src/index.ts"],"@crawlee/browser": ["../packages/browser-crawler/src/index.ts"],"@crawlee/http": ["../packages/http-crawler/src/index.ts"],- "@crawlee/linkedom": ["../packages/linkedom-crawler/src/index.ts"],- "@crawlee/jsdom": ["../packages/jsdom-crawler/src/index.ts"],"@crawlee/cheerio": ["../packages/cheerio-crawler/src/index.ts"],"@crawlee/playwright": ["../packages/playwright-crawler/src/index.ts"],"@crawlee/puppeteer": ["../packages/puppeteer-crawler/src/index.ts"],diff --git a/tsconfig.json b/tsconfig.jsonindex 4b69b7afe..cc2c5384d 100644--- a/tsconfig.json+++ b/tsconfig.json@@ -10,8 +10,6 @@"@crawlee/basic": ["./packages/basic-crawler/src/index.ts"],"@crawlee/browser": ["./packages/browser-crawler/src/index.ts"],"@crawlee/http": ["./packages/http-crawler/src/index.ts"],- "@crawlee/linkedom": ["./packages/linkedom-crawler/src/index.ts"],- "@crawlee/jsdom": ["./packages/jsdom-crawler/src/index.ts"],"@crawlee/cheerio": ["./packages/cheerio-crawler/src/index.ts"],"@crawlee/playwright": ["./packages/playwright-crawler/src/index.ts"],"@crawlee/puppeteer": ["./packages/puppeteer-crawler/src/index.ts"],diff --git a/vitest.config.mts b/vitest.config.mtsindex 03d3008d0..86a352193 100644--- a/vitest.config.mts+++ b/vitest.config.mts@@ -50,8 +50,6 @@ const baseConfig = defineConfig({{ find: '@crawlee/basic', replacement: resolve(__dirname, './packages/basic-crawler/src') },{ find: '@crawlee/browser', replacement: resolve(__dirname, './packages/browser-crawler/src') },{ find: '@crawlee/http', replacement: resolve(__dirname, './packages/http-crawler/src') },- { find: '@crawlee/linkedom', replacement: resolve(__dirname, './packages/linkedom-crawler/src') },- { find: '@crawlee/jsdom', replacement: resolve(__dirname, './packages/jsdom-crawler/src') },{ find: '@crawlee/cheerio', replacement: resolve(__dirname, './packages/cheerio-crawler/src') },{ find: '@crawlee/playwright', replacement: resolve(__dirname, './packages/playwright-crawler/src') },{ find: '@crawlee/puppeteer', replacement: resolve(__dirname, './packages/puppeteer-crawler/src') },diff --git a/website/docusaurus.config.js b/website/docusaurus.config.jsindex 07ebaeae5..3b6cde091 100644--- a/website/docusaurus.config.js+++ b/website/docusaurus.config.js@@ -12,8 +12,6 @@ const packages = ['cheerio-crawler','puppeteer-crawler','playwright-crawler',- 'jsdom-crawler',- 'linkedom-crawler','stagehand-crawler','fs-storage','utils',@@ -27,8 +25,6 @@ const packagesOrder = ['@crawlee/cheerio','@crawlee/playwright','@crawlee/puppeteer',- '@crawlee/jsdom',- '@crawlee/linkedom','@crawlee/stagehand','@crawlee/basic','@crawlee/http',diff --git a/website/sidebars.js b/website/sidebars.jsindex dbed1ce81..03472e4ac 100644--- a/website/sidebars.js+++ b/website/sidebars.js@@ -45,7 +45,6 @@ module.exports = {'guides/trace-and-monitor-crawlers','guides/avoid-blocking','guides/cookie-modals',- 'guides/jsdom-crawler-guide','guides/impit-http-client/impit-http-client','guides/got-scraping','guides/typescript-project',diff --git a/website/src/components/Highlights.jsx b/website/src/components/Highlights.jsxindex 4f43fca63..ed5d0f69b 100644--- a/website/src/components/Highlights.jsx+++ b/website/src/components/Highlights.jsx@@ -21,9 +21,8 @@ const FeatureList = [description: (<>Crawlee makes HTTP requests that <a href="https://crawlee.dev/js/docs/guides/avoid-blocking"><b>mimic browser headers and TLS fingerprints</b></a>.- It also rotates them automatically based on data about real-world traffic. Popular HTML- parsers <b><a href="https://crawlee.dev/js/docs/guides/cheerio-crawler-guide">Cheerio</a> - and <a href="https://crawlee.dev/js/docs/guides/jsdom-crawler-guide">JSDOM</a></b> are included.+ It also rotates them automatically based on data about real-world traffic. The popular HTML+ parser <b><a href="https://crawlee.dev/js/docs/guides/cheerio-crawler-guide">Cheerio</a></b> is included.</>),},