Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat(http): add DomCrawler with a pluggable DOM parser (#4104)
Jindřich Bär3f6f7bb94228
diff --git a/docs/public-api/crawlee-cheerio.api.md b/docs/public-api/crawlee-cheerio.api.mdindex 45d334529..1e124e2da 100644--- a/docs/public-api/crawlee-cheerio.api.md+++ b/docs/public-api/crawlee-cheerio.api.md@@ -4,19 +4,14 @@```ts-import type { AddRequestsBatchedResult } from '@crawlee/http';-import * as cheerio from 'cheerio';import type { CheerioAPI } from 'cheerio';-import type { ContextPipeline } from '@crawlee/http';import type { CrawlingContext } from '@crawlee/http';import type { Dictionary } from '@crawlee/types';-import type { EnqueueLinksOptions } from '@crawlee/http';+import { DOMCrawler } from '@crawlee/http';+import type { DOMCrawlingContext } from '@crawlee/http';import type { ErrorHandler } from '@crawlee/http';-import type { ExtractLinksOptions } from '@crawlee/http';import type { GetUserDataFromRequest } from '@crawlee/http';-import { HttpCrawler } from '@crawlee/http';import type { HttpCrawlerOptions } from '@crawlee/http';-import type { InternalHttpCrawlingContext } from '@crawlee/http';import type { InternalHttpHook } from '@crawlee/http';import type { RequestHandler } from '@crawlee/http';import type { RouterHandler } from '@crawlee/http';@@ -25,10 +20,8 @@ import type { RouteSchemas } from '@crawlee/http';import type { RoutesFromSchemas } from '@crawlee/http';// @public-export class CheerioCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<CheerioCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<CheerioCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+export class CheerioCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<CheerioCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends DOMCrawler<CheerioParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);- // (undocumented)- protected buildContextPipeline(): ContextPipeline<CrawlingContext, CheerioCrawlingContext>;}// @public (undocumented)@@ -39,13 +32,7 @@ Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>, Stat// @public (undocumented)export interface CheerioCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {- $: cheerio.CheerioAPI;- body: string;- enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;- extractLinks(options?: ExtractLinksOptions): Promise<string[]>;- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;- waitForSelector(selector: string, timeoutMs?: number): Promise<void>;+JSONData extends Dictionary = any> extends DOMCrawlingContext<CheerioParseResult, UserData, JSONData> {}// @public (undocumented)@@ -57,6 +44,14 @@ ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, CheerioCraexport type CheerioHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any> = InternalHttpHook<CheerioCrawlingContext<UserData, JSONData>>;+// @public (undocumented)+export interface CheerioParseResult {+ // (undocumented)+ $: CheerioAPI;+ // (undocumented)+ body: string;+}+// @public (undocumented)export type CheerioRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any> = RequestHandler<CheerioCrawlingContext<UserData, JSONData>>;diff --git a/docs/public-api/crawlee-http.api.md b/docs/public-api/crawlee-http.api.mdindex bd7de1e0e..9086bbcd0 100644--- a/docs/public-api/crawlee-http.api.md+++ b/docs/public-api/crawlee-http.api.md@@ -4,6 +4,7 @@```ts+import type { AddRequestsBatchedResult } from '@crawlee/basic';import type { Awaitable } from '@crawlee/types';import { BasicCrawler } from '@crawlee/basic';import { BasicCrawlerOptions } from '@crawlee/basic';@@ -15,7 +16,9 @@ import type { ContextPipeline as ContextPipeline_2 } from '@crawlee/core';import type { CrawlingContext } from '@crawlee/basic';import type { CrawlingContext as CrawlingContext_2 } from '@crawlee/core';import type { Dictionary } from '@crawlee/types';+import type { EnqueueLinksOptions } from '@crawlee/basic';import { ErrorHandler } from '@crawlee/basic';+import type { ExtractLinksOptions } from '@crawlee/basic';import { GetUserDataFromRequest } from '@crawlee/basic';import type { JsonValue } from 'type-fest';import { LoadedRequest } from '@crawlee/core';@@ -60,6 +63,49 @@ export function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawl// @public (undocumented)export function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;+// @public+export class DOMCrawler<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<DOMCrawlingContext<Parsed>['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+ constructor(options: DOMCrawlerOptions<Parsed, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);+ // (undocumented)+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed>>;+}++// @public (undocumented)+export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, any>, StatisticStateExtension extends object = {}> extends HttpCrawlerOptions<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+ parser: DOMParser_2<Parsed>;+}++// @public (undocumented)+export type DOMCrawlingContext<Parsed extends DOMParseResult = DOMParseResult, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler+JSONData extends Dictionary = any> = InternalHttpCrawlingContext<UserData, JSONData> & Parsed & DOMCrawlingHelpers;++// @public (undocumented)+export interface DOMCrawlingHelpers {+ enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;+ extractLinks(options?: ExtractLinksOptions): Promise<string[]>;+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;+}++// @public+interface DOMParser_2<Parsed extends DOMParseResult> {+ cleanup?(parsed: Parsed): Awaitable<void>;+ extractLinks(parsed: Parsed, selector: string, baseUrl: string): Awaitable<string[]>;+ readonly mutable?: boolean;+ // (undocumented)+ parse(context: InternalHttpCrawlingContext): Awaitable<Parsed>;+ readonly placeholderMembers: Record<keyof Parsed & string, true>;+ select(parsed: Parsed, selector: string): Awaitable<ArrayLike<unknown>>;+ toCheerio?(parsed: Parsed): Awaitable<CheerioAPI>;+}+export { DOMParser_2 as DOMParser }++// @public+export interface DOMParseResult {+ // (undocumented)+ body: string;+}+// @publicexport class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {constructor(options?: BasicCrawlerOptions<FileDownloadCrawlingContext>);diff --git a/docs/public-api/crawlee-jsdom.api.md b/docs/public-api/crawlee-jsdom.api.mdindex 6a4aef0fb..0cf817e6e 100644--- a/docs/public-api/crawlee-jsdom.api.md+++ b/docs/public-api/crawlee-jsdom.api.md@@ -4,19 +4,14 @@```ts-import type { AddRequestsBatchedResult } from '@crawlee/http';-import type { CheerioAPI } from 'cheerio';-import type { ContextPipeline } from '@crawlee/http';import type { CrawlingContext } from '@crawlee/http';import type { Dictionary } from '@crawlee/types';+import { DOMCrawler } from '@crawlee/http';+import type { DOMCrawlingContext } from '@crawlee/http';import type { DOMWindow } from 'jsdom';-import type { EnqueueLinksOptions } from '@crawlee/http';import type { ErrorHandler } from '@crawlee/http';-import type { ExtractLinksOptions } from '@crawlee/http';import type { GetUserDataFromRequest } from '@crawlee/http';-import { HttpCrawler } from '@crawlee/http';import type { HttpCrawlerOptions } from '@crawlee/http';-import type { InternalHttpCrawlingContext } from '@crawlee/http';import type { InternalHttpHook } from '@crawlee/http';import type { RequestHandler } from '@crawlee/http';import type { RouterHandler } from '@crawlee/http';@@ -34,11 +29,9 @@ export function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCr// @public (undocumented)export function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;-// @public (undocumented)-export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<JSDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+// @public+export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<JSDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends DOMCrawler<JSDOMParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);- // (undocumented)- protected buildContextPipeline(): ContextPipeline<CrawlingContext, JSDOMCrawlingContext>;getVirtualConsole(): VirtualConsole;}@@ -52,17 +45,7 @@ Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>, Stat// @public (undocumented)export interface JSDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {- // (undocumented)- body: string;- // (undocumented)- document: Document;- enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;- extractLinks(options?: ExtractLinksOptions): Promise<string[]>;- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;- waitForSelector(selector: string, timeoutMs?: number): Promise<void>;- // (undocumented)- window: DOMWindow;+JSONData extends Dictionary = any> extends DOMCrawlingContext<JSDOMParseResult, UserData, JSONData> {}// @public (undocumented)@@ -74,6 +57,16 @@ ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, JSDOMCrawlexport type JSDOMHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any> = InternalHttpHook<JSDOMCrawlingContext<UserData, JSONData>>;+// @public (undocumented)+export interface JSDOMParseResult {+ // (undocumented)+ body: string;+ // (undocumented)+ document: Document;+ // (undocumented)+ window: DOMWindow;+}+// @public (undocumented)export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;diff --git a/docs/public-api/crawlee-linkedom.api.md b/docs/public-api/crawlee-linkedom.api.mdindex 69a083fd1..e8e3e3e22 100644--- a/docs/public-api/crawlee-linkedom.api.md+++ b/docs/public-api/crawlee-linkedom.api.md@@ -4,18 +4,13 @@```ts-import type { AddRequestsBatchedResult } from '@crawlee/http';-import type { CheerioAPI } from 'cheerio';-import type { ContextPipeline } from '@crawlee/http';import type { CrawlingContext } from '@crawlee/http';import type { Dictionary } from '@crawlee/types';-import type { EnqueueLinksOptions } from '@crawlee/http';+import { DOMCrawler } from '@crawlee/http';+import type { DOMCrawlingContext } from '@crawlee/http';import type { ErrorHandler } from '@crawlee/http';-import type { ExtractLinksOptions } from '@crawlee/http';import type { GetUserDataFromRequest } from '@crawlee/http';-import { HttpCrawler } from '@crawlee/http';import type { HttpCrawlerOptions } from '@crawlee/http';-import type { InternalHttpCrawlingContext } from '@crawlee/http';import type { InternalHttpHook } from '@crawlee/http';import type { RequestHandler } from '@crawlee/http';import type { RouterHandler } from '@crawlee/http';@@ -33,10 +28,8 @@ export function createLinkeDOMRouter<Context extends LinkeDOMCrawlingContext = Lexport function createLinkeDOMRouter<Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;// @public-export class LinkeDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+export class LinkeDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends DOMCrawler<LinkeDOMParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {constructor(options?: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);- // (undocumented)- protected buildContextPipeline(): ContextPipeline<CrawlingContext, LinkeDOMCrawlingContext>;}// @public (undocumented)@@ -47,15 +40,7 @@ Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>, Stat// @public (undocumented)export interface LinkeDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {- // (undocumented)- document: Document;- enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;- extractLinks(options?: ExtractLinksOptions): Promise<string[]>;- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;- waitForSelector(selector: string, timeoutMs?: number): Promise<void>;- // (undocumented)- window: Window;+JSONData extends Dictionary = any> extends DOMCrawlingContext<LinkeDOMParseResult, UserData, JSONData> {}// @public (undocumented)@@ -67,6 +52,16 @@ ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, LinkeDOMCrexport type LinkeDOMHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any> = InternalHttpHook<LinkeDOMCrawlingContext<UserData, JSONData>>;+// @public (undocumented)+export interface LinkeDOMParseResult {+ // (undocumented)+ body: string;+ // (undocumented)+ document: Document;+ // (undocumented)+ window: Window;+}+// @public (undocumented)export type LinkeDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any> = RequestHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;diff --git a/packages/cheerio-crawler/src/index.ts b/packages/cheerio-crawler/src/index.tsindex adb102844..6d6b4a88b 100644--- a/packages/cheerio-crawler/src/index.ts+++ b/packages/cheerio-crawler/src/index.ts@@ -1,2 +1,3 @@export * from '@crawlee/http';export * from './internals/cheerio-crawler.js';+export type { CheerioParseResult } from './internals/cheerio-parser.js';diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.tsindex 4688a9df8..16d81fcac 100644--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts@@ -1,13 +1,9 @@import type {- AddRequestsBatchedResult,- ContextPipeline,CrawlingContext,- EnqueueLinksOptions,+ DOMCrawlingContext,ErrorHandler,- ExtractLinksOptions,GetUserDataFromRequest,HttpCrawlerOptions,- InternalHttpCrawlingContext,InternalHttpHook,RequestHandler,RouterHandler,@@ -15,18 +11,11 @@ import type {RouteSchemas,RoutesFromSchemas,} from '@crawlee/http';-import {- EnqueueStrategy,- HttpCrawler,- NavigationSkippedError,- resolveBaseUrlForEnqueueLinksFiltering,- Router,-} from '@crawlee/http';+import { DOMCrawler, Router } from '@crawlee/http';import type { Dictionary } from '@crawlee/types';-import { extractUrlsFromCheerio } from '@crawlee/utils/internal';-import type { CheerioAPI, CheerioOptions } from 'cheerio';-import * as cheerio from 'cheerio';-import { parseDocument } from 'htmlparser2';++import type { CheerioParseResult } from './cheerio-parser.js';+import { cheerioParser } from './cheerio-parser.js';export type CheerioErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler@@ -57,58 +46,7 @@ export type CheerioHook<export interface CheerioCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> extends InternalHttpCrawlingContext<UserData, JSONData> {- /**- * The raw HTML content of the web page as a string.- */- body: string;-- /**- * The [Cheerio](https://cheerio.js.org/) object with parsed HTML.- * Cheerio is available only for HTML and XML content types.- */- $: cheerio.CheerioAPI;-- /**- * Wait for an element matching the selector to appear. Timeout is ignored.- *- * **Example usage:**- * ```ts- * async requestHandler({ waitForSelector, parseWithCheerio }) {- * await waitForSelector('article h1');- * const $ = await parseWithCheerio();- * const title = $('title').text();- * });- * ```- */- waitForSelector(selector: string, timeoutMs?: number): Promise<void>;-- /**- * Returns Cheerio handle, this is here to unify the crawler API, so they all have this handy method.- * It has the same return type as the `$` context property, use it only if you are abstracting your workflow to- * support different context types in one handler.- * When provided with the `selector` argument, it will throw if it's not available.- *- * **Example usage:**- * ```ts- * async requestHandler({ parseWithCheerio }) {- * const $ = await parseWithCheerio();- * const title = $('title').text();- * });- * ```- */- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;-- /**- * Extracts URLs from the parsed HTML, without adding them to the request queue.- */- extractLinks(options?: ExtractLinksOptions): Promise<string[]>;-- /**- * Helper function for extracting URLs from the parsed HTML and adding them to the request queue.- */- enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;-}+> extends DOMCrawlingContext<CheerioParseResult, UserData, JSONData> {}export type CheerioRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler@@ -202,113 +140,14 @@ export class CheerioCrawler<GetUserDataFromRequest<CheerioCrawlingContext['request']>>,StatisticStateExtension extends object = {},-> extends HttpCrawler<CheerioCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+> extends DOMCrawler<CheerioParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {/*** All `CheerioCrawler` parameters are passed via an options object.*/constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>,) {- const { contextPipelineBuilder, ...rest } = options ?? {};-- super({- ...rest,- contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),- });- }-- protected override buildContextPipeline(): ContextPipeline<CrawlingContext, CheerioCrawlingContext> {- return super- .buildContextPipeline()- .compose({- action: async (context) => await this.parseContent(context),- })- .compose({ action: async (context) => await this.addHelpers(context) });- }-- private async parseContent(crawlingContext: InternalHttpCrawlingContext) {- try {- const isXml = crawlingContext.contentType.type.includes('xml');- const body = Buffer.isBuffer(crawlingContext.body)- ? crawlingContext.body.toString(crawlingContext.contentType.encoding)- : crawlingContext.body;- const dom = parseDocument(body, { decodeEntities: true, xmlMode: isXml });- const $ = cheerio.load(dom, {- xml: { decodeEntities: true, xmlMode: isXml },- } as CheerioOptions);-- return {- $,- body,- };- } catch (err) {- if (err instanceof NavigationSkippedError) {- return {- get body(): string {- throw new NavigationSkippedError(- 'The `body` property is not available - `skipNavigation` was used',- { cause: err },- );- },- get $(): CheerioAPI {- throw new NavigationSkippedError(- 'The `$` property is not available - `skipNavigation` was used',- { cause: err },- );- },- };- }-- throw err;- }- }-- private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { $: CheerioAPI }) {- const addRequests = crawlingContext.addRequests;-- const extractLinks = async (options?: ExtractLinksOptions): Promise<string[]> => {- if (!crawlingContext.$) {- throw new Error('Cannot extract links because the DOM is not available.');- }-- return extractUrlsFromCheerio(- crawlingContext.$,- options?.selector ?? 'a',- options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url,- );- };-- return {- extractLinks,- enqueueLinks: async (options: EnqueueLinksOptions = {}) => {- const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({- enqueueStrategy: options.strategy,- finalRequestUrl: crawlingContext.request.loadedUrl,- originalRequestUrl: crawlingContext.request.url,- userProvidedBaseUrl: options.baseUrl,- });-- const urls = await extractLinks(options);-- return addRequests(urls, {- ...options,- baseUrl,- strategy: options.strategy ?? EnqueueStrategy.SameHostname,- });- },- waitForSelector: async (selector: string, _timeoutMs?: number) => {- if (crawlingContext.$(selector).get().length === 0) {- throw new Error(`Selector '${selector}' not found.`);- }- },- parseWithCheerio: async (selector?: string, timeoutMs?: number) => {- if (selector) {- await crawlingContext.waitForSelector(selector, timeoutMs);- }-- return crawlingContext.$;- },- };+ super({ ...options, parser: cheerioParser() });}}diff --git a/packages/cheerio-crawler/src/internals/cheerio-parser.ts b/packages/cheerio-crawler/src/internals/cheerio-parser.tsnew file mode 100644index 000000000..24bac90a1--- /dev/null+++ b/packages/cheerio-crawler/src/internals/cheerio-parser.ts@@ -0,0 +1,35 @@+import type { DOMParser, InternalHttpCrawlingContext } from '@crawlee/http';+import { extractUrlsFromCheerio } from '@crawlee/utils/internal';+import type { CheerioAPI, CheerioOptions } from 'cheerio';+import * as cheerio from 'cheerio';+import { parseDocument } from 'htmlparser2';++export interface CheerioParseResult {+ $: CheerioAPI;+ body: string;+}++/**+ * A {@apilink DOMParser} backed by [cheerio](https://www.npmjs.com/package/cheerio). Pass it to a+ * {@apilink DOMCrawler} to get the crawling context {@apilink CheerioCrawler} provides.+ */+export function cheerioParser(): DOMParser<CheerioParseResult> {+ return {+ placeholderMembers: { $: true, body: true },+ parse(context: InternalHttpCrawlingContext) {+ const isXml = context.contentType.type.includes('xml');+ const body = Buffer.isBuffer(context.body)+ ? context.body.toString(context.contentType.encoding)+ : context.body;+ const dom = parseDocument(body, { decodeEntities: true, xmlMode: isXml });+ const $ = cheerio.load(dom, {+ xml: { decodeEntities: true, xmlMode: isXml },+ } as CheerioOptions);++ return { $, body };+ },+ extractLinks: ({ $ }, selector, baseUrl) => extractUrlsFromCheerio($, selector, baseUrl),+ select: ({ $ }, selector) => $(selector).get(),+ toCheerio: ({ $ }) => $,+ };+}diff --git a/packages/http-crawler/src/index.ts b/packages/http-crawler/src/index.tsindex b81749842..f8021aed9 100644--- a/packages/http-crawler/src/index.ts+++ b/packages/http-crawler/src/index.ts@@ -1,3 +1,4 @@export * from '@crawlee/basic';export * from './internals/http-crawler.js';+export * from './internals/dom-crawler.js';export * from './internals/file-download.js';diff --git a/packages/http-crawler/src/internals/dom-crawler.ts b/packages/http-crawler/src/internals/dom-crawler.tsnew file mode 100644index 000000000..274479bb0--- /dev/null+++ b/packages/http-crawler/src/internals/dom-crawler.ts@@ -0,0 +1,290 @@+import type {+ AddRequestsBatchedResult,+ ContextPipeline,+ CrawlingContext,+ EnqueueLinksOptions,+ ExtractLinksOptions,+ GetUserDataFromRequest,+} from '@crawlee/basic';+import { EnqueueStrategy, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering } from '@crawlee/basic';+import type { Awaitable, Dictionary } from '@crawlee/types';+import type { CheerioAPI } from 'cheerio';+import { sleep } from '@crawlee/utils';++import type { HttpCrawlerOptions, InternalHttpCrawlingContext } from './http-crawler.js';+import { HttpCrawler } from './http-crawler.js';++/**+ * The minimum a {@apilink DOMParser} has to contribute to the crawling context - the serialized document, used by+ * the {@apilink DOMCrawlingContext.parseWithCheerio|`parseWithCheerio`} helper.+ */+export interface DOMParseResult {+ body: string;+}++/**+ * Turns a response body into a DOM representation and knows how to query it. Passing one to {@apilink DOMCrawler}+ * is what makes the crawler jsdom-based, linkedom-based, or based on a DOM implementation of your own.+ *+ * **Example usage:**+ * ```ts+ * import { DOMCrawler } from 'crawlee';+ * import type { DOMParser } from 'crawlee';+ *+ * const myParser: DOMParser<{ body: string }> = {+ * // ...+ * };+ *+ * const crawler = new DOMCrawler({+ * parser: myParser,+ * async requestHandler({ body }) {+ * // ...+ * },+ * });+ * ```+ */+export interface DOMParser<Parsed extends DOMParseResult> {+ /**+ * The context members {@apilink DOMParser.parse|`parse`} contributes, mapped to `true`. Used to build the+ * placeholders that report a helpful error when the members are accessed after `skipNavigation` - the `Record`+ * type forces every key of `Parsed` to be listed, so the compiler catches an omission that would otherwise+ * yield `undefined` (rather than throwing) after `skipNavigation`.+ */+ readonly placeholderMembers: Record<keyof Parsed & string, true>;++ parse(context: InternalHttpCrawlingContext): Awaitable<Parsed>;++ /**+ * Returns the URLs the `selector` matches, resolved against `baseUrl`.+ */+ extractLinks(parsed: Parsed, selector: string, baseUrl: string): Awaitable<string[]>;++ /**+ * Returns the current matches of `selector`. Only the count is used, by+ * {@apilink DOMCrawlingContext.waitForSelector|`waitForSelector`}.+ */+ select(parsed: Parsed, selector: string): Awaitable<ArrayLike<unknown>>;++ /**+ * Whether the parse result can change after {@apilink DOMParser.parse|`parse`} returned - the case when the DOM+ * implementation runs the page scripts. If it can, {@apilink DOMCrawlingContext.waitForSelector|`waitForSelector`}+ * polls until the timeout elapses; otherwise it fails as soon as the selector does not match.+ */+ readonly mutable?: boolean;++ /**+ * Returns a Cheerio handle over the parse result, for parsers that are backed by Cheerio anyway. Without it,+ * {@apilink DOMCrawlingContext.parseWithCheerio|`parseWithCheerio`} parses+ * {@apilink DOMParseResult.body|`body`} again.+ */+ toCheerio?(parsed: Parsed): Awaitable<CheerioAPI>;++ /**+ * Releases whatever {@apilink DOMParser.parse|`parse`} allocated. Called after the request handler finishes or+ * fails, and skipped entirely when navigation was skipped.+ */+ cleanup?(parsed: Parsed): Awaitable<void>;+}++export interface DOMCrawlingHelpers {+ /**+ * Extracts URLs from the parsed DOM, without adding them to the request queue.+ */+ extractLinks(options?: ExtractLinksOptions): Promise<string[]>;++ /**+ * Helper function for extracting URLs from the parsed DOM and adding them to the request queue.+ */+ enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;++ /**+ * Wait for an element matching the selector to appear. The `timeoutMs` only has an effect when the parser is+ * {@apilink DOMParser.mutable|`mutable`} (e.g. {@apilink JSDOMCrawler} with `runScripts: true`); otherwise the+ * selector is checked once and the call resolves or throws immediately.+ * Timeout defaults to 5s.+ *+ * **Example usage:**+ * ```ts+ * async requestHandler({ waitForSelector, parseWithCheerio }) {+ * await waitForSelector('article h1');+ * const $ = await parseWithCheerio();+ * const title = $('title').text();+ * });+ * ```+ */+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;++ /**+ * Returns Cheerio handle, allowing to work with the data same way as with {@apilink CheerioCrawler}.+ * When provided with the `selector` argument, it will throw if it's not available.+ *+ * **Example usage:**+ * ```javascript+ * async requestHandler({ parseWithCheerio }) {+ * const $ = await parseWithCheerio();+ * const title = $('title').text();+ * });+ * ```+ */+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;+}++export type DOMCrawlingContext<+ Parsed extends DOMParseResult = DOMParseResult,+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler+> = InternalHttpCrawlingContext<UserData, JSONData> & Parsed & DOMCrawlingHelpers;++export interface DOMCrawlerOptions<+ Parsed extends DOMParseResult = DOMParseResult,+ ContextExtension = Dictionary<never>,+ ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension,+ Routes extends Record<keyof Routes, Dictionary> = Record<string, any>,+ StatisticStateExtension extends object = {},+> extends HttpCrawlerOptions<+ DOMCrawlingContext<Parsed>,+ ContextExtension,+ ExtendedContext,+ Routes,+ StatisticStateExtension+> {+ /**+ * The DOM implementation to parse the response bodies with. Its parse result becomes part of the crawling+ * context, so the members the request handler receives follow from the parser you pass.+ */+ parser: DOMParser<Parsed>;+}++/**+ * An {@apilink HttpCrawler} that parses each response into a DOM using the {@apilink DOMCrawlerOptions.parser|`parser`}+ * it is given, and exposes the parse result plus the {@apilink DOMCrawlingContext.enqueueLinks|`enqueueLinks`} and+ * {@apilink DOMCrawlingContext.extractLinks|`extractLinks`} helpers on the crawling context.+ *+ * {@apilink JSDOMCrawler} and {@apilink LinkeDOMCrawler} are this crawler with a parser already chosen.+ *+ * @category Crawlers+ */+export class DOMCrawler<+ Parsed extends DOMParseResult = DOMParseResult,+ ContextExtension = Dictionary<never>,+ ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension,+ Routes extends Record<keyof Routes, Dictionary> = Record<+ string,+ GetUserDataFromRequest<DOMCrawlingContext<Parsed>['request']>+ >,+ StatisticStateExtension extends object = {},+> extends HttpCrawler<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+ readonly #parser: DOMParser<Parsed>;++ constructor(+ options: DOMCrawlerOptions<Parsed, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>,+ ) {+ const { parser, contextPipelineBuilder, ...rest } = options;++ super({+ ...rest,+ contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),+ });++ this.#parser = parser;+ }++ protected override buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed>> {+ return super+ .buildContextPipeline()+ .compose({+ action: async (context) => this.#parseContent(context),+ cleanup: async (context) => {+ // The `skipNavigation` placeholders below throw on access, so there is nothing to clean up.+ if (!context.request.skipNavigation) {+ await this.#parser.cleanup?.(context as unknown as Parsed);+ }+ },+ })+ .compose({ action: async (context) => this.#addHelpers(context) });+ }++ async #parseContent(context: InternalHttpCrawlingContext): Promise<Parsed> {+ try {+ return await this.#parser.parse(context);+ } catch (err) {+ if (err instanceof NavigationSkippedError) {+ return Object.defineProperties(+ {},+ Object.fromEntries(+ (Object.keys(this.#parser.placeholderMembers) as (keyof Parsed & string)[]).map((member) => [+ member,+ {+ configurable: true,+ enumerable: true,+ get() {+ throw new NavigationSkippedError(+ `The \`${member}\` property is not available - \`skipNavigation\` was used`,+ { cause: err },+ );+ },+ },+ ]),+ ),+ ) as Parsed;+ }++ throw err;+ }+ }++ async #addHelpers(context: InternalHttpCrawlingContext & Parsed) {+ const { addRequests } = context;+ const parser = this.#parser;++ const extractLinks = async (options?: ExtractLinksOptions): Promise<string[]> =>+ parser.extractLinks(+ context,+ options?.selector ?? 'a',+ options?.baseUrl ?? context.request.loadedUrl ?? context.request.url,+ );++ const waitForSelector = async (selector: string, timeoutMs = 5_000): Promise<void> => {+ let remaining = parser.mutable ? timeoutMs : 0;++ while ((await parser.select(context, selector)).length === 0) {+ if (remaining <= 0) {+ throw new Error(`Selector '${selector}' not found.`);+ }++ await sleep(50);+ remaining -= 50;+ }+ };++ return {+ extractLinks,+ waitForSelector,+ enqueueLinks: async (options: EnqueueLinksOptions = {}) => {+ const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({+ enqueueStrategy: options.strategy,+ finalRequestUrl: context.request.loadedUrl,+ originalRequestUrl: context.request.url,+ userProvidedBaseUrl: options.baseUrl,+ });++ const urls = await extractLinks(options);++ return addRequests(urls, {+ ...options,+ baseUrl,+ strategy: options.strategy ?? EnqueueStrategy.SameHostname,+ });+ },+ async parseWithCheerio(selector?: string, _timeoutMs = 5_000) {+ const $ = (await parser.toCheerio?.(context)) ?? (await import('cheerio')).load(context.body);++ if (selector && $(selector).get().length === 0) {+ throw new Error(`Selector '${selector}' not found.`);+ }++ return $;+ },+ };+ }+}diff --git a/packages/jsdom-crawler/src/index.ts b/packages/jsdom-crawler/src/index.tsindex 905025dc8..383b91e58 100644--- a/packages/jsdom-crawler/src/index.ts+++ b/packages/jsdom-crawler/src/index.ts@@ -1,2 +1,3 @@export * from '@crawlee/http';export * from './internals/jsdom-crawler.js';+export type { JSDOMParseResult } from './internals/jsdom-parser.js';diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsindex bedbbe063..0f208d080 100644--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts@@ -1,13 +1,9 @@import type {- AddRequestsBatchedResult,- ContextPipeline,CrawlingContext,- EnqueueLinksOptions,+ DOMCrawlingContext,ErrorHandler,- ExtractLinksOptions,GetUserDataFromRequest,HttpCrawlerOptions,- InternalHttpCrawlingContext,InternalHttpHook,RequestHandler,RouterHandler,@@ -15,22 +11,14 @@ import type {RouteSchemas,RoutesFromSchemas,} from '@crawlee/http';-import {- EnqueueStrategy,- HttpCrawler,- NavigationSkippedError,- resolveBaseUrlForEnqueueLinksFiltering,- Router,-} from '@crawlee/http';+import { DOMCrawler, HttpCrawler, Router } from '@crawlee/http';import type { Dictionary } from '@crawlee/types';-import type { CheerioAPI } from 'cheerio';-import { sleep } from '@crawlee/utils';-import { parseArgument, tryAbsoluteURL } from '@crawlee/utils/internal';-import type { DOMWindow } from 'jsdom';-import { JSDOM, ResourceLoader, VirtualConsole } from 'jsdom';+import { parseArgument } from '@crawlee/utils/internal';+import { VirtualConsole } from 'jsdom';import { z } from 'zod';-import { addTimeoutToPromise } from '@apify/timeout';+import type { JSDOMParseResult } from './jsdom-parser.js';+import { jsdomParser } from './jsdom-parser.js';export type JSDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler@@ -70,51 +58,7 @@ export type JSDOMHook<export interface JSDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> extends InternalHttpCrawlingContext<UserData, JSONData> {- window: DOMWindow;- document: Document;-- body: string;-- /**- * Wait for an element matching the selector to appear.- * Timeout defaults to 5s.- *- * **Example usage:**- * ```ts- * async requestHandler({ waitForSelector, parseWithCheerio }) {- * await waitForSelector('article h1');- * const $ = await parseWithCheerio();- * const title = $('title').text();- * });- * ```- */- waitForSelector(selector: string, timeoutMs?: number): Promise<void>;-- /**- * Returns Cheerio handle, allowing to work with the data same way as with {@apilink CheerioCrawler}.- * When provided with the `selector` argument, it will first look for the selector with a 5s timeout.- *- * **Example usage:**- * ```javascript- * async requestHandler({ parseWithCheerio }) {- * const $ = await parseWithCheerio();- * const title = $('title').text();- * });- * ```- */- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;-- /**- * Extracts URLs from the parsed DOM, without adding them to the request queue.- */- extractLinks(options?: ExtractLinksOptions): Promise<string[]>;-- /**- * Helper function for extracting URLs from the parsed DOM and adding them to the request queue.- */- enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;-}+> extends DOMCrawlingContext<JSDOMParseResult, UserData, JSONData> {}export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler@@ -196,13 +140,6 @@ export type JSDOMRequestHandler<* ```* @category Crawlers*/-const resources = new ResourceLoader({- // Copy from /packages/browser-pool/src/abstract-classes/browser-plugin.ts:17- // in order not to include the entire package here- userAgent:- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',-});-export class JSDOMCrawler<ContextExtension = Dictionary<never>,ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension,@@ -211,7 +148,7 @@ export class JSDOMCrawler<GetUserDataFromRequest<JSDOMCrawlingContext['request']>>,StatisticStateExtension extends object = {},-> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+> extends DOMCrawler<JSDOMParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {/*** @internal*/@@ -224,7 +161,6 @@ export class JSDOMCrawler</** @internal */protected static override optionsSchema = z.strictObject(JSDOMCrawler.optionsShape);- #runScripts: boolean;#hideInternalConsole: boolean;#virtualConsole: VirtualConsole | null = null;@@ -240,26 +176,17 @@ export class JSDOMCrawler<super({...httpOptions,- contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),+ contextPipelineBuilder,+ parser: jsdomParser({+ runScripts,+ virtualConsole: () => this.getVirtualConsole(),+ log: () => this.log,+ }),});- this.#runScripts = runScripts;this.#hideInternalConsole = hideInternalConsole;}- protected override buildContextPipeline(): ContextPipeline<CrawlingContext, JSDOMCrawlingContext> {- return super- .buildContextPipeline()- .compose({- action: async (context) => await this.parseContent(context),- cleanup: async (context) => {- this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler);- context.window?.close();- },- })- .compose({ action: async (context) => await this.addHelpers(context) });- }-/*** Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.*@@ -291,177 +218,6 @@ export class JSDOMCrawler<}private readonly jsdomErrorHandler = (error: Error) => this.log.debug('JSDOM error from console', { error });-- private async parseContent(crawlingContext: InternalHttpCrawlingContext) {- try {- const isXml = crawlingContext.contentType.type.includes('xml');-- // TODO handle non-string- const { window } = new JSDOM(crawlingContext.body.toString(), {- url: crawlingContext.response.url,- contentType: isXml ? 'text/xml' : 'text/html',- runScripts: this.#runScripts ? 'dangerously' : undefined,- resources,- virtualConsole: this.getVirtualConsole(),- pretendToBeVisual: true,- });-- // add some stubs in place of missing API so processing won't fail- Object.defineProperty(window, 'matchMedia', {- writable: true,- value: (query: unknown): any => ({- matches: false,- media: query,- onchange: null,- addListener: () => {},- removeListener: () => {},- addEventListener: () => {},- removeEventListener: () => {},- dispatchEvent: () => {},- }),- });- window.document.createRange = () => {- const range = new window.Range();- range.getBoundingClientRect = () => ({}) as any;- range.getClientRects = () => ({ item: () => null as any, length: 0 }) as any;- return range;- };-- if (this.#runScripts) {- try {- await addTimeoutToPromise(- async () => {- return new Promise<void>((resolve) => {- window.addEventListener(- 'load',- () => {- resolve();- },- false,- );- }).catch();- },- 10_000,- 'Window.load event not fired after 10 seconds.',- ).catch();- } catch (e) {- this.log.debug((e as Error).message);- }- }-- return {- window,- get body() {- return window.document.documentElement.outerHTML;- },- get document() {- return window.document;- },- };- } catch (err) {- if (err instanceof NavigationSkippedError) {- return {- get window(): DOMWindow {- throw new NavigationSkippedError(- 'The `window` property is not available - `skipNavigation` was used',- { cause: err },- );- },- get body(): string {- throw new NavigationSkippedError(- 'The `body` property is not available - `skipNavigation` was used',- { cause: err },- );- },- get document(): Document {- throw new NavigationSkippedError(- 'The `document` property is not available - `skipNavigation` was used',- { cause: err },- );- },- };- }-- throw err;- }- }-- private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { body: string; window: DOMWindow }) {- const addRequests = crawlingContext.addRequests;-- const extractLinks = async (options?: ExtractLinksOptions): Promise<string[]> => {- if (!crawlingContext.window) {- throw new Error('Cannot extract links because the JSDOM is not available.');- }-- return extractUrlsFromWindow(- crawlingContext.window,- options?.selector ?? 'a',- options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url,- );- };-- return {- extractLinks,- enqueueLinks: async (options: EnqueueLinksOptions = {}) => {- const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({- enqueueStrategy: options.strategy,- finalRequestUrl: crawlingContext.request.loadedUrl,- originalRequestUrl: crawlingContext.request.url,- userProvidedBaseUrl: options.baseUrl,- });-- const urls = await extractLinks(options);-- return addRequests(urls, {- ...options,- baseUrl,- strategy: options.strategy ?? EnqueueStrategy.SameHostname,- });- },- async waitForSelector(selector: string, timeoutMs = 5_000) {- const cheerio = await import('cheerio');- const $ = cheerio.load(crawlingContext.body);-- if ($(selector).get().length === 0) {- if (timeoutMs) {- await sleep(50);- await this.waitForSelector(selector, Math.max(timeoutMs - 50, 0));- return;- }-- throw new Error(`Selector '${selector}' not found.`);- }- },- async parseWithCheerio(selector?: string, _timeoutMs = 5_000) {- const cheerio = await import('cheerio');- const $ = cheerio.load(crawlingContext.body);-- if (selector && $(selector).get().length === 0) {- throw new Error(`Selector '${selector}' not found.`);- }-- return $;- },- };- }-}--/**- * Extracts URLs from a given Window object.- * @ignore- */-function extractUrlsFromWindow(window: DOMWindow, selector: string, baseUrl: string): string[] {- return Array.from(window.document.querySelectorAll(selector))- .map((e: any) => e.href)- .filter((href) => href !== undefined && href !== '')- .map((href: string | undefined) => {- if (href === undefined) {- return undefined;- }- return tryAbsoluteURL(href, baseUrl);- })- .filter((href) => href !== undefined && href !== '') as string[];}/**diff --git a/packages/jsdom-crawler/src/internals/jsdom-parser.ts b/packages/jsdom-crawler/src/internals/jsdom-parser.tsnew file mode 100644index 000000000..e96eb48b7--- /dev/null+++ b/packages/jsdom-crawler/src/internals/jsdom-parser.ts@@ -0,0 +1,138 @@+import type { DOMParser, InternalHttpCrawlingContext } from '@crawlee/http';+import type { CrawleeLogger } from '@crawlee/types';+import { tryAbsoluteURL } from '@crawlee/utils/internal';+import type { DOMWindow, VirtualConsole } from 'jsdom';+import { JSDOM, ResourceLoader } from 'jsdom';++import { addTimeoutToPromise } from '@apify/timeout';++const resources = new ResourceLoader({+ // Copy from /packages/browser-pool/src/abstract-classes/browser-plugin.ts:17+ // in order not to include the entire package here+ userAgent:+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',+});++export interface JSDOMParseResult {+ window: DOMWindow;+ document: Document;+ body: string;+}++export interface JsdomParserOptions {+ /**+ * Download and run scripts.+ */+ runScripts?: boolean;++ /**+ * Resolves the `VirtualConsole` to hand to JSDOM. Called for every parse, so a crawler can create its console+ * lazily. Defaults to JSDOM's own console handling.+ */+ virtualConsole?: () => VirtualConsole;++ /**+ * Resolves the logger for JSDOM diagnostics. Called only when there is something to log, so a crawler can hand+ * over its own logger.+ */+ log?: () => CrawleeLogger;+}++/**+ * A {@apilink DOMParser} backed by [jsdom](https://www.npmjs.com/package/jsdom). Pass it to a+ * {@apilink DOMCrawler} to get the crawling context {@apilink JSDOMCrawler} provides.+ */+export function jsdomParser(options: JsdomParserOptions = {}): DOMParser<JSDOMParseResult> {+ const { runScripts = false, virtualConsole, log } = options;++ return {+ placeholderMembers: { window: true, document: true, body: true },+ mutable: runScripts,+ async parse(context: InternalHttpCrawlingContext) {+ const isXml = context.contentType.type.includes('xml');++ // TODO handle non-string+ const { window } = new JSDOM(context.body.toString(), {+ url: context.response.url,+ contentType: isXml ? 'text/xml' : 'text/html',+ runScripts: runScripts ? 'dangerously' : undefined,+ resources,+ virtualConsole: virtualConsole?.(),+ pretendToBeVisual: true,+ });++ // add some stubs in place of missing API so processing won't fail+ Object.defineProperty(window, 'matchMedia', {+ writable: true,+ value: (query: unknown): any => ({+ matches: false,+ media: query,+ onchange: null,+ addListener: () => {},+ removeListener: () => {},+ addEventListener: () => {},+ removeEventListener: () => {},+ dispatchEvent: () => {},+ }),+ });+ window.document.createRange = () => {+ const range = new window.Range();+ range.getBoundingClientRect = () => ({}) as any;+ range.getClientRects = () => ({ item: () => null as any, length: 0 }) as any;+ return range;+ };++ if (runScripts) {+ try {+ await addTimeoutToPromise(+ async () => {+ return new Promise<void>((resolve) => {+ window.addEventListener(+ 'load',+ () => {+ resolve();+ },+ false,+ );+ }).catch();+ },+ 10_000,+ 'Window.load event not fired after 10 seconds.',+ ).catch();+ } catch (e) {+ log?.().debug((e as Error).message);+ }+ }++ return {+ window,+ get body() {+ return window.document.documentElement.outerHTML;+ },+ get document() {+ return window.document;+ },+ };+ },+ extractLinks: ({ window }, selector, baseUrl) => extractUrlsFromWindow(window, selector, baseUrl),+ select: ({ document }, selector) => document.querySelectorAll(selector),+ cleanup: ({ window }) => window.close(),+ };+}++/**+ * Extracts URLs from a given Window object.+ * @ignore+ */+function extractUrlsFromWindow(window: DOMWindow, selector: string, baseUrl: string): string[] {+ return Array.from(window.document.querySelectorAll(selector))+ .map((e: any) => e.href)+ .filter((href) => href !== undefined && href !== '')+ .map((href: string | undefined) => {+ if (href === undefined) {+ return undefined;+ }+ return tryAbsoluteURL(href, baseUrl);+ })+ .filter((href) => href !== undefined && href !== '') as string[];+}diff --git a/packages/linkedom-crawler/src/index.ts b/packages/linkedom-crawler/src/index.tsindex ab8cc478d..ce2e96b04 100644--- a/packages/linkedom-crawler/src/index.ts+++ b/packages/linkedom-crawler/src/index.ts@@ -1,2 +1,3 @@export * from '@crawlee/http';export * from './internals/linkedom-crawler.js';+export type { LinkeDOMParseResult } from './internals/linkedom-parser.js';diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.tsindex adcd7b4ea..cd1b1e627 100644--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts@@ -1,13 +1,9 @@import type {- AddRequestsBatchedResult,- ContextPipeline,CrawlingContext,- EnqueueLinksOptions,+ DOMCrawlingContext,ErrorHandler,- ExtractLinksOptions,GetUserDataFromRequest,HttpCrawlerOptions,- InternalHttpCrawlingContext,InternalHttpHook,RequestHandler,RouterHandler,@@ -15,19 +11,11 @@ import type {RouteSchemas,RoutesFromSchemas,} from '@crawlee/http';-import {- EnqueueStrategy,- HttpCrawler,- NavigationSkippedError,- resolveBaseUrlForEnqueueLinksFiltering,- Router,-} from '@crawlee/http';+import { DOMCrawler, Router } from '@crawlee/http';import type { Dictionary } from '@crawlee/types';-import type { CheerioAPI } from 'cheerio';-import { sleep } from '@crawlee/utils';-import { tryAbsoluteURL } from '@crawlee/utils/internal';-import * as cheerio from 'cheerio';-import { DOMParser } from 'linkedom/cached';++import type { LinkeDOMParseResult } from './linkedom-parser.js';+import { linkedomParser } from './linkedom-parser.js';export type LinkeDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler@@ -58,54 +46,7 @@ export type LinkeDOMHook<export interface LinkeDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> extends InternalHttpCrawlingContext<UserData, JSONData> {- window: Window;- // Technically the document is not of type Document but of type either HTMLDocument or XMLDocument- // from linkedom/types/{html/xml}/document, depending on the content type of the response- // Using union of the real types would make writing the crawlers inconvenient,- // so we specify the type as the native Document type from lib.dom.d.ts- // even though it's not technically 100% correct- document: Document;-- /**- * Wait for an element matching the selector to appear.- * Timeout defaults to 5s.- *- * **Example usage:**- * ```ts- * async requestHandler({ waitForSelector, parseWithCheerio }) {- * await waitForSelector('article h1');- * const $ = await parseWithCheerio();- * const title = $('title').text();- * });- * ```- */- waitForSelector(selector: string, timeoutMs?: number): Promise<void>;-- /**- * Returns Cheerio handle, allowing to work with the data same way as with {@apilink CheerioCrawler}.- * When provided with the `selector` argument, it will first look for the selector with a 5s timeout.- *- * **Example usage:**- * ```javascript- * async requestHandler({ parseWithCheerio }) {- * const $ = await parseWithCheerio();- * const title = $('title').text();- * });- * ```- */- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;-- /**- * Extracts URLs from the parsed DOM, without adding them to the request queue.- */- extractLinks(options?: ExtractLinksOptions): Promise<string[]>;-- /**- * Helper function for extracting URLs from the parsed DOM and adding them to the request queue.- */- enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;-}+> extends DOMCrawlingContext<LinkeDOMParseResult, UserData, JSONData> {}export type LinkeDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler@@ -193,9 +134,7 @@ export class LinkeDOMCrawler<GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>>,StatisticStateExtension extends object = {},-> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {- static #parser = new DOMParser();-+> extends DOMCrawler<LinkeDOMParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {constructor(options: LinkeDOMCrawlerOptions<ContextExtension,@@ -206,143 +145,8 @@ export class LinkeDOMCrawler<StatisticStateExtension> = {},) {- const { contextPipelineBuilder, ...rest } = options;-- super({- ...rest,- contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),- });- }-- protected override buildContextPipeline(): ContextPipeline<CrawlingContext, LinkeDOMCrawlingContext> {- return super- .buildContextPipeline()- .compose({- action: async (context) => this.parseContent(context),- })- .compose({ action: async (context) => this.addHelpers(context) });- }-- private async parseContent(crawlingContext: InternalHttpCrawlingContext) {- try {- const isXml = crawlingContext.contentType.type.includes('xml');- const document = LinkeDOMCrawler.#parser.parseFromString(- crawlingContext.body.toString(),- isXml ? 'text/xml' : 'text/html',- );-- return {- window: document.defaultView,- get body() {- return document.documentElement.outerHTML;- },- get document() {- // See comment about typing in LinkeDOMCrawlingContext definition- return document as unknown as Document;- },- };- } catch (err) {- if (err instanceof NavigationSkippedError) {- return {- get window(): Window {- throw new NavigationSkippedError(- 'The `window` property is not available - `skipNavigation` was used',- { cause: err },- );- },- get body(): string {- throw new NavigationSkippedError(- 'The `body` property is not available - `skipNavigation` was used',- { cause: err },- );- },- get document(): Document {- throw new NavigationSkippedError(- 'The `document` property is not available - `skipNavigation` was used',- { cause: err },- );- },- };- }-- throw err;- }+ super({ ...options, parser: linkedomParser() });}-- private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { body: string; window: Window }) {- const addRequests = crawlingContext.addRequests;-- const extractLinks = async (options?: ExtractLinksOptions): Promise<string[]> => {- if (!crawlingContext.window) {- throw new Error('Cannot extract links because the DOM is not available.');- }-- return extractUrlsFromWindow(- crawlingContext.window,- options?.selector ?? 'a',- options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url,- );- };-- return {- extractLinks,- enqueueLinks: async (options: EnqueueLinksOptions = {}) => {- const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({- enqueueStrategy: options.strategy,- finalRequestUrl: crawlingContext.request.loadedUrl,- originalRequestUrl: crawlingContext.request.url,- userProvidedBaseUrl: options.baseUrl,- });-- const urls = await extractLinks(options);-- return addRequests(urls, {- ...options,- baseUrl,- strategy: options.strategy ?? EnqueueStrategy.SameHostname,- });- },- async waitForSelector(selector: string, timeoutMs = 5_000) {- const $ = cheerio.load(crawlingContext.body);-- if ($(selector).get().length === 0) {- if (timeoutMs) {- await sleep(50);- await this.waitForSelector(selector, Math.max(timeoutMs - 50, 0));- return;- }-- throw new Error(`Selector '${selector}' not found.`);- }- },- async parseWithCheerio(selector?: string, _timeoutMs = 5_000) {- const $ = cheerio.load(crawlingContext.body);-- if (selector && $(selector).get().length === 0) {- throw new Error(`Selector '${selector}' not found.`);- }-- return $;- },- };- }-}--/**- * Extracts URLs from a given Window object.- * @ignore- */-function extractUrlsFromWindow(window: Window, selector: string, baseUrl: string): string[] {- return Array.from(window.document.querySelectorAll(selector))- .map((e: any) => e.href)- .filter((href) => href !== undefined && href !== '')- .map((href: string | undefined) => {- if (href === undefined) {- return undefined;- }- return tryAbsoluteURL(href, baseUrl);- })- .filter((href) => href !== undefined && href !== '') as string[];}/**diff --git a/packages/linkedom-crawler/src/internals/linkedom-parser.ts b/packages/linkedom-crawler/src/internals/linkedom-parser.tsnew file mode 100644index 000000000..4bdf23c24--- /dev/null+++ b/packages/linkedom-crawler/src/internals/linkedom-parser.ts@@ -0,0 +1,60 @@+import type { DOMParser, InternalHttpCrawlingContext } from '@crawlee/http';+import { tryAbsoluteURL } from '@crawlee/utils/internal';+import { DOMParser as LinkeDOMParser } from 'linkedom/cached';++export interface LinkeDOMParseResult {+ window: Window;+ // Technically the document is not of type Document but of type either HTMLDocument or XMLDocument+ // from linkedom/types/{html/xml}/document, depending on the content type of the response+ // Using union of the real types would make writing the crawlers inconvenient,+ // so we specify the type as the native Document type from lib.dom.d.ts+ // even though it's not technically 100% correct+ document: Document;+ body: string;+}++/**+ * A {@apilink DOMParser} backed by [linkedom](https://www.npmjs.com/package/linkedom). Pass it to a+ * {@apilink DOMCrawler} to get the crawling context {@apilink LinkeDOMCrawler} provides.+ */+export function linkedomParser(): DOMParser<LinkeDOMParseResult> {+ const parser = new LinkeDOMParser();++ return {+ placeholderMembers: { window: true, document: true, body: true },+ parse(context: InternalHttpCrawlingContext) {+ const isXml = context.contentType.type.includes('xml');+ const document = parser.parseFromString(context.body.toString(), isXml ? 'text/xml' : 'text/html');++ return {+ window: document.defaultView as unknown as Window,+ get body() {+ return document.documentElement.outerHTML;+ },+ get document() {+ // See comment about typing in LinkeDOMParseResult definition+ return document as unknown as Document;+ },+ };+ },+ extractLinks: ({ window }, selector, baseUrl) => extractUrlsFromWindow(window, selector, baseUrl),+ select: ({ document }, selector) => document.querySelectorAll(selector),+ };+}++/**+ * Extracts URLs from a given Window object.+ * @ignore+ */+function extractUrlsFromWindow(window: Window, selector: string, baseUrl: string): string[] {+ return Array.from(window.document.querySelectorAll(selector))+ .map((e: any) => e.href)+ .filter((href) => href !== undefined && href !== '')+ .map((href: string | undefined) => {+ if (href === undefined) {+ return undefined;+ }+ return tryAbsoluteURL(href, baseUrl);+ })+ .filter((href) => href !== undefined && href !== '') as string[];+}diff --git a/test/core/crawlers/dom_crawler.test.ts b/test/core/crawlers/dom_crawler.test.tsindex 458d09cd3..279530be8 100644--- a/test/core/crawlers/dom_crawler.test.ts+++ b/test/core/crawlers/dom_crawler.test.ts@@ -2,9 +2,29 @@ import http from 'node:http';import type { AddressInfo } from 'node:net';import { MemoryStorageBackend, serviceLocator } from '@crawlee/core';+import type { DOMParser } from '@crawlee/http';+import { DOMCrawler } from '@crawlee/http';import { JSDOMCrawler } from '@crawlee/jsdom';import { LinkeDOMCrawler } from '@crawlee/linkedom';+interface FakeParseResult {+ title: string;+ body: string;+}++function fakeParser(): DOMParser<FakeParseResult> {+ return {+ placeholderMembers: { title: true, body: true },+ parse: (context) => {+ const body = context.body.toString();+ return { title: /<title>(.*?)<\/title>/.exec(body)?.[1] ?? '', body };+ },+ extractLinks: (parsed, _selector, baseUrl) =>+ [...parsed.body.matchAll(/<a href="([^"]+)"/g)].map(([, href]) => new URL(href, baseUrl).href),+ select: (parsed, selector) => (selector === 'title' && parsed.title ? [parsed.title] : []),+ };+}+const router = new Map<string, http.RequestListener>();router.set('/', (req, res) => {res.setHeader('content-type', 'text/html; charset=utf-8');@@ -97,3 +117,47 @@ test('LinkeDOMCrawler enqueueLinks should respect maxCrawlDepth', async () => {expect(titles).toEqual(['Depth 0', 'Depth 1']);});++test('DOMCrawler works with skipNavigation', async () => {+ const errors: string[] = [];+ const failed: string[] = [];++ const crawler = new DOMCrawler({+ parser: fakeParser(),+ maxRequestRetries: 0,+ failedRequestHandler: ({ request }, error) => {+ failed.push(`${request.url}: ${error.message}`);+ },+ requestHandler: (context) => {+ try {+ void context.title;+ } catch (error) {+ errors.push((error as Error).message);+ }+ },+ });++ await crawler.run([{ url, skipNavigation: true }]);++ expect(failed).toStrictEqual([]);+ expect(errors).toStrictEqual(['The `title` property is not available - `skipNavigation` was used']);+});++test('DOMCrawler exposes the members of the parser it is given', async () => {+ const titles: string[] = [];++ const crawler = new DOMCrawler({+ parser: fakeParser(),+ maxCrawlDepth: 1,+ maxRequestsPerCrawl: 10, // to avoid accidental runaway+ requestHandler: async ({ waitForSelector, parseWithCheerio, enqueueLinks }) => {+ await waitForSelector('title');+ titles.push((await parseWithCheerio())('title').text());+ await enqueueLinks();+ },+ });++ await crawler.run([`${url}/depth-0`]);++ expect(titles).toEqual(['Depth 0', 'Depth 1']);+});