Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import type {
AddRequestsBatchedResult,
CleanupRegistrar,
ContextPipeline,
CrawlingContext,
EnqueueLinksOptions,
ExtractLinksOptions,
GetUserDataFromRequest,
} from '@crawlee/basic';
import { EnqueueStrategy, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering } from '@crawlee/basic';
import type { Awaitable, Dictionary } from '@crawlee/types';
import type { CheerioAPI } from 'cheerio';
import { sleep } from '@crawlee/utils';
import type { HttpCrawlerOptions, InternalHttpCrawlingContext } from './http-crawler.js';
import { HttpCrawler } from './http-crawler.js';
/**
* The minimum a {@apilink DOMParser} has to contribute to the crawling context - the serialized document, used by
* the {@apilink DOMCrawlingContext.parseWithCheerio|`parseWithCheerio`} helper.
*/
export interface DOMParseResult {
body: string;
}
/**
* Turns a response body into a DOM representation and knows how to query it. Passing one to {@apilink DOMCrawler}
* is what makes the crawler jsdom-based, linkedom-based, or based on a DOM implementation of your own.
*
* **Example usage:**
* ```ts
* import { DOMCrawler } from 'crawlee';
* import type { DOMParser } from 'crawlee';
*
* const myParser: DOMParser<{ body: string }> = {
* // ...
* };
*
* const crawler = new DOMCrawler({
* parser: myParser,
* async requestHandler({ body }) {
* // ...
* },
* });
* ```
*/
export interface DOMParser<
Parsed extends DOMParseResult,
Selection extends ArrayLike<unknown> = ArrayLike<unknown>,
Single = unknown,
> {
/**
* The context members {@apilink DOMParser.parse|`parse`} contributes, mapped to `true`. Used to build the
* placeholders that report a helpful error when the members are accessed after `skipNavigation` - the `Record`
* type forces every key of `Parsed` to be listed, so the compiler catches an omission that would otherwise
* yield `undefined` (rather than throwing) after `skipNavigation`.
*/
readonly placeholderMembers: Record<keyof Parsed & string, true>;
parse(context: InternalHttpCrawlingContext): Awaitable<Parsed>;
/**
* Returns the URLs the `selector` matches, resolved against `baseUrl`.
*/
extractLinks(parsed: Parsed, selector: string, baseUrl: string): Awaitable<string[]>;
/**
* Returns the current matches of `selector`, in whatever form the DOM implementation uses - this is what
* {@apilink DOMCrawlingContext.querySelectorAll|`querySelectorAll`} hands to the request handler.
*/
querySelectorAll(parsed: Parsed, selector: string): Awaitable<Selection>;
/**
* Returns the first match of `selector`, or `undefined` when there is none - this is what
* {@apilink DOMCrawlingContext.querySelectorOne|`querySelectorOne`} hands to the request handler.
*/
querySelectorOne(parsed: Parsed, selector: string): Awaitable<Single | undefined>;
/**
* Whether the parse result can change after {@apilink DOMParser.parse|`parse`} returned - the case when the DOM
* implementation runs the page scripts. If it can, {@apilink DOMCrawlingContext.waitForSelector|`waitForSelector`}
* polls until the timeout elapses; otherwise it fails as soon as the selector does not match.
*/
readonly mutable?: boolean;
/**
* Returns a Cheerio handle over the parse result, for parsers that are backed by Cheerio anyway. Without it,
* {@apilink DOMCrawlingContext.parseWithCheerio|`parseWithCheerio`} parses
* {@apilink DOMParseResult.body|`body`} again.
*/
toCheerio?(parsed: Parsed): Awaitable<CheerioAPI>;
/**
* Releases whatever {@apilink DOMParser.parse|`parse`} allocated. Called after the request handler finishes or
* fails, and skipped entirely when navigation was skipped.
*/
cleanup?(parsed: Parsed): Awaitable<void>;
}
export interface DOMCrawlingHelpers<Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown> {
/**
* Extracts URLs from the parsed DOM, without adding them to the request queue.
*/
extractLinks(options?: ExtractLinksOptions): Promise<string[]>;
/**
* Helper function for extracting URLs from the parsed DOM and adding them to the request queue.
*/
enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;
/**
* Wait for an element matching the selector to appear. The `timeoutMs` only has an effect when the parser is
* {@apilink DOMParser.mutable|`mutable`} (e.g. {@apilink JSDOMCrawler} with `runScripts: true`); otherwise the
* selector is checked once and the call resolves or throws immediately.
* Timeout defaults to 5s.
*
* **Example usage:**
* ```ts
* async requestHandler({ waitForSelector, parseWithCheerio }) {
* await waitForSelector('article h1');
* const $ = await parseWithCheerio();
* const title = $('title').text();
* });
* ```
*/
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
/**
* Like {@apilink DOMCrawlingHelpers.waitForSelector|`waitForSelector`}, but returns the matches in the form
* the parser produces them - a Cheerio object in {@apilink CheerioCrawler}. Throws when nothing matches.
*/
querySelectorAll(selector: string, timeoutMs?: number): Promise<Selection>;
/**
* Like {@apilink DOMCrawlingHelpers.querySelectorAll|`querySelectorAll`}, but returns only the first match.
*/
querySelectorOne(selector: string, timeoutMs?: number): Promise<Single>;
/**
* Returns Cheerio handle, allowing to work with the data same way as with {@apilink CheerioCrawler}.
* When provided with the `selector` argument, it will throw if it's not available.
*
* **Example usage:**
* ```javascript
* async requestHandler({ parseWithCheerio }) {
* const $ = await parseWithCheerio();
* const title = $('title').text();
* });
* ```
*/
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
}
export type DOMCrawlingContext<
Parsed extends DOMParseResult = DOMParseResult,
Selection extends ArrayLike<unknown> = ArrayLike<unknown>,
Single = unknown,
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = Omit<InternalHttpCrawlingContext<UserData, JSONData>, keyof DOMCrawlingHelpers> &
Parsed &
DOMCrawlingHelpers<Selection, Single>;
export interface DOMCrawlerOptions<
Parsed extends DOMParseResult = DOMParseResult,
Selection extends ArrayLike<unknown> = ArrayLike<unknown>,
Single = unknown,
ContextExtension = Dictionary<never>,
ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<
Parsed,
Selection,
Single
> &
ContextExtension,
Routes extends Record<keyof Routes, Dictionary> = Record<string, any>,
StatisticStateExtension extends object = {},
> extends Omit<
HttpCrawlerOptions<
DOMCrawlingContext<Parsed, Selection, Single>,
ContextExtension,
ExtendedContext,
Routes,
StatisticStateExtension
>,
'contextPipelineBuilder'
> {
/**
* The DOM implementation to parse the response bodies with. Its parse result becomes part of the crawling
* context, so the members the request handler receives follow from the parser you pass.
*/
parser: DOMParser<Parsed, Selection, Single>;
}
/**
* An {@apilink HttpCrawler} that parses each response into a DOM using the {@apilink DOMCrawlerOptions.parser|`parser`}
* it is given, and exposes the parse result plus the {@apilink DOMCrawlingContext.enqueueLinks|`enqueueLinks`} and
* {@apilink DOMCrawlingContext.extractLinks|`extractLinks`} helpers on the crawling context.
*
* {@apilink JSDOMCrawler} and {@apilink LinkeDOMCrawler} are this crawler with a parser already chosen.
*
* @category Crawlers
*/
export class DOMCrawler<
Parsed extends DOMParseResult = DOMParseResult,
Selection extends ArrayLike<unknown> = ArrayLike<unknown>,
Single = unknown,
ContextExtension = Dictionary<never>,
ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<
Parsed,
Selection,
Single
> &
ContextExtension,
Routes extends Record<keyof Routes, Dictionary> = Record<
string,
GetUserDataFromRequest<DOMCrawlingContext<Parsed, Selection, Single>['request']>
>,
StatisticStateExtension extends object = {},
> extends HttpCrawler<
DOMCrawlingContext<Parsed, Selection, Single>,
ContextExtension,
ExtendedContext,
Routes,
StatisticStateExtension
> {
readonly #parser: DOMParser<Parsed, Selection, Single>;
constructor(
options: DOMCrawlerOptions<
Parsed,
Selection,
Single,
ContextExtension,
ExtendedContext,
Routes,
StatisticStateExtension
>,
) {
const { parser, ...rest } = options;
super({ ...rest, contextPipelineBuilder: () => this.buildContextPipeline() });
this.#parser = parser;
}
protected override buildContextPipeline(): ContextPipeline<
CrawlingContext,
DOMCrawlingContext<Parsed, Selection, Single>
> {
return super
.buildContextPipeline()
.compose(this.#parseContent.bind(this))
.compose(async (context) => this.#addHelpers(context));
}
async #parseContent(context: InternalHttpCrawlingContext, onCleanup: CleanupRegistrar): Promise<Parsed> {
try {
const parsed = await this.#parser.parse(context);
onCleanup(() => this.#parser.cleanup?.(parsed));
return parsed;
} catch (err) {
// The `skipNavigation` placeholders below throw on access, so there is nothing to clean up.
if (err instanceof NavigationSkippedError) {
return Object.defineProperties(
{},
Object.fromEntries(
(Object.keys(this.#parser.placeholderMembers) as (keyof Parsed & string)[]).map((member) => [
member,
{
configurable: true,
enumerable: true,
get() {
throw new NavigationSkippedError(
`The \`${member}\` property is not available - \`skipNavigation\` was used`,
{ cause: err },
);
},
},
]),
),
) as Parsed;
}
throw err;
}
}
async #addHelpers(context: InternalHttpCrawlingContext & Parsed) {
const { addRequests } = context;
const parser = this.#parser;
const extractLinks = async (options?: ExtractLinksOptions): Promise<string[]> =>
parser.extractLinks(
context,
options?.selector ?? 'a',
options?.baseUrl ?? context.request.loadedUrl ?? context.request.url,
);
// Polls `query` while `isEmpty` holds, for as long as a mutable parser allows; a static one gets one shot.
const poll = async <T>(
selector: string,
timeoutMs: number,
query: () => Awaitable<T>,
isEmpty: (result: T) => boolean,
): Promise<T> => {
let remaining = parser.mutable ? timeoutMs : 0;
let result = await query();
while (isEmpty(result)) {
if (remaining <= 0) {
throw new Error(`Selector '${selector}' not found.`);
}
await sleep(50);
remaining -= 50;
result = await query();
}
return result;
};
const querySelectorAll = async (selector: string, timeoutMs = 5_000): Promise<Selection> =>
poll(
selector,
timeoutMs,
() => parser.querySelectorAll(context, selector),
(matches) => matches.length === 0,
);
const waitForSelector = async (selector: string, timeoutMs?: number): Promise<void> => {
await querySelectorAll(selector, timeoutMs);
};
return {
extractLinks,
waitForSelector,
querySelectorAll,
querySelectorOne: async (selector: string, timeoutMs = 5_000): Promise<Single> =>
poll(
selector,
timeoutMs,
() => parser.querySelectorOne(context, selector),
(match) => match === undefined,
) as Promise<Single>,
enqueueLinks: async (options: EnqueueLinksOptions = {}) => {
const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
enqueueStrategy: options.strategy,
finalRequestUrl: context.request.loadedUrl,
originalRequestUrl: context.request.url,
userProvidedBaseUrl: options.baseUrl,
});
const urls = await extractLinks(options);
return addRequests(urls, {
...options,
baseUrl,
strategy: options.strategy ?? EnqueueStrategy.SameHostname,
});
},
async parseWithCheerio(selector?: string, timeoutMs?: number) {
if (selector) {
await waitForSelector(selector, timeoutMs);
}
// Import full cheerio (not cheerio/slim) to be browser-compliant for DOM Crawlers (not CheerioCrawler).
return (
(await parser.toCheerio?.(context)) ??
(await import('cheerio')).load(context.body, {
xmlMode: context.contentType.type.includes('xml'),
})
);
},
};
}
}