Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import type { Dictionary } from '@crawlee/types';
import type { CheerioAPI } from 'cheerio';
import { tryAbsoluteURL } from './extract-urls.js';
export type { CheerioAPI, Cheerio } from 'cheerio';
export type { Element } from 'domhandler';
// NOTE: We are skipping 'noscript' since it's content is evaluated as text, instead of HTML elements. That damages the results.
const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript|template)$/i;
const BLOCK_TAGS_REGEX =
/^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|dl|dt|dd|div|fieldset|legend|form|table|caption|tr|select|option|hr|article|aside|section|nav|header|footer|main|hgroup|figure|figcaption|details|summary|dialog|menu)$/i;
/**
* The function converts a HTML document to a plain text.
*
* The plain text generated by the function is similar to a text captured
* by pressing Ctrl+A and Ctrl+C on a page when loaded in a web browser.
* The function doesn't aspire to preserve the formatting or to be perfectly correct with respect to HTML specifications.
* However, it attempts to generate newlines and whitespaces in and around HTML elements
* to avoid merging distinct parts of text and thus enable extraction of data from the text (e.g. phone numbers).
*
* **Example usage**
* ```javascript
* const text = htmlToText('<html><body>Some text</body></html>');
* console.log(text);
* ```
*
* Note that the function uses [cheerio](https://www.npmjs.com/package/cheerio) to parse the HTML.
* Optionally, to avoid duplicate parsing of HTML and thus improve performance, you can pass
* an existing Cheerio object to the function instead of the HTML text.
*
* ```javascript
* import * as cheerio from 'cheerio';
* const html = '<html><body>Some text</body></html>';
* const text = htmlToText(cheerio.load(html));
* ```
* @param htmlOrCheerioElement HTML text or parsed HTML represented using a [cheerio](https://www.npmjs.com/package/cheerio) function.
* @return Plain text
*/
export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Promise<string> {
const { load } = await import('cheerio/slim');
if (!htmlOrCheerioElement) return '';
const $ = typeof htmlOrCheerioElement === 'function' ? htmlOrCheerioElement : load(htmlOrCheerioElement);
// Never read the accumulated text mid-loop: each read flattens V8's cons-string, making the traversal quadratic
const parts: string[] = [];
let last = '';
const push = (part: string) => {
if (!part) return;
parts.push(part);
last = part[part.length - 1];
};
const process = (elems: Dictionary, inPre = false) => {
const len = elems ? elems.length : 0;
for (let i = 0; i < len; i++) {
const elem = elems[i];
if (elem.type === 'text') {
// Compress spaces, unless we're inside <pre> element (including nested elements like <pre><code>)
let compr;
if (inPre) {
// Strip single newline right after `<pre>` that htmlparser2 keeps
const isFirstInPre = elem.parent?.tagName === 'pre' && elem.parent.children?.[0] === elem;
compr = isFirstInPre ? elem.data.replace(/^\r?\n/, '') : elem.data;
} else compr = elem.data.replace(/\s+/g, ' ');
// If text is empty or ends with a whitespace, don't add the leading whitespace
if (!inPre && compr.startsWith(' ') && (last === '' || /\s/.test(last))) compr = compr.substring(1);
push(compr);
} else if (elem.type === 'comment' || SKIP_TAGS_REGEX.test(elem.tagName)) {
// Skip comments and special elements
} else if (elem.tagName === 'br') {
push('\n');
} else if (elem.tagName === 'td' || elem.tagName === 'th') {
process(elem.children, inPre);
push('\t');
} else {
// Block elements must be surrounded by newlines (unless beginning of text)
const isBlockTag = BLOCK_TAGS_REGEX.test(elem.tagName);
if (isBlockTag && last !== '' && last !== '\n') push('\n');
process(elem.children, inPre || elem.tagName === 'pre');
if (isBlockTag && last !== '\n') push('\n');
}
}
};
// If HTML document has body, only convert that, otherwise convert the entire HTML
const $body = $('body');
process($body.length > 0 ? $body : $.root());
return parts.join('').trim();
}
/**
* Extracts URLs from a given Cheerio object.
*
* @param $ the Cheerio object to extract URLs from
* @param selector a CSS selector for matching link elements
* @param baseUrl a URL for resolving relative links
* @throws when a relative URL is encountered with no baseUrl set
* @return An array of absolute URLs
*/
export function extractUrlsFromCheerio($: CheerioAPI, selector = 'a', baseUrl = ''): string[] {
const base = $('base').attr('href');
const absoluteBaseUrl = base && tryAbsoluteURL(base, baseUrl);
if (absoluteBaseUrl) {
baseUrl = absoluteBaseUrl;
}
return $(selector)
.map((_i, el) => $(el).attr('href'))
.get()
.filter(Boolean)
.map((href) => {
// Throw a meaningful error when only a relative URL would be extracted instead of waiting for the Request to fail later.
const isHrefAbsolute = /^[a-z][a-z0-9+.-]*:/.test(href); // Grabbed this in 'is-absolute-url' package.
if (!isHrefAbsolute && !baseUrl) {
throw new Error(
`An extracted URL: ${href} is relative and baseUrl is not set. ` +
'Provide a baseUrl to automatically resolve relative URLs.',
);
}
return baseUrl ? tryAbsoluteURL(href, baseUrl) : href;
})
.filter(Boolean) as string[];
}