Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
perf(utils): make htmlToText linear in the number of nodes (#4322)
Martin Adámek4030aa95239b
diff --git a/packages/utils/src/internals/cheerio.ts b/packages/utils/src/internals/cheerio.tsindex a88e4bf81..50130400e 100644--- a/packages/utils/src/internals/cheerio.ts+++ b/packages/utils/src/internals/cheerio.ts@@ -44,7 +44,15 @@ export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Proif (!htmlOrCheerioElement) return '';const $ = typeof htmlOrCheerioElement === 'function' ? htmlOrCheerioElement : load(htmlOrCheerioElement);- let text = '';+ // Never read the accumulated text mid-loop: each read flattens V8's cons-string, making the traversal quadratic+ const parts: string[] = [];+ let last = '';++ const push = (part: string) => {+ if (!part) return;+ parts.push(part);+ last = part[part.length - 1];+ };const process = (elems: Dictionary) => {const len = elems ? elems.length : 0;@@ -58,21 +66,21 @@ export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Procompr = elem.parent.children?.[0] === elem ? elem.data.replace(/^\r?\n/, '') : elem.data;} else compr = elem.data.replace(/\s+/g, ' ');// If text is empty or ends with a whitespace, don't add the leading whitespace- if (compr.startsWith(' ') && /(^|\s)$/.test(text)) compr = compr.substring(1);- text += compr;+ if (compr.startsWith(' ') && (last === '' || /\s/.test(last))) compr = compr.substring(1);+ push(compr);} else if (elem.type === 'comment' || SKIP_TAGS_REGEX.test(elem.tagName)) {// Skip comments and special elements} else if (elem.tagName === 'br') {- text += '\n';+ push('\n');} else if (elem.tagName === 'td') {process(elem.children);- text += '\t';+ push('\t');} else {// Block elements must be surrounded by newlines (unless beginning of text)const isBlockTag = BLOCK_TAGS_REGEX.test(elem.tagName);- if (isBlockTag && !/(^|\n)$/.test(text)) text += '\n';+ if (isBlockTag && last !== '' && last !== '\n') push('\n');process(elem.children);- if (isBlockTag && !text.endsWith('\n')) text += '\n';+ if (isBlockTag && last !== '\n') push('\n');}}};@@ -81,7 +89,7 @@ export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Proconst $body = $('body');process($body.length > 0 ? $body : $.root());- return text.trim();+ return parts.join('').trim();}/**diff --git a/test/utils/cheerio.test.ts b/test/utils/cheerio.test.tsindex 7075aa9c0..0e89a2034 100644--- a/test/utils/cheerio.test.ts+++ b/test/utils/cheerio.test.ts@@ -107,6 +107,15 @@ describe('htmlToText()', () => {await checkHtmlToText(html, text, true);});+ test('handles documents with many nodes in linear time', async () => {+ const count = 160_000;+ const html = `<html><body><div>${'<i> a</i>'.repeat(count)}</div></body></html>`;+ const start = performance.now();+ const text = await htmlToText(html);+ expect(performance.now() - start).toBeLessThan(2_000);+ expect(text).toEqual('a'.repeat(count).split('').join(' '));+ });+test('works with Cheerio object', async () => {const html1 = '<html><body>Some text</body></html>';await checkHtmlToText(cheerio.load(html1), 'Some text');