Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

perf(utils): make htmlToText linear in the number of nodes (#4322)

Martin Adámek4030aa95239b
diff --git a/packages/utils/src/internals/cheerio.ts b/packages/utils/src/internals/cheerio.ts
index a88e4bf81..50130400e 100644
--- a/packages/utils/src/internals/cheerio.ts
+++ b/packages/utils/src/internals/cheerio.ts
@@ -44,7 +44,15 @@ export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Pro
if (!htmlOrCheerioElement) return '';
const $ = typeof htmlOrCheerioElement === 'function' ? htmlOrCheerioElement : load(htmlOrCheerioElement);
- let text = '';
+ // Never read the accumulated text mid-loop: each read flattens V8's cons-string, making the traversal quadratic
+ const parts: string[] = [];
+ let last = '';
+
+ const push = (part: string) => {
+ if (!part) return;
+ parts.push(part);
+ last = part[part.length - 1];
+ };
const process = (elems: Dictionary) => {
const len = elems ? elems.length : 0;
@@ -58,21 +66,21 @@ export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Pro
compr = elem.parent.children?.[0] === elem ? elem.data.replace(/^\r?\n/, '') : elem.data;
} else compr = elem.data.replace(/\s+/g, ' ');
// If text is empty or ends with a whitespace, don't add the leading whitespace
- if (compr.startsWith(' ') && /(^|\s)$/.test(text)) compr = compr.substring(1);
- text += compr;
+ if (compr.startsWith(' ') && (last === '' || /\s/.test(last))) compr = compr.substring(1);
+ push(compr);
} else if (elem.type === 'comment' || SKIP_TAGS_REGEX.test(elem.tagName)) {
// Skip comments and special elements
} else if (elem.tagName === 'br') {
- text += '\n';
+ push('\n');
} else if (elem.tagName === 'td') {
process(elem.children);
- text += '\t';
+ push('\t');
} else {
// Block elements must be surrounded by newlines (unless beginning of text)
const isBlockTag = BLOCK_TAGS_REGEX.test(elem.tagName);
- if (isBlockTag && !/(^|\n)$/.test(text)) text += '\n';
+ if (isBlockTag && last !== '' && last !== '\n') push('\n');
process(elem.children);
- if (isBlockTag && !text.endsWith('\n')) text += '\n';
+ if (isBlockTag && last !== '\n') push('\n');
}
}
};
@@ -81,7 +89,7 @@ export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Pro
const $body = $('body');
process($body.length > 0 ? $body : $.root());
- return text.trim();
+ return parts.join('').trim();
}
/**
diff --git a/test/utils/cheerio.test.ts b/test/utils/cheerio.test.ts
index 7075aa9c0..0e89a2034 100644
--- a/test/utils/cheerio.test.ts
+++ b/test/utils/cheerio.test.ts
@@ -107,6 +107,15 @@ describe('htmlToText()', () => {
await checkHtmlToText(html, text, true);
});
+ test('handles documents with many nodes in linear time', async () => {
+ const count = 160_000;
+ const html = `<html><body><div>${'<i> a</i>'.repeat(count)}</div></body></html>`;
+ const start = performance.now();
+ const text = await htmlToText(html);
+ expect(performance.now() - start).toBeLessThan(2_000);
+ expect(text).toEqual('a'.repeat(count).split('').join(' '));
+ });
+
test('works with Cheerio object', async () => {
const html1 = '<html><body>Some text</body></html>';
await checkHtmlToText(cheerio.load(html1), 'Some text');