Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(utils): keep whitespace of elements nested in <pre> in htmlToText (#4348)
Wei-Lun Lib04cf00cdb2a
diff --git a/packages/utils/src/internals/cheerio.ts b/packages/utils/src/internals/cheerio.tsindex 50130400e..e38d479b3 100644--- a/packages/utils/src/internals/cheerio.ts+++ b/packages/utils/src/internals/cheerio.ts@@ -54,32 +54,33 @@ export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Prolast = part[part.length - 1];};- const process = (elems: Dictionary) => {+ const process = (elems: Dictionary, inPre = false) => {const len = elems ? elems.length : 0;for (let i = 0; i < len; i++) {const elem = elems[i];if (elem.type === 'text') {- // Compress spaces, unless we're inside <pre> element+ // Compress spaces, unless we're inside <pre> element (including nested elements like <pre><code>)let compr;- if (elem.parent?.tagName === 'pre') {+ if (inPre) {// Strip single newline right after `<pre>` that htmlparser2 keeps- compr = elem.parent.children?.[0] === elem ? elem.data.replace(/^\r?\n/, '') : elem.data;+ const isFirstInPre = elem.parent?.tagName === 'pre' && elem.parent.children?.[0] === elem;+ compr = isFirstInPre ? elem.data.replace(/^\r?\n/, '') : elem.data;} else compr = elem.data.replace(/\s+/g, ' ');// If text is empty or ends with a whitespace, don't add the leading whitespace- if (compr.startsWith(' ') && (last === '' || /\s/.test(last))) compr = compr.substring(1);+ if (!inPre && compr.startsWith(' ') && (last === '' || /\s/.test(last))) compr = compr.substring(1);push(compr);} else if (elem.type === 'comment' || SKIP_TAGS_REGEX.test(elem.tagName)) {// Skip comments and special elements} else if (elem.tagName === 'br') {push('\n');} else if (elem.tagName === 'td') {- process(elem.children);+ process(elem.children, inPre);push('\t');} else {// Block elements must be surrounded by newlines (unless beginning of text)const isBlockTag = BLOCK_TAGS_REGEX.test(elem.tagName);if (isBlockTag && last !== '' && last !== '\n') push('\n');- process(elem.children);+ process(elem.children, inPre || elem.tagName === 'pre');if (isBlockTag && last !== '\n') push('\n');}}diff --git a/test/utils/cheerio.test.ts b/test/utils/cheerio.test.tsindex 0e89a2034..1d8852605 100644--- a/test/utils/cheerio.test.ts+++ b/test/utils/cheerio.test.ts@@ -74,6 +74,9 @@ describe('htmlToText()', () => {await checkHtmlToText('<b>A B C D E\n\nF G</b>', 'A B C D E F G');await checkHtmlToText('<pre>A B C D E\n\nF G</pre>', 'A B C D E\n\nF G');+ await checkHtmlToText('<pre><code>A B\n C</code></pre>', 'A B\n C');+ await checkHtmlToText('<pre>A <b>B\n <i>C D</i></b></pre>', 'A B\n C D');+ await checkHtmlToText('<pre><code><span>a</span><br><span> b</span></code></pre>', 'a\n b');await checkHtmlToText('<h1>Heading 1</h1><div><div><div><div>Deep Div</div></div></div></div><h2>Heading 2</h2>',