Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(utils): skip <template> content in htmlToText (#4399)
SwastikTripathi1ef2a93bc3ac
diff --git a/packages/utils/src/internals/cheerio.ts b/packages/utils/src/internals/cheerio.tsindex c8fb4f631..d338370bd 100644--- a/packages/utils/src/internals/cheerio.ts+++ b/packages/utils/src/internals/cheerio.ts@@ -7,7 +7,7 @@ export type { CheerioAPI, Cheerio } from 'cheerio';export type { Element } from 'domhandler';// NOTE: We are skipping 'noscript' since it's content is evaluated as text, instead of HTML elements. That damages the results.-const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript)$/i;+const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript|template)$/i;const BLOCK_TAGS_REGEX =/^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|dl|dt|dd|div|fieldset|legend|form|table|caption|tr|select|option|hr|article|aside|section|nav|header|footer|main|hgroup|figure|figcaption|details|summary|dialog|menu)$/i;diff --git a/test/utils/cheerio.test.ts b/test/utils/cheerio.test.tsindex 6f1c6fe47..9eb785346 100644--- a/test/utils/cheerio.test.ts+++ b/test/utils/cheerio.test.ts@@ -71,6 +71,11 @@ describe('htmlToText()', () => {await checkHtmlToText('<SCRIPT>Skip_scripts();</SCRIPT><div>Div1</div>', 'Div1');await checkHtmlToText('<svg>Skip svg</svg><div>Div1</div>', 'Div1');await checkHtmlToText('<canvas>Skip canvas</canvas><div>Div1</div>', 'Div1');+ await checkHtmlToText('<template><div>Skip {{ template }}</div></template><div>Div1</div>', 'Div1');+ await checkHtmlToText(+ '<ul><template x-for="item in items"><li x-text="item.name">Placeholder</li></template><li>Item 1</li></ul>',+ 'Item 1',+ );await checkHtmlToText('<b>A B C D E\n\nF G</b>', 'A B C D E F G');await checkHtmlToText('<pre>A B C D E\n\nF G</pre>', 'A B C D E\n\nF G');