Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(utils): separate HTML5 block elements and table headers in htmlToText (#4366)
SwastikTripathi589621f0d33b
diff --git a/packages/utils/src/internals/cheerio.ts b/packages/utils/src/internals/cheerio.tsindex e38d479b3..c8fb4f631 100644--- a/packages/utils/src/internals/cheerio.ts+++ b/packages/utils/src/internals/cheerio.ts@@ -9,7 +9,7 @@ export type { Element } from 'domhandler';// NOTE: We are skipping 'noscript' since it's content is evaluated as text, instead of HTML elements. That damages the results.const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript)$/i;const BLOCK_TAGS_REGEX =- /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|dl|div|fieldset|form|table|tr|select|option)$/i;+ /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|dl|dt|dd|div|fieldset|legend|form|table|caption|tr|select|option|hr|article|aside|section|nav|header|footer|main|hgroup|figure|figcaption|details|summary|dialog|menu)$/i;/*** The function converts a HTML document to a plain text.@@ -73,7 +73,7 @@ export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Pro// Skip comments and special elements} else if (elem.tagName === 'br') {push('\n');- } else if (elem.tagName === 'td') {+ } else if (elem.tagName === 'td' || elem.tagName === 'th') {process(elem.children, inPre);push('\t');} else {diff --git a/test/utils/cheerio.test.ts b/test/utils/cheerio.test.tsindex 1d8852605..6f1c6fe47 100644--- a/test/utils/cheerio.test.ts+++ b/test/utils/cheerio.test.ts@@ -100,6 +100,28 @@ describe('htmlToText()', () => {);});+ test('separates HTML5 block elements and table header cells', async () => {+ await checkHtmlToText(+ '<section>Call 555-1234</section><section>9876 Main St</section>',+ 'Call 555-1234\n9876 Main St',+ );+ await checkHtmlToText(+ '<header>Head</header><nav>Menu</nav><main><article>One</article><article>Two</article></main><aside>Side</aside><footer>Foot</footer>',+ 'Head\nMenu\nOne\nTwo\nSide\nFoot',+ );+ await checkHtmlToText('A<hr>B', 'A\nB');+ await checkHtmlToText('<dl><dt>Term</dt><dd>Definition</dd></dl>', 'Term\nDefinition');+ await checkHtmlToText(+ '<figure><img src="x.png"><figcaption>Caption</figcaption></figure>After',+ 'Caption\nAfter',+ );+ await checkHtmlToText('<details><summary>Summary</summary>Details</details>', 'Summary\nDetails');+ await checkHtmlToText(+ '<table><tr><th>Name</th><th>Phone</th></tr><tr><td>Alice</td><td>555-1234</td></tr></table>',+ 'Name\tPhone\t\nAlice\t555-1234',+ );+ });+test('handles HTML entities correctly', async () => {await checkHtmlToText('<span>á é</span>', 'á é');});