Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Development realted to Apify.utils.htmlToText(), adding tests

Jan Curn68cb0ab791ce
diff --git a/src/utils_social.js b/src/utils_social.js
index 911d8f83d..cde39e21c 100644
--- a/src/utils_social.js
+++ b/src/utils_social.js
@@ -2,6 +2,7 @@
import _ from 'underscore';
import cheerio from 'cheerio';
import log from 'apify-shared/log';
+import { htmlToText } from './utils';
// TODO: Finish docs and examples !!!
@@ -489,9 +490,7 @@ const parseHandlesFromHtml = (html, data = null) => {
const $ = cheerio.load(html, { decodeEntities: true });
if (data) data.$ = $;
- // We use ignoreHref and ignoreImage options so that the text doesn't contain links,
- // since their parts can be interpreted as e.g. phone numbers.
- const text = Apify.utils.htmlToText($);
+ const text = htmlToText($);
if (data) data.text = text;
// Find all <a> links with href tag
diff --git a/test/data/html_to_text_test.html b/test/data/html_to_text_test.html
new file mode 100644
index 000000000..87c56a06a
--- /dev/null
+++ b/test/data/html_to_text_test.html
@@ -0,0 +1,95 @@
+<html>
+<head>
+ <title>Title SHOULD NOT be converted</title>
+
+ <!-- Comments SHOULD NOT be converted -->
+</head>
+<body with='some attributes'>
+Let's start with a simple text.
+<p>
+ The ships hung in the sky, much the <a class="click" href="https://example.com/a/b/first">way that</a> bricks don't.
+</p>
+<ul>
+ <li>These aren't the Droids you're looking for</li>
+ <li some="attribute"><a href="https://example.com/a/second">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ <li><a class="click" href="https://example.com/a/b/third">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+</ul>
+
+<img src="something" alt="This should be ignored" />
+
+<!-- Comments SHOULD NOT be converted -->
+
+<table>
+ <tr class="something">
+ <td>A1</td>
+ <td attributes="are ignored">A2</td>
+ <td>A3</td>
+ </tr>
+ <tr class="something">
+ <td>B1</td>
+ <td attributes="are ignored" even="second attribute">B2</td>
+ <td>B3</td>
+ <td>B 4</td>
+ </tr>
+</table>
+
+<p>
+ This is <b>some<i> text <b>with</b></i></b> inline <span>elements</span> and HTML&nbsp;entities (&gt;bla&lt;)
+</p>
+
+<div>
+ Test<br>
+ a<br />
+ few<br>
+ line<br>
+ breaks<br>
+</div>
+
+
+
+
+ Spaces
+
+
+ in
+
+
+ an inline text should be
+
+
+ completely ignored.
+
+
+
+<pre>
+But,
+ a pre-formatted
+ block should be kept
+ pre-formatted.
+</pre>
+
+<svg>
+ These special elements SHOULD NOT BE CONVERTED.
+</svg>
+
+<script>
+ // These special elements should be completely skipped.
+ skipThis();
+</script>
+
+<style>
+ /* These special elements should be completely skipped. */
+ .skip_this {}
+</style>
+
+<canvas>
+ This should be skipped too.
+</canvas>
+
+<a class="click" href="https://another.com/a/fifth">The Greatest Science Fiction Quotes Of All Time</a>
+<p>
+ Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design,
+ just eyes. You Nexus, huh? I design your <a class="click" href="http://cool.com/">eyes</a>.
+</p>
+</body>
+</html>
diff --git a/test/utils.js b/test/utils.js
index 0b77d010c..1696dd36f 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -657,12 +657,12 @@ describe('utils.openRemoteStorage()', async () => {
});
});
-const checkHtmlToText = (html, expectedText) => {
+const checkHtmlToText = (html, expectedText, hasBody = false) => {
const text1 = utils.htmlToText(html);
expect(text1).to.eql(expectedText);
// Test embedding into <body> gives the same result
- if (typeof html === 'string') {
+ if (typeof html === 'string' && !hasBody) {
const html2 = `
<html>
<head>
@@ -745,8 +745,36 @@ describe('utils.htmlToText()', () => {
checkHtmlToText('<span>&aacute; &eacute;</span>', 'á é');
});
- xit('handles complex HTML document', () => {
- // TODO: Test some longer HTML document
+ it('handles larger HTML documents', () => {
+ const html1 = fs.readFileSync(path.join(__dirname, 'data', 'html_to_text_test.html'), 'utf8');
+
+ // Careful here - don't change spaces in the text below or the test will break!
+ /* eslint-disable no-tabs */
+ checkHtmlToText(
+ html1,
+ `Let's start with a simple text.
+The ships hung in the sky, much the way that bricks don't.
+These aren't the Droids you're looking for
+I'm sorry, Dave. I'm afraid I can't do that.
+I'm sorry, Dave. I'm afraid I can't do that.
+A1 A2 A3
+B1 B2 B3 B 4
+This is some text with inline elements and HTML entities (>bla<)
+Test
+a
+few
+line
+breaks
+Spaces in an inline text should be completely ignored.
+But,
+ a pre-formatted
+ block should be kept
+ pre-formatted.
+The Greatest Science Fiction Quotes Of All Time
+Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design, just eyes. You Nexus, huh? I design your eyes.`,
+ true,
+ );
+ /* eslint-enable no-tabs */
});
it('works with Cheerio object', () => {
@@ -757,4 +785,3 @@ describe('utils.htmlToText()', () => {
checkHtmlToText(cheerio.load(html2, { decodeEntities: true }), 'Text outside of body');
});
});
-