Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: cheerio issues with HTML serialization (#1113)
Ondra Urbane0ae762a7e40
diff --git a/.github/workflows/check-ts-support.yml b/.github/workflows/check-ts-support.ymlindex 81ad28c50..7a576834e 100644--- a/.github/workflows/check-ts-support.yml+++ b/.github/workflows/check-ts-support.yml@@ -13,7 +13,7 @@ jobs:runs-on: ubuntu-lateststeps:- uses: actions/checkout@v2- - uses: actions/setup-node@v1+ - uses: actions/setup-node@v2with:node-version: 16registry-url: https://registry.npmjs.org/diff --git a/.github/workflows/check.yml b/.github/workflows/check.ymlindex cbac74930..28100fb1d 100644--- a/.github/workflows/check.yml+++ b/.github/workflows/check.yml@@ -21,7 +21,7 @@ jobs:steps:- uses: actions/checkout@v2- name: Use Node.js ${{ matrix.node-version }}- uses: actions/setup-node@v1+ uses: actions/setup-node@v2with:node-version: ${{ matrix.node-version }}- name: Install Dependencies@@ -34,7 +34,9 @@ jobs:runs-on: ubuntu-lateststeps:- uses: actions/checkout@v2- - uses: actions/setup-node@v1+ - uses: actions/setup-node@v2+ with:+ node-version: 16- name: Install Dependenciesrun: npm install- name: Run Linterdiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 0ef426391..4a562c63b 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -51,7 +51,7 @@ jobs:uses: actions/checkout@v2-name: Use Node.js 16- uses: actions/setup-node@v1+ uses: actions/setup-node@v2with:node-version: 16-@@ -72,7 +72,7 @@ jobs:-uses: actions/checkout@v2-- uses: actions/setup-node@v1+ uses: actions/setup-node@v2with:node-version: 16registry-url: https://registry.npmjs.org/diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 963b7b88b..0ce32f5b7 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,7 @@+2.0.2 / BETA+====================+- Fix serialization issues in `CheerioCrawler` caused by parser conflicts in recent versions of `cheerio`.+2.0.1 / 2021/08/06====================- Use `got-scraping` 2.0.1 until fully compatible.diff --git a/package.json b/package.jsonindex 471132540..3c9aff6cc 100644--- a/package.json+++ b/package.json@@ -57,7 +57,6 @@"@apify/ps-tree": "^1.1.4","@apify/storage-local": "^2.0.1","@apify/utilities": "^1.1.2",- "@types/cheerio": "^0.22.30","@types/domhandler": "^2.4.2","@types/node": "^15.14.2","@types/socket.io": "^2.1.13",diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex fb01a0aba..82e658437 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -553,7 +553,15 @@ class CheerioCrawler extends BasicCrawler {request.loadedUrl = response.url;- const $ = dom ? cheerio.load(dom, { xmlMode: isXml }) : null;+ const $ = dom+ ? cheerio.load(dom, {+ xmlMode: isXml,+ // Recent versions of cheerio use parse5 as the HTML parser/serializer. It's more strict than htmlparser2+ // and not good for scraping. It also does not have a great streaming interface.+ // Here we tell cheerio to use htmlparser2 for serialization, otherwise the conflict produces weird errors.+ _useHtmlParser2: true,+ })+ : null;crawlingContext.$ = $;crawlingContext.contentType = contentType;diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex 22f730f84..f3ecb36b7 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -31,6 +31,52 @@ const responseSamples = {+ '</item>\n'+ '</items>',image: fs.readFileSync(path.join(__dirname, 'data/apify.png')),+ html: '<!doctype html>\n'+ + '<html>\n'+ + '<head>\n'+ + ' <title>Example Domain</title>\n'+ + '\n'+ + ' <meta charset="utf-8">\n'+ + ' <meta http-equiv="Content-type" content="text/html; charset=utf-8">\n'+ + ' <meta name="viewport" content="width=device-width, initial-scale=1">\n'+ + ' <style type="text/css">\n'+ + ' body {\n'+ + ' background-color: #f0f0f2;\n'+ + ' margin: 0;\n'+ + ' padding: 0;\n'+ + ' font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n' // eslint-disable-line max-len+ + ' \n'+ + ' }\n'+ + ' div {\n'+ + ' width: 600px;\n'+ + ' margin: 5em auto;\n'+ + ' padding: 2em;\n'+ + ' background-color: #fdfdff;\n'+ + ' border-radius: 0.5em;\n'+ + ' box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);\n'+ + ' }\n'+ + ' a:link, a:visited {\n'+ + ' color: #38488f;\n'+ + ' text-decoration: none;\n'+ + ' }\n'+ + ' @media (max-width: 700px) {\n'+ + ' div {\n'+ + ' margin: 0 auto;\n'+ + ' width: auto;\n'+ + ' }\n'+ + ' }\n'+ + ' </style> \n'+ + '</head>\n'+ + '\n'+ + '<body>\n'+ + '<div>\n'+ + ' <h1>Example Domain</h1>\n'+ + ' <p>This domain is for use in illustrative examples in documents. You may use this\n'+ + ' domain in literature without prior coordination or asking for permission.</p>\n'+ + ' <p><a href="https://www.iana.org/domains/example">More information...</a></p>\n'+ + '</div>\n'+ + '</body>\n'+ + '</html>\n',};const app = express();@@ -64,6 +110,10 @@ app.get('/mirror', (req, res) => {res.send('<html><head><title>Title</title></head><body>DATA</body></html>');});+app.get('/html-type', (req, res) => {+ res.type('html').send(responseSamples.html);+});+app.get('/json-type', (req, res) => {res.json(responseSamples.json);});@@ -252,6 +302,24 @@ describe('CheerioCrawler', () => {await cheerioCrawler.run();});+ test('should serialize body and html', async () => {+ expect.assertions(2);+ const sources = [`http://${HOST}:${port}/html-type`];+ const requestList = await Apify.openRequestList(null, sources);++ const cheerioCrawler = new Apify.CheerioCrawler({+ requestList,+ maxRequestRetries: 0,+ maxConcurrency: 1,+ handlePageFunction: async ({ $, body }) => {+ expect(body).toBe(responseSamples.html);+ expect($.html()).toBe(body);+ },+ });++ await cheerioCrawler.run();+ });+describe('should timeout', () => {let ll;beforeAll(() => {diff --git a/test/typescript/crawlers/cheerio_crawler.test.ts b/test/typescript/crawlers/cheerio_crawler.test.tsindex 5a796f567..e7168e8aa 100644--- a/test/typescript/crawlers/cheerio_crawler.test.ts+++ b/test/typescript/crawlers/cheerio_crawler.test.ts@@ -50,7 +50,7 @@ describe('CheerioCrawler TS', () => {test('Can pass around and call `handler({ var }: { var: Type})`', async () => {// This form can also be easily reused as above.// Auto-completion works on defined input variables in parameter list.- const y = async ({$}: { $?: cheerio.Selector }) => {+ const y = async ({$}: { $?: cheerio.CheerioAPI }) => {expect($!('a').attr('href')).toEqual('#');};