Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Bump htmlparser2 from 4.1.0 to 5.0.1 (#817)
dependabot-preview[bot]433f67e91354
diff --git a/package.json b/package.jsonindex c7036ea91..b45723960 100644--- a/package.json+++ b/package.json@@ -66,7 +66,7 @@"content-type": "^1.0.4","express": "^4.17.1","fs-extra": "^9.0.1",- "htmlparser2": "^4.1.0",+ "htmlparser2": "^5.0.1","iconv-lite": "^0.6.2","jquery": "^3.5.1","mime-types": "^2.1.27",diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 6cd1236b1..ae894681f 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -3,6 +3,7 @@ import { readStreamToString, concatStreamToBuffer } from 'apify-shared/streams_uimport * as cheerio from 'cheerio';import * as contentTypeParser from 'content-type';import * as htmlparser from 'htmlparser2';+import { WritableStream } from 'htmlparser2/lib/WritableStream';import * as iconv from 'iconv-lite';import ow from 'ow';import * as _ from 'underscore';@@ -727,7 +728,8 @@ class CheerioCrawler {if (err) reject(err);else resolve(dom);});- const parser = new htmlparser.WritableStream(domHandler, { decodeEntities: true });+ const parser = new WritableStream(domHandler, { decodeEntities: true });+ parser.on('error', reject);response.on('error', reject).pipe(parser);});}