Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix CheerioCrawler when other than Node supported encoding is used
Ondra Urban1dea5c80dfb2
diff --git a/package.json b/package.jsonindex 801d64ac2..578dde728 100644--- a/package.json+++ b/package.json@@ -57,6 +57,7 @@"express": "^4.17.1","fs-extra": "^8.1.0","htmlparser2": "^3.10.1",+ "iconv-lite": "^0.5.0","jquery": "^3.4.1","mime-types": "^2.1.24","proxy-chain": "^0.2.7",diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 8e3f3e472..32bc4fc0b 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -2,6 +2,7 @@import _ from 'underscore';import cheerio from 'cheerio';import htmlparser from 'htmlparser2';+import * as iconv from 'iconv-lite';import util from 'util';import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';@@ -355,7 +356,7 @@ class CheerioCrawler {*/async _handleRequestFunction({ request, autoscaledPool }) {if (this.prepareRequestFunction) await this.prepareRequestFunction({ request });- const { dom, isXmlOrHtml, body, contentType, responseStream: response } = await addTimeoutToPromise(+ const { dom, isXml, body, contentType, response } = await addTimeoutToPromise(this._requestFunction({ request }),this.requestTimeoutMillis,`CheerioCrawler: request timed out after ${this.requestTimeoutMillis / 1000} seconds.`,@@ -363,14 +364,14 @@ class CheerioCrawler {request.loadedUrl = response.url;- const $ = dom ? cheerio.load(dom, { xmlMode: isXmlOrHtml }) : null;+ const $ = dom ? cheerio.load(dom, { xmlMode: isXml }) : null;const context = {$,// Using a getter here not to break the original API// and lazy load the HTML only when needed.get html() {log.deprecated('CheerioCrawler: The "html" parameter of handlePageFunction is deprecated, use "body" instead.');- return dom && !isXmlOrHtml ? $.html({ decodeEntities: false }) : undefined;+ return dom && !isXml && $.html({ decodeEntities: false });},get json() {if (contentType.type !== 'application/json') return null;@@ -379,7 +380,7 @@ class CheerioCrawler {},get body() {if (dom) {- return isXmlOrHtml ? $.xml() : $.html({ decodeEntities: false });+ return isXml ? $.xml() : $.html({ decodeEntities: false });}return body;},@@ -416,11 +417,14 @@ class CheerioCrawler {throw e;}}+const { statusCode } = responseStream;- const contentType = parseContentTypeFromResponse(responseStream);- const { type, parameters: { encoding } } = contentType;+ const { type, charset } = parseContentTypeFromResponse(responseStream);+ const { response, encoding } = this._encodeResponse(request, responseStream, charset);+ const contentType = { type, encoding };+if (statusCode >= 500) {- const body = await readStreamToString(responseStream, encoding);+ const body = await readStreamToString(response, encoding);// Errors are often sent as JSON, so attempt to parse them,// despite Accept header being set to text/html.@@ -434,11 +438,11 @@ class CheerioCrawler {// It's not a JSON so it's probably some text. Get the first 100 chars of it.throw new Error(`CheerioCrawler: ${statusCode} - Internal Server Error: ${body.substr(0, 100)}`);} else if (type === 'text/html' || type === 'application/xhtml+xml' || type === 'application/xml') {- const dom = await this._parseHtmlToDom(responseStream);- return ({ dom, isXmlOrHtml: type.includes('xml'), responseStream, contentType });+ const dom = await this._parseHtmlToDom(response);+ return ({ dom, isXml: type.includes('xml'), response, contentType });} else {- const body = await concatStreamToBuffer(responseStream);- return { body, responseStream, contentType };+ const body = await concatStreamToBuffer(response);+ return { body, response, contentType };}}@@ -501,6 +505,26 @@ class CheerioCrawler {return null;}+ _encodeResponse(request, response, encoding) {+ if (!encoding || Buffer.isEncoding(encoding)) return { response, encoding };++ if (iconv.encodingExists(encoding)) {+ const finalEncoding = 'utf8';+ const encodeStream = iconv.encodeStream(finalEncoding);+ const decodeStream = iconv.decodeStream(encoding).on('error', err => encodeStream.emit('error', err));+ response.on('error', err => decodeStream.emit('error', err));+ const encodedResponse = response.pipe(decodeStream).pipe(encodeStream);+ encodedResponse.statusCode = response.statusCode;+ encodedResponse.headers = response.headers;+ encodedResponse.url = response.url;+ return {+ response: encodedResponse,+ encoding: finalEncoding,+ };+ }++ throw new Error(`CheerioCrawler: Resource ${request.url} served with unsupported charset/encoding: ${encoding}`);+ }async _parseHtmlToDom(response) {return new Promise((resolve, reject) => {diff --git a/src/utils.js b/src/utils.jsindex c65df32e6..cd12ffc25 100644--- a/src/utils.js+++ b/src/utils.js@@ -636,7 +636,7 @@ export const printOutdatedSdkWarning = () => {/*** Gets parsed content type from response object* @param {Object} response - HTTP response object- * @return {Object} parsedContentType+ * @return {{ type: string, charset: string }}* @ignore*/export const parseContentTypeFromResponse = (response) => {@@ -663,7 +663,10 @@ export const parseContentTypeFromResponse = (response) => {parsedContentType = contentTypeParser.parse(contentTypeFromExtname);}- return parsedContentType;+ return {+ type: parsedContentType.type,+ charset: parsedContentType.parameters.charset,+ };};/**diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex 7c2dc96ee..927b5d59c 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -670,7 +670,7 @@ describe('CheerioCrawler', () => {const opts = crawler._getRequestOptions(request);proxies.push(opts.proxyUrl);// it needs to return something valid- return { dom: {}, responseStream: responseMock };+ return { dom: {}, response: responseMock };};const shuffled = crawler.proxyUrls;diff --git a/test/utils.test.js b/test/utils.test.jsindex 9c507e400..205fce74a 100644--- a/test/utils.test.js+++ b/test/utils.test.js@@ -933,33 +933,33 @@ describe('utils.parseContentTypeFromResponse', () => {test('should parse content type from header', () => {const parsed = utils.parseContentTypeFromResponse({ url: 'http://example.com', headers: { 'content-type': 'text/html; charset=utf-8' } });expect(parsed.type).toBe('text/html');- expect(parsed.parameters.charset).toBe('utf-8');+ expect(parsed.charset).toBe('utf-8');});test('should parse content type from file extension', () => {const parsedHtml = utils.parseContentTypeFromResponse({ url: 'http://www.example.com/foo/file.html?someparam=foo', headers: {} });expect(parsedHtml.type).toBe('text/html');- expect(parsedHtml.parameters.charset).toBe('utf-8');+ expect(parsedHtml.charset).toBe('utf-8');const parsedTxt = utils.parseContentTypeFromResponse({ url: 'http://www.example.com/foo/file.txt', headers: {} });expect(parsedTxt.type).toBe('text/plain');- expect(parsedTxt.parameters.charset).toBe('utf-8');+ expect(parsedTxt.charset).toBe('utf-8');});test('should return default content type for bad content type headers', () => {const parsedWithoutCt = utils.parseContentTypeFromResponse({ url: 'http://www.example.com/foo/file', headers: {} });expect(parsedWithoutCt.type).toBe('application/octet-stream');- expect(parsedWithoutCt.parameters.charset).toBe('utf-8');+ expect(parsedWithoutCt.charset).toBe('utf-8');const parsedBadHeader = utils.parseContentTypeFromResponse({url: 'http://www.example.com/foo/file.html',headers: { 'content-type': 'text/html,text/html' },});expect(parsedBadHeader.type).toBe('text/html');- expect(parsedBadHeader.parameters.charset).toBe('utf-8');+ expect(parsedBadHeader.charset).toBe('utf-8');const parsedReallyBad = utils.parseContentTypeFromResponse({ url: 'http://www.example.com/foo', headers: { 'content-type': 'crazy-stuff' } });expect(parsedReallyBad.type).toBe('application/octet-stream');- expect(parsedReallyBad.parameters.charset).toBe('utf-8');+ expect(parsedReallyBad.charset).toBe('utf-8');});});