Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add option to set response encoding to CheerioCrawler (#632)

Ondra Urban84c17acceeb4
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 272a10812..7f28c7dc7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,9 @@
+NEXT
+====================
+- Add `limit` option to `Apify.utils.enqueueLinks()` for situations when full crawls are not needed.
+- Add `responseEncoding` option to `CheerioCrawler` to allow users to provide a fall-back
+ (or forced) encoding of responses in situations where websites serve invalid encoding info.
+
0.20.2 / 2020-03-09
====================
- Fix an error where persistence of `SessionPool` would fail if a cookie included invalid
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
index bde86218c..1fea91cdb 100644
--- a/src/crawlers/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -106,25 +106,6 @@ const DEFAULT_AUTOSCALED_POOL_OPTIONS = {
* @property {RequestQueue} [requestQueue]
* Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
* Either `requestList` or `requestQueue` option must be provided (or both).
- * @property {RequestAsBrowserOptions} [requestOptions]
- * Represents the options passed to the {@link requestAsBrowser} function that makes the HTTP requests to fetch the web pages.
- * Provided `requestOptions` are added to internal defaults that cannot be overridden to ensure
- * the operation of `CheerioCrawler` and all its options. Headers will not be merged,
- * use {@link RequestList} and/or {@link RequestQueue} to initialize your {@link Request} with the
- * correct headers or use `prepareRequestFunction` to modify your {@link Request} dynamically.
- * If you need more granular control over your requests, use {@link BasicCrawler}.
- *
- * The mandatory internal defaults that **CANNOT BE OVERRIDDEN** by `requestOptions`:
- * ```
- * {
- * url, // Provided by RequestList and/or RequestQueue
- * method, // Provided by RequestList and/or RequestQueue
- * headers, // Provided by RequestList and/or RequestQueue
- * payload, // Provided by RequestList and/or RequestQueue
- * strictSSL, // Use ignoreSslErrors
- * proxy, // Use useApifyProxy or proxyUrls
- * }
- * ```
* @property {PrepareRequest} [prepareRequestFunction]
* A function that executes before the HTTP request is made to the target resource.
* This function is suitable for setting dynamic properties such as cookies to the {@link Request}.
@@ -188,6 +169,24 @@ const DEFAULT_AUTOSCALED_POOL_OPTIONS = {
* An array of <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types"
* target="_blank">MIME types</a> you want the crawler to load and process.
* By default, only `text/html` and `application/xhtml+xml` MIME types are supported.
+ * @property {string} [suggestResponseEncoding]
+ * By default `CheerioCrawler` will extract correct encoding from the HTTP response headers.
+ * Sadly, there are some websites which use invalid headers. Those are encoded using the UTF-8 encoding.
+ * If those sites actually use a different encoding, the response will be corrupted. You can use
+ * `suggestResponseEncoding` to fall back to a certain encoding, if you know that your target website uses it.
+ * To force a certain encoding, disregarding the response headers, use {@link CheerioCrawlerOptions.forceResponseEncoding}
+ * ```
+ * // Will fall back to windows-1250 encoding if none found
+ * suggestResponseEncoding: 'windows-1250'
+ * ```
+ * @property {string} [forceResponseEncoding]
+ * By default `CheerioCrawler` will extract correct encoding from the HTTP response headers. Use `forceResponseEncoding`
+ * to force a certain encoding, disregarding the response headers.
+ * To only provide a default for missing encodings, use {@link CheerioCrawlerOptions.suggestResponseEncoding}
+ * ```
+ * // Will force windows-1250 encoding even if headers say otherwise
+ * forceResponseEncoding: 'windows-1250'
+ * ```
* @property {number} [maxRequestRetries=3]
* Indicates how many times the request is retried if either `requestFunction` or `handlePageFunction` fails.
* @property {number} [maxRequestsPerCrawl]
@@ -319,6 +318,8 @@ class CheerioCrawler {
apifyProxySession,
proxyUrls,
additionalMimeTypes = [],
+ suggestResponseEncoding,
+ forceResponseEncoding,
// Autoscaled pool shorthands
minConcurrency,
@@ -348,9 +349,12 @@ class CheerioCrawler {
checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe [String]');
checkParamOrThrow(prepareRequestFunction, 'options.prepareRequestFunction', 'Maybe Function');
checkParamOrThrow(additionalMimeTypes, 'options.additionalMimeTypes', '[String]');
+ checkParamOrThrow(suggestResponseEncoding, 'options.suggestResponseEncoding', 'Maybe String');
+ checkParamOrThrow(forceResponseEncoding, 'options.forceResponseEncoding', 'Maybe String');
checkParamOrThrow(useSessionPool, 'options.useSessionPool', 'Boolean');
checkParamOrThrow(sessionPoolOptions, 'options.sessionPoolOptions', 'Object');
checkParamOrThrow(persistCookiesPerSession, 'options.persistCookiesPerSession', 'Boolean');
+
// Enforce valid proxy configuration
if (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');
if (useApifyProxy && proxyUrls) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrls"!');
@@ -365,7 +369,16 @@ class CheerioCrawler {
this.supportedMimeTypes = new Set(DEFAULT_MIME_TYPES);
if (additionalMimeTypes.length) this._extendSupportedMimeTypes(additionalMimeTypes);
- this.requestOptions = requestOptions;
+
+ if (requestOptions) {
+ this.requestOptions = requestOptions;
+ log.deprecated('CheerioCrawler: options.requestOptions is deprecated. Use options.prepareRequestFunction instead.');
+ }
+
+ if (suggestResponseEncoding && forceResponseEncoding) {
+ log.warning('CheerioCrawler: Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
+ }
+
this.handlePageFunction = handlePageFunction;
this.handlePageTimeoutMillis = handlePageTimeoutSecs * 1000;
this.requestTimeoutMillis = requestTimeoutSecs * 1000;
@@ -374,6 +387,8 @@ class CheerioCrawler {
this.apifyProxyGroups = apifyProxyGroups;
this.apifyProxySession = apifyProxySession;
this.proxyUrls = _.shuffle(proxyUrls);
+ this.suggestResponseEncoding = suggestResponseEncoding;
+ this.forceResponseEncoding = forceResponseEncoding;
this.lastUsedProxyUrlIndex = 0;
this.prepareRequestFunction = prepareRequestFunction;
this.persistCookiesPerSession = persistCookiesPerSession;
@@ -603,11 +618,23 @@ class CheerioCrawler {
}
_encodeResponse(request, response, encoding) {
- if (!encoding || Buffer.isEncoding(encoding)) return { response, encoding };
+ if (this.forceResponseEncoding) {
+ encoding = this.forceResponseEncoding;
+ } else if (!encoding && this.suggestResponseEncoding) {
+ encoding = this.suggestResponseEncoding;
+ }
+
+ // Fall back to utf-8 if we still don't have encoding.
+ const utf8 = 'utf8';
+ if (!encoding) return { response, encoding: utf8 };
+
+ // This means that the encoding is one of Node.js supported
+ // encodings and we don't need to re-encode it.
+ if (Buffer.isEncoding(encoding)) return { response, encoding };
+ // Try to re-encode a variety of unsupported encodings to utf-8
if (iconv.encodingExists(encoding)) {
- const finalEncoding = 'utf8';
- const encodeStream = iconv.encodeStream(finalEncoding);
+ const encodeStream = iconv.encodeStream(utf8);
const decodeStream = iconv.decodeStream(encoding).on('error', err => encodeStream.emit('error', err));
response.on('error', err => decodeStream.emit('error', err));
const encodedResponse = response.pipe(decodeStream).pipe(encodeStream);
@@ -616,7 +643,7 @@ class CheerioCrawler {
encodedResponse.url = response.url;
return {
response: encodedResponse,
- encoding: finalEncoding,
+ encoding: utf8,
};
}
diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.js
index 8411af7f3..2813956a1 100644
--- a/test/crawlers/cheerio_crawler.test.js
+++ b/test/crawlers/cheerio_crawler.test.js
@@ -4,6 +4,8 @@ import { ENV_VARS } from 'apify-shared/consts';
import express from 'express';
import bodyParser from 'body-parser';
import sinon from 'sinon';
+import { Readable } from 'stream';
+import * as iconv from 'iconv-lite';
import log from '../../build/utils_log';
import Apify from '../../build';
import { sleep } from '../../build/utils';
@@ -501,6 +503,63 @@ describe('CheerioCrawler', () => {
});
});
+ describe('should use response encoding', () => {
+ const requestList = new Apify.RequestList({
+ sources: ['http://useless.x'],
+ });
+ const html = '<html>Žluťoučký kůň</html>';
+
+
+ test('as a fallback', async () => {
+ const suggestResponseEncoding = 'windows-1250';
+ const buf = iconv.encode(html, suggestResponseEncoding);
+ // Ensure it's really encoded.
+ expect(buf.toString('utf8')).not.toBe(html);
+
+ const crawler = new Apify.CheerioCrawler({
+ requestList,
+ handlePageFunction: async () => {},
+ suggestResponseEncoding,
+ });
+
+ const stream = Readable.from([buf]);
+
+ const { response, encoding } = crawler._encodeResponse({}, stream);
+ expect(encoding).toBe('utf8');
+ for await (const chunk of response) {
+ const string = chunk.toString('utf8');
+ expect(string).toBe(html);
+ }
+ });
+
+ test('always when using "!"', async () => {
+ const forceResponseEncoding = 'win1250';
+ const buf = iconv.encode(html, forceResponseEncoding);
+ // Ensure it's really encoded.
+ expect(buf.toString('utf8')).not.toBe(html);
+
+ const crawler = new Apify.CheerioCrawler({
+ requestList,
+ handlePageFunction: async () => {},
+ forceResponseEncoding,
+ });
+
+ const stream = Readable.from([buf]);
+
+ const { response, encoding } = crawler._encodeResponse({}, stream, 'ascii');
+ expect(encoding).toBe('utf8');
+ for await (const chunk of response) {
+ const string = chunk.toString('utf8');
+ expect(string).toBe(html);
+ }
+ });
+ });
+
+ test('should encode responses using responseEncoding', async () => {
+
+
+ });
+
describe('proxy', () => {
let requestList;
beforeEach(async () => {