Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fixed parsing for bad content-type header in CheerionCrawler #498

drobnikjd24d64149391
diff --git a/package.json b/package.json
index 1db2c553b..b04b3a439 100644
--- a/package.json
+++ b/package.json
@@ -59,7 +59,7 @@
"fs-extra": "^8.1.0",
"htmlparser2": "^3.10.1",
"jquery": "^3.4.1",
- "mime": "^2.3.1",
+ "mime-types": "^2.1.24",
"proxy-chain": "^0.2.7",
"request": "^2.88.0",
"request-promise-native": "^1.0.5",
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
index dc5cb1357..4e6ca9cca 100644
--- a/src/crawlers/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -8,7 +8,7 @@ import { checkParamOrThrow } from 'apify-client/build/utils';
import contentTypeParser from 'content-type';
import { readStreamToString, concatStreamToBuffer } from 'apify-shared/streams_utilities';
import BasicCrawler from './basic_crawler';
-import { addTimeoutToPromise } from '../utils';
+import { addTimeoutToPromise, parseContentTypeFromResponse } from '../utils';
import { getApifyProxyUrl } from '../actor';
import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';
import { requestAsBrowser } from '../utils_request';
@@ -415,9 +415,9 @@ class CheerioCrawler {
throw e;
}
}
- const { statusCode, headers } = responseStream;
- const contentType = contentTypeParser.parse(headers['content-type']);
- const { type, encoding } = contentType;
+ const { statusCode } = responseStream;
+ const contentType = parseContentTypeFromResponse(responseStream);
+ const { type, parameters: { encoding } } = contentType;
if (statusCode >= 500) {
const body = await readStreamToString(responseStream, encoding);
@@ -456,9 +456,8 @@ class CheerioCrawler {
stream: true,
useCaseSensitiveHeaders: true,
abortFunction: (res) => {
- const { statusCode, headers } = res;
-
- const { type } = contentTypeParser.parse(headers['content-type']);
+ const { statusCode } = res;
+ const { type } = parseContentTypeFromResponse(res);
if (statusCode === 406) {
request.noRetry = true;
diff --git a/src/key_value_store.js b/src/key_value_store.js
index f757bc47b..eccd6f082 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -2,7 +2,7 @@ import fs from 'fs-extra';
import path from 'path';
import { promisify } from 'util';
import contentTypeParser from 'content-type';
-import mime from 'mime';
+import mime from 'mime-types';
import LruCache from 'apify-shared/lru_cache';
import { KEY_VALUE_STORE_KEY_REGEX } from 'apify-shared/regexs';
import { ENV_VARS, LOCAL_STORAGE_SUBDIRS, KEY_VALUE_STORE_KEYS } from 'apify-shared/consts';
@@ -385,7 +385,7 @@ export class KeyValueStoreLocal {
try {
const result = await this._handleFile(key, readFilePromised);
return result
- ? parseBody(result.returnValue, mime.getType(result.fileName))
+ ? parseBody(result.returnValue, mime.contentType(result.fileName))
: null;
} catch (err) {
throw new Error(`Error reading file '${key}' in directory '${this.localStoragePath}' referred by ${ENV_VARS.LOCAL_STORAGE_DIR} environment variable: ${err.message}`); // eslint-disable-line
@@ -413,7 +413,7 @@ export class KeyValueStoreLocal {
value = maybeStringify(value, optionsCopy);
const contentType = contentTypeParser.parse(optionsCopy.contentType).type;
- const extension = mime.getExtension(contentType) || DEFAULT_LOCAL_FILE_EXTENSION;
+ const extension = mime.extension(contentType) || DEFAULT_LOCAL_FILE_EXTENSION;
const filePath = this._getPath(`${key}.${extension}`);
try {
diff --git a/src/utils.js b/src/utils.js
index 41c32e80b..c65df32e6 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -15,6 +15,9 @@ import { getRandomInt } from 'apify-shared/utilities';
import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { version as apifyClientVersion } from 'apify-client/package.json';
+import mime from 'mime-types';
+import path from 'path';
+import { URL } from 'url';
import { version as apifyVersion } from '../package.json';
import { USER_AGENT_LIST } from './constants';
@@ -297,7 +300,7 @@ export const isProduction = () => process.env.NODE_ENV === 'production';
*
* @ignore
*/
-export const ensureDirExists = path => ensureDirPromised(path);
+export const ensureDirExists = dirPath => ensureDirPromised(dirPath);
/**
* Helper function that returns the first key from plan object.
@@ -630,6 +633,39 @@ export const printOutdatedSdkWarning = () => {
Read more about Apify SDK versioning at: https://help.apify.com/en/articles/3184510-updates-and-versioning-of-apify-sdk`);
};
+/**
+ * Gets parsed content type from response object
+ * @param {Object} response - HTTP response object
+ * @return {Object} parsedContentType
+ * @ignore
+ */
+export const parseContentTypeFromResponse = (response) => {
+ checkParamOrThrow(response, 'response', 'Object');
+ checkParamOrThrow(response.url, 'response.url', 'String');
+ checkParamOrThrow(response.headers, 'response.headers', 'Object');
+
+ const { url, headers } = response;
+ let parsedContentType;
+
+ if (headers['content-type']) {
+ try {
+ parsedContentType = contentTypeParser.parse(headers['content-type']);
+ } catch (err) {
+ // Can not parse content type from Content-Type header. Try to parse it from file extension.
+ }
+ }
+
+ // Parse content type from file extension as fallback
+ if (!parsedContentType) {
+ const parsedUrl = new URL(url);
+ const contentTypeFromExtname = mime.contentType(path.extname(parsedUrl.pathname))
+ || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5
+ parsedContentType = contentTypeParser.parse(contentTypeFromExtname);
+ }
+
+ return parsedContentType;
+};
+
/**
* A namespace that contains various utilities.
*
@@ -655,4 +691,5 @@ export const publicUtils = {
URL_NO_COMMAS_REGEX,
URL_WITH_COMMAS_REGEX,
createRequestDebugInfo,
+ parseContentTypeFromResponse,
};
diff --git a/test/utils.js b/test/utils.js
index bef4e48d8..b7716f4dc 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -915,3 +915,38 @@ describe('utils.printOutdatedSdkWarning()', () => {
delete process.env[ENV_VARS.SDK_LATEST_VERSION];
});
});
+
+describe('utils.parseContentTypeFromResponse', () => {
+ it('should parse content type from header', () => {
+ const parsed = utils.parseContentTypeFromResponse({ url: 'http://example.com', headers: { 'content-type': 'text/html; charset=utf-8' } });
+ expect(parsed.type).to.be.eql('text/html');
+ expect(parsed.parameters.charset).to.be.eql('utf-8');
+ });
+
+ it('should parse content type from file extension', () => {
+ const parsedHtml = utils.parseContentTypeFromResponse({ url: 'http://www.example.com/foo/file.html?someparam=foo', headers: {} });
+ expect(parsedHtml.type).to.be.eql('text/html');
+ expect(parsedHtml.parameters.charset).to.be.eql('utf-8');
+
+ const parsedTxt = utils.parseContentTypeFromResponse({ url: 'http://www.example.com/foo/file.txt', headers: {} });
+ expect(parsedTxt.type).to.be.eql('text/plain');
+ expect(parsedTxt.parameters.charset).to.be.eql('utf-8');
+ });
+
+ it('should return default content type for bad content type headers', () => {
+ const parsedWithoutCt = utils.parseContentTypeFromResponse({ url: 'http://www.example.com/foo/file', headers: {} });
+ expect(parsedWithoutCt.type).to.be.eql('application/octet-stream');
+ expect(parsedWithoutCt.parameters.charset).to.be.eql('utf-8');
+
+ const parsedBadHeader = utils.parseContentTypeFromResponse({
+ url: 'http://www.example.com/foo/file.html',
+ headers: { 'content-type': 'text/html,text/html' },
+ });
+ expect(parsedBadHeader.type).to.be.eql('text/html');
+ expect(parsedBadHeader.parameters.charset).to.be.eql('utf-8');
+
+ const parsedReallyBad = utils.parseContentTypeFromResponse({ url: 'http://www.example.com/foo', headers: { 'content-type': 'crazy-stuff' } });
+ expect(parsedReallyBad.type).to.be.eql('application/octet-stream');
+ expect(parsedReallyBad.parameters.charset).to.be.eql('utf-8');
+ });
+});