Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore(deps): update content-type to v3 and drop @types/content-type (#4200)

renovate[bot]4fcfa4d50420
diff --git a/package.json b/package.json
index 58274b083..3e31140df 100644
--- a/package.json
+++ b/package.json
@@ -86,7 +86,6 @@
"@playwright/browser-chromium": "1.63.0",
"@playwright/browser-firefox": "1.63.0",
"@playwright/browser-webkit": "1.63.0",
- "@types/content-type": "^1.1.8",
"@types/deep-equal": "^1.0.4",
"@types/domhandler": "^3.1.0",
"@types/express": "^5.0.1",
diff --git a/packages/core/package.json b/packages/core/package.json
index 9e6544f24..9953597ed 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -60,7 +60,7 @@
"@sapphire/async-queue": "^1.5.5",
"@standard-schema/spec": "^1.0.0",
"@vladfrangu/async_event_emitter": "^2.4.6",
- "content-type": "^1.0.5",
+ "content-type": "^3.1.1",
"csv-stringify": "^6.5.2",
"json5": "^2.2.3",
"mime-types": "^3.0.1",
diff --git a/packages/core/src/storages/key_value_store_codec.ts b/packages/core/src/storages/key_value_store_codec.ts
index 0479e6663..efc1c91ad 100644
--- a/packages/core/src/storages/key_value_store_codec.ts
+++ b/packages/core/src/storages/key_value_store_codec.ts
@@ -1,5 +1,5 @@
import type { Dictionary } from '@crawlee/types';
-import contentTypeParser from 'content-type';
+import { isTypeValid, parse as parseContentType } from 'content-type';
import JSON5 from 'json5';
import { jsonStringifyExtended } from '@apify/utilities';
@@ -88,16 +88,12 @@ export function parseValue(
// No content type at all → we have no basis for interpretation; hand back the raw value.
if (contentTypeHeader === null) return body;
- let contentType: string;
- let charset: BufferEncoding;
- try {
- const result = contentTypeParser.parse(contentTypeHeader);
- contentType = result.type;
- charset = result.parameters.charset as BufferEncoding;
- } catch {
- // Unparseable header → keep the original value rather than a mangled string.
- return body;
- }
+ const result = parseContentType(contentTypeHeader);
+ // Unparseable header → keep the original value rather than a mangled string.
+ if (!isTypeValid(result.type)) return body;
+
+ const contentType = result.type;
+ const charset = result.parameters.charset as BufferEncoding;
// If we can't successfully interpret it, we return the original value rather than mangling it.
if (!areDataStringifiable(contentType, charset)) return body;
diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.json
index fdb551f68..2330fa7fd 100644
--- a/packages/http-crawler/package.json
+++ b/packages/http-crawler/package.json
@@ -54,9 +54,8 @@
"@crawlee/http-client": "workspace:*",
"@crawlee/types": "workspace:*",
"@crawlee/utils": "workspace:*",
- "@types/content-type": "^1.1.8",
"cheerio": "^1.0.0",
- "content-type": "^1.0.5",
+ "content-type": "^3.1.1",
"iconv-lite": "^0.7.2",
"mime-types": "^3.0.1",
"tslib": "^2.8.1",
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 7cdd46b7f..88786ba6c 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -33,8 +33,7 @@ import { ResponseWithUrl } from '@crawlee/http-client';
import type { Awaitable, Dictionary, ISession } from '@crawlee/types';
import { parseArgument, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
import type { CheerioAPI } from 'cheerio';
-import type { RequestLike, ResponseLike } from 'content-type';
-import contentTypeParser from 'content-type';
+import { isTypeValid, parse as parseContentType } from 'content-type';
import iconv from 'iconv-lite';
import type { JsonValue } from 'type-fest';
import { z } from 'zod';
@@ -846,19 +845,19 @@ export class HttpCrawler<
/**
* Checks and extends supported mime types
*/
- #extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]) {
+ #extendSupportedMimeTypes(additionalMimeTypes: string[]) {
for (const mimeType of additionalMimeTypes) {
if (mimeType === '*/*') {
this.#supportedMimeTypes.add(mimeType);
continue;
}
- try {
- const parsedType = contentTypeParser.parse(mimeType);
- this.#supportedMimeTypes.add(parsedType.type);
- } catch (err) {
+ const parsedType = parseContentType(mimeType);
+ if (!isTypeValid(parsedType.type)) {
throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
}
+
+ this.#supportedMimeTypes.add(parsedType.type);
}
}
diff --git a/packages/http-crawler/src/internals/utils.ts b/packages/http-crawler/src/internals/utils.ts
index 09a28a0a4..5170ace5a 100644
--- a/packages/http-crawler/src/internals/utils.ts
+++ b/packages/http-crawler/src/internals/utils.ts
@@ -3,7 +3,7 @@ import { Readable } from 'node:stream';
import type { HttpRequest, HttpRequestOptions } from '@crawlee/types';
import { applySearchParams, parseArgument } from '@crawlee/utils/internal';
-import contentTypeParser from 'content-type';
+import { isTypeValid, parse as parseContentType } from 'content-type';
import mime from 'mime-types';
import { z } from 'zod';
@@ -84,11 +84,9 @@ export function parseContentTypeFromResponse(response: Response): { type: string
let parsedContentType;
if (headers.get('content-type')) {
- try {
- parsedContentType = contentTypeParser.parse(headers.get('content-type') as string);
- } catch {
- // Can not parse content type from Content-Type header. Try to parse it from file extension.
- }
+ const parsed = parseContentType(headers.get('content-type') as string);
+ // Can not parse content type from Content-Type header. Try to parse it from file extension.
+ if (isTypeValid(parsed.type)) parsedContentType = parsed;
}
// Parse content type from file extension as fallback
@@ -96,7 +94,7 @@ export function parseContentTypeFromResponse(response: Response): { type: string
const parsedUrl = new URL(url);
const contentTypeFromExtname =
mime.contentType(extname(parsedUrl.pathname)) || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5
- parsedContentType = contentTypeParser.parse(contentTypeFromExtname);
+ parsedContentType = parseContentType(contentTypeFromExtname);
}
return {
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index f1f3ce72d..378d3608c 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -274,9 +274,6 @@ importers:
'@playwright/browser-webkit':
specifier: 1.63.0
version: 1.63.0
- '@types/content-type':
- specifier: ^1.1.8
- version: 1.1.9
'@types/deep-equal':
specifier: ^1.0.4
version: 1.0.4
@@ -718,8 +715,8 @@ importers:
specifier: ^2.4.6
version: 2.4.7
content-type:
- specifier: ^1.0.5
- version: 1.0.5
+ specifier: ^3.1.1
+ version: 3.1.1
csv-stringify:
specifier: ^6.5.2
version: 6.9.0
@@ -849,15 +846,12 @@ importers:
'@crawlee/utils':
specifier: workspace:*
version: link:../utils
- '@types/content-type':
- specifier: ^1.1.8
- version: 1.1.9
cheerio:
specifier: ^1.0.0
version: 1.2.0
content-type:
- specifier: ^1.0.5
- version: 1.0.5
+ specifier: ^3.1.1
+ version: 3.1.1
iconv-lite:
specifier: ^0.7.2
version: 0.7.3
@@ -5068,9 +5062,6 @@ packages:
resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==}
- '@types/[email protected]':
- resolution: {integrity: sha512-Hq9IMnfekuOCsEmYl4QX2HBrT+XsfXiupfrLLY8Dcf3Puf4BkBOxSbWYTITSOQAhJoYPBez+b4MJRpIYL65z8A==}
-
resolution: {integrity: sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw==}
@@ -18389,8 +18380,6 @@ snapshots:
dependencies:
'@types/node': 24.19.1
- '@types/[email protected]': {}
-
'@types/[email protected]': {}
diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts
index 909285a0e..a1a85b0d0 100644
--- a/test/core/crawlers/http_crawler.test.ts
+++ b/test/core/crawlers/http_crawler.test.ts
@@ -338,6 +338,12 @@ test('invalid content type defaults to octet-stream', async () => {
]);
});
+test('rejects an unparseable mime type in additionalMimeTypes', () => {
+ expect(() => new HttpCrawler({ additionalMimeTypes: ['not a mime type'], requestHandler: async () => {} })).toThrow(
+ 'Can not parse mime type not a mime type from "options.additionalMimeTypes".',
+ );
+});
+
test('decodes charset from http-equiv meta tag when absent in HTTP header', async () => {
const results: string[] = [];