Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: read content encoding hints from `<meta>` tags (#3780)

Jindřich Bär1dc8812a4abd
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index bea23a631..2c16a4e76 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -37,7 +37,7 @@ import type { JsonValue } from 'type-fest';
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
-import { parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
+import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
/**
* Default mime types, which HttpScraper supports.
@@ -643,6 +643,15 @@ export class HttpCrawler<
// It's not a JSON, so it's probably some text. Get the first 100 chars of it.
throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
} else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
+ if (!charset && !this.forceResponseEncoding) {
+ const rawBytes = Buffer.from(await response.arrayBuffer());
+ const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
+ const charsetToUse = metaCharset ?? this.suggestResponseEncoding ?? 'utf-8';
+ const body = iconv.encodingExists(charsetToUse)
+ ? iconv.decode(rawBytes, charsetToUse)
+ : rawBytes.toString('utf8');
+ return { response, contentType: { type, encoding: 'utf-8' as BufferEncoding }, body };
+ }
return { response, contentType, body: await reencodedResponse.text() };
} else {
const body = Buffer.from(await reencodedResponse.bytes());
diff --git a/packages/http-crawler/src/internals/utils.ts b/packages/http-crawler/src/internals/utils.ts
index 4111da0ea..3631571f1 100644
--- a/packages/http-crawler/src/internals/utils.ts
+++ b/packages/http-crawler/src/internals/utils.ts
@@ -59,6 +59,18 @@ export function processHttpRequestOptions({
return { ...request, body, url, headers };
}
+/**
+ * Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
+ * declared via `<meta charset>` or `<meta http-equiv="Content-Type" content="...;charset=...">`.
+ * This implements a simplified version of the HTML spec's byte-stream prescan algorithm.
+ */
+export function extractCharsetFromHtmlBytes(bytes: Buffer): string | undefined {
+ // latin1 preserves byte values for ASCII-compatible encodings, making the meta tags readable
+ const prescan = bytes.subarray(0, 1024).toString('latin1');
+ const match = /<meta[^>]+\bcharset\s*=\s*["']?\s*([^"'\s;>]+)/i.exec(prescan);
+ return match?.[1];
+}
+
/**
* Gets parsed content type from response object
* @param response HTTP response object
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index ddb1ecbaa..26307b020 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -695,6 +695,22 @@ describe('CheerioCrawler', () => {
expect(await response.text()).toBe(html);
});
+ test('via http-equiv meta tag when no charset in HTTP header', async () => {
+ let context: CheerioCrawlingContext | null = null;
+
+ const crawler = new CheerioCrawler({
+ requestHandler: (ctx) => {
+ context = ctx;
+ },
+ });
+
+ await crawler.run([`${serverAddress}/special/meta-charset`]);
+
+ context = context as unknown as CheerioCrawlingContext;
+ expect(context?.body).toContain('Žluťoučký kůň');
+ expect(context?.$('body').text()).toContain('Žluťoučký kůň');
+ });
+
test('Cheerio decodes html entities', async () => {
let context: CheerioCrawlingContext | null = null;
diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts
index b82b65fb1..d99593fef 100644
--- a/test/core/crawlers/http_crawler.test.ts
+++ b/test/core/crawlers/http_crawler.test.ts
@@ -4,6 +4,7 @@ import { Readable } from 'node:stream';
import { HttpCrawler, SessionPool } from '@crawlee/http';
import { ResponseWithUrl } from '@crawlee/http-client';
+import iconv from 'iconv-lite';
import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator.js';
const router = new Map<string, http.RequestListener>();
@@ -66,6 +67,20 @@ router.set('/403-with-octet-stream', (req, res) => {
res.end();
});
+router.set('/meta-charset', (req, res) => {
+ const text = 'Žluťoučký kůň';
+ const html = `<html><head><meta http-equiv="Content-Type" content="text/html; charset=windows-1250"></head><body>${text}</body></html>`;
+ res.setHeader('content-type', 'text/html'); // no charset in HTTP header
+ res.end(iconv.encode(html, 'windows-1250'));
+});
+
+router.set('/meta-charset-html5', (req, res) => {
+ const text = 'Žluťoučký kůň';
+ const html = `<html><head><meta charset="windows-1250"></head><body>${text}</body></html>`;
+ res.setHeader('content-type', 'text/html'); // no charset in HTTP header
+ res.end(iconv.encode(html, 'windows-1250'));
+});
+
let server: http.Server;
let url: string;
@@ -214,6 +229,36 @@ test('invalid content type defaults to octet-stream', async () => {
]);
});
+test('decodes charset from http-equiv meta tag when absent in HTTP header', async () => {
+ const results: string[] = [];
+
+ const crawler = new HttpCrawler({
+ maxRequestRetries: 0,
+ requestHandler: ({ body }) => {
+ results.push(body as string);
+ },
+ });
+
+ await crawler.run([`${url}/meta-charset`]);
+
+ expect(results[0]).toContain('Žluťoučký kůň');
+});
+
+test('decodes charset from HTML5 meta charset attribute when absent in HTTP header', async () => {
+ const results: string[] = [];
+
+ const crawler = new HttpCrawler({
+ maxRequestRetries: 0,
+ requestHandler: ({ body }) => {
+ results.push(body as string);
+ },
+ });
+
+ await crawler.run([`${url}/meta-charset-html5`]);
+
+ expect(results[0]).toContain('Žluťoučký kůň');
+});
+
test('handles cookies from redirects', async () => {
const results: string[] = [];
diff --git a/test/shared/_helper.ts b/test/shared/_helper.ts
index 45ca9ad20..13afd60d5 100644
--- a/test/shared/_helper.ts
+++ b/test/shared/_helper.ts
@@ -7,6 +7,7 @@ import bodyParser from 'body-parser';
import { entries } from 'crawlee';
import type { Application } from 'express';
import express from 'express';
+import iconv from 'iconv-lite';
export const startExpressAppPromise = async (app: Application, port: number) => {
return new Promise<Server>((resolve) => {
@@ -337,6 +338,13 @@ export async function runExampleComServer(): Promise<[Server, number]> {
res.type('html').send('&quot;&lt;&gt;"<>');
});
+ special.get('/meta-charset', (_req, res) => {
+ const text = 'Žluťoučký kůň';
+ const html = `<html><head><meta http-equiv="Content-Type" content="text/html; charset=windows-1250"></head><body>${text}</body></html>`;
+ res.setHeader('content-type', 'text/html');
+ res.end(iconv.encode(html, 'windows-1250'));
+ });
+
special.get('/set-cookie', (req, res) => {
const cookieName = (req.query.name as string) || 'testCookie';
const cookieValue = (req.query.value as string) || 'testValue';