Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix: decode fetched pages with their declared <meta> charset

Mohammad Hijjawi630742b6cc47
diff --git a/deeptutor/tools/web_fetch.py b/deeptutor/tools/web_fetch.py
index a48e9a869..d99205d97 100644
--- a/deeptutor/tools/web_fetch.py
+++ b/deeptutor/tools/web_fetch.py
@@ -23,6 +23,7 @@ arguments, not a human):
from __future__ import annotations
+import codecs
from dataclasses import dataclass
import ipaddress
import logging
@@ -224,21 +225,91 @@ async def _bounded_read(response: httpx.Response, limit: int) -> str:
"""Stream-read at most ``limit`` bytes from ``response`` then stop.
Avoids holding hundreds of MB if a server (or an LLM-supplied URL)
- points at a huge resource. Encoding falls back from response.encoding
- → utf-8 with replacement.
+ points at a huge resource. The charset comes from
+ :py:func:`_detect_encoding`; undecodable bytes are replaced.
"""
buf = bytearray()
async for chunk in response.aiter_bytes():
buf.extend(chunk)
if len(buf) >= limit:
break
- encoding = response.encoding or "utf-8"
+ encoding = _detect_encoding(bytes(buf), response.headers.get("content-type", ""))
try:
return buf.decode(encoding, errors="replace")
except (LookupError, TypeError):
return buf.decode("utf-8", errors="replace")
+# Byte-order marks take precedence over every declared charset (WHATWG
+# "decode"). ``utf-8-sig`` also drops the mark from the decoded text.
+_BOMS = (
+ (b"\xef\xbb\xbf", "utf-8-sig"),
+ (b"\xff\xfe", "utf-16"),
+ (b"\xfe\xff", "utf-16"),
+)
+_HEADER_CHARSET_RE = re.compile(r"""charset\s*=\s*["']?([\w.:-]+)""", re.IGNORECASE)
+# ``<meta charset="gbk">`` and ``<meta http-equiv="Content-Type"
+# content="text/html; charset=gbk">``; browsers look for either in the
+# first 1024 bytes when the HTTP header names no charset.
+_META_CHARSET_RE = re.compile(
+ rb"""<meta[^>]+?charset\s*=\s*["']?\s*([\w.:-]+)""",
+ re.IGNORECASE,
+)
+_META_PRESCAN_BYTES = 1024
+# Labels pages commonly declare for a superset encoding. Browsers decode them
+# as the superset (WHATWG Encoding), so a page labelled ``gb2312`` that uses
+# GBK-only characters still renders; Python's strict codecs would not.
+_CHARSET_SUPERSETS = {
+ "gb2312": "gb18030",
+ "gbk": "gb18030",
+ "x-gbk": "gb18030",
+ "iso-8859-1": "cp1252",
+ "latin1": "cp1252",
+ "latin-1": "cp1252",
+ "us-ascii": "cp1252",
+ "ascii": "cp1252",
+}
+
+
+def _usable_charset(label: str | bytes | None) -> str | None:
+ if not label:
+ return None
+ if isinstance(label, bytes):
+ label = label.decode("ascii", errors="ignore")
+ name = label.strip().lower()
+ name = _CHARSET_SUPERSETS.get(name, name)
+ try:
+ codecs.lookup(name)
+ except LookupError:
+ return None
+ return name
+
+
+def _detect_encoding(body: bytes, content_type: str) -> str:
+ """Pick the charset for ``body`` the way a browser would.
+
+ Order: byte-order mark, the ``charset`` parameter of the HTTP
+ ``Content-Type``, a ``<meta>`` charset declaration near the top of the
+ document, then UTF-8. Without the ``<meta>`` step every page that only
+ declares its charset in the markup (common for GBK / Shift_JIS / EUC-KR
+ sites) is decoded as UTF-8 and comes back as replacement characters.
+ """
+ for bom, encoding in _BOMS:
+ if body.startswith(bom):
+ return encoding
+ header_match = _HEADER_CHARSET_RE.search(content_type or "")
+ declared = _usable_charset(header_match.group(1) if header_match else None)
+ if declared:
+ return declared
+ meta_match = _META_CHARSET_RE.search(body[:_META_PRESCAN_BYTES])
+ sniffed = _usable_charset(meta_match.group(1) if meta_match else None)
+ # A UTF-16 label in ASCII-compatible markup is a mislabel; browsers
+ # treat it as UTF-8, and decoding it as UTF-16 would garble everything.
+ if sniffed and not sniffed.startswith("utf-16"):
+ return sniffed
+ return "utf-8"
+
+
def _extract_readable(html_or_text: str, base_url: str = "") -> tuple[str, str]:
"""Return ``(title, body_text)`` extracted from an HTML string.
diff --git a/tests/tools/test_web_fetch.py b/tests/tools/test_web_fetch.py
index a9be4f7be..7a10c9060 100644
--- a/tests/tools/test_web_fetch.py
+++ b/tests/tools/test_web_fetch.py
@@ -312,6 +312,61 @@ async def test_fetch_truncates_at_max_chars() -> None:
assert len(outcome.markdown) <= 220 # cap + marker headroom
+# "喆" is outside GB2312, so pages labelled gb2312 only render it when the
+# label is decoded as its GBK superset, as browsers do.
+_GBK_TEXT = "傅里叶变换将信号分解为不同频率的正弦波。——喆"
+
+
+ "meta",
+ [
+ '<meta charset="gbk">',
+ '<meta http-equiv="Content-Type" content="text/html; charset=gb2312">',
+ ],
+)
+async def test_fetch_honours_meta_charset_when_header_has_none(meta: str) -> None:
+ body = f"<html><head>{meta}<title>笔记</title></head><body><p>{_GBK_TEXT}</p></body></html>"
+ response = _StubResponse(
+ body=body.encode("gbk"),
+ headers={"content-type": "text/html"},
+ )
+
+ outcome = await fetch_url_as_markdown(
+ "https://example.com/gbk",
+ client_factory=_factory_returning(response),
+ host_validator=_ALLOW_ALL,
+ )
+
+ assert outcome.ok is True
+ assert outcome.title == "笔记"
+ assert _GBK_TEXT in outcome.markdown
+ assert "�" not in outcome.markdown
+
+
+async def test_fetch_header_charset_wins_over_meta_and_bom_wins_over_both() -> None:
+ body = f'<html><head><meta charset="gbk"></head><body><p>{_GBK_TEXT}</p></body></html>'
+ header_utf8 = _StubResponse(
+ body=body.encode("utf-8"),
+ headers={"content-type": 'text/html; charset="UTF-8"'},
+ )
+ bom_utf8 = _StubResponse(
+ body=b"\xef\xbb\xbf" + body.encode("utf-8"),
+ headers={"content-type": "text/html; charset=iso-8859-1"},
+ )
+
+ for response in (header_utf8, bom_utf8):
+ outcome = await fetch_url_as_markdown(
+ "https://example.com/utf8",
+ client_factory=_factory_returning(response),
+ host_validator=_ALLOW_ALL,
+ )
+ assert outcome.ok is True
+ assert _GBK_TEXT in outcome.markdown
+ assert "" not in outcome.markdown
+
+
@pytest.mark.asyncio
async def test_fetch_propagates_http_error_as_outcome_not_exception() -> None:
outcome = await fetch_url_as_markdown(