DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix: decode fetched pages with their declared <meta> charset
Mohammad Hijjawi630742b6cc47
diff --git a/deeptutor/tools/web_fetch.py b/deeptutor/tools/web_fetch.pyindex a48e9a869..d99205d97 100644--- a/deeptutor/tools/web_fetch.py+++ b/deeptutor/tools/web_fetch.py@@ -23,6 +23,7 @@ arguments, not a human):from __future__ import annotations+import codecsfrom dataclasses import dataclassimport ipaddressimport logging@@ -224,21 +225,91 @@ async def _bounded_read(response: httpx.Response, limit: int) -> str:"""Stream-read at most ``limit`` bytes from ``response`` then stop.Avoids holding hundreds of MB if a server (or an LLM-supplied URL)- points at a huge resource. Encoding falls back from response.encoding- → utf-8 with replacement.+ points at a huge resource. The charset comes from+ :py:func:`_detect_encoding`; undecodable bytes are replaced."""buf = bytearray()async for chunk in response.aiter_bytes():buf.extend(chunk)if len(buf) >= limit:break- encoding = response.encoding or "utf-8"+ encoding = _detect_encoding(bytes(buf), response.headers.get("content-type", ""))try:return buf.decode(encoding, errors="replace")except (LookupError, TypeError):return buf.decode("utf-8", errors="replace")+# Byte-order marks take precedence over every declared charset (WHATWG+# "decode"). ``utf-8-sig`` also drops the mark from the decoded text.+_BOMS = (+ (b"\xef\xbb\xbf", "utf-8-sig"),+ (b"\xff\xfe", "utf-16"),+ (b"\xfe\xff", "utf-16"),+)+_HEADER_CHARSET_RE = re.compile(r"""charset\s*=\s*["']?([\w.:-]+)""", re.IGNORECASE)+# ``<meta charset="gbk">`` and ``<meta http-equiv="Content-Type"+# content="text/html; charset=gbk">``; browsers look for either in the+# first 1024 bytes when the HTTP header names no charset.+_META_CHARSET_RE = re.compile(+ rb"""<meta[^>]+?charset\s*=\s*["']?\s*([\w.:-]+)""",+ re.IGNORECASE,+)+_META_PRESCAN_BYTES = 1024+# Labels pages commonly declare for a superset encoding. Browsers decode them+# as the superset (WHATWG Encoding), so a page labelled ``gb2312`` that uses+# GBK-only characters still renders; Python's strict codecs would not.+_CHARSET_SUPERSETS = {+ "gb2312": "gb18030",+ "gbk": "gb18030",+ "x-gbk": "gb18030",+ "iso-8859-1": "cp1252",+ "latin1": "cp1252",+ "latin-1": "cp1252",+ "us-ascii": "cp1252",+ "ascii": "cp1252",+}+++def _usable_charset(label: str | bytes | None) -> str | None:+ if not label:+ return None+ if isinstance(label, bytes):+ label = label.decode("ascii", errors="ignore")+ name = label.strip().lower()+ name = _CHARSET_SUPERSETS.get(name, name)+ try:+ codecs.lookup(name)+ except LookupError:+ return None+ return name+++def _detect_encoding(body: bytes, content_type: str) -> str:+ """Pick the charset for ``body`` the way a browser would.++ Order: byte-order mark, the ``charset`` parameter of the HTTP+ ``Content-Type``, a ``<meta>`` charset declaration near the top of the+ document, then UTF-8. Without the ``<meta>`` step every page that only+ declares its charset in the markup (common for GBK / Shift_JIS / EUC-KR+ sites) is decoded as UTF-8 and comes back as replacement characters.+ """+ for bom, encoding in _BOMS:+ if body.startswith(bom):+ return encoding+ header_match = _HEADER_CHARSET_RE.search(content_type or "")+ declared = _usable_charset(header_match.group(1) if header_match else None)+ if declared:+ return declared+ meta_match = _META_CHARSET_RE.search(body[:_META_PRESCAN_BYTES])+ sniffed = _usable_charset(meta_match.group(1) if meta_match else None)+ # A UTF-16 label in ASCII-compatible markup is a mislabel; browsers+ # treat it as UTF-8, and decoding it as UTF-16 would garble everything.+ if sniffed and not sniffed.startswith("utf-16"):+ return sniffed+ return "utf-8"++def _extract_readable(html_or_text: str, base_url: str = "") -> tuple[str, str]:"""Return ``(title, body_text)`` extracted from an HTML string.diff --git a/tests/tools/test_web_fetch.py b/tests/tools/test_web_fetch.pyindex a9be4f7be..7a10c9060 100644--- a/tests/tools/test_web_fetch.py+++ b/tests/tools/test_web_fetch.py@@ -312,6 +312,61 @@ async def test_fetch_truncates_at_max_chars() -> None:assert len(outcome.markdown) <= 220 # cap + marker headroom+# "喆" is outside GB2312, so pages labelled gb2312 only render it when the+# label is decoded as its GBK superset, as browsers do.+_GBK_TEXT = "傅里叶变换将信号分解为不同频率的正弦波。——喆"+++ "meta",+ [+ '<meta charset="gbk">',+ '<meta http-equiv="Content-Type" content="text/html; charset=gb2312">',+ ],+)+async def test_fetch_honours_meta_charset_when_header_has_none(meta: str) -> None:+ body = f"<html><head>{meta}<title>笔记</title></head><body><p>{_GBK_TEXT}</p></body></html>"+ response = _StubResponse(+ body=body.encode("gbk"),+ headers={"content-type": "text/html"},+ )++ outcome = await fetch_url_as_markdown(+ "https://example.com/gbk",+ client_factory=_factory_returning(response),+ host_validator=_ALLOW_ALL,+ )++ assert outcome.ok is True+ assert outcome.title == "笔记"+ assert _GBK_TEXT in outcome.markdown+ assert "�" not in outcome.markdown+++async def test_fetch_header_charset_wins_over_meta_and_bom_wins_over_both() -> None:+ body = f'<html><head><meta charset="gbk"></head><body><p>{_GBK_TEXT}</p></body></html>'+ header_utf8 = _StubResponse(+ body=body.encode("utf-8"),+ headers={"content-type": 'text/html; charset="UTF-8"'},+ )+ bom_utf8 = _StubResponse(+ body=b"\xef\xbb\xbf" + body.encode("utf-8"),+ headers={"content-type": "text/html; charset=iso-8859-1"},+ )++ for response in (header_utf8, bom_utf8):+ outcome = await fetch_url_as_markdown(+ "https://example.com/utf8",+ client_factory=_factory_returning(response),+ host_validator=_ALLOW_ALL,+ )+ assert outcome.ok is True+ assert _GBK_TEXT in outcome.markdown+ assert "" not in outcome.markdown++@pytest.mark.asyncioasync def test_fetch_propagates_http_error_as_outcome_not_exception() -> None:outcome = await fetch_url_as_markdown(