Skip to content

AI turns documents or topics into real, native PowerPoint decks—with native shapes, transitions and animations, data-backed charts and tables on demand, audio narration from speaker notes, and support for your own .pptx templates. · by Hugo He

Fix web page charset decoding

hugohe37f22682d7f4f
diff --git a/skills/ppt-master/scripts/source_to_md/web_to_md.py b/skills/ppt-master/scripts/source_to_md/web_to_md.py
index 5cba9b25..49fa5ff1 100644
--- a/skills/ppt-master/scripts/source_to_md/web_to_md.py
+++ b/skills/ppt-master/scripts/source_to_md/web_to_md.py
@@ -27,6 +27,7 @@ TLS fingerprint handling:
"""
import argparse
+import codecs
import datetime
import io
import json
@@ -70,6 +71,95 @@ def _http_get(url: str, *, headers: dict | None = None, timeout: int | None = No
return requests.get(url, headers=headers, timeout=timeout,
verify=verify, stream=stream)
+
+def _normalize_charset(charset: str | None) -> str:
+ """Return a Python codec name when the declared charset is usable."""
+ if not charset:
+ return ""
+ charset = charset.strip().strip('"').strip("'").lower()
+ if not charset:
+ return ""
+ try:
+ return codecs.lookup(charset).name
+ except LookupError:
+ return ""
+
+
+def _charset_from_headers(headers: dict) -> str:
+ content_type = headers.get("Content-Type") or headers.get("content-type") or ""
+ match = re.search(r"charset\s*=\s*([^;\s]+)", content_type, re.I)
+ return _normalize_charset(match.group(1)) if match else ""
+
+
+def _charset_from_html(raw: bytes) -> str:
+ """Extract a charset declaration from the first chunk of HTML bytes."""
+ head = raw[:8192]
+ patterns = [
+ rb"<meta[^>]+charset=[\"']?\s*([a-zA-Z0-9_\-]+)",
+ rb"<meta[^>]+content=[\"'][^\"']*charset=\s*([a-zA-Z0-9_\-]+)",
+ ]
+ for pattern in patterns:
+ match = re.search(pattern, head, re.I)
+ if match:
+ return _normalize_charset(match.group(1).decode("ascii", "ignore"))
+ return ""
+
+
+def _decode_quality_score(text: str) -> int:
+ """Score obvious decode artifacts; lower is better."""
+ mojibake_markers = [
+ "�", "锟", "Ã", "Â", "â€", "’", "“", "â€\x9d",
+ "琚", "佸", "鍦", "涓", "鏄", "寤", "骞", "鏈", "鏃", "鈥",
+ ]
+ marker_hits = sum(text.count(marker) for marker in mojibake_markers)
+ control_hits = sum(1 for ch in text if ord(ch) < 32 and ch not in "\t\n\r")
+ return marker_hits * 20 + control_hits * 10 + text.count("\ufffd") * 50
+
+
+def _decode_response_text(response) -> str:
+ """Decode HTTP response bytes without letting guessed encodings override declarations."""
+ raw = response.content
+ declared = [
+ _charset_from_headers(response.headers),
+ _charset_from_html(raw),
+ ]
+ if raw.startswith(codecs.BOM_UTF8):
+ declared.insert(0, "utf-8-sig")
+
+ seen = set()
+ declared = [enc for enc in declared if enc and not (enc in seen or seen.add(enc))]
+ for enc in declared:
+ try:
+ return raw.decode(enc)
+ except UnicodeDecodeError:
+ continue
+
+ candidates = []
+ for enc in [
+ getattr(response, "encoding", None),
+ getattr(response, "apparent_encoding", None),
+ "utf-8",
+ "gb18030",
+ "big5",
+ ]:
+ enc = _normalize_charset(enc)
+ if enc and enc not in candidates:
+ candidates.append(enc)
+
+ decoded = []
+ for enc in candidates:
+ try:
+ text = raw.decode(enc)
+ except UnicodeDecodeError:
+ continue
+ decoded.append((_decode_quality_score(text), enc, text))
+
+ if decoded:
+ decoded.sort(key=lambda item: item[0])
+ return decoded[0][2]
+
+ return raw.decode("utf-8", errors="replace")
+
try:
from PIL import Image
PILLOW_AVAILABLE = True
@@ -132,12 +222,7 @@ def fetch_url(url: str) -> str:
timeout=CONFIG["timeout"], verify=False)
response.raise_for_status()
- # Enhanced encoding detection (requests handles this well usually, but we force apparent_encoding for Chinese)
- # curl_cffi exposes the same .apparent_encoding attribute
- if hasattr(response, "apparent_encoding") and response.apparent_encoding:
- response.encoding = response.apparent_encoding
-
- return response.text
+ return _decode_response_text(response)
except Exception as e:
raise Exception(f"Failed to fetch {url}: {str(e)}")