Skip to content

AI turns documents or topics into real, native PowerPoint decks—with native shapes, transitions and animations, data-backed charts and tables on demand, audio narration from speaker notes, and support for your own .pptx templates. · by Hugo He

fix(doc-to-md): decode HTML by BOM and charset declaration, not forced UTF-8

hugohe334d88c1af3e0
diff --git a/skills/ppt-master/scripts/source_to_md/doc_to_md.py b/skills/ppt-master/scripts/source_to_md/doc_to_md.py
index 212ec031..4c14a2bb 100644
--- a/skills/ppt-master/scripts/source_to_md/doc_to_md.py
+++ b/skills/ppt-master/scripts/source_to_md/doc_to_md.py
@@ -21,6 +21,7 @@ All paths produce the same output convention:
import argparse
import base64
+import codecs
import hashlib
import json
import mimetypes
@@ -1260,13 +1261,31 @@ def _convert_html(input_file: Path, out_file: Path) -> str:
return ""
try:
- from bs4 import BeautifulSoup
+ from bs4 import BeautifulSoup, UnicodeDammit
except ImportError:
print("[ERROR] beautifulsoup4 not installed. Run: pip install beautifulsoup4")
return ""
media_dir, rel_media_dir = _ensure_media_dir(out_file)
- raw_html = input_file.read_text(encoding="utf-8", errors="replace")
+ # BOM, then the page's own charset declaration, then UTF-8; a forced UTF-8
+ # read with errors="replace" turned GBK pages into U+FFFD without a word.
+ raw_bytes = input_file.read_bytes()
+ dammit = UnicodeDammit(raw_bytes, user_encodings=["utf-8"], is_html=True)
+ raw_html = dammit.unicode_markup or ""
+ has_bom = raw_bytes.startswith((codecs.BOM_UTF8, codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE))
+ if (
+ dammit.contains_replacement_characters
+ or (
+ not has_bom
+ and not dammit.declared_html_encoding
+ and dammit.original_encoding not in ("utf-8", "ascii")
+ )
+ ):
+ print(
+ f"[WARN] {input_file.name}: no charset declaration and not UTF-8; "
+ f"decoded as {dammit.original_encoding} — check the converted text",
+ file=sys.stderr,
+ )
# Strip non-content elements (head/style/script) so metadata doesn't leak into MD
soup = BeautifulSoup(raw_html, "html.parser")
diff --git a/skills/ppt-master/scripts/tests/test_source_to_md_content_loss_docx.py b/skills/ppt-master/scripts/tests/test_source_to_md_content_loss_docx.py
index 6eb3aed4..eafcb0a1 100644
--- a/skills/ppt-master/scripts/tests/test_source_to_md_content_loss_docx.py
+++ b/skills/ppt-master/scripts/tests/test_source_to_md_content_loss_docx.py
@@ -29,6 +29,20 @@ sys.path.insert(0, str(SCRIPTS_DIR / "source_to_md"))
import doc_to_md # noqa: E402
+class HtmlCharsetTests(unittest.TestCase):
+ def test_declared_gbk_page_is_decoded_not_replaced(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ source = Path(tmp) / "page.html"
+ source.write_bytes(
+ '<html><head><meta charset="gbk"></head><body><p>中文收入</p></body></html>'
+ .encode("gbk")
+ )
+ with redirect_stdout(io.StringIO()):
+ markdown = doc_to_md.convert_to_markdown(str(source))
+ self.assertIn("中文收入", markdown)
+ self.assertNotIn("\ufffd", markdown)
+
+
class NotebookContentLossTests(unittest.TestCase):
def test_attachments_do_not_overwrite_named_outputs_or_each_other(self) -> None:
try: