Skip to content

AI turns documents or topics into real, native PowerPoint decks—with native shapes, transitions and animations, data-backed charts and tables on demand, audio narration from speaker notes, and support for your own .pptx templates. · by Hugo He

fix(scripts): source conversion keeps Excel precision, detects source encodings, and preserves inline spacing and absolute links from web pages

hugohe38fc4f89eeade
diff --git a/skills/ppt-master/scripts/source_to_md.py b/skills/ppt-master/scripts/source_to_md.py
index 2a29d231..a4f96033 100644
--- a/skills/ppt-master/scripts/source_to_md.py
+++ b/skills/ppt-master/scripts/source_to_md.py
@@ -21,6 +21,7 @@ Dependencies:
from __future__ import annotations
import argparse
+import codecs
import json
import os
import subprocess
@@ -145,20 +146,54 @@ def write_passthrough(
"""Copy text-like input to Markdown and write the profile sidecar."""
source = Path(input_arg)
try:
- text = source.read_text(encoding="utf-8", errors="replace")
+ raw = source.read_bytes()
except OSError as exc:
print(f"[ERROR] Cannot read {source}: {exc}", file=sys.stderr)
return 1
+ encodings = ("utf-8", "gb18030")
+ if raw.startswith(codecs.BOM_UTF8):
+ encodings = ("utf-8-sig",)
+ elif raw.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
+ encodings = ("utf-16",)
+ for encoding in encodings:
+ try:
+ text = raw.decode(encoding)
+ break
+ except UnicodeDecodeError:
+ continue
+ else:
+ print(
+ f"[ERROR] Cannot decode {source} as {' or '.join(encodings)}. "
+ "Save the source as UTF-8 text and retry.",
+ file=sys.stderr,
+ )
+ return 1
+ if any(ord(char) < 32 and char not in "\t\n\r\f" for char in text):
+ print(f"[ERROR] Binary control characters in {source}; provide a text file.", file=sys.stderr)
+ return 1
+ if encoding != "utf-8" and output.resolve() == source.resolve():
+ print(
+ f"[ERROR] {source} uses {encoding}; choose a different -o path for UTF-8 output.",
+ file=sys.stderr,
+ )
+ return 1
+
output.parent.mkdir(parents=True, exist_ok=True)
if output.resolve() != source.resolve():
- output.write_text(text, encoding="utf-8")
+ output.write_bytes(text.encode("utf-8"))
+ warnings = []
+ if encoding != "utf-8":
+ warnings.append(f"Detected source encoding: {encoding}; converted to UTF-8.")
profile = write_conversion_profile(
input_path=input_arg,
markdown_path=output,
converter="source_to_md.py",
conversion_type=conversion_type,
+ warnings=warnings,
)
+ for warning in warnings:
+ print(f"[INFO] {warning}")
_print_status(f"[OK] Saved Markdown to: {output}")
_print_status(f" Wrote conversion profile -> {profile}")
print_output(output)
diff --git a/skills/ppt-master/scripts/source_to_md/excel_to_md.py b/skills/ppt-master/scripts/source_to_md/excel_to_md.py
index 54506a1f..0defd41f 100644
--- a/skills/ppt-master/scripts/source_to_md/excel_to_md.py
+++ b/skills/ppt-master/scripts/source_to_md/excel_to_md.py
@@ -79,7 +79,9 @@ def _format_cell_value(value: Any) -> str:
if isinstance(value, time):
return value.isoformat(timespec="seconds")
if isinstance(value, float):
- return _markdown_escape(f"{value:g}")
+ # Excel shows 15 significant digits; that keeps 1234567.89 exact while
+ # 0.1 + 0.2 reads as 0.3 rather than its binary expansion.
+ return _markdown_escape(repr(float(f"{value:.15g}")).removesuffix(".0"))
return _markdown_escape(str(value))
diff --git a/skills/ppt-master/scripts/source_to_md/web_to_md.py b/skills/ppt-master/scripts/source_to_md/web_to_md.py
index 9df5636f..bd423e91 100644
--- a/skills/ppt-master/scripts/source_to_md/web_to_md.py
+++ b/skills/ppt-master/scripts/source_to_md/web_to_md.py
@@ -215,14 +215,14 @@ CONFIG = {
}
-def fetch_url(url: str) -> str:
+def fetch_url(url: str) -> tuple[str, str]:
"""Fetch a web page with explicit headers and encoding detection.
Args:
url: Target URL.
Returns:
- The response body as text.
+ The response body as text and the final URL after redirects.
"""
headers = {
"User-Agent": CONFIG["user_agent"],
@@ -235,7 +235,7 @@ def fetch_url(url: str) -> str:
timeout=CONFIG["timeout"], verify=False)
response.raise_for_status()
- return _decode_response_text(response)
+ return _decode_response_text(response), response.url
except Exception as e:
raise Exception(f"Failed to fetch {url}: {str(e)}")
@@ -714,7 +714,9 @@ def element_to_markdown(element: Tag | NavigableString | None) -> str:
return f"{content} "
-def simple_html_to_markdown_traversal(soup: Tag | BeautifulSoup | None) -> str:
+def simple_html_to_markdown_traversal(
+ soup: Tag | BeautifulSoup | None, page_url: str = "",
+) -> str:
"""Convert HTML content to Markdown using BeautifulSoup traversal."""
lines = []
@@ -727,17 +729,17 @@ def simple_html_to_markdown_traversal(soup: Tag | BeautifulSoup | None) -> str:
if isinstance(node, NavigableString):
text = str(node)
# Normalize whitespace but keep single spaces
- text = re.sub(r'\s+', ' ', text)
- if text.strip():
- return text
- return ""
+ return re.sub(r'\s+', ' ', text)
if node.name in ['script', 'style', 'comment', 'meta', 'link']:
return ""
# Handle Block Elements
is_block = node.name in ['p', 'div', 'h1', 'h2', 'h3', 'h4',
- 'h5', 'h6', 'li', 'blockquote', 'pre', 'hr', 'table', 'tr']
+ 'h5', 'h6', 'li', 'blockquote', 'pre', 'hr', 'table', 'tr',
+ 'section', 'article', 'main', 'header', 'footer', 'nav',
+ 'aside', 'ul', 'ol', 'dl', 'dt', 'dd', 'figure',
+ 'figcaption', 'address', 'details', 'summary']
# Pre-processing
prefix = ""
@@ -762,6 +764,8 @@ def simple_html_to_markdown_traversal(soup: Tag | BeautifulSoup | None) -> str:
elif node.name == 'pre':
# Extract raw text from pre to preserve formatting
return f"\n\n```\n{node.get_text()}\n```\n\n"
+ elif is_block:
+ prefix, suffix = "\n\n", "\n\n"
# Inline formatting
if node.name in ['strong', 'b']:
@@ -772,7 +776,9 @@ def simple_html_to_markdown_traversal(soup: Tag | BeautifulSoup | None) -> str:
prefix, suffix = "`", "`"
elif node.name == 'a':
href = node.get('href')
- if href and not href.startswith('javascript:'):
+ if href and not href.lower().startswith('javascript:'):
+ if not href.startswith('#') and urlparse(href).scheme.lower() not in {'mailto', 'tel'}:
+ href = urljoin(page_url, href)
prefix = "["
suffix = f"]({href})"
else:
@@ -789,7 +795,18 @@ def simple_html_to_markdown_traversal(soup: Tag | BeautifulSoup | None) -> str:
for child in node.children:
res = traverse(child)
if res:
- inner_text += res
+ if isinstance(child, NavigableString) and not res.strip():
+ if inner_text and not inner_text.endswith((' ', '\n')):
+ inner_text += ' '
+ else:
+ if res.startswith('\n'):
+ inner_text = inner_text.rstrip(' ')
+ elif inner_text.endswith((' ', '\n')) and child.name != 'br':
+ res = res.lstrip(' ')
+ inner_text += res
+
+ if is_block:
+ inner_text = inner_text.strip()
# Post-processing for tables (simplified)
if node.name == 'tr':
@@ -892,10 +909,12 @@ def process_url(
"""
print(f"\n[Fetching] {url}")
try:
- html = fetch_url(url)
+ html, page_url = fetch_url(url)
if is_plain_text_document(url, html):
return _save_plain_text_document(url, html, output_file)
soup = BeautifulSoup(html, 'html.parser')
+ base = soup.find('base', href=True)
+ base_url = urljoin(page_url, base['href']) if base else page_url
# Extract Metadata
metadata = extract_metadata(soup, url)
@@ -924,15 +943,15 @@ def process_url(
image_count = 0
if download_images:
image_count = download_and_rewrite_images(
- content_div, url, image_dir, rel_image_prefix)
+ content_div, base_url, image_dir, rel_image_prefix)
else:
- rewrite_images_to_remote_urls(content_div, url)
+ rewrite_images_to_remote_urls(content_div, base_url)
if image_count:
print(f" [OK] Images: {image_count} saved to {image_dir}")
# Convert to MD
# Note: We pass the element to our traversal function
- markdown_text = simple_html_to_markdown_traversal(content_div)
+ markdown_text = simple_html_to_markdown_traversal(content_div, base_url)
print(f" [OK] Content: {len(markdown_text)} chars")
# Construct content
diff --git a/skills/ppt-master/scripts/tests/test_source_to_md_cli.py b/skills/ppt-master/scripts/tests/test_source_to_md_cli.py
index 4c7c41ee..b2192eef 100644
--- a/skills/ppt-master/scripts/tests/test_source_to_md_cli.py
+++ b/skills/ppt-master/scripts/tests/test_source_to_md_cli.py
@@ -4,12 +4,16 @@
from __future__ import annotations
import argparse
+import codecs
+import io
import json
import subprocess
import sys
import tempfile
import unittest
+from contextlib import redirect_stdout
from pathlib import Path
+from unittest.mock import Mock, patch
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
@@ -22,6 +26,8 @@ WEB_BACKEND_DIR = SCRIPTS_DIR / "source_to_md"
if str(WEB_BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(WEB_BACKEND_DIR))
from web_to_md import is_plain_text_document # noqa: E402
+import web_to_md # noqa: E402
+from excel_to_md import _format_cell_value # noqa: E402
def _args(**overrides: object) -> argparse.Namespace:
@@ -113,6 +119,119 @@ class RawTextUrlTests(unittest.TestCase):
self.assertFalse(source_to_md._skips_images(_args(images="all")))
+class ExcelCellValueTests(unittest.TestCase):
+ def test_float_values_keep_excel_display_precision(self) -> None:
+ # 15 significant digits, the precision Excel itself displays.
+ cases = [
+ (1234567.89, "1234567.89"),
+ (0.1 + 0.2, "0.3"),
+ (100.0, "100"),
+ (1e-7, "1e-07"),
+ (-0.5, "-0.5"),
+ (1.2345678901234567, "1.23456789012346"),
+ (-0.0, "-0"),
+ (1e20, "1e+20"),
+ ]
+ for value, expected in cases:
+ with self.subTest(value=value):
+ result = _format_cell_value(value)
+ self.assertEqual(result, expected)
+ self.assertAlmostEqual(float(result), value, delta=abs(value) * 1e-14)
+
+
+class WebTraversalTests(unittest.TestCase):
+ def test_inline_whitespace_is_preserved_once(self) -> None:
+ cases = [
+ ("<strong>Hello</strong> <em>World</em>", "**Hello** *World*"),
+ ('See <a href="x">here</a> now', "See [here](x) now"),
+ ("<code>a</code> <code>b</code>", "`a` `b`"),
+ ("<strong>Hello</strong> \n\t <em>World</em>", "**Hello** *World*"),
+ ("<strong>Hello</strong> <!-- comment --> <em>World</em>", "**Hello** *World*"),
+ ("<strong>Hello</strong><em>World</em>", "**Hello***World*"),
+ ]
+ for html, expected in cases:
+ with self.subTest(html=html):
+ soup = web_to_md.BeautifulSoup(f"<p>{html}</p>", "html.parser")
+ self.assertEqual(web_to_md.simple_html_to_markdown_traversal(soup), expected)
+
+ def test_block_whitespace_does_not_leak_into_paragraphs(self) -> None:
+ cases = [
+ ("<p>one</p> \n <p>two</p>", "one\n\ntwo"),
+ ("<div>one</div> \n <div>two</div>", "one\n\ntwo"),
+ ("<section>one</section> \n <section>two</section>", "one\n\ntwo"),
+ ("<ul> <li>one</li> <li>two</li> </ul>", "- one\n- two"),
+ ("<p> lead</p>", "lead"),
+ ("<p> lead</p><p>trail </p>", "lead\n\ntrail"),
+ ("<p> <strong>Hello</strong> <em>World</em> </p>", "**Hello** *World*"),
+ ]
+ for html, expected in cases:
+ with self.subTest(html=html):
+ soup = web_to_md.BeautifulSoup(html, "html.parser")
+ self.assertEqual(web_to_md.simple_html_to_markdown_traversal(soup), expected)
+
+ def test_preformatted_text_and_explicit_line_breaks_keep_spacing(self) -> None:
+ cases = [
+ ("<pre> a\n b </pre>", "```\n a\n b \n```"),
+ ("<p>one<br>two</p>", "one \ntwo"),
+ ]
+ for html, expected in cases:
+ with self.subTest(html=html):
+ soup = web_to_md.BeautifulSoup(html, "html.parser")
+ self.assertEqual(web_to_md.simple_html_to_markdown_traversal(soup), expected)
+
+ def test_links_resolve_relative_targets_and_keep_special_targets(self) -> None:
+ cases = [
+ ("../target", "[label](https://example.com/target)"),
+ ("/abs", "[label](https://example.com/abs)"),
+ ("//cdn.x/y", "[label](https://cdn.x/y)"),
+ ("#frag", "[label](#frag)"),
+ ("mailto:[email protected]", "[label](mailto:[email protected])"),
+ ("tel:+123456789", "[label](tel:+123456789)"),
+ ("javascript:void(0)", "label"),
+ ("JavaScript:void(0)", "label"),
+ ]
+ for href, expected in cases:
+ with self.subTest(href=href):
+ soup = web_to_md.BeautifulSoup(f'<p><a href="{href}">label</a></p>', "html.parser")
+ self.assertEqual(
+ web_to_md.simple_html_to_markdown_traversal(soup, "https://example.com/a/page"),
+ expected,
+ )
+
+ def test_process_url_resolves_against_redirect_and_first_base_href(self) -> None:
+ cases = [
+ ("", "https://redirect.example/docs/"),
+ ('<base href="https://base.example/root/">', "https://base.example/root/"),
+ ('<base href="../assets/">', "https://redirect.example/assets/"),
+ ('<base target="_blank"><base href="/first/"><base href="/ignored/">',
+ "https://redirect.example/first/"),
+ ]
+ for base, expected_base in cases:
+ with self.subTest(base=base), tempfile.TemporaryDirectory() as tmp:
+ html = (
+ f"<html><head><title>Links</title>{base}</head><body>"
+ '<p><a href="target">target</a> <img src="pic.png" alt="pic"></p>'
+ "</body></html>"
+ )
+ response = Mock(
+ content=html.encode("utf-8"),
+ headers={"Content-Type": "text/html; charset=utf-8"},
+ encoding="utf-8",
+ apparent_encoding="utf-8",
+ url="https://redirect.example/docs/page",
+ )
+ response.raise_for_status.return_value = None
+ output = Path(tmp) / "page.md"
+ with patch.object(web_to_md, "_http_get", return_value=response), redirect_stdout(io.StringIO()):
+ result = web_to_md.process_url(
+ "https://original.example/start", str(output), download_images=False,
+ )
+ self.assertTrue(result[0], result[2])
+ markdown = output.read_text(encoding="utf-8")
+ self.assertIn(f"[target]({expected_base}target)", markdown)
+ self.assertIn(f"![pic]({expected_base}pic.png)", markdown)
+
+
class SourceCollisionTests(unittest.TestCase):
def setUp(self) -> None:
self.temporary = tempfile.TemporaryDirectory()
@@ -210,6 +329,67 @@ class SourceCollisionTests(unittest.TestCase):
self.assertEqual(self._outputs(result), [source])
self.assertEqual(source.read_text(encoding="utf-8"), "ORIGINAL-MARKDOWN\n")
+ def test_passthrough_decodes_supported_encodings_without_changing_text(self) -> None:
+ text = "# 中文金额 123.45\r\n\r\nKeep spaces\tand tabs.\r\n"
+ utf8 = text.encode("utf-8")
+ cases = [
+ ("utf8", utf8, "utf-8"),
+ ("utf8_bom", codecs.BOM_UTF8 + utf8, "utf-8-sig"),
+ ("utf16_le", codecs.BOM_UTF16_LE + text.encode("utf-16-le"), "utf-16"),
+ ("utf16_be", codecs.BOM_UTF16_BE + text.encode("utf-16-be"), "utf-16"),
+ ("gb18030", text.encode("gb18030"), "gb18030"),
+ ]
+ for name, raw, encoding in cases:
+ for suffix in (".txt", ".md"):
+ with self.subTest(encoding=name, suffix=suffix):
+ source = self.root / f"{name}{suffix}"
+ output = self.root / f"{name}_{suffix[1:]}_output.md"
+ source.write_bytes(raw)
+ result = self._run_cli(str(source), "-o", str(output))
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(self._outputs(result), [output])
+ self.assertEqual(output.read_bytes(), utf8)
+ self.assertEqual(source.read_bytes(), raw)
+ profile = json.loads(output.with_suffix(".conversion_profile.json").read_text(encoding="utf-8"))
+ if encoding == "utf-8":
+ self.assertEqual(profile["warnings"], [])
+ else:
+ self.assertIn(encoding, result.stdout)
+ self.assertIn(encoding, " ".join(profile["warnings"]))
+
+ def test_invalid_passthrough_fails_before_writing_outputs(self) -> None:
+ cases = [
+ bytes(range(256)),
+ b"\x00\x01binary",
+ codecs.BOM_UTF8 + b"\xff",
+ codecs.BOM_UTF16_LE + b"A\x00B",
+ codecs.BOM_UTF16_BE + b"\x00AB",
+ ]
+ for index, raw in enumerate(cases):
+ with self.subTest(raw=raw):
+ source = self.root / f"invalid_{index}.txt"
+ output = self.root / f"output_{index}" / "result.md"
+ source.write_bytes(raw)
+ result = self._run_cli(str(source), "-o", str(output))
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("[ERROR]", result.stderr)
+ self.assertEqual(self._outputs(result), [])
+ self.assertFalse(output.parent.exists())
+ self.assertEqual(source.read_bytes(), raw)
+
+ def test_non_utf8_passthrough_requires_a_distinct_output_path(self) -> None:
+ for encoding in ("utf-8-sig", "utf-16", "gb18030"):
+ with self.subTest(encoding=encoding):
+ source = self.root / f"{encoding}.md"
+ raw = "中文原稿\n".encode(encoding)
+ source.write_bytes(raw)
+ result = self._run_cli(str(source))
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("-o", result.stderr)
+ self.assertEqual(self._outputs(result), [])
+ self.assertEqual(source.read_bytes(), raw)
+ self.assertFalse(source.with_suffix(".conversion_profile.json").exists())
+
if __name__ == "__main__":
unittest.main()