AI turns documents or topics into real, native PowerPoint decks—with native shapes, transitions and animations, data-backed charts and tables on demand, audio narration from speaker notes, and support for your own .pptx templates. · by Hugo He
fix(web_to_md): verify TLS by default and refuse non-public fetch targets
hugohe331a20f2d73a2
diff --git a/skills/ppt-master/scripts/docs/conversion.md b/skills/ppt-master/scripts/docs/conversion.mdindex 2af14d0e..cd2b29bd 100644--- a/skills/ppt-master/scripts/docs/conversion.md+++ b/skills/ppt-master/scripts/docs/conversion.md@@ -621,6 +621,9 @@ Error: PPTX-to-SVG conversion failed: Invalid DrawingML sRGB color structureConvert web pages to Markdown and download images locally by default. Use`--no-images` to retain remote image links without downloading their files.+Pages, images, and redirect targets must be public HTTP(S) hosts with valid+TLS; `--insecure` skips certificate checks, `--allow-private-hosts` admits+intranet and loopback addresses.```bashpython3 scripts/source_to_md/web_to_md.py https://example.com/articlediff --git a/skills/ppt-master/scripts/source_to_md/web_to_md.py b/skills/ppt-master/scripts/source_to_md/web_to_md.pyindex 8e3271d2..58b76894 100644--- a/skills/ppt-master/scripts/source_to_md/web_to_md.py+++ b/skills/ppt-master/scripts/source_to_md/web_to_md.py@@ -32,9 +32,11 @@ import argparseimport codecsimport datetimeimport io+import ipaddressimport jsonimport osimport re+import socketimport sysimport timefrom pathlib import Path@@ -76,21 +78,65 @@ except ImportError:_CURL_IMPERSONATE = None+class _UnsafeUrlError(ValueError):+ """Reject a URL that cannot be verified as a public HTTP(S) target."""+++def _validate_public_url(url: str) -> None:+ """Reject non-HTTP(S) URLs and hosts resolving to non-public addresses."""+ try:+ parsed = urlparse(url)+ hostname = parsed.hostname+ except ValueError as exc:+ raise _UnsafeUrlError(f"Invalid URL: {exc}") from exc+ if parsed.scheme not in {"http", "https"} or not hostname:+ raise _UnsafeUrlError("Only HTTP(S) URLs with a hostname are allowed")+ try:+ addresses = [ipaddress.ip_address(hostname)]+ except ValueError:+ try:+ addresses = [+ ipaddress.ip_address(info[4][0])+ for info in socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM)+ ]+ except (OSError, ValueError) as exc:+ raise _UnsafeUrlError(f"Cannot validate URL hostname {hostname}: {exc}") from exc+ if not addresses:+ raise _UnsafeUrlError(f"Cannot resolve URL hostname {hostname}")+ if CONFIG["allow_private_hosts"]:+ return+ for address in addresses:+ if (address.is_loopback or address.is_link_local+ or address.is_private or address.is_unspecified):+ raise _UnsafeUrlError(+ f"Refusing non-public URL target: {hostname} resolves to {address} "+ "(pass --allow-private-hosts for intranet or localhost pages)"+ )++def _http_get(url: str, *, headers: dict | None = None, timeout: int | None = None,- verify: bool = False, stream: bool = False):+ verify: bool = True, stream: bool = False):"""HTTP GET with curl_cffi preferred, requests fallback.Using curl_cffi lets this script fetch sites that reject Python's defaultTLS fingerprint (notably mp.weixin.qq.com). Signature mirrors the subset ofrequests.get() this script actually uses."""+ _validate_public_url(url)if curl_requests is not None:- return curl_requests.get(+ response = curl_requests.get(url, headers=headers, timeout=timeout,verify=verify, impersonate=_CURL_IMPERSONATE, stream=stream,)- return requests.get(url, headers=headers, timeout=timeout,- verify=verify, stream=stream)+ else:+ response = requests.get(url, headers=headers, timeout=timeout,+ verify=verify, stream=stream)+ try:+ _validate_public_url(response.url)+ except _UnsafeUrlError:+ response.close()+ raise+ return responsedef _normalize_charset(charset: str | None) -> str:@@ -194,6 +240,8 @@ except ImportError:CONFIG = {"output_dir": "./projects","timeout": 30,+ "insecure": False,+ "allow_private_hosts": False,"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",# Specific content identifiers often found in Chinese CMS (Gov/News)"content_selectors": [@@ -241,7 +289,7 @@ def fetch_url(url: str) -> tuple[str, str]:try:response = _http_get(url, headers=headers,- timeout=CONFIG["timeout"], verify=False)+ timeout=CONFIG["timeout"], verify=not CONFIG["insecure"])response.raise_for_status()return _decode_response_text(response), response.url@@ -377,7 +425,7 @@ def download_and_rewrite_images(abs_url,headers={"User-Agent": CONFIG["user_agent"]},timeout=CONFIG["timeout"],- verify=False,+ verify=not CONFIG["insecure"],)resp.raise_for_status()filename = build_image_filename(@@ -455,6 +503,8 @@ def download_and_rewrite_images("occurrences": [],}saved += 1+ except _UnsafeUrlError:+ raiseexcept Exception as e:print(f" [WARN] Skip image {abs_url}: {e}")continue@@ -1040,8 +1090,23 @@ def main(argv: list[str] | None = None) -> int:action="store_true",help="Keep remote image links without downloading image files",)+ parser.add_argument(+ "--insecure",+ action="store_true",+ help="Disable TLS certificate verification (only for self-signed development targets)",+ )+ parser.add_argument(+ "--allow-private-hosts",+ action="store_true",+ help="Allow pages, images, and redirect targets on loopback, link-local, "+ "and private networks (intranet or localhost pages you trust)",+ )args = parser.parse_args(argv)+ CONFIG["insecure"] = args.insecure+ CONFIG["allow_private_hosts"] = args.allow_private_hosts+ if args.insecure:+ print("[WARN] TLS certificate verification disabled (--insecure)", file=sys.stderr)if args.dir:CONFIG["output_dir"] = args.dir@@ -1099,8 +1164,4 @@ def main(argv: list[str] | None = None) -> int:if __name__ == "__main__":- if not _HELP_REQUESTED:- # Disable warnings for verify=False if needed, though often useful to see- import urllib3- urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)raise SystemExit(main())diff --git a/skills/ppt-master/scripts/tests/test_web_to_md_security.py b/skills/ppt-master/scripts/tests/test_web_to_md_security.pynew file mode 100644index 00000000..ab2b5491--- /dev/null+++ b/skills/ppt-master/scripts/tests/test_web_to_md_security.py@@ -0,0 +1,168 @@+#!/usr/bin/env python3+"""PPT Master - Web Conversion Security Tests++Check URL validation, TLS defaults, and failure paths with in-memory responses.++Usage:+ python3 -m unittest discover -s skills/ppt-master/scripts/tests -p test_web_to_md_security.py++Dependencies:+ requests, beautifulsoup4+"""++import io+import socket+import sys+import unittest+from contextlib import redirect_stderr, redirect_stdout+from pathlib import Path+from unittest.mock import Mock, patch++SCRIPTS_DIR = Path(__file__).resolve().parents[1]+if str(SCRIPTS_DIR) not in sys.path:+ sys.path.insert(0, str(SCRIPTS_DIR))++import source_to_md # noqa: E402++WEB_BACKEND_DIR = SCRIPTS_DIR / 'source_to_md'+if str(WEB_BACKEND_DIR) not in sys.path:+ sys.path.insert(0, str(WEB_BACKEND_DIR))++import web_to_md # noqa: E402+++PUBLIC_URL = 'https://8.8.8.8/page'+++class PublicUrlTests(unittest.TestCase):+ def setUp(self) -> None:+ dns_patch = patch.object(web_to_md.socket, 'getaddrinfo', side_effect=AssertionError('Unexpected DNS'))+ dns_patch.start()+ self.addCleanup(dns_patch.stop)++ def test_non_public_literal_addresses_are_rejected_without_dns(self) -> None:+ for host in ('169.254.169.254', '10.0.0.1', '172.16.0.1', '192.168.0.1',+ '127.0.0.1', '0.0.0.0', '[::1]', '[::]', '[fe80::1]', '[fc00::1]',+ '[::ffff:127.0.0.1]'):+ with self.subTest(host=host), self.assertRaisesRegex(ValueError, 'non-public'):+ web_to_md._validate_public_url(f'http://{host}/')++ def test_public_literal_addresses_are_accepted_without_dns(self) -> None:+ for host in ('8.8.8.8', '[2001:4860:4860::8888]'):+ with self.subTest(host=host):+ self.assertIsNone(web_to_md._validate_public_url(f'https://{host}/'))++ def test_non_http_or_malformed_urls_are_rejected(self) -> None:+ for url in ('file:///etc/passwd', 'ftp://8.8.8.8/', 'data:text/plain,hello',+ '//8.8.8.8/', 'http:///missing-host', 'http://[::1'):+ with self.subTest(url=url), self.assertRaises(ValueError):+ web_to_md._validate_public_url(url)++ def test_every_dns_answer_must_be_public(self) -> None:+ public = (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('8.8.8.8', 0))+ private = (socket.AF_INET6, socket.SOCK_STREAM, 6, '', ('fe80::1', 0, 0, 0))+ with patch.object(web_to_md.socket, 'getaddrinfo', return_value=[public]) as resolve:+ self.assertIsNone(web_to_md._validate_public_url('https://page.example/'))+ resolve.assert_called_once_with('page.example', None, type=socket.SOCK_STREAM)+ with patch.object(web_to_md.socket, 'getaddrinfo', return_value=[public, private]):+ with self.assertRaisesRegex(ValueError, 'non-public'):+ web_to_md._validate_public_url('https://page.example/')++ def test_dns_failure_or_empty_answers_fail_closed(self) -> None:+ with patch.object(web_to_md.socket, 'getaddrinfo', side_effect=socket.gaierror('unresolved')):+ with self.assertRaisesRegex(ValueError, 'Cannot validate URL hostname'):+ web_to_md._validate_public_url('https://page.example/')+ with patch.object(web_to_md.socket, 'getaddrinfo', return_value=[]):+ with self.assertRaisesRegex(ValueError, 'Cannot resolve URL hostname'):+ web_to_md._validate_public_url('https://page.example/')++ def test_initial_url_is_checked_before_either_backend_fetches(self) -> None:+ for use_curl in (False, True):+ backend = Mock()+ with self.subTest(curl=use_curl), \+ patch.object(web_to_md, 'curl_requests', backend if use_curl else None), \+ patch.object(web_to_md.requests, 'get', backend.get):+ with self.assertRaisesRegex(ValueError, 'non-public'):+ web_to_md._http_get('http://169.254.169.254/latest/meta-data/')+ backend.get.assert_not_called()++ def test_final_url_is_checked_and_response_closed_for_both_backends(self) -> None:+ for use_curl in (False, True):+ for final_url in ('http://10.0.0.1/private', 'file:///etc/passwd'):+ response = Mock(url=final_url)+ backend = Mock()+ backend.get.return_value = response+ with self.subTest(curl=use_curl, final_url=final_url), \+ patch.object(web_to_md, 'curl_requests', backend if use_curl else None), \+ patch.object(web_to_md.requests, 'get', backend.get):+ with self.assertRaises(ValueError):+ web_to_md._http_get(PUBLIC_URL)+ response.close.assert_called_once_with()++ def test_unsafe_page_or_image_redirect_does_not_write_output(self) -> None:+ page = Mock(+ url=PUBLIC_URL,+ content=b'<html><head><title>Page</title></head><body>'+ b'<article><p>Content</p><img src="/image.png"></article></body></html>',+ headers={'Content-Type': 'text/html; charset=utf-8'},+ )+ for unsafe_image in (False, True):+ blocked = Mock(url='http://169.254.169.254/secret')+ responses = [page, blocked] if unsafe_image else [blocked]+ with self.subTest(image=unsafe_image), \+ patch.object(web_to_md, 'curl_requests', None), \+ patch.object(web_to_md.requests, 'get', side_effect=responses), \+ patch.object(web_to_md.os, 'makedirs'), \+ patch('builtins.open') as write, redirect_stdout(io.StringIO()):+ result = web_to_md.process_url(PUBLIC_URL, '/unused/result.md')+ self.assertFalse(result[0])+ self.assertIn('non-public', result[2])+ write.assert_not_called()+ blocked.close.assert_called_once_with()+++class TlsVerificationTests(unittest.TestCase):+ def test_both_http_backends_verify_by_default_and_honor_explicit_opt_out(self) -> None:+ for use_curl in (False, True):+ for kwargs, expected in (({}, True), ({'verify': False}, False)):+ backend = Mock()+ backend.get.return_value = Mock(url=PUBLIC_URL)+ with self.subTest(curl=use_curl, kwargs=kwargs), \+ patch.object(web_to_md, 'curl_requests', backend if use_curl else None), \+ patch.object(web_to_md.requests, 'get', backend.get):+ web_to_md._http_get(PUBLIC_URL, **kwargs)+ self.assertIs(backend.get.call_args.kwargs['verify'], expected)+ if use_curl:+ self.assertEqual(backend.get.call_args.kwargs['impersonate'], web_to_md._CURL_IMPERSONATE)++ def test_pages_and_images_share_the_insecure_config(self) -> None:+ response = Mock(url=PUBLIC_URL, content=b'page', headers={'Content-Type': 'text/plain; charset=utf-8'})+ for insecure in (False, True):+ with self.subTest(insecure=insecure), patch.dict(web_to_md.CONFIG, insecure=insecure):+ with patch.object(web_to_md, '_http_get', return_value=response) as fetch:+ web_to_md.fetch_url(PUBLIC_URL)+ self.assertIs(fetch.call_args.kwargs['verify'], not insecure)+ content = web_to_md.BeautifulSoup('<img src="/image.png">', 'html.parser')+ with patch.object(web_to_md, '_http_get', side_effect=web_to_md._UnsafeUrlError('blocked')) as fetch, \+ patch.object(web_to_md.os, 'makedirs'):+ with self.assertRaises(web_to_md._UnsafeUrlError):+ web_to_md.download_and_rewrite_images(content, PUBLIC_URL, '/unused/images', 'images')+ self.assertIs(fetch.call_args.kwargs['verify'], not insecure)++ def test_cli_insecure_is_opt_in_and_warns(self) -> None:+ with patch.dict(web_to_md.CONFIG), \+ patch.object(web_to_md, 'process_url', return_value=(True, PUBLIC_URL, None, None)), \+ redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()) as errors:+ self.assertEqual(web_to_md.main([PUBLIC_URL, '--insecure']), 0)+ self.assertIs(web_to_md.CONFIG['insecure'], True)+ self.assertIn('TLS certificate verification disabled', errors.getvalue())+ self.assertEqual(web_to_md.main([PUBLIC_URL]), 0)+ self.assertIs(web_to_md.CONFIG['insecure'], False)++ def test_dispatcher_forwards_insecure_to_web_backend(self) -> None:+ args, unknown = source_to_md.build_parser().parse_known_args([PUBLIC_URL, '--insecure'])+ with patch.object(source_to_md, 'run_backend', return_value=1) as run:+ self.assertEqual(source_to_md.dispatch_single(PUBLIC_URL, 'web', '/unused/result.md', args, unknown), 1)+ command, script_name = run.call_args.args+ self.assertEqual(script_name, 'web_to_md.py')+ self.assertEqual(command.count('--insecure'), 1)