AI turns documents or topics into real, native PowerPoint decks—with native shapes, transitions and animations, data-backed charts and tables on demand, audio narration from speaker notes, and support for your own .pptx templates. · by Hugo He
fix(web_to_md): validate the host the client actually connects to, and pin connections to the validated addresses
hugohe33b45af2c3dab
diff --git a/skills/ppt-master/scripts/source_to_md/web_to_md.py b/skills/ppt-master/scripts/source_to_md/web_to_md.pyindex 3d685a2f..2e589e7f 100644--- a/skills/ppt-master/scripts/source_to_md/web_to_md.py+++ b/skills/ppt-master/scripts/source_to_md/web_to_md.py@@ -19,6 +19,12 @@ TLS fingerprint handling:'curl_cffi' is unavailable, it silently falls back to plain 'requests' — sonon-blocking sites still work without the extra dependency.+ Public-only requests pin validated addresses: curl_cffi uses CURLOPT_RESOLVE+ while requests uses connection-local validation and numeric socket targets.+ Both retain the URL hostname for TLS verification. Environment proxies are+ disabled in this mode because a proxy could resolve the target again;+ --allow-private-hosts restores proxy use and unrestricted address resolution.+Install for WeChat / Chinese-portal coverage:pip install curl_cffi@@ -76,6 +82,7 @@ if not _HELP_REQUESTED:try:import requestsfrom bs4 import BeautifulSoup, Comment, NavigableString, Tag+ from urllib3.util import parse_urlexcept ImportError:print("Error: This script requires 'requests' and 'beautifulsoup4'.", file=sys.stderr)print("Please run: pip install requests beautifulsoup4", file=sys.stderr)@@ -84,9 +91,11 @@ if not _HELP_REQUESTED:# Prefer curl_cffi for TLS-fingerprint impersonation (bypasses JA3 blocking on# sites like WeChat). Fall back to plain requests when it's not installed.try:+ from curl_cffi import CurlOpt # type: ignorefrom curl_cffi import requests as curl_requests # type: ignore_CURL_IMPERSONATE = "chrome120"except ImportError:+ CurlOpt = Nonecurl_requests = None_CURL_IMPERSONATE = None@@ -100,15 +109,43 @@ _NON_PUBLIC_IPV4_NETWORKS = tuple(ipaddress.ip_network(network) for network in ())-def _validate_public_url(url: str) -> None:- """Reject non-HTTP(S) URLs and hosts resolving to non-public addresses."""+def _check_url_characters(url: str) -> None:+ """Reject characters clients could strip or reinterpret before parsing."""+ if "\\" in url or any(char.isspace() or unicodedata.category(char) == "Cc" for char in url):+ raise _UnsafeUrlError("Refusing URL containing backslash, whitespace, or control characters")+++def _prepare_url(url: str) -> str:+ """Return the client's URL only when both parsers agree on its hostname."""+ _check_url_characters(url)try:parsed = urlparse(url)- hostname = parsed.hostname- except ValueError as exc:+ if parsed.scheme not in {"http", "https"} or not parsed.hostname:+ raise _UnsafeUrlError("Only HTTP(S) URLs with a hostname are allowed")+ prepared = requests.PreparedRequest()+ prepared.prepare_url(url, None)+ hostname = parse_url(prepared.url).host+ # The raw hostname must survive preparation unchanged, so a host the+ # client decodes differently (percent-encoded labels) is refused. An IDN+ # host legitimately becomes punycode; compare its IDNA form. urllib3+ # keeps IPv6 brackets; urlparse does not.+ raw_host = parsed.hostname+ if not raw_host.isascii():+ try:+ raw_host = raw_host.encode("idna").decode("ascii")+ except UnicodeError as exc:+ raise _UnsafeUrlError(f"Invalid internationalized hostname: {exc}") from exc+ if not hostname or raw_host.lower() != hostname.strip("[]").lower():+ raise _UnsafeUrlError("Refusing URL with inconsistent parsed hostnames")+ return prepared.url+ except (ValueError, requests.exceptions.RequestException) as exc:+ if isinstance(exc, _UnsafeUrlError):+ raiseraise _UnsafeUrlError(f"Invalid URL: {exc}") from exc- if parsed.scheme not in {"http", "https"} or not hostname:- raise _UnsafeUrlError("Only HTTP(S) URLs with a hostname are allowed")+++def _resolve_public_addresses(hostname: str) -> list:+ """Resolve once and validate every address before any socket is created."""try:addresses = [ipaddress.ip_address(hostname)]except ValueError:@@ -122,28 +159,98 @@ def _validate_public_url(url: str) -> None:if not addresses:raise _UnsafeUrlError(f"Cannot resolve URL hostname {hostname}")if CONFIG["allow_private_hosts"]:- return+ return addressesfor address in addresses:+ scoped = getattr(address, "scope_id", None)if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped:address = address.ipv4_mappedif (address.is_loopback or address.is_link_localor address.is_private or address.is_unspecifiedor address.is_multicast or address.is_reserved+ or scopedor any(address in network for network in _NON_PUBLIC_IPV4_NETWORKS)):raise _UnsafeUrlError(f"Refusing non-public URL target: {hostname} resolves to {address} ""(pass --allow-private-hosts for intranet or localhost pages)")+ return list(dict.fromkeys(addresses))+++def _validate_public_url(url: str) -> tuple[str, list]:+ """Return the prepared URL and its validated addresses for transport use."""+ prepared = _prepare_url(url)+ hostname = parse_url(prepared).host.strip("[]")+ return prepared, _resolve_public_addresses(hostname)+++class _PublicConnectionMixin:+ """Keep urllib3's hostname/TLS handling but connect only to checked IPs."""++ def _new_conn(self):+ from urllib3.exceptions import ConnectTimeoutError, NewConnectionError+ from urllib3.util import Timeout++ addresses = _resolve_public_addresses(self._dns_host)+ error = None+ for address in addresses:+ sock = None+ try:+ family = socket.AF_INET6 if address.version == 6 else socket.AF_INET+ sock = socket.socket(family, socket.SOCK_STREAM)+ for option in self.socket_options or ():+ sock.setsockopt(*option)+ if self.timeout is not Timeout.DEFAULT_TIMEOUT:+ sock.settimeout(self.timeout)+ if self.source_address:+ sock.bind(self.source_address)+ # Numeric addresses go straight to connect; no second DNS lookup.+ sock.connect((str(address), self.port))+ sys.audit("http.client.connect", self, self.host, self.port)+ return sock+ except OSError as exc:+ error = exc+ if sock is not None:+ sock.close()+ if isinstance(error, socket.timeout):+ raise ConnectTimeoutError(self, f"Connection to {self.host} timed out") from error+ raise NewConnectionError(self, f"Failed to establish a new connection: {error}") from error+++def _public_http_adapter():+ """Build isolated pools without patching requests or urllib3 globally."""+ from urllib3.connection import HTTPConnection, HTTPSConnection+ from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool++ class PublicHTTPConnection(_PublicConnectionMixin, HTTPConnection):+ pass++ class PublicHTTPSConnection(_PublicConnectionMixin, HTTPSConnection):+ pass++ class PublicHTTPConnectionPool(HTTPConnectionPool):+ ConnectionCls = PublicHTTPConnection++ class PublicHTTPSConnectionPool(HTTPSConnectionPool):+ ConnectionCls = PublicHTTPSConnection++ adapter = requests.adapters.HTTPAdapter()+ adapter.poolmanager.pool_classes_by_scheme = {+ "http": PublicHTTPConnectionPool,+ "https": PublicHTTPSConnectionPool,+ }+ return adapterdef _validate_response_redirect(response, **kwargs) -> None:"""Check requests redirects before its redirect engine sends the next hop."""try:- _validate_public_url(response.url)+ response_url, _ = _validate_public_url(response.url)if response.is_redirect:# Match requests' decoding of the HTTP Location header.location = response.headers["location"].encode("latin1").decode("utf8")- _validate_public_url(urljoin(response.url, location))+ _check_url_characters(location)+ next_url, _ = _validate_public_url(urljoin(response_url, location))+ response.headers["location"] = next_urlexcept _UnsafeUrlError:response.close()raise@@ -151,16 +258,23 @@ def _validate_response_redirect(response, **kwargs) -> None:def _curl_http_get(url: str, *, headers: dict | None, timeout: int | None,verify: bool, stream: bool):- """Follow curl redirects explicitly, retaining scoped response cookies."""+ """Pin each curl hop with CURLOPT_RESOLVE, retaining Chrome impersonation."""cookies = requests.cookies.RequestsCookieJar()headers = requests.structures.CaseInsensitiveDict(headers or {})max_redirects = requests.models.DEFAULT_REDIRECT_LIMITfor redirect_count in range(max_redirects + 1):- _validate_public_url(url)+ url, addresses = _validate_public_url(url)+ curl_options = {}+ if not CONFIG["allow_private_hosts"]:+ target = parse_url(url)+ port = target.port if target.port is not None else (443 if target.scheme == "https" else 80)+ # IPv6 addresses need brackets in libcurl's host:port:address list.+ pinned = ",".join(f"[{ip}]" if ip.version == 6 else str(ip) for ip in addresses)+ curl_options = {CurlOpt.RESOLVE: [f"{target.host}:{port}:{pinned}"], CurlOpt.PROXY: ""}response = curl_requests.get(url, headers=headers, timeout=timeout, verify=verify,impersonate=_CURL_IMPERSONATE, stream=stream,- allow_redirects=False, cookies=cookies,+ allow_redirects=False, cookies=cookies, curl_options=curl_options, quote=False,)try:_validate_public_url(response.url)@@ -173,8 +287,8 @@ def _curl_http_get(url: str, *, headers: dict | None, timeout: int | None,if not location:return responsetry:- next_url = urljoin(response.url, location)- _validate_public_url(next_url)+ _check_url_characters(location)+ next_url, _ = _validate_public_url(urljoin(response.url, location))if redirect_count >= max_redirects:raise requests.exceptions.TooManyRedirects(f"Exceeded {max_redirects} redirects.", response=response,@@ -202,21 +316,21 @@ def _http_get(url: str, *, headers: dict | None = None, timeout: int | None = NoTLS fingerprint (notably mp.weixin.qq.com). Signature mirrors the subset ofrequests.get() this script actually uses."""- _validate_public_url(url)if curl_requests is not None:+ return _curl_http_get(+ url, headers=headers, timeout=timeout, verify=verify, stream=stream,+ )+ url, _ = _validate_public_url(url)+ with requests.Session() as session:if not CONFIG["allow_private_hosts"]:- return _curl_http_get(- url, headers=headers, timeout=timeout, verify=verify, stream=stream,- )- response = curl_requests.get(url, headers=headers, timeout=timeout,- verify=verify, impersonate=_CURL_IMPERSONATE,- stream=stream)- else:- hooks = None if CONFIG["allow_private_hosts"] else {- "response": _validate_response_redirect,- }- response = requests.get(url, headers=headers, timeout=timeout,- verify=verify, stream=stream, hooks=hooks)+ session.trust_env = False+ adapter = _public_http_adapter()+ session.mount("http://", adapter)+ session.mount("https://", adapter)+ response = session.get(+ url, headers=headers, timeout=timeout, verify=verify, stream=stream,+ hooks={"response": _validate_response_redirect},+ )try:_validate_public_url(response.url)except _UnsafeUrlError:@@ -516,6 +630,7 @@ def resolve_content_image_url(img: Tag, page_url: str) -> str | None:src = value.strip()if not src or src.startswith(("data:", "javascript:", "blob:", "#")):continue+ _check_url_characters(src)resolved = urljoin(page_url, src)parsed = urlparse(resolved)if parsed.scheme in {"http", "https"} and parsed.netloc:diff --git a/skills/ppt-master/scripts/tests/test_audit_r3_sources.py b/skills/ppt-master/scripts/tests/test_audit_r3_sources.pyindex aa217b6f..476cd907 100644--- a/skills/ppt-master/scripts/tests/test_audit_r3_sources.py+++ b/skills/ppt-master/scripts/tests/test_audit_r3_sources.py@@ -145,7 +145,7 @@ class SourceHttpSecurityTests(unittest.TestCase):backend = Mock()with self.subTest(host=host, curl=use_curl), \patch.object(web_to_md, "curl_requests", backend if use_curl else None), \- patch.object(requests, "get", backend.get):+ patch.object(requests.Session, "get", backend.get):with self.assertRaisesRegex(ValueError, "non-public"):web_to_md._http_get(f"http://{host}/")backend.get.assert_not_called()@@ -158,7 +158,7 @@ class SourceHttpSecurityTests(unittest.TestCase):non_public = (family, socket.SOCK_STREAM, 6, "", (address, 0))with self.subTest(host=host), \patch.object(socket, "getaddrinfo", return_value=[public, non_public]), \- patch.object(requests, "get") as transport:+ patch.object(requests.Session, "get") as transport:with self.assertRaisesRegex(ValueError, "non-public"):web_to_md._http_get("https://source.example/")transport.assert_not_called()@@ -176,7 +176,7 @@ class SourceHttpSecurityTests(unittest.TestCase):with self.subTest(host=host):url = f"http://{host}/"response = Mock(url=url)- with patch.object(requests, "get", return_value=response) as transport:+ with patch.object(requests.Session, "get", return_value=response) as transport:self.assertIs(web_to_md._http_get(url), response)transport.assert_called_once()@@ -458,6 +458,7 @@ class CurlSourceRedirectTests(unittest.TestCase):def setUp(self) -> None:for guard in (patch.dict(web_to_md.CONFIG, allow_private_hosts=False),+ patch.object(web_to_md, "CurlOpt", SimpleNamespace(RESOLVE=10203, PROXY=10004)),patch.object(socket, "getaddrinfo", side_effect=AssertionError("Unexpected DNS")),patch.object(socket.socket, "connect", side_effect=AssertionError("Unexpected network")),):@@ -534,15 +535,20 @@ class CurlSourceRedirectTests(unittest.TestCase):for response in responses:response.close.assert_called_once_with()- def test_a01_curl_private_opt_in_keeps_automatic_redirects(self) -> None:- response = Mock(url=self.PRIVATE)- backend = SimpleNamespace(get=Mock(return_value=response))+ def test_a01_curl_private_opt_in_keeps_redirects_without_pinning(self) -> None:+ routes = {+ self.START: (302, {"Location": self.PRIVATE}),+ self.PRIVATE: (302, {"Location": self.FINAL}),+ self.FINAL: (200, {}),+ }with patch.dict(web_to_md.CONFIG, allow_private_hosts=True), \- patch.object(web_to_md, "curl_requests", backend):- self.assertIs(web_to_md._http_get(self.START), response)- backend.get.assert_called_once()- self.assertNotIn("allow_redirects", backend.get.call_args.kwargs)- response.close.assert_not_called()+ _memory_curl(routes) as (visited, responses):+ self.assertIs(web_to_md._http_get(self.START), responses[-1])+ self.assertEqual([item["url"] for item in visited], [self.START, self.PRIVATE, self.FINAL])+ for item in visited:+ self.assertEqual(item["kwargs"]["curl_options"], {})+ self.assertIs(item["kwargs"]["allow_redirects"], False)+ responses[-1].close.assert_not_called()def test_a01_curl_nonredirect_or_missing_location_returns_response(self) -> None:for status, headers in ((200, {"Location": self.PRIVATE}), (302, {})):diff --git a/skills/ppt-master/scripts/tests/test_web_to_md_security.py b/skills/ppt-master/scripts/tests/test_web_to_md_security.pyindex ab2b5491..acf4b83d 100644--- a/skills/ppt-master/scripts/tests/test_web_to_md_security.py+++ b/skills/ppt-master/scripts/tests/test_web_to_md_security.py@@ -12,12 +12,17 @@ Dependencies:import ioimport socket+import sslimport sys+import tempfileimport unittest-from contextlib import redirect_stderr, redirect_stdout+from contextlib import contextmanager, redirect_stderr, redirect_stdoutfrom pathlib import Path+from types import SimpleNamespacefrom unittest.mock import Mock, patch+from urllib3.util.ssl_ import resolve_cert_reqs+SCRIPTS_DIR = Path(__file__).resolve().parents[1]if str(SCRIPTS_DIR) not in sys.path:sys.path.insert(0, str(SCRIPTS_DIR))@@ -34,11 +39,62 @@ import web_to_md # noqa: E402PUBLIC_URL = 'https://8.8.8.8/page'+class _CurlHeaders(web_to_md.requests.structures.CaseInsensitiveDict):+ def get_list(self, name):+ return [self[name]] if name in self else []+++@contextmanager+def _offline_transport(routes, use_curl):+ """Replace transmission while retaining requests' preparation and redirects."""+ visited = []+ responses = []++ def respond(url):+ visited.append(url)+ status, headers = routes[url]+ response = web_to_md.requests.Response()+ response.url = url+ response.status_code = status+ response.headers = _CurlHeaders(headers)+ response._content = b'public content'+ response.raw = io.BytesIO(response._content)+ responses.append(response)+ return response++ def send(adapter, request, **kwargs):+ response = respond(request.url)+ response.request = request+ return response++ backend = SimpleNamespace(get=Mock(side_effect=lambda url, **kwargs: respond(url)))+ with patch.object(web_to_md, 'curl_requests', backend if use_curl else None), \+ patch.object(web_to_md.requests.adapters.HTTPAdapter, 'send', send):+ yield visited, responses+++def _dns_answer(address, port=0):+ family = socket.AF_INET6 if ':' in address else socket.AF_INET+ return [(family, socket.SOCK_STREAM, 6, '', (address, port))]+++def _http_socket():+ """Provide a socket substitute to exercise the real urllib3 connection path."""+ sock = Mock()+ sock.makefile.side_effect = lambda *args, **kwargs: io.BytesIO(+ b'HTTP/1.1 200 OK\r\nContent-Length: 6\r\nConnection: close\r\n\r\npublic'+ )+ return sock++class PublicUrlTests(unittest.TestCase):def setUp(self) -> None:dns_patch = patch.object(web_to_md.socket, 'getaddrinfo', side_effect=AssertionError('Unexpected DNS'))dns_patch.start()self.addCleanup(dns_patch.stop)+ options = patch.object(web_to_md, 'CurlOpt', SimpleNamespace(RESOLVE=10203, PROXY=10004))+ options.start()+ self.addCleanup(options.stop)def test_non_public_literal_addresses_are_rejected_without_dns(self) -> None:for host in ('169.254.169.254', '10.0.0.1', '172.16.0.1', '192.168.0.1',@@ -50,7 +106,9 @@ class PublicUrlTests(unittest.TestCase):def test_public_literal_addresses_are_accepted_without_dns(self) -> None:for host in ('8.8.8.8', '[2001:4860:4860::8888]'):with self.subTest(host=host):- self.assertIsNone(web_to_md._validate_public_url(f'https://{host}/'))+ url, addresses = web_to_md._validate_public_url(f'https://{host}/')+ self.assertEqual(url, f'https://{host}/')+ self.assertEqual([str(ip) for ip in addresses], [host.strip('[]')])def test_non_http_or_malformed_urls_are_rejected(self) -> None:for url in ('file:///etc/passwd', 'ftp://8.8.8.8/', 'data:text/plain,hello',@@ -62,7 +120,9 @@ class PublicUrlTests(unittest.TestCase):public = (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('8.8.8.8', 0))private = (socket.AF_INET6, socket.SOCK_STREAM, 6, '', ('fe80::1', 0, 0, 0))with patch.object(web_to_md.socket, 'getaddrinfo', return_value=[public]) as resolve:- self.assertIsNone(web_to_md._validate_public_url('https://page.example/'))+ url, addresses = web_to_md._validate_public_url('https://page.example/')+ self.assertEqual(url, 'https://page.example/')+ self.assertEqual([str(ip) for ip in addresses], ['8.8.8.8'])resolve.assert_called_once_with('page.example', None, type=socket.SOCK_STREAM)with patch.object(web_to_md.socket, 'getaddrinfo', return_value=[public, private]):with self.assertRaisesRegex(ValueError, 'non-public'):@@ -81,7 +141,7 @@ class PublicUrlTests(unittest.TestCase):backend = Mock()with self.subTest(curl=use_curl), \patch.object(web_to_md, 'curl_requests', backend if use_curl else None), \- patch.object(web_to_md.requests, 'get', backend.get):+ patch.object(web_to_md.requests.Session, 'get', backend.get):with self.assertRaisesRegex(ValueError, 'non-public'):web_to_md._http_get('http://169.254.169.254/latest/meta-data/')backend.get.assert_not_called()@@ -94,7 +154,7 @@ class PublicUrlTests(unittest.TestCase):backend.get.return_value = responsewith self.subTest(curl=use_curl, final_url=final_url), \patch.object(web_to_md, 'curl_requests', backend if use_curl else None), \- patch.object(web_to_md.requests, 'get', backend.get):+ patch.object(web_to_md.requests.Session, 'get', backend.get):with self.assertRaises(ValueError):web_to_md._http_get(PUBLIC_URL)response.close.assert_called_once_with()@@ -111,7 +171,7 @@ class PublicUrlTests(unittest.TestCase):responses = [page, blocked] if unsafe_image else [blocked]with self.subTest(image=unsafe_image), \patch.object(web_to_md, 'curl_requests', None), \- patch.object(web_to_md.requests, 'get', side_effect=responses), \+ patch.object(web_to_md.requests.Session, 'get', side_effect=responses), \patch.object(web_to_md.os, 'makedirs'), \patch('builtins.open') as write, redirect_stdout(io.StringIO()):result = web_to_md.process_url(PUBLIC_URL, '/unused/result.md')@@ -122,6 +182,11 @@ class PublicUrlTests(unittest.TestCase):class TlsVerificationTests(unittest.TestCase):+ def setUp(self) -> None:+ options = patch.object(web_to_md, 'CurlOpt', SimpleNamespace(RESOLVE=10203, PROXY=10004))+ options.start()+ self.addCleanup(options.stop)+def test_both_http_backends_verify_by_default_and_honor_explicit_opt_out(self) -> None:for use_curl in (False, True):for kwargs, expected in (({}, True), ({'verify': False}, False)):@@ -129,7 +194,7 @@ class TlsVerificationTests(unittest.TestCase):backend.get.return_value = Mock(url=PUBLIC_URL)with self.subTest(curl=use_curl, kwargs=kwargs), \patch.object(web_to_md, 'curl_requests', backend if use_curl else None), \- patch.object(web_to_md.requests, 'get', backend.get):+ patch.object(web_to_md.requests.Session, 'get', backend.get):web_to_md._http_get(PUBLIC_URL, **kwargs)self.assertIs(backend.get.call_args.kwargs['verify'], expected)if use_curl:@@ -166,3 +231,253 @@ class TlsVerificationTests(unittest.TestCase):command, script_name = run.call_args.argsself.assertEqual(script_name, 'web_to_md.py')self.assertEqual(command.count('--insecure'), 1)+++class TransportBindingTests(unittest.TestCase):+ def setUp(self) -> None:+ for guard in (+ patch.dict(web_to_md.CONFIG, allow_private_hosts=False),+ patch.object(web_to_md, 'CurlOpt', SimpleNamespace(RESOLVE=10203, PROXY=10004)),+ patch.object(web_to_md.socket, 'getaddrinfo', side_effect=AssertionError('Unexpected DNS')),+ patch.object(web_to_md.socket, 'socket', side_effect=AssertionError('Unexpected socket')),+ ):+ guard.start()+ self.addCleanup(guard.stop)++ def test_ambiguous_urls_are_refused_before_either_transport_is_created(self) -> None:+ urls = (+ 'http://127.0.0.1\\@example.com/',+ 'http://8.8.8.8/path\\name',+ ' http://8.8.8.8/',+ 'http://8.8.8.8/a b',+ 'http://8.8.8.8/\tpath',+ 'http://8.8.8.8/\r\npath',+ 'http://8.8.8.8/\x00path',+ 'http://8.8.8.8/\x7fpath',+ 'http://8.8.8.8/\x85path',+ 'http://%65xample.com/',+ )+ for use_curl in (False, True):+ for allow_private in (False, True):+ for url in urls:+ backend = Mock()+ with self.subTest(curl=use_curl, private=allow_private, url=url), \+ patch.dict(web_to_md.CONFIG, allow_private_hosts=allow_private), \+ patch.object(web_to_md, 'curl_requests', backend if use_curl else None), \+ patch.object(web_to_md.requests, 'Session') as session:+ with self.assertRaises(web_to_md._UnsafeUrlError):+ web_to_md._http_get(url)+ backend.get.assert_not_called()+ session.assert_not_called()+ socket.socket.assert_not_called()++ def test_ambiguous_image_urls_are_refused_before_transport(self) -> None:+ for use_curl in (False, True):+ for src in ('http://127.0.0.1\\@example.com/', 'http://8.8.8.8/	image.png'):+ content = web_to_md.BeautifulSoup(f'<img src="{src}">', 'html.parser')+ backend = Mock()+ with self.subTest(curl=use_curl, src=src), tempfile.TemporaryDirectory() as tmp, \+ patch.object(web_to_md, 'curl_requests', backend if use_curl else None), \+ patch.object(web_to_md.requests, 'Session') as session:+ with self.assertRaises(web_to_md._UnsafeUrlError):+ web_to_md.download_and_rewrite_images(content, PUBLIC_URL, tmp, 'images')+ self.assertEqual(list(Path(tmp).iterdir()), [])+ backend.get.assert_not_called()+ session.assert_not_called()+ socket.socket.assert_not_called()++ def test_every_redirect_rejects_ambiguous_locations_even_with_private_opt_in(self) -> None:+ middle = 'https://8.8.8.8/middle'+ for use_curl in (False, True):+ for allow_private in (False, True):+ for location in ('http://127.0.0.1\\@example.com/', '/\tpath', 'http://%65xample.com/'):+ routes = {PUBLIC_URL: (302, {'Location': middle}), middle: (302, {'Location': location})}+ with self.subTest(curl=use_curl, private=allow_private, location=location), \+ patch.dict(web_to_md.CONFIG, allow_private_hosts=allow_private), \+ _offline_transport(routes, use_curl) as (visited, responses):+ with self.assertRaises(web_to_md._UnsafeUrlError):+ web_to_md._http_get(PUBLIC_URL)+ self.assertEqual(visited, [PUBLIC_URL, middle])+ self.assertTrue(responses[-1].raw.closed)+ socket.socket.assert_not_called()++ def test_both_backends_send_the_prepared_url_on_every_public_hop(self) -> None:+ start = 'http://public.example/~start'+ final = 'http://public.example/~end'+ routes = {start: (302, {'Location': '/%7eend'}), final: (200, {})}+ for use_curl in (False, True):+ with self.subTest(curl=use_curl), \+ patch.object(socket, 'getaddrinfo', return_value=_dns_answer('8.8.8.8')), \+ _offline_transport(routes, use_curl) as (visited, _):+ response = web_to_md._http_get('HTTP://PUBLIC.EXAMPLE/%7estart')+ self.assertEqual(response.content, b'public content')+ self.assertEqual(visited, [start, final])++ def test_requests_rebinding_is_refused_before_socket_creation(self) -> None:+ for scheme in ('http', 'https'):+ for rebound in (_dns_answer('169.254.169.254'),+ _dns_answer('8.8.8.8') + _dns_answer('169.254.169.254')):+ with self.subTest(scheme=scheme, rebound=rebound), \+ patch.object(web_to_md, 'curl_requests', None), \+ patch.object(socket, 'getaddrinfo', side_effect=[_dns_answer('8.8.8.8'), rebound]) as dns:+ with self.assertRaisesRegex(web_to_md._UnsafeUrlError, 'non-public'):+ web_to_md._http_get(f'{scheme}://rebind.example/')+ self.assertEqual(dns.call_count, 2)+ socket.socket.assert_not_called()++ def test_requests_rebinding_on_a_redirect_is_refused_before_its_socket(self) -> None:+ sock = _http_socket()+ sock.makefile.side_effect = lambda *args: io.BytesIO(+ b'HTTP/1.1 302 Found\r\nLocation: http://rebind.example/private\r\n'+ b'Content-Length: 0\r\nConnection: close\r\n\r\n'+ )+ with patch.object(web_to_md, 'curl_requests', None), \+ patch.object(socket, 'socket', return_value=sock) as create_socket, \+ patch.object(socket, 'getaddrinfo', side_effect=[+ _dns_answer('8.8.8.8'), _dns_answer('169.254.169.254'),+ ]) as dns:+ with self.assertRaisesRegex(web_to_md._UnsafeUrlError, 'non-public'):+ web_to_md._http_get('http://8.8.8.8/start')+ self.assertEqual(dns.call_count, 2)+ create_socket.assert_called_once()+ sock.connect.assert_called_once_with(('8.8.8.8', 80))++ def test_curl_rebinding_on_a_redirect_is_refused_before_transport(self) -> None:+ routes = {PUBLIC_URL: (302, {'Location': 'http://rebind.example/private'})}+ with _offline_transport(routes, True) as (visited, responses), \+ patch.object(socket, 'getaddrinfo', side_effect=[+ _dns_answer('8.8.8.8'), _dns_answer('169.254.169.254'),+ ]) as dns:+ with self.assertRaisesRegex(web_to_md._UnsafeUrlError, 'non-public'):+ web_to_md._http_get(PUBLIC_URL)+ self.assertEqual(dns.call_count, 2)+ self.assertEqual(visited, [PUBLIC_URL])+ self.assertTrue(responses[0].raw.closed)++ def test_requests_public_connections_keep_host_and_tls_verification(self) -> None:+ for scheme in ('http', 'https'):+ sock = _http_socket()+ with self.subTest(scheme=scheme), patch.object(web_to_md, 'curl_requests', None), \+ patch.object(socket, 'getaddrinfo', return_value=_dns_answer('8.8.8.8')), \+ patch.object(socket, 'socket', return_value=sock), \+ patch.dict(web_to_md.os.environ, HTTP_PROXY='http://127.0.0.1:9',+ HTTPS_PROXY='http://127.0.0.1:9', ALL_PROXY='http://127.0.0.1:9'), \+ patch('urllib3.connection._ssl_wrap_socket_and_match_hostname',+ return_value=SimpleNamespace(socket=sock, is_verified=True)) as tls:+ response = web_to_md._http_get(f'{scheme}://public.example/')+ self.assertEqual(response.content, b'public')+ sock.connect.assert_called_once_with(('8.8.8.8', 443 if scheme == 'https' else 80))+ self.assertIn(b'Host: public.example\r\n', sock.sendall.call_args.args[0])+ if scheme == 'https':+ self.assertEqual(tls.call_args.kwargs['server_hostname'], 'public.example')+ self.assertEqual(resolve_cert_reqs(tls.call_args.kwargs['cert_reqs']), ssl.CERT_REQUIRED)+ self.assertIsNone(tls.call_args.kwargs['assert_hostname'])++ def test_requests_connects_only_to_validated_ipv4_and_ipv6_answers(self) -> None:+ first, second = _http_socket(), _http_socket()+ first.connect.side_effect = OSError('unreachable')+ answers = _dns_answer('8.8.8.8') + _dns_answer('2001:4860:4860::8888')+ with patch.object(web_to_md, 'curl_requests', None), \+ patch.object(socket, 'getaddrinfo', return_value=answers), \+ patch.object(socket, 'socket', side_effect=[first, second]):+ response = web_to_md._http_get('http://public.example:8080/')+ self.assertEqual(response.content, b'public')+ first.connect.assert_called_once_with(('8.8.8.8', 8080))+ first.close.assert_called_once()+ second.connect.assert_called_once_with(('2001:4860:4860::8888', 8080))++ def test_curl_pins_public_addresses_despite_rebinding_at_transport(self) -> None:+ connected = []++ def get(url, **kwargs):+ # The answer available at connect time has changed. A pinned backend+ # uses RESOLVE instead of consulting it for its destination.+ rebound = socket.getaddrinfo('mp.weixin.qq.com', None, type=socket.SOCK_STREAM)+ self.assertEqual(rebound[0][4][0], '169.254.169.254')+ options = kwargs['curl_options']+ self.assertEqual(options[web_to_md.CurlOpt.RESOLVE], ['mp.weixin.qq.com:443:8.8.8.8'])+ connected.extend(options[web_to_md.CurlOpt.RESOLVE][0].split(':')[2:])+ self.assertEqual(options[web_to_md.CurlOpt.PROXY], '')+ self.assertEqual(kwargs['impersonate'], web_to_md._CURL_IMPERSONATE)+ self.assertIs(kwargs['allow_redirects'], False)+ return Mock(url=url, status_code=200)++ with patch.object(web_to_md, 'curl_requests', SimpleNamespace(get=get)), \+ patch.object(socket, 'getaddrinfo', side_effect=[+ _dns_answer('8.8.8.8'), _dns_answer('169.254.169.254'), _dns_answer('8.8.8.8'),+ ]):+ web_to_md._http_get('https://mp.weixin.qq.com/')+ self.assertEqual(connected, ['8.8.8.8'])++ def test_curl_recomputes_pins_for_each_redirect_and_explicit_port(self) -> None:+ calls = []+ start = 'https://mp.weixin.qq.com/start'+ final = 'https://images.example:8443/photo'++ def get(url, **kwargs):+ calls.append((url, kwargs['curl_options'][web_to_md.CurlOpt.RESOLVE]))+ headers = _CurlHeaders({'Location': final}) if url == start else _CurlHeaders()+ return Mock(url=url, status_code=302 if url == start else 200, headers=headers)++ def resolve(host, *args, **kwargs):+ return _dns_answer('8.8.8.8' if host == 'mp.weixin.qq.com' else '2001:4860:4860::8888')++ with patch.object(web_to_md, 'curl_requests', SimpleNamespace(get=get)), \+ patch.object(socket, 'getaddrinfo', side_effect=resolve):+ web_to_md._http_get(start, stream=True)+ self.assertEqual(calls, [+ (start, ['mp.weixin.qq.com:443:8.8.8.8']),+ (final, ['images.example:8443:[2001:4860:4860::8888]']),+ ])++ def test_private_opt_in_uses_the_regular_requests_connection(self) -> None:+ sock = _http_socket()+ with patch.dict(web_to_md.CONFIG, allow_private_hosts=True), \+ patch.dict(web_to_md.os.environ, {'NO_PROXY': '*', 'no_proxy': '*'}), \+ patch.object(web_to_md, 'curl_requests', None), \+ patch.object(socket, 'getaddrinfo', return_value=_dns_answer('127.0.0.1', 80)), \+ patch.object(socket, 'socket', return_value=sock):+ self.assertEqual(web_to_md._http_get('http://127.0.0.1/').content, b'public')+ sock.connect.assert_called_once_with(('127.0.0.1', 80))++ def test_installed_curl_api_applies_resolve_without_network(self) -> None:+ try:+ from curl_cffi import Curl, CurlOpt, requests as curl_requests+ except ImportError:+ self.skipTest('optional curl_cffi is not installed')+ applied = {}+ original_setopt = Curl.setopt++ def setopt(curl, option, value):+ applied[option] = value+ return original_setopt(curl, option, value)++ url = 'https://mp.weixin.qq.com/'+ response = Mock(url=url, status_code=200)+ with patch.object(web_to_md, 'CurlOpt', CurlOpt), \+ patch.object(web_to_md, 'curl_requests', curl_requests), \+ patch.object(socket, 'getaddrinfo', return_value=_dns_answer('8.8.8.8')), \+ patch.object(Curl, 'setopt', setopt), patch.object(Curl, 'perform') as perform, \+ patch.object(curl_requests.Session, '_parse_response', return_value=response):+ self.assertIs(web_to_md._http_get(url), response)+ perform.assert_called_once()+ self.assertEqual(applied[CurlOpt.RESOLVE], ['mp.weixin.qq.com:443:8.8.8.8'])+ self.assertEqual(applied[CurlOpt.PROXY], '')+ socket.socket.assert_not_called()+++class PreparedUrlEdgeTests(unittest.TestCase):+ """The parser-agreement check compares the prepared URL, and HTML padding is not ambiguity."""++ def test_idn_hosts_are_prepared_to_punycode_not_refused(self) -> None:+ prepared = web_to_md._prepare_url("https://例子.测试/路径")+ self.assertTrue(prepared.startswith("https://xn--fsqu00a.xn--0zwm56d/"), prepared)++ def test_padded_image_src_is_stripped_before_the_character_check(self) -> None:+ soup = web_to_md.BeautifulSoup('<img src=" https://8.8.8.8/a.png ">', "html.parser")+ self.assertEqual(+ web_to_md.resolve_content_image_url(soup.img, "https://8.8.8.8/"), "https://8.8.8.8/a.png"+ )+ soup = web_to_md.BeautifulSoup('<img src="https://8.8.8.8/a b.png">', "html.parser")+ with self.assertRaises(web_to_md._UnsafeUrlError):+ web_to_md.resolve_content_image_url(soup.img, "https://8.8.8.8/")