Skip to content

AI turns documents or topics into real, native PowerPoint decks—with native shapes, transitions and animations, data-backed charts and tables on demand, audio narration from speaker notes, and support for your own .pptx templates. · by Hugo He

fix(web_to_md): validate the host the client actually connects to, and pin connections to the validated addresses

hugohe33b45af2c3dab
diff --git a/skills/ppt-master/scripts/source_to_md/web_to_md.py b/skills/ppt-master/scripts/source_to_md/web_to_md.py
index 3d685a2f..2e589e7f 100644
--- a/skills/ppt-master/scripts/source_to_md/web_to_md.py
+++ b/skills/ppt-master/scripts/source_to_md/web_to_md.py
@@ -19,6 +19,12 @@ TLS fingerprint handling:
'curl_cffi' is unavailable, it silently falls back to plain 'requests' — so
non-blocking sites still work without the extra dependency.
+ Public-only requests pin validated addresses: curl_cffi uses CURLOPT_RESOLVE
+ while requests uses connection-local validation and numeric socket targets.
+ Both retain the URL hostname for TLS verification. Environment proxies are
+ disabled in this mode because a proxy could resolve the target again;
+ --allow-private-hosts restores proxy use and unrestricted address resolution.
+
Install for WeChat / Chinese-portal coverage:
pip install curl_cffi
@@ -76,6 +82,7 @@ if not _HELP_REQUESTED:
try:
import requests
from bs4 import BeautifulSoup, Comment, NavigableString, Tag
+ from urllib3.util import parse_url
except ImportError:
print("Error: This script requires 'requests' and 'beautifulsoup4'.", file=sys.stderr)
print("Please run: pip install requests beautifulsoup4", file=sys.stderr)
@@ -84,9 +91,11 @@ if not _HELP_REQUESTED:
# Prefer curl_cffi for TLS-fingerprint impersonation (bypasses JA3 blocking on
# sites like WeChat). Fall back to plain requests when it's not installed.
try:
+ from curl_cffi import CurlOpt # type: ignore
from curl_cffi import requests as curl_requests # type: ignore
_CURL_IMPERSONATE = "chrome120"
except ImportError:
+ CurlOpt = None
curl_requests = None
_CURL_IMPERSONATE = None
@@ -100,15 +109,43 @@ _NON_PUBLIC_IPV4_NETWORKS = tuple(ipaddress.ip_network(network) for network in (
))
-def _validate_public_url(url: str) -> None:
- """Reject non-HTTP(S) URLs and hosts resolving to non-public addresses."""
+def _check_url_characters(url: str) -> None:
+ """Reject characters clients could strip or reinterpret before parsing."""
+ if "\\" in url or any(char.isspace() or unicodedata.category(char) == "Cc" for char in url):
+ raise _UnsafeUrlError("Refusing URL containing backslash, whitespace, or control characters")
+
+
+def _prepare_url(url: str) -> str:
+ """Return the client's URL only when both parsers agree on its hostname."""
+ _check_url_characters(url)
try:
parsed = urlparse(url)
- hostname = parsed.hostname
- except ValueError as exc:
+ if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+ raise _UnsafeUrlError("Only HTTP(S) URLs with a hostname are allowed")
+ prepared = requests.PreparedRequest()
+ prepared.prepare_url(url, None)
+ hostname = parse_url(prepared.url).host
+ # The raw hostname must survive preparation unchanged, so a host the
+ # client decodes differently (percent-encoded labels) is refused. An IDN
+ # host legitimately becomes punycode; compare its IDNA form. urllib3
+ # keeps IPv6 brackets; urlparse does not.
+ raw_host = parsed.hostname
+ if not raw_host.isascii():
+ try:
+ raw_host = raw_host.encode("idna").decode("ascii")
+ except UnicodeError as exc:
+ raise _UnsafeUrlError(f"Invalid internationalized hostname: {exc}") from exc
+ if not hostname or raw_host.lower() != hostname.strip("[]").lower():
+ raise _UnsafeUrlError("Refusing URL with inconsistent parsed hostnames")
+ return prepared.url
+ except (ValueError, requests.exceptions.RequestException) as exc:
+ if isinstance(exc, _UnsafeUrlError):
+ raise
raise _UnsafeUrlError(f"Invalid URL: {exc}") from exc
- if parsed.scheme not in {"http", "https"} or not hostname:
- raise _UnsafeUrlError("Only HTTP(S) URLs with a hostname are allowed")
+
+
+def _resolve_public_addresses(hostname: str) -> list:
+ """Resolve once and validate every address before any socket is created."""
try:
addresses = [ipaddress.ip_address(hostname)]
except ValueError:
@@ -122,28 +159,98 @@ def _validate_public_url(url: str) -> None:
if not addresses:
raise _UnsafeUrlError(f"Cannot resolve URL hostname {hostname}")
if CONFIG["allow_private_hosts"]:
- return
+ return addresses
for address in addresses:
+ scoped = getattr(address, "scope_id", None)
if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped:
address = address.ipv4_mapped
if (address.is_loopback or address.is_link_local
or address.is_private or address.is_unspecified
or address.is_multicast or address.is_reserved
+ or scoped
or any(address in network for network in _NON_PUBLIC_IPV4_NETWORKS)):
raise _UnsafeUrlError(
f"Refusing non-public URL target: {hostname} resolves to {address} "
"(pass --allow-private-hosts for intranet or localhost pages)"
)
+ return list(dict.fromkeys(addresses))
+
+
+def _validate_public_url(url: str) -> tuple[str, list]:
+ """Return the prepared URL and its validated addresses for transport use."""
+ prepared = _prepare_url(url)
+ hostname = parse_url(prepared).host.strip("[]")
+ return prepared, _resolve_public_addresses(hostname)
+
+
+class _PublicConnectionMixin:
+ """Keep urllib3's hostname/TLS handling but connect only to checked IPs."""
+
+ def _new_conn(self):
+ from urllib3.exceptions import ConnectTimeoutError, NewConnectionError
+ from urllib3.util import Timeout
+
+ addresses = _resolve_public_addresses(self._dns_host)
+ error = None
+ for address in addresses:
+ sock = None
+ try:
+ family = socket.AF_INET6 if address.version == 6 else socket.AF_INET
+ sock = socket.socket(family, socket.SOCK_STREAM)
+ for option in self.socket_options or ():
+ sock.setsockopt(*option)
+ if self.timeout is not Timeout.DEFAULT_TIMEOUT:
+ sock.settimeout(self.timeout)
+ if self.source_address:
+ sock.bind(self.source_address)
+ # Numeric addresses go straight to connect; no second DNS lookup.
+ sock.connect((str(address), self.port))
+ sys.audit("http.client.connect", self, self.host, self.port)
+ return sock
+ except OSError as exc:
+ error = exc
+ if sock is not None:
+ sock.close()
+ if isinstance(error, socket.timeout):
+ raise ConnectTimeoutError(self, f"Connection to {self.host} timed out") from error
+ raise NewConnectionError(self, f"Failed to establish a new connection: {error}") from error
+
+
+def _public_http_adapter():
+ """Build isolated pools without patching requests or urllib3 globally."""
+ from urllib3.connection import HTTPConnection, HTTPSConnection
+ from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool
+
+ class PublicHTTPConnection(_PublicConnectionMixin, HTTPConnection):
+ pass
+
+ class PublicHTTPSConnection(_PublicConnectionMixin, HTTPSConnection):
+ pass
+
+ class PublicHTTPConnectionPool(HTTPConnectionPool):
+ ConnectionCls = PublicHTTPConnection
+
+ class PublicHTTPSConnectionPool(HTTPSConnectionPool):
+ ConnectionCls = PublicHTTPSConnection
+
+ adapter = requests.adapters.HTTPAdapter()
+ adapter.poolmanager.pool_classes_by_scheme = {
+ "http": PublicHTTPConnectionPool,
+ "https": PublicHTTPSConnectionPool,
+ }
+ return adapter
def _validate_response_redirect(response, **kwargs) -> None:
"""Check requests redirects before its redirect engine sends the next hop."""
try:
- _validate_public_url(response.url)
+ response_url, _ = _validate_public_url(response.url)
if response.is_redirect:
# Match requests' decoding of the HTTP Location header.
location = response.headers["location"].encode("latin1").decode("utf8")
- _validate_public_url(urljoin(response.url, location))
+ _check_url_characters(location)
+ next_url, _ = _validate_public_url(urljoin(response_url, location))
+ response.headers["location"] = next_url
except _UnsafeUrlError:
response.close()
raise
@@ -151,16 +258,23 @@ def _validate_response_redirect(response, **kwargs) -> None:
def _curl_http_get(url: str, *, headers: dict | None, timeout: int | None,
verify: bool, stream: bool):
- """Follow curl redirects explicitly, retaining scoped response cookies."""
+ """Pin each curl hop with CURLOPT_RESOLVE, retaining Chrome impersonation."""
cookies = requests.cookies.RequestsCookieJar()
headers = requests.structures.CaseInsensitiveDict(headers or {})
max_redirects = requests.models.DEFAULT_REDIRECT_LIMIT
for redirect_count in range(max_redirects + 1):
- _validate_public_url(url)
+ url, addresses = _validate_public_url(url)
+ curl_options = {}
+ if not CONFIG["allow_private_hosts"]:
+ target = parse_url(url)
+ port = target.port if target.port is not None else (443 if target.scheme == "https" else 80)
+ # IPv6 addresses need brackets in libcurl's host:port:address list.
+ pinned = ",".join(f"[{ip}]" if ip.version == 6 else str(ip) for ip in addresses)
+ curl_options = {CurlOpt.RESOLVE: [f"{target.host}:{port}:{pinned}"], CurlOpt.PROXY: ""}
response = curl_requests.get(
url, headers=headers, timeout=timeout, verify=verify,
impersonate=_CURL_IMPERSONATE, stream=stream,
- allow_redirects=False, cookies=cookies,
+ allow_redirects=False, cookies=cookies, curl_options=curl_options, quote=False,
)
try:
_validate_public_url(response.url)
@@ -173,8 +287,8 @@ def _curl_http_get(url: str, *, headers: dict | None, timeout: int | None,
if not location:
return response
try:
- next_url = urljoin(response.url, location)
- _validate_public_url(next_url)
+ _check_url_characters(location)
+ next_url, _ = _validate_public_url(urljoin(response.url, location))
if redirect_count >= max_redirects:
raise requests.exceptions.TooManyRedirects(
f"Exceeded {max_redirects} redirects.", response=response,
@@ -202,21 +316,21 @@ def _http_get(url: str, *, headers: dict | None = None, timeout: int | None = No
TLS fingerprint (notably mp.weixin.qq.com). Signature mirrors the subset of
requests.get() this script actually uses.
"""
- _validate_public_url(url)
if curl_requests is not None:
+ return _curl_http_get(
+ url, headers=headers, timeout=timeout, verify=verify, stream=stream,
+ )
+ url, _ = _validate_public_url(url)
+ with requests.Session() as session:
if not CONFIG["allow_private_hosts"]:
- return _curl_http_get(
- url, headers=headers, timeout=timeout, verify=verify, stream=stream,
- )
- response = curl_requests.get(url, headers=headers, timeout=timeout,
- verify=verify, impersonate=_CURL_IMPERSONATE,
- stream=stream)
- else:
- hooks = None if CONFIG["allow_private_hosts"] else {
- "response": _validate_response_redirect,
- }
- response = requests.get(url, headers=headers, timeout=timeout,
- verify=verify, stream=stream, hooks=hooks)
+ session.trust_env = False
+ adapter = _public_http_adapter()
+ session.mount("http://", adapter)
+ session.mount("https://", adapter)
+ response = session.get(
+ url, headers=headers, timeout=timeout, verify=verify, stream=stream,
+ hooks={"response": _validate_response_redirect},
+ )
try:
_validate_public_url(response.url)
except _UnsafeUrlError:
@@ -516,6 +630,7 @@ def resolve_content_image_url(img: Tag, page_url: str) -> str | None:
src = value.strip()
if not src or src.startswith(("data:", "javascript:", "blob:", "#")):
continue
+ _check_url_characters(src)
resolved = urljoin(page_url, src)
parsed = urlparse(resolved)
if parsed.scheme in {"http", "https"} and parsed.netloc:
diff --git a/skills/ppt-master/scripts/tests/test_audit_r3_sources.py b/skills/ppt-master/scripts/tests/test_audit_r3_sources.py
index aa217b6f..476cd907 100644
--- a/skills/ppt-master/scripts/tests/test_audit_r3_sources.py
+++ b/skills/ppt-master/scripts/tests/test_audit_r3_sources.py
@@ -145,7 +145,7 @@ class SourceHttpSecurityTests(unittest.TestCase):
backend = Mock()
with self.subTest(host=host, curl=use_curl), \
patch.object(web_to_md, "curl_requests", backend if use_curl else None), \
- patch.object(requests, "get", backend.get):
+ patch.object(requests.Session, "get", backend.get):
with self.assertRaisesRegex(ValueError, "non-public"):
web_to_md._http_get(f"http://{host}/")
backend.get.assert_not_called()
@@ -158,7 +158,7 @@ class SourceHttpSecurityTests(unittest.TestCase):
non_public = (family, socket.SOCK_STREAM, 6, "", (address, 0))
with self.subTest(host=host), \
patch.object(socket, "getaddrinfo", return_value=[public, non_public]), \
- patch.object(requests, "get") as transport:
+ patch.object(requests.Session, "get") as transport:
with self.assertRaisesRegex(ValueError, "non-public"):
web_to_md._http_get("https://source.example/")
transport.assert_not_called()
@@ -176,7 +176,7 @@ class SourceHttpSecurityTests(unittest.TestCase):
with self.subTest(host=host):
url = f"http://{host}/"
response = Mock(url=url)
- with patch.object(requests, "get", return_value=response) as transport:
+ with patch.object(requests.Session, "get", return_value=response) as transport:
self.assertIs(web_to_md._http_get(url), response)
transport.assert_called_once()
@@ -458,6 +458,7 @@ class CurlSourceRedirectTests(unittest.TestCase):
def setUp(self) -> None:
for guard in (
patch.dict(web_to_md.CONFIG, allow_private_hosts=False),
+ patch.object(web_to_md, "CurlOpt", SimpleNamespace(RESOLVE=10203, PROXY=10004)),
patch.object(socket, "getaddrinfo", side_effect=AssertionError("Unexpected DNS")),
patch.object(socket.socket, "connect", side_effect=AssertionError("Unexpected network")),
):
@@ -534,15 +535,20 @@ class CurlSourceRedirectTests(unittest.TestCase):
for response in responses:
response.close.assert_called_once_with()
- def test_a01_curl_private_opt_in_keeps_automatic_redirects(self) -> None:
- response = Mock(url=self.PRIVATE)
- backend = SimpleNamespace(get=Mock(return_value=response))
+ def test_a01_curl_private_opt_in_keeps_redirects_without_pinning(self) -> None:
+ routes = {
+ self.START: (302, {"Location": self.PRIVATE}),
+ self.PRIVATE: (302, {"Location": self.FINAL}),
+ self.FINAL: (200, {}),
+ }
with patch.dict(web_to_md.CONFIG, allow_private_hosts=True), \
- patch.object(web_to_md, "curl_requests", backend):
- self.assertIs(web_to_md._http_get(self.START), response)
- backend.get.assert_called_once()
- self.assertNotIn("allow_redirects", backend.get.call_args.kwargs)
- response.close.assert_not_called()
+ _memory_curl(routes) as (visited, responses):
+ self.assertIs(web_to_md._http_get(self.START), responses[-1])
+ self.assertEqual([item["url"] for item in visited], [self.START, self.PRIVATE, self.FINAL])
+ for item in visited:
+ self.assertEqual(item["kwargs"]["curl_options"], {})
+ self.assertIs(item["kwargs"]["allow_redirects"], False)
+ responses[-1].close.assert_not_called()
def test_a01_curl_nonredirect_or_missing_location_returns_response(self) -> None:
for status, headers in ((200, {"Location": self.PRIVATE}), (302, {})):
diff --git a/skills/ppt-master/scripts/tests/test_web_to_md_security.py b/skills/ppt-master/scripts/tests/test_web_to_md_security.py
index ab2b5491..acf4b83d 100644
--- a/skills/ppt-master/scripts/tests/test_web_to_md_security.py
+++ b/skills/ppt-master/scripts/tests/test_web_to_md_security.py
@@ -12,12 +12,17 @@ Dependencies:
import io
import socket
+import ssl
import sys
+import tempfile
import unittest
-from contextlib import redirect_stderr, redirect_stdout
+from contextlib import contextmanager, redirect_stderr, redirect_stdout
from pathlib import Path
+from types import SimpleNamespace
from unittest.mock import Mock, patch
+from urllib3.util.ssl_ import resolve_cert_reqs
+
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
if str(SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPTS_DIR))
@@ -34,11 +39,62 @@ import web_to_md # noqa: E402
PUBLIC_URL = 'https://8.8.8.8/page'
+class _CurlHeaders(web_to_md.requests.structures.CaseInsensitiveDict):
+ def get_list(self, name):
+ return [self[name]] if name in self else []
+
+
+@contextmanager
+def _offline_transport(routes, use_curl):
+ """Replace transmission while retaining requests' preparation and redirects."""
+ visited = []
+ responses = []
+
+ def respond(url):
+ visited.append(url)
+ status, headers = routes[url]
+ response = web_to_md.requests.Response()
+ response.url = url
+ response.status_code = status
+ response.headers = _CurlHeaders(headers)
+ response._content = b'public content'
+ response.raw = io.BytesIO(response._content)
+ responses.append(response)
+ return response
+
+ def send(adapter, request, **kwargs):
+ response = respond(request.url)
+ response.request = request
+ return response
+
+ backend = SimpleNamespace(get=Mock(side_effect=lambda url, **kwargs: respond(url)))
+ with patch.object(web_to_md, 'curl_requests', backend if use_curl else None), \
+ patch.object(web_to_md.requests.adapters.HTTPAdapter, 'send', send):
+ yield visited, responses
+
+
+def _dns_answer(address, port=0):
+ family = socket.AF_INET6 if ':' in address else socket.AF_INET
+ return [(family, socket.SOCK_STREAM, 6, '', (address, port))]
+
+
+def _http_socket():
+ """Provide a socket substitute to exercise the real urllib3 connection path."""
+ sock = Mock()
+ sock.makefile.side_effect = lambda *args, **kwargs: io.BytesIO(
+ b'HTTP/1.1 200 OK\r\nContent-Length: 6\r\nConnection: close\r\n\r\npublic'
+ )
+ return sock
+
+
class PublicUrlTests(unittest.TestCase):
def setUp(self) -> None:
dns_patch = patch.object(web_to_md.socket, 'getaddrinfo', side_effect=AssertionError('Unexpected DNS'))
dns_patch.start()
self.addCleanup(dns_patch.stop)
+ options = patch.object(web_to_md, 'CurlOpt', SimpleNamespace(RESOLVE=10203, PROXY=10004))
+ options.start()
+ self.addCleanup(options.stop)
def test_non_public_literal_addresses_are_rejected_without_dns(self) -> None:
for host in ('169.254.169.254', '10.0.0.1', '172.16.0.1', '192.168.0.1',
@@ -50,7 +106,9 @@ class PublicUrlTests(unittest.TestCase):
def test_public_literal_addresses_are_accepted_without_dns(self) -> None:
for host in ('8.8.8.8', '[2001:4860:4860::8888]'):
with self.subTest(host=host):
- self.assertIsNone(web_to_md._validate_public_url(f'https://{host}/'))
+ url, addresses = web_to_md._validate_public_url(f'https://{host}/')
+ self.assertEqual(url, f'https://{host}/')
+ self.assertEqual([str(ip) for ip in addresses], [host.strip('[]')])
def test_non_http_or_malformed_urls_are_rejected(self) -> None:
for url in ('file:///etc/passwd', 'ftp://8.8.8.8/', 'data:text/plain,hello',
@@ -62,7 +120,9 @@ class PublicUrlTests(unittest.TestCase):
public = (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('8.8.8.8', 0))
private = (socket.AF_INET6, socket.SOCK_STREAM, 6, '', ('fe80::1', 0, 0, 0))
with patch.object(web_to_md.socket, 'getaddrinfo', return_value=[public]) as resolve:
- self.assertIsNone(web_to_md._validate_public_url('https://page.example/'))
+ url, addresses = web_to_md._validate_public_url('https://page.example/')
+ self.assertEqual(url, 'https://page.example/')
+ self.assertEqual([str(ip) for ip in addresses], ['8.8.8.8'])
resolve.assert_called_once_with('page.example', None, type=socket.SOCK_STREAM)
with patch.object(web_to_md.socket, 'getaddrinfo', return_value=[public, private]):
with self.assertRaisesRegex(ValueError, 'non-public'):
@@ -81,7 +141,7 @@ class PublicUrlTests(unittest.TestCase):
backend = Mock()
with self.subTest(curl=use_curl), \
patch.object(web_to_md, 'curl_requests', backend if use_curl else None), \
- patch.object(web_to_md.requests, 'get', backend.get):
+ patch.object(web_to_md.requests.Session, 'get', backend.get):
with self.assertRaisesRegex(ValueError, 'non-public'):
web_to_md._http_get('http://169.254.169.254/latest/meta-data/')
backend.get.assert_not_called()
@@ -94,7 +154,7 @@ class PublicUrlTests(unittest.TestCase):
backend.get.return_value = response
with self.subTest(curl=use_curl, final_url=final_url), \
patch.object(web_to_md, 'curl_requests', backend if use_curl else None), \
- patch.object(web_to_md.requests, 'get', backend.get):
+ patch.object(web_to_md.requests.Session, 'get', backend.get):
with self.assertRaises(ValueError):
web_to_md._http_get(PUBLIC_URL)
response.close.assert_called_once_with()
@@ -111,7 +171,7 @@ class PublicUrlTests(unittest.TestCase):
responses = [page, blocked] if unsafe_image else [blocked]
with self.subTest(image=unsafe_image), \
patch.object(web_to_md, 'curl_requests', None), \
- patch.object(web_to_md.requests, 'get', side_effect=responses), \
+ patch.object(web_to_md.requests.Session, 'get', side_effect=responses), \
patch.object(web_to_md.os, 'makedirs'), \
patch('builtins.open') as write, redirect_stdout(io.StringIO()):
result = web_to_md.process_url(PUBLIC_URL, '/unused/result.md')
@@ -122,6 +182,11 @@ class PublicUrlTests(unittest.TestCase):
class TlsVerificationTests(unittest.TestCase):
+ def setUp(self) -> None:
+ options = patch.object(web_to_md, 'CurlOpt', SimpleNamespace(RESOLVE=10203, PROXY=10004))
+ options.start()
+ self.addCleanup(options.stop)
+
def test_both_http_backends_verify_by_default_and_honor_explicit_opt_out(self) -> None:
for use_curl in (False, True):
for kwargs, expected in (({}, True), ({'verify': False}, False)):
@@ -129,7 +194,7 @@ class TlsVerificationTests(unittest.TestCase):
backend.get.return_value = Mock(url=PUBLIC_URL)
with self.subTest(curl=use_curl, kwargs=kwargs), \
patch.object(web_to_md, 'curl_requests', backend if use_curl else None), \
- patch.object(web_to_md.requests, 'get', backend.get):
+ patch.object(web_to_md.requests.Session, 'get', backend.get):
web_to_md._http_get(PUBLIC_URL, **kwargs)
self.assertIs(backend.get.call_args.kwargs['verify'], expected)
if use_curl:
@@ -166,3 +231,253 @@ class TlsVerificationTests(unittest.TestCase):
command, script_name = run.call_args.args
self.assertEqual(script_name, 'web_to_md.py')
self.assertEqual(command.count('--insecure'), 1)
+
+
+class TransportBindingTests(unittest.TestCase):
+ def setUp(self) -> None:
+ for guard in (
+ patch.dict(web_to_md.CONFIG, allow_private_hosts=False),
+ patch.object(web_to_md, 'CurlOpt', SimpleNamespace(RESOLVE=10203, PROXY=10004)),
+ patch.object(web_to_md.socket, 'getaddrinfo', side_effect=AssertionError('Unexpected DNS')),
+ patch.object(web_to_md.socket, 'socket', side_effect=AssertionError('Unexpected socket')),
+ ):
+ guard.start()
+ self.addCleanup(guard.stop)
+
+ def test_ambiguous_urls_are_refused_before_either_transport_is_created(self) -> None:
+ urls = (
+ 'http://127.0.0.1\\@example.com/',
+ 'http://8.8.8.8/path\\name',
+ ' http://8.8.8.8/',
+ 'http://8.8.8.8/a b',
+ 'http://8.8.8.8/\tpath',
+ 'http://8.8.8.8/\r\npath',
+ 'http://8.8.8.8/\x00path',
+ 'http://8.8.8.8/\x7fpath',
+ 'http://8.8.8.8/\x85path',
+ 'http://%65xample.com/',
+ )
+ for use_curl in (False, True):
+ for allow_private in (False, True):
+ for url in urls:
+ backend = Mock()
+ with self.subTest(curl=use_curl, private=allow_private, url=url), \
+ patch.dict(web_to_md.CONFIG, allow_private_hosts=allow_private), \
+ patch.object(web_to_md, 'curl_requests', backend if use_curl else None), \
+ patch.object(web_to_md.requests, 'Session') as session:
+ with self.assertRaises(web_to_md._UnsafeUrlError):
+ web_to_md._http_get(url)
+ backend.get.assert_not_called()
+ session.assert_not_called()
+ socket.socket.assert_not_called()
+
+ def test_ambiguous_image_urls_are_refused_before_transport(self) -> None:
+ for use_curl in (False, True):
+ for src in ('http://127.0.0.1\\@example.com/', 'http://8.8.8.8/	image.png'):
+ content = web_to_md.BeautifulSoup(f'<img src="{src}">', 'html.parser')
+ backend = Mock()
+ with self.subTest(curl=use_curl, src=src), tempfile.TemporaryDirectory() as tmp, \
+ patch.object(web_to_md, 'curl_requests', backend if use_curl else None), \
+ patch.object(web_to_md.requests, 'Session') as session:
+ with self.assertRaises(web_to_md._UnsafeUrlError):
+ web_to_md.download_and_rewrite_images(content, PUBLIC_URL, tmp, 'images')
+ self.assertEqual(list(Path(tmp).iterdir()), [])
+ backend.get.assert_not_called()
+ session.assert_not_called()
+ socket.socket.assert_not_called()
+
+ def test_every_redirect_rejects_ambiguous_locations_even_with_private_opt_in(self) -> None:
+ middle = 'https://8.8.8.8/middle'
+ for use_curl in (False, True):
+ for allow_private in (False, True):
+ for location in ('http://127.0.0.1\\@example.com/', '/\tpath', 'http://%65xample.com/'):
+ routes = {PUBLIC_URL: (302, {'Location': middle}), middle: (302, {'Location': location})}
+ with self.subTest(curl=use_curl, private=allow_private, location=location), \
+ patch.dict(web_to_md.CONFIG, allow_private_hosts=allow_private), \
+ _offline_transport(routes, use_curl) as (visited, responses):
+ with self.assertRaises(web_to_md._UnsafeUrlError):
+ web_to_md._http_get(PUBLIC_URL)
+ self.assertEqual(visited, [PUBLIC_URL, middle])
+ self.assertTrue(responses[-1].raw.closed)
+ socket.socket.assert_not_called()
+
+ def test_both_backends_send_the_prepared_url_on_every_public_hop(self) -> None:
+ start = 'http://public.example/~start'
+ final = 'http://public.example/~end'
+ routes = {start: (302, {'Location': '/%7eend'}), final: (200, {})}
+ for use_curl in (False, True):
+ with self.subTest(curl=use_curl), \
+ patch.object(socket, 'getaddrinfo', return_value=_dns_answer('8.8.8.8')), \
+ _offline_transport(routes, use_curl) as (visited, _):
+ response = web_to_md._http_get('HTTP://PUBLIC.EXAMPLE/%7estart')
+ self.assertEqual(response.content, b'public content')
+ self.assertEqual(visited, [start, final])
+
+ def test_requests_rebinding_is_refused_before_socket_creation(self) -> None:
+ for scheme in ('http', 'https'):
+ for rebound in (_dns_answer('169.254.169.254'),
+ _dns_answer('8.8.8.8') + _dns_answer('169.254.169.254')):
+ with self.subTest(scheme=scheme, rebound=rebound), \
+ patch.object(web_to_md, 'curl_requests', None), \
+ patch.object(socket, 'getaddrinfo', side_effect=[_dns_answer('8.8.8.8'), rebound]) as dns:
+ with self.assertRaisesRegex(web_to_md._UnsafeUrlError, 'non-public'):
+ web_to_md._http_get(f'{scheme}://rebind.example/')
+ self.assertEqual(dns.call_count, 2)
+ socket.socket.assert_not_called()
+
+ def test_requests_rebinding_on_a_redirect_is_refused_before_its_socket(self) -> None:
+ sock = _http_socket()
+ sock.makefile.side_effect = lambda *args: io.BytesIO(
+ b'HTTP/1.1 302 Found\r\nLocation: http://rebind.example/private\r\n'
+ b'Content-Length: 0\r\nConnection: close\r\n\r\n'
+ )
+ with patch.object(web_to_md, 'curl_requests', None), \
+ patch.object(socket, 'socket', return_value=sock) as create_socket, \
+ patch.object(socket, 'getaddrinfo', side_effect=[
+ _dns_answer('8.8.8.8'), _dns_answer('169.254.169.254'),
+ ]) as dns:
+ with self.assertRaisesRegex(web_to_md._UnsafeUrlError, 'non-public'):
+ web_to_md._http_get('http://8.8.8.8/start')
+ self.assertEqual(dns.call_count, 2)
+ create_socket.assert_called_once()
+ sock.connect.assert_called_once_with(('8.8.8.8', 80))
+
+ def test_curl_rebinding_on_a_redirect_is_refused_before_transport(self) -> None:
+ routes = {PUBLIC_URL: (302, {'Location': 'http://rebind.example/private'})}
+ with _offline_transport(routes, True) as (visited, responses), \
+ patch.object(socket, 'getaddrinfo', side_effect=[
+ _dns_answer('8.8.8.8'), _dns_answer('169.254.169.254'),
+ ]) as dns:
+ with self.assertRaisesRegex(web_to_md._UnsafeUrlError, 'non-public'):
+ web_to_md._http_get(PUBLIC_URL)
+ self.assertEqual(dns.call_count, 2)
+ self.assertEqual(visited, [PUBLIC_URL])
+ self.assertTrue(responses[0].raw.closed)
+
+ def test_requests_public_connections_keep_host_and_tls_verification(self) -> None:
+ for scheme in ('http', 'https'):
+ sock = _http_socket()
+ with self.subTest(scheme=scheme), patch.object(web_to_md, 'curl_requests', None), \
+ patch.object(socket, 'getaddrinfo', return_value=_dns_answer('8.8.8.8')), \
+ patch.object(socket, 'socket', return_value=sock), \
+ patch.dict(web_to_md.os.environ, HTTP_PROXY='http://127.0.0.1:9',
+ HTTPS_PROXY='http://127.0.0.1:9', ALL_PROXY='http://127.0.0.1:9'), \
+ patch('urllib3.connection._ssl_wrap_socket_and_match_hostname',
+ return_value=SimpleNamespace(socket=sock, is_verified=True)) as tls:
+ response = web_to_md._http_get(f'{scheme}://public.example/')
+ self.assertEqual(response.content, b'public')
+ sock.connect.assert_called_once_with(('8.8.8.8', 443 if scheme == 'https' else 80))
+ self.assertIn(b'Host: public.example\r\n', sock.sendall.call_args.args[0])
+ if scheme == 'https':
+ self.assertEqual(tls.call_args.kwargs['server_hostname'], 'public.example')
+ self.assertEqual(resolve_cert_reqs(tls.call_args.kwargs['cert_reqs']), ssl.CERT_REQUIRED)
+ self.assertIsNone(tls.call_args.kwargs['assert_hostname'])
+
+ def test_requests_connects_only_to_validated_ipv4_and_ipv6_answers(self) -> None:
+ first, second = _http_socket(), _http_socket()
+ first.connect.side_effect = OSError('unreachable')
+ answers = _dns_answer('8.8.8.8') + _dns_answer('2001:4860:4860::8888')
+ with patch.object(web_to_md, 'curl_requests', None), \
+ patch.object(socket, 'getaddrinfo', return_value=answers), \
+ patch.object(socket, 'socket', side_effect=[first, second]):
+ response = web_to_md._http_get('http://public.example:8080/')
+ self.assertEqual(response.content, b'public')
+ first.connect.assert_called_once_with(('8.8.8.8', 8080))
+ first.close.assert_called_once()
+ second.connect.assert_called_once_with(('2001:4860:4860::8888', 8080))
+
+ def test_curl_pins_public_addresses_despite_rebinding_at_transport(self) -> None:
+ connected = []
+
+ def get(url, **kwargs):
+ # The answer available at connect time has changed. A pinned backend
+ # uses RESOLVE instead of consulting it for its destination.
+ rebound = socket.getaddrinfo('mp.weixin.qq.com', None, type=socket.SOCK_STREAM)
+ self.assertEqual(rebound[0][4][0], '169.254.169.254')
+ options = kwargs['curl_options']
+ self.assertEqual(options[web_to_md.CurlOpt.RESOLVE], ['mp.weixin.qq.com:443:8.8.8.8'])
+ connected.extend(options[web_to_md.CurlOpt.RESOLVE][0].split(':')[2:])
+ self.assertEqual(options[web_to_md.CurlOpt.PROXY], '')
+ self.assertEqual(kwargs['impersonate'], web_to_md._CURL_IMPERSONATE)
+ self.assertIs(kwargs['allow_redirects'], False)
+ return Mock(url=url, status_code=200)
+
+ with patch.object(web_to_md, 'curl_requests', SimpleNamespace(get=get)), \
+ patch.object(socket, 'getaddrinfo', side_effect=[
+ _dns_answer('8.8.8.8'), _dns_answer('169.254.169.254'), _dns_answer('8.8.8.8'),
+ ]):
+ web_to_md._http_get('https://mp.weixin.qq.com/')
+ self.assertEqual(connected, ['8.8.8.8'])
+
+ def test_curl_recomputes_pins_for_each_redirect_and_explicit_port(self) -> None:
+ calls = []
+ start = 'https://mp.weixin.qq.com/start'
+ final = 'https://images.example:8443/photo'
+
+ def get(url, **kwargs):
+ calls.append((url, kwargs['curl_options'][web_to_md.CurlOpt.RESOLVE]))
+ headers = _CurlHeaders({'Location': final}) if url == start else _CurlHeaders()
+ return Mock(url=url, status_code=302 if url == start else 200, headers=headers)
+
+ def resolve(host, *args, **kwargs):
+ return _dns_answer('8.8.8.8' if host == 'mp.weixin.qq.com' else '2001:4860:4860::8888')
+
+ with patch.object(web_to_md, 'curl_requests', SimpleNamespace(get=get)), \
+ patch.object(socket, 'getaddrinfo', side_effect=resolve):
+ web_to_md._http_get(start, stream=True)
+ self.assertEqual(calls, [
+ (start, ['mp.weixin.qq.com:443:8.8.8.8']),
+ (final, ['images.example:8443:[2001:4860:4860::8888]']),
+ ])
+
+ def test_private_opt_in_uses_the_regular_requests_connection(self) -> None:
+ sock = _http_socket()
+ with patch.dict(web_to_md.CONFIG, allow_private_hosts=True), \
+ patch.dict(web_to_md.os.environ, {'NO_PROXY': '*', 'no_proxy': '*'}), \
+ patch.object(web_to_md, 'curl_requests', None), \
+ patch.object(socket, 'getaddrinfo', return_value=_dns_answer('127.0.0.1', 80)), \
+ patch.object(socket, 'socket', return_value=sock):
+ self.assertEqual(web_to_md._http_get('http://127.0.0.1/').content, b'public')
+ sock.connect.assert_called_once_with(('127.0.0.1', 80))
+
+ def test_installed_curl_api_applies_resolve_without_network(self) -> None:
+ try:
+ from curl_cffi import Curl, CurlOpt, requests as curl_requests
+ except ImportError:
+ self.skipTest('optional curl_cffi is not installed')
+ applied = {}
+ original_setopt = Curl.setopt
+
+ def setopt(curl, option, value):
+ applied[option] = value
+ return original_setopt(curl, option, value)
+
+ url = 'https://mp.weixin.qq.com/'
+ response = Mock(url=url, status_code=200)
+ with patch.object(web_to_md, 'CurlOpt', CurlOpt), \
+ patch.object(web_to_md, 'curl_requests', curl_requests), \
+ patch.object(socket, 'getaddrinfo', return_value=_dns_answer('8.8.8.8')), \
+ patch.object(Curl, 'setopt', setopt), patch.object(Curl, 'perform') as perform, \
+ patch.object(curl_requests.Session, '_parse_response', return_value=response):
+ self.assertIs(web_to_md._http_get(url), response)
+ perform.assert_called_once()
+ self.assertEqual(applied[CurlOpt.RESOLVE], ['mp.weixin.qq.com:443:8.8.8.8'])
+ self.assertEqual(applied[CurlOpt.PROXY], '')
+ socket.socket.assert_not_called()
+
+
+class PreparedUrlEdgeTests(unittest.TestCase):
+ """The parser-agreement check compares the prepared URL, and HTML padding is not ambiguity."""
+
+ def test_idn_hosts_are_prepared_to_punycode_not_refused(self) -> None:
+ prepared = web_to_md._prepare_url("https://例子.测试/路径")
+ self.assertTrue(prepared.startswith("https://xn--fsqu00a.xn--0zwm56d/"), prepared)
+
+ def test_padded_image_src_is_stripped_before_the_character_check(self) -> None:
+ soup = web_to_md.BeautifulSoup('<img src=" https://8.8.8.8/a.png ">', "html.parser")
+ self.assertEqual(
+ web_to_md.resolve_content_image_url(soup.img, "https://8.8.8.8/"), "https://8.8.8.8/a.png"
+ )
+ soup = web_to_md.BeautifulSoup('<img src="https://8.8.8.8/a b.png">', "html.parser")
+ with self.assertRaises(web_to_md._UnsafeUrlError):
+ web_to_md.resolve_content_image_url(soup.img, "https://8.8.8.8/")