Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(capture): end special-scheme URL authorities at a backslash (#2767)
Kaoru082277be0f6666df
diff --git a/bridge/mitmproxy/capture.py b/bridge/mitmproxy/capture.pyindex 412306012..c5ca704f6 100644--- a/bridge/mitmproxy/capture.py+++ b/bridge/mitmproxy/capture.py@@ -6,11 +6,13 @@ The containing mitmdump process and its configuration directory are REA-owned.import base64import hashlib+import ipaddressimport jsonimport mathimport reimport resourcefrom pathlib import Path+from urllib.parse import unquote_to_bytesfrom mitmproxy import ctx, versionfrom mitmproxy.io import tnetstring@@ -46,6 +48,8 @@ SCHEME_PREFIX = r"^[\x00-\x20]*([a-z][a-z0-9+.-]*):"NETWORK_PREFIX = r"^[\x00-\x20]*[/\\]{2}"URL_IGNORABLE_CONTROLS = "\t\n\r"URL_CONTROL_TRANSLATION = {ord(char): None for char in URL_IGNORABLE_CONTROLS}+C0_CONTROL_OR_SPACE = "".join(chr(code) for code in range(0x21))+FORBIDDEN_DOMAIN_CODE_POINTS = set(C0_CONTROL_OR_SPACE) | set("#%/:<>?@[\\]^|\x7f")def authority_start(text, base_scheme=None):@@ -54,11 +58,14 @@ def authority_start(text, base_scheme=None):Follows WHATWG URL parsing as src/domain/recordedTransportUrl.ts does:special schemes (and scheme-relative references) skip every leading "/"or "\\" before the authority, so `https://\\user:pass@host` still has- userinfo. File URLs carry no userinfo and keep their "//" boundary.+ userinfo. File URLs never carry userinfo: an "@" in a file host makes+ WHATWG reject the whole URL, so there are no credentials to remove."""scheme = re.match(SCHEME_PREFIX, text, flags=re.I)name = scheme.group(1).lower() if scheme is not None else None- if name is not None and name in SPECIAL_SCHEMES and name != "file":+ if name == "file" or (name is None and base_scheme == "file"):+ return None+ if name is not None and name in SPECIAL_SCHEMES:# A same-scheme Location without two slash-equivalent separators is# relative to the recorded request, so its @ belongs to the path.following = text[scheme.end():scheme.end() + 2]@@ -69,7 +76,7 @@ def authority_start(text, base_scheme=None):if scheme is not None:if text[scheme.end():scheme.end() + 2] != "//":return None- return scheme.end() + 2, name in SPECIAL_SCHEMES+ return scheme.end() + 2, False# A non-special base treats backslashes as authority data and only# recognizes two forward slashes as a network reference.if base_scheme is not None and base_scheme not in SPECIAL_SCHEMES:@@ -79,13 +86,87 @@ def authority_start(text, base_scheme=None):if prefix is None:return Nonestart = prefix.end()- if base_scheme == "file":- return start, Truewhile start < len(text) and text[start] in "/\\":start += 1return start, True+IPV4_DIGITS = {8: "01234567", 10: "0123456789", 16: "0123456789abcdefABCDEF"}+++def ipv4_number(part):+ """WHATWG IPv4 number parser: decimal, 0x hex or 0-prefixed octal, else None."""+ if part == "":+ return None+ radix, digits = 10, part+ if part[:2] in ("0x", "0X"):+ radix, digits = 16, part[2:]+ elif len(part) > 1 and part[0] == "0":+ radix, digits = 8, part[1:]+ if digits == "":+ return 0+ if any(char not in IPV4_DIGITS[radix] for char in digits):+ return None+ # Bound decimal text before int(): Python limits decimal conversions to+ # 4300 digits, and anything past 10 significant digits exceeds IPv4.+ significant = digits.lstrip("0")+ if radix == 10 and len(significant) > 10:+ return 2**32+ return int(digits, radix)+++def domain_parses(domain):+ """Apply WHATWG's ends-in-a-number check and IPv4 parser to a domain."""+ parts = domain.split(".")+ if parts[-1] == "" and len(parts) > 1:+ parts.pop()+ last = parts[-1]+ if not ((last != "" and last.isascii() and last.isdigit()) or ipv4_number(last) is not None):+ return True+ numbers = [ipv4_number(part) for part in parts]+ if len(parts) > 4 or any(number is None for number in numbers):+ return False+ return all(number <= 255 for number in numbers[:-1]) and numbers[-1] < 256 ** (5 - len(numbers))+++def special_host_parses(host_port):+ """Whether WHATWG could accept a special authority's host and port.++ A rejected host fails the whole URL, so it has no credentials to remove.+ This mirrors the forbidden-code-point, empty-host, IPv4, IPv6 and port+ rules for ASCII hosts. UTS #46 mapping is not modeled: a non-ASCII host+ may map to an accepted one (fullwidth digits, ideographic full stops,+ ignored code points), so it is treated as accepted and still redacts.+ """+ if host_port.startswith("["):+ host_end = host_port.find("]") + 1+ literal = host_port[1 : host_end - 1]+ if host_end == 0 or "%" in literal:+ return False+ try:+ ipaddress.IPv6Address(literal)+ except ValueError:+ return False+ else:+ colon = host_port.find(":")+ host_end = len(host_port) if colon < 0 else colon+ host = unquote_to_bytes(host_port[:host_end]).decode("utf-8", errors="replace")+ # Mapping never removes ASCII, so these rejections hold either way.+ if host == "" or any(char in FORBIDDEN_DOMAIN_CODE_POINTS for char in host):+ return False+ if host.isascii() and not domain_parses(host.lower()):+ return False+ port = host_port[host_end:]+ if port == "":+ return True+ digits = port[1:]+ if port[0] != ":" or not (digits == "" or (digits.isascii() and digits.isdigit())):+ return False+ # Bound the text first: int() refuses decimal strings over 4300 digits.+ significant = digits.lstrip("0")+ return len(significant) <= 5 and int(significant or "0") <= 65535++def without_userinfo(value, base_scheme=None):"""Remove an authority's userinfo using the WHATWG authority boundary.@@ -96,7 +177,8 @@ def without_userinfo(value, base_scheme=None):text = value.decode("latin-1") if binary else value# WHATWG removes these controls before interpreting scheme/separators.# Keep that parse view separate from the exact source spelling and bytes.- parsed_text = text.translate(URL_CONTROL_TRANSLATION)+ # It also drops trailing C0 controls and spaces; offsets are unchanged.+ parsed_text = text.translate(URL_CONTROL_TRANSLATION).rstrip(C0_CONTROL_OR_SPACE)located = authority_start(parsed_text, base_scheme)if located is None:return value@@ -106,7 +188,7 @@ def without_userinfo(value, base_scheme=None):while end < len(parsed_text) and parsed_text[end] not in terminators:end += 1at = parsed_text.rfind("@", start, end)- if at < start:+ if at < start or (special and not special_host_parses(parsed_text[at + 1 : end])):return valueparsed_offset = 0raw_start = 0diff --git a/bridge/mitmproxy/test_capture.py b/bridge/mitmproxy/test_capture.pyindex f0a1106b1..3863312cb 100644--- a/bridge/mitmproxy/test_capture.py+++ b/bridge/mitmproxy/test_capture.py@@ -165,6 +165,97 @@ class TransportUserinfoTests(unittest.TestCase):def test_file_urls_keep_path_text(self):self.assertEqual(capture.without_userinfo("file:///a@b"), "file:///a@b")+ def test_file_urls_reject_userinfo_and_keep_their_source(self):+ # WHATWG rejects an "@" in a file host, so nothing parses as credentials.+ self.assertEqual(+ capture.without_userinfo("file://user:pass@host/x"), "file://user:pass@host/x"+ )+ for value in [b"\\\\user:pass@host/x", b"/\\user:pass@host/x", b"//user:pass@host/x"]:+ with self.subTest(value=value):+ text, sidecar, redactions = location(value, b"file")+ self.assertEqual(text, value.decode("ascii"))+ self.assertEqual(sidecar["state"], "retained")+ self.assertEqual(redactions, [])++ def test_rejected_special_hosts_keep_their_source(self):+ # A rejected host or port fails the whole URL, so it has no userinfo.+ for value in [+ "https:user@%",+ "https://user@",+ "https://user:pass@h^st/x",+ "https://user@ho st/x",+ "https://user@host:99999/x",+ "https://user@host:12a",+ "https://user@[::1",+ ]:+ with self.subTest(value=value):+ self.assertEqual(capture.without_userinfo(value), value)+ text, sidecar, redactions = location(b"https://user:pass@h^st/x", b"https")+ self.assertEqual(text, "https://user:pass@h^st/x")+ self.assertEqual(sidecar["state"], "retained")+ self.assertEqual(redactions, [])++ def test_accepted_special_hosts_still_remove_userinfo(self):+ for value, expected in [+ ("https:user@%41", "https:%41"),+ ("https://user@[::1]:8443/x", "https://[::1]:8443/x"),+ ("https://user@host:/x", "https://host:/x"),+ ("wss://user:pass@host:65535", "wss://host:65535"),+ ("https://[email protected]/", "https://0x7f.1/"),+ ("https://[email protected]./", "https://1.2.3.4./"),+ ("https://user@4294967295/", "https://4294967295/"),+ ("https://[email protected]/", "https://1.2.3.08a/"),+ ("https://user@[::ffff:1.2.3.4]/", "https://[::ffff:1.2.3.4]/"),+ ]:+ with self.subTest(value=value):+ self.assertEqual(capture.without_userinfo(value), expected)++ def test_unicode_hosts_that_may_map_to_accepted_hosts_remove_userinfo(self):+ # UTS #46 maps these to 1.2.3.4; mapping is not modeled, so redact.+ for host in ["%EF%BC%91.2.3.4", "1。2.3.4", "%C2%AD1.2.3.4", "1.2.3.4"]:+ with self.subTest(host=host):+ self.assertEqual(+ capture.without_userinfo(f"https://user:pw@{host}/"), f"https://{host}/"+ )+ text, sidecar, redactions = location(b"https://user:pw@%EF%BC%91.2.3.4/", b"https")+ self.assertEqual(text, "https://%EF%BC%91.2.3.4/")+ self.assertEqual(sidecar["state"], "redacted")+ self.assertEqual(len(redactions), 1)+ # ASCII rejections still hold for non-ASCII hosts.+ self.assertEqual(capture.without_userinfo("https://user@1^/"), "https://user@1^/")++ def test_trailing_controls_and_spaces_do_not_hide_userinfo(self):+ # WHATWG strips trailing C0 controls and spaces before parsing.+ for suffix in [b" ", b"\x01", b"\x00 \x1f"]:+ with self.subTest(suffix=suffix):+ value = b"https://user:[email protected]" + suffix+ text, sidecar, redactions = location(value, b"https")+ self.assertEqual(text, "https://example.test" + suffix.decode("latin-1"))+ self.assertEqual(sidecar["state"], "redacted")+ self.assertEqual(len(redactions), 1)++ def test_rejected_ip_literals_keep_their_source(self):+ for value in [+ "https://user:[email protected]/path",+ "https://[email protected]/",+ "https://[email protected]/",+ "https://user@4294967296/",+ "https://[email protected]/",+ "https://[email protected]/",+ "https://user@1." + "9" * 5000 + "/",+ "https://user@[1::2::3]/",+ "https://user@[::1%25eth0]/",+ ]:+ with self.subTest(value=value[:40]):+ self.assertEqual(capture.without_userinfo(value), value)++ def test_overlong_ports_are_compared_without_numeric_overflow(self):+ # Python 3.11+ refuses int() on decimal strings over 4300 digits.+ accepted = "https://user@host:" + "0" * 5000 + "80/x"+ self.assertEqual(capture.without_userinfo(accepted), "https://host:" + "0" * 5000 + "80/x")+ rejected = "https://user@host:" + "9" * 5000 + "/x"+ self.assertEqual(capture.without_userinfo(rejected), rejected)+def test_non_special_schemes_keep_backslashes_in_the_authority(self):self.assertEqual(capture.without_userinfo("foo://a\\b@host/x"), "foo://host/x")self.assertEqual(capture.without_userinfo(b"//a\\b@host"), b"//a\\b@host")