Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(website): handle UTF-8 output and slash refs in link audits (#4123)
kuxiaowo3f35eacad282
diff --git a/scripts/audit-website-links.py b/scripts/audit-website-links.pyindex cff49da0b..389887090 100644--- a/scripts/audit-website-links.py+++ b/scripts/audit-website-links.py@@ -19,7 +19,7 @@ from website import authored_markdown@lru_cache(maxsize=None)def github_api(endpoint):"""Read GitHub evidence through the caller's gh installation and authentication."""- result = subprocess.run(["gh", "api", endpoint], capture_output=True, text=True, timeout=20)+ result = subprocess.run(["gh", "api", endpoint], capture_output=True, encoding="utf-8", timeout=20)if result.returncode:raise ValueError(result.stderr.strip())return json.loads(result.stdout)@@ -32,7 +32,8 @@ def github_target(url):return Nonerepo = f"repos/{parts[0]}/{parts[1]}"if url.hostname == "raw.githubusercontent.com" and len(parts) >= 4:- return f"{repo}/contents/{'/'.join(parts[3:])}?ref={quote(parts[2], safe='')}"+ # Raw URLs omit the blob segment; share ref/path resolution below.+ parts = [*parts[:2], "blob", *parts[2:]]if len(parts) == 2:return repokind = parts[2]diff --git a/scripts/test-website.py b/scripts/test-website.pyindex 0b60cff7e..c6f091cde 100644--- a/scripts/test-website.py+++ b/scripts/test-website.py@@ -7,7 +7,10 @@ import ioimport jsonfrom pathlib import Pathimport shutil+import subprocess+import sysimport tempfile+from textwrap import dedentfrom threading import Threadimport unittestfrom unittest.mock import patch@@ -35,6 +38,113 @@ def load_script(name):PREPARE = load_script("prepare-website")VERIFY = load_script("verify-website")WEBSITE = load_script("website")+AUDIT = load_script("audit-website-links")+++class WebsiteLinkAuditChecks(unittest.TestCase):+ """Check GitHub transport and URL classification without live network access."""++ def test_github_json_is_decoded_as_utf8_under_a_legacy_locale(self):+ # Exercise real subprocess pipe decoding with UTF-8 mode disabled. Only+ # the external gh executable and the default locale are substituted.+ program = dedent('''\+ import importlib.util+ import json+ from pathlib import Path+ import subprocess+ import sys+ from unittest.mock import patch++ scripts = Path(sys.argv[1])+ sys.path.insert(0, str(scripts))+ spec = importlib.util.spec_from_file_location("audit", scripts / "audit-website-links.py")+ audit = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(audit)+ payload = json.dumps({"title": "Release \\u2603"}, ensure_ascii=False).encode("utf-8")+ run = subprocess.run++ def gh(command, **options):+ assert command == ["gh", "api", "repos/example/project"]+ return run([sys.executable, "-c", "import sys; sys.stdout.buffer.write(" + repr(payload) + ")"], **options)++ with patch("locale.getencoding", return_value="cp936"), patch.object(audit.subprocess, "run", side_effect=gh):+ print(json.dumps(audit.github_api("repos/example/project")))+ ''')+ result = subprocess.run(+ [sys.executable, "-X", "utf8=0", "-c", program, str(ROOT / "scripts")],+ capture_output=True, encoding="utf-8", timeout=20,+ )+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertEqual(json.loads(result.stdout), {"title": "Release \u2603"})++ def test_raw_and_browser_links_resolve_the_same_ref_and_encoded_path(self):+ # Prefix lookalikes must not win, and tags use the same ref/path split.+ references = {+ "repos/example/project/git/matching-refs/heads/topic": [+ {"ref": "refs/heads/topic"},+ {"ref": "refs/heads/topic/nested"},+ {"ref": "refs/heads/topic/nestedness"},+ ],+ "repos/example/project/git/matching-refs/tags/topic": [+ {"ref": "refs/tags/topic/nested/v1"},+ ],+ "repos/example/project/git/matching-refs/heads/topic%2Fnested": [+ {"ref": "refs/heads/topic/nested"},+ {"ref": "refs/heads/topic/nestedness"},+ ],+ "repos/example/project/git/matching-refs/tags/topic%2Fnested": [+ {"ref": "refs/tags/topic/nested/v1"},+ ],+ }+ cases = (+ ("topic/nested/guide%20%23%3F.md", "guide%20%23%3F.md", "topic%2Fnested"),+ ("topic/nested/v1/README.md", "README.md", "topic%2Fnested%2Fv1"),+ ("topic%2Fnested/README.md", "README.md", "topic%2Fnested"),+ )+ for suffix, path, ref in cases:+ endpoint = f"repos/example/project/contents/{path}?ref={ref}"+ responses = {**references, endpoint: {"name": "README.md"}}++ def github_api(target):+ if target not in responses:+ raise ValueError("gh: Not Found (HTTP 404)")+ return responses[target]++ for base in (+ "https://raw.githubusercontent.com/example/project/",+ "https://github.com/example/project/blob/",+ "https://github.com/example/project/tree/",+ ):+ with self.subTest(url=base + suffix), patch.object(AUDIT, "github_api", side_effect=github_api):+ report = AUDIT.audit_url((base + suffix, ["fixture.html"]))+ self.assertEqual(report["status"], "ok")+ self.assertEqual(report["endpoint"], endpoint)++ def test_plain_refs_and_pinned_commits_keep_their_contents_target(self):+ for ref in ("main", "a" * 40):+ responses = {+ "repos/example/project/git/matching-refs/heads/main": [{"ref": "refs/heads/main"}],+ "repos/example/project/git/matching-refs/tags/main": [],+ f"repos/example/project/contents/README.md?ref={ref}": {"name": "README.md"},+ }+ for base in ("https://raw.githubusercontent.com/example/project/", "https://github.com/example/project/blob/"):+ with self.subTest(ref=ref, base=base), patch.object(AUDIT, "github_api", side_effect=responses.__getitem__):+ report = AUDIT.audit_url((base + ref + "/README.md", ["fixture.html"]))+ self.assertEqual(report["status"], "ok")+ self.assertEqual(report["endpoint"], f"repos/example/project/contents/README.md?ref={ref}")++ def test_missing_contents_and_unavailable_github_probes_remain_distinct(self):+ cases = (+ (ValueError("gh: Not Found (HTTP 404)"), "broken"),+ (ValueError("gh: Forbidden (HTTP 403)"), "unknown"),+ (subprocess.TimeoutExpired(["gh", "api"], 20), "unknown"),+ )+ for failure, status in cases:+ for base in ("https://raw.githubusercontent.com/example/project/", "https://github.com/example/project/blob/"):+ with self.subTest(failure=str(failure), base=base), patch.object(AUDIT, "github_api", side_effect=failure):+ report = AUDIT.audit_url((base + "a" * 40 + "/missing.md", ["fixture.html"]))+ self.assertEqual(report["status"], status)+ self.assertEqual(report["reason"], str(failure))class WebsiteSearchChecks(unittest.TestCase):