Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(website): handle UTF-8 output and slash refs in link audits (#4123)

kuxiaowo3f35eacad282
diff --git a/scripts/audit-website-links.py b/scripts/audit-website-links.py
index cff49da0b..389887090 100644
--- a/scripts/audit-website-links.py
+++ b/scripts/audit-website-links.py
@@ -19,7 +19,7 @@ from website import authored_markdown
@lru_cache(maxsize=None)
def github_api(endpoint):
"""Read GitHub evidence through the caller's gh installation and authentication."""
- result = subprocess.run(["gh", "api", endpoint], capture_output=True, text=True, timeout=20)
+ result = subprocess.run(["gh", "api", endpoint], capture_output=True, encoding="utf-8", timeout=20)
if result.returncode:
raise ValueError(result.stderr.strip())
return json.loads(result.stdout)
@@ -32,7 +32,8 @@ def github_target(url):
return None
repo = f"repos/{parts[0]}/{parts[1]}"
if url.hostname == "raw.githubusercontent.com" and len(parts) >= 4:
- return f"{repo}/contents/{'/'.join(parts[3:])}?ref={quote(parts[2], safe='')}"
+ # Raw URLs omit the blob segment; share ref/path resolution below.
+ parts = [*parts[:2], "blob", *parts[2:]]
if len(parts) == 2:
return repo
kind = parts[2]
diff --git a/scripts/test-website.py b/scripts/test-website.py
index 0b60cff7e..c6f091cde 100644
--- a/scripts/test-website.py
+++ b/scripts/test-website.py
@@ -7,7 +7,10 @@ import io
import json
from pathlib import Path
import shutil
+import subprocess
+import sys
import tempfile
+from textwrap import dedent
from threading import Thread
import unittest
from unittest.mock import patch
@@ -35,6 +38,113 @@ def load_script(name):
PREPARE = load_script("prepare-website")
VERIFY = load_script("verify-website")
WEBSITE = load_script("website")
+AUDIT = load_script("audit-website-links")
+
+
+class WebsiteLinkAuditChecks(unittest.TestCase):
+ """Check GitHub transport and URL classification without live network access."""
+
+ def test_github_json_is_decoded_as_utf8_under_a_legacy_locale(self):
+ # Exercise real subprocess pipe decoding with UTF-8 mode disabled. Only
+ # the external gh executable and the default locale are substituted.
+ program = dedent('''\
+ import importlib.util
+ import json
+ from pathlib import Path
+ import subprocess
+ import sys
+ from unittest.mock import patch
+
+ scripts = Path(sys.argv[1])
+ sys.path.insert(0, str(scripts))
+ spec = importlib.util.spec_from_file_location("audit", scripts / "audit-website-links.py")
+ audit = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(audit)
+ payload = json.dumps({"title": "Release \\u2603"}, ensure_ascii=False).encode("utf-8")
+ run = subprocess.run
+
+ def gh(command, **options):
+ assert command == ["gh", "api", "repos/example/project"]
+ return run([sys.executable, "-c", "import sys; sys.stdout.buffer.write(" + repr(payload) + ")"], **options)
+
+ with patch("locale.getencoding", return_value="cp936"), patch.object(audit.subprocess, "run", side_effect=gh):
+ print(json.dumps(audit.github_api("repos/example/project")))
+ ''')
+ result = subprocess.run(
+ [sys.executable, "-X", "utf8=0", "-c", program, str(ROOT / "scripts")],
+ capture_output=True, encoding="utf-8", timeout=20,
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(json.loads(result.stdout), {"title": "Release \u2603"})
+
+ def test_raw_and_browser_links_resolve_the_same_ref_and_encoded_path(self):
+ # Prefix lookalikes must not win, and tags use the same ref/path split.
+ references = {
+ "repos/example/project/git/matching-refs/heads/topic": [
+ {"ref": "refs/heads/topic"},
+ {"ref": "refs/heads/topic/nested"},
+ {"ref": "refs/heads/topic/nestedness"},
+ ],
+ "repos/example/project/git/matching-refs/tags/topic": [
+ {"ref": "refs/tags/topic/nested/v1"},
+ ],
+ "repos/example/project/git/matching-refs/heads/topic%2Fnested": [
+ {"ref": "refs/heads/topic/nested"},
+ {"ref": "refs/heads/topic/nestedness"},
+ ],
+ "repos/example/project/git/matching-refs/tags/topic%2Fnested": [
+ {"ref": "refs/tags/topic/nested/v1"},
+ ],
+ }
+ cases = (
+ ("topic/nested/guide%20%23%3F.md", "guide%20%23%3F.md", "topic%2Fnested"),
+ ("topic/nested/v1/README.md", "README.md", "topic%2Fnested%2Fv1"),
+ ("topic%2Fnested/README.md", "README.md", "topic%2Fnested"),
+ )
+ for suffix, path, ref in cases:
+ endpoint = f"repos/example/project/contents/{path}?ref={ref}"
+ responses = {**references, endpoint: {"name": "README.md"}}
+
+ def github_api(target):
+ if target not in responses:
+ raise ValueError("gh: Not Found (HTTP 404)")
+ return responses[target]
+
+ for base in (
+ "https://raw.githubusercontent.com/example/project/",
+ "https://github.com/example/project/blob/",
+ "https://github.com/example/project/tree/",
+ ):
+ with self.subTest(url=base + suffix), patch.object(AUDIT, "github_api", side_effect=github_api):
+ report = AUDIT.audit_url((base + suffix, ["fixture.html"]))
+ self.assertEqual(report["status"], "ok")
+ self.assertEqual(report["endpoint"], endpoint)
+
+ def test_plain_refs_and_pinned_commits_keep_their_contents_target(self):
+ for ref in ("main", "a" * 40):
+ responses = {
+ "repos/example/project/git/matching-refs/heads/main": [{"ref": "refs/heads/main"}],
+ "repos/example/project/git/matching-refs/tags/main": [],
+ f"repos/example/project/contents/README.md?ref={ref}": {"name": "README.md"},
+ }
+ for base in ("https://raw.githubusercontent.com/example/project/", "https://github.com/example/project/blob/"):
+ with self.subTest(ref=ref, base=base), patch.object(AUDIT, "github_api", side_effect=responses.__getitem__):
+ report = AUDIT.audit_url((base + ref + "/README.md", ["fixture.html"]))
+ self.assertEqual(report["status"], "ok")
+ self.assertEqual(report["endpoint"], f"repos/example/project/contents/README.md?ref={ref}")
+
+ def test_missing_contents_and_unavailable_github_probes_remain_distinct(self):
+ cases = (
+ (ValueError("gh: Not Found (HTTP 404)"), "broken"),
+ (ValueError("gh: Forbidden (HTTP 403)"), "unknown"),
+ (subprocess.TimeoutExpired(["gh", "api"], 20), "unknown"),
+ )
+ for failure, status in cases:
+ for base in ("https://raw.githubusercontent.com/example/project/", "https://github.com/example/project/blob/"):
+ with self.subTest(failure=str(failure), base=base), patch.object(AUDIT, "github_api", side_effect=failure):
+ report = AUDIT.audit_url((base + "a" * 40 + "/missing.md", ["fixture.html"]))
+ self.assertEqual(report["status"], status)
+ self.assertEqual(report["reason"], str(failure))
class WebsiteSearchChecks(unittest.TestCase):