AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
fix(http): strip credentials from direct request redirects (#1154)
Adam Shapira74af583fa012
diff --git a/changelog.d/1062.security.md b/changelog.d/1062.security.mdnew file mode 100644index 00000000..a6d2d13c--- /dev/null+++ b/changelog.d/1062.security.md@@ -0,0 +1 @@+Route `github.py` and `transcribe.py` through `http.open_request` so their bearer tokens are also dropped on cross-origin redirects; both built their own Request and called `urllib.request.urlopen` directly, bypassing the handler added for #1062.diff --git a/skills/last30days/scripts/lib/github.py b/skills/last30days/scripts/lib/github.pyindex efc2f3e7..c438cf69 100644--- a/skills/last30days/scripts/lib/github.py+++ b/skills/last30days/scripts/lib/github.py@@ -102,7 +102,7 @@ def _fetch_json(req = urllib.request.Request(url, headers=headers)try:- with urllib.request.urlopen(req, timeout=timeout) as resp:+ with http.open_request(req, timeout) as resp:body = resp.read().decode("utf-8")return json.loads(body)except urllib.error.HTTPError as e:@@ -583,7 +583,7 @@ def _fetch_readme_snippet(repo: str, token: str, max_chars: int = 500) -> Optionreq = urllib.request.Request(url, headers=headers)try:- with urllib.request.urlopen(req, timeout=10) as resp:+ with http.open_request(req, 10) as resp:raw = resp.read().decode("utf-8", errors="replace")except (urllib.error.HTTPError, urllib.error.URLError, OSError, TimeoutError):return Nonediff --git a/skills/last30days/scripts/lib/http.py b/skills/last30days/scripts/lib/http.pyindex 57bcd83c..773cb35d 100644--- a/skills/last30days/scripts/lib/http.py+++ b/skills/last30days/scripts/lib/http.py@@ -134,6 +134,17 @@ def _open_request(req, timeout):return current(req, timeout=timeout)return _opener.open(req, timeout=timeout)++def open_request(req, timeout):+ """Open ``req`` with cross-origin credential stripping (see #1062).++ Public entry point for modules that build their own ``Request`` and would+ otherwise call ``urllib.request.urlopen`` directly, bypassing the redirect+ handler installed above.+ """+ return _open_request(req, timeout)++_failure_sink: ContextVar[Optional[list["HTTPError"]]] = ContextVar("last30days_http_failure_sink",default=None,diff --git a/skills/last30days/scripts/lib/transcribe.py b/skills/last30days/scripts/lib/transcribe.pyindex d3191fb3..3a6346d3 100644--- a/skills/last30days/scripts/lib/transcribe.py+++ b/skills/last30days/scripts/lib/transcribe.py@@ -226,7 +226,7 @@ def _post_audio(provider: str, path: str, api_key: str, timeout: float) -> Optiomethod="POST",)usage.begin(provider)- with urllib.request.urlopen(req, timeout=timeout) as resp:+ with http.open_request(req, timeout) as resp:data = json.loads(resp.read().decode("utf-8"))return data.get("text")diff --git a/tests/test_github_redirect_auth.py b/tests/test_github_redirect_auth.pynew file mode 100644index 00000000..4d3f4c1a--- /dev/null+++ b/tests/test_github_redirect_auth.py@@ -0,0 +1,246 @@+"""Modules that build their own Request must not leak credentials on redirect.++``lib/http.py`` installs a redirect handler that drops credential headers when a+3xx changes origin, but that only protects callers that go through it. Several+modules build a ``urllib.request.Request`` themselves; ``github.py`` and+``transcribe.py`` attach a bearer token to theirs, so they must use+``http.open_request`` rather than ``urllib.request.urlopen``.+"""++import ast+import http.server+import pathlib+import socketserver+import threading+from urllib.request import Request++import pytest++from lib import github, transcribe+from lib import http as l30d_http+++class _CaptureHandler(http.server.BaseHTTPRequestHandler):+ captured: dict = {}++ def do_GET(self):+ type(self).captured = {k.lower(): v for k, v in self.headers.items()}+ body = b'{"ok":true}'+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)++ def log_message(self, *_args):+ pass+++def _serve(handler):+ httpd = socketserver.TCPServer(("127.0.0.1", 0), handler)+ threading.Thread(target=httpd.serve_forever, daemon=True).start()+ return httpd+++def test_open_request_strips_bearer_across_origin():+ attacker = _serve(_CaptureHandler)+ _CaptureHandler.captured = {}++ class Victim(http.server.BaseHTTPRequestHandler):+ def do_GET(self):+ self.send_response(302)+ self.send_header(+ "Location", f"http://127.0.0.1:{attacker.server_address[1]}/steal"+ )+ self.send_header("Content-Length", "0")+ self.end_headers()++ def log_message(self, *_args):+ pass++ victim = _serve(Victim)+ req = Request(+ f"http://127.0.0.1:{victim.server_address[1]}/start",+ headers={"Authorization": "Bearer ghp_sentinel_token"},+ )+ try:+ with l30d_http.open_request(req, 10) as resp:+ resp.read()+ finally:+ victim.shutdown()+ attacker.shutdown()+ victim.server_close()+ attacker.server_close()++ assert "authorization" not in _CaptureHandler.captured, (+ "bearer token survived a cross-origin redirect: "+ f"{_CaptureHandler.captured.get('authorization')!r}"+ )++[email protected]("caller", ["github_json", "github_readme", "transcribe"])[email protected]("cross_origin", [False, True], ids=["same_origin", "cross_origin"])+def test_credential_callers_handle_redirects(caller, cross_origin, monkeypatch, tmp_path):+ captured = []+ payload = b'{"ok":true,"text":"loopback transcript"}'+ token = "dummy_redirect_caller_token"++ class Handler(http.server.BaseHTTPRequestHandler):+ def do_GET(self):+ captured.append((self.path, dict(self.headers.items())))+ if self.path == "/start":+ self.send_response(302)+ self.send_header("Location", destination)+ self.send_header("Content-Length", "0")+ self.end_headers()+ else:+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(payload)))+ self.end_headers()+ self.wfile.write(payload)++ def do_POST(self):+ self.rfile.read(int(self.headers.get("Content-Length", "0")))+ self.do_GET()++ def log_message(self, *_args):+ pass++ origin = _serve(Handler)+ target = _serve(Handler) if cross_origin else origin+ start = f"http://127.0.0.1:{origin.server_address[1]}/start"+ destination = f"http://127.0.0.1:{target.server_address[1]}/result"+ try:+ if caller == "github_json":+ assert github._fetch_json(start, token=token) == {+ "ok": True, "text": "loopback transcript",+ }+ elif caller == "github_readme":+ def loopback_request(url, *args, **kwargs):+ if url == "https://api.github.com/repos/owner/repo/readme":+ url = start+ return Request(url, *args, **kwargs)++ monkeypatch.setattr(github.urllib.request, "Request", loopback_request)+ assert github._fetch_readme_snippet("owner/repo", token) == payload.decode()+ else:+ monkeypatch.setitem(transcribe._PROVIDER_ENDPOINTS, "groq", start)+ audio = tmp_path / "audio.mp3"+ audio.write_bytes(b"dummy audio")+ assert transcribe._post_audio("groq", str(audio), token, 10) == "loopback transcript"+ finally:+ origin.shutdown()+ origin.server_close()+ if target is not origin:+ target.shutdown()+ target.server_close()++ assert [path for path, _ in captured] == ["/start", "/result"]+ original_headers, final_headers = (+ {key.lower(): value for key, value in headers.items()}+ for _, headers in captured+ )+ assert original_headers["authorization"] == f"Bearer {token}"+ assert final_headers.get("authorization") == (None if cross_origin else f"Bearer {token}")+ assert final_headers["user-agent"] == original_headers["user-agent"]+++def _dotted_name(node) -> str:+ """Render a dotted attribute chain (``urllib.request``) or "" if not one."""+ parts = []+ while isinstance(node, ast.Attribute):+ parts.append(node.attr)+ node = node.value+ if not isinstance(node, ast.Name):+ return ""+ parts.append(node.id)+ return ".".join(reversed(parts))+++def _direct_urlopen_calls(source: str) -> list[int]:+ """Line numbers of calls that reach ``urllib.request.urlopen`` in ``source``.++ Resolved through the module's imports rather than by text match. Searching+ for the literal ``urllib.request.urlopen(`` would miss+ ``from urllib.request import urlopen``, a form this repository already uses+ in four other modules, so a future edit to a credential-bearing module could+ reintroduce the bypass while keeping the guard green.+ """+ tree = ast.parse(source)++ # Names bound to the urlopen function itself.+ urlopen_names: set[str] = set()+ # Names bound to the urllib.request module.+ module_names: set[str] = {"urllib.request"}++ for node in ast.walk(tree):+ if isinstance(node, ast.ImportFrom):+ if node.module == "urllib.request":+ for alias in node.names:+ if alias.name == "urlopen":+ urlopen_names.add(alias.asname or alias.name)+ elif node.module == "urllib":+ for alias in node.names:+ if alias.name == "request":+ module_names.add(alias.asname or alias.name)+ elif isinstance(node, ast.Import):+ for alias in node.names:+ if alias.name == "urllib.request":+ # `import urllib.request` binds "urllib"; an asname binds that.+ module_names.add(alias.asname or "urllib.request")+ elif alias.name == "urllib":+ module_names.add(f"{alias.asname or 'urllib'}.request")++ hits: list[int] = []+ for node in ast.walk(tree):+ if not isinstance(node, ast.Call):+ continue+ func = node.func+ if isinstance(func, ast.Name) and func.id in urlopen_names:+ hits.append(node.lineno)+ elif isinstance(func, ast.Attribute) and func.attr == "urlopen":+ if _dotted_name(func.value) in module_names:+ hits.append(node.lineno)+ return sorted(hits)+++def test_credential_bearing_modules_do_not_call_urlopen_directly():+ """Regression guard: these modules attach a bearer token to their Request."""+ lib = pathlib.Path(l30d_http.__file__).parent+ for name in ("github.py", "transcribe.py"):+ source = (lib / name).read_text(encoding="utf-8")+ hits = _direct_urlopen_calls(source)+ assert hits == [], (+ f"{name} calls urllib.request.urlopen directly at line(s) "+ f"{hits}, bypassing the cross-origin credential strip in "+ "http.open_request"+ )+++def test_urlopen_guard_detects_aliased_forms():+ """The guard itself must not be fooled by an import alias.++ Without this, the guard could silently stop working: a text search for+ ``urllib.request.urlopen(`` passes on every aliased form below.+ """+ detected = (+ "import urllib.request\nurllib.request.urlopen(req)\n",+ "from urllib.request import urlopen\nurlopen(req)\n",+ "from urllib.request import urlopen as uo\nuo(req)\n",+ "from urllib import request\nrequest.urlopen(req)\n",+ "from urllib import request as r\nr.urlopen(req)\n",+ "import urllib.request as ur\nur.urlopen(req)\n",+ )+ for source in detected:+ assert _direct_urlopen_calls(source), f"missed: {source!r}"++ ignored = (+ # Routed through the protected wrapper — the whole point of the fix.+ "from . import http\nhttp.open_request(req, timeout=10)\n",+ # An unrelated urlopen on some other object must not trip the guard.+ "session.urlopen(req)\n",+ "from urllib.request import Request\nRequest(url)\n",+ )+ for source in ignored:+ assert not _direct_urlopen_calls(source), f"false positive: {source!r}"