Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

fix(http): strip credentials from direct request redirects (#1154)

Adam Shapira74af583fa012
diff --git a/changelog.d/1062.security.md b/changelog.d/1062.security.md
new file mode 100644
index 00000000..a6d2d13c
--- /dev/null
+++ b/changelog.d/1062.security.md
@@ -0,0 +1 @@
+Route `github.py` and `transcribe.py` through `http.open_request` so their bearer tokens are also dropped on cross-origin redirects; both built their own Request and called `urllib.request.urlopen` directly, bypassing the handler added for #1062.
diff --git a/skills/last30days/scripts/lib/github.py b/skills/last30days/scripts/lib/github.py
index efc2f3e7..c438cf69 100644
--- a/skills/last30days/scripts/lib/github.py
+++ b/skills/last30days/scripts/lib/github.py
@@ -102,7 +102,7 @@ def _fetch_json(
req = urllib.request.Request(url, headers=headers)
try:
- with urllib.request.urlopen(req, timeout=timeout) as resp:
+ with http.open_request(req, timeout) as resp:
body = resp.read().decode("utf-8")
return json.loads(body)
except urllib.error.HTTPError as e:
@@ -583,7 +583,7 @@ def _fetch_readme_snippet(repo: str, token: str, max_chars: int = 500) -> Option
req = urllib.request.Request(url, headers=headers)
try:
- with urllib.request.urlopen(req, timeout=10) as resp:
+ with http.open_request(req, 10) as resp:
raw = resp.read().decode("utf-8", errors="replace")
except (urllib.error.HTTPError, urllib.error.URLError, OSError, TimeoutError):
return None
diff --git a/skills/last30days/scripts/lib/http.py b/skills/last30days/scripts/lib/http.py
index 57bcd83c..773cb35d 100644
--- a/skills/last30days/scripts/lib/http.py
+++ b/skills/last30days/scripts/lib/http.py
@@ -134,6 +134,17 @@ def _open_request(req, timeout):
return current(req, timeout=timeout)
return _opener.open(req, timeout=timeout)
+
+def open_request(req, timeout):
+ """Open ``req`` with cross-origin credential stripping (see #1062).
+
+ Public entry point for modules that build their own ``Request`` and would
+ otherwise call ``urllib.request.urlopen`` directly, bypassing the redirect
+ handler installed above.
+ """
+ return _open_request(req, timeout)
+
+
_failure_sink: ContextVar[Optional[list["HTTPError"]]] = ContextVar(
"last30days_http_failure_sink",
default=None,
diff --git a/skills/last30days/scripts/lib/transcribe.py b/skills/last30days/scripts/lib/transcribe.py
index d3191fb3..3a6346d3 100644
--- a/skills/last30days/scripts/lib/transcribe.py
+++ b/skills/last30days/scripts/lib/transcribe.py
@@ -226,7 +226,7 @@ def _post_audio(provider: str, path: str, api_key: str, timeout: float) -> Optio
method="POST",
)
usage.begin(provider)
- with urllib.request.urlopen(req, timeout=timeout) as resp:
+ with http.open_request(req, timeout) as resp:
data = json.loads(resp.read().decode("utf-8"))
return data.get("text")
diff --git a/tests/test_github_redirect_auth.py b/tests/test_github_redirect_auth.py
new file mode 100644
index 00000000..4d3f4c1a
--- /dev/null
+++ b/tests/test_github_redirect_auth.py
@@ -0,0 +1,246 @@
+"""Modules that build their own Request must not leak credentials on redirect.
+
+``lib/http.py`` installs a redirect handler that drops credential headers when a
+3xx changes origin, but that only protects callers that go through it. Several
+modules build a ``urllib.request.Request`` themselves; ``github.py`` and
+``transcribe.py`` attach a bearer token to theirs, so they must use
+``http.open_request`` rather than ``urllib.request.urlopen``.
+"""
+
+import ast
+import http.server
+import pathlib
+import socketserver
+import threading
+from urllib.request import Request
+
+import pytest
+
+from lib import github, transcribe
+from lib import http as l30d_http
+
+
+class _CaptureHandler(http.server.BaseHTTPRequestHandler):
+ captured: dict = {}
+
+ def do_GET(self):
+ type(self).captured = {k.lower(): v for k, v in self.headers.items()}
+ body = b'{"ok":true}'
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def log_message(self, *_args):
+ pass
+
+
+def _serve(handler):
+ httpd = socketserver.TCPServer(("127.0.0.1", 0), handler)
+ threading.Thread(target=httpd.serve_forever, daemon=True).start()
+ return httpd
+
+
+def test_open_request_strips_bearer_across_origin():
+ attacker = _serve(_CaptureHandler)
+ _CaptureHandler.captured = {}
+
+ class Victim(http.server.BaseHTTPRequestHandler):
+ def do_GET(self):
+ self.send_response(302)
+ self.send_header(
+ "Location", f"http://127.0.0.1:{attacker.server_address[1]}/steal"
+ )
+ self.send_header("Content-Length", "0")
+ self.end_headers()
+
+ def log_message(self, *_args):
+ pass
+
+ victim = _serve(Victim)
+ req = Request(
+ f"http://127.0.0.1:{victim.server_address[1]}/start",
+ headers={"Authorization": "Bearer ghp_sentinel_token"},
+ )
+ try:
+ with l30d_http.open_request(req, 10) as resp:
+ resp.read()
+ finally:
+ victim.shutdown()
+ attacker.shutdown()
+ victim.server_close()
+ attacker.server_close()
+
+ assert "authorization" not in _CaptureHandler.captured, (
+ "bearer token survived a cross-origin redirect: "
+ f"{_CaptureHandler.captured.get('authorization')!r}"
+ )
+
+
[email protected]("caller", ["github_json", "github_readme", "transcribe"])
[email protected]("cross_origin", [False, True], ids=["same_origin", "cross_origin"])
+def test_credential_callers_handle_redirects(caller, cross_origin, monkeypatch, tmp_path):
+ captured = []
+ payload = b'{"ok":true,"text":"loopback transcript"}'
+ token = "dummy_redirect_caller_token"
+
+ class Handler(http.server.BaseHTTPRequestHandler):
+ def do_GET(self):
+ captured.append((self.path, dict(self.headers.items())))
+ if self.path == "/start":
+ self.send_response(302)
+ self.send_header("Location", destination)
+ self.send_header("Content-Length", "0")
+ self.end_headers()
+ else:
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(payload)))
+ self.end_headers()
+ self.wfile.write(payload)
+
+ def do_POST(self):
+ self.rfile.read(int(self.headers.get("Content-Length", "0")))
+ self.do_GET()
+
+ def log_message(self, *_args):
+ pass
+
+ origin = _serve(Handler)
+ target = _serve(Handler) if cross_origin else origin
+ start = f"http://127.0.0.1:{origin.server_address[1]}/start"
+ destination = f"http://127.0.0.1:{target.server_address[1]}/result"
+ try:
+ if caller == "github_json":
+ assert github._fetch_json(start, token=token) == {
+ "ok": True, "text": "loopback transcript",
+ }
+ elif caller == "github_readme":
+ def loopback_request(url, *args, **kwargs):
+ if url == "https://api.github.com/repos/owner/repo/readme":
+ url = start
+ return Request(url, *args, **kwargs)
+
+ monkeypatch.setattr(github.urllib.request, "Request", loopback_request)
+ assert github._fetch_readme_snippet("owner/repo", token) == payload.decode()
+ else:
+ monkeypatch.setitem(transcribe._PROVIDER_ENDPOINTS, "groq", start)
+ audio = tmp_path / "audio.mp3"
+ audio.write_bytes(b"dummy audio")
+ assert transcribe._post_audio("groq", str(audio), token, 10) == "loopback transcript"
+ finally:
+ origin.shutdown()
+ origin.server_close()
+ if target is not origin:
+ target.shutdown()
+ target.server_close()
+
+ assert [path for path, _ in captured] == ["/start", "/result"]
+ original_headers, final_headers = (
+ {key.lower(): value for key, value in headers.items()}
+ for _, headers in captured
+ )
+ assert original_headers["authorization"] == f"Bearer {token}"
+ assert final_headers.get("authorization") == (None if cross_origin else f"Bearer {token}")
+ assert final_headers["user-agent"] == original_headers["user-agent"]
+
+
+def _dotted_name(node) -> str:
+ """Render a dotted attribute chain (``urllib.request``) or "" if not one."""
+ parts = []
+ while isinstance(node, ast.Attribute):
+ parts.append(node.attr)
+ node = node.value
+ if not isinstance(node, ast.Name):
+ return ""
+ parts.append(node.id)
+ return ".".join(reversed(parts))
+
+
+def _direct_urlopen_calls(source: str) -> list[int]:
+ """Line numbers of calls that reach ``urllib.request.urlopen`` in ``source``.
+
+ Resolved through the module's imports rather than by text match. Searching
+ for the literal ``urllib.request.urlopen(`` would miss
+ ``from urllib.request import urlopen``, a form this repository already uses
+ in four other modules, so a future edit to a credential-bearing module could
+ reintroduce the bypass while keeping the guard green.
+ """
+ tree = ast.parse(source)
+
+ # Names bound to the urlopen function itself.
+ urlopen_names: set[str] = set()
+ # Names bound to the urllib.request module.
+ module_names: set[str] = {"urllib.request"}
+
+ for node in ast.walk(tree):
+ if isinstance(node, ast.ImportFrom):
+ if node.module == "urllib.request":
+ for alias in node.names:
+ if alias.name == "urlopen":
+ urlopen_names.add(alias.asname or alias.name)
+ elif node.module == "urllib":
+ for alias in node.names:
+ if alias.name == "request":
+ module_names.add(alias.asname or alias.name)
+ elif isinstance(node, ast.Import):
+ for alias in node.names:
+ if alias.name == "urllib.request":
+ # `import urllib.request` binds "urllib"; an asname binds that.
+ module_names.add(alias.asname or "urllib.request")
+ elif alias.name == "urllib":
+ module_names.add(f"{alias.asname or 'urllib'}.request")
+
+ hits: list[int] = []
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Call):
+ continue
+ func = node.func
+ if isinstance(func, ast.Name) and func.id in urlopen_names:
+ hits.append(node.lineno)
+ elif isinstance(func, ast.Attribute) and func.attr == "urlopen":
+ if _dotted_name(func.value) in module_names:
+ hits.append(node.lineno)
+ return sorted(hits)
+
+
+def test_credential_bearing_modules_do_not_call_urlopen_directly():
+ """Regression guard: these modules attach a bearer token to their Request."""
+ lib = pathlib.Path(l30d_http.__file__).parent
+ for name in ("github.py", "transcribe.py"):
+ source = (lib / name).read_text(encoding="utf-8")
+ hits = _direct_urlopen_calls(source)
+ assert hits == [], (
+ f"{name} calls urllib.request.urlopen directly at line(s) "
+ f"{hits}, bypassing the cross-origin credential strip in "
+ "http.open_request"
+ )
+
+
+def test_urlopen_guard_detects_aliased_forms():
+ """The guard itself must not be fooled by an import alias.
+
+ Without this, the guard could silently stop working: a text search for
+ ``urllib.request.urlopen(`` passes on every aliased form below.
+ """
+ detected = (
+ "import urllib.request\nurllib.request.urlopen(req)\n",
+ "from urllib.request import urlopen\nurlopen(req)\n",
+ "from urllib.request import urlopen as uo\nuo(req)\n",
+ "from urllib import request\nrequest.urlopen(req)\n",
+ "from urllib import request as r\nr.urlopen(req)\n",
+ "import urllib.request as ur\nur.urlopen(req)\n",
+ )
+ for source in detected:
+ assert _direct_urlopen_calls(source), f"missed: {source!r}"
+
+ ignored = (
+ # Routed through the protected wrapper — the whole point of the fix.
+ "from . import http\nhttp.open_request(req, timeout=10)\n",
+ # An unrelated urlopen on some other object must not trip the guard.
+ "session.urlopen(req)\n",
+ "from urllib.request import Request\nRequest(url)\n",
+ )
+ for source in ignored:
+ assert not _direct_urlopen_calls(source), f"false positive: {source!r}"