AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
"""Modules that build their own Request must not leak credentials on redirect.
``lib/http.py`` installs a redirect handler that drops credential headers when a
3xx changes origin, but that only protects callers that go through it. Several
modules build a ``urllib.request.Request`` themselves; ``github.py`` and
``transcribe.py`` attach a bearer token to theirs, so they must use
``http.open_request`` rather than ``urllib.request.urlopen``.
"""
import ast
import http.server
import pathlib
import socketserver
import threading
from urllib.request import Request
import pytest
from lib import github, transcribe
from lib import http as l30d_http
class _CaptureHandler(http.server.BaseHTTPRequestHandler):
captured: dict = {}
def do_GET(self):
type(self).captured = {k.lower(): v for k, v in self.headers.items()}
body = b'{"ok":true}'
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *_args):
pass
def _serve(handler):
httpd = socketserver.TCPServer(("127.0.0.1", 0), handler)
threading.Thread(target=httpd.serve_forever, daemon=True).start()
return httpd
def test_open_request_strips_bearer_across_origin():
attacker = _serve(_CaptureHandler)
_CaptureHandler.captured = {}
class Victim(http.server.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(302)
self.send_header(
"Location", f"http://127.0.0.1:{attacker.server_address[1]}/steal"
)
self.send_header("Content-Length", "0")
self.end_headers()
def log_message(self, *_args):
pass
victim = _serve(Victim)
req = Request(
f"http://127.0.0.1:{victim.server_address[1]}/start",
headers={"Authorization": "Bearer ghp_sentinel_token"},
)
try:
with l30d_http.open_request(req, 10) as resp:
resp.read()
finally:
victim.shutdown()
attacker.shutdown()
victim.server_close()
attacker.server_close()
assert "authorization" not in _CaptureHandler.captured, (
"bearer token survived a cross-origin redirect: "
f"{_CaptureHandler.captured.get('authorization')!r}"
)
@pytest.mark.parametrize("caller", ["github_json", "github_readme", "transcribe"])
@pytest.mark.parametrize("cross_origin", [False, True], ids=["same_origin", "cross_origin"])
def test_credential_callers_handle_redirects(caller, cross_origin, monkeypatch, tmp_path):
captured = []
payload = b'{"ok":true,"text":"loopback transcript"}'
token = "dummy_redirect_caller_token"
class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
captured.append((self.path, dict(self.headers.items())))
if self.path == "/start":
self.send_response(302)
self.send_header("Location", destination)
self.send_header("Content-Length", "0")
self.end_headers()
else:
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
def do_POST(self):
self.rfile.read(int(self.headers.get("Content-Length", "0")))
self.do_GET()
def log_message(self, *_args):
pass
origin = _serve(Handler)
target = _serve(Handler) if cross_origin else origin
start = f"http://127.0.0.1:{origin.server_address[1]}/start"
destination = f"http://127.0.0.1:{target.server_address[1]}/result"
try:
if caller == "github_json":
assert github._fetch_json(start, token=token) == {
"ok": True, "text": "loopback transcript",
}
elif caller == "github_readme":
def loopback_request(url, *args, **kwargs):
if url == "https://api.github.com/repos/owner/repo/readme":
url = start
return Request(url, *args, **kwargs)
monkeypatch.setattr(github.urllib.request, "Request", loopback_request)
assert github._fetch_readme_snippet("owner/repo", token) == payload.decode()
else:
monkeypatch.setitem(transcribe._PROVIDER_ENDPOINTS, "groq", start)
audio = tmp_path / "audio.mp3"
audio.write_bytes(b"dummy audio")
assert transcribe._post_audio("groq", str(audio), token, 10) == "loopback transcript"
finally:
origin.shutdown()
origin.server_close()
if target is not origin:
target.shutdown()
target.server_close()
assert [path for path, _ in captured] == ["/start", "/result"]
original_headers, final_headers = (
{key.lower(): value for key, value in headers.items()}
for _, headers in captured
)
assert original_headers["authorization"] == f"Bearer {token}"
assert final_headers.get("authorization") == (None if cross_origin else f"Bearer {token}")
assert final_headers["user-agent"] == original_headers["user-agent"]
def _dotted_name(node) -> str:
"""Render a dotted attribute chain (``urllib.request``) or "" if not one."""
parts = []
while isinstance(node, ast.Attribute):
parts.append(node.attr)
node = node.value
if not isinstance(node, ast.Name):
return ""
parts.append(node.id)
return ".".join(reversed(parts))
def _direct_urlopen_calls(source: str) -> list[int]:
"""Line numbers of calls that reach ``urllib.request.urlopen`` in ``source``.
Resolved through the module's imports rather than by text match. Searching
for the literal ``urllib.request.urlopen(`` would miss
``from urllib.request import urlopen``, a form this repository already uses
in four other modules, so a future edit to a credential-bearing module could
reintroduce the bypass while keeping the guard green.
"""
tree = ast.parse(source)
# Names bound to the urlopen function itself.
urlopen_names: set[str] = set()
# Names bound to the urllib.request module.
module_names: set[str] = {"urllib.request"}
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom):
if node.module == "urllib.request":
for alias in node.names:
if alias.name == "urlopen":
urlopen_names.add(alias.asname or alias.name)
elif node.module == "urllib":
for alias in node.names:
if alias.name == "request":
module_names.add(alias.asname or alias.name)
elif isinstance(node, ast.Import):
for alias in node.names:
if alias.name == "urllib.request":
# `import urllib.request` binds "urllib"; an asname binds that.
module_names.add(alias.asname or "urllib.request")
elif alias.name == "urllib":
module_names.add(f"{alias.asname or 'urllib'}.request")
hits: list[int] = []
for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
func = node.func
if isinstance(func, ast.Name) and func.id in urlopen_names:
hits.append(node.lineno)
elif isinstance(func, ast.Attribute) and func.attr == "urlopen":
if _dotted_name(func.value) in module_names:
hits.append(node.lineno)
return sorted(hits)
def test_credential_bearing_modules_do_not_call_urlopen_directly():
"""Regression guard: these modules attach a bearer token to their Request."""
lib = pathlib.Path(l30d_http.__file__).parent
for name in ("github.py", "transcribe.py"):
source = (lib / name).read_text(encoding="utf-8")
hits = _direct_urlopen_calls(source)
assert hits == [], (
f"{name} calls urllib.request.urlopen directly at line(s) "
f"{hits}, bypassing the cross-origin credential strip in "
"http.open_request"
)
def test_urlopen_guard_detects_aliased_forms():
"""The guard itself must not be fooled by an import alias.
Without this, the guard could silently stop working: a text search for
``urllib.request.urlopen(`` passes on every aliased form below.
"""
detected = (
"import urllib.request\nurllib.request.urlopen(req)\n",
"from urllib.request import urlopen\nurlopen(req)\n",
"from urllib.request import urlopen as uo\nuo(req)\n",
"from urllib import request\nrequest.urlopen(req)\n",
"from urllib import request as r\nr.urlopen(req)\n",
"import urllib.request as ur\nur.urlopen(req)\n",
)
for source in detected:
assert _direct_urlopen_calls(source), f"missed: {source!r}"
ignored = (
# Routed through the protected wrapper — the whole point of the fix.
"from . import http\nhttp.open_request(req, timeout=10)\n",
# An unrelated urlopen on some other object must not trip the guard.
"session.urlopen(req)\n",
"from urllib.request import Request\nRequest(url)\n",
)
for source in ignored:
assert not _direct_urlopen_calls(source), f"false positive: {source!r}"