AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
from __future__ import annotations
import json
import os
from unittest.mock import MagicMock
import pytest
import last30days as cli
from lib import http, pipeline, schema
def _response(body: str):
response = MagicMock()
response.__enter__.return_value = response
response.__exit__.return_value = False
response.read.return_value = body.encode("utf-8")
response.status = 200
return response
def test_http_recording_scrubs_credentials_and_replays_offline(tmp_path, monkeypatch):
monkeypatch.setattr(http.urllib.request, "urlopen", lambda *_args, **_kwargs: _response('{"items": [{"url": "https://example.test/item"}]}'))
fixture_dir = tmp_path / "fixture"
with http.recording_requests(fixture_dir):
live = http.get("https://api.example.test/search?api_key=live-secret&q=agents")
fixture_text = (fixture_dir / "http.json").read_text(encoding="utf-8")
assert "live-secret" not in fixture_text
assert "%3Credacted%3E" in fixture_text
monkeypatch.setattr(
http.urllib.request,
"urlopen",
lambda *_args, **_kwargs: pytest.fail("fixture replay attempted the network"),
)
with http.replaying_requests(fixture_dir):
replayed = http.get("https://api.example.test/search?api_key=another-secret&q=agents")
assert replayed == live
def test_http_recording_redacts_credentials_echoed_in_response_values(tmp_path, monkeypatch):
monkeypatch.setattr(
http.urllib.request,
"urlopen",
lambda *_args, **_kwargs: _response('{"echo": "Bearer live-secret"}'),
)
fixture_dir = tmp_path / "fixture"
with http.recording_requests(fixture_dir):
http.get(
"https://api.example.test/profile",
headers={"Authorization": "Bearer live-secret"},
)
fixture_text = (fixture_dir / "http.json").read_text(encoding="utf-8")
assert "live-secret" not in fixture_text
assert '"echo": "<redacted>"' in fixture_text
def test_http_recording_scrubs_app_password_and_session_jwts(tmp_path, monkeypatch):
"""A Bluesky session exchange puts the app password in the request body and
both JWTs in the response. Redaction is key-name driven, so every one of
those names has to be recognized."""
monkeypatch.setattr(
http.urllib.request,
"urlopen",
lambda *_args, **_kwargs: _response(json.dumps({
"accessJwt": "eyJhbGciOi.ACCESS-SENTINEL",
"refreshJwt": "eyJhbGciOi.REFRESH-SENTINEL",
"handle": "me.bsky.social",
})),
)
fixture_dir = tmp_path / "fixture"
with http.recording_requests(fixture_dir):
http.post(
"https://bsky.social/xrpc/com.atproto.server.createSession",
json_data={
"identifier": "me.bsky.social",
"password": "abcd-efgh-ijkl-SENTINEL",
},
)
fixture_path = fixture_dir / "http.json"
fixture_text = fixture_path.read_text(encoding="utf-8")
assert "abcd-efgh-ijkl-SENTINEL" not in fixture_text
assert "ACCESS-SENTINEL" not in fixture_text
assert "REFRESH-SENTINEL" not in fixture_text
# Non-secret fields still round-trip, so the fixture stays useful.
assert "me.bsky.social" in fixture_text
# And the file is not world-readable.
if os.name != "nt":
assert fixture_path.stat().st_mode & 0o777 == 0o600
def test_recorded_fixture_is_private_from_creation_not_after_a_chmod(
tmp_path, monkeypatch
):
"""Tightening the mode after writing leaves the credentials in a
world-readable file for the length of the write. Assert the temp file is
opened 0600, since a final-mode check passes either way."""
monkeypatch.setattr(
http.urllib.request,
"urlopen",
lambda *_args, **_kwargs: _response('{"ok": true}'),
)
opened: list[tuple[str, int]] = []
real_open = os.open
def _recording_open(path, flags, mode=0o777, **kwargs):
opened.append((str(path), mode))
return real_open(path, flags, mode, **kwargs)
monkeypatch.setattr(http.os, "open", _recording_open)
fixture_dir = tmp_path / "fixture"
with http.recording_requests(fixture_dir):
http.get("https://api.example.test/thing")
tmp_opens = [
(path, mode) for path, mode in opened if path.endswith(".http.json.tmp")
]
assert tmp_opens, "the fixture temp file must be created via os.open with a mode"
assert all(mode == 0o600 for _path, mode in tmp_opens), tmp_opens
def test_is_secret_key_covers_credential_names_without_over_matching():
for name in (
"password", "passwd", "app_password", "BSKY_APP_PASSWORD",
"accessJwt", "refreshJwt", "jwt", "passphrase", "credential",
"api_key", "apiKey", "x_api_key", "Authorization", "cookie",
"secret", "token", "access_token",
):
assert http._is_secret_key(name), name
for name in ("monkey", "handle", "identifier", "url", "title", "did"):
assert not http._is_secret_key(name), name
def test_aborted_recording_does_not_overwrite_existing_fixture(tmp_path):
fixture = tmp_path / "http.json"
fixture.write_text("existing fixture\n", encoding="utf-8")
with pytest.raises(RuntimeError, match="capture failed"):
with http.recording_requests(fixture):
raise RuntimeError("capture failed")
assert fixture.read_text(encoding="utf-8") == "existing fixture\n"
def test_http_replay_rejects_unrecorded_requests(tmp_path):
fixture = tmp_path / "http.json"
fixture.write_text(
json.dumps({"format": "last30days-http-fixture/v1", "exchanges": []}),
encoding="utf-8",
)
with pytest.raises(AssertionError, match="Unrecorded HTTP request"), \
http.replaying_requests(fixture):
http.get("https://example.test/not-recorded")
def test_cli_backed_source_results_record_at_the_module_seam(tmp_path):
fixture_dir = tmp_path / "fixture"
request = {
"source": "digg",
"topic": "agents",
"search_query": "agents",
"date_range": ["2026-06-10", "2026-07-10"],
"depth": "quick",
}
value = [[{"url": "https://di.gg/ai/example"}], {"provider": "fixture"}]
with http.recording_requests(fixture_dir):
http.fixture_source_record(request, value)
with http.replaying_requests(fixture_dir):
matched, replayed = http.fixture_source_replay(request)
assert matched is True
assert replayed == value
def test_module_seam_capture_omits_nested_http_exchanges(tmp_path, monkeypatch):
monkeypatch.setattr(http.urllib.request, "urlopen", lambda *_args, **_kwargs: _response('{"ok": true}'))
fixture_dir = tmp_path / "fixture"
request = {
"source": "youtube",
"topic": "agents",
"search_query": "agents",
"date_range": ["2026-06-10", "2026-07-10"],
"depth": "quick",
}
with http.recording_requests(fixture_dir):
with http.fixture_module_capture(True):
http.get("https://api.example.test/nested-enrichment")
http.fixture_source_record(request, [[], {}])
payload = json.loads((fixture_dir / "http.json").read_text(encoding="utf-8"))
assert payload["exchanges"] == []
assert len(payload["source_exchanges"]) == 1
def test_module_seam_records_and_replays_adapter_failures(tmp_path, monkeypatch):
fixture_dir = tmp_path / "fixture"
kwargs = {
"source": "digg",
"topic": "agents",
"subquery": type("SubQuery", (), {"search_query": "agents"})(),
"date_range": ("2026-06-10", "2026-07-10"),
"depth": "quick",
}
monkeypatch.setattr(
pipeline,
"_retrieve_stream_impl",
lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("adapter failed")),
)
with http.recording_requests(fixture_dir):
with pytest.raises(RuntimeError, match="adapter failed"):
pipeline._retrieve_stream(**kwargs)
payload = json.loads((fixture_dir / "http.json").read_text(encoding="utf-8"))
assert payload["source_exchanges"] == [
{
"request": {
"source": "digg",
"topic": "agents",
"search_query": "agents",
"date_range": ["2026-06-10", "2026-07-10"],
"depth": "quick",
},
"type": "error",
"error": {
"exception_type": "RuntimeError",
"message": "adapter failed",
"outcome_state": None,
},
}
]
monkeypatch.setattr(
pipeline,
"_retrieve_stream_impl",
lambda **_kwargs: pytest.fail("replay called the live adapter"),
)
with http.replaying_requests(fixture_dir):
with pytest.raises(http.RecordedSourceError, match="adapter failed") as replayed:
pipeline._retrieve_stream(**kwargs)
assert replayed.value.exception_type == "RuntimeError"
@pytest.mark.parametrize("source", ["youtube", "digg"])
def test_post_ranking_cli_enrichment_records_and_replays(
tmp_path,
monkeypatch,
source,
):
fixture_dir = tmp_path / source
item = schema.SourceItem(
item_id="item-1",
source=source,
title="Fixture item",
body="Fixture body",
url=f"https://example.test/{source}/item-1",
engagement={"postCount": 1} if source == "digg" else {},
metadata={"clusterUrlId": "cluster-1"} if source == "digg" else {},
)
enrichment_calls = []
if source == "youtube":
def enrich(items, **_kwargs):
enrichment_calls.append(([entry.item_id for entry in items], _kwargs))
items[0].metadata["transcript_snippet"] = "recorded transcript"
monkeypatch.setattr(pipeline.youtube_yt, "backfill_transcripts", enrich)
expected_metadata = {"transcript_snippet": "recorded transcript"}
expected_kwargs = {"topic": "agents", "depth": "quick", "token": None}
else:
def enrich(items, **_kwargs):
enrichment_calls.append(([entry.item_id for entry in items], _kwargs))
items[0].metadata["posts"] = [{"url": "https://x.com/example/status/1"}]
return items
monkeypatch.setattr(pipeline.digg, "enrich_source_items", enrich)
expected_metadata = {
"clusterUrlId": "cluster-1",
"posts": [{"url": "https://x.com/example/status/1"}],
}
expected_kwargs = {"top_k": 3}
with http.recording_requests(fixture_dir):
recorded = pipeline._finalize_items_by_source(
{source: [item]}, topic="agents", depth="quick",
)
assert enrichment_calls == [(["item-1"], expected_kwargs)]
assert len(recorded[source]) == 1
assert recorded[source][0].metadata == expected_metadata
replay_item = schema.SourceItem(
item_id="item-1",
source=source,
title="Fixture item",
body="Fixture body",
url=f"https://example.test/{source}/item-1",
engagement={"postCount": 1} if source == "digg" else {},
metadata={"clusterUrlId": "cluster-1"} if source == "digg" else {},
)
monkeypatch.setattr(
pipeline.youtube_yt if source == "youtube" else pipeline.digg,
"backfill_transcripts" if source == "youtube" else "enrich_source_items",
lambda *_args, **_kwargs: pytest.fail("replay executed CLI enrichment"),
)
with http.replaying_requests(fixture_dir):
replayed = pipeline._finalize_items_by_source(
{source: [replay_item]}, topic="agents", depth="quick",
)
assert len(replayed[source]) == 1
assert replayed[source][0].metadata == expected_metadata
assert enrichment_calls == [(["item-1"], expected_kwargs)]
def test_record_fixtures_flag_is_dev_only_and_hidden_from_help():
parser = cli.build_parser()
args = parser.parse_args(["topic", "--record-fixtures", "tmp/eval-topic"])
assert args.record_fixtures == "tmp/eval-topic"
assert "--record-fixtures" not in parser.format_help()