AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
fix: restore keyless web search on DuckDuckGo-blocked IPs; stop reddit enrichment from poisoning web results
Mekiwi2626d0ff2fca
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 88f3e15d..170f1d80 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Fixed++- Keyless web search now works on hosts where DuckDuckGo anomaly-blocks the egress IP (a 202 challenge page with no results — common on datacenter/VPS IPs). Added Startpage as a second keyless rung (DuckDuckGo → Startpage → configured SearXNG), so the web floor still returns results there. Also hardened `_strip_html` to drop `<style>`/`<script>` contents so inline CSS can't leak into a title or snippet.+- Web/grounding results are no longer discarded when one of them is a reddit.com URL whose enrichment fetch fails. Reddit enrichment is a best-effort secondary fetch; its HTTP failures (e.g. a 403 on a datacenter IP) were being attributed to the whole web source, which then reported "0 items — HTTP 403" despite having retrieved good results. Its failures are now isolated from the source's outcome.+## [3.15.0] - 2026-07-14### Addeddiff --git a/skills/last30days/scripts/lib/grounding.py b/skills/last30days/scripts/lib/grounding.pyindex 0e1985f5..b91dd7f2 100644--- a/skills/last30days/scripts/lib/grounding.py+++ b/skills/last30days/scripts/lib/grounding.py@@ -275,7 +275,13 @@ def web_search(else:return [], {}if items and not _reddit_excluded(config):- items = _enrich_reddit_items(items)+ # Reddit enrichment is a best-effort secondary fetch on already-retrieved+ # web results. Isolate its HTTP failures in a throwaway capture sink so a+ # reddit.com fetch failure (e.g. a 403 on a datacenter IP) is not+ # attributed to the web/grounding source itself — which would otherwise+ # discard the successfully retrieved results and report the source failed.+ with http.capture_failures():+ items = _enrich_reddit_items(items)return items, artifactdiff --git a/skills/last30days/scripts/lib/web_search_keyless.py b/skills/last30days/scripts/lib/web_search_keyless.pyindex 4664ba26..443e2d90 100644--- a/skills/last30days/scripts/lib/web_search_keyless.py+++ b/skills/last30days/scripts/lib/web_search_keyless.py@@ -9,10 +9,12 @@ It must never run on a host that has native search (the model does it betterthere) or preempt a configured paid backend. The pipeline/grounding layer ownsthat gating; this module just performs the search when asked.-Two vendor-neutral rungs, both stdlib-only via :mod:`http`:+Three vendor-neutral rungs, all stdlib-only via :mod:`http`:1. DuckDuckGo HTML endpoint (no key, no instance to maintain).- 2. A configurable SearXNG instance returning JSON (``LAST30DAYS_SEARXNG_URL``),- tried when the primary yields nothing.+ 2. Startpage HTML results, tried when DuckDuckGo yields nothing — notably+ when DuckDuckGo anomaly-blocks a datacenter IP with a 202 challenge page.+ 3. A configurable SearXNG instance returning JSON (``LAST30DAYS_SEARXNG_URL``),+ tried when the HTML rungs yield nothing.Never raises. Returns results in the same dict shape as the paid backends in:mod:`grounding` so they flow through normalize/score/dedupe unchanged. On total@@ -37,6 +39,10 @@ _DDG_HTML_URL = "https://html.duckduckgo.com/html/"_KEYLESS_RELEVANCE = 0.6_TAG_RE = re.compile(r"<[^>]+>")+# Strip <style>/<script> blocks *including their contents* before dropping tags,+# so inline CSS/JS text (e.g. Startpage's emotion styles) never leaks into a+# title or snippet.+_STYLE_SCRIPT_RE = re.compile(r"<(style|script)\b[^>]*>.*?</\1>", re.IGNORECASE | re.DOTALL)_RESULT_A_RE = re.compile(r'class="result__a"[^>]*href="(?P<href>[^"]+)"[^>]*>(?P<title>.*?)</a>',re.IGNORECASE | re.DOTALL,@@ -46,6 +52,21 @@ _SNIPPET_RE = re.compile(re.IGNORECASE | re.DOTALL,)+_STARTPAGE_HTML_URL = "https://www.startpage.com/sp/search"+# Startpage marks each organic hit with an <a class="result-title result-link …"+# href="<target>">…<h2 …>title</h2></a>, and the description in a following+# <p class="…description…">. Class names carry hashed emotion suffixes, so match+# on the stable "result-title" / "description" substrings.+_SP_RESULT_RE = re.compile(+ r'<a\b[^>]*class="[^"]*result-title[^"]*"[^>]*href="(?P<href>https?://[^"]+)"[^>]*>(?P<inner>.*?)</a>',+ re.IGNORECASE | re.DOTALL,+)+_SP_H2_RE = re.compile(r"<h2\b[^>]*>(?P<title>.*?)</h2>", re.IGNORECASE | re.DOTALL)+_SP_DESC_RE = re.compile(+ r'<p\b[^>]*class="[^"]*description[^"]*"[^>]*>(?P<snippet>.*?)</p>',+ re.IGNORECASE | re.DOTALL,+)+def _domain(url: str) -> str:# Normalize identically to grounding._domain (strip + lowercase) so keyless@@ -57,7 +78,8 @@ def _domain(url: str) -> str:def _strip_html(fragment: str) -> str:- return html.unescape(_TAG_RE.sub("", fragment or "")).strip()+ without_blocks = _STYLE_SCRIPT_RE.sub("", fragment or "")+ return html.unescape(_TAG_RE.sub("", without_blocks)).strip()def _unwrap_ddg_redirect(href: str) -> str:@@ -81,6 +103,11 @@ def keyless_search("""Run keyless web search; returns (items, artifact). Never raises."""items = _search_ddg(query, count)used = "ddg"+ if not items:+ # DuckDuckGo anomaly-blocks datacenter IPs (202 challenge page); fall+ # back to Startpage, which still serves organic results there.+ items = _search_startpage(query, count)+ used = "startpage"if not items:searxng_url = (config.get("LAST30DAYS_SEARXNG_URL") or "").strip()if searxng_url:@@ -123,6 +150,38 @@ def _search_ddg(query: str, count: int) -> list[dict]:return items+def _search_startpage(query: str, count: int) -> list[dict]:+ """Keyless rung 2: Startpage's HTML results page. Unlike DuckDuckGo's HTML+ endpoint (which anomaly-blocks datacenter IPs with a 202 challenge page),+ Startpage returns organic results to a plain browser-UA GET, making it the+ working floor on hosts DuckDuckGo refuses. Never raises."""+ url = f"{_STARTPAGE_HTML_URL}?{urlencode({'query': query})}"+ text = http.get_text(url, accept="text/html", retries=2)+ if not text:+ return []+ items: list[dict] = []+ result_matches = list(_SP_RESULT_RE.finditer(text))+ desc_matches = list(_SP_DESC_RE.finditer(text))+ for match in result_matches:+ if len(items) >= count:+ break+ target = html.unescape(match.group("href"))+ if not target.startswith("http"):+ continue+ h2 = _SP_H2_RE.search(match.group("inner"))+ title = _strip_html(h2.group("title") if h2 else match.group("inner"))+ if not title:+ continue+ # First description block that appears after this result's title anchor.+ snippet = ""+ for desc in desc_matches:+ if desc.start() > match.end():+ snippet = _strip_html(desc.group("snippet"))+ break+ items.append(_to_item(len(items), title, target, snippet))+ return items++def _search_searxng(query: str, count: int, instance_url: str) -> list[dict]:base = instance_url.rstrip("/")url = f"{base}/search?{urlencode({'q': query, 'format': 'json'})}"diff --git a/tests/test_grounding_v3.py b/tests/test_grounding_v3.pyindex 26aa2a21..f083845e 100644--- a/tests/test_grounding_v3.py+++ b/tests/test_grounding_v3.py@@ -350,5 +350,34 @@ class RedditEnrichItemsTests(unittest.TestCase):msg=f"Expected a rate-limit stderr message, got: {captured_stderr!r}",)+class RedditEnrichmentIsolationTests(unittest.TestCase):+ def test_enrichment_http_failure_does_not_poison_web_source(self):+ """A reddit.com enrichment fetch failure (e.g. a 403 on a datacenter IP)+ is a secondary operation on already-retrieved web results; it must not be+ attributed to the web/grounding source and discard those results."""+ from lib import http++ retrieved = [+ {"url": "https://www.reddit.com/r/x/comments/1/abc/", "title": "T", "snippet": "s"},+ ]++ def fake_enrich(items):+ # The enricher swallows the error, but the http layer records the+ # terminal failure into whatever capture sink is currently active.+ http._record_failure(http.HTTPError("Blocked", status_code=403))+ return items++ with http.capture_failures() as source_sink:+ with patch.object(grounding, "web_search_keyless") as wsk, \+ patch.object(grounding, "_enrich_reddit_items", side_effect=fake_enrich):+ wsk.keyless_search.return_value = (list(retrieved), {"keyless_backend": "startpage"})+ items, _ = grounding.web_search(+ "q", ("2026-02-25", "2026-03-27"), {}, backend="keyless")++ self.assertEqual(len(items), 1)+ # The enrichment 403 is isolated in its own sink; the source's sink is clean.+ self.assertEqual(source_sink, [])++if __name__ == "__main__":unittest.main()diff --git a/tests/test_web_search_keyless.py b/tests/test_web_search_keyless.pyindex 0cc3bdb4..8c28e338 100644--- a/tests/test_web_search_keyless.py+++ b/tests/test_web_search_keyless.py@@ -65,3 +65,60 @@ class TestKeylessSearch:with mock.patch.object(web_search_keyless.http, "get_text", return_value=html):items, _ = web_search_keyless.keyless_search("topic", ("2026-02-25", "2026-03-27"), {})assert items == []+++# Startpage marks each hit with an <a class="result-title result-link …"> whose+# href is the target and whose <h2> is the title; the emotion <style> block+# (inline CSS) must never leak into the parsed title. Description follows in a+# <p class="…description…">.+_STARTPAGE_HTML = """+<a class="result-title result-link css-1bggj8v" href="https://example.com/post" data-testid="gl-title-link">+ <style data-emotion="css i3irj7">.css-i3irj7{line-height:18px;color:#2E39B3;}</style>+ <h2 class="wgl-title css-i3irj7">First & Best Result</h2>+</a>+<p class="description css-abc">A snippet about the topic.</p>+<a class="result-title result-link css-1bggj8v" href="https://news.example.org/a" data-testid="gl-title-link">+ <h2 class="wgl-title css-i3irj7">Second result</h2>+</a>+<p class="description css-abc">Second snippet.</p>+"""+++class TestStartpageFallback:+ def _get_text(self, startpage_html):+ # DuckDuckGo yields nothing (its datacenter-IP 202 challenge page has no+ # result anchors); Startpage yields real results.+ def side_effect(url, *args, **kwargs):+ return startpage_html if "startpage.com" in url else ""+ return side_effect++ def test_startpage_used_when_ddg_empty(self):+ with mock.patch.object(+ web_search_keyless.http, "get_text",+ side_effect=self._get_text(_STARTPAGE_HTML),+ ):+ items, artifact = web_search_keyless.keyless_search(+ "topic", ("2026-02-25", "2026-03-27"), {})+ assert artifact["keyless_backend"] == "startpage"+ assert artifact.get("reason") is None+ assert len(items) == 2+ assert items[0]["url"] == "https://example.com/post"+ # The inline <style> CSS must not bleed into the title.+ assert items[0]["title"] == "First & Best Result"+ assert "css-" not in items[0]["title"]+ assert items[0]["snippet"] == "A snippet about the topic."+ assert items[0]["source_domain"] == "example.com"++ def test_ddg_preferred_over_startpage(self):+ # When DuckDuckGo returns results, Startpage is not consulted.+ with mock.patch.object(web_search_keyless.http, "get_text", return_value=_DDG_HTML):+ _, artifact = web_search_keyless.keyless_search(+ "topic", ("2026-02-25", "2026-03-27"), {})+ assert artifact["keyless_backend"] == "ddg"+++class TestStripHtml:+ def test_strips_style_block_contents(self):+ got = web_search_keyless._strip_html(+ "<style>.a{color:red}</style><h2>Title</h2>")+ assert got == "Title"