Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

fix: restore keyless web search on DuckDuckGo-blocked IPs; stop reddit enrichment from poisoning web results

Mekiwi2626d0ff2fca
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 88f3e15d..170f1d80 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Fixed
+
+- Keyless web search now works on hosts where DuckDuckGo anomaly-blocks the egress IP (a 202 challenge page with no results — common on datacenter/VPS IPs). Added Startpage as a second keyless rung (DuckDuckGo → Startpage → configured SearXNG), so the web floor still returns results there. Also hardened `_strip_html` to drop `<style>`/`<script>` contents so inline CSS can't leak into a title or snippet.
+- Web/grounding results are no longer discarded when one of them is a reddit.com URL whose enrichment fetch fails. Reddit enrichment is a best-effort secondary fetch; its HTTP failures (e.g. a 403 on a datacenter IP) were being attributed to the whole web source, which then reported "0 items — HTTP 403" despite having retrieved good results. Its failures are now isolated from the source's outcome.
+
## [3.15.0] - 2026-07-14
### Added
diff --git a/skills/last30days/scripts/lib/grounding.py b/skills/last30days/scripts/lib/grounding.py
index 0e1985f5..b91dd7f2 100644
--- a/skills/last30days/scripts/lib/grounding.py
+++ b/skills/last30days/scripts/lib/grounding.py
@@ -275,7 +275,13 @@ def web_search(
else:
return [], {}
if items and not _reddit_excluded(config):
- items = _enrich_reddit_items(items)
+ # Reddit enrichment is a best-effort secondary fetch on already-retrieved
+ # web results. Isolate its HTTP failures in a throwaway capture sink so a
+ # reddit.com fetch failure (e.g. a 403 on a datacenter IP) is not
+ # attributed to the web/grounding source itself — which would otherwise
+ # discard the successfully retrieved results and report the source failed.
+ with http.capture_failures():
+ items = _enrich_reddit_items(items)
return items, artifact
diff --git a/skills/last30days/scripts/lib/web_search_keyless.py b/skills/last30days/scripts/lib/web_search_keyless.py
index 4664ba26..443e2d90 100644
--- a/skills/last30days/scripts/lib/web_search_keyless.py
+++ b/skills/last30days/scripts/lib/web_search_keyless.py
@@ -9,10 +9,12 @@ It must never run on a host that has native search (the model does it better
there) or preempt a configured paid backend. The pipeline/grounding layer owns
that gating; this module just performs the search when asked.
-Two vendor-neutral rungs, both stdlib-only via :mod:`http`:
+Three vendor-neutral rungs, all stdlib-only via :mod:`http`:
1. DuckDuckGo HTML endpoint (no key, no instance to maintain).
- 2. A configurable SearXNG instance returning JSON (``LAST30DAYS_SEARXNG_URL``),
- tried when the primary yields nothing.
+ 2. Startpage HTML results, tried when DuckDuckGo yields nothing — notably
+ when DuckDuckGo anomaly-blocks a datacenter IP with a 202 challenge page.
+ 3. A configurable SearXNG instance returning JSON (``LAST30DAYS_SEARXNG_URL``),
+ tried when the HTML rungs yield nothing.
Never raises. Returns results in the same dict shape as the paid backends in
:mod:`grounding` so they flow through normalize/score/dedupe unchanged. On total
@@ -37,6 +39,10 @@ _DDG_HTML_URL = "https://html.duckduckgo.com/html/"
_KEYLESS_RELEVANCE = 0.6
_TAG_RE = re.compile(r"<[^>]+>")
+# Strip <style>/<script> blocks *including their contents* before dropping tags,
+# so inline CSS/JS text (e.g. Startpage's emotion styles) never leaks into a
+# title or snippet.
+_STYLE_SCRIPT_RE = re.compile(r"<(style|script)\b[^>]*>.*?</\1>", re.IGNORECASE | re.DOTALL)
_RESULT_A_RE = re.compile(
r'class="result__a"[^>]*href="(?P<href>[^"]+)"[^>]*>(?P<title>.*?)</a>',
re.IGNORECASE | re.DOTALL,
@@ -46,6 +52,21 @@ _SNIPPET_RE = re.compile(
re.IGNORECASE | re.DOTALL,
)
+_STARTPAGE_HTML_URL = "https://www.startpage.com/sp/search"
+# Startpage marks each organic hit with an <a class="result-title result-link …"
+# href="<target>">…<h2 …>title</h2></a>, and the description in a following
+# <p class="…description…">. Class names carry hashed emotion suffixes, so match
+# on the stable "result-title" / "description" substrings.
+_SP_RESULT_RE = re.compile(
+ r'<a\b[^>]*class="[^"]*result-title[^"]*"[^>]*href="(?P<href>https?://[^"]+)"[^>]*>(?P<inner>.*?)</a>',
+ re.IGNORECASE | re.DOTALL,
+)
+_SP_H2_RE = re.compile(r"<h2\b[^>]*>(?P<title>.*?)</h2>", re.IGNORECASE | re.DOTALL)
+_SP_DESC_RE = re.compile(
+ r'<p\b[^>]*class="[^"]*description[^"]*"[^>]*>(?P<snippet>.*?)</p>',
+ re.IGNORECASE | re.DOTALL,
+)
+
def _domain(url: str) -> str:
# Normalize identically to grounding._domain (strip + lowercase) so keyless
@@ -57,7 +78,8 @@ def _domain(url: str) -> str:
def _strip_html(fragment: str) -> str:
- return html.unescape(_TAG_RE.sub("", fragment or "")).strip()
+ without_blocks = _STYLE_SCRIPT_RE.sub("", fragment or "")
+ return html.unescape(_TAG_RE.sub("", without_blocks)).strip()
def _unwrap_ddg_redirect(href: str) -> str:
@@ -81,6 +103,11 @@ def keyless_search(
"""Run keyless web search; returns (items, artifact). Never raises."""
items = _search_ddg(query, count)
used = "ddg"
+ if not items:
+ # DuckDuckGo anomaly-blocks datacenter IPs (202 challenge page); fall
+ # back to Startpage, which still serves organic results there.
+ items = _search_startpage(query, count)
+ used = "startpage"
if not items:
searxng_url = (config.get("LAST30DAYS_SEARXNG_URL") or "").strip()
if searxng_url:
@@ -123,6 +150,38 @@ def _search_ddg(query: str, count: int) -> list[dict]:
return items
+def _search_startpage(query: str, count: int) -> list[dict]:
+ """Keyless rung 2: Startpage's HTML results page. Unlike DuckDuckGo's HTML
+ endpoint (which anomaly-blocks datacenter IPs with a 202 challenge page),
+ Startpage returns organic results to a plain browser-UA GET, making it the
+ working floor on hosts DuckDuckGo refuses. Never raises."""
+ url = f"{_STARTPAGE_HTML_URL}?{urlencode({'query': query})}"
+ text = http.get_text(url, accept="text/html", retries=2)
+ if not text:
+ return []
+ items: list[dict] = []
+ result_matches = list(_SP_RESULT_RE.finditer(text))
+ desc_matches = list(_SP_DESC_RE.finditer(text))
+ for match in result_matches:
+ if len(items) >= count:
+ break
+ target = html.unescape(match.group("href"))
+ if not target.startswith("http"):
+ continue
+ h2 = _SP_H2_RE.search(match.group("inner"))
+ title = _strip_html(h2.group("title") if h2 else match.group("inner"))
+ if not title:
+ continue
+ # First description block that appears after this result's title anchor.
+ snippet = ""
+ for desc in desc_matches:
+ if desc.start() > match.end():
+ snippet = _strip_html(desc.group("snippet"))
+ break
+ items.append(_to_item(len(items), title, target, snippet))
+ return items
+
+
def _search_searxng(query: str, count: int, instance_url: str) -> list[dict]:
base = instance_url.rstrip("/")
url = f"{base}/search?{urlencode({'q': query, 'format': 'json'})}"
diff --git a/tests/test_grounding_v3.py b/tests/test_grounding_v3.py
index 26aa2a21..f083845e 100644
--- a/tests/test_grounding_v3.py
+++ b/tests/test_grounding_v3.py
@@ -350,5 +350,34 @@ class RedditEnrichItemsTests(unittest.TestCase):
msg=f"Expected a rate-limit stderr message, got: {captured_stderr!r}",
)
+class RedditEnrichmentIsolationTests(unittest.TestCase):
+ def test_enrichment_http_failure_does_not_poison_web_source(self):
+ """A reddit.com enrichment fetch failure (e.g. a 403 on a datacenter IP)
+ is a secondary operation on already-retrieved web results; it must not be
+ attributed to the web/grounding source and discard those results."""
+ from lib import http
+
+ retrieved = [
+ {"url": "https://www.reddit.com/r/x/comments/1/abc/", "title": "T", "snippet": "s"},
+ ]
+
+ def fake_enrich(items):
+ # The enricher swallows the error, but the http layer records the
+ # terminal failure into whatever capture sink is currently active.
+ http._record_failure(http.HTTPError("Blocked", status_code=403))
+ return items
+
+ with http.capture_failures() as source_sink:
+ with patch.object(grounding, "web_search_keyless") as wsk, \
+ patch.object(grounding, "_enrich_reddit_items", side_effect=fake_enrich):
+ wsk.keyless_search.return_value = (list(retrieved), {"keyless_backend": "startpage"})
+ items, _ = grounding.web_search(
+ "q", ("2026-02-25", "2026-03-27"), {}, backend="keyless")
+
+ self.assertEqual(len(items), 1)
+ # The enrichment 403 is isolated in its own sink; the source's sink is clean.
+ self.assertEqual(source_sink, [])
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_web_search_keyless.py b/tests/test_web_search_keyless.py
index 0cc3bdb4..8c28e338 100644
--- a/tests/test_web_search_keyless.py
+++ b/tests/test_web_search_keyless.py
@@ -65,3 +65,60 @@ class TestKeylessSearch:
with mock.patch.object(web_search_keyless.http, "get_text", return_value=html):
items, _ = web_search_keyless.keyless_search("topic", ("2026-02-25", "2026-03-27"), {})
assert items == []
+
+
+# Startpage marks each hit with an <a class="result-title result-link …"> whose
+# href is the target and whose <h2> is the title; the emotion <style> block
+# (inline CSS) must never leak into the parsed title. Description follows in a
+# <p class="…description…">.
+_STARTPAGE_HTML = """
+<a class="result-title result-link css-1bggj8v" href="https://example.com/post" data-testid="gl-title-link">
+ <style data-emotion="css i3irj7">.css-i3irj7{line-height:18px;color:#2E39B3;}</style>
+ <h2 class="wgl-title css-i3irj7">First &amp; Best Result</h2>
+</a>
+<p class="description css-abc">A snippet about the topic.</p>
+<a class="result-title result-link css-1bggj8v" href="https://news.example.org/a" data-testid="gl-title-link">
+ <h2 class="wgl-title css-i3irj7">Second result</h2>
+</a>
+<p class="description css-abc">Second snippet.</p>
+"""
+
+
+class TestStartpageFallback:
+ def _get_text(self, startpage_html):
+ # DuckDuckGo yields nothing (its datacenter-IP 202 challenge page has no
+ # result anchors); Startpage yields real results.
+ def side_effect(url, *args, **kwargs):
+ return startpage_html if "startpage.com" in url else ""
+ return side_effect
+
+ def test_startpage_used_when_ddg_empty(self):
+ with mock.patch.object(
+ web_search_keyless.http, "get_text",
+ side_effect=self._get_text(_STARTPAGE_HTML),
+ ):
+ items, artifact = web_search_keyless.keyless_search(
+ "topic", ("2026-02-25", "2026-03-27"), {})
+ assert artifact["keyless_backend"] == "startpage"
+ assert artifact.get("reason") is None
+ assert len(items) == 2
+ assert items[0]["url"] == "https://example.com/post"
+ # The inline <style> CSS must not bleed into the title.
+ assert items[0]["title"] == "First & Best Result"
+ assert "css-" not in items[0]["title"]
+ assert items[0]["snippet"] == "A snippet about the topic."
+ assert items[0]["source_domain"] == "example.com"
+
+ def test_ddg_preferred_over_startpage(self):
+ # When DuckDuckGo returns results, Startpage is not consulted.
+ with mock.patch.object(web_search_keyless.http, "get_text", return_value=_DDG_HTML):
+ _, artifact = web_search_keyless.keyless_search(
+ "topic", ("2026-02-25", "2026-03-27"), {})
+ assert artifact["keyless_backend"] == "ddg"
+
+
+class TestStripHtml:
+ def test_strips_style_block_contents(self):
+ got = web_search_keyless._strip_html(
+ "<style>.a{color:red}</style><h2>Title</h2>")
+ assert got == "Title"