AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
fix(grounding): keep undated keyed-web results instead of dropping the whole response (#928) (#1038)
Terry Li60ec4a049dfa
diff --git a/changelog.d/928.fixed.md b/changelog.d/928.fixed.mdnew file mode 100644index 00000000..8d135cd8--- /dev/null+++ b/changelog.d/928.fixed.md@@ -0,0 +1 @@+Brave, Exa, and Serper web results with unknown dates now survive the research pipeline when the provider bounds the requested window. Parallel applies a provider start-date filter and keeps undated results for current-window searches. Unknown dates remain labeled as such.diff --git a/skills/last30days/scripts/lib/grounding.py b/skills/last30days/scripts/lib/grounding.pyindex 99ec6475..681fd7af 100644--- a/skills/last30days/scripts/lib/grounding.py+++ b/skills/last30days/scripts/lib/grounding.py@@ -6,7 +6,7 @@ import reimport sysimport urllib.parsefrom dataclasses import dataclass-from datetime import datetime, timedelta+from datetime import datetime, timedelta, timezonefrom urllib.parse import urlparsefrom . import dates, env, http, parallel_mcp, schema, web_search_keyless@@ -65,7 +65,7 @@ def brave_search(for i, r in enumerate((data.get("web", {}).get("results", []))[:count]):raw_date = r.get("page_age") or ""pub_date = _normalize_date(raw_date[:10]) if raw_date else None- if not _in_date_range(pub_date, date_range):+ if _known_date_out_of_range(pub_date, date_range):continueitems.append({"id": f"WB{i + 1}",@@ -76,6 +76,7 @@ def brave_search("date": pub_date,"relevance": 0.8,"why_relevant": "Brave web search",+ "metadata": {"date_window_basis": "server_bounds"},})artifact = {"label": "brave", "webSearchQueries": [query], "resultCount": len(items)}return items, artifact@@ -110,7 +111,7 @@ def exa_search(continueraw_date = r.get("publishedDate") or ""pub_date = _normalize_date(raw_date.split("T")[0] if "T" in raw_date else raw_date[:10]) if raw_date else None- if not _in_date_range(pub_date, date_range):+ if _known_date_out_of_range(pub_date, date_range):continueitems.append({"id": f"WE{i + 1}",@@ -121,6 +122,7 @@ def exa_search("date": pub_date,"relevance": 0.8,"why_relevant": "Exa web search",+ "metadata": {"date_window_basis": "server_bounds"},})artifact = {"label": "exa", "webSearchQueries": [query], "resultCount": len(items)}return items, artifact@@ -147,7 +149,9 @@ def serper_search(for i, r in enumerate((data.get("organic", []))[:count]):raw_date = r.get("date") or ""pub_date = _parse_serper_date(raw_date)- if not _in_date_range(pub_date, date_range):+ if pub_date is None and _SERPER_RELATIVE_RE.match(raw_date.strip()):+ continue+ if _known_date_out_of_range(pub_date, date_range):continueitems.append({"id": f"WS{i + 1}",@@ -158,6 +162,7 @@ def serper_search("date": pub_date,"relevance": 0.8,"why_relevant": "Serper web search",+ "metadata": {"date_window_basis": "server_bounds"},})artifact = {"label": "serper", "webSearchQueries": [query], "resultCount": len(items)}return items, artifact@@ -170,12 +175,16 @@ def serper_search(def parallel_search(query: str, date_range: tuple[str, str], api_key: str, count: int = 5,) -> tuple[list[dict], dict]:+ current_window = date_range[1] >= datetime.now(timezone.utc).date().isoformat()data = http.request("POST", "https://api.parallel.ai/v1/search",- headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},+ headers={"x-api-key": api_key, "Content-Type": "application/json"},json_data={"search_queries": [query],- "advanced_settings": {"max_results": count},+ "advanced_settings": {+ "max_results": count,+ "source_policy": {"after_date": date_range[0]},+ },},timeout=15,)@@ -188,7 +197,9 @@ def parallel_search(continueraw_date = r.get("publish_date") or ""pub_date = _normalize_date(raw_date[:10]) if raw_date else None- if not _in_date_range(pub_date, date_range):+ if _known_date_out_of_range(pub_date, date_range):+ continue+ if pub_date is None and not current_window:continueitems.append({"id": f"WP{i + 1}",@@ -199,6 +210,9 @@ def parallel_search("date": pub_date,"relevance": 0.8,"why_relevant": "Parallel AI web search",+ "metadata": {+ "date_window_basis": "server_start_current" if current_window else "server_start"+ },})artifact = {"label": "parallel", "webSearchQueries": [query], "resultCount": len(items)}return items, artifact@@ -248,7 +262,7 @@ def _parse_serper_relative(raw: str, now: datetime | None = None) -> str | None:return Noneunit = {"min": "minute", "hr": "hour"}.get(m.group(2), m.group(2))try:- amount = int(m.group(1))+ amount = int(m.group(1).lstrip("0") or "0")if unit == "minute":return (anchor - timedelta(minutes=amount)).date().isoformat()if unit == "hour":@@ -426,5 +440,10 @@ def _in_date_range(pub_date: str | None, date_range: tuple[str, str]) -> bool:return date_range[0] <= pub_date <= date_range[1]+def _known_date_out_of_range(pub_date: str | None, date_range: tuple[str, str]) -> bool:+ """True when a parseable date falls outside the window. Unknown dates stay."""+ return pub_date is not None and not _in_date_range(pub_date, date_range)++def _domain(url: str) -> str:return urlparse(url).netloc.strip().lower()diff --git a/skills/last30days/scripts/lib/normalize.py b/skills/last30days/scripts/lib/normalize.pyindex 352233f3..ebd39148 100644--- a/skills/last30days/scripts/lib/normalize.py+++ b/skills/last30days/scripts/lib/normalize.py@@ -18,7 +18,12 @@ def filter_by_date_range(filtered: list[schema.SourceItem] = []for item in items:if not item.published_at:- if not require_date:+ basis = item.metadata.get("date_window_basis") if item.source == "grounding" else None+ if (+ not require_date+ or basis == "server_bounds"+ or basis == "server_start_current"+ ):filtered.append(item)continueif item.published_at < from_date or item.published_at > to_date:diff --git a/tests/test_grounding_v3.py b/tests/test_grounding_v3.pyindex 2d824c60..860a907d 100644--- a/tests/test_grounding_v3.py+++ b/tests/test_grounding_v3.py@@ -5,7 +5,7 @@ import unittestimport urllib.errorfrom contextlib import contextmanagerfrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServer-from datetime import datetime+from datetime import datetime, timedelta, timezonefrom unittest.mock import patchfrom lib import grounding, parallel_mcp@@ -31,17 +31,19 @@ class BraveSearchTests(unittest.TestCase):{"title": "Undated Article","url": "https://example.com/undated",- "description": "Should also be filtered",+ "description": "Kept because date is unknown",}]}}with patch("lib.grounding.http.request", return_value=mock_response) as mock_req:items, artifact = grounding.brave_search("test", ("2026-02-25", "2026-03-27"), "fake-key")- self.assertEqual(1, len(items))+ self.assertEqual(2, len(items))self.assertEqual("Test Article", items[0]["title"])self.assertEqual("https://example.com/article", items[0]["url"])self.assertEqual("2026-03-10", items[0]["date"])+ self.assertEqual("Undated Article", items[1]["title"])+ self.assertIsNone(items[1]["date"])self.assertEqual("brave", artifact["label"])call_url = mock_req.call_args.args[1]self.assertIn("freshness=2026-02-25to2026-03-27", call_url)@@ -66,15 +68,17 @@ class SerperSearchTests(unittest.TestCase):{"title": "Undated Result","link": "https://example.com/undated",- "snippet": "Should also be filtered",+ "snippet": "Kept because date is unknown",}]}with patch("lib.grounding.http.request", return_value=mock_response):items, artifact = grounding.serper_search("test", ("2026-02-25", "2026-03-27"), "fake-key")- self.assertEqual(1, len(items))+ self.assertEqual(2, len(items))self.assertEqual("Serper Result", items[0]["title"])self.assertEqual("2026-03-15", items[0]["date"])+ self.assertEqual("Undated Result", items[1]["title"])+ self.assertIsNone(items[1]["date"])self.assertEqual("serper", artifact["label"])def test_serper_search_keeps_relative_dates(self):@@ -212,6 +216,12 @@ class SerperRelativeDateTests(unittest.TestCase):with self.subTest(raw=raw):self.assertIsNone(grounding._parse_serper_date(raw, self.NOON))+ def test_long_zero_padded_age_is_still_parseable(self):+ self.assertEqual(+ "2026-03-26",+ grounding._parse_serper_date("0" * 5000 + "1 day ago", self.NOON),+ )+class ExaSearchTests(unittest.TestCase):def test_exa_search_filters_to_in_range_dated_items(self):@@ -234,19 +244,21 @@ class ExaSearchTests(unittest.TestCase):{"title": "Undated Exa Result","url": "https://example.com/undated-exa",- "text": "No date means filtered",+ "text": "Kept because date is unknown",},]}with patch("lib.grounding.http.request", return_value=mock_response) as mock_req:items, artifact = grounding.exa_search("test", ("2026-02-25", "2026-03-27"), "fake-exa-key")- self.assertEqual(1, len(items))+ self.assertEqual(2, len(items))self.assertEqual("Exa Result", items[0]["title"])self.assertEqual("https://example.com/exa", items[0]["url"])self.assertEqual("2026-03-15", items[0]["date"])+ self.assertEqual("Undated Exa Result", items[1]["title"])+ self.assertIsNone(items[1]["date"])self.assertTrue(items[0]["id"].startswith("WE"))self.assertEqual("exa", artifact["label"])- self.assertEqual(1, artifact["resultCount"])+ self.assertEqual(2, artifact["resultCount"])# Verify API callcall_args = mock_req.call_argsself.assertEqual("POST", call_args.args[0])@@ -262,13 +274,15 @@ class ExaSearchTests(unittest.TestCase):class ParallelSearchTests(unittest.TestCase):def test_parallel_search_filters_to_in_range_dated_items(self):+ today = datetime.now(timezone.utc).date()+ date_range = ((today - timedelta(days=30)).isoformat(), today.isoformat())mock_response = {"results": [{"title": "Parallel Result","url": "https://example.com/parallel","snippet": "A parallel snippet",- "publish_date": "2026-03-15T00:00:00Z",+ "publish_date": f"{today.isoformat()}T00:00:00Z",},{"title": "Old Parallel Result",@@ -279,27 +293,26 @@ class ParallelSearchTests(unittest.TestCase):{"title": "Undated Parallel Result","url": "https://example.com/undated-parallel",- "snippet": "Should also be filtered",+ "snippet": "Kept because date is unknown",},]}with patch("lib.grounding.http.request", return_value=mock_response) as mock_req:items, artifact = grounding.parallel_search(- "test", ("2026-02-25", "2026-03-27"), "fake-parallel-key"+ "test", date_range, "fake-parallel-key")- self.assertEqual(1, len(items))+ self.assertEqual(2, len(items))self.assertEqual("Parallel Result", items[0]["title"])self.assertEqual("https://example.com/parallel", items[0]["url"])- self.assertEqual("2026-03-15", items[0]["date"])+ self.assertEqual(today.isoformat(), items[0]["date"])+ self.assertEqual("Undated Parallel Result", items[1]["title"])+ self.assertIsNone(items[1]["date"])self.assertTrue(items[0]["id"].startswith("WP"))self.assertEqual("parallel", artifact["label"])- self.assertEqual(1, artifact["resultCount"])+ self.assertEqual(2, artifact["resultCount"])self.assertEqual("POST", mock_req.call_args.args[0])self.assertEqual("https://api.parallel.ai/v1/search", mock_req.call_args.args[1])- self.assertEqual(- "Bearer fake-parallel-key",- mock_req.call_args.kwargs["headers"]["Authorization"],- )+ self.assertEqual("fake-parallel-key", mock_req.call_args.kwargs["headers"]["x-api-key"])def test_parallel_search_returns_empty_for_no_results(self):with patch("lib.grounding.http.request", return_value={"results": []}):diff --git a/tests/test_keyed_web_date_filter.py b/tests/test_keyed_web_date_filter.pynew file mode 100644index 00000000..06bea607--- /dev/null+++ b/tests/test_keyed_web_date_filter.py@@ -0,0 +1,309 @@+"""Keyed web results with unknown dates keep their date uncertainty (#928).++Brave, Exa, and Serper bound both ends at the provider. Parallel bounds the+start date and retains undated results only for current-window searches.+"""++from datetime import datetime, timedelta, timezone+from unittest.mock import patch++import pytest++from lib import grounding, normalize, render++DATE_RANGE = ("2026-02-25", "2026-03-27")+TODAY = datetime.now(timezone.utc).date()+LIVE_DATE_RANGE = ((TODAY - timedelta(days=30)).isoformat(), TODAY.isoformat())+++ ("search", "payload"),+ [+ (grounding.brave_search, {"web": {"results": [+ {"title": "Undated", "url": "https://example.com/brave"},+ ]}}),+ (grounding.exa_search, {"results": [+ {"title": "Undated", "url": "https://example.com/exa"},+ ]}),+ (grounding.serper_search, {"organic": [+ {"title": "Undated", "link": "https://example.com/serper"},+ ]}),+ ],+)+def test_server_bounded_undated_result_survives_normalization(search, payload):+ with patch("lib.grounding.http.request", return_value=payload):+ raw, _ = search("test", DATE_RANGE, "fake-key")+ normalized = normalize.normalize_source_items("grounding", raw, *DATE_RANGE)+ assert len(normalized) == 1+ assert normalized[0].published_at is None+ assert normalized[0].date_confidence == "low"+ assert render._format_date(normalized[0]) == "date unknown [date:low]"+++def test_parallel_undated_result_survives_only_for_current_window():+ today = datetime.now(timezone.utc).date()+ date_range = ((today - timedelta(days=30)).isoformat(), today.isoformat())+ payload = {"results": [{+ "title": "Undated", "url": "https://example.com/parallel",+ }]}+ with patch("lib.grounding.http.request", return_value=payload) as request:+ raw, _ = grounding.parallel_search("test", date_range, "fake-key")+ assert request.call_args.kwargs["json_data"]["advanced_settings"]["source_policy"] == {+ "after_date": date_range[0]+ }+ normalized = normalize.normalize_source_items("grounding", raw, *date_range)+ assert len(normalized) == 1+ assert normalized[0].published_at is None+ assert normalized[0].date_confidence == "low"+ assert render._format_date(normalized[0]) == "date unknown [date:low]"+++def test_parallel_historical_window_drops_undated_result():+ today = datetime.now(timezone.utc).date()+ date_range = (+ (today - timedelta(days=60)).isoformat(),+ (today - timedelta(days=30)).isoformat(),+ )+ payload = {"results": [{+ "title": "Undated", "url": "https://example.com/parallel",+ }]}+ with patch("lib.grounding.http.request", return_value=payload):+ raw, artifact = grounding.parallel_search("test", date_range, "fake-key")+ assert raw == []+ assert artifact["resultCount"] == 0+++def test_parallel_undated_result_survives_utc_midnight_between_stages():+ today = datetime.now(timezone.utc).date()+ date_range = (+ (today - timedelta(days=8)).isoformat(),+ (today - timedelta(days=1)).isoformat(),+ )+ payload = {"results": [{+ "title": "Undated", "url": "https://example.com/parallel",+ }]}+ with patch("lib.grounding.http.request", return_value=payload), \+ patch("lib.grounding.datetime") as retrieval_clock:+ retrieval_clock.now.return_value = datetime.fromisoformat(+ f"{date_range[1]}T23:59:00+00:00"+ )+ raw, _ = grounding.parallel_search("test", date_range, "fake-key")+ normalized = normalize.normalize_source_items("grounding", raw, *date_range)+ assert len(normalized) == 1+ assert normalized[0].published_at is None+ assert normalized[0].date_confidence == "low"+++def _titles(items):+ return [item["title"] for item in items]+++class TestSerperKeepsUndatedOrganicResults:+ def test_all_undated_organic_results_are_returned(self):+ payload = {+ "organic": [+ {"title": "A", "link": "https://a.example/1", "snippet": "a"},+ {"title": "B", "link": "https://b.example/2", "snippet": "b"},+ {"title": "C", "link": "https://c.example/3", "snippet": "c"},+ ]+ }+ with patch("lib.grounding.http.request", return_value=payload):+ items, artifact = grounding.serper_search(+ "OpenAI", DATE_RANGE, "fake-key", count=5+ )+ assert _titles(items) == ["A", "B", "C"]+ assert artifact["resultCount"] == 3+ assert all(item["date"] is None for item in items)++ def test_drops_only_known_out_of_range_dates(self):+ payload = {+ "organic": [+ {"title": "Undated A", "link": "https://a.example/1", "snippet": "a"},+ {+ "title": "Out of range",+ "link": "https://b.example/2",+ "snippet": "b",+ "date": "Apr 28, 2025",+ },+ {"title": "Undated C", "link": "https://c.example/3", "snippet": "c"},+ {+ "title": "In range",+ "link": "https://d.example/4",+ "snippet": "d",+ "date": "Mar 15, 2026",+ },+ {"title": "Undated E", "link": "https://e.example/5", "snippet": "e"},+ ]+ }+ with patch("lib.grounding.http.request", return_value=payload):+ items, artifact = grounding.serper_search(+ "OpenAI", DATE_RANGE, "fake-key", count=5+ )+ assert _titles(items) == ["Undated A", "Undated C", "In range", "Undated E"]+ assert artifact["resultCount"] == 4+ assert items[2]["date"] == "2026-03-15"+ assert items[0]["date"] is None++ def test_unparseable_date_is_kept(self):+ payload = {+ "organic": [+ {+ "title": "Garbage date",+ "link": "https://a.example/1",+ "snippet": "a",+ "date": "not a date",+ },+ ]+ }+ with patch("lib.grounding.http.request", return_value=payload):+ items, artifact = grounding.serper_search(+ "OpenAI", DATE_RANGE, "fake-key", count=5+ )+ assert _titles(items) == ["Garbage date"]+ assert artifact["resultCount"] == 1+ assert items[0]["date"] is None+++class TestBraveKeepsUndatedResults:+ def test_all_undated_results_are_returned(self):+ payload = {+ "web": {+ "results": [+ {+ "title": "U",+ "url": "https://u.example/",+ "description": "u",+ }+ ]+ }+ }+ with patch("lib.grounding.http.request", return_value=payload):+ items, artifact = grounding.brave_search(+ "OpenAI", DATE_RANGE, "fake-key", count=5+ )+ assert _titles(items) == ["U"]+ assert artifact["resultCount"] == 1+ assert items[0]["date"] is None++ def test_drops_only_known_out_of_range_dates(self):+ payload = {+ "web": {+ "results": [+ {+ "title": "In range",+ "url": "https://example.com/article",+ "description": "ok",+ "page_age": "2026-03-10T00:00:00",+ },+ {+ "title": "Old",+ "url": "https://example.com/old",+ "description": "old",+ "page_age": "2025-12-10T00:00:00",+ },+ {+ "title": "Undated",+ "url": "https://example.com/undated",+ "description": "no date",+ },+ ]+ }+ }+ with patch("lib.grounding.http.request", return_value=payload):+ items, artifact = grounding.brave_search(+ "test", DATE_RANGE, "fake-key"+ )+ assert _titles(items) == ["In range", "Undated"]+ assert artifact["resultCount"] == 2+++class TestExaKeepsUndatedResults:+ def test_all_undated_results_are_returned(self):+ payload = {+ "results": [+ {"title": "U", "url": "https://u.example/", "text": "u"},+ ]+ }+ with patch("lib.grounding.http.request", return_value=payload):+ items, artifact = grounding.exa_search(+ "OpenAI", DATE_RANGE, "fake-key", count=5+ )+ assert _titles(items) == ["U"]+ assert artifact["resultCount"] == 1+ assert items[0]["date"] is None++ def test_drops_only_known_out_of_range_dates(self):+ payload = {+ "results": [+ {+ "title": "In range",+ "url": "https://example.com/exa",+ "text": "ok",+ "publishedDate": "2026-03-15T00:00:00.000Z",+ },+ {+ "title": "Old",+ "url": "https://example.com/old-exa",+ "text": "old",+ "publishedDate": "2025-12-01T00:00:00.000Z",+ },+ {+ "title": "Undated",+ "url": "https://example.com/undated-exa",+ "text": "no date",+ },+ ]+ }+ with patch("lib.grounding.http.request", return_value=payload):+ items, artifact = grounding.exa_search("test", DATE_RANGE, "fake-key")+ assert _titles(items) == ["In range", "Undated"]+ assert artifact["resultCount"] == 2+++class TestParallelKeepsUndatedResults:+ def test_all_undated_results_are_returned(self):+ payload = {+ "results": [+ {+ "title": "U",+ "url": "https://u.example/",+ "excerpts": ["u"],+ }+ ]+ }+ with patch("lib.grounding.http.request", return_value=payload):+ items, artifact = grounding.parallel_search(+ "OpenAI", LIVE_DATE_RANGE, "fake-key", count=5+ )+ assert _titles(items) == ["U"]+ assert artifact["resultCount"] == 1+ assert items[0]["date"] is None++ def test_drops_only_known_out_of_range_dates(self):+ payload = {+ "results": [+ {+ "title": "In range",+ "url": "https://example.com/parallel",+ "excerpts": ["ok"],+ "publish_date": f"{LIVE_DATE_RANGE[1]}T00:00:00Z",+ },+ {+ "title": "Old",+ "url": "https://example.com/old-parallel",+ "excerpts": ["old"],+ "publish_date": "2025-12-01T00:00:00Z",+ },+ {+ "title": "Undated",+ "url": "https://example.com/undated-parallel",+ "excerpts": ["no date"],+ },+ ]+ }+ with patch("lib.grounding.http.request", return_value=payload):+ items, artifact = grounding.parallel_search(+ "test", LIVE_DATE_RANGE, "fake-key"+ )+ assert _titles(items) == ["In range", "Undated"]+ assert artifact["resultCount"] == 2