AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
Fix dev.to lookback for historical research windows (#1296)
vectorforgelabsio13a6cc5fd1f2
diff --git a/changelog.d/+devto-historical-window.fixed.md b/changelog.d/+devto-historical-window.fixed.mdnew file mode 100644index 00000000..1298d522--- /dev/null+++ b/changelog.d/+devto-historical-window.fixed.md@@ -0,0 +1 @@+dev.to now finds articles for research windows that end before today. It pages back through newer results within a 45-second budget, keeps only articles inside the window, and includes the whole first day of every window.diff --git a/skills/last30days/scripts/lib/devto.py b/skills/last30days/scripts/lib/devto.pyindex 61c568fa..0f2e8c3a 100644--- a/skills/last30days/scripts/lib/devto.py+++ b/skills/last30days/scripts/lib/devto.py@@ -4,16 +4,21 @@ Uses the public Forem API at dev.to (no key):- ``/api/articles?tag=<tag>&top=<days>`` returns the most-engaged articles fora tag in the window. Precise, but only as good as the tag.-- ``/api/articles/search?q=<term>&top=<days>`` is free-text search sorted by- recency. Single distinctive terms match well; multi-word queries match- loosely, so every search hit must pass the engine's token-overlap relevance- check before it is kept.+- ``/api/articles/search?q=<term>&top=<days>`` is free-text search sorted+ roughly by recency. Single distinctive terms match well; multi-word queries+ match loosely, so every search hit must pass the engine's token-overlap+ relevance check before it is kept.- ``/api/comments?a_id=<id>`` returns the comment tree for an article.The source is off by default. Host-inferred tags (``--devto-tags``) turn iton for a run; ``INCLUDE_SOURCES=devto`` or ``--search devto`` turn it onwithout tags, in which case only the keyword search runs. An unknown tagreturns ``[]`` with HTTP 200, so a wrong tag stays quiet instead of failing.++Both endpoints count ``top`` back from now and accept no end date, sort, or+date-range parameter. A window that ends before today therefore pages+through everything published since its start and filters locally, within+``HISTORICAL_MAX_PAGES`` and ``HISTORICAL_BUDGET_SECONDS``."""from __future__ import annotations@@ -48,6 +53,11 @@ SEARCH_RELEVANCE_FLOOR = 0.25# Total wall-clock cap per request. The socket timeout alone bounds each# read, so a response trickling in small chunks could otherwise run on.REQUEST_DEADLINE_SECONDS = 20.0+# Forem's per_page ceiling. A busy term ("kubernetes") has ~4,000 search hits+# over 78 days, about 2 MB and 4-5 s per full page.+HISTORICAL_PER_PAGE = 1000+HISTORICAL_MAX_PAGES = 10+HISTORICAL_BUDGET_SECONDS = 45.0ENRICH_WAIT_SECONDS = 25.0_TAG_RE = re.compile(r"[^a-z0-9]")@@ -76,11 +86,22 @@ def resolve_tags(tags: list[str] | None, depth: str = "default") -> list[str]:return resolved[: TAG_LIMITS.get(depth, TAG_LIMITS["default"])]+def _today() -> datetime.date:+ return datetime.datetime.now(datetime.timezone.utc).date()+++def _is_historical(to_date: str | None) -> bool:+ return bool(to_date) and str(to_date) < _today().isoformat()++def _window_days(from_date: str | None, to_date: str | None) -> int:try:start = datetime.date.fromisoformat(from_date or "")- end = datetime.date.fromisoformat(to_date or "") if to_date else datetime.date.today()- return max(1, (end - start).days)+ # Forem counts top=N back from now, including for historical windows,+ # so the reach always runs to today and _in_window enforces to_date.+ # The extra day keeps all of from_date: a now-anchored cutoff of+ # exactly N days lands partway through it.+ return max(1, (_today() - start).days + 1)except ValueError:return 30@@ -98,7 +119,9 @@ def clear_cache() -> None:_URL_LOCKS.clear()-def _get_list(url: str, headers: dict[str, str] | None = None) -> list[dict[str, Any]]:+def _get_list(+ url: str, headers: dict[str, str] | None = None, deadline: float | None = None,+) -> list[dict[str, Any]]:# Concurrent subquery streams can request the same URL; the per-URL lock# makes the second caller wait for the first fetch instead of repeating it.with _LOCKS_GUARD:@@ -107,7 +130,10 @@ def _get_list(url: str, headers: dict[str, str] | None = None) -> list[dict[str,if url not in _RESPONSE_CACHE:data = http.request("GET", url, headers=headers, timeout=15, retries=2,- deadline_monotonic=time.monotonic() + REQUEST_DEADLINE_SECONDS,+ deadline_monotonic=min(+ time.monotonic() + REQUEST_DEADLINE_SECONDS,+ deadline if deadline is not None else float("inf"),+ ),owned_get=True,)if not isinstance(data, list):@@ -118,8 +144,58 @@ def _get_list(url: str, headers: dict[str, str] | None = None) -> list[dict[str,return [dict(row) for row in _RESPONSE_CACHE[url]]+def _published(article: dict[str, Any]) -> str:+ return str(article.get("published_at") or "")[:10]+++def _fetch_rows(+ base_url: str,+ params: dict[str, Any],+ to_date: str | None,+ *,+ label: str,+ deadline: float,+ headers: dict[str, str] | None = None,+) -> list[dict[str, Any]]:+ """Fetch one page for a current window, or page through a past one.++ Every page after the first stops at ``deadline``, which the caller shares+ across all endpoints so a deep run's tags cannot each spend the budget.+ """+ if not _is_historical(to_date):+ return _get_list(f"{base_url}?{urllib.parse.urlencode(params)}", headers=headers)+ rows: list[dict[str, Any]] = []+ for page in range(1, HISTORICAL_MAX_PAGES + 1):+ if page > 1 and time.monotonic() >= deadline:+ _log(f"{label}: stopped after {page - 1} pages at the time budget; older in-window articles may be missing")+ return rows+ page_params = {**params, "per_page": HISTORICAL_PER_PAGE, "page": page}+ url = f"{base_url}?{urllib.parse.urlencode(page_params)}"+ try:+ batch = _get_list(url, headers=headers, deadline=deadline if page > 1 else None)+ except Exception as exc:+ if page == 1:+ raise+ _log(f"{label}: page {page} failed ({exc}); keeping the {page - 1} pages fetched")+ return rows+ rows.extend(batch)+ # top=N bounds the result set at from_date, so a short page is the end.+ if len(batch) < HISTORICAL_PER_PAGE:+ return rows+ _log(f"{label}: stopped at {HISTORICAL_MAX_PAGES} pages; older in-window articles may be missing")+ return rows+++def _top_engaged(rows: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]:+ return sorted(+ rows,+ key=lambda row: int(row.get("public_reactions_count") or row.get("positive_reactions_count") or 0),+ reverse=True,+ )[:limit]++def _in_window(article: dict[str, Any], from_date: str | None, to_date: str | None) -> bool:- published = str(article.get("published_at") or "")[:10]+ published = _published(article)if not published:return Trueif from_date and published < from_date:@@ -151,24 +227,28 @@ def search_devto(endpoint is recorded in ``errors`` and the other endpoint still counts."""days = _window_days(from_date, to_date)+ historical = _is_historical(to_date)+ paging_deadline = time.monotonic() + HISTORICAL_BUDGET_SECONDSresolved_tags = resolve_tags(tags, depth)articles: dict[int, dict[str, Any]] = {}errors: list[str] = []for tag in resolved_tags:- url = f"{ARTICLES_URL}?{urllib.parse.urlencode({'tag': tag, 'top': days, 'per_page': TAG_PER_PAGE})}"+ params = {"tag": tag, "top": days, "per_page": TAG_PER_PAGE}try:- rows = _get_list(url)+ rows = _fetch_rows(ARTICLES_URL, params, to_date, label=f"tag '{tag}'", deadline=paging_deadline)except Exception as exc:errors.append(f"tag {tag}: {exc}")_log(f"tag feed failed for '{tag}': {exc}")continue+ rows = [row for row in rows if row.get("id") is not None and _in_window(row, from_date, to_date)]+ if historical:+ rows = _top_engaged(rows, TAG_PER_PAGE)kept = 0for row in rows:- if row.get("id") is not None and _in_window(row, from_date, to_date):- row.setdefault("_devto_match", f"tag:{tag}")- articles.setdefault(row["id"], row)- kept += 1+ row.setdefault("_devto_match", f"tag:{tag}")+ articles.setdefault(row["id"], row)+ kept += 1_log(f"tag '{tag}': {kept} articles in window")# Planner queries quote phrases for engines with phrase syntax; Forem@@ -176,21 +256,31 @@ def search_devto(query = (search_query or topic or "").replace('"', " ").strip()query = " ".join(query.split())if query:- params = {"q": query, "top": days, "per_page": SEARCH_PER_PAGE.get(depth, SEARCH_PER_PAGE["default"])}- url = f"{SEARCH_URL}?{urllib.parse.urlencode(params)}"+ per_page = SEARCH_PER_PAGE.get(depth, SEARCH_PER_PAGE["default"])+ params = {"q": query, "top": days, "per_page": per_page}try:- rows = _get_list(url, headers=_SEARCH_HEADERS)+ rows = _fetch_rows(+ SEARCH_URL, params, to_date, label=f"search '{query}'",+ deadline=paging_deadline, headers=_SEARCH_HEADERS,+ )except Exception as exc:errors.append(f"search: {exc}")_log(f"search failed for '{query}': {exc}")rows = []kept = dropped = 0+ relevant = []for row in rows:if row.get("id") is None or not _in_window(row, from_date, to_date):continueif token_overlap_relevance(topic or query, _article_text(row)) < SEARCH_RELEVANCE_FLOOR:dropped += 1continue+ relevant.append(row)+ if historical:+ # A current window keeps the newest page; a past one keeps the+ # most-engaged in-window hits so the item count stays comparable.+ relevant = _top_engaged(relevant, per_page)+ for row in relevant:row.setdefault("_devto_match", "search")articles.setdefault(row["id"], row)kept += 1diff --git a/tests/test_devto.py b/tests/test_devto.pyindex e739ebbc..3b5b8594 100644--- a/tests/test_devto.py+++ b/tests/test_devto.py@@ -1,5 +1,6 @@"""dev.to source: tag and search retrieval, relevance gating, comments."""+import datetimeimport urllib.parseimport pytest@@ -7,6 +8,14 @@ import pytestfrom lib import devto, normalize, pipeline+TODAY = datetime.date(2026, 10, 11)++[email protected](autouse=True)+def _fixed_today(monkeypatch):+ monkeypatch.setattr(devto, "_today", lambda: TODAY)++@pytest.fixture(autouse=True)def _fresh_cache():devto.clear_cache()@@ -103,7 +112,7 @@ def test_tag_feed_and_search_merge_dedupe_and_filter(monkeypatch):assert ids == [1, 2, 3]assert "error" not in resulttag_call = next(url for url in api.calls if "tag=" in url)- assert "top=30" in tag_call+ assert "top=31" in tag_calldef test_articles_outside_the_window_are_dropped(monkeypatch):@@ -115,6 +124,118 @@ def test_articles_outside_the_window_are_dropped(monkeypatch):assert [article["id"] for article in result["articles"]] == [1][email protected]("tags", [None, ["rust"]])+def test_historical_window_reaches_back_from_today_and_keeps_upper_bound(monkeypatch, tags):+ historical = _article(1, "Rust async", tags=("rust",), published="2026-08-10T12:00:00Z")+ recent = _article(2, "Rust async current", tags=("rust",), published="2026-10-10T12:00:00Z")+ calls = []++ def relative_window_api(method, url, headers=None, **kwargs):+ query = dict(urllib.parse.parse_qsl(urllib.parse.urlparse(url).query))+ days = int(query["top"])+ calls.append(days)+ assert (query["per_page"], query["page"]) == ("1000", "1")+ cutoff = TODAY - datetime.timedelta(days=days)+ return [row for row in (historical, recent)+ if datetime.date.fromisoformat(row["published_at"][:10]) >= cutoff]++ monkeypatch.setattr(devto.http, "request", relative_window_api)+ result = devto.search_devto("rust async", "2026-08-01", "2026-08-31", tags=tags)+ assert [row["id"] for row in result["articles"]] == [1]+ assert calls and all(days == 72 for days in calls)+ assert "error" not in result++[email protected]("end", [None, "2026-10-11"])+def test_current_window_reach_covers_all_of_from_date(end):+ assert devto._window_days("2026-09-11", end) == 31+++def _paged_api(pages, calls, fail_page=None, path="/api/articles"):+ def api(method, url, headers=None, **kwargs):+ parsed = urllib.parse.urlparse(url)+ if parsed.path != path:+ return []+ query = dict(urllib.parse.parse_qsl(parsed.query))+ page = int(query.get("page", "1"))+ calls.append(page)+ if page == fail_page:+ raise OSError("page down")+ return pages[page - 1] if page <= len(pages) else []+ return api+++def _filler(start_id, count, published):+ return [_article(start_id + i, f"Rust filler {i}", tags=("rust",), published=published) for i in range(count)]++[email protected]("tags", [None, ["rust"]])+def test_historical_window_pages_past_newer_articles(monkeypatch, tags):+ in_window = _article(1, "Rust async", tags=("rust",), published="2026-08-10T12:00:00Z")+ calls = []+ pages = [_filler(1000, devto.HISTORICAL_PER_PAGE, "2026-10-05T12:00:00Z"), [in_window]]+ path = "/api/articles" if tags else "/api/articles/search"+ monkeypatch.setattr(devto.http, "request", _paged_api(pages, calls, path=path))+ result = devto.search_devto("rust async", "2026-08-01", "2026-08-31", tags=tags)+ assert [row["id"] for row in result["articles"]] == [1]+ assert calls == [1, 2]+++def test_historical_paging_stops_at_the_page_cap(monkeypatch):+ monkeypatch.setattr(devto, "HISTORICAL_MAX_PAGES", 3)+ calls = []+ pages = [_filler(1000 * n, devto.HISTORICAL_PER_PAGE, "2026-10-05T12:00:00Z") for n in range(1, 6)]+ monkeypatch.setattr(devto.http, "request", _paged_api(pages, calls))+ devto.search_devto("rust", "2026-08-01", "2026-08-31", tags=["rust"], search_query="")+ assert calls == [1, 2, 3]+++def test_historical_paging_stops_at_the_time_budget(monkeypatch):+ monkeypatch.setattr(devto, "HISTORICAL_BUDGET_SECONDS", 0.0)+ calls = []+ pages = [_filler(1000 * n, devto.HISTORICAL_PER_PAGE, "2026-10-05T12:00:00Z") for n in range(1, 4)]+ monkeypatch.setattr(devto.http, "request", _paged_api(pages, calls))+ devto.search_devto("rust", "2026-08-01", "2026-08-31", tags=["rust"], search_query="")+ assert calls == [1]+++def test_one_paging_budget_covers_every_endpoint(monkeypatch):+ clock = [0.0]+ monkeypatch.setattr(devto.time, "monotonic", lambda: clock[0])+ calls = []+ full = _filler(1000, devto.HISTORICAL_PER_PAGE, "2026-10-05T12:00:00Z")+ paged = _paged_api([full] * 5, calls)++ def slow_api(method, url, **kwargs):+ clock[0] += devto.HISTORICAL_BUDGET_SECONDS / 2 + 1+ return paged(method, url, **kwargs)++ monkeypatch.setattr(devto.http, "request", slow_api)+ devto.search_devto("rust", "2026-08-01", "2026-08-31", depth="deep", tags=["rust", "go", "zig"], search_query="")+ assert calls == [1, 2, 1, 1]+++def test_later_page_failure_keeps_the_pages_already_fetched(monkeypatch):+ calls = []+ pages = [_filler(1000, devto.HISTORICAL_PER_PAGE, "2026-08-10T12:00:00Z")]+ monkeypatch.setattr(devto.http, "request", _paged_api(pages, calls, fail_page=2))+ result = devto.search_devto("rust", "2026-08-01", "2026-08-31", tags=["rust"], search_query="")+ assert calls == [1, 2]+ assert len(result["articles"]) == devto.TAG_PER_PAGE+ assert "error" not in result+++def test_historical_window_keeps_the_most_engaged_in_window_articles(monkeypatch):+ rows = [+ _article(i, f"Rust post {i}", tags=("rust",), published="2026-08-10T12:00:00Z", reactions=i)+ for i in range(1, 51)+ ]+ rows.append(_article(99, "Rust hot but late", tags=("rust",), published="2026-09-10T12:00:00Z", reactions=500))+ monkeypatch.setattr(devto.http, "request", _paged_api([rows], []))+ result = devto.search_devto("rust", "2026-08-01", "2026-08-31", tags=["rust"], search_query="")+ assert sorted(row["id"] for row in result["articles"]) == list(range(21, 51))++def test_unknown_tag_and_offtopic_search_stay_quiet(monkeypatch):_install(monkeypatch, FakeApi(search=[_article(9, "Weekly frontend news", tags=("webdev",))]))result = devto.search_devto("Taylor Swift", "2026-09-11", "2026-10-11")