Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

fix: sample Arctic listings across subreddits within request deadlines (#1306)

yachunchen638da9716b0e
diff --git a/changelog.d/+arctic-windowed-scores.fixed.md b/changelog.d/+arctic-windowed-scores.fixed.md
new file mode 100644
index 00000000..18051219
--- /dev/null
+++ b/changelog.d/+arctic-windowed-scores.fixed.md
@@ -0,0 +1 @@
+Keyless Reddit discovery no longer collapses to 1-point stubs when shreddit is blocked: the arctic-shift supplement now slices the research window with `after`/`before` and keeps the highest-scored sampled posts, restoring real engagement numbers (a 30-day window on the test topic went from 6pts to ~3,000pts). Slice requests rotate across subreddits so a slow first subreddit cannot consume the entire discovery budget, and successful partial results survive failed slices or an expired deadline.
diff --git a/skills/last30days/scripts/lib/reddit_arctic.py b/skills/last30days/scripts/lib/reddit_arctic.py
index 84837e30..f16d7fc6 100644
--- a/skills/last30days/scripts/lib/reddit_arctic.py
+++ b/skills/last30days/scripts/lib/reddit_arctic.py
@@ -30,13 +30,12 @@ PACE_SECONDS = 0.4 # gap between batches; arctic-shift answers 422 "slow down"
CACHE_MAX = 4096 # hard size bound so the in-run memo can never grow unbounded
# Listing-lane knobs. Base limits mirror reddit_listing's DEPTH_LIMITS so callers
# get the same per-depth volume. The supplement multiplier is applied when the
-# caller requested multiple sorts (top/hot/new) — arctic-shift has no sort lanes,
-# so we fetch more posts to increase the chance of covering what the failed
-# shreddit lanes would have returned.
+# caller requested multiple sorts (top/hot/new) — arctic-shift has no sort lanes
+# (only asc/desc), so we fetch more posts to increase the chance of covering
+# what the failed shreddit lanes would have returned. Windowed runs
+# (from_date/to_date) instead slice the window and keep the highest-scored
+# posts, which recovers window-top engagement without sort lanes.
#
-# KNOWN LIMITATION: Arctic-shift is recency-only (sort=desc). It has no top/hot/
-# new/rising lanes — failed shreddit sort lanes are supplemented with recent
-# posts, not lane-specific results. This is a fundamental backend constraint.
_LISTING_DEPTH_LIMITS = {"quick": 10, "default": 25, "deep": 50}
_LISTING_SUPPLEMENT_MULTIPLIER = 2 # fetch 2x posts when supplementing multi-sort requests
# Total deadline for listing fetches to prevent unbounded stalls when many
@@ -165,6 +164,8 @@ def fetch_listings(
sorts: Optional[List[str]] = None,
timeframe: str = "month",
limit: Optional[int] = None,
+ from_date: Optional[str] = None,
+ to_date: Optional[str] = None,
) -> List[Dict[str, Any]]:
"""Scored subreddit listings from the arctic-shift archive, keyless.
@@ -172,12 +173,15 @@ def fetch_listings(
partials), which datacenter IPs get HTTP 403 on. Arctic-shift serves recent
posts with real score/num_comments from any IP.
- Arctic-shift has no top/hot/new lanes, only recency. When ``sorts`` contains
- multiple entries (e.g., dedicated lanes requesting top+hot+new), we fetch
- more posts per subreddit to partially compensate for the missing lane
- coverage — the caller's engagement ranking does the final sorting.
-
- Best-effort, never raises: returns ``[]`` on any failure.
+ Arctic-shift has no top/hot/new lanes (only asc/desc). Without a date
+ window the fetch is recency-only, and the archive freezes score at ingest,
+ so the freshest ~2-3 days read as 1-point stubs. When the caller supplies
+ ``from_date``/``to_date``, the window is sliced into ~7-day chunks queried
+ with ``after``/``before`` and the highest-scored sampled posts are kept per
+ subreddit. Slice requests rotate across subreddits under a shared deadline;
+ successful partial results survive failures. The 100-row cap per slice
+ means the sample can miss popular posts in busy subreddits. Best-effort,
+ never raises: returns ``[]`` when no requests succeed.
"""
if not subreddits:
return []
@@ -185,30 +189,69 @@ def fetch_listings(
# When multiple sorts were requested, fetch more posts to compensate for
# arctic-shift's lack of sort lanes.
n = base * _LISTING_SUPPLEMENT_MULTIPLIER if sorts and len(sorts) > 1 else base
- out: List[Dict[str, Any]] = []
- # Process all requested subreddits with pacing and a total deadline to
- # prevent unbounded stalls when arctic is slow or unreachable.
- deadline = time.time() + _LISTING_DEADLINE_SECONDS
- fetched_count = 0
+ # Window + engagement sort: the archive freezes score at ingest time, so
+ # posts from the last ~2-3 days read as 1-point stubs and `before` ending
+ # today can return null data. Scope the query with after/before, slice the
+ # window into ~7-day chunks (API caps 100 rows/request, newest-first, so
+ # one request only covers the last day of a month window), and keep the
+ # highest-scored n per subreddit (sort has no top lane — only asc/desc).
+ window = bool(from_date or to_date)
+ slices: Optional[List[tuple]] = None
+ if window:
+ from datetime import date as _date, timedelta as _timedelta
+ try:
+ start = _date.fromisoformat(from_date) if from_date else None
+ end = _date.fromisoformat(to_date) if to_date else _date.today()
+ except ValueError:
+ start, end = None, None
+ if end and not start:
+ start = end - _timedelta(days=30)
+ if start and end and start < end:
+ span = (end - start).days
+ n_slices = min(4, max(1, span // 7))
+ step = max(1, -(-span // n_slices))
+ slices = []
+ cursor = start
+ while cursor < end:
+ slices.append((cursor, min(cursor + _timedelta(days=step), end)))
+ cursor += _timedelta(days=step)
+ req_limit = 100 if window else n
+ posts_by_sub: Dict[str, List[Dict[str, Any]]] = {}
for sub in subreddits:
- if time.time() >= deadline:
- _log(f"listing deadline reached after {fetched_count} subs; skipping remaining")
- break
sub = sub.removeprefix("r/").strip()
- if not sub or sub.lower() == "all":
- continue
+ if sub and sub.lower() != "all":
+ posts_by_sub.setdefault(sub, [])
+ seen_by_sub = {sub: set() for sub in posts_by_sub}
+ # Give every subreddit a slice before spending the shared budget on more
+ # slices for any one subreddit.
+ requests = (
+ (sub, span_pair)
+ for span_pair in (slices or [None])
+ for sub in posts_by_sub
+ )
+ deadline = time.monotonic() + _LISTING_DEADLINE_SECONDS
+ for fetched_count, (sub, span_pair) in enumerate(requests):
if fetched_count:
- time.sleep(PACE_SECONDS)
- fetched_count += 1
+ time.sleep(min(PACE_SECONDS, max(0, deadline - time.monotonic())))
+ now = time.monotonic()
+ remaining = deadline - now
+ if remaining <= 0:
+ _log(f"listing deadline reached after {fetched_count} requests; keeping partial results")
+ break
+ if span_pair:
+ win = f"&after={span_pair[0].isoformat()}&before={span_pair[1].isoformat()}"
+ else:
+ win = f"&after={from_date}" if from_date else ""
+ win += f"&before={to_date}" if to_date else ""
try:
- # Use retries=1 (single attempt) so retries don't exceed our deadline.
- # The deadline handles overall timing; per-request retries would
- # multiply the delay unpredictably.
+ # DNS recovery may retry even with retries=1. Bound the whole
+ # request so those retries cannot consume other subreddits' time.
data = http.get(
- f"{SEARCH_API}?subreddit={sub}&limit={n}&sort=desc",
+ f"{SEARCH_API}?subreddit={sub}&limit={req_limit}&sort=desc{win}",
headers={"User-Agent": http.BROWSER_USER_AGENT},
- timeout=TIMEOUT,
+ timeout=min(TIMEOUT, remaining),
retries=1,
+ deadline_monotonic=min(deadline, now + TIMEOUT),
)
except Exception as e: # network error / non-200 — degrade, never raise
_log(f"listing search failed r/{sub}: {e}")
@@ -221,8 +264,16 @@ def fetch_listings(
if not isinstance(row, dict):
continue
post = _normalize_listing_row(row, query)
- if post["url"]:
- out.append(post)
+ if post["url"] and post["url"] not in seen_by_sub[sub]:
+ seen_by_sub[sub].add(post["url"])
+ posts_by_sub[sub].append(post)
+
+ out: List[Dict[str, Any]] = []
+ for sub_posts in posts_by_sub.values():
+ if window:
+ sub_posts.sort(key=lambda p: -(p["score"] or 0))
+ sub_posts = sub_posts[:n]
+ out.extend(sub_posts)
seen: set = set()
unique: List[Dict[str, Any]] = []
diff --git a/skills/last30days/scripts/lib/reddit_keyless.py b/skills/last30days/scripts/lib/reddit_keyless.py
index 7f0a5f9d..a8610326 100644
--- a/skills/last30days/scripts/lib/reddit_keyless.py
+++ b/skills/last30days/scripts/lib/reddit_keyless.py
@@ -70,6 +70,8 @@ def _scored_listings(
depth: str = "default",
query: str = "",
sorts: Optional[List[str]] = None,
+ from_date: Optional[str] = None,
+ to_date: Optional[str] = None,
) -> List[Dict[str, Any]]:
"""Scored subreddit listings: shreddit partials, arctic-shift supplement.
@@ -91,7 +93,8 @@ def _scored_listings(
if subreddits:
try:
arctic_posts = reddit_arctic.fetch_listings(
- subreddits, depth=depth, query=query, sorts=sorts
+ subreddits, depth=depth, query=query, sorts=sorts,
+ from_date=from_date, to_date=to_date,
)
except Exception as exc: # the fallback must never break the pipeline
_log(f"arctic-shift listing supplement failed: {exc}")
@@ -124,7 +127,8 @@ def _discover(
dedicated_posts: List[Dict[str, Any]] = []
if dedicated_subreddits:
dedicated_posts = _scored_listings(
- dedicated_subreddits, depth=depth, query=topic, sorts=DEDICATED_SORTS
+ dedicated_subreddits, depth=depth, query=topic, sorts=DEDICATED_SORTS,
+ from_date=from_date, to_date=to_date,
)
for p in dedicated_posts:
p["dedicated"] = True
@@ -141,7 +145,11 @@ def _discover(
# are on-topic; include them as scored discovery. A bare run fetches no
# listings, so high-upvote off-topic posts cannot flood the results.
listing_posts = (
- _scored_listings(subreddits, depth=depth, query=topic) if subreddits else []
+ _scored_listings(
+ subreddits, depth=depth, query=topic, from_date=from_date, to_date=to_date
+ )
+ if subreddits
+ else []
)
_log(
f"Tier 1 (site search) {len(search_posts)} posts; "
diff --git a/tests/test_reddit_arctic.py b/tests/test_reddit_arctic.py
index 78a52c70..d57d0904 100644
--- a/tests/test_reddit_arctic.py
+++ b/tests/test_reddit_arctic.py
@@ -1,10 +1,13 @@
"""Tests for reddit_arctic — keyless post-score lookup via arctic-shift (U6)."""
+import socket
from unittest import mock
+from urllib.error import URLError
+from urllib.parse import parse_qs, urlsplit
import pytest
-from lib import reddit_arctic
+from lib import reddit_arctic, reddit_keyless
@pytest.fixture(autouse=True)
@@ -70,6 +73,19 @@ def _listing_row(pid="abc123", title="matcha farm tour", score=406, ncmt=88,
}
+def listing_clock(monkeypatch):
+ now = [0.0]
+
+ def advance(seconds):
+ now[0] += seconds
+
+ monkeypatch.setattr(reddit_arctic.time, "time", lambda: now[0])
+ monkeypatch.setattr(reddit_arctic.time, "monotonic", lambda: now[0])
+ monkeypatch.setattr(reddit_arctic.time, "sleep", advance)
+ return advance
+
+
class TestFetchListings:
"""fetch_listings serves scored subreddit listings from the archive —
the keyless fallback for hosts where shreddit partials 403."""
@@ -92,8 +108,10 @@ class TestFetchListings:
assert post["url"].startswith("https://www.reddit.com/r/tea/comments/")
assert post["why_relevant"] == "Reddit listing (arctic-shift)"
# one call per subreddit, recent-first, depth-default volume
- assert "subreddit=tea" in g.call_args[0][0]
- assert "limit=25" in g.call_args[0][0]
+ assert g.call_count == 1
+ assert parse_qs(urlsplit(g.call_args[0][0]).query) == {
+ "subreddit": ["tea"], "limit": ["25"], "sort": ["desc"],
+ }
def test_strips_r_prefix_and_skips_all(self):
with mock.patch.object(reddit_arctic.http, "get",
@@ -183,3 +201,182 @@ class TestFetchListings:
with mock.patch.object(reddit_arctic.http, "get", return_value=_resp(rows)):
out = reddit_arctic.fetch_scores(["ok", "bad"])
assert out == {"ok": {"score": 7, "num_comments": 3}}
+
+
+class TestWindowedFetchListings:
+ def test_dns_retries_cannot_spend_other_subreddits_request_time(self, listing_clock):
+ requested = []
+
+ def open_request(request, timeout):
+ requested.append(parse_qs(urlsplit(request.full_url).query)["subreddit"][0])
+ listing_clock(timeout)
+ raise URLError(socket.gaierror(socket.EAI_AGAIN, "temporary DNS failure"))
+
+ with mock.patch.object(reddit_arctic.http, "_open_request", side_effect=open_request):
+ out = reddit_arctic.fetch_listings(
+ ["tea", "coffee", "cocoa"], from_date="2026-09-20", to_date="2026-10-20",
+ )
+
+ assert out == []
+ assert requested == ["tea", "coffee", "cocoa"]
+ assert reddit_arctic.time.monotonic() <= reddit_arctic._LISTING_DEADLINE_SECONDS
+
+ def test_slow_first_subreddit_does_not_starve_other_subreddits(self, listing_clock):
+ requested = []
+
+ def get(url, **kwargs):
+ sub = parse_qs(urlsplit(url).query)["subreddit"][0]
+ requested.append(sub)
+ listing_clock(kwargs["timeout"])
+ if sub == "tea":
+ raise TimeoutError("slow subreddit")
+ return _resp([_listing_row(
+ pid=sub, subreddit=sub, permalink=f"/r/{sub}/comments/{sub}/x/",
+ )])
+
+ with mock.patch.object(reddit_arctic.http, "get", side_effect=get):
+ out = reddit_arctic.fetch_listings(
+ ["tea", "coffee", "cocoa"], from_date="2026-09-20", to_date="2026-10-20",
+ )
+
+ assert requested == ["tea", "coffee", "cocoa"]
+ assert [post["subreddit"] for post in out] == ["coffee", "cocoa"]
+ assert reddit_arctic.time.time() <= reddit_arctic._LISTING_DEADLINE_SECONDS
+
+ @pytest.mark.parametrize(("start", "end", "boundaries"), [
+ ("2026-09-20", "2026-10-20", [
+ ("2026-09-20", "2026-09-28"),
+ ("2026-09-28", "2026-10-06"),
+ ("2026-10-06", "2026-10-14"),
+ ("2026-10-14", "2026-10-20"),
+ ]),
+ ("2026-10-19", "2026-10-20", [("2026-10-19", "2026-10-20")]),
+ ])
+ def test_slices_cover_requested_boundaries(self, listing_clock, start, end, boundaries):
+ with mock.patch.object(reddit_arctic.http, "get", return_value=_resp([])) as get:
+ reddit_arctic.fetch_listings(["tea"], from_date=start, to_date=end)
+
+ queries = [parse_qs(urlsplit(call.args[0]).query) for call in get.call_args_list]
+ assert [(query["after"][0], query["before"][0]) for query in queries] == boundaries
+ assert all(query["limit"] == ["100"] and query["sort"] == ["desc"] for query in queries)
+
+ def test_visits_every_subreddit_before_the_next_slice(self, listing_clock):
+ with mock.patch.object(reddit_arctic.http, "get", return_value=_resp([])) as get:
+ reddit_arctic.fetch_listings(
+ ["tea", "coffee"], from_date="2026-09-20", to_date="2026-10-20",
+ )
+
+ queries = [parse_qs(urlsplit(call.args[0]).query) for call in get.call_args_list]
+ assert [query["subreddit"][0] for query in queries] == ["tea", "coffee"] * 4
+ assert [query["after"][0] for query in queries] == [
+ "2026-09-20", "2026-09-20", "2026-09-28", "2026-09-28",
+ "2026-10-06", "2026-10-06", "2026-10-14", "2026-10-14",
+ ]
+
+ def test_keeps_highest_scores_across_slices_per_subreddit(self, listing_clock):
+ scores = {"tea": iter([500, 1, 300, 2]), "coffee": iter([4, 70, 10, 60])}
+
+ def get(url, **kwargs):
+ sub = parse_qs(urlsplit(url).query)["subreddit"][0]
+ score = next(scores[sub])
+ return _resp([_listing_row(
+ pid=f"{sub}{score}", subreddit=sub, score=str(score),
+ permalink=f"/r/{sub}/comments/{sub}{score}/x/",
+ )])
+
+ with mock.patch.object(reddit_arctic.http, "get", side_effect=get):
+ out = reddit_arctic.fetch_listings(
+ ["tea", "coffee"], limit=2, from_date="2026-09-20", to_date="2026-10-20",
+ )
+
+ assert [(post["subreddit"], post["score"]) for post in out] == [
+ ("tea", 500), ("tea", 300), ("coffee", 70), ("coffee", 60),
+ ]
+
+ def test_duplicate_slice_rows_do_not_consume_selection_slots(self, listing_clock):
+ best = _listing_row(score=100)
+ second = _listing_row(pid="second", score=90, permalink="/r/tea/comments/second/x/")
+ with mock.patch.object(reddit_arctic.http, "get", side_effect=[
+ _resp([best]), _resp([best, second]), _resp([]), _resp([]),
+ ]):
+ out = reddit_arctic.fetch_listings(
+ ["tea"], limit=2, from_date="2026-09-20", to_date="2026-10-20",
+ )
+
+ assert [post["metadata"]["post_id"] for post in out] == ["abc123", "second"]
+
+ @pytest.mark.parametrize("failure", [TimeoutError("unreachable"), {"data": None}])
+ def test_failed_slice_keeps_other_slices(self, listing_clock, failure):
+ later = _listing_row(pid="later", score=900, permalink="/r/tea/comments/later/x/")
+ with mock.patch.object(reddit_arctic.http, "get", side_effect=[
+ _resp([_listing_row()]), failure, _resp([later]), _resp([]),
+ ]) as get:
+ out = reddit_arctic.fetch_listings(
+ ["tea"], from_date="2026-09-20", to_date="2026-10-20",
+ )
+
+ assert get.call_count == 4
+ assert [post["metadata"]["post_id"] for post in out] == ["later", "abc123"]
+
+ def test_deadline_keeps_best_posts_from_completed_slices(self, listing_clock):
+ scores = iter([10, 90, 70, 1000])
+ timeouts = []
+
+ def get(url, **kwargs):
+ timeouts.append(kwargs["timeout"])
+ listing_clock(kwargs["timeout"])
+ score = next(scores)
+ return _resp([_listing_row(
+ pid=str(score), score=score, permalink=f"/r/tea/comments/{score}/x/",
+ )])
+
+ with mock.patch.object(reddit_arctic.http, "get", side_effect=get):
+ out = reddit_arctic.fetch_listings(
+ ["tea"], limit=2, from_date="2026-09-20", to_date="2026-10-20",
+ )
+
+ assert [post["score"] for post in out] == [90, 70]
+ assert timeouts == pytest.approx([15, 15, 14.2])
+ assert reddit_arctic.time.time() <= reddit_arctic._LISTING_DEADLINE_SECONDS
+
+ def test_pacing_does_not_start_request_after_deadline(self, listing_clock):
+ def get(url, **kwargs):
+ listing_clock(0.8)
+ return _resp([_listing_row()])
+
+ with mock.patch.object(reddit_arctic, "_LISTING_DEADLINE_SECONDS", 1), \
+ mock.patch.object(reddit_arctic.http, "get", side_effect=get) as request:
+ out = reddit_arctic.fetch_listings(
+ ["tea", "coffee"], from_date="2026-09-20", to_date="2026-10-20",
+ )
+
+ assert request.call_count == 1
+ assert [post["metadata"]["post_id"] for post in out] == ["abc123"]
+ assert reddit_arctic.time.time() == pytest.approx(1)
+
+ def test_discovery_forwards_dates_to_dedicated_and_category_listings(self, listing_clock):
+ queries = []
+
+ def get(url, **kwargs):
+ query = parse_qs(urlsplit(url).query)
+ queries.append(query)
+ sub = query["subreddit"][0]
+ return _resp([_listing_row(
+ pid=sub, subreddit=sub, permalink=f"/r/{sub}/comments/{sub}/x/",
+ )])
+
+ with mock.patch.object(reddit_keyless.reddit_search, "search", return_value=[]), \
+ mock.patch.object(reddit_keyless.reddit_listing, "fetch_listings", return_value=[]), \
+ mock.patch.object(reddit_arctic.http, "get", side_effect=get):
+ out = reddit_keyless._discover(
+ "matcha", "quick", ["tea"], dedicated_subreddits=["matcha"],
+ from_date="2026-09-20", to_date="2026-10-20",
+ )
+
+ assert [post["subreddit"] for post in out] == ["matcha", "tea"]
+ assert out[0]["dedicated"] is True
+ for sub in ("matcha", "tea"):
+ sub_queries = [query for query in queries if query["subreddit"] == [sub]]
+ assert len(sub_queries) == 4
+ assert sub_queries[0]["after"] == ["2026-09-20"]
+ assert sub_queries[-1]["before"] == ["2026-10-20"]
diff --git a/tests/test_reddit_keyless.py b/tests/test_reddit_keyless.py
index 1b6e8068..53e59aa5 100644
--- a/tests/test_reddit_keyless.py
+++ b/tests/test_reddit_keyless.py
@@ -432,7 +432,10 @@ class TestScoredListingsFallback:
return_value=[arctic_post]) as arctic:
out = reddit_keyless._scored_listings(["tea"], depth="quick", query="matcha")
assert out == [arctic_post]
- arctic.assert_called_once_with(["tea"], depth="quick", query="matcha", sorts=None)
+ arctic.assert_called_once_with(
+ ["tea"], depth="quick", query="matcha", sorts=None,
+ from_date=None, to_date=None,
+ )
def test_shreddit_and_arctic_both_called_deduped(self):
"""Shreddit and arctic are both called; arctic supplements missing posts."""
@@ -477,7 +480,8 @@ class TestScoredListingsFallback:
["Kanye"], depth="default", query="Kanye", sorts=["top", "hot", "new"]
)
arctic.assert_called_once_with(
- ["Kanye"], depth="default", query="Kanye", sorts=["top", "hot", "new"]
+ ["Kanye"], depth="default", query="Kanye", sorts=["top", "hot", "new"],
+ from_date=None, to_date=None,
)
def test_arctic_supplements_all_subreddits(self):