AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
fix: sample Arctic listings across subreddits within request deadlines (#1306)
yachunchen638da9716b0e
diff --git a/changelog.d/+arctic-windowed-scores.fixed.md b/changelog.d/+arctic-windowed-scores.fixed.mdnew file mode 100644index 00000000..18051219--- /dev/null+++ b/changelog.d/+arctic-windowed-scores.fixed.md@@ -0,0 +1 @@+Keyless Reddit discovery no longer collapses to 1-point stubs when shreddit is blocked: the arctic-shift supplement now slices the research window with `after`/`before` and keeps the highest-scored sampled posts, restoring real engagement numbers (a 30-day window on the test topic went from 6pts to ~3,000pts). Slice requests rotate across subreddits so a slow first subreddit cannot consume the entire discovery budget, and successful partial results survive failed slices or an expired deadline.diff --git a/skills/last30days/scripts/lib/reddit_arctic.py b/skills/last30days/scripts/lib/reddit_arctic.pyindex 84837e30..f16d7fc6 100644--- a/skills/last30days/scripts/lib/reddit_arctic.py+++ b/skills/last30days/scripts/lib/reddit_arctic.py@@ -30,13 +30,12 @@ PACE_SECONDS = 0.4 # gap between batches; arctic-shift answers 422 "slow down"CACHE_MAX = 4096 # hard size bound so the in-run memo can never grow unbounded# Listing-lane knobs. Base limits mirror reddit_listing's DEPTH_LIMITS so callers# get the same per-depth volume. The supplement multiplier is applied when the-# caller requested multiple sorts (top/hot/new) — arctic-shift has no sort lanes,-# so we fetch more posts to increase the chance of covering what the failed-# shreddit lanes would have returned.+# caller requested multiple sorts (top/hot/new) — arctic-shift has no sort lanes+# (only asc/desc), so we fetch more posts to increase the chance of covering+# what the failed shreddit lanes would have returned. Windowed runs+# (from_date/to_date) instead slice the window and keep the highest-scored+# posts, which recovers window-top engagement without sort lanes.#-# KNOWN LIMITATION: Arctic-shift is recency-only (sort=desc). It has no top/hot/-# new/rising lanes — failed shreddit sort lanes are supplemented with recent-# posts, not lane-specific results. This is a fundamental backend constraint._LISTING_DEPTH_LIMITS = {"quick": 10, "default": 25, "deep": 50}_LISTING_SUPPLEMENT_MULTIPLIER = 2 # fetch 2x posts when supplementing multi-sort requests# Total deadline for listing fetches to prevent unbounded stalls when many@@ -165,6 +164,8 @@ def fetch_listings(sorts: Optional[List[str]] = None,timeframe: str = "month",limit: Optional[int] = None,+ from_date: Optional[str] = None,+ to_date: Optional[str] = None,) -> List[Dict[str, Any]]:"""Scored subreddit listings from the arctic-shift archive, keyless.@@ -172,12 +173,15 @@ def fetch_listings(partials), which datacenter IPs get HTTP 403 on. Arctic-shift serves recentposts with real score/num_comments from any IP.- Arctic-shift has no top/hot/new lanes, only recency. When ``sorts`` contains- multiple entries (e.g., dedicated lanes requesting top+hot+new), we fetch- more posts per subreddit to partially compensate for the missing lane- coverage — the caller's engagement ranking does the final sorting.-- Best-effort, never raises: returns ``[]`` on any failure.+ Arctic-shift has no top/hot/new lanes (only asc/desc). Without a date+ window the fetch is recency-only, and the archive freezes score at ingest,+ so the freshest ~2-3 days read as 1-point stubs. When the caller supplies+ ``from_date``/``to_date``, the window is sliced into ~7-day chunks queried+ with ``after``/``before`` and the highest-scored sampled posts are kept per+ subreddit. Slice requests rotate across subreddits under a shared deadline;+ successful partial results survive failures. The 100-row cap per slice+ means the sample can miss popular posts in busy subreddits. Best-effort,+ never raises: returns ``[]`` when no requests succeed."""if not subreddits:return []@@ -185,30 +189,69 @@ def fetch_listings(# When multiple sorts were requested, fetch more posts to compensate for# arctic-shift's lack of sort lanes.n = base * _LISTING_SUPPLEMENT_MULTIPLIER if sorts and len(sorts) > 1 else base- out: List[Dict[str, Any]] = []- # Process all requested subreddits with pacing and a total deadline to- # prevent unbounded stalls when arctic is slow or unreachable.- deadline = time.time() + _LISTING_DEADLINE_SECONDS- fetched_count = 0+ # Window + engagement sort: the archive freezes score at ingest time, so+ # posts from the last ~2-3 days read as 1-point stubs and `before` ending+ # today can return null data. Scope the query with after/before, slice the+ # window into ~7-day chunks (API caps 100 rows/request, newest-first, so+ # one request only covers the last day of a month window), and keep the+ # highest-scored n per subreddit (sort has no top lane — only asc/desc).+ window = bool(from_date or to_date)+ slices: Optional[List[tuple]] = None+ if window:+ from datetime import date as _date, timedelta as _timedelta+ try:+ start = _date.fromisoformat(from_date) if from_date else None+ end = _date.fromisoformat(to_date) if to_date else _date.today()+ except ValueError:+ start, end = None, None+ if end and not start:+ start = end - _timedelta(days=30)+ if start and end and start < end:+ span = (end - start).days+ n_slices = min(4, max(1, span // 7))+ step = max(1, -(-span // n_slices))+ slices = []+ cursor = start+ while cursor < end:+ slices.append((cursor, min(cursor + _timedelta(days=step), end)))+ cursor += _timedelta(days=step)+ req_limit = 100 if window else n+ posts_by_sub: Dict[str, List[Dict[str, Any]]] = {}for sub in subreddits:- if time.time() >= deadline:- _log(f"listing deadline reached after {fetched_count} subs; skipping remaining")- breaksub = sub.removeprefix("r/").strip()- if not sub or sub.lower() == "all":- continue+ if sub and sub.lower() != "all":+ posts_by_sub.setdefault(sub, [])+ seen_by_sub = {sub: set() for sub in posts_by_sub}+ # Give every subreddit a slice before spending the shared budget on more+ # slices for any one subreddit.+ requests = (+ (sub, span_pair)+ for span_pair in (slices or [None])+ for sub in posts_by_sub+ )+ deadline = time.monotonic() + _LISTING_DEADLINE_SECONDS+ for fetched_count, (sub, span_pair) in enumerate(requests):if fetched_count:- time.sleep(PACE_SECONDS)- fetched_count += 1+ time.sleep(min(PACE_SECONDS, max(0, deadline - time.monotonic())))+ now = time.monotonic()+ remaining = deadline - now+ if remaining <= 0:+ _log(f"listing deadline reached after {fetched_count} requests; keeping partial results")+ break+ if span_pair:+ win = f"&after={span_pair[0].isoformat()}&before={span_pair[1].isoformat()}"+ else:+ win = f"&after={from_date}" if from_date else ""+ win += f"&before={to_date}" if to_date else ""try:- # Use retries=1 (single attempt) so retries don't exceed our deadline.- # The deadline handles overall timing; per-request retries would- # multiply the delay unpredictably.+ # DNS recovery may retry even with retries=1. Bound the whole+ # request so those retries cannot consume other subreddits' time.data = http.get(- f"{SEARCH_API}?subreddit={sub}&limit={n}&sort=desc",+ f"{SEARCH_API}?subreddit={sub}&limit={req_limit}&sort=desc{win}",headers={"User-Agent": http.BROWSER_USER_AGENT},- timeout=TIMEOUT,+ timeout=min(TIMEOUT, remaining),retries=1,+ deadline_monotonic=min(deadline, now + TIMEOUT),)except Exception as e: # network error / non-200 — degrade, never raise_log(f"listing search failed r/{sub}: {e}")@@ -221,8 +264,16 @@ def fetch_listings(if not isinstance(row, dict):continuepost = _normalize_listing_row(row, query)- if post["url"]:- out.append(post)+ if post["url"] and post["url"] not in seen_by_sub[sub]:+ seen_by_sub[sub].add(post["url"])+ posts_by_sub[sub].append(post)++ out: List[Dict[str, Any]] = []+ for sub_posts in posts_by_sub.values():+ if window:+ sub_posts.sort(key=lambda p: -(p["score"] or 0))+ sub_posts = sub_posts[:n]+ out.extend(sub_posts)seen: set = set()unique: List[Dict[str, Any]] = []diff --git a/skills/last30days/scripts/lib/reddit_keyless.py b/skills/last30days/scripts/lib/reddit_keyless.pyindex 7f0a5f9d..a8610326 100644--- a/skills/last30days/scripts/lib/reddit_keyless.py+++ b/skills/last30days/scripts/lib/reddit_keyless.py@@ -70,6 +70,8 @@ def _scored_listings(depth: str = "default",query: str = "",sorts: Optional[List[str]] = None,+ from_date: Optional[str] = None,+ to_date: Optional[str] = None,) -> List[Dict[str, Any]]:"""Scored subreddit listings: shreddit partials, arctic-shift supplement.@@ -91,7 +93,8 @@ def _scored_listings(if subreddits:try:arctic_posts = reddit_arctic.fetch_listings(- subreddits, depth=depth, query=query, sorts=sorts+ subreddits, depth=depth, query=query, sorts=sorts,+ from_date=from_date, to_date=to_date,)except Exception as exc: # the fallback must never break the pipeline_log(f"arctic-shift listing supplement failed: {exc}")@@ -124,7 +127,8 @@ def _discover(dedicated_posts: List[Dict[str, Any]] = []if dedicated_subreddits:dedicated_posts = _scored_listings(- dedicated_subreddits, depth=depth, query=topic, sorts=DEDICATED_SORTS+ dedicated_subreddits, depth=depth, query=topic, sorts=DEDICATED_SORTS,+ from_date=from_date, to_date=to_date,)for p in dedicated_posts:p["dedicated"] = True@@ -141,7 +145,11 @@ def _discover(# are on-topic; include them as scored discovery. A bare run fetches no# listings, so high-upvote off-topic posts cannot flood the results.listing_posts = (- _scored_listings(subreddits, depth=depth, query=topic) if subreddits else []+ _scored_listings(+ subreddits, depth=depth, query=topic, from_date=from_date, to_date=to_date+ )+ if subreddits+ else [])_log(f"Tier 1 (site search) {len(search_posts)} posts; "diff --git a/tests/test_reddit_arctic.py b/tests/test_reddit_arctic.pyindex 78a52c70..d57d0904 100644--- a/tests/test_reddit_arctic.py+++ b/tests/test_reddit_arctic.py@@ -1,10 +1,13 @@"""Tests for reddit_arctic — keyless post-score lookup via arctic-shift (U6)."""+import socketfrom unittest import mock+from urllib.error import URLError+from urllib.parse import parse_qs, urlsplitimport pytest-from lib import reddit_arctic+from lib import reddit_arctic, reddit_keyless@pytest.fixture(autouse=True)@@ -70,6 +73,19 @@ def _listing_row(pid="abc123", title="matcha farm tour", score=406, ncmt=88,}+def listing_clock(monkeypatch):+ now = [0.0]++ def advance(seconds):+ now[0] += seconds++ monkeypatch.setattr(reddit_arctic.time, "time", lambda: now[0])+ monkeypatch.setattr(reddit_arctic.time, "monotonic", lambda: now[0])+ monkeypatch.setattr(reddit_arctic.time, "sleep", advance)+ return advance++class TestFetchListings:"""fetch_listings serves scored subreddit listings from the archive —the keyless fallback for hosts where shreddit partials 403."""@@ -92,8 +108,10 @@ class TestFetchListings:assert post["url"].startswith("https://www.reddit.com/r/tea/comments/")assert post["why_relevant"] == "Reddit listing (arctic-shift)"# one call per subreddit, recent-first, depth-default volume- assert "subreddit=tea" in g.call_args[0][0]- assert "limit=25" in g.call_args[0][0]+ assert g.call_count == 1+ assert parse_qs(urlsplit(g.call_args[0][0]).query) == {+ "subreddit": ["tea"], "limit": ["25"], "sort": ["desc"],+ }def test_strips_r_prefix_and_skips_all(self):with mock.patch.object(reddit_arctic.http, "get",@@ -183,3 +201,182 @@ class TestFetchListings:with mock.patch.object(reddit_arctic.http, "get", return_value=_resp(rows)):out = reddit_arctic.fetch_scores(["ok", "bad"])assert out == {"ok": {"score": 7, "num_comments": 3}}+++class TestWindowedFetchListings:+ def test_dns_retries_cannot_spend_other_subreddits_request_time(self, listing_clock):+ requested = []++ def open_request(request, timeout):+ requested.append(parse_qs(urlsplit(request.full_url).query)["subreddit"][0])+ listing_clock(timeout)+ raise URLError(socket.gaierror(socket.EAI_AGAIN, "temporary DNS failure"))++ with mock.patch.object(reddit_arctic.http, "_open_request", side_effect=open_request):+ out = reddit_arctic.fetch_listings(+ ["tea", "coffee", "cocoa"], from_date="2026-09-20", to_date="2026-10-20",+ )++ assert out == []+ assert requested == ["tea", "coffee", "cocoa"]+ assert reddit_arctic.time.monotonic() <= reddit_arctic._LISTING_DEADLINE_SECONDS++ def test_slow_first_subreddit_does_not_starve_other_subreddits(self, listing_clock):+ requested = []++ def get(url, **kwargs):+ sub = parse_qs(urlsplit(url).query)["subreddit"][0]+ requested.append(sub)+ listing_clock(kwargs["timeout"])+ if sub == "tea":+ raise TimeoutError("slow subreddit")+ return _resp([_listing_row(+ pid=sub, subreddit=sub, permalink=f"/r/{sub}/comments/{sub}/x/",+ )])++ with mock.patch.object(reddit_arctic.http, "get", side_effect=get):+ out = reddit_arctic.fetch_listings(+ ["tea", "coffee", "cocoa"], from_date="2026-09-20", to_date="2026-10-20",+ )++ assert requested == ["tea", "coffee", "cocoa"]+ assert [post["subreddit"] for post in out] == ["coffee", "cocoa"]+ assert reddit_arctic.time.time() <= reddit_arctic._LISTING_DEADLINE_SECONDS++ @pytest.mark.parametrize(("start", "end", "boundaries"), [+ ("2026-09-20", "2026-10-20", [+ ("2026-09-20", "2026-09-28"),+ ("2026-09-28", "2026-10-06"),+ ("2026-10-06", "2026-10-14"),+ ("2026-10-14", "2026-10-20"),+ ]),+ ("2026-10-19", "2026-10-20", [("2026-10-19", "2026-10-20")]),+ ])+ def test_slices_cover_requested_boundaries(self, listing_clock, start, end, boundaries):+ with mock.patch.object(reddit_arctic.http, "get", return_value=_resp([])) as get:+ reddit_arctic.fetch_listings(["tea"], from_date=start, to_date=end)++ queries = [parse_qs(urlsplit(call.args[0]).query) for call in get.call_args_list]+ assert [(query["after"][0], query["before"][0]) for query in queries] == boundaries+ assert all(query["limit"] == ["100"] and query["sort"] == ["desc"] for query in queries)++ def test_visits_every_subreddit_before_the_next_slice(self, listing_clock):+ with mock.patch.object(reddit_arctic.http, "get", return_value=_resp([])) as get:+ reddit_arctic.fetch_listings(+ ["tea", "coffee"], from_date="2026-09-20", to_date="2026-10-20",+ )++ queries = [parse_qs(urlsplit(call.args[0]).query) for call in get.call_args_list]+ assert [query["subreddit"][0] for query in queries] == ["tea", "coffee"] * 4+ assert [query["after"][0] for query in queries] == [+ "2026-09-20", "2026-09-20", "2026-09-28", "2026-09-28",+ "2026-10-06", "2026-10-06", "2026-10-14", "2026-10-14",+ ]++ def test_keeps_highest_scores_across_slices_per_subreddit(self, listing_clock):+ scores = {"tea": iter([500, 1, 300, 2]), "coffee": iter([4, 70, 10, 60])}++ def get(url, **kwargs):+ sub = parse_qs(urlsplit(url).query)["subreddit"][0]+ score = next(scores[sub])+ return _resp([_listing_row(+ pid=f"{sub}{score}", subreddit=sub, score=str(score),+ permalink=f"/r/{sub}/comments/{sub}{score}/x/",+ )])++ with mock.patch.object(reddit_arctic.http, "get", side_effect=get):+ out = reddit_arctic.fetch_listings(+ ["tea", "coffee"], limit=2, from_date="2026-09-20", to_date="2026-10-20",+ )++ assert [(post["subreddit"], post["score"]) for post in out] == [+ ("tea", 500), ("tea", 300), ("coffee", 70), ("coffee", 60),+ ]++ def test_duplicate_slice_rows_do_not_consume_selection_slots(self, listing_clock):+ best = _listing_row(score=100)+ second = _listing_row(pid="second", score=90, permalink="/r/tea/comments/second/x/")+ with mock.patch.object(reddit_arctic.http, "get", side_effect=[+ _resp([best]), _resp([best, second]), _resp([]), _resp([]),+ ]):+ out = reddit_arctic.fetch_listings(+ ["tea"], limit=2, from_date="2026-09-20", to_date="2026-10-20",+ )++ assert [post["metadata"]["post_id"] for post in out] == ["abc123", "second"]++ @pytest.mark.parametrize("failure", [TimeoutError("unreachable"), {"data": None}])+ def test_failed_slice_keeps_other_slices(self, listing_clock, failure):+ later = _listing_row(pid="later", score=900, permalink="/r/tea/comments/later/x/")+ with mock.patch.object(reddit_arctic.http, "get", side_effect=[+ _resp([_listing_row()]), failure, _resp([later]), _resp([]),+ ]) as get:+ out = reddit_arctic.fetch_listings(+ ["tea"], from_date="2026-09-20", to_date="2026-10-20",+ )++ assert get.call_count == 4+ assert [post["metadata"]["post_id"] for post in out] == ["later", "abc123"]++ def test_deadline_keeps_best_posts_from_completed_slices(self, listing_clock):+ scores = iter([10, 90, 70, 1000])+ timeouts = []++ def get(url, **kwargs):+ timeouts.append(kwargs["timeout"])+ listing_clock(kwargs["timeout"])+ score = next(scores)+ return _resp([_listing_row(+ pid=str(score), score=score, permalink=f"/r/tea/comments/{score}/x/",+ )])++ with mock.patch.object(reddit_arctic.http, "get", side_effect=get):+ out = reddit_arctic.fetch_listings(+ ["tea"], limit=2, from_date="2026-09-20", to_date="2026-10-20",+ )++ assert [post["score"] for post in out] == [90, 70]+ assert timeouts == pytest.approx([15, 15, 14.2])+ assert reddit_arctic.time.time() <= reddit_arctic._LISTING_DEADLINE_SECONDS++ def test_pacing_does_not_start_request_after_deadline(self, listing_clock):+ def get(url, **kwargs):+ listing_clock(0.8)+ return _resp([_listing_row()])++ with mock.patch.object(reddit_arctic, "_LISTING_DEADLINE_SECONDS", 1), \+ mock.patch.object(reddit_arctic.http, "get", side_effect=get) as request:+ out = reddit_arctic.fetch_listings(+ ["tea", "coffee"], from_date="2026-09-20", to_date="2026-10-20",+ )++ assert request.call_count == 1+ assert [post["metadata"]["post_id"] for post in out] == ["abc123"]+ assert reddit_arctic.time.time() == pytest.approx(1)++ def test_discovery_forwards_dates_to_dedicated_and_category_listings(self, listing_clock):+ queries = []++ def get(url, **kwargs):+ query = parse_qs(urlsplit(url).query)+ queries.append(query)+ sub = query["subreddit"][0]+ return _resp([_listing_row(+ pid=sub, subreddit=sub, permalink=f"/r/{sub}/comments/{sub}/x/",+ )])++ with mock.patch.object(reddit_keyless.reddit_search, "search", return_value=[]), \+ mock.patch.object(reddit_keyless.reddit_listing, "fetch_listings", return_value=[]), \+ mock.patch.object(reddit_arctic.http, "get", side_effect=get):+ out = reddit_keyless._discover(+ "matcha", "quick", ["tea"], dedicated_subreddits=["matcha"],+ from_date="2026-09-20", to_date="2026-10-20",+ )++ assert [post["subreddit"] for post in out] == ["matcha", "tea"]+ assert out[0]["dedicated"] is True+ for sub in ("matcha", "tea"):+ sub_queries = [query for query in queries if query["subreddit"] == [sub]]+ assert len(sub_queries) == 4+ assert sub_queries[0]["after"] == ["2026-09-20"]+ assert sub_queries[-1]["before"] == ["2026-10-20"]diff --git a/tests/test_reddit_keyless.py b/tests/test_reddit_keyless.pyindex 1b6e8068..53e59aa5 100644--- a/tests/test_reddit_keyless.py+++ b/tests/test_reddit_keyless.py@@ -432,7 +432,10 @@ class TestScoredListingsFallback:return_value=[arctic_post]) as arctic:out = reddit_keyless._scored_listings(["tea"], depth="quick", query="matcha")assert out == [arctic_post]- arctic.assert_called_once_with(["tea"], depth="quick", query="matcha", sorts=None)+ arctic.assert_called_once_with(+ ["tea"], depth="quick", query="matcha", sorts=None,+ from_date=None, to_date=None,+ )def test_shreddit_and_arctic_both_called_deduped(self):"""Shreddit and arctic are both called; arctic supplements missing posts."""@@ -477,7 +480,8 @@ class TestScoredListingsFallback:["Kanye"], depth="default", query="Kanye", sorts=["top", "hot", "new"])arctic.assert_called_once_with(- ["Kanye"], depth="default", query="Kanye", sorts=["top", "hot", "new"]+ ["Kanye"], depth="default", query="Kanye", sorts=["top", "hot", "new"],+ from_date=None, to_date=None,)def test_arctic_supplements_all_subreddits(self):