Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

feat(reddit): throttle keyless Reddit tiers under fan-out (U4)

Matt Van Hornca3fbbe377bf
diff --git a/skills/last30days/scripts/lib/http.py b/skills/last30days/scripts/lib/http.py
index a8d694f2..2d8f6880 100644
--- a/skills/last30days/scripts/lib/http.py
+++ b/skills/last30days/scripts/lib/http.py
@@ -4,6 +4,7 @@ import json
import re
import socket
import sys
+import threading
import time
import urllib.error
import urllib.request
@@ -270,6 +271,54 @@ def get_text(
return None
+class RateLimiter:
+ """Thread-safe minimum-interval throttle for an endpoint family.
+
+ The keyless source tiers run under the pipeline's ThreadPoolExecutor, so a
+ multi-subquery run can fire many requests at the same host at once. A bare
+ per-request retry budget does not prevent that stampede — it only reacts
+ after a 429. This spaces calls so concurrent futures sharing one limiter
+ cannot burst past ``min_interval`` seconds between requests.
+ """
+
+ def __init__(self, min_interval: float):
+ self.min_interval = min_interval
+ self._last = 0.0
+ self._lock = threading.Lock()
+
+ def acquire(self) -> None:
+ """Block until at least ``min_interval`` has elapsed since the last call."""
+ with self._lock:
+ now = time.monotonic()
+ wait = self.min_interval - (now - self._last)
+ if wait > 0:
+ time.sleep(wait)
+ now = time.monotonic()
+ self._last = now
+
+
+# Shared across all keyless Reddit tiers (RSS, listing, shreddit) so their
+# combined fan-out is throttled as one family, not three independent bursts.
+REDDIT_KEYLESS_LIMITER = RateLimiter(min_interval=0.5)
+
+
+def reddit_keyless_get_text(
+ url: str,
+ timeout: int = DEFAULT_TIMEOUT,
+ retries: int = 2,
+ accept: str = "*/*",
+ headers: Optional[Dict[str, str]] = None,
+) -> Optional[str]:
+ """get_text for the keyless Reddit tiers, throttled by a shared limiter.
+
+ Same contract as :func:`get_text` (returns None on any failure) but spaces
+ requests via :data:`REDDIT_KEYLESS_LIMITER` so a broad multi-query run does
+ not stampede Reddit's keyless endpoints and trip blocks.
+ """
+ REDDIT_KEYLESS_LIMITER.acquire()
+ return get_text(url, timeout=timeout, retries=retries, accept=accept, headers=headers)
+
+
def scrapecreators_headers(token: str) -> Dict[str, str]:
"""Build ScrapeCreators request headers (x-api-key + JSON content type)."""
return {
diff --git a/skills/last30days/scripts/lib/reddit_listing.py b/skills/last30days/scripts/lib/reddit_listing.py
index 7f2fcccd..f3b92d2b 100644
--- a/skills/last30days/scripts/lib/reddit_listing.py
+++ b/skills/last30days/scripts/lib/reddit_listing.py
@@ -128,8 +128,8 @@ def _listing_url(subreddit: str, sort: str) -> str:
def _fetch_one(subreddit: str, sort: str, query: str) -> List[Dict[str, Any]]:
try:
- text = http.get_text(_listing_url(subreddit, sort), timeout=LISTING_TIMEOUT,
- accept="text/html")
+ text = http.reddit_keyless_get_text(_listing_url(subreddit, sort), timeout=LISTING_TIMEOUT,
+ accept="text/html")
return parse_cards(text, query) if text else []
except Exception as e:
_log(f"listing fetch failed r/{subreddit} {sort}: {e}")
diff --git a/skills/last30days/scripts/lib/reddit_rss.py b/skills/last30days/scripts/lib/reddit_rss.py
index b7c3576a..19054389 100644
--- a/skills/last30days/scripts/lib/reddit_rss.py
+++ b/skills/last30days/scripts/lib/reddit_rss.py
@@ -173,7 +173,7 @@ def _build_urls(query: str, depth: str, subreddits: Optional[List[str]]) -> List
def _fetch_feed(url: str, query: str) -> List[Dict[str, Any]]:
"""Fetch and parse one feed. Never raises."""
try:
- text = http.get_text(url, timeout=FEED_TIMEOUT, accept="application/atom+xml")
+ text = http.reddit_keyless_get_text(url, timeout=FEED_TIMEOUT, accept="application/atom+xml")
return _parse_feed(text, query) if text else []
except Exception as e: # defensive: a single bad feed must not sink the run
_log(f"feed fetch failed for {url}: {e}")
diff --git a/skills/last30days/scripts/lib/reddit_shreddit.py b/skills/last30days/scripts/lib/reddit_shreddit.py
index d222ac09..b0546397 100644
--- a/skills/last30days/scripts/lib/reddit_shreddit.py
+++ b/skills/last30days/scripts/lib/reddit_shreddit.py
@@ -162,7 +162,7 @@ def fetch_comments(
return {"top_comments": [], "comment_insights": [], "num_comments": None}
sub, post_id = ref
- html_text = http.get_text(_svc_url(sub, post_id), timeout=timeout, accept="text/html")
+ html_text = http.reddit_keyless_get_text(_svc_url(sub, post_id), timeout=timeout, accept="text/html")
if not html_text:
return {"top_comments": [], "comment_insights": [], "num_comments": None}
diff --git a/tests/test_reddit_keyless_backoff.py b/tests/test_reddit_keyless_backoff.py
new file mode 100644
index 00000000..f2af0da7
--- /dev/null
+++ b/tests/test_reddit_keyless_backoff.py
@@ -0,0 +1,51 @@
+"""Tests for the shared keyless-Reddit throttle (U4)."""
+
+from unittest import mock
+
+from lib import http, reddit_rss
+
+
+class TestRateLimiter:
+ def test_first_acquire_does_not_sleep(self):
+ limiter = http.RateLimiter(min_interval=0.5)
+ with mock.patch.object(http.time, "monotonic", return_value=100.0), \
+ mock.patch.object(http.time, "sleep") as slept:
+ limiter.acquire()
+ slept.assert_not_called()
+
+ def test_second_acquire_sleeps_to_maintain_interval(self):
+ limiter = http.RateLimiter(min_interval=0.5)
+ # First call at t=100, second call 0.1s later -> must wait ~0.4s.
+ times = iter([100.0, 100.1, 100.5])
+ with mock.patch.object(http.time, "monotonic", side_effect=lambda: next(times)), \
+ mock.patch.object(http.time, "sleep") as slept:
+ limiter.acquire()
+ limiter.acquire()
+ slept.assert_called_once()
+ waited = slept.call_args.args[0]
+ assert abs(waited - 0.4) < 1e-6
+
+ def test_no_sleep_when_interval_already_elapsed(self):
+ limiter = http.RateLimiter(min_interval=0.5)
+ times = iter([100.0, 102.0]) # 2s gap > interval
+ with mock.patch.object(http.time, "monotonic", side_effect=lambda: next(times)), \
+ mock.patch.object(http.time, "sleep") as slept:
+ limiter.acquire()
+ limiter.acquire()
+ slept.assert_not_called()
+
+
+class TestRedditKeylessGetText:
+ def test_acquires_limiter_then_delegates(self):
+ with mock.patch.object(http.REDDIT_KEYLESS_LIMITER, "acquire") as acq, \
+ mock.patch.object(http, "get_text", return_value="body") as gt:
+ out = http.reddit_keyless_get_text("https://www.reddit.com/x.rss", accept="application/atom+xml")
+ assert out == "body"
+ acq.assert_called_once()
+ gt.assert_called_once()
+
+ def test_reddit_rss_routes_through_throttle(self):
+ # The RSS tier must use the throttled helper, not raw get_text.
+ with mock.patch.object(reddit_rss.http, "reddit_keyless_get_text", return_value=None) as throttled:
+ reddit_rss.search_rss("test query")
+ assert throttled.called