AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
feat(reddit): throttle keyless Reddit tiers under fan-out (U4)
Matt Van Hornca3fbbe377bf
diff --git a/skills/last30days/scripts/lib/http.py b/skills/last30days/scripts/lib/http.pyindex a8d694f2..2d8f6880 100644--- a/skills/last30days/scripts/lib/http.py+++ b/skills/last30days/scripts/lib/http.py@@ -4,6 +4,7 @@ import jsonimport reimport socketimport sys+import threadingimport timeimport urllib.errorimport urllib.request@@ -270,6 +271,54 @@ def get_text(return None+class RateLimiter:+ """Thread-safe minimum-interval throttle for an endpoint family.++ The keyless source tiers run under the pipeline's ThreadPoolExecutor, so a+ multi-subquery run can fire many requests at the same host at once. A bare+ per-request retry budget does not prevent that stampede — it only reacts+ after a 429. This spaces calls so concurrent futures sharing one limiter+ cannot burst past ``min_interval`` seconds between requests.+ """++ def __init__(self, min_interval: float):+ self.min_interval = min_interval+ self._last = 0.0+ self._lock = threading.Lock()++ def acquire(self) -> None:+ """Block until at least ``min_interval`` has elapsed since the last call."""+ with self._lock:+ now = time.monotonic()+ wait = self.min_interval - (now - self._last)+ if wait > 0:+ time.sleep(wait)+ now = time.monotonic()+ self._last = now+++# Shared across all keyless Reddit tiers (RSS, listing, shreddit) so their+# combined fan-out is throttled as one family, not three independent bursts.+REDDIT_KEYLESS_LIMITER = RateLimiter(min_interval=0.5)+++def reddit_keyless_get_text(+ url: str,+ timeout: int = DEFAULT_TIMEOUT,+ retries: int = 2,+ accept: str = "*/*",+ headers: Optional[Dict[str, str]] = None,+) -> Optional[str]:+ """get_text for the keyless Reddit tiers, throttled by a shared limiter.++ Same contract as :func:`get_text` (returns None on any failure) but spaces+ requests via :data:`REDDIT_KEYLESS_LIMITER` so a broad multi-query run does+ not stampede Reddit's keyless endpoints and trip blocks.+ """+ REDDIT_KEYLESS_LIMITER.acquire()+ return get_text(url, timeout=timeout, retries=retries, accept=accept, headers=headers)++def scrapecreators_headers(token: str) -> Dict[str, str]:"""Build ScrapeCreators request headers (x-api-key + JSON content type)."""return {diff --git a/skills/last30days/scripts/lib/reddit_listing.py b/skills/last30days/scripts/lib/reddit_listing.pyindex 7f2fcccd..f3b92d2b 100644--- a/skills/last30days/scripts/lib/reddit_listing.py+++ b/skills/last30days/scripts/lib/reddit_listing.py@@ -128,8 +128,8 @@ def _listing_url(subreddit: str, sort: str) -> str:def _fetch_one(subreddit: str, sort: str, query: str) -> List[Dict[str, Any]]:try:- text = http.get_text(_listing_url(subreddit, sort), timeout=LISTING_TIMEOUT,- accept="text/html")+ text = http.reddit_keyless_get_text(_listing_url(subreddit, sort), timeout=LISTING_TIMEOUT,+ accept="text/html")return parse_cards(text, query) if text else []except Exception as e:_log(f"listing fetch failed r/{subreddit} {sort}: {e}")diff --git a/skills/last30days/scripts/lib/reddit_rss.py b/skills/last30days/scripts/lib/reddit_rss.pyindex b7c3576a..19054389 100644--- a/skills/last30days/scripts/lib/reddit_rss.py+++ b/skills/last30days/scripts/lib/reddit_rss.py@@ -173,7 +173,7 @@ def _build_urls(query: str, depth: str, subreddits: Optional[List[str]]) -> Listdef _fetch_feed(url: str, query: str) -> List[Dict[str, Any]]:"""Fetch and parse one feed. Never raises."""try:- text = http.get_text(url, timeout=FEED_TIMEOUT, accept="application/atom+xml")+ text = http.reddit_keyless_get_text(url, timeout=FEED_TIMEOUT, accept="application/atom+xml")return _parse_feed(text, query) if text else []except Exception as e: # defensive: a single bad feed must not sink the run_log(f"feed fetch failed for {url}: {e}")diff --git a/skills/last30days/scripts/lib/reddit_shreddit.py b/skills/last30days/scripts/lib/reddit_shreddit.pyindex d222ac09..b0546397 100644--- a/skills/last30days/scripts/lib/reddit_shreddit.py+++ b/skills/last30days/scripts/lib/reddit_shreddit.py@@ -162,7 +162,7 @@ def fetch_comments(return {"top_comments": [], "comment_insights": [], "num_comments": None}sub, post_id = ref- html_text = http.get_text(_svc_url(sub, post_id), timeout=timeout, accept="text/html")+ html_text = http.reddit_keyless_get_text(_svc_url(sub, post_id), timeout=timeout, accept="text/html")if not html_text:return {"top_comments": [], "comment_insights": [], "num_comments": None}diff --git a/tests/test_reddit_keyless_backoff.py b/tests/test_reddit_keyless_backoff.pynew file mode 100644index 00000000..f2af0da7--- /dev/null+++ b/tests/test_reddit_keyless_backoff.py@@ -0,0 +1,51 @@+"""Tests for the shared keyless-Reddit throttle (U4)."""++from unittest import mock++from lib import http, reddit_rss+++class TestRateLimiter:+ def test_first_acquire_does_not_sleep(self):+ limiter = http.RateLimiter(min_interval=0.5)+ with mock.patch.object(http.time, "monotonic", return_value=100.0), \+ mock.patch.object(http.time, "sleep") as slept:+ limiter.acquire()+ slept.assert_not_called()++ def test_second_acquire_sleeps_to_maintain_interval(self):+ limiter = http.RateLimiter(min_interval=0.5)+ # First call at t=100, second call 0.1s later -> must wait ~0.4s.+ times = iter([100.0, 100.1, 100.5])+ with mock.patch.object(http.time, "monotonic", side_effect=lambda: next(times)), \+ mock.patch.object(http.time, "sleep") as slept:+ limiter.acquire()+ limiter.acquire()+ slept.assert_called_once()+ waited = slept.call_args.args[0]+ assert abs(waited - 0.4) < 1e-6++ def test_no_sleep_when_interval_already_elapsed(self):+ limiter = http.RateLimiter(min_interval=0.5)+ times = iter([100.0, 102.0]) # 2s gap > interval+ with mock.patch.object(http.time, "monotonic", side_effect=lambda: next(times)), \+ mock.patch.object(http.time, "sleep") as slept:+ limiter.acquire()+ limiter.acquire()+ slept.assert_not_called()+++class TestRedditKeylessGetText:+ def test_acquires_limiter_then_delegates(self):+ with mock.patch.object(http.REDDIT_KEYLESS_LIMITER, "acquire") as acq, \+ mock.patch.object(http, "get_text", return_value="body") as gt:+ out = http.reddit_keyless_get_text("https://www.reddit.com/x.rss", accept="application/atom+xml")+ assert out == "body"+ acq.assert_called_once()+ gt.assert_called_once()++ def test_reddit_rss_routes_through_throttle(self):+ # The RSS tier must use the throttled helper, not raw get_text.+ with mock.patch.object(reddit_rss.http, "reddit_keyless_get_text", return_value=None) as throttled:+ reddit_rss.search_rss("test query")+ assert throttled.called