AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
perf(pipeline): cap inner ThreadPool to 4 under competitor fanout (#435)
Ilia Alshanetskyccca10bbaf84
diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.pyindex e0732d3d..e709ae4f 100644--- a/skills/last30days/scripts/last30days.py+++ b/skills/last30days/scripts/last30days.py@@ -820,6 +820,7 @@ def main() -> int:lookback_days=args.lookback_days,github_user=github_user,github_repos=github_repos,+ internal_subrun=comp_enabled,hiring_signals_mode=args.hiring_signals,)r.artifacts["resolved"] = {diff --git a/skills/last30days/scripts/lib/pipeline.py b/skills/last30days/scripts/lib/pipeline.pyindex a20570dc..93d5c3be 100644--- a/skills/last30days/scripts/lib/pipeline.py+++ b/skills/last30days/scripts/lib/pipeline.py@@ -178,6 +178,18 @@ def diagnose(config: dict[str, Any], requested_sources: list[str] | None = None)}+def _inner_max_workers(stream_count: int, *, internal_subrun: bool) -> int:+ """Worker-pool size for the per-stream fanout inside a single pipeline run.++ Top-level runs use up to 16 workers. Subruns of ``run_competitor_fanout``+ cap the inner pool to 4 so a six-way competitor fan-out stays below+ roughly 30 worker threads in aggregate instead of ~96.+ """+ if internal_subrun:+ return max(2, min(4, stream_count or 1))+ return max(4, min(16, stream_count or 1))++def run(*,topic: str,@@ -356,7 +368,7 @@ def run(for source in subquery.sourcesif source in available)- max_workers = max(4, min(16, stream_count or 1))+ max_workers = _inner_max_workers(stream_count, internal_subrun=internal_subrun)with ThreadPoolExecutor(max_workers=max_workers) as executor:for subquery in plan.subqueries:for source in subquery.sources:diff --git a/tests/test_pipeline_v3.py b/tests/test_pipeline_v3.pyindex 85d9cc03..c1b9867a 100644--- a/tests/test_pipeline_v3.py+++ b/tests/test_pipeline_v3.py@@ -1114,5 +1114,45 @@ class TestExcludeSourcesEndToEnd(unittest.TestCase):self.assertNotIn("tiktok", sources)self.assertNotIn("instagram", sources)++class TestInnerMaxWorkers(unittest.TestCase):+ """Cap inner ThreadPoolExecutor concurrency under competitor fanout.++ Without the cap, six competitor sub-runs each open their own+ ``ThreadPoolExecutor(max_workers=16)``, peaking around 96 worker threads+ that all hammer the same upstream APIs. ``internal_subrun=True`` should+ reduce the inner pool so the nested fanout stays bounded.+ """++ def test_normal_run_uses_full_ceiling(self):+ self.assertEqual(pipeline._inner_max_workers(20, internal_subrun=False), 16)+ self.assertEqual(pipeline._inner_max_workers(10, internal_subrun=False), 10)+ self.assertEqual(pipeline._inner_max_workers(1, internal_subrun=False), 4)++ def test_subrun_caps_at_four(self):+ self.assertEqual(pipeline._inner_max_workers(20, internal_subrun=True), 4)+ self.assertEqual(pipeline._inner_max_workers(10, internal_subrun=True), 4)+ self.assertEqual(pipeline._inner_max_workers(3, internal_subrun=True), 3)+ self.assertEqual(pipeline._inner_max_workers(1, internal_subrun=True), 2)++ def test_subrun_caps_total_concurrency_below_uncapped(self):+ # Derive the outer cap from fanout so this test stays meaningful if+ # MAX_PARALLEL_SUBRUNS is bumped. The contract under test is "subrun+ # mode meaningfully reduces total inner-thread count", not a magic+ # number tied to today's value of MAX_PARALLEL_SUBRUNS=6.+ from lib import fanout+ max_subruns = fanout.MAX_PARALLEL_SUBRUNS+ capped = pipeline._inner_max_workers(20, internal_subrun=True) * max_subruns+ uncapped = pipeline._inner_max_workers(20, internal_subrun=False) * max_subruns+ self.assertLess(capped, uncapped, f"capped={capped} not < uncapped={uncapped}")+ # The cap must cut total concurrency to at most half of the un-capped+ # value; otherwise the cap is doing real work.+ self.assertLessEqual(+ capped,+ uncapped // 2,+ f"capped {capped} should be at most half of uncapped {uncapped}",+ )++if __name__ == "__main__":unittest.main()