Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

perf(pipeline): cap inner ThreadPool to 4 under competitor fanout (#435)

Ilia Alshanetskyccca10bbaf84
diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py
index e0732d3d..e709ae4f 100644
--- a/skills/last30days/scripts/last30days.py
+++ b/skills/last30days/scripts/last30days.py
@@ -820,6 +820,7 @@ def main() -> int:
lookback_days=args.lookback_days,
github_user=github_user,
github_repos=github_repos,
+ internal_subrun=comp_enabled,
hiring_signals_mode=args.hiring_signals,
)
r.artifacts["resolved"] = {
diff --git a/skills/last30days/scripts/lib/pipeline.py b/skills/last30days/scripts/lib/pipeline.py
index a20570dc..93d5c3be 100644
--- a/skills/last30days/scripts/lib/pipeline.py
+++ b/skills/last30days/scripts/lib/pipeline.py
@@ -178,6 +178,18 @@ def diagnose(config: dict[str, Any], requested_sources: list[str] | None = None)
}
+def _inner_max_workers(stream_count: int, *, internal_subrun: bool) -> int:
+ """Worker-pool size for the per-stream fanout inside a single pipeline run.
+
+ Top-level runs use up to 16 workers. Subruns of ``run_competitor_fanout``
+ cap the inner pool to 4 so a six-way competitor fan-out stays below
+ roughly 30 worker threads in aggregate instead of ~96.
+ """
+ if internal_subrun:
+ return max(2, min(4, stream_count or 1))
+ return max(4, min(16, stream_count or 1))
+
+
def run(
*,
topic: str,
@@ -356,7 +368,7 @@ def run(
for source in subquery.sources
if source in available
)
- max_workers = max(4, min(16, stream_count or 1))
+ max_workers = _inner_max_workers(stream_count, internal_subrun=internal_subrun)
with ThreadPoolExecutor(max_workers=max_workers) as executor:
for subquery in plan.subqueries:
for source in subquery.sources:
diff --git a/tests/test_pipeline_v3.py b/tests/test_pipeline_v3.py
index 85d9cc03..c1b9867a 100644
--- a/tests/test_pipeline_v3.py
+++ b/tests/test_pipeline_v3.py
@@ -1114,5 +1114,45 @@ class TestExcludeSourcesEndToEnd(unittest.TestCase):
self.assertNotIn("tiktok", sources)
self.assertNotIn("instagram", sources)
+
+class TestInnerMaxWorkers(unittest.TestCase):
+ """Cap inner ThreadPoolExecutor concurrency under competitor fanout.
+
+ Without the cap, six competitor sub-runs each open their own
+ ``ThreadPoolExecutor(max_workers=16)``, peaking around 96 worker threads
+ that all hammer the same upstream APIs. ``internal_subrun=True`` should
+ reduce the inner pool so the nested fanout stays bounded.
+ """
+
+ def test_normal_run_uses_full_ceiling(self):
+ self.assertEqual(pipeline._inner_max_workers(20, internal_subrun=False), 16)
+ self.assertEqual(pipeline._inner_max_workers(10, internal_subrun=False), 10)
+ self.assertEqual(pipeline._inner_max_workers(1, internal_subrun=False), 4)
+
+ def test_subrun_caps_at_four(self):
+ self.assertEqual(pipeline._inner_max_workers(20, internal_subrun=True), 4)
+ self.assertEqual(pipeline._inner_max_workers(10, internal_subrun=True), 4)
+ self.assertEqual(pipeline._inner_max_workers(3, internal_subrun=True), 3)
+ self.assertEqual(pipeline._inner_max_workers(1, internal_subrun=True), 2)
+
+ def test_subrun_caps_total_concurrency_below_uncapped(self):
+ # Derive the outer cap from fanout so this test stays meaningful if
+ # MAX_PARALLEL_SUBRUNS is bumped. The contract under test is "subrun
+ # mode meaningfully reduces total inner-thread count", not a magic
+ # number tied to today's value of MAX_PARALLEL_SUBRUNS=6.
+ from lib import fanout
+ max_subruns = fanout.MAX_PARALLEL_SUBRUNS
+ capped = pipeline._inner_max_workers(20, internal_subrun=True) * max_subruns
+ uncapped = pipeline._inner_max_workers(20, internal_subrun=False) * max_subruns
+ self.assertLess(capped, uncapped, f"capped={capped} not < uncapped={uncapped}")
+ # The cap must cut total concurrency to at most half of the un-capped
+ # value; otherwise the cap is doing real work.
+ self.assertLessEqual(
+ capped,
+ uncapped // 2,
+ f"capped {capped} should be at most half of uncapped {uncapped}",
+ )
+
+
if __name__ == "__main__":
unittest.main()