AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
refactor: extract subprocess cleanup into shared subproc helper (#210)
Ilia Alshanetskybbf892aecca1
diff --git a/skills/last30days/scripts/lib/bird_x.py b/skills/last30days/scripts/lib/bird_x.pyindex d2b723fa..72f9bab0 100644--- a/skills/last30days/scripts/lib/bird_x.py+++ b/skills/last30days/scripts/lib/bird_x.py@@ -7,13 +7,11 @@ See scripts/lib/vendor/bird-search/package.json for authoritative version.import jsonimport os-import signalimport shutil-import subprocessimport sysfrom pathlib import Path-from . import http, log+from . import http, log, subprocfrom datetime import datetimefrom typing import Any, Dict, List, Optional, Tuple@@ -168,62 +166,51 @@ def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:"--json",]- # Use process groups for clean cleanup on timeout/kill- preexec = os.setsid if hasattr(os, 'setsid') else None+ pid_holder: list[int] = []- try:- proc = subprocess.Popen(- cmd,- stdout=subprocess.PIPE,- stderr=subprocess.PIPE,- text=True,- encoding="utf-8",- errors="replace",- preexec_fn=preexec,- env=_subprocess_env(),- )-- # Register for cleanup tracking (if available)+ def _register(pid: int) -> None:+ pid_holder.append(pid)try:- from last30days import register_child_pid, unregister_child_pid- register_child_pid(proc.pid)+ from last30days import register_child_pid+ register_child_pid(pid)except ImportError:pass- try:- stdout, stderr = proc.communicate(timeout=timeout)- except subprocess.TimeoutExpired:- # Kill the entire process group- try:- os.killpg(os.getpgid(proc.pid), signal.SIGTERM)- except (ProcessLookupError, PermissionError, OSError):- proc.kill()- proc.wait(timeout=5)- return {"error": f"Search timed out after {timeout}s", "items": []}- finally:+ try:+ result = subproc.run_with_timeout(+ cmd,+ timeout=timeout,+ env=_subprocess_env(),+ on_pid=_register,+ )+ except subproc.SubprocTimeout:+ return {"error": f"Search timed out after {timeout}s", "items": []}+ except Exception as e:+ return {"error": str(e), "items": []}+ finally:+ if pid_holder:try:from last30days import unregister_child_pid- unregister_child_pid(proc.pid)+ unregister_child_pid(pid_holder[0])except Exception:pass- if proc.returncode != 0:- error = stderr.strip() if stderr else "Bird search failed"- return {"error": error, "items": []}+ if result.returncode != 0:+ error = result.stderr.strip() or "Bird search failed"+ return {"error": error, "items": []}- output = stdout.strip() if stdout else ""- if not output:- return {"items": []}+ output = result.stdout.strip()+ if not output:+ return {"items": []}+ try:parsed = json.loads(output)- if isinstance(parsed, list):- return {"items": parsed}- return parsed-except json.JSONDecodeError as e:return {"error": f"Invalid JSON response: {e}", "items": []}- except Exception as e:- return {"error": str(e), "items": []}++ if isinstance(parsed, list):+ return {"items": parsed}+ return parseddef search_x(@@ -330,47 +317,29 @@ def search_handles("--json",]- preexec = os.setsid if hasattr(os, 'setsid') else None-try:- proc = subprocess.Popen(- cmd,- stdout=subprocess.PIPE,- stderr=subprocess.PIPE,- text=True,- encoding="utf-8",- errors="replace",- preexec_fn=preexec,- env=_subprocess_env(),- )-- try:- stdout, stderr = proc.communicate(timeout=15)- except subprocess.TimeoutExpired:- try:- os.killpg(os.getpgid(proc.pid), signal.SIGTERM)- except (ProcessLookupError, PermissionError, OSError):- proc.kill()- proc.wait(timeout=5)- _log(f"Handle search timed out for @{handle}")- return []+ result = subproc.run_with_timeout(cmd, timeout=15, env=_subprocess_env())+ except subproc.SubprocTimeout:+ _log(f"Handle search timed out for @{handle}")+ return []+ except OSError as e:+ _log(f"Handle search error for @{handle}: {e}")+ return []- if proc.returncode != 0:- _log(f"Handle search failed for @{handle}: {(stderr or '').strip()}")- return []+ if result.returncode != 0:+ _log(f"Handle search failed for @{handle}: {result.stderr.strip()}")+ return []- output = (stdout or "").strip()- if not output:- return []+ output = result.stdout.strip()+ if not output:+ return []+ try:response = json.loads(output)- return parse_bird_response(response, query=core_topic)-except json.JSONDecodeError:_log(f"Invalid JSON from handle search for @{handle}")- except (OSError, subprocess.SubprocessError) as e:- _log(f"Handle search error for @{handle}: {e}")- return []+ return []+ return parse_bird_response(response, query=core_topic)from concurrent.futures import ThreadPoolExecutor, as_completeddiff --git a/skills/last30days/scripts/lib/subproc.py b/skills/last30days/scripts/lib/subproc.pynew file mode 100644index 00000000..bc5772e4--- /dev/null+++ b/skills/last30days/scripts/lib/subproc.py@@ -0,0 +1,94 @@+"""Subprocess helpers: safe timeout + process-group cleanup.++Used by bird_x.py (Node.js Bird search) and youtube_yt.py (yt-dlp search+and transcript download). Both need the same os.setsid/killpg cleanup+dance on timeout to avoid orphaning child processes.+"""++from __future__ import annotations++import os+import signal+import subprocess+from dataclasses import dataclass+from typing import Optional, Sequence+++class SubprocTimeout(Exception):+ """Raised when a subprocess exceeds its timeout and is killed."""+++@dataclass+class SubprocResult:+ """Result of a subprocess run that captured stdout and stderr."""++ returncode: int+ stdout: str+ stderr: str+++def run_with_timeout(+ cmd: Sequence[str],+ *,+ timeout: int,+ env: Optional[dict] = None,+ on_pid: Optional[callable] = None,+) -> SubprocResult:+ """Run a subprocess with process-group cleanup on timeout.++ Spawns ``cmd`` inside its own process group via ``os.setsid`` where+ available. If ``communicate(timeout=...)`` raises ``TimeoutExpired``,+ signals ``SIGTERM`` to the entire group, falls back to ``proc.kill()``+ if the signal fails, then waits up to 5 seconds for cleanup, and+ raises ``SubprocTimeout``.++ Args:+ cmd: Command and arguments to spawn.+ timeout: Timeout in seconds passed to ``communicate()``.+ env: Optional environment dict. If None, inherits parent env.+ on_pid: Optional callable invoked with the child PID right after+ spawn. Used by bird_x.py to register child PIDs for cleanup+ tracking. Exceptions raised by the callback are suppressed.++ Returns:+ SubprocResult with returncode, stdout, and stderr as strings.++ Raises:+ SubprocTimeout: If the process exceeded ``timeout``.+ FileNotFoundError: If the executable is not found.+ OSError: For other spawn failures.+ """+ preexec = os.setsid if hasattr(os, "setsid") else None++ proc = subprocess.Popen(+ list(cmd),+ stdout=subprocess.PIPE,+ stderr=subprocess.PIPE,+ text=True,+ encoding="utf-8",+ errors="replace",+ preexec_fn=preexec,+ env=env,+ )++ if on_pid is not None:+ try:+ on_pid(proc.pid)+ except Exception:+ pass++ try:+ stdout, stderr = proc.communicate(timeout=timeout)+ except subprocess.TimeoutExpired:+ try:+ os.killpg(os.getpgid(proc.pid), signal.SIGTERM)+ except (ProcessLookupError, PermissionError, OSError):+ proc.kill()+ proc.wait(timeout=5)+ raise SubprocTimeout(f"Command {cmd[0]} timed out after {timeout}s")++ return SubprocResult(+ returncode=proc.returncode,+ stdout=stdout or "",+ stderr=stderr or "",+ )diff --git a/skills/last30days/scripts/lib/youtube_yt.py b/skills/last30days/scripts/lib/youtube_yt.pyindex 7d71f727..03001c27 100644--- a/skills/last30days/scripts/lib/youtube_yt.py+++ b/skills/last30days/scripts/lib/youtube_yt.py@@ -8,11 +8,8 @@ Inspired by Peter Steinberger's toolchain approach (yt-dlp + summarize CLI).import jsonimport math-import osimport re-import signalimport shutil-import subprocessimport sysimport tempfileimport urllib.error@@ -37,7 +34,7 @@ TRANSCRIPT_LIMITS = {# Max words to keep from each transcriptTRANSCRIPT_MAX_WORDS = 5000-from . import http, log+from . import http, log, subprocfrom .relevance import token_overlap_relevance as _compute_relevance@@ -227,30 +224,16 @@ def search_youtube("--no-download",]- preexec = os.setsid if hasattr(os, 'setsid') else None-try:- proc = subprocess.Popen(- cmd,- stdout=subprocess.PIPE,- stderr=subprocess.PIPE,- text=True,- preexec_fn=preexec,- )- try:- stdout, stderr = proc.communicate(timeout=120)- except subprocess.TimeoutExpired:- try:- os.killpg(os.getpgid(proc.pid), signal.SIGTERM)- except (ProcessLookupError, PermissionError, OSError):- proc.kill()- proc.wait(timeout=5)- _log("YouTube search timed out (120s)")- return {"items": [], "error": "Search timed out"}+ result = subproc.run_with_timeout(cmd, timeout=120)+ except subproc.SubprocTimeout:+ _log("YouTube search timed out (120s)")+ return {"items": [], "error": "Search timed out"}except FileNotFoundError:return {"items": [], "error": "yt-dlp not found"}- if not (stdout or "").strip():+ stdout = result.stdout+ if not stdout.strip():_log("YouTube search returned 0 results")return {"items": []}@@ -452,25 +435,10 @@ def _fetch_transcript_ytdlp(video_id: str, temp_dir: str) -> Optional[str]:f"https://www.youtube.com/watch?v={video_id}",]- preexec = os.setsid if hasattr(os, 'setsid') else None-try:- proc = subprocess.Popen(- cmd,- stdout=subprocess.PIPE,- stderr=subprocess.PIPE,- text=True,- preexec_fn=preexec,- )- try:- proc.communicate(timeout=30)- except subprocess.TimeoutExpired:- try:- os.killpg(os.getpgid(proc.pid), signal.SIGTERM)- except (ProcessLookupError, PermissionError, OSError):- proc.kill()- proc.wait(timeout=5)- return None+ subproc.run_with_timeout(cmd, timeout=30)+ except subproc.SubprocTimeout:+ return Noneexcept FileNotFoundError:return None@@ -556,7 +524,7 @@ def fetch_transcripts_parallel(vid = futures[future]try:results[vid] = future.result()- except (OSError, subprocess.SubprocessError) as exc:+ except OSError as exc:_log(f"Transcript fetch error for {vid}: {exc}")results[vid] = Noneexcept Exception as exc:diff --git a/tests/test_env_v3.py b/tests/test_env_v3.pyindex c5a5b860..a79ec690 100644--- a/tests/test_env_v3.py+++ b/tests/test_env_v3.py@@ -30,8 +30,11 @@ class EnvV3Tests(unittest.TestCase):self.assertEqual("b", bird_x._credentials["CT0"])def test_bird_auth_never_checks_browser_cookies(self):+ # The guarantee: is_bird_authenticated() must not spawn any child+ # process to probe for cookies. All subprocess paths in bird_x go+ # through subproc.run_with_timeout, so patching that covers it.with mock.patch("lib.bird_x.is_bird_installed", return_value=True), mock.patch(- "lib.bird_x.subprocess.run",+ "lib.bird_x.subproc.run_with_timeout",side_effect=AssertionError("browser-cookie whoami should not run"),):bird_x._credentials.clear()diff --git a/tests/test_subproc.py b/tests/test_subproc.pynew file mode 100644index 00000000..e9f939db--- /dev/null+++ b/tests/test_subproc.py@@ -0,0 +1,102 @@+"""Tests for scripts/lib/subproc.py.++Covers the process-group cleanup path, timeout behavior, success path,+PID callback wiring, and environment inheritance.+"""++import sys+import unittest+from pathlib import Path+from unittest.mock import patch++sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))++from lib import subproc+++class TestRunWithTimeout(unittest.TestCase):+ def test_success_returns_stdout(self):+ result = subproc.run_with_timeout(+ ["sh", "-c", "echo hello"],+ timeout=5,+ )+ self.assertEqual(result.returncode, 0)+ self.assertEqual(result.stdout.strip(), "hello")+ self.assertEqual(result.stderr, "")++ def test_nonzero_exit_returns_returncode_not_exception(self):+ result = subproc.run_with_timeout(+ ["sh", "-c", "exit 3"],+ timeout=5,+ )+ self.assertEqual(result.returncode, 3)++ def test_captures_stderr(self):+ result = subproc.run_with_timeout(+ ["sh", "-c", "echo err >&2"],+ timeout=5,+ )+ self.assertEqual(result.stderr.strip(), "err")++ def test_timeout_raises_subproctimeout(self):+ with self.assertRaises(subproc.SubprocTimeout):+ subproc.run_with_timeout(+ ["sh", "-c", "sleep 10"],+ timeout=1,+ )++ def test_timeout_kills_process_group(self):+ """A slow child inside a shell should be killed when the group is signaled."""+ with self.assertRaises(subproc.SubprocTimeout):+ # Parent shell spawns a child that sleeps long.+ # Without process-group cleanup, the child would orphan.+ subproc.run_with_timeout(+ ["sh", "-c", "sleep 10 & wait"],+ timeout=1,+ )++ def test_missing_command_raises_oserror(self):+ """Missing executables raise FileNotFoundError (or PermissionError on+ some filesystems if a same-named junk file exists)."""+ with self.assertRaises(OSError):+ subproc.run_with_timeout(+ ["/nonexistent-path/last30days-test-no-such-bin"],+ timeout=5,+ )++ def test_env_is_passed_through(self):+ result = subproc.run_with_timeout(+ ["sh", "-c", "echo $LAST30DAYS_TEST_VAR"],+ timeout=5,+ env={"LAST30DAYS_TEST_VAR": "custom_value", "PATH": "/usr/bin:/bin"},+ )+ self.assertEqual(result.stdout.strip(), "custom_value")++ def test_on_pid_callback_receives_pid(self):+ seen_pids = []+ subproc.run_with_timeout(+ ["sh", "-c", "true"],+ timeout=5,+ on_pid=lambda pid: seen_pids.append(pid),+ )+ self.assertEqual(len(seen_pids), 1)+ self.assertIsInstance(seen_pids[0], int)+ self.assertGreater(seen_pids[0], 0)++ def test_on_pid_callback_exceptions_are_suppressed(self):+ """If the PID callback raises, the subprocess should still run to completion."""+ def raising_callback(pid):+ raise RuntimeError("boom")++ # Should not raise, callback exception is swallowed.+ result = subproc.run_with_timeout(+ ["sh", "-c", "echo ok"],+ timeout=5,+ on_pid=raising_callback,+ )+ self.assertEqual(result.returncode, 0)+ self.assertEqual(result.stdout.strip(), "ok")+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_youtube_yt.py b/tests/test_youtube_yt.pyindex a68ab862..6128d6ab 100644--- a/tests/test_youtube_yt.py+++ b/tests/test_youtube_yt.py@@ -63,24 +63,26 @@ class TestYouTubeEngagementZero(unittest.TestCase):class TestYtDlpFlags(unittest.TestCase):+ def _fake_result(self, stdout: str = "", returncode: int = 0):+ from lib.subproc import SubprocResult+ return SubprocResult(returncode=returncode, stdout=stdout, stderr="")+def test_search_ignores_global_config_and_browser_cookies(self):- proc = _DummyProc()with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \- mock.patch.object(youtube_yt.subprocess, "Popen", return_value=proc) as popen_mock:+ mock.patch.object(youtube_yt.subproc, "run_with_timeout", return_value=self._fake_result()) as run_mock:youtube_yt.search_youtube("Claude Code", "2026-02-01", "2026-03-01")- cmd = popen_mock.call_args.args[0]+ cmd = run_mock.call_args.args[0]self.assertIn("--ignore-config", cmd)self.assertIn("--no-cookies-from-browser", cmd)def test_transcript_fetch_ignores_global_config_and_browser_cookies(self):- proc = _DummyProc()with tempfile.TemporaryDirectory() as temp_dir, \mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \- mock.patch.object(youtube_yt.subprocess, "Popen", return_value=proc) as popen_mock:+ mock.patch.object(youtube_yt.subproc, "run_with_timeout", return_value=self._fake_result()) as run_mock:youtube_yt.fetch_transcript("abc123", temp_dir)- cmd = popen_mock.call_args.args[0]+ cmd = run_mock.call_args.args[0]self.assertIn("--ignore-config", cmd)self.assertIn("--no-cookies-from-browser", cmd)