AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
fix: bound doctor and Reddit transport lifetimes (#1229)
Ilia Alshanetsky7f582ad8c2e1
diff --git a/changelog.d/+test-audit-deadlines.fixed.md b/changelog.d/+test-audit-deadlines.fixed.mdnew file mode 100644index 00000000..fdc9a890--- /dev/null+++ b/changelog.d/+test-audit-deadlines.fixed.md@@ -0,0 +1,5 @@+Doctor source probes and Reddit comment enrichment now stop stalled HTTP reads within their operation budgets and reap the transport workers. Reddit keeps comments that completed in time, and a zero enrichment budget starts no requests.++GET worker launch failures no longer leave unknown API-spend records for requests that were never delivered.++Doctor preserves completed Reddit HTTP 429 responses as unverified when the remaining probe budget cannot fit a retry.diff --git a/skills/last30days/scripts/lib/bounded_get.py b/skills/last30days/scripts/lib/bounded_get.pynew file mode 100644index 00000000..e1194014--- /dev/null+++ b/skills/last30days/scripts/lib/bounded_get.py@@ -0,0 +1,156 @@+"""Owned GET transport for operation deadlines, including DNS and body reads."""++from __future__ import annotations++import base64+from email.message import Message+import io+import json+import socket+import ssl+import sys+import threading+import time+import urllib.error+import urllib.request+from pathlib import Path++if __package__ in (None, ""):+ sys.path.insert(0, str(Path(__file__).resolve().parent.parent))++from lib import subproc++# Reserve TERM and KILL/reap time inside the caller's operation budget.+CLEANUP_GRACE_SECONDS = 0.1+++class GetTimeout(TimeoutError):+ def __init__(self, message, *, started):+ super().__init__(message)+ self.started = started+++class GetLaunchError(OSError):+ """GET worker launch failed before the worker received a request."""+++def _error_record(exc):+ if isinstance(exc, urllib.error.URLError):+ reason = exc.reason+ return {"kind": "URLError", "reason": _error_record(reason) if isinstance(reason, BaseException) else str(reason)}+ return {"kind": type(exc).__name__, "message": str(exc), "errno": getattr(exc, "errno", None)}+++def _error_from_record(record):+ if record["kind"] == "URLError":+ reason = record["reason"]+ return urllib.error.URLError(_error_from_record(reason) if isinstance(reason, dict) else reason)+ kinds = {+ "gaierror": socket.gaierror,+ "TimeoutError": TimeoutError,+ "ConnectionResetError": ConnectionResetError,+ "ConnectionRefusedError": ConnectionRefusedError,+ "OSError": OSError,+ "SSLError": ssl.SSLError,+ "SSLCertVerificationError": ssl.SSLCertVerificationError,+ }+ kind = kinds.get(record["kind"], OSError)+ if record.get("errno") is not None:+ return kind(record["errno"], record["message"])+ return kind(record["message"])+++def get(+ req: urllib.request.Request,+ *,+ timeout: float,+ deadline_monotonic: float,+ cancel: threading.Event | None = None,+ read_body: bool = True,+ read_error_body: bool = True,+):+ if req.get_method() != "GET":+ raise ValueError("bounded transport supports GET only")+ payload = {+ "url": req.full_url,+ "headers": req.headers,+ "unredirected_headers": req.unredirected_hdrs,+ "origin_req_host": req.origin_req_host,+ "timeout": timeout,+ "read_body": read_body,+ "read_error_body": read_error_body,+ }+ transport_deadline = deadline_monotonic - 2 * CLEANUP_GRACE_SECONDS+ try:+ result = subproc.run_with_timeout(+ [sys.executable, "-I", str(Path(__file__).resolve())],+ timeout=max(0, transport_deadline - time.monotonic()),+ deadline_monotonic=transport_deadline,+ input_text=json.dumps(payload),+ cancel=cancel,+ cleanup_grace=CLEANUP_GRACE_SECONDS,+ )+ except subproc.SubprocTimeout as exc:+ raise GetTimeout("GET exceeded operation deadline or was cancelled", started=exc.started) from exc+ except OSError as exc:+ if not getattr(exc, "_last30days_subproc_launch_failed", False):+ raise+ raise GetLaunchError(*exc.args) from exc+ if result.returncode != 0:+ raise OSError("bounded GET worker failed")+ record = json.loads(result.stdout)+ if "error" in record:+ raise _error_from_record(record["error"])+ body = base64.b64decode(record["body"]) if record["body"] is not None else None+ error = None+ if record.get("http_error") is not None:+ headers = Message()+ for key, value in record["headers"]:+ headers[key] = value+ error = urllib.error.HTTPError(+ record["url"], record["status"], record["http_error"],+ headers, io.BytesIO(body or b""),+ )+ return record["status"], body, error+++def _main():+ from lib import http++ payload = json.load(sys.stdin)+ req = urllib.request.Request(+ payload["url"], headers=payload["headers"], method="GET",+ origin_req_host=payload["origin_req_host"],+ )+ for key, value in payload["unredirected_headers"].items():+ req.add_unredirected_header(key, value)+ error_reason = None+ try:+ try:+ response = http.open_request(req, payload["timeout"])+ except urllib.error.HTTPError as exc:+ response = exc+ error_reason = str(exc.reason)+ with response:+ body = None+ read_body = payload["read_error_body"] if error_reason is not None else payload["read_body"]+ if read_body:+ try:+ body = response.read()+ except OSError:+ if error_reason is None:+ raise+ result = {+ "status": getattr(response, "status", 200) or 200,+ "body": base64.b64encode(body).decode("ascii") if body is not None else None,+ "headers": list(response.headers.items()),+ "url": response.geturl(),+ "http_error": error_reason,+ }+ except Exception as exc:+ result = {"error": _error_record(exc)}+ json.dump(result, sys.stdout)+++if __name__ == "__main__":+ _main()diff --git a/skills/last30days/scripts/lib/doctor.py b/skills/last30days/scripts/lib/doctor.pyindex 45009225..1a0a3d69 100644--- a/skills/last30days/scripts/lib/doctor.py+++ b/skills/last30days/scripts/lib/doctor.py@@ -1882,6 +1882,7 @@ def _http_ok(blocked_statuses: frozenset = frozenset(),headers: Optional[Dict[str, str]] = None,body_check: Optional[Callable[[str], Optional[str]]] = None,+ deadline_monotonic: Optional[float] = None,) -> tuple:"""Reachability check: a 4xx still means the endpoint responded; 5xx or aconnection/timeout error means it did not.@@ -1899,6 +1900,20 @@ def _http_ok(req = urllib.request.Request(url, headers=headers or {"User-Agent": "last30days-doctor"})+ if deadline_monotonic is not None:+ from . import bounded_get++ code, body, error = bounded_get.get(+ req, timeout=timeout, deadline_monotonic=deadline_monotonic,+ read_body=body_check is not None, read_error_body=False,+ )+ if error is not None:+ return _verdict(error.code)+ if body_check is not None and code < 300:+ reason = body_check((body or b"").decode("utf-8", errors="replace"))+ if reason:+ return False, f"HTTP {code} but blocked: {reason}"+ return _verdict(code)with urllib.request.urlopen(req, timeout=timeout) as resp:code = getattr(resp, "status", 200) or 200if body_check is not None and code < 300:@@ -1910,6 +1925,10 @@ def _http_ok(except urllib.error.HTTPError as exc:return _verdict(exc.code)except Exception as exc:+ from . import bounded_get++ if isinstance(exc, bounded_get.GetTimeout):+ return False, "probe exceeded deadline"return False, f"{type(exc).__name__}: {exc}"@@ -1926,6 +1945,10 @@ def _transient_probe_detail(name: str, detail: str) -> bool:def _probe_source(name: str, config: Dict[str, Any], timeout: float) -> Optional[Dict[str, Any]]:+ deadline = time.monotonic() + timeout+ expired = {"ok": False, "detail": "probe exceeded deadline", "probed": True}+ if timeout <= 0:+ return expiredurl = _HTTP_PROBE_URLS.get(name)if url:blocked = _PROBE_BLOCKED_STATUSES.get(name, frozenset())@@ -1934,11 +1957,29 @@ def _probe_source(name: str, config: Dict[str, Any], timeout: float) -> Optional"blocked_statuses": blocked,"headers": headers,"body_check": _PROBE_BODY_CHECKS.get(name),+ "deadline_monotonic": deadline,}ok, detail = _http_ok(url, timeout, **probe_kwargs)+ if time.monotonic() >= deadline or detail == "probe exceeded deadline":+ return expiredif not ok and _transient_probe_detail(name, detail):+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ return expired+ if remaining <= _PROBE_RETRY_DELAY_SECONDS:+ return {+ "ok": False,+ "transient": True,+ "detail": f"{detail} (retry skipped: insufficient probe budget)",+ "probed": True,+ }time.sleep(_PROBE_RETRY_DELAY_SECONDS)- ok, detail = _http_ok(url, timeout, **probe_kwargs)+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ return expired+ ok, detail = _http_ok(url, remaining, **probe_kwargs)+ if time.monotonic() >= deadline or detail == "probe exceeded deadline":+ return expiredif not ok and _transient_probe_detail(name, detail):return {"ok": False,@@ -1950,7 +1991,9 @@ def _probe_source(name: str, config: Dict[str, Any], timeout: float) -> Optionalcli = CLI_DEPENDENCIES.get(name)if cli:try:- probe = health.probe_dependency(cli)+ probe = health.probe_dependency(cli, timeout=max(0, deadline - time.monotonic()))+ if time.monotonic() >= deadline:+ return expiredexcept Exception as exc:return {"ok": False, "detail": f"{type(exc).__name__}: {exc}", "probed": True}return {"ok": bool(probe.ok), "detail": probe.detail, "probed": True}@@ -1973,7 +2016,7 @@ def _probe_sources(config: Dict[str, Any], timeout: int) -> Dict[str, Dict[str,}for name, fut in futures.items():try:- res = fut.result(timeout=timeout + 1)+ res = fut.result()except concurrent.futures.TimeoutError:res = {"ok": False, "detail": "probe exceeded deadline", "probed": True}except Exception as exc:diff --git a/skills/last30days/scripts/lib/health.py b/skills/last30days/scripts/lib/health.pyindex 69f842c4..f44abdc3 100644--- a/skills/last30days/scripts/lib/health.py+++ b/skills/last30days/scripts/lib/health.py@@ -16,10 +16,13 @@ from __future__ import annotationsimport osimport shutilimport subprocess+import timefrom dataclasses import dataclassfrom pathlib import Pathfrom typing import Dict, Iterable, List, Optional, Tuple+from . import subproc+# Health states, best to worst.OK = "ok"DEGRADED = "degraded" # ran, but returned less than expected@@ -388,6 +391,7 @@ def probe_dependency(name: str, timeout: float = PROBE_TIMEOUT) -> DependencyProdef _probe_dependency_uncached(name: str, timeout: float) -> DependencyProbe:+ deadline = time.monotonic() + timeoutresolved = shutil.which(name)if resolved is None:off_path = _off_path_binary(name)@@ -412,11 +416,13 @@ def _probe_dependency_uncached(name: str, timeout: float) -> DependencyProbe:command = [name] + _VERSION_ARGS.get(name, ["--version"])try:- proc = subprocess.run(+ cleanup_grace = 0.1+ proc = subproc.run_with_timeout(command,- capture_output=True,- text=True,- timeout=timeout,+ timeout=max(0, deadline - time.monotonic() - 2 * cleanup_grace),+ deadline_monotonic=deadline - 2 * cleanup_grace,+ cleanup_grace=cleanup_grace,+ capture_limit_bytes=64 * 1024,)except (FileNotFoundError, OSError) as exc:prescription, manager = _prescription(name, "reinstall")@@ -427,7 +433,7 @@ def _probe_dependency_uncached(name: str, timeout: float) -> DependencyProbe:prescription=prescription,owner_pkg_manager=manager,)- except subprocess.TimeoutExpired:+ except subproc.SubprocTimeout:prescription, manager = _prescription(name, "reinstall")return DependencyProbe(name=name,diff --git a/skills/last30days/scripts/lib/http.py b/skills/last30days/scripts/lib/http.pyindex 773cb35d..f09fb543 100644--- a/skills/last30days/scripts/lib/http.py+++ b/skills/last30days/scripts/lib/http.py@@ -18,7 +18,7 @@ from pathlib import Pathfrom typing import Any, Callable, Dict, Optional, Unionfrom urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit, quote-from . import health, usage+from . import bounded_get, health, usagefrom . import log as _logDEFAULT_TIMEOUT = 30@@ -776,6 +776,7 @@ def request(raw: bool = False,deadline_monotonic: float | None = None,cancel: threading.Event | None = None,+ owned_get: bool = False,) -> Union[Dict[str, Any], str]:"""Make an HTTP request and return JSON response.@@ -792,6 +793,8 @@ def request(raw: If True, return raw response text instead of parsed JSONdeadline_monotonic: Optional absolute monotonic deadline shared by allattempts and retry delays.+ owned_get: Run GET transport in a child that is killed and reaped at+ its deadline. Requires deadline_monotonic; other methods are refused.Returns:Parsed JSON response as dict, or raw text string if raw=True.@@ -799,6 +802,10 @@ def request(Raises:HTTPError: On request failure"""+ if owned_get and (method != "GET" or json_data is not None):+ raise ValueError("owned transport requires a bodyless GET")+ if owned_get and deadline_monotonic is None:+ raise ValueError("owned GET requires an operation deadline")headers = headers or {}headers.setdefault("User-Agent", USER_AGENT)@@ -892,6 +899,26 @@ def request(request_timeout: float,) -> tuple[int, str | None, urllib.error.HTTPError | None]:"""Stop waiting at the wall deadline, even during DNS or body reads."""+ if owned_get:+ try:+ status, body, error = bounded_get.get(+ req, timeout=request_timeout,+ deadline_monotonic=deadline_monotonic, cancel=cancel,+ )+ except bounded_get.GetLaunchError:+ usage.cancel(charge)+ raise+ except bounded_get.GetTimeout as exc:+ if not exc.started:+ usage.cancel(charge)+ raise deadline_error() from exc+ try:+ decoded = body.decode("utf-8") if body is not None else None+ except UnicodeDecodeError:+ if error is None:+ raise+ decoded = None+ return status, decoded, errorif deadline_monotonic is None and cancel is None:return open_and_read(request_timeout)wait_deadline = deadline_monotonic or (time.monotonic() + request_timeout)diff --git a/skills/last30days/scripts/lib/reddit.py b/skills/last30days/scripts/lib/reddit.pyindex e56b5d92..1f9cf7cc 100644--- a/skills/last30days/scripts/lib/reddit.py+++ b/skills/last30days/scripts/lib/reddit.py@@ -13,6 +13,7 @@ import reimport sysimport threadingimport time+from contextlib import ExitStackfrom collections import Counterfrom concurrent.futures import ThreadPoolExecutor, as_completed, wait as futures_waitfrom datetime import date, datetime, timezone@@ -404,6 +405,9 @@ def _subreddit_search(def fetch_post_comments(url: str,token: str,+ *,+ deadline_monotonic: float | None = None,+ cancel: threading.Event | None = None,) -> List[Dict[str, Any]]:"""Fetch comments for a Reddit post via ScrapeCreators.@@ -421,6 +425,9 @@ def fetch_post_comments(params={"url": url},timeout=30,retries=2,+ deadline_monotonic=deadline_monotonic,+ cancel=cancel,+ owned_get=deadline_monotonic is not None,)return data.get("comments", data.get("data", []))except http.HTTPError as e:@@ -669,7 +676,7 @@ def enrich_with_comments(config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])max_comments = config["comment_enrichments"]- if not items or not token or max_comments <= 0:+ if not items or not token or max_comments <= 0 or budget_seconds <= 0:return items# Select the top threads by total engagement (upvotes + comment count),@@ -680,11 +687,23 @@ def enrich_with_comments(_log(f"Enriching comments for {len(top_items)} posts (by total engagement)")start = time.monotonic()+ deadline = start + budget_seconds+ cancel = threading.Event()++ def fetch(item):+ if cancel.is_set() or time.monotonic() >= deadline:+ return None+ comments = fetch_post_comments(+ item.get("url", ""), token,+ deadline_monotonic=deadline, cancel=cancel,+ )+ return comments if time.monotonic() < deadline else None- with ThreadPoolExecutor(max_workers=min(4, len(top_items))) as executor:+ with ThreadPoolExecutor(max_workers=min(4, len(top_items))) as executor, ExitStack() as cleanup:+ cleanup.callback(cancel.set)futures = {http.submit_with_context(- executor, fetch_post_comments, item.get("url", ""), token,+ executor, fetch, item,): itemfor item in top_itemsif item.get("url")@@ -693,6 +712,8 @@ def enrich_with_comments(# Wait with budget instead of unbounded as_completedremaining = max(0, budget_seconds - (time.monotonic() - start))done, not_done = futures_wait(futures, timeout=remaining)+ if not_done:+ cancel.set()enriched_count = 0for future in done:diff --git a/skills/last30days/scripts/lib/subproc.py b/skills/last30days/scripts/lib/subproc.pyindex 0716f1f4..a2d693e1 100644--- a/skills/last30days/scripts/lib/subproc.py+++ b/skills/last30days/scripts/lib/subproc.py@@ -10,8 +10,10 @@ from __future__ import annotationsimport osimport signalimport subprocess+import tempfileimport threadingimport time+from contextlib import ExitStackfrom dataclasses import dataclassfrom typing import Optional, Sequence@@ -19,6 +21,10 @@ from typing import Optional, Sequenceclass SubprocTimeout(Exception):"""Raised when a subprocess exceeds its timeout and is killed."""+ def __init__(self, message: str = "", *, started: bool = True):+ super().__init__(message)+ self.started = started+# Live run_with_timeout children, process-wide. Each child is a session# leader (os.setsid), so a group kill aimed at the engine never reaches it;@@ -122,17 +128,22 @@ class SubprocResult:def run_with_timeout(cmd: Sequence[str],*,- timeout: int,+ timeout: float,env: Optional[dict] = None,on_pid: Optional[callable] = None,+ input_text: Optional[str] = None,+ deadline_monotonic: Optional[float] = None,+ cancel: Optional[threading.Event] = None,+ cleanup_grace: float = 5.0,+ capture_limit_bytes: Optional[int] = None,) -> SubprocResult:"""Run a subprocess with process-group cleanup on timeout.- Spawns ``cmd`` inside its own process group via ``os.setsid`` where+ Spawns ``cmd`` inside its own process group via ``start_new_session`` whereavailable. If ``communicate(timeout=...)`` raises ``TimeoutExpired``,signals ``SIGTERM`` to the entire group, falls back to ``proc.kill()``- if the signal fails, then waits up to 5 seconds for cleanup, and- raises ``SubprocTimeout``.+ if the signal fails, then waits up to ``cleanup_grace`` seconds before+ escalating to SIGKILL and waiting once more to reap the child.Args:cmd: Command and arguments to spawn.@@ -141,43 +152,83 @@ def run_with_timeout(on_pid: Optional callable invoked with the child PID right afterspawn. Exceptions raised by the callback are suppressed. Thechild is registered for cleanup_children() regardless.+ input_text: Optional UTF-8 text supplied on stdin, never argv.+ deadline_monotonic: Absolute operation deadline, including startup.+ cancel: Optional event checked while waiting for the command.+ cleanup_grace: Maximum TERM wait and subsequent KILL/reap wait, each.+ capture_limit_bytes: Capture to temporary files instead of pipes,+ reading at most this many bytes from each stream after exit.Returns:SubprocResult with returncode, stdout, and stderr as strings.Raises:- SubprocTimeout: If the process exceeded ``timeout``.+ SubprocTimeout: If the process timed out, was cancelled, or its+ absolute deadline elapsed.FileNotFoundError: If the executable is not found.OSError: For other spawn failures."""- preexec = os.setsid if hasattr(os, "setsid") else None-- proc = subprocess.Popen(- list(cmd),- stdout=subprocess.PIPE,- stderr=subprocess.PIPE,- text=True,- encoding="utf-8",- errors="replace",- preexec_fn=preexec,- env=env,- )-- if on_pid is not None:- try:- on_pid(proc.pid)- except Exception:- pass-- register_child_pid(proc.pid)+ if deadline_monotonic is not None and time.monotonic() >= deadline_monotonic:+ raise SubprocTimeout("Command deadline exceeded before spawn", started=False)+ if cancel is not None and cancel.is_set():+ raise SubprocTimeout("Command cancelled before spawn", started=False)+ if capture_limit_bytes is not None and capture_limit_bytes < 0:+ raise ValueError("capture limit must be nonnegative")+ own_group = hasattr(os, "setsid") and hasattr(os, "killpg")+ capture = ExitStack()+ try:+ stdout_file = capture.enter_context(tempfile.TemporaryFile()) if capture_limit_bytes is not None else None+ stderr_file = capture.enter_context(tempfile.TemporaryFile()) if capture_limit_bytes is not None else None+ proc = subprocess.Popen(+ list(cmd),+ stdout=stdout_file if stdout_file is not None else subprocess.PIPE,+ stderr=stderr_file if stderr_file is not None else subprocess.PIPE,+ stdin=subprocess.PIPE if input_text is not None else None,+ text=True,+ encoding="utf-8",+ errors="replace",+ start_new_session=own_group,+ env=env,+ )+ except BaseException as exc:+ if isinstance(exc, OSError):+ exc._last30days_subproc_launch_failed = True+ capture.close()+ raisetry:+ register_child_pid(proc.pid)+ if on_pid is not None:+ try:+ on_pid(proc.pid)+ except Exception:+ passtry:- stdout, stderr = proc.communicate(timeout=timeout)+ deadline = time.monotonic() + timeout+ if deadline_monotonic is not None:+ deadline = min(deadline, deadline_monotonic)+ first = True+ while True:+ remaining = deadline - time.monotonic()+ if remaining <= 0 or (cancel is not None and cancel.is_set()):+ raise subprocess.TimeoutExpired(cmd, timeout)+ try:+ stdout, stderr = proc.communicate(+ input=input_text if first else None,+ timeout=min(remaining, 0.05) if cancel is not None else remaining,+ )+ if deadline_monotonic is not None and time.monotonic() >= deadline_monotonic:+ raise subprocess.TimeoutExpired(cmd, timeout)+ break+ except subprocess.TimeoutExpired:+ if cancel is None or time.monotonic() >= deadline or cancel.is_set():+ raise+ finally:+ first = Falseexcept subprocess.TimeoutExpired:- term_deadline = time.monotonic() + 5+ term_deadline = time.monotonic() + cleanup_gracepgid = Nonetry:- if preexec is not None and hasattr(os, "killpg"):+ if own_group:pgid = proc.pidos.killpg(pgid, signal.SIGTERM)else:@@ -186,7 +237,7 @@ def run_with_timeout(pgid = Noneproc.kill()try:- proc.wait(timeout=5)+ proc.wait(timeout=cleanup_grace)except subprocess.TimeoutExpired:# Child ignored SIGTERM (or our killpg lost the race); escalate.try:@@ -197,7 +248,7 @@ def run_with_timeout(except (ProcessLookupError, PermissionError, OSError, AttributeError):proc.kill()try:- proc.wait(timeout=5)+ proc.wait(timeout=cleanup_grace)except subprocess.TimeoutExpired:pass # process unkillable (e.g. D-state); leave as zombieelse:@@ -211,8 +262,27 @@ def run_with_timeout(breaktime.sleep(min(_CLEANUP_POLL_SECONDS, remaining))raise SubprocTimeout(f"Command {cmd[0]} timed out after {timeout}s")+ if capture_limit_bytes is not None:+ stdout_file.seek(0)+ stderr_file.seek(0)+ stdout = stdout_file.read(capture_limit_bytes).decode("utf-8", errors="replace")+ stderr = stderr_file.read(capture_limit_bytes).decode("utf-8", errors="replace")+ except SubprocTimeout:+ raise+ except BaseException:+ if proc.poll() is None:+ _kill_child_group(proc.pid, getattr(signal, "SIGKILL", signal.SIGTERM))+ try:+ proc.wait(timeout=cleanup_grace)+ except subprocess.TimeoutExpired:+ pass+ raisefinally:unregister_child_pid(proc.pid)+ for pipe in (proc.stdin, proc.stdout, proc.stderr):+ if pipe is not None:+ pipe.close()+ capture.close()return SubprocResult(returncode=proc.returncode,diff --git a/tests/test_deadline_lifecycle.py b/tests/test_deadline_lifecycle.pynew file mode 100644index 00000000..8c58bed2--- /dev/null+++ b/tests/test_deadline_lifecycle.py@@ -0,0 +1,607 @@+import errno+import json+import os+import socket+import signal+import sqlite3+import sys+import threading+import time+from types import SimpleNamespace+import urllib.error+import urllib.request+from concurrent.futures import ThreadPoolExecutor+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from urllib.parse import parse_qs, urlencode, urlsplit+from unittest.mock import patch++import pytest++from lib import bounded_get, doctor, health, http, reddit, subproc, usage+++def children(monkeypatch):+ processes = []+ popen = subproc.subprocess.Popen++ def start(*args, **kwargs):+ process = popen(*args, **kwargs)+ processes.append(process)+ return process++ monkeypatch.setattr(subproc.subprocess, "Popen", start)+ yield processes+ assert processes+ assert all(process.poll() is not None for process in processes)+ assert all(process.pid not in subproc._child_pids for process in processes)+++def local_http():+ started = threading.Event()+ release = threading.Event()+ finished = threading.Event()++ class Handler(BaseHTTPRequestHandler):+ def log_message(self, *args):+ pass++ def do_GET(self):+ parsed = urlsplit(self.path)+ query = parse_qs(parsed.query)+ if parsed.path == "/redirect":+ self.send_response(302)+ self.send_header("Location", query["to"][0])+ self.end_headers()+ return+ status = int(parsed.path.rsplit("/", 1)[1]) if parsed.path.startswith("/status/") else 200+ if parsed.path == "/held-error":+ status = 403+ if query.get("url") == ["failed"]:+ status = 402+ self.send_response(status)+ self.send_header("Retry-After", "0")+ self.end_headers()+ hold = parsed.path in ("/hold", "/held-error") or query.get("url") == ["slow"]+ if hold:+ started.set()+ try:+ while not release.wait(0.02):+ self.wfile.write(b" ")+ self.wfile.flush()+ except (BrokenPipeError, ConnectionResetError):+ pass+ finally:+ finished.set()+ return+ if parsed.path == "/echo":+ self.wfile.write(json.dumps(dict(self.headers)).encode())+ return+ if status != 200:+ self.wfile.write(b'{"error":"credits exhausted"}')+ return+ self.wfile.write(json.dumps({"comments": [{+ "body": "The fast comment survives.", "score": 17, "author": "alice",+ }]}).encode())++ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ server.daemon_threads = True+ thread = threading.Thread(target=server.serve_forever)+ thread.start()+ try:+ yield f"http://127.0.0.1:{server.server_port}", started, release, finished+ finally:+ release.set()+ if started.is_set():+ assert finished.wait(2), "fixture request did not finish"+ server.shutdown()+ server.server_close()+ thread.join(2)+ assert not thread.is_alive()+++def test_doctor_returns_before_held_response_is_released(local_http, children):+ url, started, release, _ = local_http+ result = {}+ returned = threading.Event()++ def run():+ result.update(doctor._probe_sources({}, timeout=0.8))+ returned.set()++ with patch.object(doctor, "_probeable_sources", return_value=("reddit",)), \+ patch.dict(doctor._HTTP_PROBE_URLS, {"reddit": f"{url}/hold"}):+ caller = threading.Thread(target=run)+ caller.start()+ try:+ assert started.wait(2), "real HTTP body read was not reached"+ assert any(process.poll() is None for process in children)+ assert returned.wait(1.0), "doctor waited for the stalled response"+ assert all(process.poll() is not None for process in children), "owned HTTP worker survived timeout"+ assert result["reddit"] == {+ "ok": False, "detail": "probe exceeded deadline", "probed": True,+ }+ assert doctor.audit_state("reddit", {"tier": doctor.TIER_OK},+ probe_result=result["reddit"]) == doctor.AUDIT_NOT_WORKING+ finally:+ release.set()+ caller.join(3)+ assert not caller.is_alive()++[email protected]("status,expected_audit", [+ (429, doctor.AUDIT_UNVERIFIED),+ (403, doctor.AUDIT_NOT_WORKING),+])+def test_short_doctor_budget_preserves_completed_reddit_status(local_http, children, status, expected_audit):+ url, _, _, _ = local_http+ with patch.dict(doctor._HTTP_PROBE_URLS, {"reddit": f"{url}/status/{status}"}), \+ patch.object(doctor.time, "sleep") as sleep:+ result = doctor._probe_source("reddit", {}, 1)+ assert not result["ok"]+ assert result["probed"]+ assert result["detail"].startswith(f"HTTP {status}")+ assert result.get("transient", False) is (status == 429)+ assert doctor.audit_state("reddit", {"tier": doctor.TIER_OK},+ probe_result=result) == expected_audit+ assert len(children) == 1+ sleep.assert_not_called()+++def test_reddit_keeps_fast_comments_and_reaps_stalled_transport(local_http, children, monkeypatch):+ url, started, release, _ = local_http+ monkeypatch.setattr(reddit, "SCRAPECREATORS_BASE", url)+ items = [+ {"id": "fast", "url": "fast", "engagement": {"score": 10}},+ {"id": "slow", "url": "slow", "engagement": {"score": 20}},+ ]+ returned = threading.Event()+ result = []++ def run():+ result.extend(reddit.enrich_with_comments(items, "dummy-key", depth="quick", budget_seconds=1.2))+ returned.set()++ caller = threading.Thread(target=run)+ caller.start()+ try:+ assert started.wait(2), "real HTTP body read was not reached"+ assert returned.wait(1.8), "Reddit waited for the stalled response"+ assert len(result) == 2+ assert result[0]["top_comments"][0]["excerpt"] == "The fast comment survives."+ assert result[0]["top_comments"][0]["score"] == 17+ assert "top_comments" not in result[1]+ assert all(process.poll() is not None for process in children)+ before_release = json.dumps(result, sort_keys=True)+ finally:+ release.set()+ caller.join(3)+ assert not caller.is_alive()+ assert json.dumps(result, sort_keys=True) == before_release++[email protected]("path,body_check,expected", [+ ("/hold", None, True),+ ("/held-error", lambda body: "must not run", True),+])+def test_doctor_header_only_modes_do_not_read_bodies(local_http, children, path, body_check, expected):+ url, _, _, _ = local_http+ result = doctor._http_ok(f"{url}{path}", 2, body_check=body_check, deadline_monotonic=time.monotonic() + 2)+ assert result == (expected, "HTTP 403" if path == "/held-error" else "HTTP 200")++[email protected]("status,state", [(401, health.AUTH_FAILED), (402, health.PAYMENT_REQUIRED), (429, health.RATE_LIMITED)])+def test_owned_get_preserves_error_body_and_failure_capture(local_http, children, status, state):+ url, _, _, _ = local_http+ with http.capture_failures() as failures:+ with pytest.raises(http.HTTPError) as caught:+ http.get(f"{url}/status/{status}", retries=1, max_429_retries=1,+ deadline_monotonic=time.monotonic() + 2, owned_get=True)+ assert caught.value.status_code == status+ assert caught.value.body == '{"error":"credits exhausted"}'+ assert caught.value.outcome_state == state+ assert failures == [caught.value]++[email protected]("cross_origin", [False, True])+def test_owned_get_preserves_redirect_auth_policy(local_http, children, cross_origin):+ url, _, _, _ = local_http+ target = url.replace("127.0.0.1", "localhost") if cross_origin else url+ endpoint = f"{url}/redirect?{urlencode({'to': target + '/echo'})}"+ result = http.get(endpoint, headers={"Authorization": "Bearer dummy", "x-api-key": "dummy-key"},+ deadline_monotonic=time.monotonic() + 3, owned_get=True)+ lowered = {key.lower(): value for key, value in result.items()}+ if cross_origin:+ assert "authorization" not in lowered+ assert "x-api-key" not in lowered+ else:+ assert lowered["authorization"] == "Bearer dummy"+ assert lowered["x-api-key"] == "dummy-key"+++def test_owned_get_cancellation_reaps_transport(local_http, children):+ url, started, release, _ = local_http+ cancel = threading.Event()+ returned = threading.Event()+ caught = []++ def run():+ try:+ bounded_get.get(urllib.request.Request(f"{url}/hold"), timeout=5,+ deadline_monotonic=time.monotonic() + 5, cancel=cancel)+ except bounded_get.GetTimeout as exc:+ caught.append(exc)+ finally:+ returned.set()++ caller = threading.Thread(target=run)+ caller.start()+ try:+ assert started.wait(2)+ cancel.set()+ assert returned.wait(1)+ assert len(caught) == 1+ assert all(process.poll() is not None for process in children)+ finally:+ release.set()+ caller.join(3)+ assert not caller.is_alive()+++def test_stalled_error_body_is_reaped_and_captured_as_timeout(local_http, children):+ url, started, release, _ = local_http+ with http.capture_failures() as failures:+ try:+ with pytest.raises(http.DeadlineExceeded) as caught:+ http.get(f"{url}/held-error", retries=1,+ deadline_monotonic=time.monotonic() + 0.8, owned_get=True)+ assert started.is_set(), "real HTTP error body read was not reached"+ assert caught.value.outcome_state == health.TIMEOUT+ assert failures == [caught.value]+ assert all(process.poll() is not None for process in children)+ finally:+ release.set()+++def test_reddit_comment_worker_preserves_parent_failure_capture(local_http, children, monkeypatch):+ url, _, _, _ = local_http+ monkeypatch.setattr(reddit, "SCRAPECREATORS_BASE", url)+ items = [{"id": "failed", "url": "failed", "engagement": {"score": 10}}]+ with http.capture_failures() as failures:+ result = reddit.enrich_with_comments(items, "dummy-key", budget_seconds=2)+ assert result is items+ assert "top_comments" not in result[0]+ assert len(failures) == 1+ assert failures[0].status_code == 402+ assert failures[0].outcome_state == health.PAYMENT_REQUIRED+ assert failures[0].body == '{"error":"credits exhausted"}'+++def test_dns_error_wire_record_retains_retry_signal():+ restored = bounded_get._error_from_record({+ "kind": "URLError", "reason": {"kind": "gaierror", "errno": socket.EAI_AGAIN,+ "message": "temporary DNS failure"},+ })+ assert isinstance(restored, urllib.error.URLError)+ assert isinstance(restored.reason, socket.gaierror)+ assert restored.reason.errno == socket.EAI_AGAIN+ assert http._is_dns_failure(restored)+++def test_owned_get_replay_skips_child_and_request_accounting(local_http, children, tmp_path):+ url, _, _, _ = local_http+ fixture = tmp_path / "fixture"+ with http.recording_requests(fixture):+ recorded = http.get(f"{url}/echo", deadline_monotonic=time.monotonic() + 2, owned_get=True)+ with patch.object(bounded_get, "get", side_effect=AssertionError("replay launched transport")), \+ patch.object(usage, "begin_http") as begin, http.replaying_requests(fixture):+ replayed = http.get(f"{url}/echo", deadline_monotonic=time.monotonic() + 2, owned_get=True)+ assert replayed == recorded+ begin.assert_not_called()+++def test_unstarted_owned_get_cancels_request_charge(local_http):+ url, _, _, _ = local_http+ charge = ("scrapecreators", "request", "dummy-attempt")+ with patch.object(usage, "begin_http", return_value=charge) as begin, \+ patch.object(usage, "cancel") as cancel, patch.object(subproc.subprocess, "Popen") as spawn:+ with pytest.raises(http.DeadlineExceeded):+ http.get(f"{url}/echo", deadline_monotonic=time.monotonic() + 0.1, owned_get=True)+ begin.assert_called_once()+ cancel.assert_called_once_with(charge)+ spawn.assert_not_called()+++def paid_owned_get(local_http, tmp_path, monkeypatch):+ """Keep real provider accounting while restricting worker destinations to loopback."""+ local_url, _, _, _ = local_http+ paid_url = "https://api.scrapecreators.com/v1/reddit/post"+ journal = tmp_path / "usage.db"+ usage.create_journal(journal)+ monkeypatch.setenv(usage.JOURNAL_ENV, str(journal))+ guarded_get = bounded_get.get++ def local_transport(req, **kwargs):+ assert req.full_url == paid_url+ return guarded_get(urllib.request.Request(f"{local_url}/echo"), **kwargs)++ monkeypatch.setattr(bounded_get, "get", local_transport)+ monkeypatch.setattr(http.time, "sleep", lambda _: None)+ return paid_url, journal+++def _pending_charges(journal):+ with sqlite3.connect(journal) as connection:+ return connection.execute("SELECT COUNT(*) FROM attempts WHERE unknown = 1").fetchone()[0]++[email protected]("error_number", [errno.ENOENT, errno.EAGAIN, errno.EMFILE])+def test_owned_get_launch_error_preserves_errno_and_cause(local_http, monkeypatch, error_number):+ local_url, _, _, _ = local_http+ error = OSError(error_number, "GET worker launch failed")+ with patch.object(subproc.subprocess, "Popen", side_effect=error):+ with pytest.raises(bounded_get.GetLaunchError) as caught:+ bounded_get.get(urllib.request.Request(f"{local_url}/echo"), timeout=2,+ deadline_monotonic=time.monotonic() + 2)+ assert caught.value.errno == error_number+ assert caught.value.__cause__ is error++[email protected]("error_number", [errno.EAGAIN, errno.EMFILE])+def test_owned_get_spawn_errors_cancel_each_paid_retry(paid_owned_get, monkeypatch, error_number):+ url, journal = paid_owned_get+ pending_before_spawn = []++ def unavailable(*args, **kwargs):+ pending_before_spawn.append(_pending_charges(journal))+ raise OSError(error_number, "GET worker could not start")++ monkeypatch.setattr(subproc.subprocess, "Popen", unavailable)+ with http.capture_failures() as failures:+ with pytest.raises(http.HTTPError) as caught:+ http.get(url, retries=3, deadline_monotonic=time.monotonic() + 30, owned_get=True)++ assert len(pending_before_spawn) == 3+ assert _pending_charges(journal) == 0+ assert pending_before_spawn == [1, 1, 1]+ assert usage.read_journal(journal) == {+ "token_cost": 0.0, "cost_unknown": 0, "prompt_tokens": 0, "completion_tokens": 0,+ }+ assert caught.value.outcome_state == health.UNREACHABLE+ assert "GET worker could not start" in str(caught.value)+ assert failures == [caught.value]++[email protected]("error_number", [errno.EAGAIN, errno.EMFILE])[email protected]("stage", ["registration", "communication"])+def test_owned_get_post_launch_errors_keep_unknown_paid_retries(paid_owned_get, children, monkeypatch, error_number, stage):+ url, journal = paid_owned_get+ pending_before_spawn = []+ start = subproc.subprocess.Popen++ def fail(*args, **kwargs):+ raise OSError(error_number, "GET parent failed after launch")++ def fail_after_start(*args, **kwargs):+ pending_before_spawn.append(_pending_charges(journal))+ process = start(*args, **kwargs)+ if stage == "communication":+ process.communicate = fail+ return process++ monkeypatch.setattr(subproc.subprocess, "Popen", fail_after_start)+ if stage == "registration":+ monkeypatch.setattr(subproc, "register_child_pid", fail)+ with http.capture_failures() as failures:+ with pytest.raises(http.HTTPError) as caught:+ http.get(url, retries=3, deadline_monotonic=time.monotonic() + 30, owned_get=True)++ assert len(children) == 3+ assert pending_before_spawn == [1, 2, 3]+ assert _pending_charges(journal) == 3+ assert usage.read_journal(journal) == {+ "token_cost": 0.0, "cost_unknown": 1, "prompt_tokens": 0, "completion_tokens": 0,+ }+ assert caught.value.outcome_state == health.UNREACHABLE+ assert "GET parent failed after launch" in str(caught.value)+ assert failures == [caught.value]++[email protected]("kwargs", [+ {"method": "POST", "deadline_monotonic": 1},+ {"method": "GET", "json_data": {"key": "value"}, "deadline_monotonic": 1},+ {"method": "GET"},+])+def test_owned_transport_rejects_unsupported_calls_before_accounting(kwargs):+ with patch.object(usage, "begin_http") as begin, patch.object(bounded_get, "get") as transport:+ with pytest.raises(ValueError):+ http.request(url="http://example.test", owned_get=True, **kwargs)+ begin.assert_not_called()+ transport.assert_not_called()+++def test_subprocess_without_process_groups_still_reaps_child(children, monkeypatch):+ monkeypatch.setattr(subproc, "os", SimpleNamespace(kill=os.kill))+ with pytest.raises(subproc.SubprocTimeout):+ subproc.run_with_timeout([sys.executable, "-c", "import time; time.sleep(30)"],+ timeout=0.05, cleanup_grace=0.1)+ assert all(process.poll() is not None for process in children)+++def test_unexpected_parent_failure_reaps_owned_child(children, monkeypatch):+ start = subproc.subprocess.Popen++ def fail_after_start(*args, **kwargs):+ process = start(*args, **kwargs)+ def fail(**kwargs):+ raise RuntimeError("parent communication failed")+ process.communicate = fail+ return process++ monkeypatch.setattr(subproc.subprocess, "Popen", fail_after_start)+ with pytest.raises(RuntimeError, match="parent communication failed"):+ subproc.run_with_timeout([sys.executable, "-c", "import time; time.sleep(30)"],+ timeout=2, cleanup_grace=0.1)+ assert all(process.poll() is not None for process in children)++[email protected](not hasattr(os, "killpg"), reason="POSIX process-group termination")+def test_health_timeout_terminates_inherited_output_descendant(tmp_path, children, monkeypatch):+ descendant_pid = tmp_path / "descendant.pid"+ stopped = tmp_path / "descendant.stopped"+ child = (+ "import os, signal, sys, time\nfrom pathlib import Path\n"+ f"def stop(*args):\n Path({str(stopped)!r}).write_text('terminated')\n sys.exit(0)\n"+ "signal.signal(signal.SIGTERM, stop)\n"+ f"Path({str(descendant_pid)!r}).write_text(str(os.getpid()))\n"+ "time.sleep(30)\n"+ )+ command = tmp_path / "version.py"+ command.write_text(f"import subprocess, sys, time\nsubprocess.Popen([sys.executable, '-c', {child!r}])\ntime.sleep(30)\n")+ monkeypatch.setitem(health._VERSION_ARGS, sys.executable, [str(command)])+ started = time.monotonic()+ try:+ result = health._probe_dependency_uncached(sys.executable, 0.8)+ assert time.monotonic() - started < 1.5+ assert result.status == health.TIMEOUT+ assert descendant_pid.exists(), "real descendant was not reached"+ assert stopped.exists(), "timed-out version probe left its descendant running"+ assert stopped.read_text() == "terminated"+ assert all(process.poll() is not None for process in children)+ finally:+ if descendant_pid.exists() and not stopped.exists():+ try:+ os.kill(int(descendant_pid.read_text()), signal.SIGTERM)+ except ProcessLookupError:+ pass+++def test_health_real_version_command_preserves_first_line(children, monkeypatch):+ monkeypatch.setitem(health._VERSION_ARGS, sys.executable, [+ "-c", "import sys; print('1.2.3'); print('ignored second line'); print('stderr detail', file=sys.stderr)",+ ])+ result = health._probe_dependency_uncached(sys.executable, 2)+ assert result.status == health.OK+ assert result.detail == "1.2.3"+ assert result.prescription == ""+ assert all(process.stdout is None and process.stderr is None for process in children)+++def test_file_capture_returns_bounded_stdout_and_stderr(children):+ result = subproc.run_with_timeout([+ sys.executable, "-c", "import sys; print('a' * 100); print('b' * 100, file=sys.stderr)",+ ], timeout=2, capture_limit_bytes=16)+ assert result.returncode == 0+ assert result.stdout == "a" * 16+ assert result.stderr == "b" * 16+ assert all(process.stdout is None and process.stderr is None for process in children)+++def test_file_capture_without_process_groups_avoids_descendant_pipe_wait(tmp_path, children, monkeypatch):+ descendant_pid = tmp_path / "descendant.pid"+ command = tmp_path / "version.py"+ child = "import time; time.sleep(30)"+ command.write_text(+ "import subprocess, sys, time\nfrom pathlib import Path\n"+ f"child = subprocess.Popen([sys.executable, '-c', {child!r}])\n"+ f"Path({str(descendant_pid)!r}).write_text(str(child.pid))\n"+ "time.sleep(30)\n"+ )+ monkeypatch.setattr(subproc, "os", SimpleNamespace(kill=os.kill))+ started = time.monotonic()+ try:+ with pytest.raises(subproc.SubprocTimeout):+ subproc.run_with_timeout([sys.executable, str(command)], timeout=0.6,+ cleanup_grace=0.1, capture_limit_bytes=64 * 1024)+ assert time.monotonic() - started < 1.5+ assert descendant_pid.exists(), "real descendant was not reached"+ assert all(process.poll() is not None for process in children)+ assert all(process.stdout is None and process.stderr is None for process in children)+ finally:+ if descendant_pid.exists():+ try:+ os.kill(int(descendant_pid.read_text()), signal.SIGTERM)+ except ProcessLookupError:+ pass+++def test_doctor_retry_spends_one_operation_budget(monkeypatch):+ clock = [100.0]+ timeouts = []++ def get(*args, **kwargs):+ timeouts.append((args[1], kwargs["deadline_monotonic"]))+ clock[0] += 6 if len(timeouts) == 1 else 0.5+ return (False, "HTTP 429") if len(timeouts) == 1 else (True, "HTTP 200")++ monkeypatch.setattr(doctor.time, "monotonic", lambda: clock[0])+ monkeypatch.setattr(doctor.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds))+ monkeypatch.setattr(doctor, "_http_ok", get)+ assert doctor._probe_source("reddit", {}, 10)["ok"]+ assert timeouts == [(10, 110), (2, 110)]++[email protected]("elapsed", [8, 9])+def test_doctor_does_not_retry_when_backoff_exceeds_budget(monkeypatch, elapsed):+ clock = [100.0]++ def get(*args, **kwargs):+ clock[0] += elapsed+ return False, "HTTP 429"++ monkeypatch.setattr(doctor.time, "monotonic", lambda: clock[0])+ monkeypatch.setattr(doctor, "_http_ok", get)+ with patch.object(doctor.time, "sleep") as sleep:+ result = doctor._probe_source("reddit", {}, 10)+ assert result == {+ "ok": False, "transient": True,+ "detail": "HTTP 429 (retry skipped: insufficient probe budget)", "probed": True,+ }+ assert doctor.audit_state("reddit", {"tier": doctor.TIER_OK},+ probe_result=result) == doctor.AUDIT_UNVERIFIED+ sleep.assert_not_called()++[email protected]("readings", [+ (100.0, 101.0),+ (100.0, 100.5, 101.0),+])+def test_doctor_expired_429_keeps_deadline_precedence(monkeypatch, readings):+ clock = iter(readings)+ monkeypatch.setattr(doctor.time, "monotonic", lambda: next(clock))+ with patch.object(doctor, "_http_ok", return_value=(False, "HTTP 429")) as transport, \+ patch.object(doctor.time, "sleep") as sleep:+ result = doctor._probe_source("reddit", {}, 1)+ assert result == {"ok": False, "detail": "probe exceeded deadline", "probed": True}+ assert doctor.audit_state("reddit", {"tier": doctor.TIER_OK},+ probe_result=result) == doctor.AUDIT_NOT_WORKING+ transport.assert_called_once()+ sleep.assert_not_called()+++def test_doctor_cli_receives_remaining_timeout(monkeypatch):+ clock = iter([100.0, 100.25, 100.5])+ monkeypatch.setattr(doctor.time, "monotonic", lambda: next(clock))+ probe = health.DependencyProbe(name="yt-dlp", status=health.OK, detail="version")+ with patch.object(health, "probe_dependency", return_value=probe) as dependency:+ assert doctor._probe_source("youtube", {}, 5)["ok"]+ dependency.assert_called_once_with("yt-dlp", timeout=4.75)+++def test_queued_doctor_probe_gets_its_own_start_deadline(monkeypatch):+ clock = [100.0]+ deadlines = []++ def get(*args, **kwargs):+ deadlines.append(kwargs["deadline_monotonic"])+ clock[0] += 8+ return True, "HTTP 200"++ monkeypatch.setattr(doctor.time, "monotonic", lambda: clock[0])+ monkeypatch.setattr(doctor, "_http_ok", get)+ monkeypatch.setattr(doctor, "_probeable_sources", lambda: ("github", "hackernews"))+ monkeypatch.setattr(doctor.concurrent.futures, "ThreadPoolExecutor", lambda **kwargs: ThreadPoolExecutor(max_workers=1))+ assert all(result["ok"] for result in doctor._probe_sources({}, 10).values())+ assert deadlines == [110, 118]diff --git a/tests/test_doctor.py b/tests/test_doctor.pyindex 23cb27f0..d8abfa43 100644--- a/tests/test_doctor.py+++ b/tests/test_doctor.py@@ -1005,19 +1005,7 @@ class LiveProbe(unittest.TestCase):self.assertEqual(reddit_search.search_url("test"), url)def _probe_reddit_with_body(self, body):- class _Resp:- status = 200-- def read(self, *args):- return body.encode("utf-8")-- def __enter__(self):- return self-- def __exit__(self, *exc):- return False-- with mock.patch("lib.doctor.urllib.request.urlopen", return_value=_Resp()):+ with mock.patch("lib.bounded_get.get", return_value=(200, body.encode("utf-8"), None)):return doctor._probe_source("reddit", {}, 5)def test_reddit_probe_200_with_results_passes(self):@@ -1045,7 +1033,7 @@ class LiveProbe(unittest.TestCase):doctor._HTTP_PROBE_URLS["reddit"], code, "Blocked", {}, None)with (- mock.patch("lib.doctor.urllib.request.urlopen", side_effect=error),+ mock.patch("lib.bounded_get.get", return_value=(code, None, error)),# 429 buys a retry; don't pay the real backoff in the suite.mock.patch("lib.doctor.time.sleep"),):@@ -1125,7 +1113,7 @@ class LiveProbe(unittest.TestCase):error = urllib.error.HTTPError(doctor._HTTP_PROBE_URLS["github"], 403, "Forbidden", {}, None)- with mock.patch("lib.doctor.urllib.request.urlopen", side_effect=error):+ with mock.patch("lib.bounded_get.get", return_value=(403, None, error)):res = doctor._probe_source("github", {}, 5)self.assertTrue(res["ok"])@@ -1133,12 +1121,12 @@ class LiveProbe(unittest.TestCase):# Probing with a different UA measures the User-Agent, not the endpoint.seen = {}- def capture(req, timeout=None):+ def capture(req, **kwargs):seen["ua"] = req.get_header("User-agent")seen["accept"] = req.get_header("Accept")- raise urllib.error.HTTPError(req.full_url, 500, "boom", {}, None)+ return 500, None, urllib.error.HTTPError(req.full_url, 500, "boom", {}, None)- with mock.patch("lib.doctor.urllib.request.urlopen", capture):+ with mock.patch("lib.bounded_get.get", capture):doctor._probe_source("reddit", {}, 5)self.assertEqual(http.BROWSER_USER_AGENT, seen["ua"])self.assertIn("text/html", seen["accept"])@@ -1155,19 +1143,12 @@ class LiveProbe(unittest.TestCase):self.assertIn("boom", results["reddit"]["detail"])self.assertTrue(results["hackernews"]["ok"]) # others unaffected- def test_probe_deadline_never_hangs(self):- import time-- def too_slow(name, config, timeout):- time.sleep(1.3) # exceeds the timeout(0)+1s result deadline- return {"ok": True, "probed": True}-- with mock.patch("lib.doctor._probe_source", too_slow):+ def test_zero_probe_budget_starts_no_transport(self):+ with mock.patch("lib.bounded_get.get") as transport:results = doctor._probe_sources({}, timeout=0)+ transport.assert_not_called()self.assertTrue(results)- self.assertTrue(- any("deadline" in r.get("detail", "") for r in results.values())- )+ self.assertTrue(all("deadline" in r["detail"] for r in results.values()))def test_probe_result_flips_unverified_to_working(self):rec = {"tier": "ok", "status": "ok", "audit_state": doctor.AUDIT_UNVERIFIED}diff --git a/tests/test_grok_dependency.py b/tests/test_grok_dependency.pyindex 46202d2d..6f1eccdd 100644--- a/tests/test_grok_dependency.py+++ b/tests/test_grok_dependency.py@@ -41,7 +41,7 @@ def test_broken_binary_yields_a_reinstall_prescription(monkeypatch):def boom(*a, **k):raise OSError("exec format error")- monkeypatch.setattr(health.subprocess, "run", boom)+ monkeypatch.setattr(health.subproc, "run_with_timeout", boom)probe = health.probe_dependency("grok")assert probe.status == health.BROKENassert "reinstall" in probe.prescription.lower()diff --git a/tests/test_health_probe_taxonomy.py b/tests/test_health_probe_taxonomy.pyindex 2e66dad8..63d0ca02 100644--- a/tests/test_health_probe_taxonomy.py+++ b/tests/test_health_probe_taxonomy.py@@ -103,7 +103,7 @@ class TestBroken:def test_stale_shim_exec_oserror_is_broken_not_ok(self):with mock.patch.object(health.shutil, "which", side_effect=_which_map({"yt-dlp": "/x/yt-dlp", "brew": "/x/brew"})), \- mock.patch.object(health.subprocess, "run", side_effect=OSError("exec format error")):+ mock.patch.object(health.subproc, "run_with_timeout", side_effect=OSError("exec format error")):probe = health.probe_dependency("yt-dlp")assert probe.status == health.BROKENassert probe.prescription == "brew reinstall yt-dlp"@@ -112,7 +112,7 @@ class TestBroken:def test_nonzero_version_exit_is_broken(self):fake = _completed(rc=1, stderr="ModuleNotFoundError: No module named 'yt_dlp'")with mock.patch.object(health.shutil, "which", side_effect=_which_map({"yt-dlp": "/x/yt-dlp", "brew": "/x/brew"})), \- mock.patch.object(health.subprocess, "run", return_value=fake):+ mock.patch.object(health.subproc, "run_with_timeout", return_value=fake):probe = health.probe_dependency("yt-dlp")assert probe.status == health.BROKENassert "ModuleNotFoundError" in probe.detail@@ -120,7 +120,7 @@ class TestBroken:def test_broken_pp_cli_prescribes_rerunning_install(self):with mock.patch.object(health.shutil, "which", side_effect=_which_map({"digg-pp-cli": "/x/digg-pp-cli"})), \- mock.patch.object(health.subprocess, "run", side_effect=OSError("bad exec")):+ mock.patch.object(health.subproc, "run_with_timeout", side_effect=OSError("bad exec")):probe = health.probe_dependency("digg-pp-cli")assert probe.status == health.BROKENassert "printing-press-library" in probe.prescription@@ -144,8 +144,8 @@ class TestTimeout:def test_timeout_status_and_message(self):with mock.patch.object(health.shutil, "which", side_effect=_which_map({"ffmpeg": "/x/ffmpeg", "brew": "/x/brew"})), \mock.patch.object(- health.subprocess, "run",- side_effect=subprocess.TimeoutExpired(cmd=["ffmpeg"], timeout=health.PROBE_TIMEOUT),+ health.subproc, "run_with_timeout",+ side_effect=health.subproc.SubprocTimeout("ffmpeg timed out"),):probe = health.probe_dependency("ffmpeg")assert probe.status == health.TIMEOUT@@ -153,7 +153,7 @@ class TestTimeout:def test_probe_budget_is_bounded(self):with mock.patch.object(health.shutil, "which", side_effect=_which_map({"node": "/x/node", "brew": "/x/brew"})), \- mock.patch.object(health.subprocess, "run", return_value=_completed(stdout="v22.1.0")) as run:+ mock.patch.object(health.subproc, "run_with_timeout", return_value=_completed(stdout="v22.1.0")) as run:health.probe_dependency("node")assert run.call_args.kwargs["timeout"] <= health.PROBE_TIMEOUT@@ -163,7 +163,7 @@ class TestOk:def test_healthy_binary_ok_no_prescription(self):with mock.patch.object(health.shutil, "which", side_effect=_which_map({"yt-dlp": "/x/yt-dlp", "brew": "/x/brew"})), \- mock.patch.object(health.subprocess, "run", return_value=_completed(stdout="2026.06.09\n")):+ mock.patch.object(health.subproc, "run_with_timeout", return_value=_completed(stdout="2026.06.09\n")):probe = health.probe_dependency("yt-dlp")assert probe.status == health.OKassert probe.ok@@ -214,7 +214,7 @@ class TestCachingAndRegistry:def test_probe_memoized_single_subprocess(self):with mock.patch.object(health.shutil, "which", side_effect=_which_map({"node": "/x/node", "brew": "/x/brew"})), \- mock.patch.object(health.subprocess, "run", return_value=_completed(stdout="v22.1.0")) as run:+ mock.patch.object(health.subproc, "run_with_timeout", return_value=_completed(stdout="v22.1.0")) as run:first = health.probe_dependency("node")second = health.probe_dependency("node")assert run.call_count == 1@@ -222,7 +222,7 @@ class TestCachingAndRegistry:def test_clear_cache_reprobes(self):with mock.patch.object(health.shutil, "which", side_effect=_which_map({"node": "/x/node", "brew": "/x/brew"})), \- mock.patch.object(health.subprocess, "run", return_value=_completed(stdout="v22.1.0")) as run:+ mock.patch.object(health.subproc, "run_with_timeout", return_value=_completed(stdout="v22.1.0")) as run:health.probe_dependency("node")health.clear_dependency_probe_cache()health.probe_dependency("node")diff --git a/tests/test_reddit.py b/tests/test_reddit.pyindex 6189ee72..61ca7926 100644--- a/tests/test_reddit.py+++ b/tests/test_reddit.py@@ -154,7 +154,7 @@ class TestEnrichSelectsTopEngagement(unittest.TestCase):enriched_urls = []- def mock_fetch_comments(url, token):+ def mock_fetch_comments(url, token, **kwargs):enriched_urls.append(url)return [{"body": "Great thread!", "ups": 10, "author": "testuser"}]@@ -189,7 +189,7 @@ class TestEnrichSelectsTopEngagement(unittest.TestCase):enriched_urls = []- def mock_fetch_comments(url, token):+ def mock_fetch_comments(url, token, **kwargs):enriched_urls.append(url)return [{"body": "Comment", "ups": 5, "author": "user"}]@@ -231,20 +231,16 @@ class TestEnrichmentBudget(unittest.TestCase):def test_budget_zero_returns_items_unenriched(self):"""With budget=0, items are returned without enrichment (not discarded)."""- import time as _timefrom unittest.mock import patchitems = self._make_items(3)- def slow_fetch(url, token):- _time.sleep(2)- return [{"body": "comment", "score": 10, "author": "u"}]-- with patch("lib.reddit.fetch_post_comments", side_effect=slow_fetch):+ with patch("lib.reddit.fetch_post_comments") as fetch:result = enrich_with_comments(items, "fake-token", depth="quick", budget_seconds=0)-- # All 3 items returned (not discarded)+ self.assertIs(result, items)self.assertEqual(len(result), 3)+ self.assertTrue(all("top_comments" not in item for item in result))+ fetch.assert_not_called()def test_empty_items_returns_immediately(self):result = enrich_with_comments([], "fake-token", depth="default", budget_seconds=60)diff --git a/tests/test_subproc.py b/tests/test_subproc.pyindex 9f2d603a..f2106c23 100644--- a/tests/test_subproc.py+++ b/tests/test_subproc.py@@ -5,6 +5,7 @@ PID callback wiring, and environment inheritance."""import builtins+import errnoimport os as real_osimport platformimport unittest@@ -35,10 +36,28 @@ def get_shell_cmd(cmd_str: str) -> list[str]:return ["sh", "-c", cmd_str]+class TestSubprocTimeout(unittest.TestCase):+ def test_exception_class_raises_without_a_message(self):+ with self.assertRaises(subproc.SubprocTimeout) as caught:+ raise subproc.SubprocTimeout+ self.assertEqual(str(caught.exception), "")+ self.assertTrue(caught.exception.started)++class TestRunWithTimeout(unittest.TestCase):def setUp(self):self.addCleanup(setattr, subproc, "_shutting_down", False)+ def test_spawn_errors_preserve_original_type_and_errno(self):+ for error_number in (errno.ENOENT, errno.EAGAIN, errno.EMFILE):+ error = OSError(error_number, "command could not start")+ with self.subTest(error_number=error_number), \+ patch.object(subproc.subprocess, "Popen", side_effect=error):+ with self.assertRaises(type(error)) as caught:+ subproc.run_with_timeout(["missing-command"], timeout=1)+ self.assertIs(caught.exception, error)+ self.assertEqual(caught.exception.errno, error_number)+def test_success_returns_stdout(self):result = subproc.run_with_timeout(get_shell_cmd("echo hello"),@@ -405,8 +424,9 @@ time.sleep(15)def __init__(self):self.pid = 4321self.kill_count = 0+ self.stdin = self.stdout = self.stderr = None- def communicate(self, timeout=None):+ def communicate(self, timeout=None, input=None):raise TimeoutExpired(cmd="x", timeout=timeout)def wait(self, timeout=None):