Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

fix: bound doctor and Reddit transport lifetimes (#1229)

Ilia Alshanetsky7f582ad8c2e1
diff --git a/changelog.d/+test-audit-deadlines.fixed.md b/changelog.d/+test-audit-deadlines.fixed.md
new file mode 100644
index 00000000..fdc9a890
--- /dev/null
+++ b/changelog.d/+test-audit-deadlines.fixed.md
@@ -0,0 +1,5 @@
+Doctor source probes and Reddit comment enrichment now stop stalled HTTP reads within their operation budgets and reap the transport workers. Reddit keeps comments that completed in time, and a zero enrichment budget starts no requests.
+
+GET worker launch failures no longer leave unknown API-spend records for requests that were never delivered.
+
+Doctor preserves completed Reddit HTTP 429 responses as unverified when the remaining probe budget cannot fit a retry.
diff --git a/skills/last30days/scripts/lib/bounded_get.py b/skills/last30days/scripts/lib/bounded_get.py
new file mode 100644
index 00000000..e1194014
--- /dev/null
+++ b/skills/last30days/scripts/lib/bounded_get.py
@@ -0,0 +1,156 @@
+"""Owned GET transport for operation deadlines, including DNS and body reads."""
+
+from __future__ import annotations
+
+import base64
+from email.message import Message
+import io
+import json
+import socket
+import ssl
+import sys
+import threading
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+if __package__ in (None, ""):
+ sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from lib import subproc
+
+# Reserve TERM and KILL/reap time inside the caller's operation budget.
+CLEANUP_GRACE_SECONDS = 0.1
+
+
+class GetTimeout(TimeoutError):
+ def __init__(self, message, *, started):
+ super().__init__(message)
+ self.started = started
+
+
+class GetLaunchError(OSError):
+ """GET worker launch failed before the worker received a request."""
+
+
+def _error_record(exc):
+ if isinstance(exc, urllib.error.URLError):
+ reason = exc.reason
+ return {"kind": "URLError", "reason": _error_record(reason) if isinstance(reason, BaseException) else str(reason)}
+ return {"kind": type(exc).__name__, "message": str(exc), "errno": getattr(exc, "errno", None)}
+
+
+def _error_from_record(record):
+ if record["kind"] == "URLError":
+ reason = record["reason"]
+ return urllib.error.URLError(_error_from_record(reason) if isinstance(reason, dict) else reason)
+ kinds = {
+ "gaierror": socket.gaierror,
+ "TimeoutError": TimeoutError,
+ "ConnectionResetError": ConnectionResetError,
+ "ConnectionRefusedError": ConnectionRefusedError,
+ "OSError": OSError,
+ "SSLError": ssl.SSLError,
+ "SSLCertVerificationError": ssl.SSLCertVerificationError,
+ }
+ kind = kinds.get(record["kind"], OSError)
+ if record.get("errno") is not None:
+ return kind(record["errno"], record["message"])
+ return kind(record["message"])
+
+
+def get(
+ req: urllib.request.Request,
+ *,
+ timeout: float,
+ deadline_monotonic: float,
+ cancel: threading.Event | None = None,
+ read_body: bool = True,
+ read_error_body: bool = True,
+):
+ if req.get_method() != "GET":
+ raise ValueError("bounded transport supports GET only")
+ payload = {
+ "url": req.full_url,
+ "headers": req.headers,
+ "unredirected_headers": req.unredirected_hdrs,
+ "origin_req_host": req.origin_req_host,
+ "timeout": timeout,
+ "read_body": read_body,
+ "read_error_body": read_error_body,
+ }
+ transport_deadline = deadline_monotonic - 2 * CLEANUP_GRACE_SECONDS
+ try:
+ result = subproc.run_with_timeout(
+ [sys.executable, "-I", str(Path(__file__).resolve())],
+ timeout=max(0, transport_deadline - time.monotonic()),
+ deadline_monotonic=transport_deadline,
+ input_text=json.dumps(payload),
+ cancel=cancel,
+ cleanup_grace=CLEANUP_GRACE_SECONDS,
+ )
+ except subproc.SubprocTimeout as exc:
+ raise GetTimeout("GET exceeded operation deadline or was cancelled", started=exc.started) from exc
+ except OSError as exc:
+ if not getattr(exc, "_last30days_subproc_launch_failed", False):
+ raise
+ raise GetLaunchError(*exc.args) from exc
+ if result.returncode != 0:
+ raise OSError("bounded GET worker failed")
+ record = json.loads(result.stdout)
+ if "error" in record:
+ raise _error_from_record(record["error"])
+ body = base64.b64decode(record["body"]) if record["body"] is not None else None
+ error = None
+ if record.get("http_error") is not None:
+ headers = Message()
+ for key, value in record["headers"]:
+ headers[key] = value
+ error = urllib.error.HTTPError(
+ record["url"], record["status"], record["http_error"],
+ headers, io.BytesIO(body or b""),
+ )
+ return record["status"], body, error
+
+
+def _main():
+ from lib import http
+
+ payload = json.load(sys.stdin)
+ req = urllib.request.Request(
+ payload["url"], headers=payload["headers"], method="GET",
+ origin_req_host=payload["origin_req_host"],
+ )
+ for key, value in payload["unredirected_headers"].items():
+ req.add_unredirected_header(key, value)
+ error_reason = None
+ try:
+ try:
+ response = http.open_request(req, payload["timeout"])
+ except urllib.error.HTTPError as exc:
+ response = exc
+ error_reason = str(exc.reason)
+ with response:
+ body = None
+ read_body = payload["read_error_body"] if error_reason is not None else payload["read_body"]
+ if read_body:
+ try:
+ body = response.read()
+ except OSError:
+ if error_reason is None:
+ raise
+ result = {
+ "status": getattr(response, "status", 200) or 200,
+ "body": base64.b64encode(body).decode("ascii") if body is not None else None,
+ "headers": list(response.headers.items()),
+ "url": response.geturl(),
+ "http_error": error_reason,
+ }
+ except Exception as exc:
+ result = {"error": _error_record(exc)}
+ json.dump(result, sys.stdout)
+
+
+if __name__ == "__main__":
+ _main()
diff --git a/skills/last30days/scripts/lib/doctor.py b/skills/last30days/scripts/lib/doctor.py
index 45009225..1a0a3d69 100644
--- a/skills/last30days/scripts/lib/doctor.py
+++ b/skills/last30days/scripts/lib/doctor.py
@@ -1882,6 +1882,7 @@ def _http_ok(
blocked_statuses: frozenset = frozenset(),
headers: Optional[Dict[str, str]] = None,
body_check: Optional[Callable[[str], Optional[str]]] = None,
+ deadline_monotonic: Optional[float] = None,
) -> tuple:
"""Reachability check: a 4xx still means the endpoint responded; 5xx or a
connection/timeout error means it did not.
@@ -1899,6 +1900,20 @@ def _http_ok(
req = urllib.request.Request(
url, headers=headers or {"User-Agent": "last30days-doctor"}
)
+ if deadline_monotonic is not None:
+ from . import bounded_get
+
+ code, body, error = bounded_get.get(
+ req, timeout=timeout, deadline_monotonic=deadline_monotonic,
+ read_body=body_check is not None, read_error_body=False,
+ )
+ if error is not None:
+ return _verdict(error.code)
+ if body_check is not None and code < 300:
+ reason = body_check((body or b"").decode("utf-8", errors="replace"))
+ if reason:
+ return False, f"HTTP {code} but blocked: {reason}"
+ return _verdict(code)
with urllib.request.urlopen(req, timeout=timeout) as resp:
code = getattr(resp, "status", 200) or 200
if body_check is not None and code < 300:
@@ -1910,6 +1925,10 @@ def _http_ok(
except urllib.error.HTTPError as exc:
return _verdict(exc.code)
except Exception as exc:
+ from . import bounded_get
+
+ if isinstance(exc, bounded_get.GetTimeout):
+ return False, "probe exceeded deadline"
return False, f"{type(exc).__name__}: {exc}"
@@ -1926,6 +1945,10 @@ def _transient_probe_detail(name: str, detail: str) -> bool:
def _probe_source(name: str, config: Dict[str, Any], timeout: float) -> Optional[Dict[str, Any]]:
+ deadline = time.monotonic() + timeout
+ expired = {"ok": False, "detail": "probe exceeded deadline", "probed": True}
+ if timeout <= 0:
+ return expired
url = _HTTP_PROBE_URLS.get(name)
if url:
blocked = _PROBE_BLOCKED_STATUSES.get(name, frozenset())
@@ -1934,11 +1957,29 @@ def _probe_source(name: str, config: Dict[str, Any], timeout: float) -> Optional
"blocked_statuses": blocked,
"headers": headers,
"body_check": _PROBE_BODY_CHECKS.get(name),
+ "deadline_monotonic": deadline,
}
ok, detail = _http_ok(url, timeout, **probe_kwargs)
+ if time.monotonic() >= deadline or detail == "probe exceeded deadline":
+ return expired
if not ok and _transient_probe_detail(name, detail):
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ return expired
+ if remaining <= _PROBE_RETRY_DELAY_SECONDS:
+ return {
+ "ok": False,
+ "transient": True,
+ "detail": f"{detail} (retry skipped: insufficient probe budget)",
+ "probed": True,
+ }
time.sleep(_PROBE_RETRY_DELAY_SECONDS)
- ok, detail = _http_ok(url, timeout, **probe_kwargs)
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ return expired
+ ok, detail = _http_ok(url, remaining, **probe_kwargs)
+ if time.monotonic() >= deadline or detail == "probe exceeded deadline":
+ return expired
if not ok and _transient_probe_detail(name, detail):
return {
"ok": False,
@@ -1950,7 +1991,9 @@ def _probe_source(name: str, config: Dict[str, Any], timeout: float) -> Optional
cli = CLI_DEPENDENCIES.get(name)
if cli:
try:
- probe = health.probe_dependency(cli)
+ probe = health.probe_dependency(cli, timeout=max(0, deadline - time.monotonic()))
+ if time.monotonic() >= deadline:
+ return expired
except Exception as exc:
return {"ok": False, "detail": f"{type(exc).__name__}: {exc}", "probed": True}
return {"ok": bool(probe.ok), "detail": probe.detail, "probed": True}
@@ -1973,7 +2016,7 @@ def _probe_sources(config: Dict[str, Any], timeout: int) -> Dict[str, Dict[str,
}
for name, fut in futures.items():
try:
- res = fut.result(timeout=timeout + 1)
+ res = fut.result()
except concurrent.futures.TimeoutError:
res = {"ok": False, "detail": "probe exceeded deadline", "probed": True}
except Exception as exc:
diff --git a/skills/last30days/scripts/lib/health.py b/skills/last30days/scripts/lib/health.py
index 69f842c4..f44abdc3 100644
--- a/skills/last30days/scripts/lib/health.py
+++ b/skills/last30days/scripts/lib/health.py
@@ -16,10 +16,13 @@ from __future__ import annotations
import os
import shutil
import subprocess
+import time
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Tuple
+from . import subproc
+
# Health states, best to worst.
OK = "ok"
DEGRADED = "degraded" # ran, but returned less than expected
@@ -388,6 +391,7 @@ def probe_dependency(name: str, timeout: float = PROBE_TIMEOUT) -> DependencyPro
def _probe_dependency_uncached(name: str, timeout: float) -> DependencyProbe:
+ deadline = time.monotonic() + timeout
resolved = shutil.which(name)
if resolved is None:
off_path = _off_path_binary(name)
@@ -412,11 +416,13 @@ def _probe_dependency_uncached(name: str, timeout: float) -> DependencyProbe:
command = [name] + _VERSION_ARGS.get(name, ["--version"])
try:
- proc = subprocess.run(
+ cleanup_grace = 0.1
+ proc = subproc.run_with_timeout(
command,
- capture_output=True,
- text=True,
- timeout=timeout,
+ timeout=max(0, deadline - time.monotonic() - 2 * cleanup_grace),
+ deadline_monotonic=deadline - 2 * cleanup_grace,
+ cleanup_grace=cleanup_grace,
+ capture_limit_bytes=64 * 1024,
)
except (FileNotFoundError, OSError) as exc:
prescription, manager = _prescription(name, "reinstall")
@@ -427,7 +433,7 @@ def _probe_dependency_uncached(name: str, timeout: float) -> DependencyProbe:
prescription=prescription,
owner_pkg_manager=manager,
)
- except subprocess.TimeoutExpired:
+ except subproc.SubprocTimeout:
prescription, manager = _prescription(name, "reinstall")
return DependencyProbe(
name=name,
diff --git a/skills/last30days/scripts/lib/http.py b/skills/last30days/scripts/lib/http.py
index 773cb35d..f09fb543 100644
--- a/skills/last30days/scripts/lib/http.py
+++ b/skills/last30days/scripts/lib/http.py
@@ -18,7 +18,7 @@ from pathlib import Path
from typing import Any, Callable, Dict, Optional, Union
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit, quote
-from . import health, usage
+from . import bounded_get, health, usage
from . import log as _log
DEFAULT_TIMEOUT = 30
@@ -776,6 +776,7 @@ def request(
raw: bool = False,
deadline_monotonic: float | None = None,
cancel: threading.Event | None = None,
+ owned_get: bool = False,
) -> Union[Dict[str, Any], str]:
"""Make an HTTP request and return JSON response.
@@ -792,6 +793,8 @@ def request(
raw: If True, return raw response text instead of parsed JSON
deadline_monotonic: Optional absolute monotonic deadline shared by all
attempts and retry delays.
+ owned_get: Run GET transport in a child that is killed and reaped at
+ its deadline. Requires deadline_monotonic; other methods are refused.
Returns:
Parsed JSON response as dict, or raw text string if raw=True.
@@ -799,6 +802,10 @@ def request(
Raises:
HTTPError: On request failure
"""
+ if owned_get and (method != "GET" or json_data is not None):
+ raise ValueError("owned transport requires a bodyless GET")
+ if owned_get and deadline_monotonic is None:
+ raise ValueError("owned GET requires an operation deadline")
headers = headers or {}
headers.setdefault("User-Agent", USER_AGENT)
@@ -892,6 +899,26 @@ def request(
request_timeout: float,
) -> tuple[int, str | None, urllib.error.HTTPError | None]:
"""Stop waiting at the wall deadline, even during DNS or body reads."""
+ if owned_get:
+ try:
+ status, body, error = bounded_get.get(
+ req, timeout=request_timeout,
+ deadline_monotonic=deadline_monotonic, cancel=cancel,
+ )
+ except bounded_get.GetLaunchError:
+ usage.cancel(charge)
+ raise
+ except bounded_get.GetTimeout as exc:
+ if not exc.started:
+ usage.cancel(charge)
+ raise deadline_error() from exc
+ try:
+ decoded = body.decode("utf-8") if body is not None else None
+ except UnicodeDecodeError:
+ if error is None:
+ raise
+ decoded = None
+ return status, decoded, error
if deadline_monotonic is None and cancel is None:
return open_and_read(request_timeout)
wait_deadline = deadline_monotonic or (time.monotonic() + request_timeout)
diff --git a/skills/last30days/scripts/lib/reddit.py b/skills/last30days/scripts/lib/reddit.py
index e56b5d92..1f9cf7cc 100644
--- a/skills/last30days/scripts/lib/reddit.py
+++ b/skills/last30days/scripts/lib/reddit.py
@@ -13,6 +13,7 @@ import re
import sys
import threading
import time
+from contextlib import ExitStack
from collections import Counter
from concurrent.futures import ThreadPoolExecutor, as_completed, wait as futures_wait
from datetime import date, datetime, timezone
@@ -404,6 +405,9 @@ def _subreddit_search(
def fetch_post_comments(
url: str,
token: str,
+ *,
+ deadline_monotonic: float | None = None,
+ cancel: threading.Event | None = None,
) -> List[Dict[str, Any]]:
"""Fetch comments for a Reddit post via ScrapeCreators.
@@ -421,6 +425,9 @@ def fetch_post_comments(
params={"url": url},
timeout=30,
retries=2,
+ deadline_monotonic=deadline_monotonic,
+ cancel=cancel,
+ owned_get=deadline_monotonic is not None,
)
return data.get("comments", data.get("data", []))
except http.HTTPError as e:
@@ -669,7 +676,7 @@ def enrich_with_comments(
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
max_comments = config["comment_enrichments"]
- if not items or not token or max_comments <= 0:
+ if not items or not token or max_comments <= 0 or budget_seconds <= 0:
return items
# Select the top threads by total engagement (upvotes + comment count),
@@ -680,11 +687,23 @@ def enrich_with_comments(
_log(f"Enriching comments for {len(top_items)} posts (by total engagement)")
start = time.monotonic()
+ deadline = start + budget_seconds
+ cancel = threading.Event()
+
+ def fetch(item):
+ if cancel.is_set() or time.monotonic() >= deadline:
+ return None
+ comments = fetch_post_comments(
+ item.get("url", ""), token,
+ deadline_monotonic=deadline, cancel=cancel,
+ )
+ return comments if time.monotonic() < deadline else None
- with ThreadPoolExecutor(max_workers=min(4, len(top_items))) as executor:
+ with ThreadPoolExecutor(max_workers=min(4, len(top_items))) as executor, ExitStack() as cleanup:
+ cleanup.callback(cancel.set)
futures = {
http.submit_with_context(
- executor, fetch_post_comments, item.get("url", ""), token,
+ executor, fetch, item,
): item
for item in top_items
if item.get("url")
@@ -693,6 +712,8 @@ def enrich_with_comments(
# Wait with budget instead of unbounded as_completed
remaining = max(0, budget_seconds - (time.monotonic() - start))
done, not_done = futures_wait(futures, timeout=remaining)
+ if not_done:
+ cancel.set()
enriched_count = 0
for future in done:
diff --git a/skills/last30days/scripts/lib/subproc.py b/skills/last30days/scripts/lib/subproc.py
index 0716f1f4..a2d693e1 100644
--- a/skills/last30days/scripts/lib/subproc.py
+++ b/skills/last30days/scripts/lib/subproc.py
@@ -10,8 +10,10 @@ from __future__ import annotations
import os
import signal
import subprocess
+import tempfile
import threading
import time
+from contextlib import ExitStack
from dataclasses import dataclass
from typing import Optional, Sequence
@@ -19,6 +21,10 @@ from typing import Optional, Sequence
class SubprocTimeout(Exception):
"""Raised when a subprocess exceeds its timeout and is killed."""
+ def __init__(self, message: str = "", *, started: bool = True):
+ super().__init__(message)
+ self.started = started
+
# Live run_with_timeout children, process-wide. Each child is a session
# leader (os.setsid), so a group kill aimed at the engine never reaches it;
@@ -122,17 +128,22 @@ class SubprocResult:
def run_with_timeout(
cmd: Sequence[str],
*,
- timeout: int,
+ timeout: float,
env: Optional[dict] = None,
on_pid: Optional[callable] = None,
+ input_text: Optional[str] = None,
+ deadline_monotonic: Optional[float] = None,
+ cancel: Optional[threading.Event] = None,
+ cleanup_grace: float = 5.0,
+ capture_limit_bytes: Optional[int] = None,
) -> SubprocResult:
"""Run a subprocess with process-group cleanup on timeout.
- Spawns ``cmd`` inside its own process group via ``os.setsid`` where
+ Spawns ``cmd`` inside its own process group via ``start_new_session`` where
available. If ``communicate(timeout=...)`` raises ``TimeoutExpired``,
signals ``SIGTERM`` to the entire group, falls back to ``proc.kill()``
- if the signal fails, then waits up to 5 seconds for cleanup, and
- raises ``SubprocTimeout``.
+ if the signal fails, then waits up to ``cleanup_grace`` seconds before
+ escalating to SIGKILL and waiting once more to reap the child.
Args:
cmd: Command and arguments to spawn.
@@ -141,43 +152,83 @@ def run_with_timeout(
on_pid: Optional callable invoked with the child PID right after
spawn. Exceptions raised by the callback are suppressed. The
child is registered for cleanup_children() regardless.
+ input_text: Optional UTF-8 text supplied on stdin, never argv.
+ deadline_monotonic: Absolute operation deadline, including startup.
+ cancel: Optional event checked while waiting for the command.
+ cleanup_grace: Maximum TERM wait and subsequent KILL/reap wait, each.
+ capture_limit_bytes: Capture to temporary files instead of pipes,
+ reading at most this many bytes from each stream after exit.
Returns:
SubprocResult with returncode, stdout, and stderr as strings.
Raises:
- SubprocTimeout: If the process exceeded ``timeout``.
+ SubprocTimeout: If the process timed out, was cancelled, or its
+ absolute deadline elapsed.
FileNotFoundError: If the executable is not found.
OSError: For other spawn failures.
"""
- preexec = os.setsid if hasattr(os, "setsid") else None
-
- proc = subprocess.Popen(
- list(cmd),
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- text=True,
- encoding="utf-8",
- errors="replace",
- preexec_fn=preexec,
- env=env,
- )
-
- if on_pid is not None:
- try:
- on_pid(proc.pid)
- except Exception:
- pass
-
- register_child_pid(proc.pid)
+ if deadline_monotonic is not None and time.monotonic() >= deadline_monotonic:
+ raise SubprocTimeout("Command deadline exceeded before spawn", started=False)
+ if cancel is not None and cancel.is_set():
+ raise SubprocTimeout("Command cancelled before spawn", started=False)
+ if capture_limit_bytes is not None and capture_limit_bytes < 0:
+ raise ValueError("capture limit must be nonnegative")
+ own_group = hasattr(os, "setsid") and hasattr(os, "killpg")
+ capture = ExitStack()
+ try:
+ stdout_file = capture.enter_context(tempfile.TemporaryFile()) if capture_limit_bytes is not None else None
+ stderr_file = capture.enter_context(tempfile.TemporaryFile()) if capture_limit_bytes is not None else None
+ proc = subprocess.Popen(
+ list(cmd),
+ stdout=stdout_file if stdout_file is not None else subprocess.PIPE,
+ stderr=stderr_file if stderr_file is not None else subprocess.PIPE,
+ stdin=subprocess.PIPE if input_text is not None else None,
+ text=True,
+ encoding="utf-8",
+ errors="replace",
+ start_new_session=own_group,
+ env=env,
+ )
+ except BaseException as exc:
+ if isinstance(exc, OSError):
+ exc._last30days_subproc_launch_failed = True
+ capture.close()
+ raise
try:
+ register_child_pid(proc.pid)
+ if on_pid is not None:
+ try:
+ on_pid(proc.pid)
+ except Exception:
+ pass
try:
- stdout, stderr = proc.communicate(timeout=timeout)
+ deadline = time.monotonic() + timeout
+ if deadline_monotonic is not None:
+ deadline = min(deadline, deadline_monotonic)
+ first = True
+ while True:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0 or (cancel is not None and cancel.is_set()):
+ raise subprocess.TimeoutExpired(cmd, timeout)
+ try:
+ stdout, stderr = proc.communicate(
+ input=input_text if first else None,
+ timeout=min(remaining, 0.05) if cancel is not None else remaining,
+ )
+ if deadline_monotonic is not None and time.monotonic() >= deadline_monotonic:
+ raise subprocess.TimeoutExpired(cmd, timeout)
+ break
+ except subprocess.TimeoutExpired:
+ if cancel is None or time.monotonic() >= deadline or cancel.is_set():
+ raise
+ finally:
+ first = False
except subprocess.TimeoutExpired:
- term_deadline = time.monotonic() + 5
+ term_deadline = time.monotonic() + cleanup_grace
pgid = None
try:
- if preexec is not None and hasattr(os, "killpg"):
+ if own_group:
pgid = proc.pid
os.killpg(pgid, signal.SIGTERM)
else:
@@ -186,7 +237,7 @@ def run_with_timeout(
pgid = None
proc.kill()
try:
- proc.wait(timeout=5)
+ proc.wait(timeout=cleanup_grace)
except subprocess.TimeoutExpired:
# Child ignored SIGTERM (or our killpg lost the race); escalate.
try:
@@ -197,7 +248,7 @@ def run_with_timeout(
except (ProcessLookupError, PermissionError, OSError, AttributeError):
proc.kill()
try:
- proc.wait(timeout=5)
+ proc.wait(timeout=cleanup_grace)
except subprocess.TimeoutExpired:
pass # process unkillable (e.g. D-state); leave as zombie
else:
@@ -211,8 +262,27 @@ def run_with_timeout(
break
time.sleep(min(_CLEANUP_POLL_SECONDS, remaining))
raise SubprocTimeout(f"Command {cmd[0]} timed out after {timeout}s")
+ if capture_limit_bytes is not None:
+ stdout_file.seek(0)
+ stderr_file.seek(0)
+ stdout = stdout_file.read(capture_limit_bytes).decode("utf-8", errors="replace")
+ stderr = stderr_file.read(capture_limit_bytes).decode("utf-8", errors="replace")
+ except SubprocTimeout:
+ raise
+ except BaseException:
+ if proc.poll() is None:
+ _kill_child_group(proc.pid, getattr(signal, "SIGKILL", signal.SIGTERM))
+ try:
+ proc.wait(timeout=cleanup_grace)
+ except subprocess.TimeoutExpired:
+ pass
+ raise
finally:
unregister_child_pid(proc.pid)
+ for pipe in (proc.stdin, proc.stdout, proc.stderr):
+ if pipe is not None:
+ pipe.close()
+ capture.close()
return SubprocResult(
returncode=proc.returncode,
diff --git a/tests/test_deadline_lifecycle.py b/tests/test_deadline_lifecycle.py
new file mode 100644
index 00000000..8c58bed2
--- /dev/null
+++ b/tests/test_deadline_lifecycle.py
@@ -0,0 +1,607 @@
+import errno
+import json
+import os
+import socket
+import signal
+import sqlite3
+import sys
+import threading
+import time
+from types import SimpleNamespace
+import urllib.error
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.parse import parse_qs, urlencode, urlsplit
+from unittest.mock import patch
+
+import pytest
+
+from lib import bounded_get, doctor, health, http, reddit, subproc, usage
+
+
+def children(monkeypatch):
+ processes = []
+ popen = subproc.subprocess.Popen
+
+ def start(*args, **kwargs):
+ process = popen(*args, **kwargs)
+ processes.append(process)
+ return process
+
+ monkeypatch.setattr(subproc.subprocess, "Popen", start)
+ yield processes
+ assert processes
+ assert all(process.poll() is not None for process in processes)
+ assert all(process.pid not in subproc._child_pids for process in processes)
+
+
+def local_http():
+ started = threading.Event()
+ release = threading.Event()
+ finished = threading.Event()
+
+ class Handler(BaseHTTPRequestHandler):
+ def log_message(self, *args):
+ pass
+
+ def do_GET(self):
+ parsed = urlsplit(self.path)
+ query = parse_qs(parsed.query)
+ if parsed.path == "/redirect":
+ self.send_response(302)
+ self.send_header("Location", query["to"][0])
+ self.end_headers()
+ return
+ status = int(parsed.path.rsplit("/", 1)[1]) if parsed.path.startswith("/status/") else 200
+ if parsed.path == "/held-error":
+ status = 403
+ if query.get("url") == ["failed"]:
+ status = 402
+ self.send_response(status)
+ self.send_header("Retry-After", "0")
+ self.end_headers()
+ hold = parsed.path in ("/hold", "/held-error") or query.get("url") == ["slow"]
+ if hold:
+ started.set()
+ try:
+ while not release.wait(0.02):
+ self.wfile.write(b" ")
+ self.wfile.flush()
+ except (BrokenPipeError, ConnectionResetError):
+ pass
+ finally:
+ finished.set()
+ return
+ if parsed.path == "/echo":
+ self.wfile.write(json.dumps(dict(self.headers)).encode())
+ return
+ if status != 200:
+ self.wfile.write(b'{"error":"credits exhausted"}')
+ return
+ self.wfile.write(json.dumps({"comments": [{
+ "body": "The fast comment survives.", "score": 17, "author": "alice",
+ }]}).encode())
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ server.daemon_threads = True
+ thread = threading.Thread(target=server.serve_forever)
+ thread.start()
+ try:
+ yield f"http://127.0.0.1:{server.server_port}", started, release, finished
+ finally:
+ release.set()
+ if started.is_set():
+ assert finished.wait(2), "fixture request did not finish"
+ server.shutdown()
+ server.server_close()
+ thread.join(2)
+ assert not thread.is_alive()
+
+
+def test_doctor_returns_before_held_response_is_released(local_http, children):
+ url, started, release, _ = local_http
+ result = {}
+ returned = threading.Event()
+
+ def run():
+ result.update(doctor._probe_sources({}, timeout=0.8))
+ returned.set()
+
+ with patch.object(doctor, "_probeable_sources", return_value=("reddit",)), \
+ patch.dict(doctor._HTTP_PROBE_URLS, {"reddit": f"{url}/hold"}):
+ caller = threading.Thread(target=run)
+ caller.start()
+ try:
+ assert started.wait(2), "real HTTP body read was not reached"
+ assert any(process.poll() is None for process in children)
+ assert returned.wait(1.0), "doctor waited for the stalled response"
+ assert all(process.poll() is not None for process in children), "owned HTTP worker survived timeout"
+ assert result["reddit"] == {
+ "ok": False, "detail": "probe exceeded deadline", "probed": True,
+ }
+ assert doctor.audit_state("reddit", {"tier": doctor.TIER_OK},
+ probe_result=result["reddit"]) == doctor.AUDIT_NOT_WORKING
+ finally:
+ release.set()
+ caller.join(3)
+ assert not caller.is_alive()
+
+
[email protected]("status,expected_audit", [
+ (429, doctor.AUDIT_UNVERIFIED),
+ (403, doctor.AUDIT_NOT_WORKING),
+])
+def test_short_doctor_budget_preserves_completed_reddit_status(local_http, children, status, expected_audit):
+ url, _, _, _ = local_http
+ with patch.dict(doctor._HTTP_PROBE_URLS, {"reddit": f"{url}/status/{status}"}), \
+ patch.object(doctor.time, "sleep") as sleep:
+ result = doctor._probe_source("reddit", {}, 1)
+ assert not result["ok"]
+ assert result["probed"]
+ assert result["detail"].startswith(f"HTTP {status}")
+ assert result.get("transient", False) is (status == 429)
+ assert doctor.audit_state("reddit", {"tier": doctor.TIER_OK},
+ probe_result=result) == expected_audit
+ assert len(children) == 1
+ sleep.assert_not_called()
+
+
+def test_reddit_keeps_fast_comments_and_reaps_stalled_transport(local_http, children, monkeypatch):
+ url, started, release, _ = local_http
+ monkeypatch.setattr(reddit, "SCRAPECREATORS_BASE", url)
+ items = [
+ {"id": "fast", "url": "fast", "engagement": {"score": 10}},
+ {"id": "slow", "url": "slow", "engagement": {"score": 20}},
+ ]
+ returned = threading.Event()
+ result = []
+
+ def run():
+ result.extend(reddit.enrich_with_comments(items, "dummy-key", depth="quick", budget_seconds=1.2))
+ returned.set()
+
+ caller = threading.Thread(target=run)
+ caller.start()
+ try:
+ assert started.wait(2), "real HTTP body read was not reached"
+ assert returned.wait(1.8), "Reddit waited for the stalled response"
+ assert len(result) == 2
+ assert result[0]["top_comments"][0]["excerpt"] == "The fast comment survives."
+ assert result[0]["top_comments"][0]["score"] == 17
+ assert "top_comments" not in result[1]
+ assert all(process.poll() is not None for process in children)
+ before_release = json.dumps(result, sort_keys=True)
+ finally:
+ release.set()
+ caller.join(3)
+ assert not caller.is_alive()
+ assert json.dumps(result, sort_keys=True) == before_release
+
+
[email protected]("path,body_check,expected", [
+ ("/hold", None, True),
+ ("/held-error", lambda body: "must not run", True),
+])
+def test_doctor_header_only_modes_do_not_read_bodies(local_http, children, path, body_check, expected):
+ url, _, _, _ = local_http
+ result = doctor._http_ok(f"{url}{path}", 2, body_check=body_check, deadline_monotonic=time.monotonic() + 2)
+ assert result == (expected, "HTTP 403" if path == "/held-error" else "HTTP 200")
+
+
[email protected]("status,state", [(401, health.AUTH_FAILED), (402, health.PAYMENT_REQUIRED), (429, health.RATE_LIMITED)])
+def test_owned_get_preserves_error_body_and_failure_capture(local_http, children, status, state):
+ url, _, _, _ = local_http
+ with http.capture_failures() as failures:
+ with pytest.raises(http.HTTPError) as caught:
+ http.get(f"{url}/status/{status}", retries=1, max_429_retries=1,
+ deadline_monotonic=time.monotonic() + 2, owned_get=True)
+ assert caught.value.status_code == status
+ assert caught.value.body == '{"error":"credits exhausted"}'
+ assert caught.value.outcome_state == state
+ assert failures == [caught.value]
+
+
[email protected]("cross_origin", [False, True])
+def test_owned_get_preserves_redirect_auth_policy(local_http, children, cross_origin):
+ url, _, _, _ = local_http
+ target = url.replace("127.0.0.1", "localhost") if cross_origin else url
+ endpoint = f"{url}/redirect?{urlencode({'to': target + '/echo'})}"
+ result = http.get(endpoint, headers={"Authorization": "Bearer dummy", "x-api-key": "dummy-key"},
+ deadline_monotonic=time.monotonic() + 3, owned_get=True)
+ lowered = {key.lower(): value for key, value in result.items()}
+ if cross_origin:
+ assert "authorization" not in lowered
+ assert "x-api-key" not in lowered
+ else:
+ assert lowered["authorization"] == "Bearer dummy"
+ assert lowered["x-api-key"] == "dummy-key"
+
+
+def test_owned_get_cancellation_reaps_transport(local_http, children):
+ url, started, release, _ = local_http
+ cancel = threading.Event()
+ returned = threading.Event()
+ caught = []
+
+ def run():
+ try:
+ bounded_get.get(urllib.request.Request(f"{url}/hold"), timeout=5,
+ deadline_monotonic=time.monotonic() + 5, cancel=cancel)
+ except bounded_get.GetTimeout as exc:
+ caught.append(exc)
+ finally:
+ returned.set()
+
+ caller = threading.Thread(target=run)
+ caller.start()
+ try:
+ assert started.wait(2)
+ cancel.set()
+ assert returned.wait(1)
+ assert len(caught) == 1
+ assert all(process.poll() is not None for process in children)
+ finally:
+ release.set()
+ caller.join(3)
+ assert not caller.is_alive()
+
+
+def test_stalled_error_body_is_reaped_and_captured_as_timeout(local_http, children):
+ url, started, release, _ = local_http
+ with http.capture_failures() as failures:
+ try:
+ with pytest.raises(http.DeadlineExceeded) as caught:
+ http.get(f"{url}/held-error", retries=1,
+ deadline_monotonic=time.monotonic() + 0.8, owned_get=True)
+ assert started.is_set(), "real HTTP error body read was not reached"
+ assert caught.value.outcome_state == health.TIMEOUT
+ assert failures == [caught.value]
+ assert all(process.poll() is not None for process in children)
+ finally:
+ release.set()
+
+
+def test_reddit_comment_worker_preserves_parent_failure_capture(local_http, children, monkeypatch):
+ url, _, _, _ = local_http
+ monkeypatch.setattr(reddit, "SCRAPECREATORS_BASE", url)
+ items = [{"id": "failed", "url": "failed", "engagement": {"score": 10}}]
+ with http.capture_failures() as failures:
+ result = reddit.enrich_with_comments(items, "dummy-key", budget_seconds=2)
+ assert result is items
+ assert "top_comments" not in result[0]
+ assert len(failures) == 1
+ assert failures[0].status_code == 402
+ assert failures[0].outcome_state == health.PAYMENT_REQUIRED
+ assert failures[0].body == '{"error":"credits exhausted"}'
+
+
+def test_dns_error_wire_record_retains_retry_signal():
+ restored = bounded_get._error_from_record({
+ "kind": "URLError", "reason": {"kind": "gaierror", "errno": socket.EAI_AGAIN,
+ "message": "temporary DNS failure"},
+ })
+ assert isinstance(restored, urllib.error.URLError)
+ assert isinstance(restored.reason, socket.gaierror)
+ assert restored.reason.errno == socket.EAI_AGAIN
+ assert http._is_dns_failure(restored)
+
+
+def test_owned_get_replay_skips_child_and_request_accounting(local_http, children, tmp_path):
+ url, _, _, _ = local_http
+ fixture = tmp_path / "fixture"
+ with http.recording_requests(fixture):
+ recorded = http.get(f"{url}/echo", deadline_monotonic=time.monotonic() + 2, owned_get=True)
+ with patch.object(bounded_get, "get", side_effect=AssertionError("replay launched transport")), \
+ patch.object(usage, "begin_http") as begin, http.replaying_requests(fixture):
+ replayed = http.get(f"{url}/echo", deadline_monotonic=time.monotonic() + 2, owned_get=True)
+ assert replayed == recorded
+ begin.assert_not_called()
+
+
+def test_unstarted_owned_get_cancels_request_charge(local_http):
+ url, _, _, _ = local_http
+ charge = ("scrapecreators", "request", "dummy-attempt")
+ with patch.object(usage, "begin_http", return_value=charge) as begin, \
+ patch.object(usage, "cancel") as cancel, patch.object(subproc.subprocess, "Popen") as spawn:
+ with pytest.raises(http.DeadlineExceeded):
+ http.get(f"{url}/echo", deadline_monotonic=time.monotonic() + 0.1, owned_get=True)
+ begin.assert_called_once()
+ cancel.assert_called_once_with(charge)
+ spawn.assert_not_called()
+
+
+def paid_owned_get(local_http, tmp_path, monkeypatch):
+ """Keep real provider accounting while restricting worker destinations to loopback."""
+ local_url, _, _, _ = local_http
+ paid_url = "https://api.scrapecreators.com/v1/reddit/post"
+ journal = tmp_path / "usage.db"
+ usage.create_journal(journal)
+ monkeypatch.setenv(usage.JOURNAL_ENV, str(journal))
+ guarded_get = bounded_get.get
+
+ def local_transport(req, **kwargs):
+ assert req.full_url == paid_url
+ return guarded_get(urllib.request.Request(f"{local_url}/echo"), **kwargs)
+
+ monkeypatch.setattr(bounded_get, "get", local_transport)
+ monkeypatch.setattr(http.time, "sleep", lambda _: None)
+ return paid_url, journal
+
+
+def _pending_charges(journal):
+ with sqlite3.connect(journal) as connection:
+ return connection.execute("SELECT COUNT(*) FROM attempts WHERE unknown = 1").fetchone()[0]
+
+
[email protected]("error_number", [errno.ENOENT, errno.EAGAIN, errno.EMFILE])
+def test_owned_get_launch_error_preserves_errno_and_cause(local_http, monkeypatch, error_number):
+ local_url, _, _, _ = local_http
+ error = OSError(error_number, "GET worker launch failed")
+ with patch.object(subproc.subprocess, "Popen", side_effect=error):
+ with pytest.raises(bounded_get.GetLaunchError) as caught:
+ bounded_get.get(urllib.request.Request(f"{local_url}/echo"), timeout=2,
+ deadline_monotonic=time.monotonic() + 2)
+ assert caught.value.errno == error_number
+ assert caught.value.__cause__ is error
+
+
[email protected]("error_number", [errno.EAGAIN, errno.EMFILE])
+def test_owned_get_spawn_errors_cancel_each_paid_retry(paid_owned_get, monkeypatch, error_number):
+ url, journal = paid_owned_get
+ pending_before_spawn = []
+
+ def unavailable(*args, **kwargs):
+ pending_before_spawn.append(_pending_charges(journal))
+ raise OSError(error_number, "GET worker could not start")
+
+ monkeypatch.setattr(subproc.subprocess, "Popen", unavailable)
+ with http.capture_failures() as failures:
+ with pytest.raises(http.HTTPError) as caught:
+ http.get(url, retries=3, deadline_monotonic=time.monotonic() + 30, owned_get=True)
+
+ assert len(pending_before_spawn) == 3
+ assert _pending_charges(journal) == 0
+ assert pending_before_spawn == [1, 1, 1]
+ assert usage.read_journal(journal) == {
+ "token_cost": 0.0, "cost_unknown": 0, "prompt_tokens": 0, "completion_tokens": 0,
+ }
+ assert caught.value.outcome_state == health.UNREACHABLE
+ assert "GET worker could not start" in str(caught.value)
+ assert failures == [caught.value]
+
+
[email protected]("error_number", [errno.EAGAIN, errno.EMFILE])
[email protected]("stage", ["registration", "communication"])
+def test_owned_get_post_launch_errors_keep_unknown_paid_retries(paid_owned_get, children, monkeypatch, error_number, stage):
+ url, journal = paid_owned_get
+ pending_before_spawn = []
+ start = subproc.subprocess.Popen
+
+ def fail(*args, **kwargs):
+ raise OSError(error_number, "GET parent failed after launch")
+
+ def fail_after_start(*args, **kwargs):
+ pending_before_spawn.append(_pending_charges(journal))
+ process = start(*args, **kwargs)
+ if stage == "communication":
+ process.communicate = fail
+ return process
+
+ monkeypatch.setattr(subproc.subprocess, "Popen", fail_after_start)
+ if stage == "registration":
+ monkeypatch.setattr(subproc, "register_child_pid", fail)
+ with http.capture_failures() as failures:
+ with pytest.raises(http.HTTPError) as caught:
+ http.get(url, retries=3, deadline_monotonic=time.monotonic() + 30, owned_get=True)
+
+ assert len(children) == 3
+ assert pending_before_spawn == [1, 2, 3]
+ assert _pending_charges(journal) == 3
+ assert usage.read_journal(journal) == {
+ "token_cost": 0.0, "cost_unknown": 1, "prompt_tokens": 0, "completion_tokens": 0,
+ }
+ assert caught.value.outcome_state == health.UNREACHABLE
+ assert "GET parent failed after launch" in str(caught.value)
+ assert failures == [caught.value]
+
+
[email protected]("kwargs", [
+ {"method": "POST", "deadline_monotonic": 1},
+ {"method": "GET", "json_data": {"key": "value"}, "deadline_monotonic": 1},
+ {"method": "GET"},
+])
+def test_owned_transport_rejects_unsupported_calls_before_accounting(kwargs):
+ with patch.object(usage, "begin_http") as begin, patch.object(bounded_get, "get") as transport:
+ with pytest.raises(ValueError):
+ http.request(url="http://example.test", owned_get=True, **kwargs)
+ begin.assert_not_called()
+ transport.assert_not_called()
+
+
+def test_subprocess_without_process_groups_still_reaps_child(children, monkeypatch):
+ monkeypatch.setattr(subproc, "os", SimpleNamespace(kill=os.kill))
+ with pytest.raises(subproc.SubprocTimeout):
+ subproc.run_with_timeout([sys.executable, "-c", "import time; time.sleep(30)"],
+ timeout=0.05, cleanup_grace=0.1)
+ assert all(process.poll() is not None for process in children)
+
+
+def test_unexpected_parent_failure_reaps_owned_child(children, monkeypatch):
+ start = subproc.subprocess.Popen
+
+ def fail_after_start(*args, **kwargs):
+ process = start(*args, **kwargs)
+ def fail(**kwargs):
+ raise RuntimeError("parent communication failed")
+ process.communicate = fail
+ return process
+
+ monkeypatch.setattr(subproc.subprocess, "Popen", fail_after_start)
+ with pytest.raises(RuntimeError, match="parent communication failed"):
+ subproc.run_with_timeout([sys.executable, "-c", "import time; time.sleep(30)"],
+ timeout=2, cleanup_grace=0.1)
+ assert all(process.poll() is not None for process in children)
+
+
[email protected](not hasattr(os, "killpg"), reason="POSIX process-group termination")
+def test_health_timeout_terminates_inherited_output_descendant(tmp_path, children, monkeypatch):
+ descendant_pid = tmp_path / "descendant.pid"
+ stopped = tmp_path / "descendant.stopped"
+ child = (
+ "import os, signal, sys, time\nfrom pathlib import Path\n"
+ f"def stop(*args):\n Path({str(stopped)!r}).write_text('terminated')\n sys.exit(0)\n"
+ "signal.signal(signal.SIGTERM, stop)\n"
+ f"Path({str(descendant_pid)!r}).write_text(str(os.getpid()))\n"
+ "time.sleep(30)\n"
+ )
+ command = tmp_path / "version.py"
+ command.write_text(f"import subprocess, sys, time\nsubprocess.Popen([sys.executable, '-c', {child!r}])\ntime.sleep(30)\n")
+ monkeypatch.setitem(health._VERSION_ARGS, sys.executable, [str(command)])
+ started = time.monotonic()
+ try:
+ result = health._probe_dependency_uncached(sys.executable, 0.8)
+ assert time.monotonic() - started < 1.5
+ assert result.status == health.TIMEOUT
+ assert descendant_pid.exists(), "real descendant was not reached"
+ assert stopped.exists(), "timed-out version probe left its descendant running"
+ assert stopped.read_text() == "terminated"
+ assert all(process.poll() is not None for process in children)
+ finally:
+ if descendant_pid.exists() and not stopped.exists():
+ try:
+ os.kill(int(descendant_pid.read_text()), signal.SIGTERM)
+ except ProcessLookupError:
+ pass
+
+
+def test_health_real_version_command_preserves_first_line(children, monkeypatch):
+ monkeypatch.setitem(health._VERSION_ARGS, sys.executable, [
+ "-c", "import sys; print('1.2.3'); print('ignored second line'); print('stderr detail', file=sys.stderr)",
+ ])
+ result = health._probe_dependency_uncached(sys.executable, 2)
+ assert result.status == health.OK
+ assert result.detail == "1.2.3"
+ assert result.prescription == ""
+ assert all(process.stdout is None and process.stderr is None for process in children)
+
+
+def test_file_capture_returns_bounded_stdout_and_stderr(children):
+ result = subproc.run_with_timeout([
+ sys.executable, "-c", "import sys; print('a' * 100); print('b' * 100, file=sys.stderr)",
+ ], timeout=2, capture_limit_bytes=16)
+ assert result.returncode == 0
+ assert result.stdout == "a" * 16
+ assert result.stderr == "b" * 16
+ assert all(process.stdout is None and process.stderr is None for process in children)
+
+
+def test_file_capture_without_process_groups_avoids_descendant_pipe_wait(tmp_path, children, monkeypatch):
+ descendant_pid = tmp_path / "descendant.pid"
+ command = tmp_path / "version.py"
+ child = "import time; time.sleep(30)"
+ command.write_text(
+ "import subprocess, sys, time\nfrom pathlib import Path\n"
+ f"child = subprocess.Popen([sys.executable, '-c', {child!r}])\n"
+ f"Path({str(descendant_pid)!r}).write_text(str(child.pid))\n"
+ "time.sleep(30)\n"
+ )
+ monkeypatch.setattr(subproc, "os", SimpleNamespace(kill=os.kill))
+ started = time.monotonic()
+ try:
+ with pytest.raises(subproc.SubprocTimeout):
+ subproc.run_with_timeout([sys.executable, str(command)], timeout=0.6,
+ cleanup_grace=0.1, capture_limit_bytes=64 * 1024)
+ assert time.monotonic() - started < 1.5
+ assert descendant_pid.exists(), "real descendant was not reached"
+ assert all(process.poll() is not None for process in children)
+ assert all(process.stdout is None and process.stderr is None for process in children)
+ finally:
+ if descendant_pid.exists():
+ try:
+ os.kill(int(descendant_pid.read_text()), signal.SIGTERM)
+ except ProcessLookupError:
+ pass
+
+
+def test_doctor_retry_spends_one_operation_budget(monkeypatch):
+ clock = [100.0]
+ timeouts = []
+
+ def get(*args, **kwargs):
+ timeouts.append((args[1], kwargs["deadline_monotonic"]))
+ clock[0] += 6 if len(timeouts) == 1 else 0.5
+ return (False, "HTTP 429") if len(timeouts) == 1 else (True, "HTTP 200")
+
+ monkeypatch.setattr(doctor.time, "monotonic", lambda: clock[0])
+ monkeypatch.setattr(doctor.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds))
+ monkeypatch.setattr(doctor, "_http_ok", get)
+ assert doctor._probe_source("reddit", {}, 10)["ok"]
+ assert timeouts == [(10, 110), (2, 110)]
+
+
[email protected]("elapsed", [8, 9])
+def test_doctor_does_not_retry_when_backoff_exceeds_budget(monkeypatch, elapsed):
+ clock = [100.0]
+
+ def get(*args, **kwargs):
+ clock[0] += elapsed
+ return False, "HTTP 429"
+
+ monkeypatch.setattr(doctor.time, "monotonic", lambda: clock[0])
+ monkeypatch.setattr(doctor, "_http_ok", get)
+ with patch.object(doctor.time, "sleep") as sleep:
+ result = doctor._probe_source("reddit", {}, 10)
+ assert result == {
+ "ok": False, "transient": True,
+ "detail": "HTTP 429 (retry skipped: insufficient probe budget)", "probed": True,
+ }
+ assert doctor.audit_state("reddit", {"tier": doctor.TIER_OK},
+ probe_result=result) == doctor.AUDIT_UNVERIFIED
+ sleep.assert_not_called()
+
+
[email protected]("readings", [
+ (100.0, 101.0),
+ (100.0, 100.5, 101.0),
+])
+def test_doctor_expired_429_keeps_deadline_precedence(monkeypatch, readings):
+ clock = iter(readings)
+ monkeypatch.setattr(doctor.time, "monotonic", lambda: next(clock))
+ with patch.object(doctor, "_http_ok", return_value=(False, "HTTP 429")) as transport, \
+ patch.object(doctor.time, "sleep") as sleep:
+ result = doctor._probe_source("reddit", {}, 1)
+ assert result == {"ok": False, "detail": "probe exceeded deadline", "probed": True}
+ assert doctor.audit_state("reddit", {"tier": doctor.TIER_OK},
+ probe_result=result) == doctor.AUDIT_NOT_WORKING
+ transport.assert_called_once()
+ sleep.assert_not_called()
+
+
+def test_doctor_cli_receives_remaining_timeout(monkeypatch):
+ clock = iter([100.0, 100.25, 100.5])
+ monkeypatch.setattr(doctor.time, "monotonic", lambda: next(clock))
+ probe = health.DependencyProbe(name="yt-dlp", status=health.OK, detail="version")
+ with patch.object(health, "probe_dependency", return_value=probe) as dependency:
+ assert doctor._probe_source("youtube", {}, 5)["ok"]
+ dependency.assert_called_once_with("yt-dlp", timeout=4.75)
+
+
+def test_queued_doctor_probe_gets_its_own_start_deadline(monkeypatch):
+ clock = [100.0]
+ deadlines = []
+
+ def get(*args, **kwargs):
+ deadlines.append(kwargs["deadline_monotonic"])
+ clock[0] += 8
+ return True, "HTTP 200"
+
+ monkeypatch.setattr(doctor.time, "monotonic", lambda: clock[0])
+ monkeypatch.setattr(doctor, "_http_ok", get)
+ monkeypatch.setattr(doctor, "_probeable_sources", lambda: ("github", "hackernews"))
+ monkeypatch.setattr(doctor.concurrent.futures, "ThreadPoolExecutor", lambda **kwargs: ThreadPoolExecutor(max_workers=1))
+ assert all(result["ok"] for result in doctor._probe_sources({}, 10).values())
+ assert deadlines == [110, 118]
diff --git a/tests/test_doctor.py b/tests/test_doctor.py
index 23cb27f0..d8abfa43 100644
--- a/tests/test_doctor.py
+++ b/tests/test_doctor.py
@@ -1005,19 +1005,7 @@ class LiveProbe(unittest.TestCase):
self.assertEqual(reddit_search.search_url("test"), url)
def _probe_reddit_with_body(self, body):
- class _Resp:
- status = 200
-
- def read(self, *args):
- return body.encode("utf-8")
-
- def __enter__(self):
- return self
-
- def __exit__(self, *exc):
- return False
-
- with mock.patch("lib.doctor.urllib.request.urlopen", return_value=_Resp()):
+ with mock.patch("lib.bounded_get.get", return_value=(200, body.encode("utf-8"), None)):
return doctor._probe_source("reddit", {}, 5)
def test_reddit_probe_200_with_results_passes(self):
@@ -1045,7 +1033,7 @@ class LiveProbe(unittest.TestCase):
doctor._HTTP_PROBE_URLS["reddit"], code, "Blocked", {}, None
)
with (
- mock.patch("lib.doctor.urllib.request.urlopen", side_effect=error),
+ mock.patch("lib.bounded_get.get", return_value=(code, None, error)),
# 429 buys a retry; don't pay the real backoff in the suite.
mock.patch("lib.doctor.time.sleep"),
):
@@ -1125,7 +1113,7 @@ class LiveProbe(unittest.TestCase):
error = urllib.error.HTTPError(
doctor._HTTP_PROBE_URLS["github"], 403, "Forbidden", {}, None
)
- with mock.patch("lib.doctor.urllib.request.urlopen", side_effect=error):
+ with mock.patch("lib.bounded_get.get", return_value=(403, None, error)):
res = doctor._probe_source("github", {}, 5)
self.assertTrue(res["ok"])
@@ -1133,12 +1121,12 @@ class LiveProbe(unittest.TestCase):
# Probing with a different UA measures the User-Agent, not the endpoint.
seen = {}
- def capture(req, timeout=None):
+ def capture(req, **kwargs):
seen["ua"] = req.get_header("User-agent")
seen["accept"] = req.get_header("Accept")
- raise urllib.error.HTTPError(req.full_url, 500, "boom", {}, None)
+ return 500, None, urllib.error.HTTPError(req.full_url, 500, "boom", {}, None)
- with mock.patch("lib.doctor.urllib.request.urlopen", capture):
+ with mock.patch("lib.bounded_get.get", capture):
doctor._probe_source("reddit", {}, 5)
self.assertEqual(http.BROWSER_USER_AGENT, seen["ua"])
self.assertIn("text/html", seen["accept"])
@@ -1155,19 +1143,12 @@ class LiveProbe(unittest.TestCase):
self.assertIn("boom", results["reddit"]["detail"])
self.assertTrue(results["hackernews"]["ok"]) # others unaffected
- def test_probe_deadline_never_hangs(self):
- import time
-
- def too_slow(name, config, timeout):
- time.sleep(1.3) # exceeds the timeout(0)+1s result deadline
- return {"ok": True, "probed": True}
-
- with mock.patch("lib.doctor._probe_source", too_slow):
+ def test_zero_probe_budget_starts_no_transport(self):
+ with mock.patch("lib.bounded_get.get") as transport:
results = doctor._probe_sources({}, timeout=0)
+ transport.assert_not_called()
self.assertTrue(results)
- self.assertTrue(
- any("deadline" in r.get("detail", "") for r in results.values())
- )
+ self.assertTrue(all("deadline" in r["detail"] for r in results.values()))
def test_probe_result_flips_unverified_to_working(self):
rec = {"tier": "ok", "status": "ok", "audit_state": doctor.AUDIT_UNVERIFIED}
diff --git a/tests/test_grok_dependency.py b/tests/test_grok_dependency.py
index 46202d2d..6f1eccdd 100644
--- a/tests/test_grok_dependency.py
+++ b/tests/test_grok_dependency.py
@@ -41,7 +41,7 @@ def test_broken_binary_yields_a_reinstall_prescription(monkeypatch):
def boom(*a, **k):
raise OSError("exec format error")
- monkeypatch.setattr(health.subprocess, "run", boom)
+ monkeypatch.setattr(health.subproc, "run_with_timeout", boom)
probe = health.probe_dependency("grok")
assert probe.status == health.BROKEN
assert "reinstall" in probe.prescription.lower()
diff --git a/tests/test_health_probe_taxonomy.py b/tests/test_health_probe_taxonomy.py
index 2e66dad8..63d0ca02 100644
--- a/tests/test_health_probe_taxonomy.py
+++ b/tests/test_health_probe_taxonomy.py
@@ -103,7 +103,7 @@ class TestBroken:
def test_stale_shim_exec_oserror_is_broken_not_ok(self):
with mock.patch.object(health.shutil, "which", side_effect=_which_map({"yt-dlp": "/x/yt-dlp", "brew": "/x/brew"})), \
- mock.patch.object(health.subprocess, "run", side_effect=OSError("exec format error")):
+ mock.patch.object(health.subproc, "run_with_timeout", side_effect=OSError("exec format error")):
probe = health.probe_dependency("yt-dlp")
assert probe.status == health.BROKEN
assert probe.prescription == "brew reinstall yt-dlp"
@@ -112,7 +112,7 @@ class TestBroken:
def test_nonzero_version_exit_is_broken(self):
fake = _completed(rc=1, stderr="ModuleNotFoundError: No module named 'yt_dlp'")
with mock.patch.object(health.shutil, "which", side_effect=_which_map({"yt-dlp": "/x/yt-dlp", "brew": "/x/brew"})), \
- mock.patch.object(health.subprocess, "run", return_value=fake):
+ mock.patch.object(health.subproc, "run_with_timeout", return_value=fake):
probe = health.probe_dependency("yt-dlp")
assert probe.status == health.BROKEN
assert "ModuleNotFoundError" in probe.detail
@@ -120,7 +120,7 @@ class TestBroken:
def test_broken_pp_cli_prescribes_rerunning_install(self):
with mock.patch.object(health.shutil, "which", side_effect=_which_map({"digg-pp-cli": "/x/digg-pp-cli"})), \
- mock.patch.object(health.subprocess, "run", side_effect=OSError("bad exec")):
+ mock.patch.object(health.subproc, "run_with_timeout", side_effect=OSError("bad exec")):
probe = health.probe_dependency("digg-pp-cli")
assert probe.status == health.BROKEN
assert "printing-press-library" in probe.prescription
@@ -144,8 +144,8 @@ class TestTimeout:
def test_timeout_status_and_message(self):
with mock.patch.object(health.shutil, "which", side_effect=_which_map({"ffmpeg": "/x/ffmpeg", "brew": "/x/brew"})), \
mock.patch.object(
- health.subprocess, "run",
- side_effect=subprocess.TimeoutExpired(cmd=["ffmpeg"], timeout=health.PROBE_TIMEOUT),
+ health.subproc, "run_with_timeout",
+ side_effect=health.subproc.SubprocTimeout("ffmpeg timed out"),
):
probe = health.probe_dependency("ffmpeg")
assert probe.status == health.TIMEOUT
@@ -153,7 +153,7 @@ class TestTimeout:
def test_probe_budget_is_bounded(self):
with mock.patch.object(health.shutil, "which", side_effect=_which_map({"node": "/x/node", "brew": "/x/brew"})), \
- mock.patch.object(health.subprocess, "run", return_value=_completed(stdout="v22.1.0")) as run:
+ mock.patch.object(health.subproc, "run_with_timeout", return_value=_completed(stdout="v22.1.0")) as run:
health.probe_dependency("node")
assert run.call_args.kwargs["timeout"] <= health.PROBE_TIMEOUT
@@ -163,7 +163,7 @@ class TestOk:
def test_healthy_binary_ok_no_prescription(self):
with mock.patch.object(health.shutil, "which", side_effect=_which_map({"yt-dlp": "/x/yt-dlp", "brew": "/x/brew"})), \
- mock.patch.object(health.subprocess, "run", return_value=_completed(stdout="2026.06.09\n")):
+ mock.patch.object(health.subproc, "run_with_timeout", return_value=_completed(stdout="2026.06.09\n")):
probe = health.probe_dependency("yt-dlp")
assert probe.status == health.OK
assert probe.ok
@@ -214,7 +214,7 @@ class TestCachingAndRegistry:
def test_probe_memoized_single_subprocess(self):
with mock.patch.object(health.shutil, "which", side_effect=_which_map({"node": "/x/node", "brew": "/x/brew"})), \
- mock.patch.object(health.subprocess, "run", return_value=_completed(stdout="v22.1.0")) as run:
+ mock.patch.object(health.subproc, "run_with_timeout", return_value=_completed(stdout="v22.1.0")) as run:
first = health.probe_dependency("node")
second = health.probe_dependency("node")
assert run.call_count == 1
@@ -222,7 +222,7 @@ class TestCachingAndRegistry:
def test_clear_cache_reprobes(self):
with mock.patch.object(health.shutil, "which", side_effect=_which_map({"node": "/x/node", "brew": "/x/brew"})), \
- mock.patch.object(health.subprocess, "run", return_value=_completed(stdout="v22.1.0")) as run:
+ mock.patch.object(health.subproc, "run_with_timeout", return_value=_completed(stdout="v22.1.0")) as run:
health.probe_dependency("node")
health.clear_dependency_probe_cache()
health.probe_dependency("node")
diff --git a/tests/test_reddit.py b/tests/test_reddit.py
index 6189ee72..61ca7926 100644
--- a/tests/test_reddit.py
+++ b/tests/test_reddit.py
@@ -154,7 +154,7 @@ class TestEnrichSelectsTopEngagement(unittest.TestCase):
enriched_urls = []
- def mock_fetch_comments(url, token):
+ def mock_fetch_comments(url, token, **kwargs):
enriched_urls.append(url)
return [{"body": "Great thread!", "ups": 10, "author": "testuser"}]
@@ -189,7 +189,7 @@ class TestEnrichSelectsTopEngagement(unittest.TestCase):
enriched_urls = []
- def mock_fetch_comments(url, token):
+ def mock_fetch_comments(url, token, **kwargs):
enriched_urls.append(url)
return [{"body": "Comment", "ups": 5, "author": "user"}]
@@ -231,20 +231,16 @@ class TestEnrichmentBudget(unittest.TestCase):
def test_budget_zero_returns_items_unenriched(self):
"""With budget=0, items are returned without enrichment (not discarded)."""
- import time as _time
from unittest.mock import patch
items = self._make_items(3)
- def slow_fetch(url, token):
- _time.sleep(2)
- return [{"body": "comment", "score": 10, "author": "u"}]
-
- with patch("lib.reddit.fetch_post_comments", side_effect=slow_fetch):
+ with patch("lib.reddit.fetch_post_comments") as fetch:
result = enrich_with_comments(items, "fake-token", depth="quick", budget_seconds=0)
-
- # All 3 items returned (not discarded)
+ self.assertIs(result, items)
self.assertEqual(len(result), 3)
+ self.assertTrue(all("top_comments" not in item for item in result))
+ fetch.assert_not_called()
def test_empty_items_returns_immediately(self):
result = enrich_with_comments([], "fake-token", depth="default", budget_seconds=60)
diff --git a/tests/test_subproc.py b/tests/test_subproc.py
index 9f2d603a..f2106c23 100644
--- a/tests/test_subproc.py
+++ b/tests/test_subproc.py
@@ -5,6 +5,7 @@ PID callback wiring, and environment inheritance.
"""
import builtins
+import errno
import os as real_os
import platform
import unittest
@@ -35,10 +36,28 @@ def get_shell_cmd(cmd_str: str) -> list[str]:
return ["sh", "-c", cmd_str]
+class TestSubprocTimeout(unittest.TestCase):
+ def test_exception_class_raises_without_a_message(self):
+ with self.assertRaises(subproc.SubprocTimeout) as caught:
+ raise subproc.SubprocTimeout
+ self.assertEqual(str(caught.exception), "")
+ self.assertTrue(caught.exception.started)
+
+
class TestRunWithTimeout(unittest.TestCase):
def setUp(self):
self.addCleanup(setattr, subproc, "_shutting_down", False)
+ def test_spawn_errors_preserve_original_type_and_errno(self):
+ for error_number in (errno.ENOENT, errno.EAGAIN, errno.EMFILE):
+ error = OSError(error_number, "command could not start")
+ with self.subTest(error_number=error_number), \
+ patch.object(subproc.subprocess, "Popen", side_effect=error):
+ with self.assertRaises(type(error)) as caught:
+ subproc.run_with_timeout(["missing-command"], timeout=1)
+ self.assertIs(caught.exception, error)
+ self.assertEqual(caught.exception.errno, error_number)
+
def test_success_returns_stdout(self):
result = subproc.run_with_timeout(
get_shell_cmd("echo hello"),
@@ -405,8 +424,9 @@ time.sleep(15)
def __init__(self):
self.pid = 4321
self.kill_count = 0
+ self.stdin = self.stdout = self.stderr = None
- def communicate(self, timeout=None):
+ def communicate(self, timeout=None, input=None):
raise TimeoutExpired(cmd="x", timeout=timeout)
def wait(self, timeout=None):