Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
test(cdp): isolate smoke groups by process
ldm0ea80ecd6a2fa
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 785c0b123..e95b0d22f 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -134,7 +134,20 @@ jobs:continue-on-error: trueenv:MOLI_BIN: ${{ github.workspace }}/target/release/moli- run: uv run --project moli-cdp-smoke --no-sync moli-cdp-smoke+ run: >-+ uv run --project moli-cdp-smoke --no-sync+ moli-cdp-smoke+ --jobs 4+ --output-dir ${{ github.workspace }}/target/smoke/cdp-ci++ - name: Upload CDP smoke diagnostics+ if: ${{ always() && steps.cdp.outcome != 'skipped' }}+ continue-on-error: true+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: cdp-smoke-diagnostics+ path: target/smoke/cdp-ci+ if-no-files-found: warn- name: Run full WebDriver smokeid: webdriverdiff --git a/moli-cdp-smoke/README.md b/moli-cdp-smoke/README.mdindex 73565ff69..f8d2ce3fe 100644--- a/moli-cdp-smoke/README.md+++ b/moli-cdp-smoke/README.md@@ -340,11 +340,28 @@ npm ciuv run moli-cdp-smoke```-With no `--group`, the runner executes every raw, Playwright page, browser, and-repository-managed external-client group, including `inspector-routing` and the-pinned Puppeteer client. CI invokes that same unfiltered default. External-client-groups whose binaries or dependency environments are not owned by this project-remain explicit integration runs.+With no `--group`, the supervisor executes every raw, Playwright page, browser,+and repository-managed external-client group, including `inspector-routing`+and the pinned Puppeteer client. Every group runs in a separate worker process+with its own Moli process, fixture server, HTTP cache, Playwright context, and+temporary directories. One failed or wedged group therefore cannot retain+state or child processes for the next group.++The default concurrency is one worker. Use bounded parallelism explicitly;+CI runs the same unfiltered suite with four workers:++```bash+uv run moli-cdp-smoke --jobs 4+```++Each run writes `summary.json` plus one log and one result JSON per worker under+`target/smoke/cdp/<run-id>/`. Use `--output-dir` for a stable artifact path and+`--timeout` to change the per-worker wall-clock limit. The worker also retains+its own protocol-operation timeouts, so the wall limit is a final process-level+safety boundary rather than the primary synchronization mechanism.++External-client groups whose binaries or dependency environments are not owned+by this project remain explicit integration runs.List available scenario groups:@@ -407,7 +424,9 @@ ordering between the two routes. A normal debugger pause must pump mixed V8and non-V8 Main commands from the same session in send order. Aninstrumentation pause must leave Main blocked and admit only IO work.-Run the same focused group against an already-running Chromium CDP endpoint:+Run the same focused group against an already-running Chromium CDP endpoint.+The Python workers are still separate, but an explicitly supplied external+browser endpoint is shared rather than restarted by the supervisor:```bashuv run moli-cdp-smoke \@@ -472,7 +491,9 @@ MOLI_BIN=../target/release/moli uv run moli-cdp-smokeUseful environment variables:- `MOLI_BIN`: path to the `moli` binary under test.-- `MOLI_CDP_PORT`: CDP server port. Defaults to a free local port.+- `MOLI_CDP_PORT`: fixed CDP server port for single-worker debugging. By+ default every worker uses `--port 0` and reads the OS-selected endpoint from+ Moli's bound-listener log. A fixed port is rejected with parallel workers.- `MOLI_SMOKE_GROUPS`: comma-separated smoke group list. CLI `--group` takes precedence.- `MOLI_INSPECTOR_ROUTING_SCENARIOS`: comma-separated scenario names within the `inspector-routing` group.- `MOLI_SMOKE_TRACE=1`: print extra runner-side trace logs.diff --git a/moli-cdp-smoke/moli_cdp_smoke/groups/agent_browser.py b/moli-cdp-smoke/moli_cdp_smoke/groups/agent_browser.pyindex 707dba751..7f888a652 100644--- a/moli-cdp-smoke/moli_cdp_smoke/groups/agent_browser.py+++ b/moli-cdp-smoke/moli_cdp_smoke/groups/agent_browser.py@@ -14,6 +14,7 @@ from urllib.parse import urlparsefrom ..assertions import SmokeError, recordfrom ..config import clear_proxy_env+from ..process import CapturedProcessTimeout, run_captured_processfrom .tracing import _assert_cpu_profile_events, _hot_function_expression@@ -45,24 +46,17 @@ def _agent_browser_env() -> dict[str, str]:async def _agent_browser_version(binary: str) -> str:- process = await asyncio.create_subprocess_exec(- binary,- "--version",- stdout=asyncio.subprocess.PIPE,- stderr=asyncio.subprocess.PIPE,- )try:- stdout_bytes, stderr_bytes = await asyncio.wait_for(- process.communicate(), timeout=5+ process = await run_captured_process(+ [binary, "--version"],+ timeout_seconds=5,)- except asyncio.TimeoutError as error:- process.kill()- await process.communicate()+ except CapturedProcessTimeout as error:raise SmokeError("agent-browser --version timed out") from erroroutput = "\n".join(part.decode("utf-8", errors="replace").strip()- for part in (stdout_bytes, stderr_bytes)+ for part in (process.stdout, process.stderr)if part)match = re.search(r"\b(\d+)\.(\d+)\.(\d+)\b", output)@@ -113,28 +107,22 @@ async def _run_agent_browser("--json",*args,]- process = await asyncio.create_subprocess_exec(- *argv,- cwd=str(cwd),- env=_agent_browser_env(),- stdout=asyncio.subprocess.PIPE,- stderr=asyncio.subprocess.PIPE,- )try:- stdout_bytes, stderr_bytes = await asyncio.wait_for(- process.communicate(), timeout=timeout_seconds+ process = await run_captured_process(+ argv,+ timeout_seconds=timeout_seconds,+ cwd=str(cwd),+ env=_agent_browser_env(),)- except asyncio.TimeoutError as error:- process.kill()- stdout_bytes, stderr_bytes = await process.communicate()+ except CapturedProcessTimeout as error:raise SmokeError(f"agent-browser command {args!r} timed out after {timeout_seconds}s\n"- f"stdout:\n{stdout_bytes.decode(errors='replace')}\n"- f"stderr:\n{stderr_bytes.decode(errors='replace')}"+ f"stdout:\n{error.stdout.decode(errors='replace')}\n"+ f"stderr:\n{error.stderr.decode(errors='replace')}") from error- stdout = stdout_bytes.decode("utf-8", errors="replace")- stderr = stderr_bytes.decode("utf-8", errors="replace")+ stdout = process.stdout.decode("utf-8", errors="replace")+ stderr = process.stderr.decode("utf-8", errors="replace")return _parse_command_payload(stdout, stderr, argv)diff --git a/moli-cdp-smoke/moli_cdp_smoke/groups/external_process.py b/moli-cdp-smoke/moli_cdp_smoke/groups/external_process.pyindex c944f6bab..0d18be999 100644--- a/moli-cdp-smoke/moli_cdp_smoke/groups/external_process.py+++ b/moli-cdp-smoke/moli_cdp_smoke/groups/external_process.py@@ -1,6 +1,5 @@from __future__ import annotations-import asyncioimport jsonimport osimport sys@@ -9,6 +8,7 @@ from typing import Anyfrom ..assertions import SmokeErrorfrom ..config import clear_proxy_env+from ..process import CapturedProcessTimeout, run_captured_processasync def run_external_json_process(@@ -18,27 +18,28 @@ async def run_external_json_process(*,timeout_seconds: float = 45,) -> None:- process = await asyncio.create_subprocess_exec(- *argv,- stdout=asyncio.subprocess.PIPE,- stderr=asyncio.subprocess.PIPE,- env=clear_proxy_env(os.environ),- )try:- stdout, stderr = await asyncio.wait_for(- process.communicate(), timeout=timeout_seconds+ process = await run_captured_process(+ argv,+ timeout_seconds=timeout_seconds,+ env=clear_proxy_env(os.environ),)- except asyncio.TimeoutError as error:- process.kill()- stdout, stderr = await process.communicate()+ except CapturedProcessTimeout as error:+ cleanup_note = ""+ if not error.output_closed:+ cleanup_note = (+ "\nstdout/stderr remained open after process-group termination; "+ "captured output is unavailable"+ )raise SmokeError(f"{label} smoke timed out after {timeout_seconds:g}s\n"- f"stdout:\n{stdout.decode(errors='replace')}\n"- f"stderr:\n{stderr.decode(errors='replace')}"+ f"stdout:\n{error.stdout.decode(errors='replace')}\n"+ f"stderr:\n{error.stderr.decode(errors='replace')}"+ f"{cleanup_note}") from error- stdout_text = stdout.decode("utf-8", errors="replace")- stderr_text = stderr.decode("utf-8", errors="replace")+ stdout_text = process.stdout.decode("utf-8", errors="replace")+ stderr_text = process.stderr.decode("utf-8", errors="replace")if process.returncode != 0:raise SmokeError(f"{label} smoke failed with exit code {process.returncode}\n"diff --git a/moli-cdp-smoke/moli_cdp_smoke/groups/layout_screenshot.py b/moli-cdp-smoke/moli_cdp_smoke/groups/layout_screenshot.pyindex b9acc99e2..cb50d0df3 100644--- a/moli-cdp-smoke/moli_cdp_smoke/groups/layout_screenshot.py+++ b/moli-cdp-smoke/moli_cdp_smoke/groups/layout_screenshot.py@@ -6,11 +6,10 @@ import mathfrom typing import Anyfrom ..assertions import SmokeError, assert_equal, record-from ..config import reserve_portfrom ..jpeg_image import jpeg_dimensionsfrom ..png_image import decode_pngfrom ..raw_cdp import RawCdpClient, connect_raw_cdp, discover_websocket_url-from ..serve import start_moli_serve, stop_moli_serve, wait_for_cdp_server+from ..serve import start_moli_serve, stop_moli_serve, wait_for_moli_endpointSCREENSHOT_UNSUPPORTED = (@@ -648,13 +647,11 @@ async def _run_default_mock_boundary(fixture: str,results: list[dict[str, Any]],) -> None:- port = reserve_port()- serve = await start_moli_serve(port, layout=False)- endpoint = f"http://127.0.0.1:{port}"+ serve = await start_moli_serve(layout=False)+ endpoint = await wait_for_moli_endpoint(serve)client: RawCdpClient | None = Nonetarget_id: str | None = Nonetry:- await wait_for_cdp_server(endpoint, serve)client = await connect_raw_cdp(endpoint)target_id, session_id = await _open_target(client,diff --git a/moli-cdp-smoke/moli_cdp_smoke/groups/puppeteer.py b/moli-cdp-smoke/moli_cdp_smoke/groups/puppeteer.pyindex a700f3733..d39727063 100644--- a/moli-cdp-smoke/moli_cdp_smoke/groups/puppeteer.py+++ b/moli-cdp-smoke/moli_cdp_smoke/groups/puppeteer.py@@ -1,61 +1,24 @@from __future__ import annotations-import asyncio-import jsonimport os-import sysfrom pathlib import Pathfrom typing import Any-from ..assertions import SmokeError-from ..config import clear_proxy_env+from .external_process import run_external_json_processPUPPETEER_SCRIPT = Path(__file__).resolve().parents[1] / "puppeteer_smoke.mjs"async def run_puppeteer_group(endpoint: str, fixture: str, results: list[dict[str, Any]]) -> None:- node = os.environ.get("NODE", "node")- process = await asyncio.create_subprocess_exec(- node,- str(PUPPETEER_SCRIPT),- endpoint,- fixture,- stdout=asyncio.subprocess.PIPE,- stderr=asyncio.subprocess.PIPE,- env=clear_proxy_env(os.environ),+ await run_external_json_process(+ "Puppeteer",+ [+ os.environ.get("NODE", "node"),+ str(PUPPETEER_SCRIPT),+ endpoint,+ fixture,+ ],+ results,+ timeout_seconds=30,)- try:- stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=30)- except asyncio.TimeoutError as error:- process.kill()- stdout, stderr = await process.communicate()- stdout_text = stdout.decode("utf-8", errors="replace")- stderr_text = stderr.decode("utf-8", errors="replace")- raise SmokeError(- f"Puppeteer smoke timed out after 30s\nstdout:\n{stdout_text}\nstderr:\n{stderr_text}"- ) from error-- stdout_text = stdout.decode("utf-8", errors="replace")- stderr_text = stderr.decode("utf-8", errors="replace")- if process.returncode != 0:- raise SmokeError(- "Puppeteer smoke failed with exit code "- f"{process.returncode}\nstdout:\n{stdout_text}\nstderr:\n{stderr_text}"- )-- try:- payload = json.loads(stdout_text)- except json.JSONDecodeError as error:- raise SmokeError(- f"Puppeteer smoke did not return JSON\nstdout:\n{stdout_text}\nstderr:\n{stderr_text}"- ) from error-- if not payload.get("ok"):- raise SmokeError(f"Puppeteer smoke reported failure: {payload}")- payload_results = payload.get("results")- if not isinstance(payload_results, list):- raise SmokeError(f"Puppeteer smoke returned invalid results: {payload}")- results.extend(payload_results)- if os.environ.get("MOLI_SMOKE_TRACE") == "1" and stderr_text:- print(stderr_text, file=sys.stderr, end="")diff --git a/moli-cdp-smoke/moli_cdp_smoke/process.py b/moli-cdp-smoke/moli_cdp_smoke/process.pynew file mode 100644index 000000000..f3de5dd46--- /dev/null+++ b/moli-cdp-smoke/moli_cdp_smoke/process.py@@ -0,0 +1,167 @@+from __future__ import annotations++import asyncio+import os+import signal+from collections.abc import Mapping, Sequence+from dataclasses import dataclass+from os import PathLike+++_PROCESS_CLEANUP_TIMEOUT_SECONDS = 2.0+INHERIT_PROCESS_GROUP_ENV = "MOLI_SMOKE_INHERIT_PROCESS_GROUP"+++def subprocess_starts_new_session() -> bool:+ return os.name == "posix" and os.environ.get(INHERIT_PROCESS_GROUP_ENV) != "1"+++@dataclass(frozen=True)+class CapturedProcessResult:+ returncode: int+ stdout: bytes+ stderr: bytes+++class CapturedProcessTimeout(TimeoutError):+ def __init__(+ self,+ timeout_seconds: float,+ stdout: bytes,+ stderr: bytes,+ *,+ output_closed: bool,+ ) -> None:+ super().__init__(f"process timed out after {timeout_seconds:g}s")+ self.timeout_seconds = timeout_seconds+ self.stdout = stdout+ self.stderr = stderr+ self.output_closed = output_closed+++def _consume_task_result(task: asyncio.Task[object]) -> None:+ try:+ task.exception()+ except (asyncio.CancelledError, Exception):+ pass+++async def _task_completed(task: asyncio.Task[object], timeout_seconds: float) -> bool:+ done, _ = await asyncio.wait((task,), timeout=timeout_seconds)+ return task in done+++def _signal_process_tree(+ process: asyncio.subprocess.Process,+ process_signal: signal.Signals,+) -> None:+ if os.name == "posix":+ try:+ os.killpg(process.pid, process_signal)+ return+ except ProcessLookupError:+ pass+ except OSError:+ # Fall back to the direct child below. The bounded waits still keep+ # cleanup from becoming a second unbounded failure mode.+ pass++ if process.returncode is not None:+ return+ try:+ if process_signal == signal.SIGTERM:+ process.terminate()+ else:+ process.kill()+ except ProcessLookupError:+ pass+++async def _finish_communication_after_kill(+ process: asyncio.subprocess.Process,+ communication: asyncio.Task[tuple[bytes, bytes]],+) -> tuple[bytes, bytes, bool]:+ _signal_process_tree(process, signal.SIGKILL)+ if await _task_completed(communication, _PROCESS_CLEANUP_TIMEOUT_SECONDS):+ stdout, stderr = communication.result()+ return stdout, stderr, True++ # A process which escaped the new process group can still own one of the+ # inherited pipes. Do not let that escaped descriptor turn timeout cleanup+ # back into an infinite communicate() wait.+ communication.cancel()+ communication.add_done_callback(_consume_task_result)+ return b"", b"", False+++async def run_captured_process(+ argv: Sequence[str],+ *,+ timeout_seconds: float,+ cwd: str | PathLike[str] | None = None,+ env: Mapping[str, str] | None = None,+) -> CapturedProcessResult:+ if timeout_seconds <= 0:+ raise ValueError("timeout_seconds must be positive")++ process = await asyncio.create_subprocess_exec(+ *argv,+ cwd=cwd,+ env=env,+ stdout=asyncio.subprocess.PIPE,+ stderr=asyncio.subprocess.PIPE,+ # Every managed external command gets its own POSIX process group so a+ # timeout can terminate descendants which inherited stdout/stderr.+ start_new_session=subprocess_starts_new_session(),+ )+ communication = asyncio.create_task(process.communicate())+ try:+ completed = await _task_completed(communication, timeout_seconds)+ except BaseException:+ if not communication.done():+ await _finish_communication_after_kill(process, communication)+ raise++ if completed:+ stdout, stderr = communication.result()+ if process.returncode is None:+ raise RuntimeError("completed subprocess has no return code")+ return CapturedProcessResult(process.returncode, stdout, stderr)++ stdout, stderr, output_closed = await _finish_communication_after_kill(+ process,+ communication,+ )+ raise CapturedProcessTimeout(+ timeout_seconds,+ stdout,+ stderr,+ output_closed=output_closed,+ )+++async def terminate_process_tree(+ process: asyncio.subprocess.Process,+ *,+ terminate_timeout_seconds: float = 5.0,+ kill_timeout_seconds: float = 2.0,+) -> bool:+ if process.returncode is not None:+ _signal_process_tree(process, signal.SIGKILL)+ return True++ wait_task = asyncio.create_task(process.wait())+ _signal_process_tree(process, signal.SIGTERM)+ if await _task_completed(wait_task, terminate_timeout_seconds):+ # The dedicated group may still contain descendants after its leader+ # exits. The caller is tearing the service down, so clear any remaining+ # group members as well.+ _signal_process_tree(process, signal.SIGKILL)+ return True++ _signal_process_tree(process, signal.SIGKILL)+ if await _task_completed(wait_task, kill_timeout_seconds):+ return True+ wait_task.cancel()+ wait_task.add_done_callback(_consume_task_result)+ return Falsediff --git a/moli-cdp-smoke/moli_cdp_smoke/progress.py b/moli-cdp-smoke/moli_cdp_smoke/progress.pyindex aa900cd24..cfb950168 100644--- a/moli-cdp-smoke/moli_cdp_smoke/progress.py+++ b/moli-cdp-smoke/moli_cdp_smoke/progress.py@@ -6,18 +6,51 @@ from typing import Awaitable, TypeVarProgressResult = TypeVar("ProgressResult")+_CANCELLATION_GRACE_SECONDS = 5.0+++def _consume_task_result(task: asyncio.Future[object]) -> None:+ try:+ task.exception()+ except (asyncio.CancelledError, Exception):+ pass+++async def _cancel_with_grace(task: asyncio.Future[object]) -> None:+ if task.done():+ return+ task.cancel()+ done, _ = await asyncio.wait((task,), timeout=_CANCELLATION_GRACE_SECONDS)+ if task in done:+ _consume_task_result(task)+ else:+ task.add_done_callback(_consume_task_result)async def await_with_progress(label: str,awaitable: Awaitable[ProgressResult],+ *,+ timeout_seconds: float | None = None,) -> ProgressResult:+ if timeout_seconds is not None and timeout_seconds <= 0:+ raise ValueError("timeout_seconds must be positive")loop = asyncio.get_running_loop()started_at = loop.time()print(f"[moli-cdp-smoke] START {label}", file=sys.stderr, flush=True)+ task = asyncio.ensure_future(awaitable)try:- result = await awaitable+ if timeout_seconds is None:+ result = await task+ else:+ done, _ = await asyncio.wait((task,), timeout=timeout_seconds)+ if task not in done:+ raise TimeoutError(+ f"{label} timed out after {timeout_seconds:g}s"+ )+ result = task.result()except BaseException as error:+ await _cancel_with_grace(task)elapsed = loop.time() - started_atprint(f"[moli-cdp-smoke] FAIL {label} elapsed={elapsed:.3f}s "diff --git a/moli-cdp-smoke/moli_cdp_smoke/runner.py b/moli-cdp-smoke/moli_cdp_smoke/runner.pyindex 0a046f09b..aff357923 100644--- a/moli-cdp-smoke/moli_cdp_smoke/runner.py+++ b/moli-cdp-smoke/moli_cdp_smoke/runner.py@@ -15,7 +15,7 @@ from typing import Any, Awaitable, Callable, Iterablefrom playwright.async_api import async_playwrightfrom .assertions import record-from .config import clear_current_process_proxy_env, reserve_port+from .config import clear_current_process_proxy_envfrom .fixture import FixtureServerfrom .groups.action_window import run_action_window_groupfrom .groups.agent_browser import run_agent_browser_group@@ -60,7 +60,13 @@ from .groups.workers import run_workers_groupfrom .groups.xhr_sync_semantics import run_xhr_sync_semantics_groupfrom .helpers import attach_cdp_event_collectorfrom .progress import await_with_progress-from .serve import MoliServe, start_moli_serve, stop_moli_serve, wait_for_cdp_server+from .serve import (+ MoliServe,+ start_moli_serve,+ stop_moli_serve,+ wait_for_cdp_server,+ wait_for_moli_endpoint,+)from .state import SmokeState@@ -81,6 +87,13 @@ class SmokeGroup:runner: RawGroupRunner | ExternalGroupRunner | PageGroupRunner | BrowserGroupRunner+async def _await_group(group: SmokeGroup, awaitable: Awaitable[None]) -> None:+ await await_with_progress(+ f"group/{group.phase}/{group.name}",+ awaitable,+ )++RAW_GROUPS: tuple[SmokeGroup, ...] = (SmokeGroup("inspector-routing",@@ -436,8 +449,11 @@ async def run_smoke(if results is None:results = []temp_dir = Path(tempfile.mkdtemp(prefix="moli-pw-smoke-"))- playwright = await await_with_progress("playwright/start", async_playwright().start())+ playwright = Nonetry:+ playwright = await await_with_progress(+ "playwright/start", async_playwright().start()+ )browser = await await_with_progress("playwright/connect-over-cdp",playwright.chromium.connect_over_cdp(endpoint, timeout=10_000),@@ -500,36 +516,37 @@ async def run_smoke()for group in selection.page_groups:- await await_with_progress(- f"group/{group.phase}/{group.name}",+ await _await_group(+ group,group.runner(state), # type: ignore[misc])await await_with_progress("playwright/context-close", context.close())for group in selection.browser_groups:- await await_with_progress(- f"group/{group.phase}/{group.name}",+ await _await_group(+ group,group.runner(browser, fixture_server.url, results), # type: ignore[misc])return resultsfinally:await await_with_progress("playwright/browser-close", browser.close())- shutil.rmtree(temp_dir, ignore_errors=True)finally:- await await_with_progress("playwright/stop", playwright.stop())+ if playwright is not None:+ await await_with_progress("playwright/stop", playwright.stop())+ shutil.rmtree(temp_dir, ignore_errors=True)def parse_args(argv: list[str] | None = None) -> argparse.Namespace:parser = argparse.ArgumentParser(- description="Run Playwright CDP smoke workflows against moli."+ description="Run one isolated CDP smoke group against moli.")parser.add_argument("--group",action="append",default=[],help=(- "Run only the named group. May be repeated or comma-separated. "- "Defaults to every repository-managed group."+ "Run exactly one named group. The supervisor is responsible for "+ "selecting and scheduling multiple groups."),)parser.add_argument(@@ -541,68 +558,102 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:"--endpoint",help="Use an existing HTTP CDP endpoint instead of starting moli serve.",)+ parser.add_argument(+ "--result",+ type=Path,+ help="Atomically write the worker result JSON to this path.",+ )return parser.parse_args(argv)+def _emit_worker_payload(+ payload: dict[str, Any],+ result_path: Path | None,+ *,+ failed: bool,+) -> None:+ rendered = json.dumps(payload, indent=2, ensure_ascii=False) + "\n"+ if result_path is None:+ print(rendered, file=sys.stderr if failed else sys.stdout, end="")+ return+ result_path.parent.mkdir(parents=True, exist_ok=True)+ temporary = result_path.with_name(+ f".{result_path.name}.tmp-{os.getpid()}"+ )+ temporary.write_text(rendered, encoding="utf-8")+ os.replace(temporary, result_path)++async def async_main(argv: list[str] | None = None) -> int:args = parse_args(argv)if args.list_groups:print(json.dumps({"groups": group_listing()}, indent=2, ensure_ascii=False))return 0selection = resolve_group_selection(args.group)+ if len(selection.groups) != 1:+ raise RuntimeError(+ "moli-cdp-smoke-worker requires exactly one --group; "+ "use moli-cdp-smoke to schedule the suite"+ )+ selected_group = selection.groups[0]fixture = FixtureServer()+ fixture_started = Falseserve: MoliServe | None = Noneresults: list[dict[str, Any]] = []+ endpoint: str | None = None+ failures: list[str] = []try:fixture.start()+ fixture_started = Trueif args.endpoint:endpoint = args.endpoint.rstrip("/")else:port_env = os.environ.get("MOLI_CDP_PORT")- port = int(port_env) if port_env else reserve_port()- if port <= 0 or port > 65535:+ port = int(port_env) if port_env else 0+ if port < 0 or port > 65535 or (port == 0 and port_env is not None):raise RuntimeError(f"invalid MOLI_CDP_PORT: {port_env}")serve = await start_moli_serve(port)- endpoint = f"http://127.0.0.1:{port}"- await wait_for_cdp_server(endpoint, serve)- for group in selection.raw_groups:- await await_with_progress(- f"group/{group.phase}/{group.name}",- group.runner(endpoint, fixture.url, results), # type: ignore[misc]+ endpoint = await wait_for_moli_endpoint(serve)+ if endpoint is None:+ raise RuntimeError("CDP endpoint was not initialized")+ if serve is None:+ await wait_for_cdp_server(endpoint, serve)+ for current_group in selection.raw_groups:+ await _await_group(+ current_group,+ current_group.runner(endpoint, fixture.url, results), # type: ignore[misc])await run_smoke(endpoint, fixture, selection, results)- for group in selection.external_groups:- await await_with_progress(- f"group/{group.phase}/{group.name}",- group.runner(endpoint, fixture.url, results), # type: ignore[misc]+ for current_group in selection.external_groups:+ await _await_group(+ current_group,+ current_group.runner(endpoint, fixture.url, results), # type: ignore[misc])- ok = not any(result.get("ok") is False for result in results)- print(- json.dumps(- {- "ok": ok,- "endpoint": endpoint,- "fixture": fixture.url,- "results": results,- },- indent=2,- ensure_ascii=False,- )- )- return 0 if ok else 1except Exception as error:- print(- json.dumps(- {"ok": False, "error": "".join(traceback.format_exception(error)), "results": results},- indent=2,- ensure_ascii=False,- ),- file=sys.stderr,- )- return 1+ failures.append("".join(traceback.format_exception(error)))finally:- await stop_moli_serve(serve)- fixture.stop()+ try:+ await stop_moli_serve(serve)+ except Exception as error:+ failures.append("".join(traceback.format_exception(error)))+ if fixture_started:+ try:+ fixture.stop()+ except Exception as error:+ failures.append("".join(traceback.format_exception(error)))++ ok = not failures and not any(result.get("ok") is False for result in results)+ payload: dict[str, Any] = {+ "ok": ok,+ "group": selected_group.name,+ "endpoint": endpoint,+ "fixture": fixture.url,+ "results": results,+ }+ if failures:+ payload["error"] = "\n".join(failures)+ _emit_worker_payload(payload, args.result, failed=not ok)+ return 0 if ok else 1def main(argv: list[str] | None = None) -> None:diff --git a/moli-cdp-smoke/moli_cdp_smoke/serve.py b/moli-cdp-smoke/moli_cdp_smoke/serve.pyindex b7f0ebb14..88371e838 100644--- a/moli-cdp-smoke/moli_cdp_smoke/serve.py+++ b/moli-cdp-smoke/moli_cdp_smoke/serve.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport os+import reimport shutilimport sysimport tempfile@@ -12,6 +13,7 @@ from dataclasses import dataclassfrom typing import Anyfrom .config import REPO_ROOT, clear_proxy_env, moli_binary+from .process import subprocess_starts_new_session, terminate_process_tree@dataclass@@ -20,9 +22,38 @@ class MoliServe:logs: list[str]tasks: list[asyncio.Task[Any]]http_cache_dir: str+ endpoint_ready: asyncio.Future[str]-async def _collect_process_output(stream: asyncio.StreamReader | None, logs: list[str], label: str) -> None:+_ANSI_ESCAPE = re.compile(r"\x1b\[[0-9;]*m")+_LISTENING_ADDRESS = re.compile(+ r"\bprotocol server listening\b.*\baddr=127\.0\.0\.1:(?P<port>[0-9]{1,5})\b"+)+++def _moli_endpoint_from_log_line(line: str) -> str | None:+ match = _LISTENING_ADDRESS.search(_ANSI_ESCAPE.sub("", line))+ if match is None:+ return None+ port = int(match.group("port"))+ if not 0 < port <= 65535:+ return None+ return f"http://127.0.0.1:{port}"+++def _consume_task_result(task: asyncio.Task[Any]) -> None:+ try:+ task.exception()+ except asyncio.CancelledError:+ pass+++async def _collect_process_output(+ stream: asyncio.StreamReader | None,+ logs: list[str],+ label: str,+ endpoint_ready: asyncio.Future[str],+) -> None:if stream is None:returnwhile True:@@ -31,12 +62,15 @@ async def _collect_process_output(stream: asyncio.StreamReader | None, logs: lisreturntext = line.decode("utf-8", errors="replace").rstrip()logs.append(f"{label}: {text}")+ endpoint = _moli_endpoint_from_log_line(text)+ if endpoint is not None and not endpoint_ready.done():+ endpoint_ready.set_result(endpoint)if os.environ.get("MOLI_SMOKE_TRACE_BG") == "1":print(f"[moli serve {label}] {text}", file=sys.stderr, flush=True)async def start_moli_serve(- port: int,+ port: int = 0,*,layout: bool = True,extra_args: tuple[str, ...] = (),@@ -51,6 +85,8 @@ async def start_moli_serve("--port",str(port),"--resource",+ "--log-level",+ "info",]if layout:command.append("--layout")@@ -62,37 +98,44 @@ async def start_moli_serve(env=clear_proxy_env(os.environ),stdout=asyncio.subprocess.PIPE,stderr=asyncio.subprocess.PIPE,+ start_new_session=subprocess_starts_new_session(),)except Exception:shutil.rmtree(http_cache_dir, ignore_errors=True)raiselogs: list[str] = []+ endpoint_ready = asyncio.get_running_loop().create_future()tasks = [- asyncio.create_task(_collect_process_output(process.stdout, logs, "stdout")),- asyncio.create_task(_collect_process_output(process.stderr, logs, "stderr")),+ asyncio.create_task(+ _collect_process_output(process.stdout, logs, "stdout", endpoint_ready)+ ),+ asyncio.create_task(+ _collect_process_output(process.stderr, logs, "stderr", endpoint_ready)+ ),]return MoliServe(process=process,logs=logs,tasks=tasks,http_cache_dir=http_cache_dir,+ endpoint_ready=endpoint_ready,)async def stop_moli_serve(serve: MoliServe | None) -> None:if serve is None:return- if serve.process.returncode is None:- serve.process.terminate()- try:- await asyncio.wait_for(serve.process.wait(), timeout=5)- except asyncio.TimeoutError:- serve.process.kill()- await serve.process.wait()+ stopped = await terminate_process_tree(serve.process)for task in serve.tasks:task.cancel()- await asyncio.gather(*serve.tasks, return_exceptions=True)+ done, pending = await asyncio.wait(serve.tasks, timeout=2)+ for task in done:+ _consume_task_result(task)+ for task in pending:+ task.add_done_callback(_consume_task_result)shutil.rmtree(serve.http_cache_dir, ignore_errors=True)+ if not stopped:+ raise RuntimeError(f"moli serve process {serve.process.pid} did not exit after SIGKILL")def _probe_url(url: str) -> bool:@@ -120,3 +163,35 @@ async def wait_for_cdp_server(tail_text = "\n".join(serve.logs[-80:]) if serve is not None else ""tail = f"\n{tail_text}" if tail_text else ""raise RuntimeError(f"timed out waiting for CDP server at {endpoint}{tail}")+++async def wait_for_moli_endpoint(+ serve: MoliServe,+ *,+ timeout_seconds: float = 10,+) -> str:+ process_exit = asyncio.create_task(serve.process.wait())+ try:+ done, _ = await asyncio.wait(+ (serve.endpoint_ready, process_exit),+ timeout=timeout_seconds,+ return_when=asyncio.FIRST_COMPLETED,+ )+ if serve.endpoint_ready in done:+ endpoint = serve.endpoint_ready.result()+ await wait_for_cdp_server(endpoint, serve)+ return endpoint+ if process_exit in done:+ tail = "\n".join(serve.logs[-80:])+ raise RuntimeError(+ f"moli serve exited early with {process_exit.result()}\n{tail}"+ )+ tail_text = "\n".join(serve.logs[-80:])+ tail = f"\n{tail_text}" if tail_text else ""+ raise RuntimeError(+ "timed out waiting for moli to report its bound endpoint" + tail+ )+ finally:+ if not process_exit.done():+ process_exit.cancel()+ await asyncio.gather(process_exit, return_exceptions=True)diff --git a/moli-cdp-smoke/moli_cdp_smoke/supervisor.py b/moli-cdp-smoke/moli_cdp_smoke/supervisor.pynew file mode 100644index 000000000..aea7c19f3--- /dev/null+++ b/moli-cdp-smoke/moli_cdp_smoke/supervisor.py@@ -0,0 +1,403 @@+from __future__ import annotations++import argparse+import asyncio+import json+import os+import sys+import time+from dataclasses import dataclass+from datetime import UTC, datetime+from pathlib import Path+from typing import Any, Sequence++from .config import REPO_ROOT, clear_proxy_env+from .process import INHERIT_PROCESS_GROUP_ENV, terminate_process_tree+from .runner import SmokeGroup, group_listing, resolve_group_selection+++_DEFAULT_JOBS = 1+_DEFAULT_GROUP_TIMEOUT_SECONDS = 120.0+++@dataclass(frozen=True)+class WorkerJob:+ group: SmokeGroup+ attempt: int+ repeat: int+ endpoint: str | None = None++ @property+ def label(self) -> str:+ if self.repeat == 1:+ return self.group.name+ return f"{self.group.name}#{self.attempt}"++ @property+ def file_stem(self) -> str:+ if self.repeat == 1:+ return self.group.name+ return f"{self.group.name}-run-{self.attempt:02d}"+++@dataclass(frozen=True)+class WorkerOutcome:+ job: WorkerJob+ status: str+ duration_seconds: float+ exit_code: int | None+ log_path: Path+ result_path: Path+ scenario_count: int+ endpoint: str | None = None+ error: str | None = None++ @property+ def passed(self) -> bool:+ return self.status == "passed"++ def summary(self) -> dict[str, Any]:+ payload: dict[str, Any] = {+ "group": self.job.group.name,+ "attempt": self.job.attempt,+ "status": self.status,+ "durationSeconds": round(self.duration_seconds, 3),+ "exitCode": self.exit_code,+ "scenarioCount": self.scenario_count,+ "log": str(self.log_path),+ "result": str(self.result_path),+ }+ if self.error:+ payload["error"] = self.error+ if self.endpoint:+ payload["endpoint"] = self.endpoint+ return payload+++def _positive_int(raw: str) -> int:+ value = int(raw)+ if value <= 0:+ raise argparse.ArgumentTypeError("value must be positive")+ return value+++def _positive_float(raw: str) -> float:+ value = float(raw)+ if not 0 < value < float("inf"):+ raise argparse.ArgumentTypeError("value must be a finite positive number")+ return value+++def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:+ parser = argparse.ArgumentParser(+ description="Run CDP smoke groups in isolated worker processes."+ )+ parser.add_argument(+ "--group",+ action="append",+ default=[],+ help=(+ "Select a group. May be repeated or comma-separated. Defaults to "+ "every repository-managed group."+ ),+ )+ parser.add_argument(+ "--jobs",+ type=_positive_int,+ default=_DEFAULT_JOBS,+ help="Maximum isolated workers to run concurrently (default: 1).",+ )+ parser.add_argument(+ "--timeout",+ type=_positive_float,+ default=_DEFAULT_GROUP_TIMEOUT_SECONDS,+ help="Wall-clock timeout for each worker in seconds (default: 120).",+ )+ parser.add_argument(+ "--repeat",+ type=_positive_int,+ default=1,+ help="Run every selected group this many times; any failure fails the suite.",+ )+ parser.add_argument(+ "--fail-fast",+ action="store_true",+ help="Do not start queued workers after the first failure.",+ )+ parser.add_argument(+ "--output-dir",+ type=Path,+ help="Directory for per-worker logs/results and summary.json.",+ )+ parser.add_argument(+ "--endpoint",+ help=(+ "Use an existing HTTP CDP endpoint. Workers remain separate, but "+ "the external browser process is shared."+ ),+ )+ parser.add_argument(+ "--list-groups",+ action="store_true",+ help="List available groups as JSON and exit.",+ )+ return parser.parse_args(argv)+++def _default_output_dir() -> Path:+ timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")+ return REPO_ROOT / "target" / "smoke" / "cdp" / f"{timestamp}-{os.getpid()}"+++def _worker_argv(job: WorkerJob, result_path: Path) -> list[str]:+ argv = [+ sys.executable,+ "-m",+ "moli_cdp_smoke.worker",+ "--group",+ job.group.name,+ "--result",+ str(result_path),+ ]+ if job.endpoint:+ argv.extend(("--endpoint", job.endpoint))+ return argv+++def _worker_environment() -> dict[str, str]:+ env = clear_proxy_env(os.environ)+ env[INHERIT_PROCESS_GROUP_ENV] = "1"+ env["PYTHONUNBUFFERED"] = "1"+ return env+++def _read_worker_payload(result_path: Path) -> tuple[dict[str, Any] | None, str | None]:+ try:+ payload = json.loads(result_path.read_text(encoding="utf-8"))+ except FileNotFoundError:+ return None, "worker did not publish a result file"+ except (OSError, json.JSONDecodeError) as error:+ return None, f"worker result is unreadable: {error}"+ if not isinstance(payload, dict):+ return None, "worker result root is not an object"+ return payload, None+++async def _wait_for_worker(+ process: asyncio.subprocess.Process,+ timeout_seconds: float,+) -> tuple[int | None, bool]:+ wait_task = asyncio.create_task(process.wait())+ try:+ done, _ = await asyncio.wait((wait_task,), timeout=timeout_seconds)+ except BaseException:+ await terminate_process_tree(+ process,+ terminate_timeout_seconds=2,+ kill_timeout_seconds=2,+ )+ raise+ if wait_task in done:+ exit_code = wait_task.result()+ # A successful worker should have cleaned up its children. Clear any+ # process which escaped worker teardown before the process group can be+ # reused by another isolated job.+ await terminate_process_tree(+ process,+ terminate_timeout_seconds=0,+ kill_timeout_seconds=1,+ )+ return exit_code, False++ await terminate_process_tree(+ process,+ terminate_timeout_seconds=2,+ kill_timeout_seconds=2,+ )+ return process.returncode, True+++async def _run_worker_job(+ job: WorkerJob,+ output_dir: Path,+ timeout_seconds: float,+) -> WorkerOutcome:+ log_path = output_dir / f"{job.file_stem}.log"+ result_path = output_dir / f"{job.file_stem}.json"+ result_path.unlink(missing_ok=True)+ started_at = time.monotonic()+ print(f"[moli-cdp-supervisor] START {job.label}", file=sys.stderr, flush=True)+ with log_path.open("wb", buffering=0) as log_stream:+ process = await asyncio.create_subprocess_exec(+ *_worker_argv(job, result_path),+ cwd=str(REPO_ROOT),+ env=_worker_environment(),+ stdout=log_stream,+ stderr=asyncio.subprocess.STDOUT,+ start_new_session=os.name == "posix",+ )+ exit_code, timed_out = await _wait_for_worker(process, timeout_seconds)++ duration = time.monotonic() - started_at+ payload, payload_error = _read_worker_payload(result_path)+ scenario_count = 0+ endpoint = None+ if payload is not None and isinstance(payload.get("results"), list):+ scenario_count = len(payload["results"])+ if payload is not None and isinstance(payload.get("endpoint"), str):+ endpoint = payload["endpoint"]++ if timed_out:+ status = "timed_out"+ error = f"worker exceeded {timeout_seconds:g}s"+ elif payload_error:+ status = "crashed"+ error = payload_error+ elif payload is None:+ status = "crashed"+ error = "worker result is unavailable"+ elif payload.get("group") != job.group.name:+ status = "crashed"+ error = f"worker reported the wrong group: {payload.get('group')!r}"+ elif exit_code == 0 and payload.get("ok") is True:+ status = "passed"+ error = None+ else:+ status = "failed"+ worker_error = payload.get("error")+ error = str(worker_error) if worker_error else f"worker exited with {exit_code}"++ outcome = WorkerOutcome(+ job=job,+ status=status,+ duration_seconds=duration,+ exit_code=exit_code,+ log_path=log_path,+ result_path=result_path,+ scenario_count=scenario_count,+ endpoint=endpoint,+ error=error,+ )+ print(+ f"[moli-cdp-supervisor] {status.upper()} {job.label} "+ f"elapsed={duration:.3f}s",+ file=sys.stderr,+ flush=True,+ )+ return outcome+++def _log_tail(path: Path, *, lines: int = 40) -> str:+ try:+ content = path.read_text(encoding="utf-8", errors="replace").splitlines()+ except OSError as error:+ return f"<failed to read log: {error}>"+ return "\n".join(content[-lines:])+++def _write_json_atomic(path: Path, payload: dict[str, Any]) -> None:+ rendered = json.dumps(payload, indent=2, ensure_ascii=False) + "\n"+ temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")+ temporary.write_text(rendered, encoding="utf-8")+ os.replace(temporary, path)+++async def run_supervisor(args: argparse.Namespace) -> tuple[int, dict[str, Any]]:+ selection = resolve_group_selection(args.group)+ output_dir = (args.output_dir or _default_output_dir()).resolve()+ output_dir.mkdir(parents=True, exist_ok=True)+ jobs = [+ WorkerJob(+ group=group,+ attempt=attempt,+ repeat=args.repeat,+ endpoint=args.endpoint.rstrip("/") if args.endpoint else None,+ )+ for group in selection.groups+ for attempt in range(1, args.repeat + 1)+ ]+ fixed_port = os.environ.get("MOLI_CDP_PORT")+ if (+ args.endpoint is None+ and fixed_port is not None+ and args.jobs > 1+ and len(jobs) > 1+ ):+ raise RuntimeError(+ "MOLI_CDP_PORT cannot be shared by parallel isolated workers; "+ "unset it to use atomic OS-assigned ports or run with --jobs 1"+ )+ semaphore = asyncio.Semaphore(args.jobs)+ stop_starting = asyncio.Event()++ async def execute(job: WorkerJob) -> WorkerOutcome:+ async with semaphore:+ if args.fail_fast and stop_starting.is_set():+ return WorkerOutcome(+ job=job,+ status="skipped",+ duration_seconds=0,+ exit_code=None,+ log_path=output_dir / f"{job.file_stem}.log",+ result_path=output_dir / f"{job.file_stem}.json",+ scenario_count=0,+ error="not started after an earlier failure",+ )+ try:+ outcome = await _run_worker_job(job, output_dir, args.timeout)+ except Exception as error:+ outcome = WorkerOutcome(+ job=job,+ status="crashed",+ duration_seconds=0,+ exit_code=None,+ log_path=output_dir / f"{job.file_stem}.log",+ result_path=output_dir / f"{job.file_stem}.json",+ scenario_count=0,+ error=f"failed to start or supervise worker: {error}",+ )+ if not outcome.passed:+ stop_starting.set()+ return outcome++ outcomes = await asyncio.gather(*(execute(job) for job in jobs))+ ok = all(outcome.passed for outcome in outcomes)+ summary = {+ "ok": ok,+ "jobs": args.jobs,+ "timeoutSeconds": args.timeout,+ "repeat": args.repeat,+ "externalEndpoint": args.endpoint,+ "outputDirectory": str(output_dir),+ "groups": [outcome.summary() for outcome in outcomes],+ }+ _write_json_atomic(output_dir / "summary.json", summary)++ for outcome in outcomes:+ if outcome.passed or outcome.status == "skipped":+ continue+ print(+ f"\n--- {outcome.job.label} ({outcome.status}) log tail ---\n"+ f"{_log_tail(outcome.log_path)}",+ file=sys.stderr,+ )+ passed = sum(outcome.passed for outcome in outcomes)+ print(+ f"[moli-cdp-supervisor] summary {passed}/{len(outcomes)} passed; "+ f"artifacts={output_dir}",+ file=sys.stderr,+ )+ return (0 if ok else 1), summary+++async def async_main(argv: Sequence[str] | None = None) -> int:+ args = parse_args(argv)+ if args.list_groups:+ print(json.dumps({"groups": group_listing()}, indent=2, ensure_ascii=False))+ return 0+ exit_code, _ = await run_supervisor(args)+ return exit_code+++def main(argv: Sequence[str] | None = None) -> None:+ raise SystemExit(asyncio.run(async_main(argv)))diff --git a/moli-cdp-smoke/moli_cdp_smoke/worker.py b/moli-cdp-smoke/moli_cdp_smoke/worker.pynew file mode 100644index 000000000..43d02edb4--- /dev/null+++ b/moli-cdp-smoke/moli_cdp_smoke/worker.py@@ -0,0 +1,7 @@+from __future__ import annotations++from .runner import main+++if __name__ == "__main__":+ main()diff --git a/moli-cdp-smoke/pyproject.toml b/moli-cdp-smoke/pyproject.tomlindex a1cac1f4e..75c21d46b 100644--- a/moli-cdp-smoke/pyproject.toml+++ b/moli-cdp-smoke/pyproject.toml@@ -11,7 +11,8 @@ dependencies = [][project.scripts]-moli-cdp-smoke = "moli_cdp_smoke.runner:main"+moli-cdp-smoke = "moli_cdp_smoke.supervisor:main"+moli-cdp-smoke-worker = "moli_cdp_smoke.runner:main"[build-system]requires = ["hatchling"]diff --git a/moli-cdp-smoke/tests/test_external_process.py b/moli-cdp-smoke/tests/test_external_process.pynew file mode 100644index 000000000..8d663e061--- /dev/null+++ b/moli-cdp-smoke/tests/test_external_process.py@@ -0,0 +1,87 @@+from __future__ import annotations++import asyncio+import json+import os+import signal+import sys+import unittest++from moli_cdp_smoke.assertions import SmokeError+from moli_cdp_smoke.groups.external_process import run_external_json_process+from moli_cdp_smoke.process import terminate_process_tree+++class ExternalProcessTests(unittest.IsolatedAsyncioTestCase):+ async def test_successful_json_process_extends_results(self) -> None:+ expected = {"name": "external-process-probe", "ok": True}+ payload = json.dumps({"ok": True, "results": [expected]})+ results: list[dict[str, object]] = []++ await run_external_json_process(+ "test",+ [sys.executable, "-c", f"print({payload!r})"],+ results,+ timeout_seconds=5,+ )++ self.assertEqual(results, [expected])++ @unittest.skipUnless(os.name == "posix", "process-group regression is POSIX-specific")+ async def test_timeout_kills_descendants_holding_output_pipes(self) -> None:+ descendant = "import time; time.sleep(4)"+ parent = (+ "import subprocess, sys, time; "+ f"subprocess.Popen([sys.executable, '-c', {descendant!r}], "+ "stdout=sys.stdout, stderr=sys.stderr); "+ "print('descendant-started', flush=True); "+ "time.sleep(30)"+ )+ loop = asyncio.get_running_loop()+ started_at = loop.time()++ with self.assertRaisesRegex(SmokeError, "test smoke timed out") as raised:+ await run_external_json_process(+ "test",+ [sys.executable, "-c", parent],+ [],+ timeout_seconds=1,+ )++ elapsed = loop.time() - started_at+ self.assertIn("descendant-started", str(raised.exception))+ self.assertLess(+ elapsed,+ 2.5,+ "timeout cleanup waited for the descendant to close inherited pipes",+ )++ @unittest.skipUnless(os.name == "posix", "signal regression is POSIX-specific")+ async def test_service_cleanup_escalates_to_sigkill_with_a_bound(self) -> None:+ process = await asyncio.create_subprocess_exec(+ sys.executable,+ "-c",+ (+ "import signal, time; "+ "signal.signal(signal.SIGTERM, signal.SIG_IGN); "+ "print('ready', flush=True); "+ "time.sleep(30)"+ ),+ stdout=asyncio.subprocess.PIPE,+ start_new_session=True,+ )+ self.assertIsNotNone(process.stdout)+ self.assertEqual(await process.stdout.readline(), b"ready\n")++ stopped = await terminate_process_tree(+ process,+ terminate_timeout_seconds=0.05,+ kill_timeout_seconds=1,+ )++ self.assertTrue(stopped)+ self.assertEqual(process.returncode, -signal.SIGKILL)+++if __name__ == "__main__":+ unittest.main()diff --git a/moli-cdp-smoke/tests/test_runner_progress.py b/moli-cdp-smoke/tests/test_runner_progress.pyindex 5867a5a4a..c0c475ae1 100644--- a/moli-cdp-smoke/tests/test_runner_progress.py+++ b/moli-cdp-smoke/tests/test_runner_progress.py@@ -61,6 +61,35 @@ class RunnerProgressTests(unittest.IsolatedAsyncioTestCase):r"elapsed=\d+\.\d{3}s error=CancelledError$",)+ async def test_timeout_cancels_the_operation_and_reports_its_label(self) -> None:+ cancelled = asyncio.Event()++ async def wait_forever() -> None:+ try:+ await asyncio.Event().wait()+ finally:+ cancelled.set()++ stderr = io.StringIO()+ with redirect_stderr(stderr), self.assertRaisesRegex(+ TimeoutError,+ r"test/timeout timed out after 0\.01s",+ ):+ await await_with_progress(+ "test/timeout",+ wait_forever(),+ timeout_seconds=0.01,+ )++ self.assertTrue(cancelled.is_set())+ lines = stderr.getvalue().splitlines()+ self.assertEqual(lines[0], "[moli-cdp-smoke] START test/timeout")+ self.assertRegex(+ lines[1],+ r"^\[moli-cdp-smoke\] FAIL test/timeout "+ r"elapsed=\d+\.\d{3}s error=TimeoutError$",+ )+if __name__ == "__main__":unittest.main()diff --git a/moli-cdp-smoke/tests/test_serve.py b/moli-cdp-smoke/tests/test_serve.pynew file mode 100644index 000000000..2fcee549f--- /dev/null+++ b/moli-cdp-smoke/tests/test_serve.py@@ -0,0 +1,97 @@+from __future__ import annotations++import asyncio+import unittest+from typing import cast++from moli_cdp_smoke.serve import (+ MoliServe,+ _collect_process_output,+ _moli_endpoint_from_log_line,+ wait_for_moli_endpoint,+)+++class _FakeProcess:+ def __init__(self, returncode: int | None = None) -> None:+ self.returncode = returncode+ self._never_exits = asyncio.Event()++ async def wait(self) -> int:+ if self.returncode is None:+ await self._never_exits.wait()+ assert self.returncode is not None+ return self.returncode+++class ServeLogTests(unittest.IsolatedAsyncioTestCase):+ def test_bound_endpoint_is_parsed_from_plain_and_ansi_logs(self) -> None:+ plain = "INFO protocol server listening addr=127.0.0.1:41827"+ ansi = (+ "\x1b[2m2026-08-24T01:02:03Z\x1b[0m "+ "\x1b[32m INFO\x1b[0m protocol server listening "+ "\x1b[3maddr\x1b[0m\x1b[2m=\x1b[0m127.0.0.1:53109"+ )++ self.assertEqual(+ _moli_endpoint_from_log_line(plain), "http://127.0.0.1:41827"+ )+ self.assertEqual(+ _moli_endpoint_from_log_line(ansi), "http://127.0.0.1:53109"+ )+ self.assertIsNone(_moli_endpoint_from_log_line("unrelated addr=127.0.0.1:9"))+ self.assertIsNone(+ _moli_endpoint_from_log_line(+ "protocol server listening addr=127.0.0.1:99999"+ )+ )++ async def test_output_collector_publishes_the_bound_endpoint(self) -> None:+ stream = asyncio.StreamReader()+ stream.feed_data(+ b"INFO protocol server listening addr=127.0.0.1:42123\n"+ )+ stream.feed_eof()+ logs: list[str] = []+ endpoint_ready: asyncio.Future[str] = (+ asyncio.get_running_loop().create_future()+ )++ await _collect_process_output(stream, logs, "stderr", endpoint_ready)++ self.assertEqual(endpoint_ready.result(), "http://127.0.0.1:42123")+ self.assertEqual(len(logs), 1)++ async def test_wait_reports_process_exit_before_the_listening_log(self) -> None:+ endpoint_ready: asyncio.Future[str] = (+ asyncio.get_running_loop().create_future()+ )+ serve = MoliServe(+ process=cast(asyncio.subprocess.Process, _FakeProcess(7)),+ logs=["stderr: startup failed"],+ tasks=[],+ http_cache_dir="",+ endpoint_ready=endpoint_ready,+ )++ with self.assertRaisesRegex(RuntimeError, "exited early with 7"):+ await wait_for_moli_endpoint(serve, timeout_seconds=1)++ async def test_wait_for_listening_log_has_a_bounded_timeout(self) -> None:+ endpoint_ready: asyncio.Future[str] = (+ asyncio.get_running_loop().create_future()+ )+ serve = MoliServe(+ process=cast(asyncio.subprocess.Process, _FakeProcess()),+ logs=[],+ tasks=[],+ http_cache_dir="",+ endpoint_ready=endpoint_ready,+ )++ with self.assertRaisesRegex(RuntimeError, "bound endpoint"):+ await wait_for_moli_endpoint(serve, timeout_seconds=0.01)+++if __name__ == "__main__":+ unittest.main()diff --git a/moli-cdp-smoke/tests/test_supervisor.py b/moli-cdp-smoke/tests/test_supervisor.pynew file mode 100644index 000000000..4a86680c9--- /dev/null+++ b/moli-cdp-smoke/tests/test_supervisor.py@@ -0,0 +1,207 @@+from __future__ import annotations++import asyncio+import json+import os+import sys+import tempfile+import unittest+from pathlib import Path+from unittest.mock import patch++from moli_cdp_smoke.runner import GROUPS_BY_NAME+from moli_cdp_smoke.supervisor import (+ WorkerJob,+ WorkerOutcome,+ _run_worker_job,+ parse_args,+ run_supervisor,+)+++class SupervisorWorkerTests(unittest.IsolatedAsyncioTestCase):+ async def test_worker_result_is_read_from_its_isolated_file(self) -> None:+ group = GROUPS_BY_NAME["protocol"]+ job = WorkerJob(group=group, attempt=1, repeat=1)+ with tempfile.TemporaryDirectory() as temporary:+ output_dir = Path(temporary)++ def fake_argv(_: WorkerJob, result_path: Path) -> list[str]:+ payload = json.dumps(+ {+ "ok": True,+ "group": "protocol",+ "results": [{"name": "probe", "ok": True}],+ }+ )+ return [+ sys.executable,+ "-c",+ (+ "from pathlib import Path; import sys; "+ f"Path(sys.argv[1]).write_text({payload!r}, encoding='utf-8'); "+ "print('worker-log')"+ ),+ str(result_path),+ ]++ with patch("moli_cdp_smoke.supervisor._worker_argv", fake_argv):+ outcome = await _run_worker_job(job, output_dir, 5)++ self.assertTrue(outcome.passed)+ self.assertEqual(outcome.scenario_count, 1)+ self.assertEqual(outcome.log_path.read_text().strip(), "worker-log")++ async def test_worker_wall_clock_timeout_is_bounded(self) -> None:+ group = GROUPS_BY_NAME["protocol"]+ job = WorkerJob(group=group, attempt=1, repeat=1)+ with tempfile.TemporaryDirectory() as temporary:+ output_dir = Path(temporary)+ stale_result = output_dir / "protocol.json"+ stale_result.write_text(+ json.dumps(+ {+ "ok": True,+ "group": "protocol",+ "results": [{"name": "stale", "ok": True}],+ }+ ),+ encoding="utf-8",+ )++ def hanging_argv(_: WorkerJob, __: Path) -> list[str]:+ return [sys.executable, "-c", "import time; time.sleep(30)"]++ loop = asyncio.get_running_loop()+ started_at = loop.time()+ with patch("moli_cdp_smoke.supervisor._worker_argv", hanging_argv):+ outcome = await _run_worker_job(job, output_dir, 0.05)++ self.assertEqual(outcome.status, "timed_out")+ self.assertEqual(outcome.scenario_count, 0)+ self.assertFalse(stale_result.exists())+ self.assertLess(loop.time() - started_at, 2)++ @unittest.skipUnless(+ sys.platform.startswith("linux"),+ "descendant process-state regression uses Linux /proc",+ )+ async def test_worker_timeout_signals_descendants_in_its_process_group(self) -> None:+ group = GROUPS_BY_NAME["protocol"]+ job = WorkerJob(group=group, attempt=1, repeat=1)+ with tempfile.TemporaryDirectory() as temporary:+ output_dir = Path(temporary)+ child_ready = output_dir / "child-ready"+ child = f"""+import os+import time+from pathlib import Path++Path({str(child_ready)!r}).write_text(str(os.getpid()), encoding="utf-8")+while True:+ time.sleep(1)+"""+ parent = f"""+import subprocess+import sys+import time+from pathlib import Path++subprocess.Popen([sys.executable, "-c", {child!r}])+ready = Path({str(child_ready)!r})+while not ready.exists():+ time.sleep(0.01)+time.sleep(30)+"""++ def descendant_argv(_: WorkerJob, __: Path) -> list[str]:+ return [sys.executable, "-c", parent]++ with patch("moli_cdp_smoke.supervisor._worker_argv", descendant_argv):+ outcome = await _run_worker_job(job, output_dir, 0.5)++ self.assertEqual(outcome.status, "timed_out")+ self.assertTrue(child_ready.exists())+ child_pid = int(child_ready.read_text(encoding="utf-8"))+ process_state = Path(f"/proc/{child_pid}/stat")+ for _ in range(100):+ if not process_state.exists():+ break+ if process_state.read_text(encoding="utf-8").split()[2] == "Z":+ break+ await asyncio.sleep(0.01)+ else:+ self.fail(f"worker descendant {child_pid} survived process-group cleanup")+++class SupervisorSchedulingTests(unittest.IsolatedAsyncioTestCase):+ async def test_jobs_bound_allows_isolated_workers_to_run_in_parallel(self) -> None:+ with tempfile.TemporaryDirectory() as temporary:+ args = parse_args(+ [+ "--group",+ "protocol",+ "--group",+ "puppeteer",+ "--jobs",+ "2",+ "--output-dir",+ temporary,+ ]+ )+ both_started = asyncio.Event()+ active = 0+ maximum_active = 0++ async def fake_run(+ job: WorkerJob,+ output_dir: Path,+ _: float,+ ) -> WorkerOutcome:+ nonlocal active, maximum_active+ active += 1+ maximum_active = max(maximum_active, active)+ if active == 2:+ both_started.set()+ await asyncio.wait_for(both_started.wait(), timeout=1)+ active -= 1+ return WorkerOutcome(+ job=job,+ status="passed",+ duration_seconds=0,+ exit_code=0,+ log_path=output_dir / f"{job.file_stem}.log",+ result_path=output_dir / f"{job.file_stem}.json",+ scenario_count=1,+ )++ with patch("moli_cdp_smoke.supervisor._run_worker_job", fake_run):+ exit_code, summary = await run_supervisor(args)++ self.assertEqual(exit_code, 0)+ self.assertTrue(summary["ok"])+ self.assertEqual(maximum_active, 2)++ async def test_fixed_port_is_rejected_for_parallel_isolated_workers(self) -> None:+ with tempfile.TemporaryDirectory() as temporary:+ args = parse_args(+ [+ "--group",+ "protocol",+ "--group",+ "puppeteer",+ "--jobs",+ "2",+ "--output-dir",+ temporary,+ ]+ )++ with patch.dict(os.environ, {"MOLI_CDP_PORT": "9333"}), self.assertRaisesRegex(+ RuntimeError, "cannot be shared"+ ):+ await run_supervisor(args)+++if __name__ == "__main__":+ unittest.main()