Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

test(cdp): isolate smoke groups by process

ldm0ea80ecd6a2fa
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 785c0b123..e95b0d22f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -134,7 +134,20 @@ jobs:
continue-on-error: true
env:
MOLI_BIN: ${{ github.workspace }}/target/release/moli
- run: uv run --project moli-cdp-smoke --no-sync moli-cdp-smoke
+ run: >-
+ uv run --project moli-cdp-smoke --no-sync
+ moli-cdp-smoke
+ --jobs 4
+ --output-dir ${{ github.workspace }}/target/smoke/cdp-ci
+
+ - name: Upload CDP smoke diagnostics
+ if: ${{ always() && steps.cdp.outcome != 'skipped' }}
+ continue-on-error: true
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: cdp-smoke-diagnostics
+ path: target/smoke/cdp-ci
+ if-no-files-found: warn
- name: Run full WebDriver smoke
id: webdriver
diff --git a/moli-cdp-smoke/README.md b/moli-cdp-smoke/README.md
index 73565ff69..f8d2ce3fe 100644
--- a/moli-cdp-smoke/README.md
+++ b/moli-cdp-smoke/README.md
@@ -340,11 +340,28 @@ npm ci
uv run moli-cdp-smoke
```
-With no `--group`, the runner executes every raw, Playwright page, browser, and
-repository-managed external-client group, including `inspector-routing` and the
-pinned Puppeteer client. CI invokes that same unfiltered default. External-client
-groups whose binaries or dependency environments are not owned by this project
-remain explicit integration runs.
+With no `--group`, the supervisor executes every raw, Playwright page, browser,
+and repository-managed external-client group, including `inspector-routing`
+and the pinned Puppeteer client. Every group runs in a separate worker process
+with its own Moli process, fixture server, HTTP cache, Playwright context, and
+temporary directories. One failed or wedged group therefore cannot retain
+state or child processes for the next group.
+
+The default concurrency is one worker. Use bounded parallelism explicitly;
+CI runs the same unfiltered suite with four workers:
+
+```bash
+uv run moli-cdp-smoke --jobs 4
+```
+
+Each run writes `summary.json` plus one log and one result JSON per worker under
+`target/smoke/cdp/<run-id>/`. Use `--output-dir` for a stable artifact path and
+`--timeout` to change the per-worker wall-clock limit. The worker also retains
+its own protocol-operation timeouts, so the wall limit is a final process-level
+safety boundary rather than the primary synchronization mechanism.
+
+External-client groups whose binaries or dependency environments are not owned
+by this project remain explicit integration runs.
List available scenario groups:
@@ -407,7 +424,9 @@ ordering between the two routes. A normal debugger pause must pump mixed V8
and non-V8 Main commands from the same session in send order. An
instrumentation pause must leave Main blocked and admit only IO work.
-Run the same focused group against an already-running Chromium CDP endpoint:
+Run the same focused group against an already-running Chromium CDP endpoint.
+The Python workers are still separate, but an explicitly supplied external
+browser endpoint is shared rather than restarted by the supervisor:
```bash
uv run moli-cdp-smoke \
@@ -472,7 +491,9 @@ MOLI_BIN=../target/release/moli uv run moli-cdp-smoke
Useful environment variables:
- `MOLI_BIN`: path to the `moli` binary under test.
-- `MOLI_CDP_PORT`: CDP server port. Defaults to a free local port.
+- `MOLI_CDP_PORT`: fixed CDP server port for single-worker debugging. By
+ default every worker uses `--port 0` and reads the OS-selected endpoint from
+ Moli's bound-listener log. A fixed port is rejected with parallel workers.
- `MOLI_SMOKE_GROUPS`: comma-separated smoke group list. CLI `--group` takes precedence.
- `MOLI_INSPECTOR_ROUTING_SCENARIOS`: comma-separated scenario names within the `inspector-routing` group.
- `MOLI_SMOKE_TRACE=1`: print extra runner-side trace logs.
diff --git a/moli-cdp-smoke/moli_cdp_smoke/groups/agent_browser.py b/moli-cdp-smoke/moli_cdp_smoke/groups/agent_browser.py
index 707dba751..7f888a652 100644
--- a/moli-cdp-smoke/moli_cdp_smoke/groups/agent_browser.py
+++ b/moli-cdp-smoke/moli_cdp_smoke/groups/agent_browser.py
@@ -14,6 +14,7 @@ from urllib.parse import urlparse
from ..assertions import SmokeError, record
from ..config import clear_proxy_env
+from ..process import CapturedProcessTimeout, run_captured_process
from .tracing import _assert_cpu_profile_events, _hot_function_expression
@@ -45,24 +46,17 @@ def _agent_browser_env() -> dict[str, str]:
async def _agent_browser_version(binary: str) -> str:
- process = await asyncio.create_subprocess_exec(
- binary,
- "--version",
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- )
try:
- stdout_bytes, stderr_bytes = await asyncio.wait_for(
- process.communicate(), timeout=5
+ process = await run_captured_process(
+ [binary, "--version"],
+ timeout_seconds=5,
)
- except asyncio.TimeoutError as error:
- process.kill()
- await process.communicate()
+ except CapturedProcessTimeout as error:
raise SmokeError("agent-browser --version timed out") from error
output = "\n".join(
part.decode("utf-8", errors="replace").strip()
- for part in (stdout_bytes, stderr_bytes)
+ for part in (process.stdout, process.stderr)
if part
)
match = re.search(r"\b(\d+)\.(\d+)\.(\d+)\b", output)
@@ -113,28 +107,22 @@ async def _run_agent_browser(
"--json",
*args,
]
- process = await asyncio.create_subprocess_exec(
- *argv,
- cwd=str(cwd),
- env=_agent_browser_env(),
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- )
try:
- stdout_bytes, stderr_bytes = await asyncio.wait_for(
- process.communicate(), timeout=timeout_seconds
+ process = await run_captured_process(
+ argv,
+ timeout_seconds=timeout_seconds,
+ cwd=str(cwd),
+ env=_agent_browser_env(),
)
- except asyncio.TimeoutError as error:
- process.kill()
- stdout_bytes, stderr_bytes = await process.communicate()
+ except CapturedProcessTimeout as error:
raise SmokeError(
f"agent-browser command {args!r} timed out after {timeout_seconds}s\n"
- f"stdout:\n{stdout_bytes.decode(errors='replace')}\n"
- f"stderr:\n{stderr_bytes.decode(errors='replace')}"
+ f"stdout:\n{error.stdout.decode(errors='replace')}\n"
+ f"stderr:\n{error.stderr.decode(errors='replace')}"
) from error
- stdout = stdout_bytes.decode("utf-8", errors="replace")
- stderr = stderr_bytes.decode("utf-8", errors="replace")
+ stdout = process.stdout.decode("utf-8", errors="replace")
+ stderr = process.stderr.decode("utf-8", errors="replace")
return _parse_command_payload(stdout, stderr, argv)
diff --git a/moli-cdp-smoke/moli_cdp_smoke/groups/external_process.py b/moli-cdp-smoke/moli_cdp_smoke/groups/external_process.py
index c944f6bab..0d18be999 100644
--- a/moli-cdp-smoke/moli_cdp_smoke/groups/external_process.py
+++ b/moli-cdp-smoke/moli_cdp_smoke/groups/external_process.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import asyncio
import json
import os
import sys
@@ -9,6 +8,7 @@ from typing import Any
from ..assertions import SmokeError
from ..config import clear_proxy_env
+from ..process import CapturedProcessTimeout, run_captured_process
async def run_external_json_process(
@@ -18,27 +18,28 @@ async def run_external_json_process(
*,
timeout_seconds: float = 45,
) -> None:
- process = await asyncio.create_subprocess_exec(
- *argv,
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- env=clear_proxy_env(os.environ),
- )
try:
- stdout, stderr = await asyncio.wait_for(
- process.communicate(), timeout=timeout_seconds
+ process = await run_captured_process(
+ argv,
+ timeout_seconds=timeout_seconds,
+ env=clear_proxy_env(os.environ),
)
- except asyncio.TimeoutError as error:
- process.kill()
- stdout, stderr = await process.communicate()
+ except CapturedProcessTimeout as error:
+ cleanup_note = ""
+ if not error.output_closed:
+ cleanup_note = (
+ "\nstdout/stderr remained open after process-group termination; "
+ "captured output is unavailable"
+ )
raise SmokeError(
f"{label} smoke timed out after {timeout_seconds:g}s\n"
- f"stdout:\n{stdout.decode(errors='replace')}\n"
- f"stderr:\n{stderr.decode(errors='replace')}"
+ f"stdout:\n{error.stdout.decode(errors='replace')}\n"
+ f"stderr:\n{error.stderr.decode(errors='replace')}"
+ f"{cleanup_note}"
) from error
- stdout_text = stdout.decode("utf-8", errors="replace")
- stderr_text = stderr.decode("utf-8", errors="replace")
+ stdout_text = process.stdout.decode("utf-8", errors="replace")
+ stderr_text = process.stderr.decode("utf-8", errors="replace")
if process.returncode != 0:
raise SmokeError(
f"{label} smoke failed with exit code {process.returncode}\n"
diff --git a/moli-cdp-smoke/moli_cdp_smoke/groups/layout_screenshot.py b/moli-cdp-smoke/moli_cdp_smoke/groups/layout_screenshot.py
index b9acc99e2..cb50d0df3 100644
--- a/moli-cdp-smoke/moli_cdp_smoke/groups/layout_screenshot.py
+++ b/moli-cdp-smoke/moli_cdp_smoke/groups/layout_screenshot.py
@@ -6,11 +6,10 @@ import math
from typing import Any
from ..assertions import SmokeError, assert_equal, record
-from ..config import reserve_port
from ..jpeg_image import jpeg_dimensions
from ..png_image import decode_png
from ..raw_cdp import RawCdpClient, connect_raw_cdp, discover_websocket_url
-from ..serve import start_moli_serve, stop_moli_serve, wait_for_cdp_server
+from ..serve import start_moli_serve, stop_moli_serve, wait_for_moli_endpoint
SCREENSHOT_UNSUPPORTED = (
@@ -648,13 +647,11 @@ async def _run_default_mock_boundary(
fixture: str,
results: list[dict[str, Any]],
) -> None:
- port = reserve_port()
- serve = await start_moli_serve(port, layout=False)
- endpoint = f"http://127.0.0.1:{port}"
+ serve = await start_moli_serve(layout=False)
+ endpoint = await wait_for_moli_endpoint(serve)
client: RawCdpClient | None = None
target_id: str | None = None
try:
- await wait_for_cdp_server(endpoint, serve)
client = await connect_raw_cdp(endpoint)
target_id, session_id = await _open_target(
client,
diff --git a/moli-cdp-smoke/moli_cdp_smoke/groups/puppeteer.py b/moli-cdp-smoke/moli_cdp_smoke/groups/puppeteer.py
index a700f3733..d39727063 100644
--- a/moli-cdp-smoke/moli_cdp_smoke/groups/puppeteer.py
+++ b/moli-cdp-smoke/moli_cdp_smoke/groups/puppeteer.py
@@ -1,61 +1,24 @@
from __future__ import annotations
-import asyncio
-import json
import os
-import sys
from pathlib import Path
from typing import Any
-from ..assertions import SmokeError
-from ..config import clear_proxy_env
+from .external_process import run_external_json_process
PUPPETEER_SCRIPT = Path(__file__).resolve().parents[1] / "puppeteer_smoke.mjs"
async def run_puppeteer_group(endpoint: str, fixture: str, results: list[dict[str, Any]]) -> None:
- node = os.environ.get("NODE", "node")
- process = await asyncio.create_subprocess_exec(
- node,
- str(PUPPETEER_SCRIPT),
- endpoint,
- fixture,
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- env=clear_proxy_env(os.environ),
+ await run_external_json_process(
+ "Puppeteer",
+ [
+ os.environ.get("NODE", "node"),
+ str(PUPPETEER_SCRIPT),
+ endpoint,
+ fixture,
+ ],
+ results,
+ timeout_seconds=30,
)
- try:
- stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=30)
- except asyncio.TimeoutError as error:
- process.kill()
- stdout, stderr = await process.communicate()
- stdout_text = stdout.decode("utf-8", errors="replace")
- stderr_text = stderr.decode("utf-8", errors="replace")
- raise SmokeError(
- f"Puppeteer smoke timed out after 30s\nstdout:\n{stdout_text}\nstderr:\n{stderr_text}"
- ) from error
-
- stdout_text = stdout.decode("utf-8", errors="replace")
- stderr_text = stderr.decode("utf-8", errors="replace")
- if process.returncode != 0:
- raise SmokeError(
- "Puppeteer smoke failed with exit code "
- f"{process.returncode}\nstdout:\n{stdout_text}\nstderr:\n{stderr_text}"
- )
-
- try:
- payload = json.loads(stdout_text)
- except json.JSONDecodeError as error:
- raise SmokeError(
- f"Puppeteer smoke did not return JSON\nstdout:\n{stdout_text}\nstderr:\n{stderr_text}"
- ) from error
-
- if not payload.get("ok"):
- raise SmokeError(f"Puppeteer smoke reported failure: {payload}")
- payload_results = payload.get("results")
- if not isinstance(payload_results, list):
- raise SmokeError(f"Puppeteer smoke returned invalid results: {payload}")
- results.extend(payload_results)
- if os.environ.get("MOLI_SMOKE_TRACE") == "1" and stderr_text:
- print(stderr_text, file=sys.stderr, end="")
diff --git a/moli-cdp-smoke/moli_cdp_smoke/process.py b/moli-cdp-smoke/moli_cdp_smoke/process.py
new file mode 100644
index 000000000..f3de5dd46
--- /dev/null
+++ b/moli-cdp-smoke/moli_cdp_smoke/process.py
@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+import asyncio
+import os
+import signal
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from os import PathLike
+
+
+_PROCESS_CLEANUP_TIMEOUT_SECONDS = 2.0
+INHERIT_PROCESS_GROUP_ENV = "MOLI_SMOKE_INHERIT_PROCESS_GROUP"
+
+
+def subprocess_starts_new_session() -> bool:
+ return os.name == "posix" and os.environ.get(INHERIT_PROCESS_GROUP_ENV) != "1"
+
+
+@dataclass(frozen=True)
+class CapturedProcessResult:
+ returncode: int
+ stdout: bytes
+ stderr: bytes
+
+
+class CapturedProcessTimeout(TimeoutError):
+ def __init__(
+ self,
+ timeout_seconds: float,
+ stdout: bytes,
+ stderr: bytes,
+ *,
+ output_closed: bool,
+ ) -> None:
+ super().__init__(f"process timed out after {timeout_seconds:g}s")
+ self.timeout_seconds = timeout_seconds
+ self.stdout = stdout
+ self.stderr = stderr
+ self.output_closed = output_closed
+
+
+def _consume_task_result(task: asyncio.Task[object]) -> None:
+ try:
+ task.exception()
+ except (asyncio.CancelledError, Exception):
+ pass
+
+
+async def _task_completed(task: asyncio.Task[object], timeout_seconds: float) -> bool:
+ done, _ = await asyncio.wait((task,), timeout=timeout_seconds)
+ return task in done
+
+
+def _signal_process_tree(
+ process: asyncio.subprocess.Process,
+ process_signal: signal.Signals,
+) -> None:
+ if os.name == "posix":
+ try:
+ os.killpg(process.pid, process_signal)
+ return
+ except ProcessLookupError:
+ pass
+ except OSError:
+ # Fall back to the direct child below. The bounded waits still keep
+ # cleanup from becoming a second unbounded failure mode.
+ pass
+
+ if process.returncode is not None:
+ return
+ try:
+ if process_signal == signal.SIGTERM:
+ process.terminate()
+ else:
+ process.kill()
+ except ProcessLookupError:
+ pass
+
+
+async def _finish_communication_after_kill(
+ process: asyncio.subprocess.Process,
+ communication: asyncio.Task[tuple[bytes, bytes]],
+) -> tuple[bytes, bytes, bool]:
+ _signal_process_tree(process, signal.SIGKILL)
+ if await _task_completed(communication, _PROCESS_CLEANUP_TIMEOUT_SECONDS):
+ stdout, stderr = communication.result()
+ return stdout, stderr, True
+
+ # A process which escaped the new process group can still own one of the
+ # inherited pipes. Do not let that escaped descriptor turn timeout cleanup
+ # back into an infinite communicate() wait.
+ communication.cancel()
+ communication.add_done_callback(_consume_task_result)
+ return b"", b"", False
+
+
+async def run_captured_process(
+ argv: Sequence[str],
+ *,
+ timeout_seconds: float,
+ cwd: str | PathLike[str] | None = None,
+ env: Mapping[str, str] | None = None,
+) -> CapturedProcessResult:
+ if timeout_seconds <= 0:
+ raise ValueError("timeout_seconds must be positive")
+
+ process = await asyncio.create_subprocess_exec(
+ *argv,
+ cwd=cwd,
+ env=env,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ # Every managed external command gets its own POSIX process group so a
+ # timeout can terminate descendants which inherited stdout/stderr.
+ start_new_session=subprocess_starts_new_session(),
+ )
+ communication = asyncio.create_task(process.communicate())
+ try:
+ completed = await _task_completed(communication, timeout_seconds)
+ except BaseException:
+ if not communication.done():
+ await _finish_communication_after_kill(process, communication)
+ raise
+
+ if completed:
+ stdout, stderr = communication.result()
+ if process.returncode is None:
+ raise RuntimeError("completed subprocess has no return code")
+ return CapturedProcessResult(process.returncode, stdout, stderr)
+
+ stdout, stderr, output_closed = await _finish_communication_after_kill(
+ process,
+ communication,
+ )
+ raise CapturedProcessTimeout(
+ timeout_seconds,
+ stdout,
+ stderr,
+ output_closed=output_closed,
+ )
+
+
+async def terminate_process_tree(
+ process: asyncio.subprocess.Process,
+ *,
+ terminate_timeout_seconds: float = 5.0,
+ kill_timeout_seconds: float = 2.0,
+) -> bool:
+ if process.returncode is not None:
+ _signal_process_tree(process, signal.SIGKILL)
+ return True
+
+ wait_task = asyncio.create_task(process.wait())
+ _signal_process_tree(process, signal.SIGTERM)
+ if await _task_completed(wait_task, terminate_timeout_seconds):
+ # The dedicated group may still contain descendants after its leader
+ # exits. The caller is tearing the service down, so clear any remaining
+ # group members as well.
+ _signal_process_tree(process, signal.SIGKILL)
+ return True
+
+ _signal_process_tree(process, signal.SIGKILL)
+ if await _task_completed(wait_task, kill_timeout_seconds):
+ return True
+ wait_task.cancel()
+ wait_task.add_done_callback(_consume_task_result)
+ return False
diff --git a/moli-cdp-smoke/moli_cdp_smoke/progress.py b/moli-cdp-smoke/moli_cdp_smoke/progress.py
index aa900cd24..cfb950168 100644
--- a/moli-cdp-smoke/moli_cdp_smoke/progress.py
+++ b/moli-cdp-smoke/moli_cdp_smoke/progress.py
@@ -6,18 +6,51 @@ from typing import Awaitable, TypeVar
ProgressResult = TypeVar("ProgressResult")
+_CANCELLATION_GRACE_SECONDS = 5.0
+
+
+def _consume_task_result(task: asyncio.Future[object]) -> None:
+ try:
+ task.exception()
+ except (asyncio.CancelledError, Exception):
+ pass
+
+
+async def _cancel_with_grace(task: asyncio.Future[object]) -> None:
+ if task.done():
+ return
+ task.cancel()
+ done, _ = await asyncio.wait((task,), timeout=_CANCELLATION_GRACE_SECONDS)
+ if task in done:
+ _consume_task_result(task)
+ else:
+ task.add_done_callback(_consume_task_result)
async def await_with_progress(
label: str,
awaitable: Awaitable[ProgressResult],
+ *,
+ timeout_seconds: float | None = None,
) -> ProgressResult:
+ if timeout_seconds is not None and timeout_seconds <= 0:
+ raise ValueError("timeout_seconds must be positive")
loop = asyncio.get_running_loop()
started_at = loop.time()
print(f"[moli-cdp-smoke] START {label}", file=sys.stderr, flush=True)
+ task = asyncio.ensure_future(awaitable)
try:
- result = await awaitable
+ if timeout_seconds is None:
+ result = await task
+ else:
+ done, _ = await asyncio.wait((task,), timeout=timeout_seconds)
+ if task not in done:
+ raise TimeoutError(
+ f"{label} timed out after {timeout_seconds:g}s"
+ )
+ result = task.result()
except BaseException as error:
+ await _cancel_with_grace(task)
elapsed = loop.time() - started_at
print(
f"[moli-cdp-smoke] FAIL {label} elapsed={elapsed:.3f}s "
diff --git a/moli-cdp-smoke/moli_cdp_smoke/runner.py b/moli-cdp-smoke/moli_cdp_smoke/runner.py
index 0a046f09b..aff357923 100644
--- a/moli-cdp-smoke/moli_cdp_smoke/runner.py
+++ b/moli-cdp-smoke/moli_cdp_smoke/runner.py
@@ -15,7 +15,7 @@ from typing import Any, Awaitable, Callable, Iterable
from playwright.async_api import async_playwright
from .assertions import record
-from .config import clear_current_process_proxy_env, reserve_port
+from .config import clear_current_process_proxy_env
from .fixture import FixtureServer
from .groups.action_window import run_action_window_group
from .groups.agent_browser import run_agent_browser_group
@@ -60,7 +60,13 @@ from .groups.workers import run_workers_group
from .groups.xhr_sync_semantics import run_xhr_sync_semantics_group
from .helpers import attach_cdp_event_collector
from .progress import await_with_progress
-from .serve import MoliServe, start_moli_serve, stop_moli_serve, wait_for_cdp_server
+from .serve import (
+ MoliServe,
+ start_moli_serve,
+ stop_moli_serve,
+ wait_for_cdp_server,
+ wait_for_moli_endpoint,
+)
from .state import SmokeState
@@ -81,6 +87,13 @@ class SmokeGroup:
runner: RawGroupRunner | ExternalGroupRunner | PageGroupRunner | BrowserGroupRunner
+async def _await_group(group: SmokeGroup, awaitable: Awaitable[None]) -> None:
+ await await_with_progress(
+ f"group/{group.phase}/{group.name}",
+ awaitable,
+ )
+
+
RAW_GROUPS: tuple[SmokeGroup, ...] = (
SmokeGroup(
"inspector-routing",
@@ -436,8 +449,11 @@ async def run_smoke(
if results is None:
results = []
temp_dir = Path(tempfile.mkdtemp(prefix="moli-pw-smoke-"))
- playwright = await await_with_progress("playwright/start", async_playwright().start())
+ playwright = None
try:
+ playwright = await await_with_progress(
+ "playwright/start", async_playwright().start()
+ )
browser = await await_with_progress(
"playwright/connect-over-cdp",
playwright.chromium.connect_over_cdp(endpoint, timeout=10_000),
@@ -500,36 +516,37 @@ async def run_smoke(
)
for group in selection.page_groups:
- await await_with_progress(
- f"group/{group.phase}/{group.name}",
+ await _await_group(
+ group,
group.runner(state), # type: ignore[misc]
)
await await_with_progress("playwright/context-close", context.close())
for group in selection.browser_groups:
- await await_with_progress(
- f"group/{group.phase}/{group.name}",
+ await _await_group(
+ group,
group.runner(browser, fixture_server.url, results), # type: ignore[misc]
)
return results
finally:
await await_with_progress("playwright/browser-close", browser.close())
- shutil.rmtree(temp_dir, ignore_errors=True)
finally:
- await await_with_progress("playwright/stop", playwright.stop())
+ if playwright is not None:
+ await await_with_progress("playwright/stop", playwright.stop())
+ shutil.rmtree(temp_dir, ignore_errors=True)
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(
- description="Run Playwright CDP smoke workflows against moli."
+ description="Run one isolated CDP smoke group against moli."
)
parser.add_argument(
"--group",
action="append",
default=[],
help=(
- "Run only the named group. May be repeated or comma-separated. "
- "Defaults to every repository-managed group."
+ "Run exactly one named group. The supervisor is responsible for "
+ "selecting and scheduling multiple groups."
),
)
parser.add_argument(
@@ -541,68 +558,102 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
"--endpoint",
help="Use an existing HTTP CDP endpoint instead of starting moli serve.",
)
+ parser.add_argument(
+ "--result",
+ type=Path,
+ help="Atomically write the worker result JSON to this path.",
+ )
return parser.parse_args(argv)
+def _emit_worker_payload(
+ payload: dict[str, Any],
+ result_path: Path | None,
+ *,
+ failed: bool,
+) -> None:
+ rendered = json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
+ if result_path is None:
+ print(rendered, file=sys.stderr if failed else sys.stdout, end="")
+ return
+ result_path.parent.mkdir(parents=True, exist_ok=True)
+ temporary = result_path.with_name(
+ f".{result_path.name}.tmp-{os.getpid()}"
+ )
+ temporary.write_text(rendered, encoding="utf-8")
+ os.replace(temporary, result_path)
+
+
async def async_main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
if args.list_groups:
print(json.dumps({"groups": group_listing()}, indent=2, ensure_ascii=False))
return 0
selection = resolve_group_selection(args.group)
+ if len(selection.groups) != 1:
+ raise RuntimeError(
+ "moli-cdp-smoke-worker requires exactly one --group; "
+ "use moli-cdp-smoke to schedule the suite"
+ )
+ selected_group = selection.groups[0]
fixture = FixtureServer()
+ fixture_started = False
serve: MoliServe | None = None
results: list[dict[str, Any]] = []
+ endpoint: str | None = None
+ failures: list[str] = []
try:
fixture.start()
+ fixture_started = True
if args.endpoint:
endpoint = args.endpoint.rstrip("/")
else:
port_env = os.environ.get("MOLI_CDP_PORT")
- port = int(port_env) if port_env else reserve_port()
- if port <= 0 or port > 65535:
+ port = int(port_env) if port_env else 0
+ if port < 0 or port > 65535 or (port == 0 and port_env is not None):
raise RuntimeError(f"invalid MOLI_CDP_PORT: {port_env}")
serve = await start_moli_serve(port)
- endpoint = f"http://127.0.0.1:{port}"
- await wait_for_cdp_server(endpoint, serve)
- for group in selection.raw_groups:
- await await_with_progress(
- f"group/{group.phase}/{group.name}",
- group.runner(endpoint, fixture.url, results), # type: ignore[misc]
+ endpoint = await wait_for_moli_endpoint(serve)
+ if endpoint is None:
+ raise RuntimeError("CDP endpoint was not initialized")
+ if serve is None:
+ await wait_for_cdp_server(endpoint, serve)
+ for current_group in selection.raw_groups:
+ await _await_group(
+ current_group,
+ current_group.runner(endpoint, fixture.url, results), # type: ignore[misc]
)
await run_smoke(endpoint, fixture, selection, results)
- for group in selection.external_groups:
- await await_with_progress(
- f"group/{group.phase}/{group.name}",
- group.runner(endpoint, fixture.url, results), # type: ignore[misc]
+ for current_group in selection.external_groups:
+ await _await_group(
+ current_group,
+ current_group.runner(endpoint, fixture.url, results), # type: ignore[misc]
)
- ok = not any(result.get("ok") is False for result in results)
- print(
- json.dumps(
- {
- "ok": ok,
- "endpoint": endpoint,
- "fixture": fixture.url,
- "results": results,
- },
- indent=2,
- ensure_ascii=False,
- )
- )
- return 0 if ok else 1
except Exception as error:
- print(
- json.dumps(
- {"ok": False, "error": "".join(traceback.format_exception(error)), "results": results},
- indent=2,
- ensure_ascii=False,
- ),
- file=sys.stderr,
- )
- return 1
+ failures.append("".join(traceback.format_exception(error)))
finally:
- await stop_moli_serve(serve)
- fixture.stop()
+ try:
+ await stop_moli_serve(serve)
+ except Exception as error:
+ failures.append("".join(traceback.format_exception(error)))
+ if fixture_started:
+ try:
+ fixture.stop()
+ except Exception as error:
+ failures.append("".join(traceback.format_exception(error)))
+
+ ok = not failures and not any(result.get("ok") is False for result in results)
+ payload: dict[str, Any] = {
+ "ok": ok,
+ "group": selected_group.name,
+ "endpoint": endpoint,
+ "fixture": fixture.url,
+ "results": results,
+ }
+ if failures:
+ payload["error"] = "\n".join(failures)
+ _emit_worker_payload(payload, args.result, failed=not ok)
+ return 0 if ok else 1
def main(argv: list[str] | None = None) -> None:
diff --git a/moli-cdp-smoke/moli_cdp_smoke/serve.py b/moli-cdp-smoke/moli_cdp_smoke/serve.py
index b7f0ebb14..88371e838 100644
--- a/moli-cdp-smoke/moli_cdp_smoke/serve.py
+++ b/moli-cdp-smoke/moli_cdp_smoke/serve.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import os
+import re
import shutil
import sys
import tempfile
@@ -12,6 +13,7 @@ from dataclasses import dataclass
from typing import Any
from .config import REPO_ROOT, clear_proxy_env, moli_binary
+from .process import subprocess_starts_new_session, terminate_process_tree
@dataclass
@@ -20,9 +22,38 @@ class MoliServe:
logs: list[str]
tasks: list[asyncio.Task[Any]]
http_cache_dir: str
+ endpoint_ready: asyncio.Future[str]
-async def _collect_process_output(stream: asyncio.StreamReader | None, logs: list[str], label: str) -> None:
+_ANSI_ESCAPE = re.compile(r"\x1b\[[0-9;]*m")
+_LISTENING_ADDRESS = re.compile(
+ r"\bprotocol server listening\b.*\baddr=127\.0\.0\.1:(?P<port>[0-9]{1,5})\b"
+)
+
+
+def _moli_endpoint_from_log_line(line: str) -> str | None:
+ match = _LISTENING_ADDRESS.search(_ANSI_ESCAPE.sub("", line))
+ if match is None:
+ return None
+ port = int(match.group("port"))
+ if not 0 < port <= 65535:
+ return None
+ return f"http://127.0.0.1:{port}"
+
+
+def _consume_task_result(task: asyncio.Task[Any]) -> None:
+ try:
+ task.exception()
+ except asyncio.CancelledError:
+ pass
+
+
+async def _collect_process_output(
+ stream: asyncio.StreamReader | None,
+ logs: list[str],
+ label: str,
+ endpoint_ready: asyncio.Future[str],
+) -> None:
if stream is None:
return
while True:
@@ -31,12 +62,15 @@ async def _collect_process_output(stream: asyncio.StreamReader | None, logs: lis
return
text = line.decode("utf-8", errors="replace").rstrip()
logs.append(f"{label}: {text}")
+ endpoint = _moli_endpoint_from_log_line(text)
+ if endpoint is not None and not endpoint_ready.done():
+ endpoint_ready.set_result(endpoint)
if os.environ.get("MOLI_SMOKE_TRACE_BG") == "1":
print(f"[moli serve {label}] {text}", file=sys.stderr, flush=True)
async def start_moli_serve(
- port: int,
+ port: int = 0,
*,
layout: bool = True,
extra_args: tuple[str, ...] = (),
@@ -51,6 +85,8 @@ async def start_moli_serve(
"--port",
str(port),
"--resource",
+ "--log-level",
+ "info",
]
if layout:
command.append("--layout")
@@ -62,37 +98,44 @@ async def start_moli_serve(
env=clear_proxy_env(os.environ),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
+ start_new_session=subprocess_starts_new_session(),
)
except Exception:
shutil.rmtree(http_cache_dir, ignore_errors=True)
raise
logs: list[str] = []
+ endpoint_ready = asyncio.get_running_loop().create_future()
tasks = [
- asyncio.create_task(_collect_process_output(process.stdout, logs, "stdout")),
- asyncio.create_task(_collect_process_output(process.stderr, logs, "stderr")),
+ asyncio.create_task(
+ _collect_process_output(process.stdout, logs, "stdout", endpoint_ready)
+ ),
+ asyncio.create_task(
+ _collect_process_output(process.stderr, logs, "stderr", endpoint_ready)
+ ),
]
return MoliServe(
process=process,
logs=logs,
tasks=tasks,
http_cache_dir=http_cache_dir,
+ endpoint_ready=endpoint_ready,
)
async def stop_moli_serve(serve: MoliServe | None) -> None:
if serve is None:
return
- if serve.process.returncode is None:
- serve.process.terminate()
- try:
- await asyncio.wait_for(serve.process.wait(), timeout=5)
- except asyncio.TimeoutError:
- serve.process.kill()
- await serve.process.wait()
+ stopped = await terminate_process_tree(serve.process)
for task in serve.tasks:
task.cancel()
- await asyncio.gather(*serve.tasks, return_exceptions=True)
+ done, pending = await asyncio.wait(serve.tasks, timeout=2)
+ for task in done:
+ _consume_task_result(task)
+ for task in pending:
+ task.add_done_callback(_consume_task_result)
shutil.rmtree(serve.http_cache_dir, ignore_errors=True)
+ if not stopped:
+ raise RuntimeError(f"moli serve process {serve.process.pid} did not exit after SIGKILL")
def _probe_url(url: str) -> bool:
@@ -120,3 +163,35 @@ async def wait_for_cdp_server(
tail_text = "\n".join(serve.logs[-80:]) if serve is not None else ""
tail = f"\n{tail_text}" if tail_text else ""
raise RuntimeError(f"timed out waiting for CDP server at {endpoint}{tail}")
+
+
+async def wait_for_moli_endpoint(
+ serve: MoliServe,
+ *,
+ timeout_seconds: float = 10,
+) -> str:
+ process_exit = asyncio.create_task(serve.process.wait())
+ try:
+ done, _ = await asyncio.wait(
+ (serve.endpoint_ready, process_exit),
+ timeout=timeout_seconds,
+ return_when=asyncio.FIRST_COMPLETED,
+ )
+ if serve.endpoint_ready in done:
+ endpoint = serve.endpoint_ready.result()
+ await wait_for_cdp_server(endpoint, serve)
+ return endpoint
+ if process_exit in done:
+ tail = "\n".join(serve.logs[-80:])
+ raise RuntimeError(
+ f"moli serve exited early with {process_exit.result()}\n{tail}"
+ )
+ tail_text = "\n".join(serve.logs[-80:])
+ tail = f"\n{tail_text}" if tail_text else ""
+ raise RuntimeError(
+ "timed out waiting for moli to report its bound endpoint" + tail
+ )
+ finally:
+ if not process_exit.done():
+ process_exit.cancel()
+ await asyncio.gather(process_exit, return_exceptions=True)
diff --git a/moli-cdp-smoke/moli_cdp_smoke/supervisor.py b/moli-cdp-smoke/moli_cdp_smoke/supervisor.py
new file mode 100644
index 000000000..aea7c19f3
--- /dev/null
+++ b/moli-cdp-smoke/moli_cdp_smoke/supervisor.py
@@ -0,0 +1,403 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import sys
+import time
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, Sequence
+
+from .config import REPO_ROOT, clear_proxy_env
+from .process import INHERIT_PROCESS_GROUP_ENV, terminate_process_tree
+from .runner import SmokeGroup, group_listing, resolve_group_selection
+
+
+_DEFAULT_JOBS = 1
+_DEFAULT_GROUP_TIMEOUT_SECONDS = 120.0
+
+
+@dataclass(frozen=True)
+class WorkerJob:
+ group: SmokeGroup
+ attempt: int
+ repeat: int
+ endpoint: str | None = None
+
+ @property
+ def label(self) -> str:
+ if self.repeat == 1:
+ return self.group.name
+ return f"{self.group.name}#{self.attempt}"
+
+ @property
+ def file_stem(self) -> str:
+ if self.repeat == 1:
+ return self.group.name
+ return f"{self.group.name}-run-{self.attempt:02d}"
+
+
+@dataclass(frozen=True)
+class WorkerOutcome:
+ job: WorkerJob
+ status: str
+ duration_seconds: float
+ exit_code: int | None
+ log_path: Path
+ result_path: Path
+ scenario_count: int
+ endpoint: str | None = None
+ error: str | None = None
+
+ @property
+ def passed(self) -> bool:
+ return self.status == "passed"
+
+ def summary(self) -> dict[str, Any]:
+ payload: dict[str, Any] = {
+ "group": self.job.group.name,
+ "attempt": self.job.attempt,
+ "status": self.status,
+ "durationSeconds": round(self.duration_seconds, 3),
+ "exitCode": self.exit_code,
+ "scenarioCount": self.scenario_count,
+ "log": str(self.log_path),
+ "result": str(self.result_path),
+ }
+ if self.error:
+ payload["error"] = self.error
+ if self.endpoint:
+ payload["endpoint"] = self.endpoint
+ return payload
+
+
+def _positive_int(raw: str) -> int:
+ value = int(raw)
+ if value <= 0:
+ raise argparse.ArgumentTypeError("value must be positive")
+ return value
+
+
+def _positive_float(raw: str) -> float:
+ value = float(raw)
+ if not 0 < value < float("inf"):
+ raise argparse.ArgumentTypeError("value must be a finite positive number")
+ return value
+
+
+def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ description="Run CDP smoke groups in isolated worker processes."
+ )
+ parser.add_argument(
+ "--group",
+ action="append",
+ default=[],
+ help=(
+ "Select a group. May be repeated or comma-separated. Defaults to "
+ "every repository-managed group."
+ ),
+ )
+ parser.add_argument(
+ "--jobs",
+ type=_positive_int,
+ default=_DEFAULT_JOBS,
+ help="Maximum isolated workers to run concurrently (default: 1).",
+ )
+ parser.add_argument(
+ "--timeout",
+ type=_positive_float,
+ default=_DEFAULT_GROUP_TIMEOUT_SECONDS,
+ help="Wall-clock timeout for each worker in seconds (default: 120).",
+ )
+ parser.add_argument(
+ "--repeat",
+ type=_positive_int,
+ default=1,
+ help="Run every selected group this many times; any failure fails the suite.",
+ )
+ parser.add_argument(
+ "--fail-fast",
+ action="store_true",
+ help="Do not start queued workers after the first failure.",
+ )
+ parser.add_argument(
+ "--output-dir",
+ type=Path,
+ help="Directory for per-worker logs/results and summary.json.",
+ )
+ parser.add_argument(
+ "--endpoint",
+ help=(
+ "Use an existing HTTP CDP endpoint. Workers remain separate, but "
+ "the external browser process is shared."
+ ),
+ )
+ parser.add_argument(
+ "--list-groups",
+ action="store_true",
+ help="List available groups as JSON and exit.",
+ )
+ return parser.parse_args(argv)
+
+
+def _default_output_dir() -> Path:
+ timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
+ return REPO_ROOT / "target" / "smoke" / "cdp" / f"{timestamp}-{os.getpid()}"
+
+
+def _worker_argv(job: WorkerJob, result_path: Path) -> list[str]:
+ argv = [
+ sys.executable,
+ "-m",
+ "moli_cdp_smoke.worker",
+ "--group",
+ job.group.name,
+ "--result",
+ str(result_path),
+ ]
+ if job.endpoint:
+ argv.extend(("--endpoint", job.endpoint))
+ return argv
+
+
+def _worker_environment() -> dict[str, str]:
+ env = clear_proxy_env(os.environ)
+ env[INHERIT_PROCESS_GROUP_ENV] = "1"
+ env["PYTHONUNBUFFERED"] = "1"
+ return env
+
+
+def _read_worker_payload(result_path: Path) -> tuple[dict[str, Any] | None, str | None]:
+ try:
+ payload = json.loads(result_path.read_text(encoding="utf-8"))
+ except FileNotFoundError:
+ return None, "worker did not publish a result file"
+ except (OSError, json.JSONDecodeError) as error:
+ return None, f"worker result is unreadable: {error}"
+ if not isinstance(payload, dict):
+ return None, "worker result root is not an object"
+ return payload, None
+
+
+async def _wait_for_worker(
+ process: asyncio.subprocess.Process,
+ timeout_seconds: float,
+) -> tuple[int | None, bool]:
+ wait_task = asyncio.create_task(process.wait())
+ try:
+ done, _ = await asyncio.wait((wait_task,), timeout=timeout_seconds)
+ except BaseException:
+ await terminate_process_tree(
+ process,
+ terminate_timeout_seconds=2,
+ kill_timeout_seconds=2,
+ )
+ raise
+ if wait_task in done:
+ exit_code = wait_task.result()
+ # A successful worker should have cleaned up its children. Clear any
+ # process which escaped worker teardown before the process group can be
+ # reused by another isolated job.
+ await terminate_process_tree(
+ process,
+ terminate_timeout_seconds=0,
+ kill_timeout_seconds=1,
+ )
+ return exit_code, False
+
+ await terminate_process_tree(
+ process,
+ terminate_timeout_seconds=2,
+ kill_timeout_seconds=2,
+ )
+ return process.returncode, True
+
+
+async def _run_worker_job(
+ job: WorkerJob,
+ output_dir: Path,
+ timeout_seconds: float,
+) -> WorkerOutcome:
+ log_path = output_dir / f"{job.file_stem}.log"
+ result_path = output_dir / f"{job.file_stem}.json"
+ result_path.unlink(missing_ok=True)
+ started_at = time.monotonic()
+ print(f"[moli-cdp-supervisor] START {job.label}", file=sys.stderr, flush=True)
+ with log_path.open("wb", buffering=0) as log_stream:
+ process = await asyncio.create_subprocess_exec(
+ *_worker_argv(job, result_path),
+ cwd=str(REPO_ROOT),
+ env=_worker_environment(),
+ stdout=log_stream,
+ stderr=asyncio.subprocess.STDOUT,
+ start_new_session=os.name == "posix",
+ )
+ exit_code, timed_out = await _wait_for_worker(process, timeout_seconds)
+
+ duration = time.monotonic() - started_at
+ payload, payload_error = _read_worker_payload(result_path)
+ scenario_count = 0
+ endpoint = None
+ if payload is not None and isinstance(payload.get("results"), list):
+ scenario_count = len(payload["results"])
+ if payload is not None and isinstance(payload.get("endpoint"), str):
+ endpoint = payload["endpoint"]
+
+ if timed_out:
+ status = "timed_out"
+ error = f"worker exceeded {timeout_seconds:g}s"
+ elif payload_error:
+ status = "crashed"
+ error = payload_error
+ elif payload is None:
+ status = "crashed"
+ error = "worker result is unavailable"
+ elif payload.get("group") != job.group.name:
+ status = "crashed"
+ error = f"worker reported the wrong group: {payload.get('group')!r}"
+ elif exit_code == 0 and payload.get("ok") is True:
+ status = "passed"
+ error = None
+ else:
+ status = "failed"
+ worker_error = payload.get("error")
+ error = str(worker_error) if worker_error else f"worker exited with {exit_code}"
+
+ outcome = WorkerOutcome(
+ job=job,
+ status=status,
+ duration_seconds=duration,
+ exit_code=exit_code,
+ log_path=log_path,
+ result_path=result_path,
+ scenario_count=scenario_count,
+ endpoint=endpoint,
+ error=error,
+ )
+ print(
+ f"[moli-cdp-supervisor] {status.upper()} {job.label} "
+ f"elapsed={duration:.3f}s",
+ file=sys.stderr,
+ flush=True,
+ )
+ return outcome
+
+
+def _log_tail(path: Path, *, lines: int = 40) -> str:
+ try:
+ content = path.read_text(encoding="utf-8", errors="replace").splitlines()
+ except OSError as error:
+ return f"<failed to read log: {error}>"
+ return "\n".join(content[-lines:])
+
+
+def _write_json_atomic(path: Path, payload: dict[str, Any]) -> None:
+ rendered = json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
+ temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
+ temporary.write_text(rendered, encoding="utf-8")
+ os.replace(temporary, path)
+
+
+async def run_supervisor(args: argparse.Namespace) -> tuple[int, dict[str, Any]]:
+ selection = resolve_group_selection(args.group)
+ output_dir = (args.output_dir or _default_output_dir()).resolve()
+ output_dir.mkdir(parents=True, exist_ok=True)
+ jobs = [
+ WorkerJob(
+ group=group,
+ attempt=attempt,
+ repeat=args.repeat,
+ endpoint=args.endpoint.rstrip("/") if args.endpoint else None,
+ )
+ for group in selection.groups
+ for attempt in range(1, args.repeat + 1)
+ ]
+ fixed_port = os.environ.get("MOLI_CDP_PORT")
+ if (
+ args.endpoint is None
+ and fixed_port is not None
+ and args.jobs > 1
+ and len(jobs) > 1
+ ):
+ raise RuntimeError(
+ "MOLI_CDP_PORT cannot be shared by parallel isolated workers; "
+ "unset it to use atomic OS-assigned ports or run with --jobs 1"
+ )
+ semaphore = asyncio.Semaphore(args.jobs)
+ stop_starting = asyncio.Event()
+
+ async def execute(job: WorkerJob) -> WorkerOutcome:
+ async with semaphore:
+ if args.fail_fast and stop_starting.is_set():
+ return WorkerOutcome(
+ job=job,
+ status="skipped",
+ duration_seconds=0,
+ exit_code=None,
+ log_path=output_dir / f"{job.file_stem}.log",
+ result_path=output_dir / f"{job.file_stem}.json",
+ scenario_count=0,
+ error="not started after an earlier failure",
+ )
+ try:
+ outcome = await _run_worker_job(job, output_dir, args.timeout)
+ except Exception as error:
+ outcome = WorkerOutcome(
+ job=job,
+ status="crashed",
+ duration_seconds=0,
+ exit_code=None,
+ log_path=output_dir / f"{job.file_stem}.log",
+ result_path=output_dir / f"{job.file_stem}.json",
+ scenario_count=0,
+ error=f"failed to start or supervise worker: {error}",
+ )
+ if not outcome.passed:
+ stop_starting.set()
+ return outcome
+
+ outcomes = await asyncio.gather(*(execute(job) for job in jobs))
+ ok = all(outcome.passed for outcome in outcomes)
+ summary = {
+ "ok": ok,
+ "jobs": args.jobs,
+ "timeoutSeconds": args.timeout,
+ "repeat": args.repeat,
+ "externalEndpoint": args.endpoint,
+ "outputDirectory": str(output_dir),
+ "groups": [outcome.summary() for outcome in outcomes],
+ }
+ _write_json_atomic(output_dir / "summary.json", summary)
+
+ for outcome in outcomes:
+ if outcome.passed or outcome.status == "skipped":
+ continue
+ print(
+ f"\n--- {outcome.job.label} ({outcome.status}) log tail ---\n"
+ f"{_log_tail(outcome.log_path)}",
+ file=sys.stderr,
+ )
+ passed = sum(outcome.passed for outcome in outcomes)
+ print(
+ f"[moli-cdp-supervisor] summary {passed}/{len(outcomes)} passed; "
+ f"artifacts={output_dir}",
+ file=sys.stderr,
+ )
+ return (0 if ok else 1), summary
+
+
+async def async_main(argv: Sequence[str] | None = None) -> int:
+ args = parse_args(argv)
+ if args.list_groups:
+ print(json.dumps({"groups": group_listing()}, indent=2, ensure_ascii=False))
+ return 0
+ exit_code, _ = await run_supervisor(args)
+ return exit_code
+
+
+def main(argv: Sequence[str] | None = None) -> None:
+ raise SystemExit(asyncio.run(async_main(argv)))
diff --git a/moli-cdp-smoke/moli_cdp_smoke/worker.py b/moli-cdp-smoke/moli_cdp_smoke/worker.py
new file mode 100644
index 000000000..43d02edb4
--- /dev/null
+++ b/moli-cdp-smoke/moli_cdp_smoke/worker.py
@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from .runner import main
+
+
+if __name__ == "__main__":
+ main()
diff --git a/moli-cdp-smoke/pyproject.toml b/moli-cdp-smoke/pyproject.toml
index a1cac1f4e..75c21d46b 100644
--- a/moli-cdp-smoke/pyproject.toml
+++ b/moli-cdp-smoke/pyproject.toml
@@ -11,7 +11,8 @@ dependencies = [
]
[project.scripts]
-moli-cdp-smoke = "moli_cdp_smoke.runner:main"
+moli-cdp-smoke = "moli_cdp_smoke.supervisor:main"
+moli-cdp-smoke-worker = "moli_cdp_smoke.runner:main"
[build-system]
requires = ["hatchling"]
diff --git a/moli-cdp-smoke/tests/test_external_process.py b/moli-cdp-smoke/tests/test_external_process.py
new file mode 100644
index 000000000..8d663e061
--- /dev/null
+++ b/moli-cdp-smoke/tests/test_external_process.py
@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import signal
+import sys
+import unittest
+
+from moli_cdp_smoke.assertions import SmokeError
+from moli_cdp_smoke.groups.external_process import run_external_json_process
+from moli_cdp_smoke.process import terminate_process_tree
+
+
+class ExternalProcessTests(unittest.IsolatedAsyncioTestCase):
+ async def test_successful_json_process_extends_results(self) -> None:
+ expected = {"name": "external-process-probe", "ok": True}
+ payload = json.dumps({"ok": True, "results": [expected]})
+ results: list[dict[str, object]] = []
+
+ await run_external_json_process(
+ "test",
+ [sys.executable, "-c", f"print({payload!r})"],
+ results,
+ timeout_seconds=5,
+ )
+
+ self.assertEqual(results, [expected])
+
+ @unittest.skipUnless(os.name == "posix", "process-group regression is POSIX-specific")
+ async def test_timeout_kills_descendants_holding_output_pipes(self) -> None:
+ descendant = "import time; time.sleep(4)"
+ parent = (
+ "import subprocess, sys, time; "
+ f"subprocess.Popen([sys.executable, '-c', {descendant!r}], "
+ "stdout=sys.stdout, stderr=sys.stderr); "
+ "print('descendant-started', flush=True); "
+ "time.sleep(30)"
+ )
+ loop = asyncio.get_running_loop()
+ started_at = loop.time()
+
+ with self.assertRaisesRegex(SmokeError, "test smoke timed out") as raised:
+ await run_external_json_process(
+ "test",
+ [sys.executable, "-c", parent],
+ [],
+ timeout_seconds=1,
+ )
+
+ elapsed = loop.time() - started_at
+ self.assertIn("descendant-started", str(raised.exception))
+ self.assertLess(
+ elapsed,
+ 2.5,
+ "timeout cleanup waited for the descendant to close inherited pipes",
+ )
+
+ @unittest.skipUnless(os.name == "posix", "signal regression is POSIX-specific")
+ async def test_service_cleanup_escalates_to_sigkill_with_a_bound(self) -> None:
+ process = await asyncio.create_subprocess_exec(
+ sys.executable,
+ "-c",
+ (
+ "import signal, time; "
+ "signal.signal(signal.SIGTERM, signal.SIG_IGN); "
+ "print('ready', flush=True); "
+ "time.sleep(30)"
+ ),
+ stdout=asyncio.subprocess.PIPE,
+ start_new_session=True,
+ )
+ self.assertIsNotNone(process.stdout)
+ self.assertEqual(await process.stdout.readline(), b"ready\n")
+
+ stopped = await terminate_process_tree(
+ process,
+ terminate_timeout_seconds=0.05,
+ kill_timeout_seconds=1,
+ )
+
+ self.assertTrue(stopped)
+ self.assertEqual(process.returncode, -signal.SIGKILL)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/moli-cdp-smoke/tests/test_runner_progress.py b/moli-cdp-smoke/tests/test_runner_progress.py
index 5867a5a4a..c0c475ae1 100644
--- a/moli-cdp-smoke/tests/test_runner_progress.py
+++ b/moli-cdp-smoke/tests/test_runner_progress.py
@@ -61,6 +61,35 @@ class RunnerProgressTests(unittest.IsolatedAsyncioTestCase):
r"elapsed=\d+\.\d{3}s error=CancelledError$",
)
+ async def test_timeout_cancels_the_operation_and_reports_its_label(self) -> None:
+ cancelled = asyncio.Event()
+
+ async def wait_forever() -> None:
+ try:
+ await asyncio.Event().wait()
+ finally:
+ cancelled.set()
+
+ stderr = io.StringIO()
+ with redirect_stderr(stderr), self.assertRaisesRegex(
+ TimeoutError,
+ r"test/timeout timed out after 0\.01s",
+ ):
+ await await_with_progress(
+ "test/timeout",
+ wait_forever(),
+ timeout_seconds=0.01,
+ )
+
+ self.assertTrue(cancelled.is_set())
+ lines = stderr.getvalue().splitlines()
+ self.assertEqual(lines[0], "[moli-cdp-smoke] START test/timeout")
+ self.assertRegex(
+ lines[1],
+ r"^\[moli-cdp-smoke\] FAIL test/timeout "
+ r"elapsed=\d+\.\d{3}s error=TimeoutError$",
+ )
+
if __name__ == "__main__":
unittest.main()
diff --git a/moli-cdp-smoke/tests/test_serve.py b/moli-cdp-smoke/tests/test_serve.py
new file mode 100644
index 000000000..2fcee549f
--- /dev/null
+++ b/moli-cdp-smoke/tests/test_serve.py
@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import asyncio
+import unittest
+from typing import cast
+
+from moli_cdp_smoke.serve import (
+ MoliServe,
+ _collect_process_output,
+ _moli_endpoint_from_log_line,
+ wait_for_moli_endpoint,
+)
+
+
+class _FakeProcess:
+ def __init__(self, returncode: int | None = None) -> None:
+ self.returncode = returncode
+ self._never_exits = asyncio.Event()
+
+ async def wait(self) -> int:
+ if self.returncode is None:
+ await self._never_exits.wait()
+ assert self.returncode is not None
+ return self.returncode
+
+
+class ServeLogTests(unittest.IsolatedAsyncioTestCase):
+ def test_bound_endpoint_is_parsed_from_plain_and_ansi_logs(self) -> None:
+ plain = "INFO protocol server listening addr=127.0.0.1:41827"
+ ansi = (
+ "\x1b[2m2026-08-24T01:02:03Z\x1b[0m "
+ "\x1b[32m INFO\x1b[0m protocol server listening "
+ "\x1b[3maddr\x1b[0m\x1b[2m=\x1b[0m127.0.0.1:53109"
+ )
+
+ self.assertEqual(
+ _moli_endpoint_from_log_line(plain), "http://127.0.0.1:41827"
+ )
+ self.assertEqual(
+ _moli_endpoint_from_log_line(ansi), "http://127.0.0.1:53109"
+ )
+ self.assertIsNone(_moli_endpoint_from_log_line("unrelated addr=127.0.0.1:9"))
+ self.assertIsNone(
+ _moli_endpoint_from_log_line(
+ "protocol server listening addr=127.0.0.1:99999"
+ )
+ )
+
+ async def test_output_collector_publishes_the_bound_endpoint(self) -> None:
+ stream = asyncio.StreamReader()
+ stream.feed_data(
+ b"INFO protocol server listening addr=127.0.0.1:42123\n"
+ )
+ stream.feed_eof()
+ logs: list[str] = []
+ endpoint_ready: asyncio.Future[str] = (
+ asyncio.get_running_loop().create_future()
+ )
+
+ await _collect_process_output(stream, logs, "stderr", endpoint_ready)
+
+ self.assertEqual(endpoint_ready.result(), "http://127.0.0.1:42123")
+ self.assertEqual(len(logs), 1)
+
+ async def test_wait_reports_process_exit_before_the_listening_log(self) -> None:
+ endpoint_ready: asyncio.Future[str] = (
+ asyncio.get_running_loop().create_future()
+ )
+ serve = MoliServe(
+ process=cast(asyncio.subprocess.Process, _FakeProcess(7)),
+ logs=["stderr: startup failed"],
+ tasks=[],
+ http_cache_dir="",
+ endpoint_ready=endpoint_ready,
+ )
+
+ with self.assertRaisesRegex(RuntimeError, "exited early with 7"):
+ await wait_for_moli_endpoint(serve, timeout_seconds=1)
+
+ async def test_wait_for_listening_log_has_a_bounded_timeout(self) -> None:
+ endpoint_ready: asyncio.Future[str] = (
+ asyncio.get_running_loop().create_future()
+ )
+ serve = MoliServe(
+ process=cast(asyncio.subprocess.Process, _FakeProcess()),
+ logs=[],
+ tasks=[],
+ http_cache_dir="",
+ endpoint_ready=endpoint_ready,
+ )
+
+ with self.assertRaisesRegex(RuntimeError, "bound endpoint"):
+ await wait_for_moli_endpoint(serve, timeout_seconds=0.01)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/moli-cdp-smoke/tests/test_supervisor.py b/moli-cdp-smoke/tests/test_supervisor.py
new file mode 100644
index 000000000..4a86680c9
--- /dev/null
+++ b/moli-cdp-smoke/tests/test_supervisor.py
@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from moli_cdp_smoke.runner import GROUPS_BY_NAME
+from moli_cdp_smoke.supervisor import (
+ WorkerJob,
+ WorkerOutcome,
+ _run_worker_job,
+ parse_args,
+ run_supervisor,
+)
+
+
+class SupervisorWorkerTests(unittest.IsolatedAsyncioTestCase):
+ async def test_worker_result_is_read_from_its_isolated_file(self) -> None:
+ group = GROUPS_BY_NAME["protocol"]
+ job = WorkerJob(group=group, attempt=1, repeat=1)
+ with tempfile.TemporaryDirectory() as temporary:
+ output_dir = Path(temporary)
+
+ def fake_argv(_: WorkerJob, result_path: Path) -> list[str]:
+ payload = json.dumps(
+ {
+ "ok": True,
+ "group": "protocol",
+ "results": [{"name": "probe", "ok": True}],
+ }
+ )
+ return [
+ sys.executable,
+ "-c",
+ (
+ "from pathlib import Path; import sys; "
+ f"Path(sys.argv[1]).write_text({payload!r}, encoding='utf-8'); "
+ "print('worker-log')"
+ ),
+ str(result_path),
+ ]
+
+ with patch("moli_cdp_smoke.supervisor._worker_argv", fake_argv):
+ outcome = await _run_worker_job(job, output_dir, 5)
+
+ self.assertTrue(outcome.passed)
+ self.assertEqual(outcome.scenario_count, 1)
+ self.assertEqual(outcome.log_path.read_text().strip(), "worker-log")
+
+ async def test_worker_wall_clock_timeout_is_bounded(self) -> None:
+ group = GROUPS_BY_NAME["protocol"]
+ job = WorkerJob(group=group, attempt=1, repeat=1)
+ with tempfile.TemporaryDirectory() as temporary:
+ output_dir = Path(temporary)
+ stale_result = output_dir / "protocol.json"
+ stale_result.write_text(
+ json.dumps(
+ {
+ "ok": True,
+ "group": "protocol",
+ "results": [{"name": "stale", "ok": True}],
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ def hanging_argv(_: WorkerJob, __: Path) -> list[str]:
+ return [sys.executable, "-c", "import time; time.sleep(30)"]
+
+ loop = asyncio.get_running_loop()
+ started_at = loop.time()
+ with patch("moli_cdp_smoke.supervisor._worker_argv", hanging_argv):
+ outcome = await _run_worker_job(job, output_dir, 0.05)
+
+ self.assertEqual(outcome.status, "timed_out")
+ self.assertEqual(outcome.scenario_count, 0)
+ self.assertFalse(stale_result.exists())
+ self.assertLess(loop.time() - started_at, 2)
+
+ @unittest.skipUnless(
+ sys.platform.startswith("linux"),
+ "descendant process-state regression uses Linux /proc",
+ )
+ async def test_worker_timeout_signals_descendants_in_its_process_group(self) -> None:
+ group = GROUPS_BY_NAME["protocol"]
+ job = WorkerJob(group=group, attempt=1, repeat=1)
+ with tempfile.TemporaryDirectory() as temporary:
+ output_dir = Path(temporary)
+ child_ready = output_dir / "child-ready"
+ child = f"""
+import os
+import time
+from pathlib import Path
+
+Path({str(child_ready)!r}).write_text(str(os.getpid()), encoding="utf-8")
+while True:
+ time.sleep(1)
+"""
+ parent = f"""
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+subprocess.Popen([sys.executable, "-c", {child!r}])
+ready = Path({str(child_ready)!r})
+while not ready.exists():
+ time.sleep(0.01)
+time.sleep(30)
+"""
+
+ def descendant_argv(_: WorkerJob, __: Path) -> list[str]:
+ return [sys.executable, "-c", parent]
+
+ with patch("moli_cdp_smoke.supervisor._worker_argv", descendant_argv):
+ outcome = await _run_worker_job(job, output_dir, 0.5)
+
+ self.assertEqual(outcome.status, "timed_out")
+ self.assertTrue(child_ready.exists())
+ child_pid = int(child_ready.read_text(encoding="utf-8"))
+ process_state = Path(f"/proc/{child_pid}/stat")
+ for _ in range(100):
+ if not process_state.exists():
+ break
+ if process_state.read_text(encoding="utf-8").split()[2] == "Z":
+ break
+ await asyncio.sleep(0.01)
+ else:
+ self.fail(f"worker descendant {child_pid} survived process-group cleanup")
+
+
+class SupervisorSchedulingTests(unittest.IsolatedAsyncioTestCase):
+ async def test_jobs_bound_allows_isolated_workers_to_run_in_parallel(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ args = parse_args(
+ [
+ "--group",
+ "protocol",
+ "--group",
+ "puppeteer",
+ "--jobs",
+ "2",
+ "--output-dir",
+ temporary,
+ ]
+ )
+ both_started = asyncio.Event()
+ active = 0
+ maximum_active = 0
+
+ async def fake_run(
+ job: WorkerJob,
+ output_dir: Path,
+ _: float,
+ ) -> WorkerOutcome:
+ nonlocal active, maximum_active
+ active += 1
+ maximum_active = max(maximum_active, active)
+ if active == 2:
+ both_started.set()
+ await asyncio.wait_for(both_started.wait(), timeout=1)
+ active -= 1
+ return WorkerOutcome(
+ job=job,
+ status="passed",
+ duration_seconds=0,
+ exit_code=0,
+ log_path=output_dir / f"{job.file_stem}.log",
+ result_path=output_dir / f"{job.file_stem}.json",
+ scenario_count=1,
+ )
+
+ with patch("moli_cdp_smoke.supervisor._run_worker_job", fake_run):
+ exit_code, summary = await run_supervisor(args)
+
+ self.assertEqual(exit_code, 0)
+ self.assertTrue(summary["ok"])
+ self.assertEqual(maximum_active, 2)
+
+ async def test_fixed_port_is_rejected_for_parallel_isolated_workers(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ args = parse_args(
+ [
+ "--group",
+ "protocol",
+ "--group",
+ "puppeteer",
+ "--jobs",
+ "2",
+ "--output-dir",
+ temporary,
+ ]
+ )
+
+ with patch.dict(os.environ, {"MOLI_CDP_PORT": "9333"}), self.assertRaisesRegex(
+ RuntimeError, "cannot be shared"
+ ):
+ await run_supervisor(args)
+
+
+if __name__ == "__main__":
+ unittest.main()