Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
#!/usr/bin/env python3
"""Sharded, process-isolated pytest runner (the CI test engine).
Why this exists: one monolithic ``pytest tests/`` process took 75-95 minutes
on a 4-core runner and routinely hit the job timeout. A single wedged test ate
the whole budget, and module state leaked across the ~2 000 test files.
How it works (a per-file runner):
1. **Discover** every ``test_*.py`` / ``*_test.py`` under the roots.
2. **Shard** (``--shard 2/6``): longest-processing-time bin packing over the
per-file duration cache, so shards finish together. Every shard computes
the same packing, so no coordination is needed.
3. **Batch**: consecutive files of one directory run in one pytest process
(the ~1.5 s interpreter + conftest start-up is paid once per batch).
4. **Isolate on failure**: a failed or timed-out batch is re-run file by file
in fresh processes, and a failed file gets one more fresh attempt. A pass
on that attempt is reported as FLAKY, not as a failure.
5. **Bound**: every process has a wall-clock budget and is killed with its
whole process tree, so a hang costs one file, never the job. A file that
runs out of its budget alone gets one more attempt with a longer one: a
file the history has never timed starts on the generic floor, and a slow
file on a loaded runner is not a hang.
Outputs: ``--report`` (JSON: failed test ids, flaky files, counts) and
``--junit-out`` (merged JUnit XML), both in the run's temp dir unless given,
plus ``--durations-out`` (timings of files
that passed on the first attempt or alone after a timeout — a killed run has no
timing, so a hang can never ratchet its own budget).
The run must leave the checkout as it found it: a tracked file it rewrote
or a new file it left behind (outside its own outputs) fails it, see
``checkout_guard.py``.
The runner exits 1 when any test failed for real; whether that blocks the
build is ``test_ratchet.py``'s decision (known failures are baselined).
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import shlex
import signal
import subprocess
import sys
import tempfile
import threading
import time
import xml.etree.ElementTree as ET # noqa: S405 - parses our own pytest output
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from pathlib import Path
try: # imported as scripts.ci.run_tests_parallel (tests, select_tests.py)
from scripts.ci import checkout_guard
except ImportError: # run as a script: its own directory is on sys.path
import checkout_guard # type: ignore[no-redef]
REPO_ROOT = Path(__file__).resolve().parents[2]
DEFAULT_MARKERS = (
"not skip_ci and not slow and not openclaw_live and not voice_latency and not eval"
)
_SKIP_DIRS = {"__pycache__", "node_modules", "fakes", "fixtures", ".pytest_cache"}
DEFAULT_DURATION = 4.0 # seconds, for a file the cache has never seen
MIN_FILE_TIMEOUT = 300.0
MAX_FILE_TIMEOUT = 900.0
BATCH_TARGET_SECONDS = 45.0
BATCH_MAX_FILES = 12
MAX_BATCH_TIMEOUT = 1800.0
BATCH_TIMEOUT_FACTOR = 6.0 # times the batch's own history
# A file that timed out alone runs once more with this multiple of its budget.
# The floor is a guess for a file without history: on Windows a file whose
# tests each start a contained command took 160 s on a quiet runner and more
# than the 300 s floor on a loaded one, and every such timeout ejected a pull
# request from the merge queue. A real hang now costs the floor three times.
TIMEOUT_RETRY_FACTOR = 2.0
_OK_EXIT_CODES = {0, 5} # 5 = every test deselected / none collected
# A new batch waits while less than this much memory is left to commit. A local
# run shares the machine with the user's apps: eight batches on a box already
# near its commit limit once ran Windows out of virtual memory, and every
# process that allocated next (a Claude Code pane, OBS) died with 0xC0000409.
MEMORY_RESERVE_BYTES = 6 * 1024**3
MEMORY_POLL_SECONDS = 2.0
# --------------------------------------------------------------------------- discovery
def discover(roots: list[str], repo: Path = REPO_ROOT) -> list[str]:
files: set[str] = set()
for root in roots:
base = (repo / root).resolve()
if base.is_file():
files.add(base.relative_to(repo).as_posix())
continue
if not base.is_dir():
continue
for path in base.rglob("*.py"):
rel = path.relative_to(repo)
if any(part in _SKIP_DIRS for part in rel.parts):
continue
if path.name.startswith("test_") or path.name.endswith("_test.py"):
files.add(rel.as_posix())
return sorted(files)
def load_durations(path: Path | None) -> dict[str, float]:
if not path or not path.is_file():
return {}
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError) as exc:
print(f"[runner] duration cache unreadable ({exc}); using defaults", flush=True)
return {}
return {str(k): float(v) for k, v in data.items() if isinstance(v, int | float)}
def estimate(file: str, durations: dict[str, float]) -> float:
return max(0.5, durations.get(file, DEFAULT_DURATION))
def shard(files: list[str], index: int, total: int, durations: dict[str, float]) -> list[str]:
"""Deterministic LPT bin packing; returns shard ``index`` (1-based) of ``total``."""
if total <= 1:
return list(files)
bins: list[tuple[float, int, list[str]]] = [(0.0, i, []) for i in range(total)]
for file in sorted(files, key=lambda f: (-estimate(f, durations), f)):
load, idx, members = min(bins, key=lambda b: (b[0], b[1]))
members.append(file)
bins[idx] = (load + estimate(file, durations), idx, members)
return sorted(bins[index - 1][2])
def make_batches(files: list[str], durations: dict[str, float]) -> list[list[str]]:
batches: list[list[str]] = []
current: list[str] = []
current_dir = None
load = 0.0
for file in sorted(files):
directory = file.rsplit("/", 1)[0]
cost = estimate(file, durations)
# A file that alone exceeds the target always runs by itself.
if current and (
directory != current_dir
or load + cost > BATCH_TARGET_SECONDS
or len(current) >= BATCH_MAX_FILES
or cost > BATCH_TARGET_SECONDS
):
batches.append(current)
current, load = [], 0.0
current.append(file)
current_dir = directory
load += cost
if current:
batches.append(current)
return batches
def file_timeout(file: str, durations: dict[str, float]) -> float:
return min(MAX_FILE_TIMEOUT, max(MIN_FILE_TIMEOUT, 3 * durations.get(file, 0.0)))
def retry_timeout(file: str, durations: dict[str, float]) -> float:
"""Wall clock for the one attempt a timed-out file gets on its own."""
return min(MAX_FILE_TIMEOUT, TIMEOUT_RETRY_FACTOR * file_timeout(file, durations))
def batch_timeout(batch: list[str], durations: dict[str, float]) -> float:
"""Wall clock for one batch: its slowest file's budget, or a multiple of its load.
A batch is built to ~45 s of history. Summing every file's five-minute
floor gave a twelve-file batch thirty minutes, so one wedged test held a
shard until the job timeout killed it without a report. A batch killed
early costs nothing but time: each of its files runs again alone.
"""
load = sum(estimate(file, durations) for file in batch)
slowest = max(file_timeout(file, durations) for file in batch)
return min(MAX_BATCH_TIMEOUT, max(slowest, BATCH_TIMEOUT_FACTOR * load))
# --------------------------------------------------------------------------- execution
def available_memory() -> int | None:
"""Bytes the OS can still hand out, or None where it cannot be read.
Windows reports the commit headroom (RAM plus page file): that is the
limit an allocation fails against. Linux reports MemAvailable.
"""
if os.name == "nt":
import ctypes
class _MemoryStatusEx(ctypes.Structure):
_fields_ = [
("dwLength", ctypes.c_ulong),
("dwMemoryLoad", ctypes.c_ulong),
("ullTotalPhys", ctypes.c_ulonglong),
("ullAvailPhys", ctypes.c_ulonglong),
("ullTotalPageFile", ctypes.c_ulonglong),
("ullAvailPageFile", ctypes.c_ulonglong),
("ullTotalVirtual", ctypes.c_ulonglong),
("ullAvailVirtual", ctypes.c_ulonglong),
("ullAvailExtendedVirtual", ctypes.c_ulonglong),
]
status = _MemoryStatusEx()
status.dwLength = ctypes.sizeof(_MemoryStatusEx)
if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
return None
return int(status.ullAvailPageFile)
try:
with open("/proc/meminfo", encoding="ascii") as meminfo:
for line in meminfo:
if line.startswith("MemAvailable:"):
return int(line.split()[1]) * 1024
except (OSError, ValueError):
return None # macOS and other systems without /proc: no gate
return None
class MemoryGate:
"""Holds back a new batch while the machine is short of memory.
A batch only waits while another one is running, so the gate can slow a
run down but never stall it: with nothing else in flight the next batch
always starts, and its finish is what frees the memory others wait for.
"""
def __init__(
self,
reserve: int = MEMORY_RESERVE_BYTES,
probe=available_memory,
poll_seconds: float = MEMORY_POLL_SECONDS,
):
self.reserve = reserve
self.probe = probe
self.poll_seconds = poll_seconds
self.running = 0
self.waits = 0
self._cond = threading.Condition()
def _short(self) -> bool:
free = self.probe()
return free is not None and free < self.reserve
def acquire(self) -> None:
with self._cond:
warned = False
while self.running > 0 and self._short():
if not warned:
self.waits += 1
warned = True
print(
f"[runner] under {self.reserve // 1024**3} GiB memory left; "
f"waiting for one of {self.running} running batches",
flush=True,
)
self._cond.wait(self.poll_seconds)
self.running += 1
def release(self) -> None:
with self._cond:
self.running -= 1
self._cond.notify_all()
@dataclass
class RunResult:
files: list[str]
exit_code: int
duration: float
timed_out: bool
failed_ids: list[str] = field(default_factory=list)
file_seconds: dict[str, float] = field(default_factory=dict)
counts: dict[str, int] = field(default_factory=dict)
junit: Path | None = None
log_tail: str = ""
@property
def ok(self) -> bool:
return not self.timed_out and self.exit_code in _OK_EXIT_CODES and not self.failed_ids
def _kill_tree(proc: subprocess.Popen[bytes]) -> None:
try:
if os.name == "nt":
subprocess.run( # noqa: S603
["taskkill", "/F", "/T", "/PID", str(proc.pid)], # noqa: S607
capture_output=True,
check=False,
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
)
else:
os.killpg(proc.pid, signal.SIGKILL)
except (OSError, subprocess.SubprocessError) as exc:
print(f"[runner] could not kill process tree {proc.pid}: {exc}", flush=True)
try:
proc.kill()
except OSError:
pass # already gone - the tree kill above reaped it
def _module_to_file(classname: str, files: list[str]) -> str | None:
dotted = classname.replace(".", "/")
best = None
for file in files:
stem = file[:-3]
if dotted == stem or dotted.startswith(stem + "/"):
if best is None or len(stem) > len(best[:-3]):
best = file
return best
def parse_junit(path: Path, files: list[str]) -> tuple[list[str], dict[str, float], dict[str, int]]:
"""Return (failed ids, seconds per file, counts) from one pytest JUnit file."""
failed: list[str] = []
seconds: dict[str, float] = {}
counts = {"tests": 0, "passed": 0, "failed": 0, "skipped": 0}
if not path.is_file():
return failed, seconds, counts
try:
root = ET.parse(path).getroot() # noqa: S314 - our own pytest output
except ET.ParseError as exc:
print(f"[runner] unreadable JUnit {path.name}: {exc}", flush=True)
return failed, seconds, counts
for case in root.iter("testcase"):
classname = case.get("classname", "")
name = case.get("name", "")
counts["tests"] += 1
owner = _module_to_file(classname, files) or (files[0] if len(files) == 1 else "")
seconds[owner] = seconds.get(owner, 0.0) + float(case.get("time", 0) or 0)
if case.find("failure") is not None or case.find("error") is not None:
counts["failed"] += 1
failed.append(f"{classname}::{name}")
elif case.find("skipped") is not None:
counts["skipped"] += 1
else:
counts["passed"] += 1
return failed, seconds, counts
class Runner:
def __init__(
self,
args: argparse.Namespace,
durations: dict[str, float],
workdir: Path,
gate: MemoryGate | None = None,
):
self.args = args
self.durations = durations
self.workdir = workdir
self.gate = gate or MemoryGate()
self._seq = 0
self._lock = threading.Lock()
# The whole shard's wall clock. Batch budget plus isolation and retry
# runs can outlast the CI job's timeout; the job is then cancelled with
# no report and no log, so a hang names nothing (merge-queue run
# 40485520423, tests-windows (6)). Past the deadline nothing new starts
# and a running process is stopped: the report still gets written.
minutes = getattr(args, "deadline_minutes", None)
self.deadline = time.monotonic() + 60 * minutes if minutes else None
def _next_id(self) -> int:
with self._lock:
self._seq += 1
return self._seq
def run(self, files: list[str], timeout: float) -> RunResult:
if self.deadline is not None:
left = self.deadline - time.monotonic()
if left <= 0:
return RunResult(
files,
-1,
0.0,
True,
[f"{f}::<shard deadline>" for f in files],
log_tail="not started: the shard's deadline had passed",
)
timeout = min(timeout, left)
run_id = self._next_id()
junit = self.workdir / f"run-{run_id:05d}.xml"
log = self.workdir / f"run-{run_id:05d}.log"
cmd = [
sys.executable,
"-m",
"pytest",
*files,
"-q",
"-p",
"no:cacheprovider",
"-o",
"faulthandler_timeout=240",
f"--basetemp={self.workdir / f'tmp-{run_id:05d}'}",
f"--junitxml={junit}",
"-m",
self.args.markers,
*shlex.split(self.args.pytest_args or ""),
]
env = dict(os.environ)
env.setdefault("PYTHONUTF8", "1")
env["JARVIS_TEST_RUNNER_ISOLATED"] = "1"
popen_kwargs: dict[str, object] = {}
if os.name == "nt":
popen_kwargs["creationflags"] = getattr(subprocess, "CREATE_NO_WINDOW", 0)
else:
popen_kwargs["start_new_session"] = True
self.gate.acquire()
started = time.monotonic()
timed_out = False
try:
with log.open("wb") as sink:
proc = subprocess.Popen( # noqa: S603
cmd,
cwd=REPO_ROOT,
stdout=sink,
stderr=subprocess.STDOUT,
stdin=subprocess.DEVNULL,
env=env,
**popen_kwargs, # type: ignore[arg-type]
)
try:
exit_code = proc.wait(timeout=timeout)
except subprocess.TimeoutExpired:
timed_out = True
_kill_tree(proc)
exit_code = proc.wait()
finally:
self.gate.release()
duration = time.monotonic() - started
failed, seconds, counts = parse_junit(junit, files)
tail = ""
if timed_out or exit_code not in _OK_EXIT_CODES or failed:
text = log.read_text(encoding="utf-8", errors="replace")
tail = "\n".join(text.splitlines()[-60:])
if timed_out:
failed = failed or [f"{f}::<timeout {int(timeout)}s>" for f in files]
elif not failed:
failed = [f"{f}::<pytest exit {exit_code}>" for f in files]
return RunResult(
files, exit_code, duration, timed_out, failed, seconds, counts, junit, tail
)
def run_unit(self, batch: list[str]) -> dict[str, object]:
"""Run one batch with isolation-on-failure and a flaky retry."""
budget = batch_timeout(batch, self.durations)
first = self.run(batch, budget)
outcome: dict[str, object] = {"runs": [first], "flaky": [], "first_pass": [], "slow": []}
if first.ok:
outcome["first_pass"] = list(batch)
return outcome
runs: list[RunResult] = []
flaky: list[str] = []
slow: list[str] = []
for file in batch:
budget_one = file_timeout(file, self.durations)
single = first if len(batch) == 1 else self.run([file], budget_one)
if not single.ok:
timed_out = single.timed_out
again = retry_timeout(file, self.durations) if timed_out else budget_one
retry = self.run([file], again)
if retry.ok:
flaky.append(file)
if timed_out:
# Its real duration, so the next history gives it room.
slow.append(file)
print(
f"[runner] SLOW: {file} passed in {retry.duration:.0f}s "
f"after running out of {int(budget_one)}s",
flush=True,
)
single = retry
runs.append(single)
outcome["runs"] = runs
outcome["flaky"] = flaky
outcome["slow"] = slow
return outcome
# --------------------------------------------------------------------------- reporting
def merge_junit(paths: list[Path], out: Path) -> None:
merged = ET.Element("testsuites")
for path in paths:
if not path.is_file():
continue
try:
root = ET.parse(path).getroot() # noqa: S314 - our own pytest output
except ET.ParseError as exc:
print(f"[runner] skipping unreadable JUnit {path.name}: {exc}", flush=True)
continue
suites = [root] if root.tag == "testsuite" else list(root.iter("testsuite"))
merged.extend(suites)
out.parent.mkdir(parents=True, exist_ok=True)
ET.ElementTree(merged).write(out, encoding="utf-8", xml_declaration=True)
def checkout_changes(before: checkout_guard.Snapshot | None, outputs: list[Path]) -> list[str]:
"""Paths the run changed in the checkout, minus its own requested outputs."""
if before is None:
return []
after = checkout_guard.snapshot()
if after is None:
return []
return checkout_guard.changes(before, after, checkout_guard.relative_ignores(outputs))
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("roots", nargs="*", default=["tests"], help="Discovery roots.")
parser.add_argument("--files-from", type=Path, help="Explicit file list (one per line).")
parser.add_argument("--shard", default="1/1", help="i/n — run shard i of n.")
parser.add_argument("--workers", type=int, default=os.cpu_count() or 2)
parser.add_argument("--markers", default=DEFAULT_MARKERS)
parser.add_argument("--pytest-args", default="", help="Extra args for every pytest run.")
parser.add_argument("--durations", type=Path, default=REPO_ROOT / "test_durations.json")
parser.add_argument("--durations-out", type=Path)
# Defaults live in the run's temp dir: a bare run must not drop files
# into the checkout it is testing.
parser.add_argument("--report", type=Path, help="Default: <run temp dir>/test-report.json.")
parser.add_argument("--junit-out", type=Path, help="Default: <run temp dir>/report.xml.")
parser.add_argument("--list", action="store_true", help="Print the shard's files and exit.")
parser.add_argument(
"--deadline-minutes",
type=float,
help="Wall clock for the whole shard; keep it under the CI job's timeout.",
)
args = parser.parse_args(argv)
# Windows consoles default to cp1252; a test log with box-drawing
# characters must never crash the runner that is reporting it (AP-1).
for stream in (sys.stdout, sys.stderr):
if hasattr(stream, "reconfigure"):
stream.reconfigure(encoding="utf-8", errors="replace")
index, total = (int(x) for x in args.shard.split("/", 1))
durations = load_durations(args.durations)
if args.files_from:
wanted = [line.strip() for line in args.files_from.read_text(encoding="utf-8").splitlines()]
files = sorted({f for f in wanted if f and (REPO_ROOT / f).is_file()})
else:
files = discover(args.roots)
mine = shard(files, index, total, durations)
if args.list:
print("\n".join(mine))
return 0
print(
f"[runner] shard {index}/{total}: {len(mine)} of {len(files)} files, "
f"{args.workers} workers, markers={args.markers!r}",
flush=True,
)
batches = make_batches(mine, durations)
started = time.monotonic()
workdir = Path(tempfile.mkdtemp(prefix="jarvis-tests-"))
args.report = args.report or workdir / "test-report.json"
args.junit_out = args.junit_out or workdir / "report.xml"
checkout_before = checkout_guard.snapshot()
runner = Runner(args, durations, workdir)
all_runs: list[RunResult] = []
flaky: list[str] = []
first_pass: list[str] = []
slow: list[str] = []
done = 0
with ThreadPoolExecutor(max_workers=max(1, args.workers)) as pool:
futures = [pool.submit(runner.run_unit, batch) for batch in batches]
for future in futures:
outcome = future.result()
runs = outcome["runs"] # type: ignore[assignment]
all_runs.extend(runs) # type: ignore[arg-type]
flaky.extend(outcome["flaky"]) # type: ignore[arg-type]
first_pass.extend(outcome["first_pass"]) # type: ignore[arg-type]
slow.extend(outcome["slow"]) # type: ignore[arg-type]
done += 1
for run in runs: # type: ignore[attr-defined]
if not run.ok:
label = "TIMEOUT" if run.timed_out else f"FAIL exit={run.exit_code}"
names = " ".join(run.files)
print(f"\n[runner] {label}: {names}\n{run.log_tail}\n", flush=True)
if done % 20 == 0 or done == len(futures):
print(f"[runner] {done}/{len(futures)} batches done", flush=True)
failed_ids = sorted({fid for run in all_runs for fid in run.failed_ids})
counts = {"tests": 0, "passed": 0, "failed": 0, "skipped": 0}
for run in all_runs:
for key in counts:
counts[key] += run.counts.get(key, 0)
elapsed = time.monotonic() - started
report = {
"shard": f"{index}/{total}",
"file_paths": mine,
"suite_files": len(files),
"suite_digest": hashlib.sha256("\n".join(files).encode("utf-8")).hexdigest(),
"platform": sys.platform,
"files": len(mine),
"counts": counts,
"failed_ids": failed_ids,
"flaky_files": sorted(set(flaky)),
"timeouts": sorted({f for run in all_runs if run.timed_out for f in run.files}),
"elapsed_seconds": round(elapsed, 1),
}
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
merge_junit([run.junit for run in all_runs if run.junit], args.junit_out)
if args.durations_out:
fresh: dict[str, float] = {}
timed = {*first_pass, *slow}
for run in all_runs:
for file, secs in run.file_seconds.items():
if file in timed:
fresh[file] = round(secs + 1.0, 2) # + interpreter start-up share
payload = json.dumps(fresh, indent=1, sort_keys=True) + "\n"
args.durations_out.write_text(payload, encoding="utf-8")
print(
f"[runner] done in {elapsed:.0f}s: {counts['passed']} passed, {counts['failed']} failed, "
f"{counts['skipped']} skipped, {len(set(flaky))} flaky files, "
f"{len(report['timeouts'])} timed-out files",
flush=True,
)
print(f"[runner] report: {args.report} junit: {args.junit_out}", flush=True)
outputs = [args.report, args.junit_out, *([args.durations_out] if args.durations_out else [])]
changed = checkout_changes(checkout_before, outputs)
if changed:
print(
"[runner] FAIL: the run modified tracked files or left new files in the "
"checkout. Point the writer at a temp path:\n" + checkout_guard.describe(changed),
flush=True,
)
return 1
return 1 if failed_ids else 0
if __name__ == "__main__":
sys.exit(main())