Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
#!/usr/bin/env python3
"""Known-failure ratchet for the sharded test suite.
The old CI step ended in ``|| echo`` — ANY failure rode through green as long
as the passed count stayed above a floor. The suite does carry a backlog of
failures (mostly platform-specific), so going strict in one step would turn
every PR red for reasons it did not cause. The ratchet splits the two:
* a failure listed in ``scripts/ci/test-baseline-<os>.json`` is KNOWN — it is
reported, never blocking;
* any other failure is NEW — it blocks the change that introduced it;
* a known failure that now passes is reported so the baseline can shrink
(``update`` rewrites it from a full run; it may only get smaller in review);
* Historical ``flaky_files`` metadata is informational. Only exact known
test identities can be waived; a new failure in a flaky file still blocks.
Commands::
ratchet_tests.py check --baseline B.json report*.json # per shard, blocking
ratchet_tests.py summary --baseline B.json --floor report*.json
ratchet_tests.py update --out B.json report*.json # regenerate
ratchet_tests.py prune --baseline B.json --os linux run1/ run2/ run3/
ratchet_tests.py durations --out D.json dur*.json # merge timing caches
A missing baseline means no failures have been approved for that OS.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
_CRASH = re.compile(r"::<(?:timeout|pytest exit)[^>]*>$")
def normalize(test_id: str) -> str:
"""Timeouts and crashes differ only in their number between runs."""
return _CRASH.sub("::<crash>", test_id)
def load_reports(paths: list[Path]) -> list[dict]:
reports = []
for path in paths:
if path.is_dir():
reports.extend(load_reports(sorted(path.rglob("test-report*.json"))))
continue
reports.append(json.loads(path.read_text(encoding="utf-8")))
return reports
def load_baseline(path: Path) -> set[str] | None:
if not path.is_file():
return None
data = json.loads(path.read_text(encoding="utf-8"))
return {normalize(x) for x in data.get("known_failures", [])}
def load_flaky_files(path: Path | None) -> list[str]:
if not path or not path.is_file():
return []
return list(json.loads(path.read_text(encoding="utf-8")).get("flaky_files", []))
def failed_ids(reports: list[dict]) -> set[str]:
return {normalize(fid) for report in reports for fid in report.get("failed_ids", [])}
def _summary(lines: list[str]) -> None:
target = os.environ.get("GITHUB_STEP_SUMMARY")
if target:
with open(target, "a", encoding="utf-8") as handle:
handle.write("\n".join(lines) + "\n")
def partition_errors(reports: list[dict], expected_shards: int) -> list[str]:
"""Prove all files were assigned exactly once, not merely a passed-count floor."""
wanted = {f"{i}/{expected_shards}" for i in range(1, expected_shards + 1)}
if len(reports) != expected_shards or {r.get("shard") for r in reports} != wanted:
return ["missing or duplicate shard reports"]
if any(not isinstance(r.get("file_paths"), list) for r in reports):
return ["shard reports lack file-level coverage evidence"]
files = [file for r in reports for file in r["file_paths"]]
if len(files) != len(set(files)):
return ["test files were assigned to multiple shards"]
digest = hashlib.sha256("\n".join(sorted(files)).encode("utf-8")).hexdigest()
if any(r.get("suite_files") != len(files) or r.get("suite_digest") != digest for r in reports):
return ["shard coverage does not match the discovered suite"]
return []
def cmd_check(args: argparse.Namespace) -> int:
reports = load_reports(args.reports)
if not reports or any(
not isinstance(r.get("failed_ids"), list)
or not isinstance(r.get("counts"), dict)
or r.get("files", 0) < 1
or r.get("counts", {}).get("tests", 0) < 1
or (r.get("counts", {}).get("failed", 0) > 0 and not r.get("failed_ids"))
for r in reports
):
print("::error::Missing, empty, or malformed test evidence")
return 1
baseline = load_baseline(args.baseline)
failures = failed_ids(reports)
if baseline is None:
print(f"[ratchet] no baseline at {args.baseline} - every failure is new")
baseline = set()
new = sorted(failures - baseline)
known = sorted(set(failures) - set(new))
flaky = sorted({f for r in reports for f in r.get("flaky_files", [])})
print(f"[ratchet] {len(failures)} failing ids: {len(new)} new, {len(known)} known (baselined)")
for fid in flaky:
print(f" flaky (passed on retry): {fid}")
lines = [f"### Test ratchet ({args.baseline.name})", ""]
if new:
lines.append(f"**{len(new)} NEW failure(s)** — not baselined, so this change owns them:")
lines.extend(f"- `{fid}`" for fid in new)
for fid in new:
print(f"::error title=New test failure::{fid}")
else:
lines.append(f"No new failures ({len(known)} known, {len(flaky)} flaky files).")
_summary(lines)
return 1 if new else 0
def cmd_summary(args: argparse.Namespace) -> int:
reports = load_reports(args.reports)
if not reports:
print("[ratchet] no shard reports found - the test lane produced nothing")
return 1
counts = {"tests": 0, "passed": 0, "failed": 0, "skipped": 0}
for report in reports:
for key in counts:
counts[key] += report.get("counts", {}).get(key, 0)
status = 0
if args.expected_shards:
errors = partition_errors(reports, args.expected_shards)
for error in errors:
print(f"::error::{error}")
if errors:
status = 1
lines = [
f"### Test suite ({len(reports)} shards)",
"",
f"{counts['passed']} passed · {counts['failed']} failed · {counts['skipped']} skipped",
]
if args.floor:
from scripts.ci.assert_min_passed import FLOOR
if counts["passed"] < FLOOR:
lines.append(f"**FAIL: passed={counts['passed']} is below the floor {FLOOR}**")
print(f"::error::passed={counts['passed']} < FLOOR={FLOOR} (mass-skip regression?)")
status = 1
else:
lines.append(f"Min-passed floor: {counts['passed']} >= {FLOOR}")
baseline = load_baseline(args.baseline) if args.baseline else None
if baseline is not None:
fixed = sorted(baseline - failed_ids(reports))
if fixed:
lines.append("")
lines.append(
f"{len(fixed)} baseline failure(s) not observed; confirm they passed "
"rather than skipped before removing them:"
)
lines.extend(f"- `{fid}`" for fid in fixed[:50])
slowest = max(reports, key=lambda r: float(r.get("elapsed_seconds", 0)))
lines.append(f"Slowest shard: {slowest.get('shard')} ({slowest.get('elapsed_seconds')} s)")
print("\n".join(lines))
_summary(lines)
return status
def cmd_update(args: argparse.Namespace) -> int:
reports = load_reports(args.reports)
failures = sorted(failed_ids(reports))
flaky_files = load_flaky_files(args.out)
payload = {
"comment": (
"Known test failures. Generated by scripts/ci/ratchet_tests.py update from "
"one or more full CI runs on main (the union, so timing-sensitive tests that "
"flip between runs are covered); entries may only be removed by hand, never "
"added to hide a regression."
),
"flaky_files": flaky_files,
"known_failures": failures,
}
args.out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
print(f"[ratchet] wrote {len(failures)} known failures to {args.out}")
return 0
def junit_outcomes(run_dir: Path, os_name: str) -> dict[str, str]:
"""Outcome per test id in one run's merged JUnit files; any failure wins."""
import xml.etree.ElementTree as ET
outcomes: dict[str, str] = {}
for path in sorted(run_dir.rglob(f"junit-{os_name}-*.xml")):
root = ET.parse(path).getroot() # noqa: S314 - our own CI artifact
for case in root.iter("testcase"):
fid = normalize(f"{case.get('classname', '')}::{case.get('name', '')}")
if case.find("failure") is not None or case.find("error") is not None:
outcomes[fid] = "failed"
elif case.find("skipped") is not None:
outcomes.setdefault(fid, "skipped")
elif outcomes.get(fid) != "failed":
outcomes[fid] = "passed"
return outcomes
def cmd_prune(args: argparse.Namespace) -> int:
"""Drop baseline entries that PASSED (not skipped) in every given full run.
Each positional directory holds one run's downloaded ``tests-<os>-*``
artifacts. Requiring several runs keeps tests that flip between runs.
"""
data = json.loads(args.baseline.read_text(encoding="utf-8"))
known = data.get("known_failures", [])
runs = [junit_outcomes(run_dir, args.os) for run_dir in args.runs]
if not runs or any(not run for run in runs):
print("::error::every run directory needs JUnit evidence for this OS")
return 1
proven = {fid for fid in known if all(run.get(normalize(fid)) == "passed" for run in runs)}
data["known_failures"] = [fid for fid in known if fid not in proven]
args.baseline.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
print(
f"[ratchet] pruned {len(proven)} of {len(known)} known failures "
f"(passed in all {len(runs)} runs); {len(data['known_failures'])} remain"
)
return 0
def cmd_durations(args: argparse.Namespace) -> int:
merged: dict[str, float] = {}
if args.base and args.base.is_file():
merged.update(json.loads(args.base.read_text(encoding="utf-8")))
for path in args.inputs:
files = sorted(path.rglob("durations*.json")) if path.is_dir() else [path]
for file in files:
merged.update(json.loads(file.read_text(encoding="utf-8")))
payload = json.dumps(dict(sorted(merged.items())), indent=1) + "\n"
args.out.write_text(payload, encoding="utf-8")
print(f"[ratchet] duration cache: {len(merged)} files -> {args.out}")
return 0
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
sub = parser.add_subparsers(dest="cmd", required=True)
check = sub.add_parser("check")
check.add_argument("--baseline", type=Path, required=True)
check.add_argument("reports", nargs="+", type=Path)
summary = sub.add_parser("summary")
summary.add_argument("--baseline", type=Path)
summary.add_argument("--floor", action="store_true")
summary.add_argument("--expected-shards", type=int)
summary.add_argument("reports", nargs="+", type=Path)
update = sub.add_parser("update")
update.add_argument("--out", type=Path, required=True)
update.add_argument("reports", nargs="+", type=Path)
prune = sub.add_parser("prune")
prune.add_argument("--baseline", type=Path, required=True)
prune.add_argument("--os", required=True, choices=["linux", "windows", "macos"])
prune.add_argument("runs", nargs="+", type=Path)
durations = sub.add_parser("durations")
durations.add_argument("--base", type=Path)
durations.add_argument("--out", type=Path, required=True)
durations.add_argument("inputs", nargs="+", type=Path)
args = parser.parse_args(argv)
handler = {
"check": cmd_check,
"summary": cmd_summary,
"update": cmd_update,
"prune": cmd_prune,
"durations": cmd_durations,
}[args.cmd]
return handler(args)
if __name__ == "__main__":
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
sys.exit(main())