Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
fix(ci): let the baseline mark whole files as flaky
rubenluetke10-beep5fd365d9e16e
diff --git a/scripts/ci/ratchet_tests.py b/scripts/ci/ratchet_tests.pyindex 2f1e8ddeeb..3dac1fb3d1 100644--- a/scripts/ci/ratchet_tests.py+++ b/scripts/ci/ratchet_tests.py@@ -10,7 +10,10 @@ every PR red for reasons it did not cause. The ratchet splits the two:reported, never blocking;* any other failure is NEW — it blocks the change that introduced it;* a known failure that now passes is reported so the baseline can shrink- (``update`` rewrites it from a full run; it may only get smaller in review).+ (``update`` rewrites it from a full run; it may only get smaller in review);+* ``flaky_files`` lists test FILES whose tests flip between runs on a busy+ runner (timing-sensitive). Any failure inside such a file is known. It is+ kept by hand, ``update`` preserves it, and it stays short on purpose.Commands::@@ -58,6 +61,23 @@ def load_baseline(path: Path) -> set[str] | None:return {normalize(x) for x in data.get("known_failures", [])}+def load_flaky_files(path: Path | None) -> list[str]:+ if not path or not path.is_file():+ return []+ return list(json.loads(path.read_text(encoding="utf-8")).get("flaky_files", []))+++def in_flaky_file(test_id: str, flaky_files: list[str]) -> bool:+ """True when ``test_id`` (dotted ``module::name`` or ``path::<crash>``) lives+ in one of ``flaky_files`` (repo-relative ``.py`` paths)."""+ head = test_id.split("::", 1)[0]+ for file in flaky_files:+ dotted = file[:-3].replace("/", ".") if file.endswith(".py") else file+ if head == file or head == dotted or head.startswith(dotted + "."):+ return True+ return False++def failed_ids(reports: list[dict]) -> set[str]:return {normalize(fid) for report in reports for fid in report.get("failed_ids", [])}@@ -78,8 +98,9 @@ def cmd_check(args: argparse.Namespace) -> int:for fid in sorted(failures):print(f" failing: {fid}")return 0- new = sorted(failures - baseline)- known = sorted(failures & baseline)+ flaky_files = load_flaky_files(args.baseline)+ new = sorted(f for f in failures - baseline if not in_flaky_file(f, flaky_files))+ known = sorted(set(failures) - set(new))flaky = sorted({f for r in reports for f in r.get("flaky_files", [])})print(f"[ratchet] {len(failures)} failing ids: {len(new)} new, {len(known)} known (baselined)")for fid in flaky:@@ -137,6 +158,7 @@ def cmd_summary(args: argparse.Namespace) -> int:def cmd_update(args: argparse.Namespace) -> int:reports = load_reports(args.reports)failures = sorted(failed_ids(reports))+ flaky_files = load_flaky_files(args.out)payload = {"comment": ("Known test failures. Generated by scripts/ci/ratchet_tests.py update from "@@ -144,6 +166,7 @@ def cmd_update(args: argparse.Namespace) -> int:"flip between runs are covered); entries may only be removed by hand, never ""added to hide a regression."),+ "flaky_files": flaky_files,"known_failures": failures,}args.out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")diff --git a/scripts/ci/test-baseline-windows.json b/scripts/ci/test-baseline-windows.jsonindex 7b70ff83cd..bfcfd09afe 100644--- a/scripts/ci/test-baseline-windows.json+++ b/scripts/ci/test-baseline-windows.json@@ -1,5 +1,11 @@{"comment": "Known test failures. Generated by scripts/ci/ratchet_tests.py update from one or more full CI runs on main (the union, so timing-sensitive tests that flip between runs are covered); entries may only be removed by hand, never added to hide a regression.",+ "flaky_files": [+ "tests/missions/kontrollierer/test_capture_diff.py",+ "tests/missions/kontrollierer/test_external_write_verification.py",+ "tests/unit/dictation/test_polish_timeout_ramp.py",+ "tests/unit/memory/wiki/test_voice_bridge_realtime.py"+ ],"known_failures": ["tests.contract.test_society_coding_sessions::test_jarvis_chat_receives_controller_without_becoming_a_coding_cli","tests.integration.test_capability_coupling_e2e::test_hard_positive_file_ops_resolves_to_file_capability",diff --git a/tests/unit/ci/test_pipeline_scripts.py b/tests/unit/ci/test_pipeline_scripts.pyindex 510d67117e..f5ab3acf44 100644--- a/tests/unit/ci/test_pipeline_scripts.py+++ b/tests/unit/ci/test_pipeline_scripts.py@@ -340,3 +340,22 @@ def test_apply_moves_notes_under_a_dated_section(tmp_path):assert release_admit.versions(tmp_path) == ("1.1.0", "1.1.0")assert release_admit.check_identity("v1.1.0", tmp_path) == []assert release_admit.check_identity("v1.2.0", tmp_path)+++def test_any_failure_inside_a_flaky_file_is_known(tmp_path):+ baseline = tmp_path / "b.json"+ baseline.write_text(+ json.dumps({"flaky_files": ["tests/unit/x/test_timing.py"], "known_failures": []}),+ encoding="utf-8",+ )+ flip = _report(tmp_path / "r1.json", ["tests.unit.x.test_timing::test_ramp"])+ crash = _report(tmp_path / "r2.json", ["tests/unit/x/test_timing.py::<timeout 300s>"])+ other = _report(tmp_path / "r3.json", ["tests.unit.x.test_timing_other::test_a"])+ assert ratchet_tests.main(["check", "--baseline", str(baseline), str(flip)]) == 0+ assert ratchet_tests.main(["check", "--baseline", str(baseline), str(crash)]) == 0+ assert ratchet_tests.main(["check", "--baseline", str(baseline), str(other)]) == 1+ out = tmp_path / "b.json"+ assert ratchet_tests.main(["update", "--out", str(out), str(other)]) == 0+ assert json.loads(out.read_text(encoding="utf-8"))["flaky_files"] == [+ "tests/unit/x/test_timing.py"+ ]