Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(ci): let the baseline mark whole files as flaky

rubenluetke10-beep5fd365d9e16e
diff --git a/scripts/ci/ratchet_tests.py b/scripts/ci/ratchet_tests.py
index 2f1e8ddeeb..3dac1fb3d1 100644
--- a/scripts/ci/ratchet_tests.py
+++ b/scripts/ci/ratchet_tests.py
@@ -10,7 +10,10 @@ every PR red for reasons it did not cause. The ratchet splits the two:
reported, never blocking;
* any other failure is NEW — it blocks the change that introduced it;
* a known failure that now passes is reported so the baseline can shrink
- (``update`` rewrites it from a full run; it may only get smaller in review).
+ (``update`` rewrites it from a full run; it may only get smaller in review);
+* ``flaky_files`` lists test FILES whose tests flip between runs on a busy
+ runner (timing-sensitive). Any failure inside such a file is known. It is
+ kept by hand, ``update`` preserves it, and it stays short on purpose.
Commands::
@@ -58,6 +61,23 @@ def load_baseline(path: Path) -> set[str] | None:
return {normalize(x) for x in data.get("known_failures", [])}
+def load_flaky_files(path: Path | None) -> list[str]:
+ if not path or not path.is_file():
+ return []
+ return list(json.loads(path.read_text(encoding="utf-8")).get("flaky_files", []))
+
+
+def in_flaky_file(test_id: str, flaky_files: list[str]) -> bool:
+ """True when ``test_id`` (dotted ``module::name`` or ``path::<crash>``) lives
+ in one of ``flaky_files`` (repo-relative ``.py`` paths)."""
+ head = test_id.split("::", 1)[0]
+ for file in flaky_files:
+ dotted = file[:-3].replace("/", ".") if file.endswith(".py") else file
+ if head == file or head == dotted or head.startswith(dotted + "."):
+ return True
+ return False
+
+
def failed_ids(reports: list[dict]) -> set[str]:
return {normalize(fid) for report in reports for fid in report.get("failed_ids", [])}
@@ -78,8 +98,9 @@ def cmd_check(args: argparse.Namespace) -> int:
for fid in sorted(failures):
print(f" failing: {fid}")
return 0
- new = sorted(failures - baseline)
- known = sorted(failures & baseline)
+ flaky_files = load_flaky_files(args.baseline)
+ new = sorted(f for f in failures - baseline if not in_flaky_file(f, flaky_files))
+ known = sorted(set(failures) - set(new))
flaky = sorted({f for r in reports for f in r.get("flaky_files", [])})
print(f"[ratchet] {len(failures)} failing ids: {len(new)} new, {len(known)} known (baselined)")
for fid in flaky:
@@ -137,6 +158,7 @@ def cmd_summary(args: argparse.Namespace) -> int:
def cmd_update(args: argparse.Namespace) -> int:
reports = load_reports(args.reports)
failures = sorted(failed_ids(reports))
+ flaky_files = load_flaky_files(args.out)
payload = {
"comment": (
"Known test failures. Generated by scripts/ci/ratchet_tests.py update from "
@@ -144,6 +166,7 @@ def cmd_update(args: argparse.Namespace) -> int:
"flip between runs are covered); entries may only be removed by hand, never "
"added to hide a regression."
),
+ "flaky_files": flaky_files,
"known_failures": failures,
}
args.out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
diff --git a/scripts/ci/test-baseline-windows.json b/scripts/ci/test-baseline-windows.json
index 7b70ff83cd..bfcfd09afe 100644
--- a/scripts/ci/test-baseline-windows.json
+++ b/scripts/ci/test-baseline-windows.json
@@ -1,5 +1,11 @@
{
"comment": "Known test failures. Generated by scripts/ci/ratchet_tests.py update from one or more full CI runs on main (the union, so timing-sensitive tests that flip between runs are covered); entries may only be removed by hand, never added to hide a regression.",
+ "flaky_files": [
+ "tests/missions/kontrollierer/test_capture_diff.py",
+ "tests/missions/kontrollierer/test_external_write_verification.py",
+ "tests/unit/dictation/test_polish_timeout_ramp.py",
+ "tests/unit/memory/wiki/test_voice_bridge_realtime.py"
+ ],
"known_failures": [
"tests.contract.test_society_coding_sessions::test_jarvis_chat_receives_controller_without_becoming_a_coding_cli",
"tests.integration.test_capability_coupling_e2e::test_hard_positive_file_ops_resolves_to_file_capability",
diff --git a/tests/unit/ci/test_pipeline_scripts.py b/tests/unit/ci/test_pipeline_scripts.py
index 510d67117e..f5ab3acf44 100644
--- a/tests/unit/ci/test_pipeline_scripts.py
+++ b/tests/unit/ci/test_pipeline_scripts.py
@@ -340,3 +340,22 @@ def test_apply_moves_notes_under_a_dated_section(tmp_path):
assert release_admit.versions(tmp_path) == ("1.1.0", "1.1.0")
assert release_admit.check_identity("v1.1.0", tmp_path) == []
assert release_admit.check_identity("v1.2.0", tmp_path)
+
+
+def test_any_failure_inside_a_flaky_file_is_known(tmp_path):
+ baseline = tmp_path / "b.json"
+ baseline.write_text(
+ json.dumps({"flaky_files": ["tests/unit/x/test_timing.py"], "known_failures": []}),
+ encoding="utf-8",
+ )
+ flip = _report(tmp_path / "r1.json", ["tests.unit.x.test_timing::test_ramp"])
+ crash = _report(tmp_path / "r2.json", ["tests/unit/x/test_timing.py::<timeout 300s>"])
+ other = _report(tmp_path / "r3.json", ["tests.unit.x.test_timing_other::test_a"])
+ assert ratchet_tests.main(["check", "--baseline", str(baseline), str(flip)]) == 0
+ assert ratchet_tests.main(["check", "--baseline", str(baseline), str(crash)]) == 0
+ assert ratchet_tests.main(["check", "--baseline", str(baseline), str(other)]) == 1
+ out = tmp_path / "b.json"
+ assert ratchet_tests.main(["update", "--out", str(out), str(other)]) == 0
+ assert json.loads(out.read_text(encoding="utf-8"))["flaky_files"] == [
+ "tests/unit/x/test_timing.py"
+ ]