Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(ci): stop a hanging test shard before the job timeout so it still reports (#963)

Ruben Lütke088e01758bb8
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4eb82bfb3d..32e0eaa2c7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -520,7 +520,8 @@ jobs:
--shard "$SHARD/$total" \
--report "out/test-report-linux-$SHARD.json" \
--junit-out "out/junit-linux-$SHARD.xml" \
- --durations-out "out/durations-linux-$SHARD.json"
+ --durations-out "out/durations-linux-$SHARD.json" \
+ --deadline-minutes 22
- name: Ratchet (a failure not in the baseline blocks)
env:
SHARD: ${{ matrix.shard }}
@@ -575,7 +576,8 @@ jobs:
--durations test_durations_windows.json \
--report "out/test-report-windows-$SHARD.json" \
--junit-out "out/junit-windows-$SHARD.xml" \
- --durations-out "out/durations-windows-$SHARD.json"
+ --durations-out "out/durations-windows-$SHARD.json" \
+ --deadline-minutes 32
- name: Ratchet (a failure not in the baseline blocks)
env:
SHARD: ${{ matrix.shard }}
@@ -619,6 +621,7 @@ jobs:
python scripts/ci/run_tests_parallel.py tests --shard ${{ matrix.shard }}/3
--report out/test-report-macos-${{ matrix.shard }}.json
--junit-out out/junit-macos-${{ matrix.shard }}.xml
+ --deadline-minutes 50
- name: Ratchet (a failure not in the baseline blocks)
run: python scripts/ci/ratchet_tests.py check --baseline scripts/ci/test-baseline-macos.json out/test-report-macos-${{ matrix.shard }}.json
- name: Upload shard report
@@ -667,7 +670,8 @@ jobs:
python scripts/ci/run_tests_parallel.py tests --files-from mac_tests.txt \
--shard "$SHARD/$total" \
--report "out/test-report-macos-impact-$SHARD.json" \
- --junit-out "out/junit-macos-impact-$SHARD.xml"
+ --junit-out "out/junit-macos-impact-$SHARD.xml" \
+ --deadline-minutes 32
- name: Ratchet (a failure not in the baseline blocks)
env:
SHARD: ${{ matrix.shard }}
diff --git a/scripts/ci/run_tests_parallel.py b/scripts/ci/run_tests_parallel.py
index 03fe17904d..ad45fa4e0a 100644
--- a/scripts/ci/run_tests_parallel.py
+++ b/scripts/ci/run_tests_parallel.py
@@ -346,6 +346,13 @@ class Runner:
self.gate = gate or MemoryGate()
self._seq = 0
self._lock = threading.Lock()
+ # The whole shard's wall clock. Batch budget plus isolation and retry
+ # runs can outlast the CI job's timeout; the job is then cancelled with
+ # no report and no log, so a hang names nothing (merge-queue run
+ # 40485520423, tests-windows (6)). Past the deadline nothing new starts
+ # and a running process is stopped: the report still gets written.
+ minutes = getattr(args, "deadline_minutes", None)
+ self.deadline = time.monotonic() + 60 * minutes if minutes else None
def _next_id(self) -> int:
with self._lock:
@@ -353,6 +360,18 @@ class Runner:
return self._seq
def run(self, files: list[str], timeout: float) -> RunResult:
+ if self.deadline is not None:
+ left = self.deadline - time.monotonic()
+ if left <= 0:
+ return RunResult(
+ files,
+ -1,
+ 0.0,
+ True,
+ [f"{f}::<shard deadline>" for f in files],
+ log_tail="not started: the shard's deadline had passed",
+ )
+ timeout = min(timeout, left)
run_id = self._next_id()
junit = self.workdir / f"run-{run_id:05d}.xml"
log = self.workdir / f"run-{run_id:05d}.log"
@@ -484,6 +503,11 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument("--report", type=Path, default=Path("test-report.json"))
parser.add_argument("--junit-out", type=Path, default=Path("report.xml"))
parser.add_argument("--list", action="store_true", help="Print the shard's files and exit.")
+ parser.add_argument(
+ "--deadline-minutes",
+ type=float,
+ help="Wall clock for the whole shard; keep it under the CI job's timeout.",
+ )
args = parser.parse_args(argv)
# Windows consoles default to cp1252; a test log with box-drawing
# characters must never crash the runner that is reporting it (AP-1).
diff --git a/tests/unit/ci/test_pipeline_scripts.py b/tests/unit/ci/test_pipeline_scripts.py
index 2487fbb7c4..ff8130a5e7 100644
--- a/tests/unit/ci/test_pipeline_scripts.py
+++ b/tests/unit/ci/test_pipeline_scripts.py
@@ -463,6 +463,65 @@ def test_ratchet_blocks_only_new_failures(tmp_path):
assert ratchet_tests.main(["check", "--baseline", str(baseline), str(new)]) == 1
+def test_nothing_starts_after_the_shard_deadline(tmp_path):
+ from types import SimpleNamespace
+
+ runner = run_tests_parallel.Runner(SimpleNamespace(deadline_minutes=1), {}, tmp_path)
+ runner.deadline = 0.0 # long past
+
+ result = runner.run(["t/test_late.py"], 300.0)
+
+ assert result.timed_out and not result.ok
+ assert result.failed_ids == ["t/test_late.py::<shard deadline>"]
+ assert not list(tmp_path.glob("run-*.log")) # no process was started
+
+
+def test_a_hang_is_stopped_at_the_shard_deadline_not_its_batch_budget(tmp_path):
+ """The batch budget (up to 30 min) plus retries outlasted the job timeout;
+ the shard deadline stops the process so the report names the file."""
+ import time
+ from types import SimpleNamespace
+
+ repo = run_tests_parallel.REPO_ROOT
+ rel = Path(f"tests/unit/ci/_runner_hang_{tmp_path.name}")
+ probe = repo / rel
+ probe.mkdir()
+ try:
+ (probe / "test_probe_hang.py").write_text(
+ "import time\n\ndef test_hang():\n time.sleep(120)\n"
+ )
+ args = SimpleNamespace(deadline_minutes=1, markers="", pytest_args="")
+ runner = run_tests_parallel.Runner(args, {}, tmp_path)
+ runner.deadline = time.monotonic() + 3
+ started = time.monotonic()
+ result = runner.run([(rel / "test_probe_hang.py").as_posix()], 1800.0)
+ finally:
+ shutil.rmtree(probe, ignore_errors=True)
+ assert result.timed_out
+ assert time.monotonic() - started < 60
+
+
+def test_every_ci_shard_deadline_ends_before_its_job_timeout():
+ import re
+
+ import yaml
+
+ path = run_tests_parallel.REPO_ROOT / ".github" / "workflows" / "ci.yml"
+ jobs = yaml.safe_load(path.read_text(encoding="utf-8"))["jobs"]
+ seen = 0
+ for name, job in jobs.items():
+ for step in job.get("steps", []):
+ run = str(step.get("run", ""))
+ if "run_tests_parallel.py" not in run:
+ continue
+ match = re.search(r"--deadline-minutes\s+([0-9.]+)", run)
+ assert match, f"{name}: the shard has no --deadline-minutes"
+ # Room for checkout, install, the ratchet and the report upload.
+ assert float(match.group(1)) <= job["timeout-minutes"] - 6, name
+ seen += 1
+ assert seen >= 4
+
+
def test_ratchet_matches_timeouts_regardless_of_budget(tmp_path):
baseline = tmp_path / "b.json"
baseline.write_text(