Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
fix(ci): stop a hanging test shard before the job timeout so it still reports (#963)
Ruben Lütke088e01758bb8
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 4eb82bfb3d..32e0eaa2c7 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -520,7 +520,8 @@ jobs:--shard "$SHARD/$total" \--report "out/test-report-linux-$SHARD.json" \--junit-out "out/junit-linux-$SHARD.xml" \- --durations-out "out/durations-linux-$SHARD.json"+ --durations-out "out/durations-linux-$SHARD.json" \+ --deadline-minutes 22- name: Ratchet (a failure not in the baseline blocks)env:SHARD: ${{ matrix.shard }}@@ -575,7 +576,8 @@ jobs:--durations test_durations_windows.json \--report "out/test-report-windows-$SHARD.json" \--junit-out "out/junit-windows-$SHARD.xml" \- --durations-out "out/durations-windows-$SHARD.json"+ --durations-out "out/durations-windows-$SHARD.json" \+ --deadline-minutes 32- name: Ratchet (a failure not in the baseline blocks)env:SHARD: ${{ matrix.shard }}@@ -619,6 +621,7 @@ jobs:python scripts/ci/run_tests_parallel.py tests --shard ${{ matrix.shard }}/3--report out/test-report-macos-${{ matrix.shard }}.json--junit-out out/junit-macos-${{ matrix.shard }}.xml+ --deadline-minutes 50- name: Ratchet (a failure not in the baseline blocks)run: python scripts/ci/ratchet_tests.py check --baseline scripts/ci/test-baseline-macos.json out/test-report-macos-${{ matrix.shard }}.json- name: Upload shard report@@ -667,7 +670,8 @@ jobs:python scripts/ci/run_tests_parallel.py tests --files-from mac_tests.txt \--shard "$SHARD/$total" \--report "out/test-report-macos-impact-$SHARD.json" \- --junit-out "out/junit-macos-impact-$SHARD.xml"+ --junit-out "out/junit-macos-impact-$SHARD.xml" \+ --deadline-minutes 32- name: Ratchet (a failure not in the baseline blocks)env:SHARD: ${{ matrix.shard }}diff --git a/scripts/ci/run_tests_parallel.py b/scripts/ci/run_tests_parallel.pyindex 03fe17904d..ad45fa4e0a 100644--- a/scripts/ci/run_tests_parallel.py+++ b/scripts/ci/run_tests_parallel.py@@ -346,6 +346,13 @@ class Runner:self.gate = gate or MemoryGate()self._seq = 0self._lock = threading.Lock()+ # The whole shard's wall clock. Batch budget plus isolation and retry+ # runs can outlast the CI job's timeout; the job is then cancelled with+ # no report and no log, so a hang names nothing (merge-queue run+ # 40485520423, tests-windows (6)). Past the deadline nothing new starts+ # and a running process is stopped: the report still gets written.+ minutes = getattr(args, "deadline_minutes", None)+ self.deadline = time.monotonic() + 60 * minutes if minutes else Nonedef _next_id(self) -> int:with self._lock:@@ -353,6 +360,18 @@ class Runner:return self._seqdef run(self, files: list[str], timeout: float) -> RunResult:+ if self.deadline is not None:+ left = self.deadline - time.monotonic()+ if left <= 0:+ return RunResult(+ files,+ -1,+ 0.0,+ True,+ [f"{f}::<shard deadline>" for f in files],+ log_tail="not started: the shard's deadline had passed",+ )+ timeout = min(timeout, left)run_id = self._next_id()junit = self.workdir / f"run-{run_id:05d}.xml"log = self.workdir / f"run-{run_id:05d}.log"@@ -484,6 +503,11 @@ def main(argv: list[str] | None = None) -> int:parser.add_argument("--report", type=Path, default=Path("test-report.json"))parser.add_argument("--junit-out", type=Path, default=Path("report.xml"))parser.add_argument("--list", action="store_true", help="Print the shard's files and exit.")+ parser.add_argument(+ "--deadline-minutes",+ type=float,+ help="Wall clock for the whole shard; keep it under the CI job's timeout.",+ )args = parser.parse_args(argv)# Windows consoles default to cp1252; a test log with box-drawing# characters must never crash the runner that is reporting it (AP-1).diff --git a/tests/unit/ci/test_pipeline_scripts.py b/tests/unit/ci/test_pipeline_scripts.pyindex 2487fbb7c4..ff8130a5e7 100644--- a/tests/unit/ci/test_pipeline_scripts.py+++ b/tests/unit/ci/test_pipeline_scripts.py@@ -463,6 +463,65 @@ def test_ratchet_blocks_only_new_failures(tmp_path):assert ratchet_tests.main(["check", "--baseline", str(baseline), str(new)]) == 1+def test_nothing_starts_after_the_shard_deadline(tmp_path):+ from types import SimpleNamespace++ runner = run_tests_parallel.Runner(SimpleNamespace(deadline_minutes=1), {}, tmp_path)+ runner.deadline = 0.0 # long past++ result = runner.run(["t/test_late.py"], 300.0)++ assert result.timed_out and not result.ok+ assert result.failed_ids == ["t/test_late.py::<shard deadline>"]+ assert not list(tmp_path.glob("run-*.log")) # no process was started+++def test_a_hang_is_stopped_at_the_shard_deadline_not_its_batch_budget(tmp_path):+ """The batch budget (up to 30 min) plus retries outlasted the job timeout;+ the shard deadline stops the process so the report names the file."""+ import time+ from types import SimpleNamespace++ repo = run_tests_parallel.REPO_ROOT+ rel = Path(f"tests/unit/ci/_runner_hang_{tmp_path.name}")+ probe = repo / rel+ probe.mkdir()+ try:+ (probe / "test_probe_hang.py").write_text(+ "import time\n\ndef test_hang():\n time.sleep(120)\n"+ )+ args = SimpleNamespace(deadline_minutes=1, markers="", pytest_args="")+ runner = run_tests_parallel.Runner(args, {}, tmp_path)+ runner.deadline = time.monotonic() + 3+ started = time.monotonic()+ result = runner.run([(rel / "test_probe_hang.py").as_posix()], 1800.0)+ finally:+ shutil.rmtree(probe, ignore_errors=True)+ assert result.timed_out+ assert time.monotonic() - started < 60+++def test_every_ci_shard_deadline_ends_before_its_job_timeout():+ import re++ import yaml++ path = run_tests_parallel.REPO_ROOT / ".github" / "workflows" / "ci.yml"+ jobs = yaml.safe_load(path.read_text(encoding="utf-8"))["jobs"]+ seen = 0+ for name, job in jobs.items():+ for step in job.get("steps", []):+ run = str(step.get("run", ""))+ if "run_tests_parallel.py" not in run:+ continue+ match = re.search(r"--deadline-minutes\s+([0-9.]+)", run)+ assert match, f"{name}: the shard has no --deadline-minutes"+ # Room for checkout, install, the ratchet and the report upload.+ assert float(match.group(1)) <= job["timeout-minutes"] - 6, name+ seen += 1+ assert seen >= 4++def test_ratchet_matches_timeouts_regardless_of_budget(tmp_path):baseline = tmp_path / "b.json"baseline.write_text(