Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(ci): budget the unit-test watchdog against each file's own cost

cyh872a6745df7b
diff --git a/scripts/ci_unit_test_shards.py b/scripts/ci_unit_test_shards.py
index 72e5d440b..d6ce0277b 100644
--- a/scripts/ci_unit_test_shards.py
+++ b/scripts/ci_unit_test_shards.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import json
+import math
import re
import statistics
import sys
@@ -12,6 +13,13 @@ from typing import Iterable, Mapping
DEFAULT_TEST_ROOT = Path("tests")
DEFAULT_TIMINGS_PATH = Path("scripts/ci_unit_test_timings.json")
+#: How much slower than its own recorded duration a file may run before the
+#: per-file watchdog treats it as stalled rather than merely unlucky. A single
+#: wall-clock budget cannot tell those apart across a suite whose files span
+#: three orders of magnitude, so the budget is expressed as a multiple of what
+#: the file itself has actually cost.
+DEFAULT_WATCHDOG_MULTIPLIER = 8.0
+
def discover_unit_test_files(root: Path = DEFAULT_TEST_ROOT) -> list[Path]:
"""Return deterministic unit-test files, excluding Docker-backed e2e tests."""
@@ -64,6 +72,40 @@ def _timing_scale(files: Iterable[Path], timings: Mapping[str, float]) -> float:
return statistics.median(ratios) if ratios else 1.0
+def expected_seconds(
+ files: Iterable[Path],
+ timings: Mapping[str, float] | None = None,
+) -> dict[str, float]:
+ """Cost per file: its measured duration, else a scaled structural estimate."""
+ file_list = sorted(files)
+ timing_map = timings or {}
+ scale = _timing_scale(file_list, timing_map)
+ return {
+ path.as_posix(): max(1.0, float(timing_map.get(path.as_posix(), structural_weight(path) * scale)))
+ for path in file_list
+ }
+
+
+def watchdog_budget_seconds(
+ path: Path,
+ timings: Mapping[str, float] | None = None,
+ *,
+ floor_seconds: int,
+ multiplier: float = DEFAULT_WATCHDOG_MULTIPLIER,
+) -> tuple[int, float | None]:
+ """Return the watchdog budget for one file and the baseline it came from.
+
+ Only a *measured* duration raises the budget. The structural estimate the
+ planner falls back on is a relative weight, not seconds, so multiplying it
+ would hand an arbitrary deadline to exactly the files nothing is known
+ about; those keep the floor, which is what the fixed budget always was.
+ """
+ baseline = (timings or {}).get(path.as_posix())
+ if baseline is None or baseline <= 0:
+ return floor_seconds, None
+ return max(floor_seconds, math.ceil(baseline * multiplier)), baseline
+
+
def plan_shards(
files: Iterable[Path],
shard_total: int,
@@ -73,13 +115,8 @@ def plan_shards(
if shard_total < 1:
raise ValueError("shard_total must be at least 1")
file_list = sorted(files)
- timing_map = timings or {}
- scale = _timing_scale(file_list, timing_map)
- weighted_files = []
- for path in file_list:
- key = path.as_posix()
- weight = timing_map.get(key, structural_weight(path) * scale)
- weighted_files.append((max(1.0, float(weight)), path))
+ weights = expected_seconds(file_list, timings)
+ weighted_files = [(weights[path.as_posix()], path) for path in file_list]
shards: list[tuple[float, list[Path]]] = [(0.0, []) for _ in range(shard_total)]
for weight, path in sorted(weighted_files, key=lambda item: (-item[0], item[1].as_posix())):
@@ -94,13 +131,25 @@ def plan_shards(
def main(argv: list[str] | None = None) -> int:
args = list(argv if argv is not None else sys.argv[1:])
+ budget_floor: int | None = None
+ for index, value in enumerate(args):
+ if value.startswith("--budget-floor="):
+ budget_floor = int(value.removeprefix("--budget-floor=") or 0)
+ del args[index]
+ break
if len(args) != 2 or any(not value.isdigit() for value in args):
- print("Usage: ci_unit_test_shards.py SHARD_INDEX SHARD_TOTAL", file=sys.stderr)
+ print(
+ "Usage: ci_unit_test_shards.py [--budget-floor=SECONDS] SHARD_INDEX SHARD_TOTAL",
+ file=sys.stderr,
+ )
return 2
shard_index, shard_total = (int(value) for value in args)
if shard_total < 1 or shard_index >= shard_total:
print("Shard index must be less than a positive shard total.", file=sys.stderr)
return 2
+ if budget_floor is not None and budget_floor < 1:
+ print("Budget floor must be a positive number of seconds.", file=sys.stderr)
+ return 2
files = discover_unit_test_files()
timings = load_timings()
@@ -114,7 +163,13 @@ def main(argv: list[str] | None = None) -> int:
file=sys.stderr,
)
for path in selected:
- print(path.as_posix())
+ if budget_floor is None:
+ print(path.as_posix())
+ continue
+ budget, baseline = watchdog_budget_seconds(path, timings, floor_seconds=budget_floor)
+ # Tab-separated so the launcher reads the budget it must arm without
+ # paying a second interpreter start per file to recompute it.
+ print(f"{path.as_posix()}\t{budget}\t{'' if baseline is None else format(baseline, '.0f')}")
return 0
diff --git a/scripts/ci_unit_tests.sh b/scripts/ci_unit_tests.sh
index 3c10ab5ec..e44e61e74 100755
--- a/scripts/ci_unit_tests.sh
+++ b/scripts/ci_unit_tests.sh
@@ -15,6 +15,13 @@
# use the checked-in per-file timing snapshot when available and fall back to a
# deterministic file-size/test-count estimate for new or missing files.
#
+# Each file gets its own watchdog budget from that same snapshot: a multiple of
+# what the file itself has recorded, floored at CI_TEST_FILE_TIMEOUT_SECONDS. A
+# single fixed budget cannot separate "hung" from "slow" across a suite whose
+# files span three orders of magnitude -- it is generous for a 1s file and thin
+# for a 74s one, so a uniformly slow runner kills the slowest file first and
+# names it as the culprit. Files with no recorded duration keep the floor.
+#
# Excludes ``tests/e2e`` (Docker) and the ``integration`` marker (Docker +
# platform tokens) — those run in dedicated jobs. ``-p no:randomly`` keeps a
# deterministic order if pytest-randomly happens to be installed (no-op when it
@@ -60,6 +67,8 @@ if [ -z "$PYTHON_BIN" ]; then
fi
fi
+# The floor under every per-file budget, and the whole budget for any file the
+# timing snapshot does not know about.
FILE_TIMEOUT="${CI_TEST_FILE_TIMEOUT_SECONDS:-300}"
case "$FILE_TIMEOUT" in
''|*[!0-9]*)
@@ -87,26 +96,48 @@ metrics = FileMetrics(sys.argv[1], started_at)
exit_code = pytest.main(sys.argv[1:], plugins=[metrics])
metrics.emit(diagnostics, int(exit_code))
sys.exit(exit_code)
-' "$FILE_TIMEOUT")
+')
+# Emits "<path>\t<budget seconds>\t<recorded seconds, blank if unknown>".
select_unit_test_files() {
- "$PYTHON_BIN" scripts/ci_unit_test_shards.py "$SHARD_INDEX" "$SHARD_TOTAL"
+ "$PYTHON_BIN" scripts/ci_unit_test_shards.py \
+ --budget-floor="$FILE_TIMEOUT" "$SHARD_INDEX" "$SHARD_TOTAL"
}
failed=""
empty=""
selected=0
discovered=0
-while IFS= read -r f; do
+while IFS=$'\t' read -r f budget baseline; do
+ [ -n "$f" ] || continue
+ case "$budget" in ''|*[!0-9]*) budget="$FILE_TIMEOUT" ;; esac
discovered=$((discovered + 1))
selected=$((selected + 1))
started_at=$(date +%s)
- echo "Starting $f (timeout ${FILE_TIMEOUT}s)."
- "${PYTEST[@]}" "$f" -m "not integration" -p no:randomly -p no:faulthandler -o addopts="" -v
+ if [ -n "$baseline" ]; then
+ echo "Starting $f (timeout ${budget}s, budgeted from its recorded ${baseline}s)."
+ else
+ echo "Starting $f (timeout ${budget}s)."
+ fi
+ "${PYTEST[@]}" "$budget" "$f" -m "not integration" -p no:randomly -p no:faulthandler -o addopts="" -v
rc=$?
finished_at=$(date +%s)
elapsed=$((finished_at - started_at))
echo "Finished $f in ${elapsed}s with exit code $rc."
+ if [ "$rc" -ne 0 ] && [ "$elapsed" -ge "$budget" ]; then
+ # Say what the dump above is and is not. faulthandler prints whichever frame
+ # the main thread occupied when the budget expired; in a file whose cost is
+ # spread over many tests that is usually an ordinary cheap frame, and reading
+ # it as the stall site sends the next reader after an innocent test.
+ echo " ^ $f hit its ${budget}s watchdog."
+ if [ -n "$baseline" ]; then
+ echo " That budget is a multiple of this file's own recorded ${baseline}s, so being the slowest file is not enough to reach it."
+ else
+ echo " No recorded duration for this file, so it ran on the ${FILE_TIMEOUT}s floor; refresh scripts/ci_unit_test_timings.json to give it a budget of its own."
+ fi
+ echo " The traceback above is where the main thread stood when the budget expired, not necessarily where it stalled."
+ echo " Before blaming the named test, compare the 'Finished ... in Ns' lines in this shard against scripts/ci_unit_test_timings.json: if they are slow across the board, the runner was slow, not this file."
+ fi
if [ "$rc" -eq 0 ]; then
:
elif [ "$rc" -eq 5 ]; then
diff --git a/tests/test_ci_pipeline.py b/tests/test_ci_pipeline.py
index 310b00ba8..ab0fcaba1 100644
--- a/tests/test_ci_pipeline.py
+++ b/tests/test_ci_pipeline.py
@@ -124,13 +124,23 @@ def test_ci_tool_install_command_propagates_failure(tmp_path, exit_code):
assert "--only-binary=:all:" in result.stdout
-def _run_isolated_unit_files(tmp_path: Path, sources: dict[str, str], *, timeout: str = "15"):
+def _run_isolated_unit_files(
+ tmp_path: Path,
+ sources: dict[str, str],
+ *,
+ timeout: str = "15",
+ timings: dict[str, float] | None = None,
+):
(tmp_path / "tests").mkdir()
(tmp_path / "scripts").mkdir()
for name, source in sources.items():
(tmp_path / "tests" / name).write_text(source, encoding="utf-8")
shutil.copyfile(ROOT / "scripts/ci_unit_test_shards.py", tmp_path / "scripts/ci_unit_test_shards.py")
shutil.copyfile(ROOT / "scripts/ci_pytest_metrics.py", tmp_path / "scripts/ci_pytest_metrics.py")
+ if timings is not None:
+ (tmp_path / "scripts" / "ci_unit_test_timings.json").write_text(
+ json.dumps({"durations_seconds": timings}), encoding="utf-8",
+ )
return subprocess.run(
["bash", str(ROOT / "scripts/ci_unit_tests.sh")], cwd=tmp_path,
env={**os.environ, "PYTHON": sys.executable, "CI_TEST_FILE_TIMEOUT_SECONDS": timeout,
@@ -170,6 +180,13 @@ def test_unit_file_watchdog_dumps_stacks_and_continues_fail_closed(tmp_path: Pat
assert "test_after_stuck_file PASSED" in result.stdout
assert "FAILED files:\n tests/test_a_stuck.py" in result.stdout
assert "All unit test files passed." not in result.stdout
+ # The dump names whichever frame the main thread held at the deadline, which
+ # in a file whose cost is spread across many tests is an ordinary cheap one.
+ # Saying so is the difference between reading the trace as evidence and
+ # reading it as a verdict on the test it happens to name.
+ assert "^ tests/test_a_stuck.py hit its 3s watchdog." in result.stdout
+ assert "No recorded duration for this file, so it ran on the 3s floor" in result.stdout
+ assert "not necessarily where it stalled" in result.stdout
records = _metrics(result)
assert records[-1]["file"] == "tests/test_b_pass.py"
if phase == "shutdown":
@@ -179,6 +196,35 @@ def test_unit_file_watchdog_dumps_stacks_and_continues_fail_closed(tmp_path: Pat
assert len(records) == 1
+def test_unit_file_watchdog_budget_follows_the_file_own_recorded_cost(tmp_path: Path) -> None:
+ """A file the snapshot knows is slow gets headroom measured against itself.
+
+ The fixed budget this replaces could not tell "hung" from "slow": it was
+ generous for a 1s file and thin for the suite's slowest, so a uniformly
+ slow runner killed the slowest file first and named it as the culprit.
+ """
+ result = _run_isolated_unit_files(
+ tmp_path,
+ {
+ "test_a_stuck.py": "import time\ndef test_stuck():\n time.sleep(3600)\n",
+ "test_b_pass.py": "def test_after_stuck_file():\n pass\n",
+ },
+ timeout="4",
+ timings={"tests/test_a_stuck.py": 1.0},
+ )
+
+ assert result.returncode == 1, result.stdout + result.stderr
+ # 8x its own second, not the 4s floor the same run gives an unknown file.
+ assert "Starting tests/test_a_stuck.py (timeout 8s, budgeted from its recorded 1s)." in result.stdout
+ assert "Timeout (0:00:08)!" in result.stderr
+ assert "^ tests/test_a_stuck.py hit its 8s watchdog." in result.stdout
+ assert "That budget is a multiple of this file's own recorded 1s" in result.stdout
+ # An unknown file keeps the floor, and the line it prints is unchanged.
+ assert "Starting tests/test_b_pass.py (timeout 4s)." in result.stdout
+ assert "test_after_stuck_file PASSED" in result.stdout
+ assert "FAILED files:\n tests/test_a_stuck.py" in result.stdout
+
+
@pytest.mark.parametrize("failed", [False, True])
def test_unit_file_runner_preserves_assertions_and_integration_selection(tmp_path: Path, failed: bool) -> None:
result = _run_isolated_unit_files(tmp_path, {
diff --git a/tests/test_ci_unit_test_shards.py b/tests/test_ci_unit_test_shards.py
index a456b616b..4aaa9f9d8 100644
--- a/tests/test_ci_unit_test_shards.py
+++ b/tests/test_ci_unit_test_shards.py
@@ -4,7 +4,16 @@ from pathlib import Path
import pytest
-from scripts.ci_unit_test_shards import discover_unit_test_files, load_timings, plan_shards
+import json
+
+from scripts.ci_unit_test_shards import (
+ DEFAULT_WATCHDOG_MULTIPLIER,
+ discover_unit_test_files,
+ load_timings,
+ main,
+ plan_shards,
+ watchdog_budget_seconds,
+)
def _test_file(root: Path, name: str, *, tests: int = 1, lines: int = 1) -> Path:
@@ -61,3 +70,67 @@ def test_measured_plan_runs_every_discovered_file_exactly_once() -> None:
shards = plan_shards(files, 6, load_timings())
assert sorted(path for _, selected in shards for path in selected) == files
+
+
+def test_watchdog_budget_scales_with_the_file_own_recorded_cost(tmp_path: Path) -> None:
+ slow = _test_file(tmp_path, "test_slow.py")
+ quick = _test_file(tmp_path, "test_quick.py")
+ timings = {slow.as_posix(): 74.0, quick.as_posix(): 2.0}
+
+ slow_budget, slow_baseline = watchdog_budget_seconds(slow, timings, floor_seconds=300)
+ quick_budget, quick_baseline = watchdog_budget_seconds(quick, timings, floor_seconds=300)
+
+ # The slow file is the one a fixed budget squeezes: 74s against 300s leaves
+ # it barely 4x of headroom, which a merely slow runner can eat. Its budget
+ # is a multiple of its own cost, so slowness alone no longer reads as a hang.
+ assert slow_budget == int(74 * DEFAULT_WATCHDOG_MULTIPLIER)
+ assert slow_baseline == 74.0
+ # A quick file would be *cut* by the same multiple, so the floor holds.
+ assert quick_budget == 300
+ assert quick_baseline == 2.0
+
+
+def test_watchdog_budget_keeps_the_floor_for_files_with_no_recorded_cost(tmp_path: Path) -> None:
+ unknown = _test_file(tmp_path, "test_unknown.py", tests=200, lines=5000)
+
+ budget, baseline = watchdog_budget_seconds(unknown, {}, floor_seconds=300)
+
+ # The planner's structural estimate is a relative weight, not seconds.
+ # Multiplying it would hand an arbitrary deadline to exactly the files
+ # nothing is known about, however large the estimate grows.
+ assert budget == 300
+ assert baseline is None
+
+
+def test_main_emits_budgets_only_when_a_floor_is_requested(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ (tmp_path / "tests").mkdir()
+ measured = _test_file(tmp_path / "tests", "test_measured.py")
+ _test_file(tmp_path / "tests", "test_new.py")
+ (tmp_path / "scripts").mkdir()
+ (tmp_path / "scripts" / "ci_unit_test_timings.json").write_text(
+ json.dumps({"durations_seconds": {"tests/test_measured.py": 50.0}}),
+ encoding="utf-8",
+ )
+ monkeypatch.chdir(tmp_path)
+ assert measured.is_file()
+
+ assert main(["0", "1"]) == 0
+ plain = capsys.readouterr().out.splitlines()
+ # Callers that only want a file list keep reading one, unchanged.
+ assert plain == ["tests/test_measured.py", "tests/test_new.py"]
+
+ assert main(["--budget-floor=10", "0", "1"]) == 0
+ budgeted = [line.split("\t") for line in capsys.readouterr().out.splitlines()]
+ assert budgeted == [
+ ["tests/test_measured.py", str(int(50 * DEFAULT_WATCHDOG_MULTIPLIER)), "50"],
+ # Blank baseline is how the launcher knows to print the legacy line.
+ ["tests/test_new.py", "10", ""],
+ ]
+
+
+def test_main_rejects_a_budget_floor_that_is_not_a_positive_number_of_seconds() -> None:
+ assert main(["--budget-floor=0", "0", "1"]) == 2