Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(ci): budget the unit-test watchdog against each file's own cost
cyh872a6745df7b
diff --git a/scripts/ci_unit_test_shards.py b/scripts/ci_unit_test_shards.pyindex 72e5d440b..d6ce0277b 100644--- a/scripts/ci_unit_test_shards.py+++ b/scripts/ci_unit_test_shards.py@@ -3,6 +3,7 @@from __future__ import annotationsimport json+import mathimport reimport statisticsimport sys@@ -12,6 +13,13 @@ from typing import Iterable, MappingDEFAULT_TEST_ROOT = Path("tests")DEFAULT_TIMINGS_PATH = Path("scripts/ci_unit_test_timings.json")+#: How much slower than its own recorded duration a file may run before the+#: per-file watchdog treats it as stalled rather than merely unlucky. A single+#: wall-clock budget cannot tell those apart across a suite whose files span+#: three orders of magnitude, so the budget is expressed as a multiple of what+#: the file itself has actually cost.+DEFAULT_WATCHDOG_MULTIPLIER = 8.0+def discover_unit_test_files(root: Path = DEFAULT_TEST_ROOT) -> list[Path]:"""Return deterministic unit-test files, excluding Docker-backed e2e tests."""@@ -64,6 +72,40 @@ def _timing_scale(files: Iterable[Path], timings: Mapping[str, float]) -> float:return statistics.median(ratios) if ratios else 1.0+def expected_seconds(+ files: Iterable[Path],+ timings: Mapping[str, float] | None = None,+) -> dict[str, float]:+ """Cost per file: its measured duration, else a scaled structural estimate."""+ file_list = sorted(files)+ timing_map = timings or {}+ scale = _timing_scale(file_list, timing_map)+ return {+ path.as_posix(): max(1.0, float(timing_map.get(path.as_posix(), structural_weight(path) * scale)))+ for path in file_list+ }+++def watchdog_budget_seconds(+ path: Path,+ timings: Mapping[str, float] | None = None,+ *,+ floor_seconds: int,+ multiplier: float = DEFAULT_WATCHDOG_MULTIPLIER,+) -> tuple[int, float | None]:+ """Return the watchdog budget for one file and the baseline it came from.++ Only a *measured* duration raises the budget. The structural estimate the+ planner falls back on is a relative weight, not seconds, so multiplying it+ would hand an arbitrary deadline to exactly the files nothing is known+ about; those keep the floor, which is what the fixed budget always was.+ """+ baseline = (timings or {}).get(path.as_posix())+ if baseline is None or baseline <= 0:+ return floor_seconds, None+ return max(floor_seconds, math.ceil(baseline * multiplier)), baseline++def plan_shards(files: Iterable[Path],shard_total: int,@@ -73,13 +115,8 @@ def plan_shards(if shard_total < 1:raise ValueError("shard_total must be at least 1")file_list = sorted(files)- timing_map = timings or {}- scale = _timing_scale(file_list, timing_map)- weighted_files = []- for path in file_list:- key = path.as_posix()- weight = timing_map.get(key, structural_weight(path) * scale)- weighted_files.append((max(1.0, float(weight)), path))+ weights = expected_seconds(file_list, timings)+ weighted_files = [(weights[path.as_posix()], path) for path in file_list]shards: list[tuple[float, list[Path]]] = [(0.0, []) for _ in range(shard_total)]for weight, path in sorted(weighted_files, key=lambda item: (-item[0], item[1].as_posix())):@@ -94,13 +131,25 @@ def plan_shards(def main(argv: list[str] | None = None) -> int:args = list(argv if argv is not None else sys.argv[1:])+ budget_floor: int | None = None+ for index, value in enumerate(args):+ if value.startswith("--budget-floor="):+ budget_floor = int(value.removeprefix("--budget-floor=") or 0)+ del args[index]+ breakif len(args) != 2 or any(not value.isdigit() for value in args):- print("Usage: ci_unit_test_shards.py SHARD_INDEX SHARD_TOTAL", file=sys.stderr)+ print(+ "Usage: ci_unit_test_shards.py [--budget-floor=SECONDS] SHARD_INDEX SHARD_TOTAL",+ file=sys.stderr,+ )return 2shard_index, shard_total = (int(value) for value in args)if shard_total < 1 or shard_index >= shard_total:print("Shard index must be less than a positive shard total.", file=sys.stderr)return 2+ if budget_floor is not None and budget_floor < 1:+ print("Budget floor must be a positive number of seconds.", file=sys.stderr)+ return 2files = discover_unit_test_files()timings = load_timings()@@ -114,7 +163,13 @@ def main(argv: list[str] | None = None) -> int:file=sys.stderr,)for path in selected:- print(path.as_posix())+ if budget_floor is None:+ print(path.as_posix())+ continue+ budget, baseline = watchdog_budget_seconds(path, timings, floor_seconds=budget_floor)+ # Tab-separated so the launcher reads the budget it must arm without+ # paying a second interpreter start per file to recompute it.+ print(f"{path.as_posix()}\t{budget}\t{'' if baseline is None else format(baseline, '.0f')}")return 0diff --git a/scripts/ci_unit_tests.sh b/scripts/ci_unit_tests.shindex 3c10ab5ec..e44e61e74 100755--- a/scripts/ci_unit_tests.sh+++ b/scripts/ci_unit_tests.sh@@ -15,6 +15,13 @@# use the checked-in per-file timing snapshot when available and fall back to a# deterministic file-size/test-count estimate for new or missing files.#+# Each file gets its own watchdog budget from that same snapshot: a multiple of+# what the file itself has recorded, floored at CI_TEST_FILE_TIMEOUT_SECONDS. A+# single fixed budget cannot separate "hung" from "slow" across a suite whose+# files span three orders of magnitude -- it is generous for a 1s file and thin+# for a 74s one, so a uniformly slow runner kills the slowest file first and+# names it as the culprit. Files with no recorded duration keep the floor.+## Excludes ``tests/e2e`` (Docker) and the ``integration`` marker (Docker +# platform tokens) — those run in dedicated jobs. ``-p no:randomly`` keeps a# deterministic order if pytest-randomly happens to be installed (no-op when it@@ -60,6 +67,8 @@ if [ -z "$PYTHON_BIN" ]; thenfifi+# The floor under every per-file budget, and the whole budget for any file the+# timing snapshot does not know about.FILE_TIMEOUT="${CI_TEST_FILE_TIMEOUT_SECONDS:-300}"case "$FILE_TIMEOUT" in''|*[!0-9]*)@@ -87,26 +96,48 @@ metrics = FileMetrics(sys.argv[1], started_at)exit_code = pytest.main(sys.argv[1:], plugins=[metrics])metrics.emit(diagnostics, int(exit_code))sys.exit(exit_code)-' "$FILE_TIMEOUT")+')+# Emits "<path>\t<budget seconds>\t<recorded seconds, blank if unknown>".select_unit_test_files() {- "$PYTHON_BIN" scripts/ci_unit_test_shards.py "$SHARD_INDEX" "$SHARD_TOTAL"+ "$PYTHON_BIN" scripts/ci_unit_test_shards.py \+ --budget-floor="$FILE_TIMEOUT" "$SHARD_INDEX" "$SHARD_TOTAL"}failed=""empty=""selected=0discovered=0-while IFS= read -r f; do+while IFS=$'\t' read -r f budget baseline; do+ [ -n "$f" ] || continue+ case "$budget" in ''|*[!0-9]*) budget="$FILE_TIMEOUT" ;; esacdiscovered=$((discovered + 1))selected=$((selected + 1))started_at=$(date +%s)- echo "Starting $f (timeout ${FILE_TIMEOUT}s)."- "${PYTEST[@]}" "$f" -m "not integration" -p no:randomly -p no:faulthandler -o addopts="" -v+ if [ -n "$baseline" ]; then+ echo "Starting $f (timeout ${budget}s, budgeted from its recorded ${baseline}s)."+ else+ echo "Starting $f (timeout ${budget}s)."+ fi+ "${PYTEST[@]}" "$budget" "$f" -m "not integration" -p no:randomly -p no:faulthandler -o addopts="" -vrc=$?finished_at=$(date +%s)elapsed=$((finished_at - started_at))echo "Finished $f in ${elapsed}s with exit code $rc."+ if [ "$rc" -ne 0 ] && [ "$elapsed" -ge "$budget" ]; then+ # Say what the dump above is and is not. faulthandler prints whichever frame+ # the main thread occupied when the budget expired; in a file whose cost is+ # spread over many tests that is usually an ordinary cheap frame, and reading+ # it as the stall site sends the next reader after an innocent test.+ echo " ^ $f hit its ${budget}s watchdog."+ if [ -n "$baseline" ]; then+ echo " That budget is a multiple of this file's own recorded ${baseline}s, so being the slowest file is not enough to reach it."+ else+ echo " No recorded duration for this file, so it ran on the ${FILE_TIMEOUT}s floor; refresh scripts/ci_unit_test_timings.json to give it a budget of its own."+ fi+ echo " The traceback above is where the main thread stood when the budget expired, not necessarily where it stalled."+ echo " Before blaming the named test, compare the 'Finished ... in Ns' lines in this shard against scripts/ci_unit_test_timings.json: if they are slow across the board, the runner was slow, not this file."+ fiif [ "$rc" -eq 0 ]; then:elif [ "$rc" -eq 5 ]; thendiff --git a/tests/test_ci_pipeline.py b/tests/test_ci_pipeline.pyindex 310b00ba8..ab0fcaba1 100644--- a/tests/test_ci_pipeline.py+++ b/tests/test_ci_pipeline.py@@ -124,13 +124,23 @@ def test_ci_tool_install_command_propagates_failure(tmp_path, exit_code):assert "--only-binary=:all:" in result.stdout-def _run_isolated_unit_files(tmp_path: Path, sources: dict[str, str], *, timeout: str = "15"):+def _run_isolated_unit_files(+ tmp_path: Path,+ sources: dict[str, str],+ *,+ timeout: str = "15",+ timings: dict[str, float] | None = None,+):(tmp_path / "tests").mkdir()(tmp_path / "scripts").mkdir()for name, source in sources.items():(tmp_path / "tests" / name).write_text(source, encoding="utf-8")shutil.copyfile(ROOT / "scripts/ci_unit_test_shards.py", tmp_path / "scripts/ci_unit_test_shards.py")shutil.copyfile(ROOT / "scripts/ci_pytest_metrics.py", tmp_path / "scripts/ci_pytest_metrics.py")+ if timings is not None:+ (tmp_path / "scripts" / "ci_unit_test_timings.json").write_text(+ json.dumps({"durations_seconds": timings}), encoding="utf-8",+ )return subprocess.run(["bash", str(ROOT / "scripts/ci_unit_tests.sh")], cwd=tmp_path,env={**os.environ, "PYTHON": sys.executable, "CI_TEST_FILE_TIMEOUT_SECONDS": timeout,@@ -170,6 +180,13 @@ def test_unit_file_watchdog_dumps_stacks_and_continues_fail_closed(tmp_path: Patassert "test_after_stuck_file PASSED" in result.stdoutassert "FAILED files:\n tests/test_a_stuck.py" in result.stdoutassert "All unit test files passed." not in result.stdout+ # The dump names whichever frame the main thread held at the deadline, which+ # in a file whose cost is spread across many tests is an ordinary cheap one.+ # Saying so is the difference between reading the trace as evidence and+ # reading it as a verdict on the test it happens to name.+ assert "^ tests/test_a_stuck.py hit its 3s watchdog." in result.stdout+ assert "No recorded duration for this file, so it ran on the 3s floor" in result.stdout+ assert "not necessarily where it stalled" in result.stdoutrecords = _metrics(result)assert records[-1]["file"] == "tests/test_b_pass.py"if phase == "shutdown":@@ -179,6 +196,35 @@ def test_unit_file_watchdog_dumps_stacks_and_continues_fail_closed(tmp_path: Patassert len(records) == 1+def test_unit_file_watchdog_budget_follows_the_file_own_recorded_cost(tmp_path: Path) -> None:+ """A file the snapshot knows is slow gets headroom measured against itself.++ The fixed budget this replaces could not tell "hung" from "slow": it was+ generous for a 1s file and thin for the suite's slowest, so a uniformly+ slow runner killed the slowest file first and named it as the culprit.+ """+ result = _run_isolated_unit_files(+ tmp_path,+ {+ "test_a_stuck.py": "import time\ndef test_stuck():\n time.sleep(3600)\n",+ "test_b_pass.py": "def test_after_stuck_file():\n pass\n",+ },+ timeout="4",+ timings={"tests/test_a_stuck.py": 1.0},+ )++ assert result.returncode == 1, result.stdout + result.stderr+ # 8x its own second, not the 4s floor the same run gives an unknown file.+ assert "Starting tests/test_a_stuck.py (timeout 8s, budgeted from its recorded 1s)." in result.stdout+ assert "Timeout (0:00:08)!" in result.stderr+ assert "^ tests/test_a_stuck.py hit its 8s watchdog." in result.stdout+ assert "That budget is a multiple of this file's own recorded 1s" in result.stdout+ # An unknown file keeps the floor, and the line it prints is unchanged.+ assert "Starting tests/test_b_pass.py (timeout 4s)." in result.stdout+ assert "test_after_stuck_file PASSED" in result.stdout+ assert "FAILED files:\n tests/test_a_stuck.py" in result.stdout++@pytest.mark.parametrize("failed", [False, True])def test_unit_file_runner_preserves_assertions_and_integration_selection(tmp_path: Path, failed: bool) -> None:result = _run_isolated_unit_files(tmp_path, {diff --git a/tests/test_ci_unit_test_shards.py b/tests/test_ci_unit_test_shards.pyindex a456b616b..4aaa9f9d8 100644--- a/tests/test_ci_unit_test_shards.py+++ b/tests/test_ci_unit_test_shards.py@@ -4,7 +4,16 @@ from pathlib import Pathimport pytest-from scripts.ci_unit_test_shards import discover_unit_test_files, load_timings, plan_shards+import json++from scripts.ci_unit_test_shards import (+ DEFAULT_WATCHDOG_MULTIPLIER,+ discover_unit_test_files,+ load_timings,+ main,+ plan_shards,+ watchdog_budget_seconds,+)def _test_file(root: Path, name: str, *, tests: int = 1, lines: int = 1) -> Path:@@ -61,3 +70,67 @@ def test_measured_plan_runs_every_discovered_file_exactly_once() -> None:shards = plan_shards(files, 6, load_timings())assert sorted(path for _, selected in shards for path in selected) == files+++def test_watchdog_budget_scales_with_the_file_own_recorded_cost(tmp_path: Path) -> None:+ slow = _test_file(tmp_path, "test_slow.py")+ quick = _test_file(tmp_path, "test_quick.py")+ timings = {slow.as_posix(): 74.0, quick.as_posix(): 2.0}++ slow_budget, slow_baseline = watchdog_budget_seconds(slow, timings, floor_seconds=300)+ quick_budget, quick_baseline = watchdog_budget_seconds(quick, timings, floor_seconds=300)++ # The slow file is the one a fixed budget squeezes: 74s against 300s leaves+ # it barely 4x of headroom, which a merely slow runner can eat. Its budget+ # is a multiple of its own cost, so slowness alone no longer reads as a hang.+ assert slow_budget == int(74 * DEFAULT_WATCHDOG_MULTIPLIER)+ assert slow_baseline == 74.0+ # A quick file would be *cut* by the same multiple, so the floor holds.+ assert quick_budget == 300+ assert quick_baseline == 2.0+++def test_watchdog_budget_keeps_the_floor_for_files_with_no_recorded_cost(tmp_path: Path) -> None:+ unknown = _test_file(tmp_path, "test_unknown.py", tests=200, lines=5000)++ budget, baseline = watchdog_budget_seconds(unknown, {}, floor_seconds=300)++ # The planner's structural estimate is a relative weight, not seconds.+ # Multiplying it would hand an arbitrary deadline to exactly the files+ # nothing is known about, however large the estimate grows.+ assert budget == 300+ assert baseline is None+++def test_main_emits_budgets_only_when_a_floor_is_requested(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ capsys: pytest.CaptureFixture[str],+) -> None:+ (tmp_path / "tests").mkdir()+ measured = _test_file(tmp_path / "tests", "test_measured.py")+ _test_file(tmp_path / "tests", "test_new.py")+ (tmp_path / "scripts").mkdir()+ (tmp_path / "scripts" / "ci_unit_test_timings.json").write_text(+ json.dumps({"durations_seconds": {"tests/test_measured.py": 50.0}}),+ encoding="utf-8",+ )+ monkeypatch.chdir(tmp_path)+ assert measured.is_file()++ assert main(["0", "1"]) == 0+ plain = capsys.readouterr().out.splitlines()+ # Callers that only want a file list keep reading one, unchanged.+ assert plain == ["tests/test_measured.py", "tests/test_new.py"]++ assert main(["--budget-floor=10", "0", "1"]) == 0+ budgeted = [line.split("\t") for line in capsys.readouterr().out.splitlines()]+ assert budgeted == [+ ["tests/test_measured.py", str(int(50 * DEFAULT_WATCHDOG_MULTIPLIER)), "50"],+ # Blank baseline is how the launcher knows to print the legacy line.+ ["tests/test_new.py", "10", ""],+ ]+++def test_main_rejects_a_budget_floor_that_is_not_a_positive_number_of_seconds() -> None:+ assert main(["--budget-floor=0", "0", "1"]) == 2