Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(tier3): give the Windows process-tree kill longer than a plain reap (#687)
Christopher Kevin7f9d2c55f579
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 84b97103..0903f4f4 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -413,6 +413,10 @@ All notable changes to SkillEvaluator are documented in this file.- The Tier 1 PII scan no longer stalls on a long identifier before an IPv4-shaped value. A line with a30-letter name followed by `= "8.8.0.8"` took about a minute to scan, and every further letter doubled that.+- On Windows, stopping a Tier 3 Harbor run no longer fails with `Harbor process-tree cleanup could not be+ confirmed` when `taskkill` needs more than 5 seconds. It now gets up to 30 seconds, so the tree is stopped+ and the run's own timeout or output-limit error is reported. A second Ctrl+C still ends that wait early,+ and the cleanup is then reported as unconfirmed.- Keep Steps per Eval Case chart labels readable for long case IDs while preserving full IDs in hover tooltips.- Negative controls with complete, confirmed skill non-use report wrong answers as task outcomes instead of recommending skill edits; truncated or recovered attempt evidence cannot establish complete non-use.- Repeated negative-case task outcomes share one advice item, retain task-failure wording and avoid describing passing attempts as lost, keeping distinct skill fixes visible.diff --git a/src/skillevaluator/tier3/harbor/runner.py b/src/skillevaluator/tier3/harbor/runner.pyindex f19a91e4..38b72c2e 100644--- a/src/skillevaluator/tier3/harbor/runner.py+++ b/src/skillevaluator/tier3/harbor/runner.py@@ -869,6 +869,12 @@ _HARBOR_RUN_STOPPING_DETAIL = (f"2nd Ctrl+C skips wait; stopping Harbor jobs, up to {_HARBOR_RUN_CANCEL_GRACE_SECONDS:g}s of environment cleanup")_HARBOR_RUN_REAP_SECONDS = 5.0+# ``taskkill /T /F`` has to look the task tree up before it kills anything, and+# that lookup can take longer than reaping a process that was already signalled.+# Cutting it short leaves the tree running and replaces the run's own error with+# a cleanup error, so the tree kill gets its own, longer bound. A second Ctrl+C+# (``_HarborRunCancellation.forced``) still ends the wait early.+_HARBOR_WINDOWS_TREE_KILL_SECONDS = 30.0@dataclass(frozen=True)@@ -3800,6 +3806,34 @@ def _wait_for_harbor_leader(return+def _wait_for_windows_taskkill(+ taskkill: subprocess.Popen[bytes],+ timeout_seconds: float,+ *,+ end_grace: threading.Event | None,+) -> None:+ """Wait up to ``timeout_seconds`` for ``taskkill`` to exit, or until ``end_grace`` is set.++ The wait is cut into short slices so a set ``end_grace`` (a second Ctrl+C) is+ noticed within ``_HARBOR_RUN_CANCEL_POLL_SECONDS`` instead of after the whole+ budget. A ``taskkill`` that is still running when the budget runs out, or+ when ``end_grace`` is set, raises :class:`subprocess.TimeoutExpired` so the+ caller stops it and fails closed rather than treating the tree as stopped.+ """+ started = time.monotonic()+ deadline = started + timeout_seconds+ while True:+ slice_seconds = min(max(deadline - time.monotonic(), 0.0), _HARBOR_RUN_CANCEL_POLL_SECONDS)+ try:+ taskkill.wait(timeout=slice_seconds)+ except subprocess.TimeoutExpired as exc:+ now = time.monotonic()+ if now >= deadline or (end_grace is not None and end_grace.is_set()):+ raise subprocess.TimeoutExpired(exc.cmd, now - started) from None+ continue+ return++def _terminate_harbor_process_tree(process: subprocess.Popen[bytes],*,@@ -3811,7 +3845,10 @@ def _terminate_harbor_process_tree(On POSIX the group receives SIGTERM, then SIGKILL once the leader exits or``grace_seconds`` pass, whichever is first; a set ``end_grace`` skips therest of the grace period. Windows has no SIGTERM that Harbor handles, so- its task tree is always stopped with ``taskkill /T /F``.+ its task tree is always stopped with ``taskkill /T /F`` and ``grace_seconds``+ does not apply. ``taskkill`` gets up to ``_HARBOR_WINDOWS_TREE_KILL_SECONDS``;+ a set ``end_grace`` stops that wait early, kills ``taskkill`` and fails+ closed, because the tree was then not confirmed stopped."""if os.name == "posix":# POSIX process groups do not own a descendant that deliberately calls@@ -3838,7 +3875,7 @@ def _terminate_harbor_process_tree(stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,)- taskkill.wait(timeout=_HARBOR_RUN_REAP_SECONDS)+ _wait_for_windows_taskkill(taskkill, _HARBOR_WINDOWS_TREE_KILL_SECONDS, end_grace=end_grace)if taskkill.returncode != 0:taskkill_error = RuntimeError("Windows Harbor process-tree cleanup failed")except subprocess.TimeoutExpired as exc:diff --git a/tests/test_harbor_runner_status.py b/tests/test_harbor_runner_status.pyindex 1e4427f4..a70d3267 100644--- a/tests/test_harbor_runner_status.py+++ b/tests/test_harbor_runner_status.py@@ -10,9 +10,11 @@ import importlibimport jsonimport osimport signal+import subprocessimport sysimport threadingimport time+from collections.abc import Callablefrom pathlib import Pathimport pytest@@ -804,6 +806,208 @@ def test_windows_tree_cleanup_fails_closed_when_taskkill_misses_an_exited_root(assert root_process.killed is False+class _SimulatedClock:+ """Stands in for the ``time`` module so a simulated ``taskkill`` can take "seconds" without sleeping."""++ def __init__(self) -> None:+ self.now = 1000.0++ def monotonic(self) -> float:+ return self.now++ def __getattr__(self, name: str) -> object:+ return getattr(time, name)+++class _SimulatedWindowsProcess:+ """A process that exits ``exits_after`` simulated seconds after it is created, and at once when killed."""++ def __init__(+ self,+ clock: _SimulatedClock,+ *,+ pid: int,+ exits_after: float,+ on_wait: Callable[[int], None] | None = None,+ ) -> None:+ self.clock = clock+ self.pid = pid+ self.returncode: int | None = None+ self.exits_at = clock.now + exits_after+ self.killed = False+ self.on_wait = on_wait+ self.wait_timeouts: list[float | None] = []++ def poll(self) -> int | None:+ if self.returncode is None and self.clock.now >= self.exits_at:+ self.returncode = -9 if self.killed else 0+ return self.returncode++ def kill(self) -> None:+ self.killed = True+ self.exits_at = self.clock.now++ def wait(self, timeout: float | None = None) -> int:+ assert timeout is not None, "a Windows tree cleanup wait must be bounded"+ self.wait_timeouts.append(timeout)+ if self.on_wait is not None:+ self.on_wait(len(self.wait_timeouts))+ if self.poll() is None:+ if self.clock.now + timeout < self.exits_at:+ self.clock.now += timeout+ raise subprocess.TimeoutExpired(cmd="fake", timeout=timeout)+ self.clock.now = max(self.clock.now, self.exits_at)+ self.returncode = -9 if self.killed else 0+ return int(self.returncode or 0)+++def _simulate_windows_tree_cleanup(+ monkeypatch: pytest.MonkeyPatch,+ *,+ taskkill_exits_after: float,+ on_taskkill_wait: Callable[[int], None] | None = None,+) -> tuple[_SimulatedClock, _SimulatedWindowsProcess, _SimulatedWindowsProcess]:+ class WindowsOs:+ name = "nt"++ clock = _SimulatedClock()+ taskkill_process = _SimulatedWindowsProcess(+ clock, pid=99, exits_after=taskkill_exits_after, on_wait=on_taskkill_wait+ )+ root_process = _SimulatedWindowsProcess(clock, pid=4242, exits_after=0.0)+ monkeypatch.setattr(runner, "os", WindowsOs())+ monkeypatch.setattr(runner, "time", clock)+ monkeypatch.setattr(runner, "_verified_windows_taskkill_path", lambda: Path("/trusted/taskkill.exe"))+ monkeypatch.setattr(runner.subprocess, "Popen", lambda *_args, **_kwargs: taskkill_process)+ return clock, taskkill_process, root_process++[email protected]("taskkill_finishes", [True, False])+def test_windows_tree_cleanup_gives_taskkill_longer_than_a_plain_reap(+ monkeypatch: pytest.MonkeyPatch,+ taskkill_finishes: bool,+) -> None:+ """taskkill looks the task tree up before it kills it, which can outlast the budget for reaping an+ already-signalled process. Cut short, it leaves the tree running and hides the run's own error."""+ reap_seconds = runner._HARBOR_RUN_REAP_SECONDS+ poll_seconds = runner._HARBOR_RUN_CANCEL_POLL_SECONDS+ budget_seconds = runner._HARBOR_WINDOWS_TREE_KILL_SECONDS+ clock, taskkill_process, root_process = _simulate_windows_tree_cleanup(+ monkeypatch,+ taskkill_exits_after=2 * reap_seconds if taskkill_finishes else float("inf"),+ )+ started = clock.now++ if taskkill_finishes:+ runner._terminate_harbor_process_tree(root_process) # type: ignore[arg-type]++ assert taskkill_process.killed is False+ # It waited past the plain reap budget, in short slices rather than one blocking call.+ assert clock.now - started == pytest.approx(2 * reap_seconds)+ assert len(taskkill_process.wait_timeouts) > 1+ assert max(taskkill_process.wait_timeouts) <= poll_seconds+ else:+ # A taskkill that never returns is still stopped, and cleanup still fails closed.+ with pytest.raises(RuntimeError, match="process-tree cleanup could not be confirmed"):+ runner._terminate_harbor_process_tree(root_process) # type: ignore[arg-type]++ assert taskkill_process.killed is True+ # It waited out the whole tree-kill budget before giving up, then reaped on the short budget.+ assert clock.now - started == pytest.approx(budget_seconds, abs=poll_seconds)+ assert taskkill_process.wait_timeouts[-1] == reap_seconds+ assert max(taskkill_process.wait_timeouts[:-1]) <= poll_seconds+ # The root is reaped either way, on the short budget.+ assert root_process.wait_timeouts == [reap_seconds]+++def test_windows_tree_cleanup_waits_out_a_slow_taskkill_when_cancellation_is_not_forced(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A first Ctrl+C alone must not shorten the wait: the cancel path passes ``end_grace``, but it is unset."""+ budget_seconds = runner._HARBOR_WINDOWS_TREE_KILL_SECONDS+ slow_seconds = budget_seconds - 5.0+ clock, taskkill_process, root_process = _simulate_windows_tree_cleanup(+ monkeypatch, taskkill_exits_after=slow_seconds+ )+ started = clock.now+ cancellation = runner._HarborRunCancellation()+ cancellation.requested.set()++ runner._terminate_harbor_process_tree(+ root_process, # type: ignore[arg-type]+ grace_seconds=runner._HARBOR_RUN_CANCEL_GRACE_SECONDS,+ end_grace=cancellation.forced,+ )++ assert taskkill_process.killed is False+ assert clock.now - started == pytest.approx(slow_seconds)+ assert root_process.wait_timeouts == [runner._HARBOR_RUN_REAP_SECONDS]++[email protected]("forced_when", ["before-cleanup", "while-taskkill-runs"])+def test_windows_tree_cleanup_stops_waiting_for_taskkill_once_cancellation_is_forced(+ monkeypatch: pytest.MonkeyPatch,+ forced_when: str,+) -> None:+ """A second Ctrl+C says "skip wait". The 30 s taskkill budget must not outlast it, and a taskkill that was+ cut off has not confirmed the tree is gone, so cleanup still fails closed."""+ cancellation = runner._HarborRunCancellation()+ cancellation.requested.set()+ slices_before_forced = 3++ def second_ctrl_c(wait_count: int) -> None:+ if forced_when == "while-taskkill-runs" and wait_count == slices_before_forced:+ cancellation.forced.set()++ clock, taskkill_process, root_process = _simulate_windows_tree_cleanup(+ monkeypatch,+ taskkill_exits_after=float("inf"),+ on_taskkill_wait=second_ctrl_c,+ )+ if forced_when == "before-cleanup":+ cancellation.forced.set()+ started = clock.now+ poll_seconds = runner._HARBOR_RUN_CANCEL_POLL_SECONDS++ with pytest.raises(RuntimeError, match="process-tree cleanup could not be confirmed") as exc_info:+ runner._terminate_harbor_process_tree(+ root_process, # type: ignore[arg-type]+ grace_seconds=runner._HARBOR_RUN_CANCEL_GRACE_SECONDS,+ end_grace=cancellation.forced,+ )++ expected_slices = 1 if forced_when == "before-cleanup" else slices_before_forced+ # Noticed within a poll slice of the second Ctrl+C, nowhere near the 30 s budget.+ assert clock.now - started == pytest.approx(expected_slices * poll_seconds)+ assert clock.now - started < runner._HARBOR_WINDOWS_TREE_KILL_SECONDS / 10+ assert isinstance(exc_info.value.__cause__, subprocess.TimeoutExpired)+ # taskkill is stopped and reaped on the short budget, and so is the root.+ assert taskkill_process.killed is True+ assert taskkill_process.wait_timeouts[-1] == runner._HARBOR_RUN_REAP_SECONDS+ assert root_process.wait_timeouts == [runner._HARBOR_RUN_REAP_SECONDS]+++def test_windows_tree_cleanup_keeps_a_taskkill_that_finishes_despite_forced_cancellation(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Forcing only stops waiting on a taskkill that is still running; one that finishes is a verified cleanup."""+ _clock, taskkill_process, root_process = _simulate_windows_tree_cleanup(+ monkeypatch, taskkill_exits_after=runner._HARBOR_RUN_CANCEL_POLL_SECONDS / 2+ )+ forced = threading.Event()+ forced.set()++ runner._terminate_harbor_process_tree(+ root_process, # type: ignore[arg-type]+ grace_seconds=runner._HARBOR_RUN_CANCEL_GRACE_SECONDS,+ end_grace=forced,+ )++ assert taskkill_process.killed is False+ assert taskkill_process.returncode == 0+ assert root_process.wait_timeouts == [runner._HARBOR_RUN_REAP_SECONDS]++@pytest.mark.parametrize("candidate_kind", ["missing", "directory"])def test_windows_tree_cleanup_rejects_unverified_system32_taskkill(monkeypatch: pytest.MonkeyPatch,