Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(ci): stop the Tier 1 runner missing a Ctrl-C, and deflake timing-dependent tests (#568)
Christopher Kevin59b31f6546fe
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex f1d3d789..614fe73a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -327,6 +327,11 @@ All notable changes to SkillEvaluator are documented in this file.existing advice.- The comments-only consumer template now grants its required read and PR-commentpermissions only to the comment job, with no workflow-wide token permissions.+- Ctrl-C during a reusable Tier 1 workflow run is no longer missed when it+ arrives while a scan is starting. The runner waited for scan results with no+ time limit, and on Linux the interrupt could be handled on a scan thread,+ which did not wake that wait; the run then went on until a scan ended. The+ runner now checks for the interrupt every 0.1 seconds.- The behavior evidence gate no longer keeps a score for an expectation thatnames an observed artifact but also asks for something only the final answercould show. "Writes `report.py` and explains why each column was dropped" wasdiff --git a/ci/run_tier1.py b/ci/run_tier1.pyindex 9fff145c..45dc916e 100644--- a/ci/run_tier1.py+++ b/ci/run_tier1.py@@ -777,7 +777,12 @@ def run(live.append(thread)running += 1while running:- batch = [events.get()]+ try:+ batch = [events.get(timeout=POLL_SECONDS)]+ except queue.Empty:+ # Ctrl-C is acted on in this thread, but its signal handler can run on a+ # scan thread, and that does not interrupt a wait without a time limit.+ continuewhile True:try:batch.append(events.get_nowait())diff --git a/tests/ci/test_run_tier1.py b/tests/ci/test_run_tier1.pyindex 22666673..1cfd567f 100644--- a/tests/ci/test_run_tier1.py+++ b/tests/ci/test_run_tier1.py@@ -1546,8 +1546,14 @@ def test_scan_timeout_also_ends_the_scanner_tools(tmp_path, monkeypatch):path = scan_shim(tmp_path)output = tmp_path / "out"output.mkdir()- clock = types.SimpleNamespace(offset=0.0)- clock.monotonic = lambda: time.monotonic() + clock.offset+ clock = types.SimpleNamespace(offset=0.0, read=threading.Event())++ def monotonic():+ now = time.monotonic() + clock.offset+ clock.read.set()+ return now++ clock.monotonic = monotonicmonkeypatch.setattr(runner, "time", clock)outcome = {}@@ -1568,6 +1574,10 @@ def test_scan_timeout_also_ends_the_scanner_tools(tmp_path, monkeypatch):thread.start()pids = survivors = scan_pids(output, time.monotonic() + WAIT)try:+ # The scan sets its time limit after it starts the evaluator, which can write its+ # PIDs first. Move the clock only once the scan has read it, or the limit moves too.+ clock.read.clear()+ assert clock.read.wait(WAIT), "the scan did not read the clock"clock.offset = WAITthread.join(WAIT)assert not thread.is_alive() and "error" in outcome@@ -1719,6 +1729,37 @@ def test_second_ctrl_c_does_not_leave_the_runner_waiting(tmp_path):kill_quietly(survivors)[email protected](not hasattr(signal, "pthread_kill"), reason="sends a signal to one thread")+def test_ctrl_c_handled_on_a_scan_thread_still_ends_the_run(tmp_path, monkeypatch):+ # Python acts on Ctrl-C in the main thread, but the system can run the signal handler on+ # another one. Linux hands a waiting SIGINT to a thread that unblocks signals, as a scan+ # thread does when it starts an evaluator. That does not wake a main thread that waits+ # for scan events without a time limit, so the run would go on until a scan ended.+ workspace = tmp_path / "workspace"+ skill(workspace, "skills/a")+ waits, aborted = threading.Semaphore(0), []++ class Events(runner.queue.SimpleQueue):+ def get(self, *args, **kwargs):+ waits.release()+ return super().get(*args, **kwargs)++ def run_scan(argv, *, abort, **options):+ # A busy main thread acts on any signal, and this scan's start event ends its first+ # wait when that wait has no time limit. Signal once it has started a second wait.+ for _ in range(2):+ waits.acquire(timeout=WAIT)+ signal.pthread_kill(threading.get_ident(), signal.SIGINT)+ aborted.append(abort.wait(WAIT))+ return CompletedProcess(argv, 1)++ monkeypatch.setattr(runner, "queue", types.SimpleNamespace(SimpleQueue=Events, Empty=runner.queue.Empty))+ monkeypatch.setattr(runner, "run_scan", run_scan)+ with pytest.raises(KeyboardInterrupt):+ runner.run(workspace, tmp_path / "reports", {})+ assert aborted == [True]++def test_each_report_is_released_once_its_result_is_logged(tmp_path, monkeypatch):# Reports can be 32 MiB each; 100 of them must not stay in memory.workspace = tmp_path / "workspace"diff --git a/tests/test_harbor_local_mode.py b/tests/test_harbor_local_mode.pyindex 98e92f42..b9f9b933 100644--- a/tests/test_harbor_local_mode.py+++ b/tests/test_harbor_local_mode.py@@ -13,6 +13,7 @@ import jsonimport loggingimport osimport re+import selectorsimport shleximport signalimport socket@@ -161,6 +162,57 @@ def _initialized_local_environment()+def _assert_process_exits(pid: int, *, timeout: float = 5.0) -> None:+ """Assert that a process signalled by cleanup leaves the process table.++ Cleanup returns once its group probe finds no live member. On macOS that+ probe reports EPERM while a SIGKILLed descendant is still finishing its+ exit, and until then ``kill(pid, 0)`` still succeeds for that PID. A single+ immediate probe races that teardown, so allow bounded exit completion.++ This polls without yielding to the event loop, so cleanup cannot run in the+ meantime: a descendant it never signalled is still alive at the deadline.+ """+ deadline = time.monotonic() + timeout+ while True:+ try:+ os.kill(pid, 0)+ except ProcessLookupError:+ return+ assert time.monotonic() < deadline, "a descendant survived process-group cleanup"+ time.sleep(0.01)+++class _ManualClockSelector(selectors.DefaultSelector):+ """Advance a manual clock by each requested wait instead of sleeping."""++ def __init__(self) -> None:+ super().__init__()+ self.now = 0.0++ def select(self, timeout: float | None = None) -> list[tuple[selectors.SelectorKey, int]]:+ if timeout is None:+ return super().select(None)+ self.now += max(timeout, 0.0)+ return super().select(0)+++class _ManualClockEventLoop(asyncio.SelectorEventLoop):+ """Event loop whose timers elapse on a manual clock instead of the host's.++ Time advances only by the delay the loop asked to wait for, so a bound on+ timer-driven code counts the timeouts that code requested and not how long+ a loaded host descheduled the loop.+ """++ def __init__(self) -> None:+ self._manual_clock = _ManualClockSelector()+ super().__init__(self._manual_clock)++ def time(self) -> float:+ return self._manual_clock.now++def _provider(name: str, *, api_key: str = "k", base_url: str | None = None) -> ProviderConfig:return ProviderConfig(provider=name, model="m", api_key=api_key, base_url=base_url, litellm_model="m", region=None)@@ -282,8 +334,7 @@ def test_local_output_limit_fails_closed_and_reaps_process_group(assert processes[0].returncode is not Noneassert child_pid_path.is_file()child_pid = int(child_pid_path.read_text(encoding="ascii"))- with pytest.raises(ProcessLookupError):- os.kill(child_pid, 0)+ _assert_process_exits(child_pid)assert len("".join(callback_chunks).encode()) <= 8assert overflow_value not in "".join(callback_chunks)assert overflow_value not in str(caught.value)@@ -2840,8 +2891,7 @@ def test_streamed_timeout_callback_matches_result_and_contains_descendants(tmp_p)assert secret not in callback_stdoutchild_pid = int(child_pid_path.read_text(encoding="ascii"))- with pytest.raises(ProcessLookupError):- os.kill(child_pid, 0)+ _assert_process_exits(child_pid)@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)@@ -3184,7 +3234,9 @@ def test_exec_cancellation_terminates_background_descendants(tmp_path: Path) -># An interrupted sleep must not itself count as a surviving child."(printf ready > cancel-child-ready; sleep 30 && ""printf survived > cancel-child-survived) & "- "printf '%s' \"$!\" > cancel-child-pid; wait"+ # Existence signals readiness, so publish only the complete PID.+ "printf '%s' \"$!\" > cancel-child-pid.tmp && "+ "mv cancel-child-pid.tmp cancel-child-pid; wait")environment._local_command_guardrail_reason = lambda *_args: "" # type: ignore[method-assign]@@ -3275,8 +3327,7 @@ def test_streamed_exec_cancellation_reaps_descendants(assert callback_chunksassert secret not in "".join(callback_chunks)- with pytest.raises(ProcessLookupError):- os.kill(child_pid, 0)+ _assert_process_exits(child_pid)@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)@@ -3308,8 +3359,8 @@ def test_streamed_exec_cancellation_during_final_flush_reaps_descendants(assert child_pid_path.exists()assert not environment._active_processeschild_pid = int(child_pid_path.read_text(encoding="ascii"))- with pytest.raises(ProcessLookupError):- os.kill(child_pid, 0)+ # The final callback is still blocked, so containment came first.+ _assert_process_exits(child_pid)task.cancel()if repeat_cancellation:await asyncio.sleep(0)@@ -3321,8 +3372,7 @@ def test_streamed_exec_cancellation_during_final_flush_reaps_descendants(child_pid: int | None = Nonetry:child_pid = asyncio.run(exercise())- with pytest.raises(ProcessLookupError):- os.kill(child_pid, 0)+ _assert_process_exits(child_pid)finally:if child_pid is None and child_pid_path.exists():child_pid = int(child_pid_path.read_text(encoding="ascii"))@@ -3340,17 +3390,27 @@ def test_cancellation_during_timeout_reap_preserves_cancellation_and_containmentenvironment = _initialized_local_environment(tmp_path)environment._local_command_guardrail_reason = lambda *_args: "" # type: ignore[method-assign]child_pid_path = environment._workspace / "timeout-reap-cancel-child-pid"+ child_pid_tmp_path = child_pid_path.with_name(f"{child_pid_path.name}.tmp")reap_entered = asyncio.Event()original_terminate = environment._terminate_process_treecommand = ("(trap '' TERM; exec >/dev/null 2>&1; sleep 30) & "- f"printf '%s' \"$!\" > {shlex.quote(child_pid_path.name)}; wait"+ # Existence signals readiness, so publish only the complete PID.+ f"printf '%s' \"$!\" > {shlex.quote(child_pid_tmp_path.name)} && "+ f"mv {shlex.quote(child_pid_tmp_path.name)} {shlex.quote(child_pid_path.name)}; wait")async def observed_terminate(proc: asyncio.subprocess.Process,communication: asyncio.Task[tuple[bytes, bytes]] | None = None,) -> tuple[bytes, bytes]:+ # The exec timeout can expire before a loaded host has started the+ # descendant. Hold the reap until its PID is published, so that the+ # cancellation always interrupts the reap of a live descendant.+ for _ in range(400):+ if child_pid_path.exists():+ break+ await asyncio.sleep(0.01)reap_entered.set()return await original_terminate(proc, communication)@@ -3383,8 +3443,7 @@ def test_cancellation_during_timeout_reap_preserves_cancellation_and_containmentoutcome, child_pid, retained = asyncio.run(exercise())assert isinstance(outcome, asyncio.CancelledError)assert not retained- with pytest.raises(ProcessLookupError):- os.kill(child_pid, 0)+ _assert_process_exits(child_pid)finally:if child_pid is None and child_pid_path.exists():child_pid = int(child_pid_path.read_text(encoding="ascii"))@@ -3664,7 +3723,9 @@ def test_process_tree_cleanup_stays_bounded_when_communication_ignores_cancellatawait cleanupreturn finished_within_bound, communication.cancelled()- assert asyncio.run(run_cleanup()) == (True, True)+ # On the manual clock the 0.1s bound counts only the waits that cleanup+ # asked for, not the time a loaded host keeps the event loop descheduled.+ assert asyncio.run(run_cleanup(), loop_factory=_ManualClockEventLoop) == (True, True)def test_stop_reaps_all_tracked_processes(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:diff --git a/tests/test_harbor_secure_docker_environment.py b/tests/test_harbor_secure_docker_environment.pyindex 9c6afc2b..6b42f827 100644--- a/tests/test_harbor_secure_docker_environment.py+++ b/tests/test_harbor_secure_docker_environment.py@@ -35,6 +35,7 @@ from skillevaluator.tier3.harbor.adapter import generate_harbor_tasksfrom skillevaluator.tier3.harbor.runner import build_harbor_run_commandfrom skillevaluator.tier3.harbor.secure_docker_environment import (_REDACTION_SENTINEL_CANDIDATES,+ _SENSITIVE_ENV_NAME_RE,NVIDIA_BUILD_STDIN_SENTINEL,SECURE_DOCKER_ENV_IMPORT_PATH,SkillEvaluatorDockerEnvironment,@@ -51,6 +52,7 @@ from skillevaluator.tier3.harbor.secure_docker_environment import ()from skillevaluator.tier3.harbor.sensitive_stdin import NVIDIA_BUILD_KEY_STDIN_ENVfrom skillevaluator.tier3.harbor.stream_redaction import CommandOutputLimitError+from skillevaluator.tier3.output_provenance import OUTPUT_PROVENANCE_KEY_ENV_SENTINEL = "sentinel-never-visible-in-argv-or-files"@@ -96,6 +98,17 @@ def _initialized_secure_docker_environment()+def _isolate_host_sensitive_environment(monkeypatch: pytest.MonkeyPatch) -> None:+ """Keep the sidecar Compose check away from sensitive-named variables of the host shell."""+ # The check protects the value of every sensitive-named host variable, whatever its+ # length, so an ambient ``SOME_SESSION=1`` would decide whether a fixture value such as+ # ``127.0.0.1`` is retained. Call this before setting the variables the test controls.+ for name in list(os.environ):+ # conftest points the provenance key at tmp_path; leave that isolation in place.+ if name != OUTPUT_PROVENANCE_KEY_ENV and _SENSITIVE_ENV_NAME_RE.search(name):+ monkeypatch.delenv(name)++class _BufferedComposeProcess:pid = 8841@@ -897,6 +910,22 @@ def test_streaming_redactor_does_not_starve_event_loop_on_repeated_prefix() -> Nassert asyncio.run(exercise()) < 1.0+def _single_secret_scan_seconds(text: str) -> float:+ """Return how long this interpreter takes to redact ``text`` with one secret.++ Coverage tracing slows the pure-Python scan several times over. Nested-secret+ time bounds therefore fall back to a multiple of this linear baseline: tracing+ scales both alike, while a scan that grows with the nested secrets does not.++ Linear matching of the 1,000 nested secrets used below costs about two of these+ scans (the extra is mostly building the trie), so ten leaves room for noise.+ Walking every nested match at each position costs more than twenty.+ """+ started = time.perf_counter()+ _redact(text, {"a" * 8})+ return time.perf_counter() - started++def test_streaming_redactor_nested_terminal_matching_work_is_linear() -> None:secrets = {"a" * length for length in range(8, 1008)}text = "a" * 20_000@@ -905,7 +934,7 @@ def test_streaming_redactor_nested_terminal_matching_work_is_linear() -> None:output = redactor.feed(text) + redactor.finish()elapsed = time.perf_counter() - started- assert elapsed < 1.0+ assert elapsed < 1.0 or elapsed < 10 * _single_secret_scan_seconds(text)assert output == _collision_safe_redaction_marker(secrets)assert redactor.match_work_count <= 10 * len(text)@@ -989,7 +1018,7 @@ def test_compose_stream_nested_secret_union_does_not_starve_event_loop(expected = _collision_safe_redaction_marker({"a" * length for length in range(8, 1008)})assert "".join(callback_chunks) == result.stdout == expected- assert maximum_gap < 1.0+ assert maximum_gap < 1.0 or maximum_gap < 10 * _single_secret_scan_seconds("a" * 20_000)def _reference_union_redaction(text: str, secrets: set[str]) -> str:@@ -2682,6 +2711,7 @@ def test_sidecar_retains_only_structurally_required_nonsecret_compose_task_valuetmp_path: Path,monkeypatch: pytest.MonkeyPatch,) -> None:+ _isolate_host_sensitive_environment(monkeypatch)environment = _initialized_secure_docker_environment(tmp_path)compose_path = environment.environment_dir / "docker-compose.yaml"compose_path.write_text(@@ -2946,6 +2976,7 @@ def test_sidecar_retains_structurally_required_nonsecret_host_compose_value(tmp_path: Path,monkeypatch: pytest.MonkeyPatch,) -> None:+ _isolate_host_sensitive_environment(monkeypatch)environment = _initialized_secure_docker_environment(tmp_path)compose_path = environment.environment_dir / "docker-compose.yaml"compose_path.write_text(diff --git a/tests/test_model_catalog.py b/tests/test_model_catalog.pyindex af4e2eda..0df8a454 100644--- a/tests/test_model_catalog.py+++ b/tests/test_model_catalog.py@@ -213,7 +213,8 @@ def test_anthropic_pagination_fails_when_overall_deadline_expires(monkeypatch) -def test_catalog_body_trickle_cannot_extend_the_wall_clock_deadline() -> None:- payload = json.dumps({"data": [{"id": "slow-model"}]}).encode()+ # 400 bytes, one every 25ms: a client that waits for the whole body needs 10 seconds.+ payload = json.dumps({"data": [{"id": "slow-model"}]}).encode().ljust(400)class SlowBodyHandler(BaseHTTPRequestHandler):def do_GET(self) -> None:@@ -248,10 +249,11 @@ def test_catalog_body_trickle_cannot_extend_the_wall_clock_deadline() -> None:server.shutdown()thread.join(timeout=2)- # The old single response.read() waited for the complete slow body (~0.9s).- # Allow ample scheduler jitter while still proving the configured deadline is- # wall-clock bounded rather than reset by each arriving byte.- assert elapsed < 0.5+ # The old single response.read() waited for the complete slow body (10s here),+ # because every arriving byte reset the socket timeout. The bound stays far from+ # both that and the 0.15s deadline: a stalled CI runner has needed 0.86s, and+ # must not look like a body that extended the deadline.+ assert elapsed < 5.0def test_catalog_dns_resolution_cannot_extend_the_wall_clock_deadline(monkeypatch) -> None:diff --git a/tests/test_nvidia_build_bridge.py b/tests/test_nvidia_build_bridge.pyindex e3e97668..73760eab 100644--- a/tests/test_nvidia_build_bridge.py+++ b/tests/test_nvidia_build_bridge.py@@ -266,6 +266,18 @@ def _local_test_transport(endpoint: str, body: bytes) -> bytes:return response.read()+def _wait_for_bridge_server(started: Event, bridge_thread: Thread) -> None:+ """Wait for ``bridge.serve`` to report its bound server, failing at once if its thread died first.++ Binding, the ready-file fsync and thread scheduling have no latency bound on a+ loaded CI runner, so the deadline only guards against a start that hangs.+ """+ deadline = time.monotonic() + 30+ while not started.wait(timeout=0.05):+ assert bridge_thread.is_alive(), "bridge thread exited before its server was ready"+ assert time.monotonic() < deadline, "bridge server did not start"++def _read_raw_http_response(client: socket.socket) -> str:chunks: list[bytes] = []client.settimeout(2)@@ -436,7 +448,7 @@ def bridge_services(tmp_path: Path) -> _BridgeServices:daemon=True,)bridge_thread.start()- assert started.wait(timeout=1)+ _wait_for_bridge_server(started, bridge_thread)port = int(servers[0].server_port)base_url = f"http://127.0.0.1:{port}"for _ in range(40):@@ -540,7 +552,7 @@ def test_host_bridge_requires_the_per_run_client_token_before_forwarding(tmp_patdaemon=True,)thread.start()- assert started.wait(timeout=1)+ _wait_for_bridge_server(started, thread)origin = f"http://127.0.0.1:{servers[0].server_port}"try:responses_payload = {"model": "nvidia/model", "input": "Hello"}@@ -637,7 +649,7 @@ def test_bridge_rejects_authenticated_requests_after_per_run_budget_is_exhausteddaemon=True,)thread.start()- assert started.wait(timeout=1)+ _wait_for_bridge_server(started, thread)origin = f"http://127.0.0.1:{servers[0].server_port}"headers = {"Authorization": f"Bearer {client_token}"}payload = {"model": "nvidia/model", "input": "Hello"}diff --git a/tests/tier3/test_judge_failure_artifacts.py b/tests/tier3/test_judge_failure_artifacts.pyindex 9ecbf4c6..29832497 100644--- a/tests/tier3/test_judge_failure_artifacts.py+++ b/tests/tier3/test_judge_failure_artifacts.py@@ -12,6 +12,7 @@ import jsonimport loggingimport mathimport os+import selectorsimport shutilimport signalimport subprocess@@ -629,7 +630,20 @@ def test_ragas_cleanup_does_not_wait_for_cancellation_acknowledgment("""Cleanup returns before resistant work finishes, closes the client, and preserves the scorer's outcome."""verifier = _load_verifier(tmp_path)_use_ragas_eligible_judge(monkeypatch)- loop = asyncio.new_event_loop()+ # Run the loop on a virtual clock that jumps to the next timer instead of waiting for it.+ # Timers keep their order, so the rescue below fires only if cleanup really waits for the+ # resistant work, never because a stalled machine took half a second to reach cleanup.+ clock = [0.0]++ class JumpingSelector(selectors.DefaultSelector):+ def select(self, timeout=None):+ if timeout:+ clock[0] += timeout+ timeout = 0+ return super().select(timeout)++ loop = asyncio.SelectorEventLoop(JumpingSelector())+ monkeypatch.setattr(loop, "time", lambda: clock[0])real_close = loop.closerelease = asyncio.Event()rescued: list[bool] = []@@ -848,6 +862,19 @@ def test_ragas_goal_judge_obeys_the_required_judge_time_budget(return real_remaining(timeout)monkeypatch.setattr(verifier, "_remaining_judge_timeout", remaining_after_scorer)+ # Ring the alarm from the loop's own wait, which is where a real timer interrupts an idle loop.+ # A short timer armed by the scorer can expire before the scorer is suspended and raise inside+ # it, and that is a scorer failure, not the cancellation of cut-off work this case checks.+ alarm_due: list[bool] = []++ class AlarmedSelector(selectors.DefaultSelector):+ def select(self, timeout=None):+ if alarm_due:+ alarm_due.clear()+ signal.raise_signal(signal.SIGALRM)+ return super().select(timeout)++ monkeypatch.setattr(asyncio, "new_event_loop", lambda: asyncio.SelectorEventLoop(AlarmedSelector()))class FakeMetric:"""Mock metric object for Ragas async evaluation."""@@ -859,10 +886,10 @@ def test_ragas_goal_judge_obeys_the_required_judge_time_budget(if cutoff == "scorer_timeout":elapsed[0] = 1.0else:- # The real timer represents expiry of the virtual deadline.+ # The alarm represents expiry of the virtual deadline.# Fallback must see that expiry too, after scoring is cancelled.elapsed[0] = 31.0- signal.setitimer(signal.ITIMER_REAL, 0.01)+ alarm_due.append(True)try:await asyncio.sleep(scorer_sleep)except asyncio.CancelledError: