Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
test(tier3): run the Bedrock judge-deadline retry test on a manual clock (#393)
Narendran Raghavan35ef01e65d98
diff --git a/tests/test_retry.py b/tests/test_retry.pyindex 2634eeff..1872c885 100644--- a/tests/test_retry.py+++ b/tests/test_retry.py@@ -5,6 +5,7 @@from __future__ import annotations+import timeimport urllib.errorfrom collections.abc import Callablefrom datetime import UTC, datetime, timedelta@@ -371,6 +372,28 @@ def test_retry_call_with_backoff_invokes_on_retry_callback(monkeypatch: pytest.Massert callback_calls[0][1] >= 0.0+class _ManualClock:+ """Stand-in for the verifier's ``time`` module whose monotonic clock moves only on ``sleep``.++ Real setup cost (importing boto3 and building the first client can take+ seconds on a loaded host) then never eats into a judge deadline under test.+ """++ def __init__(self, sleeps: list[float]) -> None:+ self.now = 0.0+ self.sleeps = sleeps++ def monotonic(self) -> float:+ return self.now++ def sleep(self, seconds: float) -> None:+ self.sleeps.append(seconds)+ self.now += seconds++ def __getattr__(self, name: str) -> Any:+ return getattr(time, name)++class _BedrockTransportHarness:"""Test harness for driving _call_bedrock against botocore's URLLib3Session.send."""@@ -634,7 +657,9 @@ def test_bedrock_converse_aborts_when_judge_deadline_exhausted_before_retry(bedrock_harness.set_handler(lambda req: bedrock_harness.throttled_response(req, retry_after="5"))verifier = bedrock_harness.verifier- token = verifier._ACTIVE_JUDGE_DEADLINE.set(verifier.time.monotonic() + 0.5)+ clock = _ManualClock(bedrock_harness.slept)+ monkeypatch.setattr(verifier, "time", clock)+ token = verifier._ACTIVE_JUDGE_DEADLINE.set(clock.monotonic() + 0.5)try:content, error = bedrock_harness.call_bedrock()finally: