Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

test(tier3): run the Bedrock judge-deadline retry test on a manual clock (#393)

Narendran Raghavan35ef01e65d98
diff --git a/tests/test_retry.py b/tests/test_retry.py
index 2634eeff..1872c885 100644
--- a/tests/test_retry.py
+++ b/tests/test_retry.py
@@ -5,6 +5,7 @@
from __future__ import annotations
+import time
import urllib.error
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
@@ -371,6 +372,28 @@ def test_retry_call_with_backoff_invokes_on_retry_callback(monkeypatch: pytest.M
assert callback_calls[0][1] >= 0.0
+class _ManualClock:
+ """Stand-in for the verifier's ``time`` module whose monotonic clock moves only on ``sleep``.
+
+ Real setup cost (importing boto3 and building the first client can take
+ seconds on a loaded host) then never eats into a judge deadline under test.
+ """
+
+ def __init__(self, sleeps: list[float]) -> None:
+ self.now = 0.0
+ self.sleeps = sleeps
+
+ def monotonic(self) -> float:
+ return self.now
+
+ def sleep(self, seconds: float) -> None:
+ self.sleeps.append(seconds)
+ self.now += seconds
+
+ def __getattr__(self, name: str) -> Any:
+ return getattr(time, name)
+
+
class _BedrockTransportHarness:
"""Test harness for driving _call_bedrock against botocore's URLLib3Session.send."""
@@ -634,7 +657,9 @@ def test_bedrock_converse_aborts_when_judge_deadline_exhausted_before_retry(
bedrock_harness.set_handler(lambda req: bedrock_harness.throttled_response(req, retry_after="5"))
verifier = bedrock_harness.verifier
- token = verifier._ACTIVE_JUDGE_DEADLINE.set(verifier.time.monotonic() + 0.5)
+ clock = _ManualClock(bedrock_harness.slept)
+ monkeypatch.setattr(verifier, "time", clock)
+ token = verifier._ACTIVE_JUDGE_DEADLINE.set(clock.monotonic() + 0.5)
try:
content, error = bedrock_harness.call_bedrock()
finally: