Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(tier3): keep the Bedrock label on strict judge errors that move on (#359)
Narendran Raghavanaf50680cb0a7
diff --git a/src/skillevaluator/tier3/harbor/templates/eval.py b/src/skillevaluator/tier3/harbor/templates/eval.pyindex 358f8579..d0189bca 100644--- a/src/skillevaluator/tier3/harbor/templates/eval.py+++ b/src/skillevaluator/tier3/harbor/templates/eval.py@@ -2096,7 +2096,7 @@ def _call_bedrock(prompt, model, max_tokens, temperature, timeout=90, provenanceexcept _JudgeRefusedError:raiseexcept Exception as exc:- if _STRICT_JUDGE_REQUESTS_LEFT.get() is not None and _strict_judge_failure(exc)[1]:+ if _STRICT_JUDGE_REQUESTS_LEFT.get() is not None and _strict_judge_failure(exc, "bedrock")[1]:# The strict policy lets the next candidate answer for an unavailable model.raisereturn None, f"Bedrock request failed: {exc}"@@ -2228,14 +2228,16 @@ def _call_judge_model(prompt, provider, model, max_tokens, temperature, response)-def _strict_judge_failure(error):+def _strict_judge_failure(error, provider=None):"""Describe a failed strict judge request and whether the next candidate may answer instead.Only unavailability moves on: a transient HTTP status, connection failureor Bedrock throttling that outlasted its one retry, a model the providerdoes not have or the key cannot use (_should_try_fallback and_strict_model_unavailable), or a used-up per-model request limit.- Everything else ends the judge.+ Everything else ends the judge. A Bedrock error keeps the+ "Bedrock request failed" label that _call_bedrock gives the errors it+ returns, so a model that moved on reads the same as one that ended the judge."""if isinstance(error, _JudgeRefusedError):return str(error), False@@ -2246,7 +2248,8 @@ def _strict_judge_failure(error):if _strict_judge_error_stops(error):return detail, Falsereturn detail, model_unavailable or _strict_model_unavailable(error) or _is_retriable_http_status(error.code)- detail = f"Public provider call failed: {error}"+ label = "Bedrock request failed" if provider == "bedrock" else "Public provider call failed"+ detail = f"{label}: {error}"if isinstance(error, TimeoutError) and "LLM judge time budget exhausted" in str(error):return detail, Falseif _is_certificate_failure(error) or _strict_judge_error_stops(error):@@ -2283,7 +2286,7 @@ def _call_strict_judge(provider, requested_model, models, call_model):try:content, error = call_model(candidate_model, provenance)except Exception as exc:- detail, move_on = _strict_judge_failure(exc)+ detail, move_on = _strict_judge_failure(exc, provider)errors.append(f"{candidate_model}: {detail}")if not move_on:# Keep the earlier models' errors, so the reason shows why they were passed over.diff --git a/tests/tier3/test_judge_policies.py b/tests/tier3/test_judge_policies.pyindex b647a962..6dccd10f 100644--- a/tests/tier3/test_judge_policies.py+++ b/tests/tier3/test_judge_policies.py@@ -414,8 +414,46 @@ def test_strict_bedrock_stop_keeps_the_errors_of_earlier_models(verifier, monkeycontent, error = verifier.call_public_llm("prompt")assert content is None- assert error.startswith(f"{PRIMARY}: Public provider call failed: Too many requests | family/backup: ")- assert error.endswith("maxTokens: must be at most 8192")+ assert error == (+ f"{PRIMARY}: Bedrock request failed: Too many requests | "+ "family/backup: Bedrock request failed: maxTokens: must be at most 8192"+ )+++def _bedrock_endpoint_error():+ from botocore.exceptions import EndpointConnectionError++ return EndpointConnectionError(endpoint_url="https://bedrock-runtime.example/model/converse")+++ ("make_error", "requests_per_model"),+ [+ (lambda: _bedrock_error("ResourceNotFoundException", 404, "Model not found."), 1),+ (lambda: _bedrock_error("ThrottlingException", 429, "Too many requests"), 2),+ (_bedrock_endpoint_error, 2),+ ],+ ids=["model-not-found", "throttled", "endpoint-unreachable"],+)+def test_strict_bedrock_errors_that_move_on_keep_the_bedrock_label(+ verifier, monkeypatch, make_error, requests_per_model+):+ strict(monkeypatch)+ monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "bedrock")+ monkeypatch.setenv("LLM_JUDGE_FALLBACK_MODELS", "family/backup")+ error = make_error()+ client = _bedrock_client(monkeypatch, [error] * (2 * requests_per_model))++ content, detail = verifier.call_public_llm("prompt")++ assert content is None+ assert detail == (+ f"LLM judge model fallback exhausted: {PRIMARY}: Bedrock request failed: {error} | "+ f"family/backup: Bedrock request failed: {error}"+ )+ assert [call.kwargs["modelId"] for call in client.converse.call_args_list] == (+ [PRIMARY] * requests_per_model + ["family/backup"] * requests_per_model+ )def test_strict_stop_words_in_an_echoed_model_id_do_not_end_the_judge(verifier, monkeypatch):