Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): score goal_accuracy with RAGAS for eligible OpenAI judges (#286)

Narendran Raghavandb5cc9c33e45
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7afc799c..8ca47546 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -972,6 +972,21 @@ All notable changes to SkillEvaluator are documented in this file.
2.12.0) is not affected by bundle size and needs a rerun or a newer
SkillSpector. A report with both kinds of rows gets both explanations. The
scan still fails closed as `INCOMPLETE`.
+- The Tier 3 `goal_accuracy` RAGAS scorer now produces scores. It passed a
+ `SingleTurnSample` to the RAGAS 0.4 collections metric, which takes the
+ messages and reference as keyword arguments, so it failed before sending any
+ request and every eligible trial (the `openai` provider on the canonical
+ endpoint with a judge model outside the GPT-5 and GPT-6 families) fell back
+ to the prompt judge with only an info log. These trials now report
+ `goal_accuracy` with `"method": "ragas"` instead of `"custom"`, plus
+ `judge_model_used`, `judge_reasoning_effort` and `judge_fallback_used`, so
+ scores and lift can differ from earlier runs. A RAGAS failure that falls back
+ to the prompt judge is logged as a warning with the redacted error, and the
+ scorer's OpenAI client is closed after each attempt. A case scored by RAGAS
+ in one condition and by the prompt judge in the other is left out of the
+ `goal_accuracy` lift and counted in `judge_mixed_cases`, even when both used
+ the same model. GPT-5 and GPT-6 judges, including the defaults, are
+ unaffected.
### Security
diff --git a/docs/tier3-live-evaluation.mdx b/docs/tier3-live-evaluation.mdx
index 1d5482ac..450a6c2d 100644
--- a/docs/tier3-live-evaluation.mdx
+++ b/docs/tier3-live-evaluation.mdx
@@ -786,9 +786,14 @@ in [Judge evidence budgets](environment-variables.mdx#judge-evidence-budgets).
`https://api.openai.com/v1` endpoint and a judge model that is not a GPT-5 or
GPT-6 reasoning model, `goal_accuracy` first tries the RAGAS
`AgentGoalAccuracyWithReference` scorer through its own client, without the
- schema or retry handling above but within the same time budget. If RAGAS is
- unavailable, fails, or returns a score other than `0.0` or `1.0`, the prompt
- judge runs instead.
+ schema or retry handling above but within the same time budget. A RAGAS
+ score is recorded with `"method": "ragas"` and the judge-model fields above.
+ If RAGAS is unavailable, fails, or returns a score other than `0.0` or
+ `1.0`, the verifier log shows a warning with the redacted error, and the
+ prompt judge runs instead (`"method": "custom"`). A case scored by RAGAS in
+ one condition and by the prompt judge in the other counts as a judge mix
+ even with the same model, so it is left out of the `goal_accuracy` lift the
+ same way.
- **Transient failures.** HTTP 408, 429, and every 5xx response except 501 and
505 are retried, including 520–524 proxy errors and 529 overload responses.
Connection errors, timeouts, and Bedrock throttling or service-unavailable
diff --git a/src/skillevaluator/tier3/harbor/adapter.py b/src/skillevaluator/tier3/harbor/adapter.py
index bf660c30..c35f55c1 100644
--- a/src/skillevaluator/tier3/harbor/adapter.py
+++ b/src/skillevaluator/tier3/harbor/adapter.py
@@ -3529,7 +3529,6 @@ def _custom_compose_path(environment_dir: Path) -> Path | None:
_VERIFIER_DEPS = "ragas~=0.4.0 langchain-community<0.4.2 openai>=1.0 anthropic>=0.40 boto3>=1.34 idna>=3.10,<4"
_VERIFIER_IMPORT_SMOKE = (
- "from ragas import SingleTurnSample; "
"from ragas.llms.base import llm_factory; "
"from ragas.messages import AIMessage, HumanMessage; "
"from ragas.metrics.collections import AgentGoalAccuracyWithReference"
diff --git a/src/skillevaluator/tier3/harbor/collector.py b/src/skillevaluator/tier3/harbor/collector.py
index 33ba588f..e9bc495c 100644
--- a/src/skillevaluator/tier3/harbor/collector.py
+++ b/src/skillevaluator/tier3/harbor/collector.py
@@ -5273,7 +5273,9 @@ def _merge_constituent_judge_models(data: dict[str, Any], result: dict[str, Any]
def _reward_judge_models(reward: dict[str, Any], metric: str) -> set[str]:
"""Return the judge models that scored ``metric`` in one reward row.
- A multi-step logical reward carries its steps' models in ``_judge_models``.
+ A RAGAS score is tagged ``"<model> (ragas)"`` so it never pairs with a
+ prompt-judge score from the same model. A multi-step logical reward carries
+ its steps' models in ``_judge_models``.
"""
carried = reward.get("_judge_models")
if isinstance(carried, dict):
@@ -5284,7 +5286,10 @@ def _reward_judge_models(reward: dict[str, Any], metric: str) -> set[str]:
if isinstance(detail, dict) and isinstance(detail.get("judge_models"), list):
return {model for model in detail["judge_models"] if isinstance(model, str) and model}
model = detail.get("judge_model_used") if isinstance(detail, dict) else None
- return {model} if isinstance(model, str) and model else set()
+ if not isinstance(model, str) or not model:
+ return set()
+ # RAGAS and the prompt judge score differently with the same model, so a mix of the two must not pair.
+ return {f"{model} (ragas)" if detail.get("method") == "ragas" else model}
def _judge_models_by_case(
diff --git a/src/skillevaluator/tier3/harbor/templates/eval.py b/src/skillevaluator/tier3/harbor/templates/eval.py
index 881d9754..9980b8f1 100644
--- a/src/skillevaluator/tier3/harbor/templates/eval.py
+++ b/src/skillevaluator/tier3/harbor/templates/eval.py
@@ -2827,7 +2827,9 @@ def judge_goal_accuracy(question, ground_truth, agent_text, tool_summary=""):
if isinstance(score, bool) or score not in (0.0, 1.0):
raise ValueError("RAGAS returned a non-binary goal accuracy score")
except Exception as e:
- logger.info("RAGAS not available (%s), using custom prompt", e)
+ # The documented scorer changed for this trial, so say why where users will see it.
+ detail = _redact_configured_credentials(f"{type(e).__name__}: {e}")[:_JUDGE_ERROR_REASON_LIMIT]
+ logger.warning("RAGAS goal scorer failed (%s); using the custom goal judge", detail)
problem = f"RAGAS failed ({type(e).__name__}: {e}); used the custom judge"
_record_judge_attempt(
"goal_accuracy",
@@ -2876,7 +2878,6 @@ def _judge_goal_accuracy_ragas(question, ground_truth, agent_text, tool_summary)
import asyncio
from openai import AsyncOpenAI
- from ragas import SingleTurnSample
from ragas.llms.base import llm_factory
from ragas.messages import AIMessage as RagasAI
from ragas.messages import HumanMessage as RagasHuman
@@ -2893,38 +2894,71 @@ def _judge_goal_accuracy_ragas(question, ground_truth, agent_text, tool_summary)
if not request_url.endswith(suffix) or not _is_native_openai_chat_url("openai", request_url):
raise RuntimeError("RAGAS goal accuracy requires the selected canonical OpenAI provider")
client = AsyncOpenAI(api_key=api_key, base_url=request_url[: -len(suffix)])
- llm = llm_factory(_selected_judge_model(), client=client)
-
- metric = AgentGoalAccuracyWithReference(llm=llm)
- user_input = [
- RagasHuman(content=question),
- RagasAI(content=agent_text),
- ]
-
- sample = SingleTurnSample(user_input=user_input, reference=ground_truth)
-
loop = asyncio.new_event_loop()
try:
+ # Build the scorer inside the try so a setup failure still closes the client.
+ judge_model = _selected_judge_model()
+ llm = llm_factory(judge_model, client=client)
+ metric = AgentGoalAccuracyWithReference(llm=llm)
+ user_input = [
+ RagasHuman(content=question),
+ RagasAI(content=agent_text),
+ ]
+ # Collections metrics take the messages and reference as keywords, not a SingleTurnSample.
result = loop.run_until_complete(
asyncio.wait_for(
- metric.ascore(sample),
+ metric.ascore(user_input=user_input, reference=ground_truth),
timeout=_remaining_judge_timeout(_resolve_judge_wall_time_budget()),
)
)
finally:
- loop.close()
+
+ def consume_cleanup_task_exception(task):
+ # Tasks can fail after a cleanup timeout; never let asyncio log their exception text.
+ if not task.cancelled():
+ task.exception()
+
+ try:
+ # The judge deadline alarm can stop the loop mid-score; cancel that work before closing the client.
+ pending = asyncio.all_tasks(loop)
+ for task in pending:
+ task.add_done_callback(consume_cleanup_task_exception)
+ task.cancel()
+ if pending:
+ _finished, unfinished = loop.run_until_complete(asyncio.wait(pending, timeout=5.0))
+ if unfinished:
+ logger.warning("RAGAS judge task cancellation timed out: %d unfinished tasks", len(unfinished))
+ except Exception as error:
+ logger.warning("RAGAS judge task cancellation failed: %s", type(error).__name__)
+ finally:
+ try:
+ # Closing the client must still be attempted if scoring tasks could not be drained.
+ close_task = loop.create_task(client.close())
+ close_task.add_done_callback(consume_cleanup_task_exception)
+ _finished, unfinished = loop.run_until_complete(asyncio.wait({close_task}, timeout=5.0))
+ if unfinished:
+ close_task.cancel()
+ logger.warning("RAGAS judge client close timed out")
+ else:
+ close_task.result()
+ except (Exception, asyncio.CancelledError) as error:
+ # Cleanup must never hide the score or the scorer's own failure.
+ logger.warning("RAGAS judge client close failed: %s", type(error).__name__)
+ finally:
+ loop.close()
score = float(result.value) if hasattr(result, "value") else float(result)
if score not in (0.0, 1.0):
# Also rejects NaN; the caller then runs the prompt judge.
raise ValueError("RAGAS returned a non-binary goal accuracy score")
+ provenance = {"provider": "openai", "model": judge_model}
+ _record_judge_call(provenance, judge_model, judge_model)
return {
"score": score,
"reason": "RAGAS AgentGoalAccuracyWithReference",
"achieved": score == 1.0,
"method": "ragas",
- "provider": "openai",
- "model": _selected_judge_model(),
+ **provenance,
}
diff --git a/tests/test_harbor_metrics_truth.py b/tests/test_harbor_metrics_truth.py
index f784113e..8a57ceb4 100644
--- a/tests/test_harbor_metrics_truth.py
+++ b/tests/test_harbor_metrics_truth.py
@@ -698,6 +698,63 @@ def test_lift_leaves_out_multistep_cases_whose_steps_used_a_fallback_judge() ->
assert lift["judge_mixed_cases"] == {"accuracy": 1}
+def test_lift_leaves_out_goal_cases_scored_by_ragas_in_one_arm_only() -> None:
+ def goal_reward(case_id: str, score: float, method: str, step: int | None = None) -> dict:
+ reward = {
+ "entry_id": case_id,
+ "_trial_name": f"{case_id}__attempt1",
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(DEFAULT_METRICS, 0.5),
+ "goal_accuracy": score,
+ "details": {"goal_accuracy": {"score": score, "method": method, "judge_model_used": "gpt-4.1"}},
+ }
+ if step is not None:
+ reward |= {"_trial_root_name": f"{case_id}__attempt1", "_step_name": f"step-{step}"}
+ return reward
+
+ with_scores = dict.fromkeys(DEFAULT_METRICS, 0.5) | {"goal_accuracy": 0.9}
+ without_scores = dict.fromkeys(DEFAULT_METRICS, 0.5) | {"goal_accuracy": 0.2}
+ expected = {"with_skill": 0.8, "without_skill": 0.4, "delta": 0.4, "direction": "up"}
+
+ # The same model scored both arms, but RAGAS fell back to the prompt judge in one of them.
+ with_rewards = [goal_reward("case-1", 1.0, "ragas"), goal_reward("case-2", 0.8, "ragas")]
+ without_rewards = [goal_reward("case-1", 0.0, "custom"), goal_reward("case-2", 0.4, "ragas")]
+ lift = _compute_lift(with_scores, without_scores, with_rewards, without_rewards, ["case-1", "case-2"])
+ assert lift["goal_accuracy"] == expected
+ assert lift["judge_mixed_cases"] == {"goal_accuracy": 1}
+
+ # Two prompt-judged arms with the same model still pair.
+ same = _compute_lift(
+ with_scores,
+ without_scores,
+ [goal_reward("case-1", 0.8, "custom")],
+ [goal_reward("case-1", 0.4, "custom")],
+ ["case-1"],
+ )
+ assert "judge_mixed_cases" not in same
+
+ # A multi-step attempt whose steps used different scorers is left out too.
+ multistep_with = _logical_attempt_rewards(
+ [
+ goal_reward("case-1", 1.0, "ragas", step=1),
+ goal_reward("case-1", 1.0, "custom", step=2),
+ goal_reward("case-2", 0.8, "ragas", step=1),
+ goal_reward("case-2", 0.8, "ragas", step=2),
+ ]
+ )
+ multistep_without = _logical_attempt_rewards(
+ [
+ goal_reward("case-1", 0.0, "ragas", step=1),
+ goal_reward("case-1", 0.0, "ragas", step=2),
+ goal_reward("case-2", 0.4, "ragas", step=1),
+ goal_reward("case-2", 0.4, "ragas", step=2),
+ ]
+ )
+ lift = _compute_lift(with_scores, without_scores, multistep_with, multistep_without, ["case-1", "case-2"])
+ assert lift["goal_accuracy"] == expected
+ assert lift["judge_mixed_cases"] == {"goal_accuracy": 1}
+
+
def test_pass_summary_marks_incomplete_reward_unscored() -> None:
reward = {
"entry_id": "case-001",
diff --git a/tests/tier3/test_judge_failure_artifacts.py b/tests/tier3/test_judge_failure_artifacts.py
index b2e7c620..b9e2b3e2 100644
--- a/tests/tier3/test_judge_failure_artifacts.py
+++ b/tests/tier3/test_judge_failure_artifacts.py
@@ -6,8 +6,10 @@
from __future__ import annotations
import asyncio
+import gc
import importlib.util
import json
+import logging
import math
import os
import shutil
@@ -16,8 +18,10 @@ import subprocess
import sys
import time
import urllib.error
+from contextlib import suppress
from pathlib import Path
from types import ModuleType, SimpleNamespace
+from typing import ClassVar
import pytest
@@ -342,45 +346,72 @@ def test_required_judge_restores_alarm_handler_after_teardown_interrupt(
assert signal.getitimer(signal.ITIMER_REAL)[0] == 0
-def test_ragas_goal_judge_obeys_the_required_judge_time_budget(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
- """Ensure Ragas goal scorer returns before its Harbor budget expires."""
- verifier = _load_verifier(tmp_path)
+_RAGAS_TEST_KEY = "test-key-fake"
+
+
+def _use_ragas_eligible_judge(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Select a canonical OpenAI non-reasoning judge, which routes goal_accuracy to Ragas."""
monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "openai")
- monkeypatch.setenv("OPENAI_API_KEY", "test-key-fake")
- monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
- monkeypatch.delenv("SKILL_EVAL_LLM_BASE_URL", raising=False)
- monkeypatch.setattr(verifier, "_ragas_goal_accuracy_enabled", lambda: True)
- monkeypatch.setattr(verifier, "_JUDGE_WALL_TIME_BUDGET_SEC", 0.01)
+ monkeypatch.setenv("OPENAI_API_KEY", _RAGAS_TEST_KEY)
+ monkeypatch.setenv("LLM_JUDGE_MODEL", "gpt-4.1")
+ for name in (
+ "SKILL_EVAL_JUDGE_MODEL",
+ "SKILL_EVAL_LLM_MODEL",
+ "LLM_JUDGE_FALLBACK_MODELS",
+ "OPENAI_BASE_URL",
+ "SKILL_EVAL_LLM_BASE_URL",
+ # Character facts select the prompt goal judge instead of Ragas.
+ "SKILL_EVAL_JUDGE_CHARACTER_FACTS",
+ ):
+ monkeypatch.delenv(name, raising=False)
+
+
+class _FakeAsyncOpenAI:
+ """Record how the Ragas scorer builds and closes its OpenAI client."""
+
+ instances: ClassVar[list[_FakeAsyncOpenAI]] = []
+ close_error: ClassVar[BaseException | None] = None
+
+ def __init__(self, **kwargs):
+ self.kwargs = kwargs
+ self.closed = 0
+ type(self).instances.append(self)
+
+ async def close(self) -> None:
+ self.closed += 1
+ if self.close_error is not None:
+ raise self.close_error
+
+
+def _install_fake_ragas(monkeypatch: pytest.MonkeyPatch, metric: type) -> type[_FakeAsyncOpenAI]:
+ """Install the Ragas 0.4 modules and OpenAI client the verifier imports, with ``metric`` as the scorer."""
class FakeMessage:
- """Mock message object for Ragas sample input."""
+ """Mock message object for Ragas input."""
def __init__(self, content: str):
self.content = content
- class FakeMetric:
- """Mock metric object for Ragas async evaluation."""
+ class RejectedSample:
+ """Collections metrics do not take a sample object."""
- def __init__(self, llm):
- self.llm = llm
-
- async def ascore(self, _sample):
- await asyncio.sleep(0.05)
- return SimpleNamespace(value=1.0)
+ def __init__(self, **_kwargs):
+ raise AssertionError("Ragas collections metrics take keywords, not a SingleTurnSample")
+ client = type("FakeAsyncOpenAI", (_FakeAsyncOpenAI,), {"instances": []})
fake_ragas = ModuleType("ragas")
- fake_ragas.SingleTurnSample = lambda **_kwargs: object()
+ fake_ragas.SingleTurnSample = RejectedSample
fake_ragas_llms = ModuleType("ragas.llms")
fake_ragas_llms_base = ModuleType("ragas.llms.base")
- fake_ragas_llms_base.llm_factory = lambda *_args, **_kwargs: object()
+ fake_ragas_llms_base.llm_factory = lambda model, *, client: SimpleNamespace(model=model, client=client)
fake_ragas_messages = ModuleType("ragas.messages")
fake_ragas_messages.AIMessage = FakeMessage
fake_ragas_messages.HumanMessage = FakeMessage
fake_ragas_metrics = ModuleType("ragas.metrics")
fake_ragas_collections = ModuleType("ragas.metrics.collections")
- fake_ragas_collections.AgentGoalAccuracyWithReference = FakeMetric
+ fake_ragas_collections.AgentGoalAccuracyWithReference = metric
fake_openai = ModuleType("openai")
- fake_openai.AsyncOpenAI = lambda **_kwargs: object()
+ fake_openai.AsyncOpenAI = client
for name, module in {
"ragas": fake_ragas,
"ragas.llms": fake_ragas_llms,
@@ -391,6 +422,343 @@ def test_ragas_goal_judge_obeys_the_required_judge_time_budget(tmp_path: Path, m
"openai": fake_openai,
}.items():
monkeypatch.setitem(sys.modules, name, module)
+ return client
+
+
+def test_ragas_goal_scorer_uses_the_collections_keyword_contract(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """An eligible OpenAI judge scores goal_accuracy with Ragas instead of silently using the custom judge."""
+ verifier = _load_verifier(tmp_path)
+ _use_ragas_eligible_judge(monkeypatch)
+ calls: list[tuple[str, list[str], str]] = []
+
+ class CollectionsMetric:
+ """Mirror Ragas 0.4 AgentGoalAccuracyWithReference.ascore(user_input, reference)."""
+
+ def __init__(self, llm):
+ self.llm = llm
+
+ async def ascore(self, user_input, reference):
+ calls.append((self.llm.model, [message.content for message in user_input], reference))
+ return SimpleNamespace(value=1.0)
+
+ client = _install_fake_ragas(monkeypatch, CollectionsMetric)
+ monkeypatch.setattr(
+ verifier,
+ "_judge_goal_accuracy_custom",
+ lambda *_args, **_kwargs: pytest.fail("Ragas must score without falling back to the custom judge"),
+ )
+
+ assert verifier._ragas_goal_accuracy_enabled()
+ result = verifier._call_required_judge(
+ "goal_accuracy", verifier.judge_goal_accuracy, "question", "ground truth", "agent response"
+ )
+
+ assert calls == [("gpt-4.1", ["question", "agent response"], "ground truth")]
+ assert result["score"] == 1.0
+ assert result["method"] == "ragas"
+ assert (result["provider"], result["model"]) == ("openai", "gpt-4.1")
+ assert result["judge_model_used"] == "gpt-4.1"
+ assert result["judge_fallback_used"] is False
+ assert [instance.kwargs["base_url"] for instance in client.instances] == ["https://api.openai.com/v1"]
+ assert [instance.closed for instance in client.instances] == [1]
+
+
[email protected]("close_fails", [False, True], ids=["close-ok", "close-fails"])
+def test_ragas_goal_scorer_failure_warns_and_closes_its_client(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, close_fails: bool
+) -> None:
+ """A Ragas failure is logged as a redacted warning, and the client is closed before the custom judge runs."""
+ verifier = _load_verifier(tmp_path)
+ _use_ragas_eligible_judge(monkeypatch)
+
+ class FailingMetric:
+ """Fail the way a provider error surfaces through Ragas."""
+
+ def __init__(self, llm):
+ self.llm = llm
+
+ async def ascore(self, **_kwargs):
+ raise RuntimeError(f"Error code: 401 - Incorrect API key provided: {_RAGAS_TEST_KEY}")
+
+ client = _install_fake_ragas(monkeypatch, FailingMetric)
+ if close_fails:
+ client.close_error = OSError("socket close failed")
+ custom_result = {"score": 0.0, "reason": "custom judgment", "method": "custom"}
+
+ def custom_judge(*_args, **_kwargs):
+ assert [instance.closed for instance in client.instances] == [1]
+ return custom_result
+
+ monkeypatch.setattr(verifier, "_judge_goal_accuracy_custom", custom_judge)
+
+ with caplog.at_level(logging.WARNING, logger=verifier.logger.name):
+ result = verifier.judge_goal_accuracy("question", "ground truth", "agent response")
+
+ assert result == custom_result
+ warnings = [record.getMessage() for record in caplog.records if record.levelno == logging.WARNING]
+ fallback_warning = "RAGAS goal scorer failed (RuntimeError: Error code: 401"
+ assert any(message.startswith(fallback_warning) for message in warnings), warnings
+ assert ("RAGAS judge client close failed: OSError" in warnings) is close_fails
+ assert all(_RAGAS_TEST_KEY not in message for message in warnings)
+
+
+def test_ragas_goal_scorer_setup_failure_closes_its_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ """A failure while building the scorer, before any score request, must not leak the client."""
+ verifier = _load_verifier(tmp_path)
+ _use_ragas_eligible_judge(monkeypatch)
+
+ class BrokenMetric:
+ """Fail at construction, as an incompatible Ragas release would."""
+
+ def __init__(self, **_kwargs):
+ raise TypeError("incompatible Ragas release")
+
+ client = _install_fake_ragas(monkeypatch, BrokenMetric)
+ custom_result = {"score": 0.0, "reason": "custom judgment", "method": "custom"}
+ monkeypatch.setattr(verifier, "_judge_goal_accuracy_custom", lambda *_args, **_kwargs: custom_result)
+
+ assert verifier.judge_goal_accuracy("question", "ground truth", "agent response") == custom_result
+ assert [instance.closed for instance in client.instances] == [1]
+
+
[email protected]("scorer_fails", [False, True], ids=["score", "scorer-error"])
+def test_ragas_client_close_cancellation_preserves_the_scorer_outcome(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, scorer_fails: bool
+) -> None:
+ """A cancelled close operation must not replace a completed score or the scorer's original exception."""
+ verifier = _load_verifier(tmp_path)
+ _use_ragas_eligible_judge(monkeypatch)
+ scorer_error = RuntimeError("original scorer failure")
+
+ class Metric:
+ def __init__(self, llm):
+ self.llm = llm
+
+ async def ascore(self, **_kwargs):
+ if scorer_fails:
+ raise scorer_error
+ return SimpleNamespace(value=1.0)
+
+ client = _install_fake_ragas(monkeypatch, Metric)
+ client.close_error = asyncio.CancelledError(_RAGAS_TEST_KEY)
+
+ with caplog.at_level(logging.WARNING, logger=verifier.logger.name):
+ if scorer_fails:
+ with pytest.raises(RuntimeError) as exc_info:
+ verifier._judge_goal_accuracy_ragas("question", "ground truth", "response", "")
+ assert exc_info.value is scorer_error
+ else:
+ result = verifier._judge_goal_accuracy_ragas("question", "ground truth", "response", "")
+ assert result["score"] == 1.0
+
+ assert [instance.closed for instance in client.instances] == [1]
+ warnings = [record.getMessage() for record in caplog.records]
+ assert "RAGAS judge client close failed: CancelledError" in warnings
+ assert all(_RAGAS_TEST_KEY not in message for message in warnings)
+
+
[email protected]("stubborn", ["scoring_task", "client_close"])
[email protected]("scorer_fails", [False, True], ids=["score", "scorer-error"])
+def test_ragas_cleanup_does_not_wait_for_cancellation_acknowledgment(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+ stubborn: str,
+ scorer_fails: bool,
+) -> None:
+ """Cleanup returns before resistant work finishes, closes the client, and preserves the scorer's outcome."""
+ verifier = _load_verifier(tmp_path)
+ _use_ragas_eligible_judge(monkeypatch)
+ loop = asyncio.new_event_loop()
+ real_close = loop.close
+ release = asyncio.Event()
+ rescued: list[bool] = []
+ scoring_tasks: list[asyncio.Task[None]] = []
+ scorer_error = RuntimeError("original scorer failure")
+
+ async def refuse_cancellation():
+ while not release.is_set():
+ with suppress(asyncio.CancelledError):
+ await release.wait()
+
+ class Metric:
+ def __init__(self, llm):
+ self.llm = llm
+
+ async def ascore(self, **_kwargs):
+ if stubborn == "scoring_task":
+ scoring_tasks.append(loop.create_task(refuse_cancellation()))
+ await asyncio.sleep(0)
+ if scorer_fails:
+ raise scorer_error
+ return SimpleNamespace(value=1.0)
+
+ client = _install_fake_ragas(monkeypatch, Metric)
+ if stubborn == "client_close":
+
+ async def resistant_close(self):
+ self.closed += 1
+ await refuse_cancellation()
+
+ monkeypatch.setattr(client, "close", resistant_close)
+
+ # Shorten cleanup waits without replacing asyncio's cancellation behavior.
+ real_wait_for, real_wait = asyncio.wait_for, asyncio.wait
+
+ async def short_wait_for(awaitable, timeout):
+ return await real_wait_for(awaitable, 0.01 if timeout == 5.0 else timeout)
+
+ async def short_wait(tasks, *, timeout=None, **kwargs):
+ return await real_wait(tasks, timeout=0.01 if timeout == 5.0 else timeout, **kwargs)
+
+ monkeypatch.setattr(asyncio, "wait_for", short_wait_for)
+ monkeypatch.setattr(asyncio, "wait", short_wait)
+ monkeypatch.setattr(asyncio, "new_event_loop", lambda: loop)
+ # Keep this independent loop open so the test can finish every task after the assertion.
+ monkeypatch.setattr(loop, "close", lambda: None)
+
+ def rescue_blocked_cleanup():
+ rescued.append(True)
+ release.set()
+
+ rescue = loop.call_later(0.5, rescue_blocked_cleanup)
+ try:
+ with caplog.at_level(logging.WARNING, logger=verifier.logger.name):
+ if scorer_fails:
+ with pytest.raises(RuntimeError) as exc_info:
+ verifier._judge_goal_accuracy_ragas("question", "ground truth", "response", "")
+ assert exc_info.value is scorer_error
+ else:
+ result = verifier._judge_goal_accuracy_ragas("question", "ground truth", "response", "")
+ assert result["score"] == 1.0
+ assert result["method"] == "ragas"
+
+ assert not rescued, "cleanup waited for cancellation-resistant work to finish"
+ assert [instance.closed for instance in client.instances] == [1]
+ assert any("timed out" in record.getMessage() for record in caplog.records)
+ finally:
+ rescue.cancel()
+ release.set()
+
+ async def drain_test_tasks():
+ pending = asyncio.all_tasks(loop) - {asyncio.current_task()}
+ if pending:
+ await asyncio.gather(*pending, return_exceptions=True)
+
+ loop.run_until_complete(drain_test_tasks())
+ assert not asyncio.all_tasks(loop)
+ real_close()
+
+
+def test_ragas_cleanup_consumes_late_task_exceptions(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ """A task failing after the drain timeout must not leak credentials through asyncio's exception handler."""
+ verifier = _load_verifier(tmp_path)
+ _use_ragas_eligible_judge(monkeypatch)
+ loop = asyncio.new_event_loop()
+ release = asyncio.Event()
+ late_failures: list[bool] = []
+ scoring_tasks: list[asyncio.Task[None]] = []
+ loop_errors: list[dict] = []
+ loop.set_exception_handler(lambda _loop, context: loop_errors.append(context))
+
+ async def fail_after_client_close_starts():
+ try:
+ await asyncio.sleep(30)
+ except asyncio.CancelledError:
+ await release.wait()
+ late_failures.append(True)
+ raise RuntimeError(_RAGAS_TEST_KEY) from None
+
+ class Metric:
+ def __init__(self, llm):
+ self.llm = llm
+
+ async def ascore(self, **_kwargs):
+ scoring_tasks.append(loop.create_task(fail_after_client_close_starts()))
+ await asyncio.sleep(0)
+ return SimpleNamespace(value=1.0)
+
+ client = _install_fake_ragas(monkeypatch, Metric)
+
+ async def close(self):
+ self.closed += 1
+ release.set()
+ # Let the abandoned scoring task fail while client cleanup is still running.
+ await asyncio.sleep(0)
+ await asyncio.sleep(0)
+
+ real_wait = asyncio.wait
+
+ async def short_wait(tasks, *, timeout=None, **kwargs):
+ return await real_wait(tasks, timeout=0.01 if timeout == 5.0 else timeout, **kwargs)
+
+ monkeypatch.setattr(client, "close", close)
+ monkeypatch.setattr(asyncio, "wait", short_wait)
+ monkeypatch.setattr(asyncio, "new_event_loop", lambda: loop)
+
+ with caplog.at_level(logging.WARNING, logger=verifier.logger.name):
+ result = verifier._judge_goal_accuracy_ragas("question", "ground truth", "response", "")
+
+ assert result["score"] == 1.0
+ assert [instance.closed for instance in client.instances] == [1]
+ assert late_failures == [True]
+ assert any("task cancellation timed out" in record.getMessage() for record in caplog.records)
+ # Finalize the abandoned task's exception cycle before checking asyncio's handler.
+ scoring_tasks.clear()
+ gc.collect()
+ assert not loop_errors
+ assert all(_RAGAS_TEST_KEY not in record.getMessage() for record in caplog.records)
+
+
[email protected]("cutoff", ["scorer_timeout", "deadline_alarm"])
+def test_ragas_goal_judge_obeys_the_required_judge_time_budget(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, cutoff: str
+) -> None:
+ """Ensure Ragas goal scorer returns before its Harbor budget expires and still closes its client."""
+ verifier = _load_verifier(tmp_path)
+ _use_ragas_eligible_judge(monkeypatch)
+ cancelled: list[bool] = []
+ if cutoff == "scorer_timeout":
+ # Without an interval timer only the scorer's own wait_for bounds it.
+ monkeypatch.setattr(verifier, "signal", SimpleNamespace())
+ monkeypatch.setattr(verifier, "_JUDGE_WALL_TIME_BUDGET_SEC", 0.01)
+ scorer_sleep = 0.05
+ else:
+ if not hasattr(signal, "setitimer") or signal.getitimer(signal.ITIMER_REAL)[0] > 0:
+ pytest.skip("free POSIX interval timer required")
+ monkeypatch.setattr(verifier, "_JUDGE_WALL_TIME_BUDGET_SEC", 0.05)
+ scorer_sleep = 30.0
+ real_remaining = verifier._remaining_judge_timeout
+ scorer_bounds: list[float] = []
+
+ def remaining_after_scorer(timeout):
+ # Leave the scorer's first bound open so the deadline alarm stops the loop mid-score.
+ if not scorer_bounds:
+ scorer_bounds.append(timeout)
+ return 60.0
+ return real_remaining(timeout)
+
+ monkeypatch.setattr(verifier, "_remaining_judge_timeout", remaining_after_scorer)
+
+ class FakeMetric:
+ """Mock metric object for Ragas async evaluation."""
+
+ def __init__(self, llm):
+ self.llm = llm
+
+ async def ascore(self, **_kwargs):
+ try:
+ await asyncio.sleep(scorer_sleep)
+ except asyncio.CancelledError:
+ cancelled.append(True)
+ raise
+ return SimpleNamespace(value=1.0)
+
+ client = _install_fake_ragas(monkeypatch, FakeMetric)
monkeypatch.setattr(
verifier.urllib.request,
"urlopen",
@@ -403,6 +771,9 @@ def test_ragas_goal_judge_obeys_the_required_judge_time_budget(tmp_path: Path, m
assert result["status"] == "error"
assert result["score"] is None
+ # The cut-off scoring work is cancelled and the client is closed once, not left to the loop's teardown.
+ assert cancelled == [True]
+ assert [instance.closed for instance in client.instances] == [1]
def test_verifier_main_keeps_genuine_zero_judge_verdicts_scoreable(