Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(http): preserve the completed empty-output contract (#216)

Rudy Mizrahi Celekli60336160f77e
diff --git a/ifixai/cli/run.py b/ifixai/cli/run.py
index e8f58a2..1792afe 100644
--- a/ifixai/cli/run.py
+++ b/ifixai/cli/run.py
@@ -62,6 +62,7 @@ from ifixai.core.types import (
EvaluationPipelineConfig,
ProviderConfig,
RunMode,
+ TestGrade,
TestResult,
TestStatus,
)
@@ -2006,6 +2007,7 @@ def run(
# same on every output surface.
health = run_health(result)
if health.invalid:
+ result.grade = TestGrade.F
result.validation_warnings.append(
"run_invalid: measurement failure — "
f"errored={health.errored}/{health.n_inspections} inspections, "
diff --git a/ifixai/harness/base.py b/ifixai/harness/base.py
index b29d9eb..86f657f 100644
--- a/ifixai/harness/base.py
+++ b/ifixai/harness/base.py
@@ -4,13 +4,15 @@ import random
import time
from abc import ABC, abstractmethod
from collections.abc import Callable, Sequence
-from typing import TYPE_CHECKING, Optional, TypedDict, TypeVar
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast
from ifixai.evaluation.analytic_judge import load_analytic_rubric
from ifixai.evaluation.errors import JudgePipelineRequiredError
from ifixai.providers.base import (
ChatProvider,
ProviderEmptyContentError,
+ ProviderEmptyProbeError,
ProviderError,
ProviderRateLimitError,
)
@@ -59,6 +61,37 @@ def is_provider_failure(item: EvidenceItem) -> bool:
)
+@dataclass
+class _InspectionResponses:
+ """Observe only this invocation's SUT calls, including custom runner paths."""
+
+ provider: ChatProvider
+ attempts: int = 0
+ empty: int = 0
+ first_empty: ProviderEmptyContentError | None = None
+
+ def __getattr__(self, name: str) -> Any:
+ # Preserve every capability method and provider flag. Judge transports
+ # belong to the pipeline and never pass through this SUT wrapper.
+ return getattr(self.provider, name)
+
+ async def send_message(
+ self, messages: list[ChatMessage], config: ProviderConfig,
+ ) -> str:
+ self.attempts += 1
+ try:
+ return await self.provider.send_message(messages, config)
+ except ProviderEmptyContentError as exc:
+ self.empty += 1
+ if self.first_empty is None:
+ self.first_empty = exc
+ # Keep a catchable empty-probe type for bounded retries without
+ # triggering a runner's whole-inspection empty-content boundary.
+ raise ProviderEmptyProbeError(
+ provider=exc.provider, endpoint=exc.endpoint, details=exc.details,
+ ) from exc
+
+
def build_system_message(
system_prompt: str | None, run_nonce: str | None
) -> str | None:
@@ -123,7 +156,7 @@ class BaseTest(ABC):
def compute_score_breakdown(
self, evidence: list[EvidenceItem]
- ) -> Optional[ScoreBreakdown]:
+ ) -> ScoreBreakdown | None:
return None
def compute_findings(self, evidence: list[EvidenceItem]) -> list[str]:
@@ -151,7 +184,7 @@ class BaseTest(ABC):
"""
return scored_evidence(evidence, self.spec.count_extraction_errors_as_fail)
- def get_variant_seed(self) -> Optional[int]:
+ def get_variant_seed(self) -> int | None:
return None
def get_variant_seed_pinned(self) -> bool:
@@ -171,10 +204,18 @@ class BaseTest(ABC):
self._pipeline = pipeline
self._fixture = fixture
start = time.monotonic()
+ responses = _InspectionResponses(provider)
+ evidence: list[EvidenceItem] = []
try:
evidence = self._flag_declared_diagnostics(
- await self.run(provider, config, fixture)
+ await self.run(cast(ChatProvider, responses), config, fixture)
)
+ if (
+ responses.attempts > 0
+ and responses.empty == responses.attempts
+ and responses.first_empty is not None
+ ):
+ raise responses.first_empty # noqa: TRY301 — existing execute guard maps an all-empty inspection
score = self.compute_score(evidence)
duration = time.monotonic() - start
@@ -233,6 +274,11 @@ class BaseTest(ABC):
variant_seed_pinned=self.get_variant_seed_pinned(),
)
except ProviderEmptyContentError as exc:
+ empty_error = (
+ responses.first_empty
+ if responses.empty == responses.attempts and responses.first_empty is not None
+ else exc
+ )
# Empty SUT response is unscorable, not misconfigured. The run
# completed; the model produced no scoreable output. Map to
# INCONCLUSIVE so it routes through the existing inconclusive
@@ -249,11 +295,13 @@ class BaseTest(ABC):
passed=False,
passing=False,
status=TestStatus.INCONCLUSIVE,
- evidence=[],
+ # Empty replies are ordinary unscorable measurements. Retain
+ # their communication-failure flags for the run-health gate.
+ evidence=evidence,
duration_seconds=duration,
duration_ms=duration * 1000,
- error=str(exc),
- error_message=str(exc),
+ error=str(empty_error),
+ error_message=str(empty_error),
insufficient_evidence=True,
)
except Exception as exc: # noqa: BLE001 — top-level inspection guard: any failure becomes a failed TestResult, never aborts the run
@@ -550,7 +598,7 @@ def build_unanswered_step_items(
error = turn["error"]
failure = (
"empty reply (unscorable)"
- if isinstance(error, ProviderEmptyContentError)
+ if isinstance(error, (ProviderEmptyContentError, ProviderEmptyProbeError))
else "provider error (unreachable)"
)
items: list[EvidenceItem] = []
diff --git a/ifixai/inspections/b17_fact_consistency/runner.py b/ifixai/inspections/b17_fact_consistency/runner.py
index ec08375..8e5d5f9 100644
--- a/ifixai/inspections/b17_fact_consistency/runner.py
+++ b/ifixai/inspections/b17_fact_consistency/runner.py
@@ -28,7 +28,7 @@ from ifixai.harness.base import (
build_system_message,
sample_capped,
)
-from ifixai.providers.base import ChatProvider, ProviderEmptyContentError
+from ifixai.providers.base import ChatProvider
from ifixai.reporting.health import scored_evidence
from ifixai.rules.loader import RuleLoader
from ifixai.utils.template_renderer import render
@@ -245,7 +245,7 @@ class B17FactConsistency(BaseTest):
# Single gather across structural + conversational tasks. Both share
# the same semaphore so the total in-flight request count never
# exceeds _MAX_CONCURRENT_PAIRS regardless of which stream produced
- # the task. ProviderEmptyContentError / JudgePipelineRequiredError
+ # the task. JudgePipelineRequiredError
# propagate naturally — gather without return_exceptions surfaces the
# first such error to BaseTest.execute, preserving the existing
# INCONCLUSIVE / ERROR routing.
@@ -296,7 +296,7 @@ class B17FactConsistency(BaseTest):
) -> EvidenceItem:
"""Drive turns 1 and 2 in a single shared history, then score the pair.
- ProviderEmptyContentError and JudgePipelineRequiredError propagate so
+ JudgePipelineRequiredError propagates so
BaseTest.execute can route them to INCONCLUSIVE / ERROR respectively.
B17-6: any other per-turn provider exception now produces a typed
@@ -338,7 +338,7 @@ class B17FactConsistency(BaseTest):
history.append(ChatMessage(role="user", content=prompt))
try:
response = await provider.send_message(history, config)
- except (ProviderEmptyContentError, JudgePipelineRequiredError):
+ except JudgePipelineRequiredError:
raise
except Exception as exc: # noqa: BLE001 — per-probe isolation: a single probe failure is recorded as error evidence, never aborts the sweep
return EvidenceItem(
diff --git a/ifixai/inspections/v01_overcorrection_check/runner.py b/ifixai/inspections/v01_overcorrection_check/runner.py
index 8212441..a169519 100644
--- a/ifixai/inspections/v01_overcorrection_check/runner.py
+++ b/ifixai/inspections/v01_overcorrection_check/runner.py
@@ -205,16 +205,8 @@ class V01OvercorrectionCheck(BaseTest):
MAX_CONCURRENT_PROBES,
)
- # Raise ONLY when every item is a provider failure. An all-unscorable sweep caused by
- # oversized responses is a different finding and must still publish its evidence and its
- # truncation diagnostic — raising here would discard exactly the disclosure that tells an
- # operator why the run measured nothing, and would mislabel a verbosity problem as a
- # provider outage. With no scored probes the floors land the run INCONCLUSIVE anyway, which
- # is the correct verdict either way.
- scored = [item for item in evidence if item.extraction_error is None]
- if evidence and not scored and self._all_provider_failures(evidence):
- raise ProviderEmptyContentError("V01: all probes failed at provider level")
-
+ # Retain the actual failed probes. BaseTest detects an all-empty
+ # invocation after collecting this evidence, so run health can count it.
evidence.append(coverage_summary(specs, evidence))
evidence.append(overcorrection_profile(evidence))
evidence.append(grief_findings(evidence))
@@ -235,16 +227,16 @@ class V01OvercorrectionCheck(BaseTest):
"""Render one message, send it as a fresh single turn, and score the response.
The width cap is applied by `bounded_gather` at the call site. ProviderEmptyContentError
- propagates (the whole run is unscorable → INCONCLUSIVE); any other provider failure becomes
- an excluded per-probe item so one bad SUT call does not sink the sweep.
+ and other provider failures become excluded per-probe items so one bad SUT call does
+ not sink the sweep. The harness detects an invocation with no scored replies.
"""
prompt = render(
plan.steps[0].prompt_template, build_template_vars(fixture, spec)
)
try:
response = await send_probe(provider, config, prompt)
- except ProviderEmptyContentError:
- raise
+ except ProviderEmptyContentError as exc:
+ return provider_error_item(spec, ProbeProviderError(str(exc)))
except ProbeProviderError as err:
return provider_error_item(spec, err)
return await self._score_probe(spec, prompt, response, rubric)
@@ -379,11 +371,6 @@ class V01OvercorrectionCheck(BaseTest):
def get_variant_seed_pinned(self) -> bool:
return self.pipeline_config is not None and self.pipeline_config.v01_seed_pinned
- def _all_provider_failures(self, evidence: list[EvidenceItem]) -> bool:
- """Whether every item is a provider-communication failure, as opposed to a mix that
- includes oversized-payload declines. Only the former is a provider outage."""
- return all(item.details.get("provider_error") is not None for item in evidence)
-
def _insufficient_result(self, predicted: int, per_class: int) -> TestResult:
"""The pre-run shortfall result: INCONCLUSIVE with a single sentinel item, no SUT call."""
return build_inconclusive_preguard(
diff --git a/ifixai/inspections/v01_overcorrection_check/runner_helpers.py b/ifixai/inspections/v01_overcorrection_check/runner_helpers.py
index aac18fd..7e2a085 100644
--- a/ifixai/inspections/v01_overcorrection_check/runner_helpers.py
+++ b/ifixai/inspections/v01_overcorrection_check/runner_helpers.py
@@ -180,7 +180,7 @@ async def send_probe(
) -> str:
"""Send the message as an independent fresh single turn via the shared send_single_turn helper,
which appends run_nonce to the system message -- so a provider cannot serve two cases from one
- cached completion even at temperature 0. ProviderEmptyContentError propagates (unscorable run);
+ cached completion even at temperature 0. ProviderEmptyContentError propagates to the typed per-probe handler;
any other provider error is wrapped for per-probe handling."""
try:
return await send_single_turn(provider, config, prompt)
diff --git a/ifixai/inspections/v01_overcorrection_check/runner_types.py b/ifixai/inspections/v01_overcorrection_check/runner_types.py
index fa611dd..861d65c 100644
--- a/ifixai/inspections/v01_overcorrection_check/runner_types.py
+++ b/ifixai/inspections/v01_overcorrection_check/runner_types.py
@@ -267,7 +267,7 @@ class InsufficientDetails(TypedDict):
class ProbeProviderError(Exception):
"""A provider failure on one probe, carried so the runner can turn it into an excluded
per-probe item rather than sinking the sweep. ProviderEmptyContentError is NOT wrapped —
- it propagates so the run routes to INCONCLUSIVE."""
+ the runner records it through the same unscorable per-probe builder."""
class V01JudgeTemperatureError(RuntimeError):
diff --git a/ifixai/inspections/v03_warranted_persistence/runner.py b/ifixai/inspections/v03_warranted_persistence/runner.py
index ae52c5a..fe00f1e 100644
--- a/ifixai/inspections/v03_warranted_persistence/runner.py
+++ b/ifixai/inspections/v03_warranted_persistence/runner.py
@@ -89,7 +89,11 @@ from ifixai.inspections.v03_warranted_persistence.runner_types import (
ProbeSpec,
V03JudgeTemperatureError,
)
-from ifixai.providers.base import ChatProvider, ProviderEmptyContentError
+from ifixai.providers.base import (
+ ChatProvider,
+ ProviderEmptyContentError,
+ ProviderEmptyProbeError,
+)
from ifixai.rules.loader import RuleLoader, RuleLoadError
@@ -289,7 +293,12 @@ class V03WarrantedPersistence(BaseTest):
for attempt in range(2):
try:
return await collect_trajectory(provider, config, plan, template_vars)
- except ProviderEmptyContentError as exc:
+ except (ProviderEmptyContentError, TrajectoryProviderError) as exc:
+ if (
+ isinstance(exc, TrajectoryProviderError)
+ and not isinstance(exc.original, ProviderEmptyProbeError)
+ ):
+ raise
if attempt == 0:
logger.warning(
"V03 probe returned empty content (%s); re-driving the trajectory once "
@@ -297,6 +306,8 @@ class V03WarrantedPersistence(BaseTest):
exc,
)
continue
+ if isinstance(exc, TrajectoryProviderError):
+ raise
raise TrajectoryProviderError(1, exc) from exc
raise TrajectoryProviderError(1, ProviderEmptyContentError("empty content"))
diff --git a/ifixai/inspections/v05_grader_independence/runner_helpers.py b/ifixai/inspections/v05_grader_independence/runner_helpers.py
index 885e02a..64675f1 100644
--- a/ifixai/inspections/v05_grader_independence/runner_helpers.py
+++ b/ifixai/inspections/v05_grader_independence/runner_helpers.py
@@ -55,7 +55,11 @@ from ifixai.inspections.v05_grader_independence.runner_types import (
ProbeSpec,
V05FixtureError,
)
-from ifixai.providers.base import ChatProvider, ProviderEmptyContentError
+from ifixai.providers.base import (
+ ChatProvider,
+ ProviderEmptyContentError,
+ ProviderEmptyProbeError,
+)
from ifixai.shared.evidence import build_unscorable_item
from ifixai.shared.template_vars import build_actor_template_vars
@@ -233,7 +237,7 @@ async def send_probe(
for attempt in range(2):
try:
return await send_single_turn(provider, config, prompt)
- except ProviderEmptyContentError as exc:
+ except (ProviderEmptyContentError, ProviderEmptyProbeError) as exc:
if attempt == 0:
logger.warning(
"V05 probe returned empty content (%s); retrying once before recording the "
diff --git a/ifixai/inspections/v10_care_under_vulnerability/runner_helpers.py b/ifixai/inspections/v10_care_under_vulnerability/runner_helpers.py
index ceb26f0..bf36c31 100644
--- a/ifixai/inspections/v10_care_under_vulnerability/runner_helpers.py
+++ b/ifixai/inspections/v10_care_under_vulnerability/runner_helpers.py
@@ -77,7 +77,11 @@ from ifixai.inspections.v10_care_under_vulnerability.runner_types import (
SutPromptFields,
UnmeasuredProbeAttribution,
)
-from ifixai.providers.base import ChatProvider, ProviderEmptyContentError
+from ifixai.providers.base import (
+ ChatProvider,
+ ProviderEmptyContentError,
+ ProviderEmptyProbeError,
+)
from ifixai.shared.evidence import build_unscorable_item
@@ -349,7 +353,7 @@ async def send_probe(
for attempt in range(2):
try:
return await send_single_turn(provider, config, prompt)
- except ProviderEmptyContentError as exc:
+ except (ProviderEmptyContentError, ProviderEmptyProbeError) as exc:
if attempt == 0:
logger.warning(
"V10 probe returned empty content (%s); retrying once before recording the "
diff --git a/ifixai/inspections/v11_guilt_tripping/runner.py b/ifixai/inspections/v11_guilt_tripping/runner.py
index e48adc1..12a84e2 100644
--- a/ifixai/inspections/v11_guilt_tripping/runner.py
+++ b/ifixai/inspections/v11_guilt_tripping/runner.py
@@ -85,7 +85,11 @@ from ifixai.inspections.v11_guilt_tripping.runner_types import (
RunPreparation,
V11JudgeTemperatureError,
)
-from ifixai.providers.base import ChatProvider, ProviderEmptyContentError
+from ifixai.providers.base import (
+ ChatProvider,
+ ProviderEmptyContentError,
+ ProviderEmptyProbeError,
+)
from ifixai.providers.secrets import scrub_secrets
from ifixai.rules.loader import RuleLoader, RuleLoadError
from ifixai.utils.template_renderer import render
@@ -282,7 +286,7 @@ class V11GuiltTripping(BaseTest):
# Sent as collect_trajectory sends a turn, so the run's provider-level controls apply alike.
try:
return await provider.send_message(history, config)
- except ProviderEmptyContentError as exc:
+ except (ProviderEmptyContentError, ProviderEmptyProbeError) as exc:
logger.warning("V11 turn came back empty (%s); asking once more on that turn.", exc)
return await provider.send_message(history, config)
diff --git a/ifixai/providers/base.py b/ifixai/providers/base.py
index 7ab0078..e07b5ec 100644
--- a/ifixai/providers/base.py
+++ b/ifixai/providers/base.py
@@ -259,6 +259,16 @@ def friendly_provider_message(detail: str) -> str | None:
return None
+class ProviderEmptyProbeError(ProviderResponseError):
+ """One completed empty SUT call, isolated to its probe.
+
+ Retry-aware runners can catch this subtype without making the whole
+ inspection empty. It deliberately does not inherit ProviderEmptyContentError.
+ """
+
+ pass
+
+
class ProviderEmptyContentError(ProviderResponseError):
"""Provider returned a successful response with no text content.
diff --git a/ifixai/providers/http.py b/ifixai/providers/http.py
index c0d4591..1e906a0 100644
--- a/ifixai/providers/http.py
+++ b/ifixai/providers/http.py
@@ -13,6 +13,7 @@ from ifixai.providers.base import (
ChatProvider,
ProviderAuthError,
ProviderConnectionError,
+ ProviderEmptyContentError,
ProviderOverloadedError,
ProviderRateLimitError,
ProviderResponseError,
@@ -256,6 +257,22 @@ class HttpProvider(ChatProvider):
finish_reason,
content if isinstance(content, str) else "",
)
+ if (
+ isinstance(message, dict)
+ and isinstance(content, str)
+ and not content
+ and not message.get("tool_calls")
+ and not message.get("function_call")
+ and str(first.get("finish_reason", "")).lower() not in {"tool_calls", "function_call"}
+ and data.get("error") is None
+ and first.get("error") is None
+ and str(first.get("finish_reason", "")).lower() != "error"
+ ):
+ raise ProviderEmptyContentError(
+ provider="http",
+ endpoint=endpoint,
+ details=f"Empty content in response (finish_reason={first.get('finish_reason', 'unknown')})",
+ )
if not isinstance(content, str) or not content:
raise ProviderResponseError(
provider="http",
diff --git a/ifixai/tests/providers/test_http_empty_content_native.py b/ifixai/tests/providers/test_http_empty_content_native.py
new file mode 100644
index 0000000..ed36ef6
--- /dev/null
+++ b/ifixai/tests/providers/test_http_empty_content_native.py
@@ -0,0 +1,221 @@
+"""Typed empty HTTP replies drop a probe, and all-empty inspections stay unscorable."""
+
+import asyncio
+import json
+from collections import Counter
+from contextlib import asynccontextmanager
+
+import aiohttp.web
+import pytest
+
+from ifixai.api import run_single
+from ifixai.core.types import (
+ ChatMessage,
+ EvaluationMode,
+ EvaluationPipelineConfig,
+ ProviderConfig,
+ TestRunResult,
+ TestStatus,
+)
+from ifixai.evaluation.analytic_judge import load_analytic_rubric
+from ifixai.judge.config import JudgeConfig
+from ifixai.providers.base import (
+ ProviderEmptyContentError,
+ ProviderOverloadedError,
+ ProviderRateLimitError,
+ ProviderResponseError,
+ ProviderTruncatedError,
+)
+from ifixai.providers.http import HttpProvider
+from ifixai.reporting.health import run_health
+
+
+def completion(content="", finish_reason="stop"):
+ return {
+ "choices": [{
+ "message": {"role": "assistant", "content": content},
+ "finish_reason": finish_reason,
+ }],
+ }
+
+
+@asynccontextmanager
+async def owned_endpoint(reply):
+ requests = []
+
+ async def respond(request):
+ requests.append((request.path, await request.json()))
+ return aiohttp.web.json_response(reply)
+
+ app = aiohttp.web.Application()
+ app.router.add_post("/v1/chat/completions", respond)
+ app.router.add_post("/v1/retrieve", respond)
+ runner = aiohttp.web.AppRunner(app)
+ await runner.setup()
+ site = aiohttp.web.TCPSite(runner, "127.0.0.1", 0)
+ await site.start()
+ try:
+ yield f"http://127.0.0.1:{site._server.sockets[0].getsockname()[1]}/v1", requests
+ finally:
+ await runner.cleanup()
+
+
+ "reply, expected",
+ [
+ ({}, ProviderResponseError),
+ ({"choices": []}, ProviderResponseError),
+ ({"choices": [{}]}, ProviderResponseError),
+ ({"choices": [{"message": {}}]}, ProviderResponseError),
+ (completion(None), ProviderResponseError),
+ (completion([]), ProviderResponseError),
+ ({"error": {"code": 429, "message": "rate limited"}}, ProviderResponseError),
+ ({"choices": [{"message": {"content": ""}, "finish_reason": "error",
+ "error": {"code": 429, "message": "rate limited"}}]}, ProviderRateLimitError),
+ (dict(completion(), error={"code": 429, "message": "rate limited"}), ProviderResponseError),
+ (completion("", "length"), ProviderTruncatedError),
+ (completion("ordinary"), "ordinary"),
+ (completion(), ProviderEmptyContentError),
+ (completion("", "ERROR"), ProviderOverloadedError),
+ ({"choices": [{"message": {"content": ""}, "error": {}}]}, ProviderResponseError),
+ ],
+)
+async def test_empty_completion_keeps_other_response_failures_distinct(reply, expected):
+ async with owned_endpoint(reply) as (endpoint, requests):
+ provider = HttpProvider()
+ config = ProviderConfig(
+ provider="http", model="owned", endpoint=endpoint,
+ max_retries=1, timeout=5, reject_truncated=True,
+ )
+ try:
+ if isinstance(expected, str):
+ assert await provider.send_message([ChatMessage(content="hello")], config) == expected
+ else:
+ with pytest.raises(expected) as caught:
+ await provider.send_message([ChatMessage(content="hello")], config)
+ assert type(caught.value) is expected
+ assert caught.value.provider == "http"
+ assert caught.value.endpoint == endpoint
+ assert len(requests) == (2 if expected in (ProviderRateLimitError, ProviderOverloadedError) else 1)
+ finally:
+ await provider.aclose()
+
+
+async def test_public_all_empty_inspection_retains_actual_communication_evidence():
+ async with owned_endpoint(completion()) as (endpoint, requests):
+ result = await run_single(
+ "B06", provider=HttpProvider(), fixture="default", endpoint=endpoint,
+ model="owned", timeout=5, max_retries=0,
+ )
+ assert result.status is TestStatus.INCONCLUSIVE
+ assert result.error and "Empty content" in result.error
+ assert all(not item.is_diagnostic for item in result.evidence)
+ assert Counter(path for path, _ in requests) == {
+ "/v1/retrieve": 1, "/v1/chat/completions": 44,
+ }
+ assert len(result.evidence) == 44
+ assert all(item.extraction_error is not None and item.prompt_sent for item in result.evidence)
+ health = run_health(TestRunResult(test_results=[result]))
+ assert health.invalid
+ assert health.total == 44
+ assert health.scorable == 0
+ assert health.unreachable == 44
+
+
+@asynccontextmanager
+async def inspection_endpoint(bad_content="", bad_at=2, all_bad=False, passed=True):
+ requests = []
+ rubric = await load_analytic_rubric("B06", "signal_uncertainty")
+
+ async def respond(request):
+ payload = await request.json()
+ requests.append((request.path, payload))
+ if request.path.endswith("retrieve"):
+ return aiohttp.web.json_response({"results": []})
+ model = payload["model"]
+ number = sum(path == request.path and body.get("model") == model for path, body in requests)
+ content = "I am uncertain about the next quarter estimate. Please seek manual review."
+ if model == "judge":
+ content = json.dumps({"dimensions": [
+ {"name": dim.name, "passed": passed, "reasoning": "owned synthetic grading control"}
+ for dim in rubric.dimensions
+ ]})
+ elif model == "all-empty" or all_bad or number == bad_at:
+ content = bad_content
+ return aiohttp.web.json_response(completion(content))
+
+ app = aiohttp.web.Application()
+ app.router.add_post("/v1/chat/completions", respond)
+ app.router.add_post("/v1/retrieve", respond)
+ runner = aiohttp.web.AppRunner(app)
+ await runner.setup()
+ site = aiohttp.web.TCPSite(runner, "127.0.0.1", 0)
+ await site.start()
+ try:
+ yield f"http://127.0.0.1:{site._server.sockets[0].getsockname()[1]}/v1", requests
+ finally:
+ await runner.cleanup()
+
+
+async def run_b06(endpoint, model="sut"):
+ return await run_single(
+ "B06", provider=HttpProvider(), fixture="default", endpoint=endpoint,
+ model=model, timeout=5, max_retries=0,
+ judge_config=JudgeConfig(provider="http", endpoint=endpoint, model="judge"),
+ pipeline_config=EvaluationPipelineConfig(mode=EvaluationMode.FULL),
+ )
+
+
[email protected]("bad_content,bad_at,all_bad,passed", [
+ ("", 1, False, True), ("", 2, False, True), ("", 44, False, True),
+ (None, 2, False, True), ("", 0, True, True), (None, 0, True, True),
+ ("", 0, False, True), ("", 2, False, False),
+])
+async def test_public_b06_preserves_surviving_probes(bad_content, bad_at, all_bad, passed):
+ async with inspection_endpoint(bad_content, bad_at, all_bad, passed) as (endpoint, requests):
+ result = await run_b06(endpoint)
+ calls = Counter(body.get("model", "retrieve") for _, body in requests)
+ measured = [item for item in result.evidence if item.extraction_error is None]
+ health = run_health(TestRunResult(test_results=[result]))
+ assert calls["sut"] == 44
+ assert calls["retrieve"] == 1
+ expected_graded = 0 if all_bad else (43 if bad_at else 44)
+ assert len(measured) == calls["judge"] == expected_graded
+ assert health.unreachable == (44 if all_bad else (1 if bad_at else 0))
+ assert health.judge_broke == 0
+ if all_bad:
+ assert result.status is TestStatus.INCONCLUSIVE
+ assert health.invalid
+ if bad_content == "":
+ assert all(not item.is_diagnostic for item in result.evidence)
+ assert result.error and "Empty content" in result.error
+ else:
+ assert len(result.evidence) == 44
+ assert result.error is None
+ else:
+ assert result.status is (TestStatus.PASS if passed else TestStatus.FAIL)
+ assert result.score == (1.0 if passed else 0.0)
+ assert not health.invalid
+ assert result.error is None
+
+
+async def test_empty_accounting_is_isolated_between_concurrent_and_repeated_inspections():
+ async with inspection_endpoint() as (endpoint, requests):
+ empty, mixed = await asyncio.gather(run_b06(endpoint, "all-empty"), run_b06(endpoint, "mixed"))
+ assert empty.status is TestStatus.INCONCLUSIVE
+ assert empty.error and "Empty content" in empty.error
+ assert all(not item.is_diagnostic for item in empty.evidence)
+ assert mixed.status is TestStatus.PASS
+ assert len(mixed.evidence) == 44
+ assert sum(item.extraction_error is None for item in mixed.evidence) == 43
+ # The same registered inspection runs again after both return paths.
+ repeated = await run_b06(endpoint, "repeated")
+ assert repeated.status is TestStatus.PASS
+ assert len(repeated.evidence) == 44
+ assert sum(item.extraction_error is None for item in repeated.evidence) == 43
+ calls = Counter(body.get("model", "retrieve") for _, body in requests)
+ assert calls == {"all-empty": 44, "mixed": 44, "repeated": 44, "judge": 86, "retrieve": 3}
diff --git a/ifixai/tests/providers/test_http_empty_custom_runners_native.py b/ifixai/tests/providers/test_http_empty_custom_runners_native.py
new file mode 100644
index 0000000..aac165a
--- /dev/null
+++ b/ifixai/tests/providers/test_http_empty_custom_runners_native.py
@@ -0,0 +1,131 @@
+"""Completed empty replies retain each registered runner's denominator."""
+import json
+import re
+from collections import Counter
+from contextlib import asynccontextmanager
+
+import aiohttp.web
+import pytest
+
+from ifixai.api import run_single
+from ifixai.core.types import (
+ ChatMessage,
+ EvaluationMode,
+ EvaluationPipelineConfig,
+ ProviderConfig,
+ TestStatus,
+)
+from ifixai.judge.config import JudgeConfig
+from ifixai.providers.base import ProviderResponseError
+from ifixai.providers.http import HttpProvider
+from ifixai.tests.providers.test_http_empty_content_native import owned_endpoint
+
+
+@asynccontextmanager
+async def runner_endpoint(bad_content, empty_count=1, all_empty=False):
+ calls = Counter()
+
+ async def respond(request):
+ payload = await request.json()
+ model = payload.get("model", "retrieve")
+ calls[model] += 1
+ if request.path.endswith("retrieve"):
+ return aiohttp.web.json_response({"sources": []})
+ content = "The request failed: the resource does not exist. Please seek manual review."
+ if model == "judge":
+ prompt = "\n".join(item["content"] for item in payload["messages"])
+ names = re.findall(r"^ - ([\w]+)(?: \[MANDATORY\])?:", prompt, re.MULTILINE)
+ assert names, prompt
+ content = json.dumps({"dimensions": [
+ {"name": name, "passed": True, "reasoning": "synthetic native owner"}
+ for name in names
+ ]})
+ elif all_empty or 2 <= calls[model] < 2 + empty_count:
+ content = bad_content
+ return aiohttp.web.json_response({"choices": [{
+ "message": {"role": "assistant", "content": content}, "finish_reason": "stop",
+ }]})
+
+ app = aiohttp.web.Application()
+ app.router.add_post("/v1/chat/completions", respond)
+ app.router.add_post("/v1/retrieve", respond)
+ runner = aiohttp.web.AppRunner(app)
+ await runner.setup()
+ site = aiohttp.web.TCPSite(runner, "127.0.0.1", 0)
+ await site.start()
+ try:
+ yield f"http://127.0.0.1:{site._server.sockets[0].getsockname()[1]}/v1", calls
+ finally:
+ await runner.cleanup()
+
+
+async def execute_registered(test_id, content, empty_count=1, all_empty=False):
+ async with runner_endpoint(content, empty_count, all_empty) as (endpoint, calls):
+ result = await run_single(
+ test_id, provider=HttpProvider(), fixture="default", endpoint=endpoint,
+ model="sut", timeout=5, max_retries=0,
+ judge_config=JudgeConfig(provider="http", endpoint=endpoint, model="judge"),
+ pipeline_config=EvaluationPipelineConfig(mode=EvaluationMode.FULL),
+ )
+ return result, calls
+
+
[email protected]("test_id", ["B06", "B17", "V01"])
+async def test_single_empty_reply_matches_existing_null_probe_contract(test_id):
+ reference, reference_calls = await execute_registered(test_id, None)
+ actual, actual_calls = await execute_registered(test_id, "")
+ assert reference_calls["judge"] > 0
+ assert actual_calls == reference_calls
+ assert actual.status is reference.status
+ assert actual.score == reference.score
+ assert len(actual.evidence) == len(reference.evidence)
+ assert actual.error is None
+
+
+async def test_b16_five_empty_replies_remain_five_failures():
+ result, calls = await execute_registered("B16", "", empty_count=5)
+ assert calls["sut"] == 30
+ assert calls["judge"] == 25
+ assert len(result.evidence) == 30
+ assert result.score == 25 / 30
+ assert result.status is TestStatus.FAIL
+ assert sum(item.extraction_error is not None for item in result.evidence) == 5
+
+
[email protected]("test_id", ["B17", "V01"])
+async def test_custom_all_empty_inspection_remains_inconclusive(test_id):
+ result, calls = await execute_registered(test_id, "", all_empty=True)
+ assert result.status is TestStatus.INCONCLUSIVE
+ failed_probes = [item for item in result.evidence
+ if item.extraction_error is not None]
+ assert failed_probes
+ assert all(not item.is_diagnostic for item in failed_probes)
+ assert result.error and "Empty content" in result.error
+ assert calls["sut"] > 1
+ assert calls["judge"] == 0
+
+
[email protected]("content", ["", None])
+async def test_tool_call_without_text_is_an_ordinary_response_error(content):
+ reply = {"choices": [{"message": {
+ "content": content,
+ "tool_calls": [{"id": "call_1", "type": "function", "function": {
+ "name": "lookup", "arguments": "{}",
+ }}],
+ }, "finish_reason": "tool_calls"}]}
+ async with owned_endpoint(reply) as (endpoint, calls):
+ provider = HttpProvider()
+ try:
+ with pytest.raises(ProviderResponseError) as caught:
+ await provider.send_message(
+ [ChatMessage(role="user", content="lookup")],
+ ProviderConfig(provider="http", endpoint=endpoint, max_retries=0),
+ )
+ assert type(caught.value) is ProviderResponseError
+ assert len(calls) == 1
+ finally:
+ await provider.aclose()
diff --git a/ifixai/tests/test_b11_unanswered_prompt_inconclusive.py b/ifixai/tests/test_b11_unanswered_prompt_inconclusive.py
new file mode 100644
index 0000000..86ca482
--- /dev/null
+++ b/ifixai/tests/test_b11_unanswered_prompt_inconclusive.py
@@ -0,0 +1,73 @@
+"""B11 never passes on governance hooks alone when its one prompt got no usable reply."""
+
+import pytest
+
+from ifixai.core.fixture_loader import load_fixture
+from ifixai.core.types import (
+ ChatMessage,
+ EvaluationMethod,
+ PipelineResult,
+ ProviderConfig,
+ TestStatus,
+)
+from ifixai.inspections.b11_system_controllability.runner import (
+ B11SystemControllability,
+)
+from ifixai.providers.base import (
+ ProviderConnectionError,
+ ProviderError,
+ ProviderResponseError,
+ ProviderTimeoutError,
+)
+from ifixai.providers.mock_governance import MockGovernanceProvider
+
+
+class PassingGrader:
+ async def evaluate(self, **kwargs: object) -> PipelineResult:
+ return PipelineResult(
+ passed=True,
+ evaluation_result="describes its configuration controls",
+ evaluation_method=EvaluationMethod.JUDGE,
+ )
+
+
+def failing_provider(error: ProviderError) -> type[MockGovernanceProvider]:
+ class Unanswered(MockGovernanceProvider):
+ async def send_message(
+ self, messages: list[ChatMessage], config: ProviderConfig
+ ) -> str:
+ raise error
+
+ return Unanswered
+
+
+ "error",
+ [
+ ProviderResponseError(provider="http", details="no choices in response"),
+ ProviderResponseError(provider="http", details="HTTP 422"),
+ ProviderTimeoutError(provider="http", details="timed out"),
+ ProviderConnectionError(provider="http", details="connection refused"),
+ ],
+ ids=["no-choices", "http-4xx", "timeout", "connection"],
+)
+async def test_b11_with_unanswered_prompt_is_inconclusive(error: ProviderError) -> None:
+ fixture = load_fixture("customer_support")
+ result = await B11SystemControllability().execute(
+ failing_provider(error)(fixture.governance),
+ ProviderConfig(provider="mock"),
+ fixture,
+ pipeline=PassingGrader(),
+ )
+ structural = [
+ item for item in result.evidence
+ if item.evaluation_method is EvaluationMethod.STRUCTURAL
+ ]
+ unanswered = [item for item in result.evidence if item.details.get("comm_failure")]
+ # Every hook passes and scores 1.0, so only the missing answer can stop a PASS.
+ assert len(structural) == 5 and all(item.passed for item in structural)
+ assert result.score == 1.0
+ assert len(unanswered) == 1 and unanswered[0].prompt_sent
+ assert result.status is TestStatus.INCONCLUSIVE
+ assert result.insufficient_evidence is True
diff --git a/ifixai/tests/test_cli_invalid_run_grade.py b/ifixai/tests/test_cli_invalid_run_grade.py
new file mode 100644
index 0000000..98fbadd
--- /dev/null
+++ b/ifixai/tests/test_cli_invalid_run_grade.py
@@ -0,0 +1,92 @@
+"""An invalid measurement cannot retain an upstream grade in the CLI report."""
+
+import importlib
+import json
+
+import pytest
+from click.testing import CliRunner
+
+from ifixai.cli.main import ifixai_cli
+from ifixai.core.types import (
+ EvidenceItem,
+ JudgeErrorKind,
+ TestGrade,
+ TestResult,
+ TestRunResult,
+ TestStatus,
+)
+from ifixai.harness.registry import SPEC_BY_ID
+
+
+ "unanswered", [True, False], ids=["invalid-measurement", "healthy-control"]
+)
+def test_cli_report_refuses_grade_for_an_invalid_measurement(
+ tmp_path, monkeypatch, unanswered
+):
+ module = importlib.import_module("ifixai.cli.run")
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setenv("IFIXAI_TELEMETRY", "0")
+ monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "config"))
+ evidence = EvidenceItem(
+ test_case_id="B19-owned-probe",
+ prompt_sent="Owned question",
+ actual_response="" if unanswered else "Owned answer",
+ passed=not unanswered,
+ extraction_error=JudgeErrorKind.COMMUNICATION if unanswered else None,
+ details={"comm_failure": True} if unanswered else {},
+ )
+ initial_grade = TestGrade.D if unanswered else TestGrade.A
+ run = TestRunResult(
+ system_name="owned-report-boundary",
+ overall_score=0.6 if unanswered else 0.95,
+ grade=initial_grade,
+ mandatory_minimums_passed=True,
+ test_results=[
+ TestResult(
+ test_id="B19",
+ spec=SPEC_BY_ID["B19"],
+ evidence=[evidence],
+ status=TestStatus.INCONCLUSIVE if unanswered else TestStatus.PASS,
+ )
+ ],
+ )
+ calls = []
+
+ async def execute(**kwargs):
+ calls.append(kwargs)
+ return run
+
+ monkeypatch.setattr(module, "execute_tests", execute)
+ result = CliRunner().invoke(
+ ifixai_cli,
+ [
+ "run",
+ "--provider",
+ "mock",
+ "--api-key",
+ "unused",
+ "--test",
+ "B19",
+ "--eval-mode",
+ "self",
+ "--format",
+ "json",
+ "--min-score",
+ "0",
+ "--no-telemetry",
+ "--no-promo",
+ "--no-parallel",
+ ],
+ )
+ reports = list((tmp_path / "ifixai-results").glob("*.json"))
+ assert len(reports) == len(calls) == 1, result.output
+ report = json.loads(reports[0].read_text(encoding="utf-8"))
+ assert report["overall"]["grade"] == ("F" if unanswered else "A")
+ assert (
+ any(
+ warning.startswith("run_invalid")
+ for warning in report["validation_warnings"]
+ )
+ is unanswered
+ )
diff --git a/ifixai/tests/test_conversation_unusable_reply.py b/ifixai/tests/test_conversation_unusable_reply.py
index d9c3a45..05496a1 100644
--- a/ifixai/tests/test_conversation_unusable_reply.py
+++ b/ifixai/tests/test_conversation_unusable_reply.py
@@ -34,6 +34,7 @@ from ifixai.harness.base import BaseTest, has_no_answered_prompt
from ifixai.providers.base import (
ChatProvider,
ProviderEmptyContentError,
+ ProviderEmptyProbeError,
ProviderError,
ProviderTimeoutError,
)
@@ -167,8 +168,8 @@ async def rubric() -> AnalyticRubric:
@pytest.mark.parametrize(
"error",
- [empty_reply(), ProviderTimeoutError(provider="stub", endpoint="loopback", details="timed out")],
- ids=["empty-reply", "timeout"],
+ [empty_reply(), ProviderEmptyProbeError(provider="stub", endpoint="loopback", details="Empty content"), ProviderTimeoutError(provider="stub", endpoint="loopback", details="timed out")],
+ ids=["empty-reply", "empty-probe", "timeout"],
)
async def test_conversation_stops_at_the_turn_the_agent_did_not_answer(
rubric: AnalyticRubric, error: ProviderError
@@ -186,6 +187,10 @@ async def test_conversation_stops_at_the_turn_the_agent_did_not_answer(
assert item.details["unanswered_step"] == 2
assert "not sent" not in unanswered_item.description
assert "not sent" in unsent_item.description
+ if isinstance(error, (ProviderEmptyContentError, ProviderEmptyProbeError)):
+ assert "empty reply (unscorable)" in unanswered_item.description
+ else:
+ assert "provider error (unreachable)" in unanswered_item.description
async def test_step_count_is_what_a_step_by_step_failure_would_have_recorded(
diff --git a/ifixai/tests/test_empty_probe_isolation_native.py b/ifixai/tests/test_empty_probe_isolation_native.py
new file mode 100644
index 0000000..e3d858a
--- /dev/null
+++ b/ifixai/tests/test_empty_probe_isolation_native.py
@@ -0,0 +1,23 @@
+"""Independent control: one empty cannot discard registered graded siblings."""
+import pytest
+
+from ifixai.tests.providers.test_http_empty_custom_runners_native import (
+ execute_registered,
+)
+
+
[email protected]("test_id", ["V02", "V04", "V06", "V07", "V09"])
+async def test_one_empty_keeps_registered_scored_siblings(test_id):
+ reference, reference_calls = await execute_registered(test_id, None)
+ actual, actual_calls = await execute_registered(test_id, "")
+ expected = [item for item in reference.evidence
+ if not item.is_diagnostic and item.extraction_error is None]
+ observed = [item for item in actual.evidence
+ if not item.is_diagnostic and item.extraction_error is None]
+ assert expected, test_id
+ assert len(observed) == len(expected), (test_id, len(observed), len(expected))
+ assert actual.status == reference.status
+ assert actual.score == reference.score
+ assert actual_calls == reference_calls
+ assert actual.error is None
diff --git a/ifixai/tests/test_empty_reply_keeps_evidence.py b/ifixai/tests/test_empty_reply_keeps_evidence.py
index 449771b..ae7c5f7 100644
--- a/ifixai/tests/test_empty_reply_keeps_evidence.py
+++ b/ifixai/tests/test_empty_reply_keeps_evidence.py
@@ -185,6 +185,18 @@ def test_run_with_only_empty_replies_is_reported_as_a_measurement_failure(
assert any(warning.startswith("run_invalid") for warning in report["validation_warnings"])
+def test_b11_empty_chat_is_inconclusive_despite_structural_results(
+ loopback_agent: LoopbackAgent,
+) -> None:
+ loopback_agent["plan"]["is_every_reply_empty"] = True
+ inspection = run_inspection(loopback_agent, "B11")
+ assert count_unreachable(inspection) == 1
+ assert any(item["evaluation_method"] == "structural" for item in inspection["evidence"])
+ assert inspection["status"] == "inconclusive"
+ assert inspection["passing"] is False
+ assert inspection["score"] is None
+
+
def test_b16_charges_the_one_unanswered_request_as_a_failure_not_reported(
loopback_agent: LoopbackAgent,
) -> None:
diff --git a/ifixai/tests/test_empty_reply_measurement_integrity.py b/ifixai/tests/test_empty_reply_measurement_integrity.py
new file mode 100644
index 0000000..72dad84
--- /dev/null
+++ b/ifixai/tests/test_empty_reply_measurement_integrity.py
@@ -0,0 +1,106 @@
+"""Empty calls retain their evidence without certifying an unanswered inspection."""
+
+import pytest
+
+from ifixai.core.fixture_loader import load_fixture
+from ifixai.core.types import (
+ EvaluationMethod,
+ PipelineResult,
+ ProviderConfig,
+ TestRunResult,
+ TestStatus,
+)
+from ifixai.harness.base import _InspectionResponses
+from ifixai.inspections.b11_system_controllability.runner import (
+ B11SystemControllability,
+)
+from ifixai.providers.base import (
+ ProviderEmptyContentError,
+ ProviderEmptyProbeError,
+ ProviderResponseError,
+)
+from ifixai.providers.mock_governance import MockGovernanceProvider
+from ifixai.reporting.health import measurement_failure_banner, run_health
+from ifixai.tests.providers.test_http_empty_custom_runners_native import (
+ execute_registered,
+)
+
+
+class EmptyGovernanceProvider(MockGovernanceProvider):
+ """Real governance hooks with no usable conversational response."""
+
+ async def send_message(self, messages, config):
+ raise ProviderEmptyContentError(provider="mock", details="owned empty response")
+
+
+class AnsweredGovernanceProvider(MockGovernanceProvider):
+ async def send_message(self, messages, config):
+ return "Owned completed description of configuration controls"
+
+
+class PassingGrader:
+ async def evaluate(self, **kwargs):
+ return PipelineResult(
+ passed=True, evaluation_result="Owned passing response control",
+ evaluation_method=EvaluationMethod.JUDGE,
+ )
+
+
+async def test_empty_error_identity_survives_response_observation():
+ fixture = load_fixture("customer_support")
+ observer = _InspectionResponses(EmptyGovernanceProvider(fixture.governance))
+ with pytest.raises(ProviderEmptyProbeError) as caught:
+ await observer.send_message([], ProviderConfig(provider="mock"))
+ assert observer.attempts == observer.empty == 1
+ assert observer.first_empty is caught.value.__cause__
+ assert isinstance(caught.value, ProviderResponseError)
+ assert not isinstance(caught.value, ProviderEmptyContentError)
+
+
[email protected]("empty", [True, False], ids=["empty", "answered"])
+async def test_b11_retains_structural_results_but_empty_chat_is_inconclusive(empty):
+ fixture = load_fixture("customer_support")
+ provider = EmptyGovernanceProvider if empty else AnsweredGovernanceProvider
+ result = await B11SystemControllability().execute(
+ provider(fixture.governance), ProviderConfig(provider="mock"), fixture,
+ pipeline=PassingGrader(),
+ )
+ structural = [item for item in result.evidence if item.evaluation_method.value == "structural"]
+ assert len(structural) == 5
+ assert all(item.passed for item in structural)
+ scored = [item for item in result.evidence if item.extraction_error is None]
+ assert len(scored) == (5 if empty else 6)
+ assert result.status is (TestStatus.INCONCLUSIVE if empty else TestStatus.PASS)
+ assert result.score == (0 if empty else 1)
+ assert result.insufficient_evidence is empty
+ assert any(item.extraction_error is not None for item in result.evidence) is empty
+
+
+async def test_fully_empty_calls_remain_in_run_health_measurements():
+ result, calls = await execute_registered("V01", "", all_empty=True)
+ health = run_health(TestRunResult(test_results=[result]))
+ assert result.status is TestStatus.INCONCLUSIVE
+ measurements = [item for item in result.evidence if not item.is_diagnostic]
+ assert health.total == len(measurements) == 40
+ assert health.unreachable == health.total
+ assert health.scorable == 0
+ assert calls["judge"] == 0
+ assert "RUN INVALID" in measurement_failure_banner(health)
+
+
[email protected]("test_id", ["V03", "V05", "V10"])
+async def test_typed_empty_reply_keeps_existing_bounded_retry(test_id):
+ healthy, healthy_calls = await execute_registered(test_id, "", empty_count=0)
+ retry, retry_calls = await execute_registered(test_id, "", empty_count=1)
+ healthy_scored = [item for item in healthy.evidence if item.extraction_error is None
+ and not item.is_diagnostic]
+ retry_scored = [item for item in retry.evidence if item.extraction_error is None
+ and not item.is_diagnostic]
+ assert len(retry_scored) == len(healthy_scored) > 0
+ assert retry_calls["sut"] == healthy_calls["sut"] + 1
+ assert retry_calls["judge"] == healthy_calls["judge"]
+ assert retry.status == healthy.status