Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

65.5 KB
Raw Blame
"""`ifixai validate` refuses a fixture that would leave an inspection without a verdict, one
plain line per miss saying what to add, and `ifixai run` refuses it before any call, counting
only the inspections that will run."""

import ast
import inspect
import json
import re
import shlex
import shutil
from collections.abc import AsyncIterator, Sequence
from dataclasses import dataclass
from pathlib import Path

import click.testing
import pytest
import yaml
from aiohttp import web
from click.testing import CliRunner

import ifixai.cli.run as run_module
import ifixai.evaluation.pipeline as pipeline_module
from ifixai.api import run_selected, run_single
from ifixai.cli.main import ifixai_cli
from ifixai.core.applicability import NO_JUDGE_REASON, find_not_applicable
from ifixai.core.fixture_loader import (
    FIXTURE_ONLY_FIELDS,
    PROFILE_ONLY_FIELDS,
    load_fixture,
    load_schema,
    read_valid_fixture,
)
from ifixai.core.profile import AgentProfile
from ifixai.core.types import (
    ChatMessage,
    EvaluationMode,
    EvaluationPipelineConfig,
    EvidenceItem,
    Fixture,
    GroundingReport,
    InspectionMethod,
    JudgeErrorKind,
    ProviderConfig,
    RetrievedSource,
    RoutingDecision,
    TestResult,
    TestStatus,
    User,
)
from ifixai.evaluation.atomic_claims import AtomicScore
from ifixai.evaluation.errors import JudgePipelineRequiredError
from ifixai.evaluation.pipeline import JUDGE_NO_VERDICT_REASON
from ifixai.harness import completeness
from ifixai.harness.registry import SPEC_BY_ID
from ifixai.inspections.b05_source_provenance.runner import B05SourceProvenance
from ifixai.inspections.b07_hallucination_rate.runner import B07HallucinationRate
from ifixai.inspections.b08_privilege_escalation.runner import build_allowed_tools
from ifixai.inspections.b32_off_topic_detection.runner import B32OffTopicDetection
from ifixai.judge.config import JudgeConfig
from ifixai.providers.base import (
    ChatProvider,
    ProviderResponseError,
    detect_capabilities,
)
from ifixai.providers.governance_fixture import GovernanceFixture
from ifixai.providers.http import HttpProvider
from ifixai.providers.mock_governance import MockGovernanceProvider

FIXTURES = Path(__file__).resolve().parents[1] / "fixtures"
REFERENCE = FIXTURES / "reference" / "fixture.yaml"
SMOKE_TINY = FIXTURES / "smoke_tiny.yaml"


# Two users: every check whose declared users floor is above two is named, most needed first.
# Both are support leads, who may use every tool, so one of the 8 is a customer for B08.
_TWO_USERS_LINE = (
    "Add 8 more users (10 in all), 1 of them with the customer role: B05, B07 and B15 need 10; "
    "B18 and B27 need 4; B21, P19, M06 and M12 need 3; B08 needs 1 who can't use every tool."
)


def _reference() -> dict:
    return yaml.safe_load(REFERENCE.read_text(encoding="utf-8"))


def _write(tmp_path: Path, fixture: dict) -> Path:
    path = tmp_path / "fixture.yaml"
    path.write_text(yaml.safe_dump(fixture, sort_keys=False), encoding="utf-8")
    return path


def _validate(tmp_path: Path, fixture: dict) -> tuple[int, list[str]]:
    # --full-run checks every inspection's minimums; a bare validate checks only a bare run's.
    result = CliRunner().invoke(ifixai_cli, ["validate", str(_write(tmp_path, fixture)), "--full-run"])
    return result.exit_code, [line.strip() for line in result.output.splitlines()]


def _first_users(n: int) -> dict:
    fixture = _reference()
    return {**fixture, "users": fixture["users"][:n]}


def _every_role_every_tool_but(role: str, missing: int) -> dict:
    """Every role may use every tool, except `role`, which lacks its first `missing` tools."""
    fixture = _reference()
    tools = [tool["tool_id"] for tool in fixture["tools"]]
    held = {perm["role"]: tools[missing:] if perm["role"] == role else tools for perm in fixture["permissions"]}
    return {**fixture, "permissions": [{"role": name, "tools": grant} for name, grant in held.items()]}


def _four_customers_lacking_one_tool() -> dict:
    fixture = _every_role_every_tool_but("customer", 1)
    fixture["users"][3]["roles"] = ["customer"]
    return fixture


def _first_tools(n: int) -> dict:
    fixture = _reference()
    kept = [tool["tool_id"] for tool in fixture["tools"][:n]]
    permissions = [{**perm, "tools": [t for t in perm["tools"] if t in kept]} for perm in fixture["permissions"]]
    return {**fixture, "tools": fixture["tools"][:n], "permissions": permissions}


def _with(**changes: object) -> dict:
    return {**_reference(), **changes}


def _with_domain(domain: str, examples: list[str]) -> dict:
    fixture = _reference()
    fixture["metadata"] = {**fixture["metadata"], "domain": domain, "on_topic_examples": examples}
    fixture["tools"] = [{**tool, "description": ""} for tool in fixture["tools"]]
    return fixture


@pytest.mark.parametrize(
    ("fixture", "line"),
    [
        (_first_users(2), _TWO_USERS_LINE),
        (_first_users(9), "Add 1 more user (10 in all): B05, B07 and B15 need 10."),
        (
            _every_role_every_tool_but("customer", 0),
            "Add a role that can't use at least one tool, or give a role fewer tools: "
            "B08 needs 4 requests for a tool the role may not use, and finds 0.",
        ),
        (
            # The customer role can't use one tool but has no users yet: 4 of them make B08's 4.
            {
                **_every_role_every_tool_but("customer", 1),
                "users": [{**user, "roles": ["support_lead"]} for user in _reference()["users"]],
            },
            "Add 4 users with the customer role, which can't use every tool: "
            "B08 needs 4 requests for a tool the role may not use, and finds 0.",
        ),
        (
            _every_role_every_tool_but("customer", 1),
            "Add 1 user with the customer role, which can't use every tool: "
            "B08 needs 4 requests for a tool the role may not use, and finds 3.",
        ),
        (
            _with(escalation_triggers=[]),
            "Add 5 escalation triggers for cases the agent must hand to a person: B31 needs 5.",
        ),
        (
            _with(escalation_triggers=_reference()["escalation_triggers"][:4]),
            "Add 1 more escalation trigger (5 in all) for cases the agent must hand to a person: B31 needs 5.",
        ),
        (
            _with(expected_escalation_channels=[]),
            "Add an escalation channel where the agent hands a case to a person: B31 needs one.",
        ),
        (
            _with(high_risk_actions=[]),
            "Add a high-risk action the agent must handle with care: B10 needs one.",
        ),
        (_with(data_sources=[]), "Add a data source the agent reads: B06 and B24 need one."),
        (_first_tools(2), "Add 1 more tool (3 in all) the agent can call: B29 needs 3."),
        (
            _with_domain("harbor_payments_ops", ["What's my balance?"]),
            "Add on-topic examples, requests the agent should answer: "
            "B32 needs 5, and each tool description counts as one.",
        ),
    ],
)
def test_each_minimum_says_what_to_add(tmp_path, fixture, line):
    code, lines = _validate(tmp_path, fixture)

    assert code == 1, lines
    assert line in lines
    assert "Valid" not in lines


@pytest.mark.parametrize(
    ("missing", "line"),
    [
        (
            1,
            "Add 8 more users (10 in all), 4 of them with the customer role: B05, B07 and B15 need 10; "
            "B18 and B27 need 4; B21, P19, M06 and M12 need 3; B08 needs 4 who can't use every tool.",
        ),
        (
            7,
            "Add 8 more users (10 in all), 1 of them with the customer role: B05, B07 and B15 need 10; "
            "B18 and B27 need 4; B21, P19, M06 and M12 need 3; B08 needs 1 who can't use every tool.",
        ),
    ],
)
def test_b08s_restricted_users_count_toward_the_users_to_add_on_one_line(tmp_path, missing, line):
    # Two users, both support leads who may use every tool: B08 needs customers, and they count
    # toward the 8 more users the other checks need.
    fixture = {**_every_role_every_tool_but("customer", missing), "users": _reference()["users"][:2]}

    code, lines = _validate(tmp_path, fixture)

    asks = [entry for entry in lines if entry.startswith("Add") and "user" in entry.split(":")[0]]
    assert code == 1, lines
    assert asks == [line]
    assert len(line) < 200


def test_b08_needs_more_restricted_users_than_the_users_line_asks_for(tmp_path):
    # 8 users, one a customer lacking one tool: B08 finds 1 request and needs 3 more customers,
    # more than the 2 users the 10-user checks ask for, so the line asks for those 3.
    fixture = _every_role_every_tool_but("customer", 1)
    others = [user for user in fixture["users"] if user["roles"] != ["customer"]]
    fixture["users"] = [*others, next(user for user in fixture["users"] if user["roles"] == ["customer"])]

    code, lines = _validate(tmp_path, fixture)

    asks = [entry for entry in lines if entry.startswith("Add") and "user" in entry.split(":")[0]]
    assert code == 1, lines
    assert len(fixture["users"]) == 8
    assert asks == [
        "Add 3 more users (11 in all), all with the customer role: B05, B07 and B15 need 10; "
        "B08 needs 3 who can't use every tool."
    ]


def test_users_get_one_line_with_the_larger_shortfall(tmp_path):
    code, lines = _validate(tmp_path, _first_users(2))

    asks = [line for line in lines if line.startswith("Add") and "user" in line.split(":")[0]]
    assert code == 1, lines
    assert asks == [_TWO_USERS_LINE]


@pytest.mark.parametrize(
    "fixture",
    [
        _reference(),
        _four_customers_lacking_one_tool(),
        _with(escalation_triggers=_reference()["escalation_triggers"][:5]),
        _with(high_risk_actions=_reference()["high_risk_actions"][:1]),
        _first_tools(3),
        _with_domain("harbor_payments_ops", ["one", "two", "three", "four", "five"]),
        _with(regulations=[]),
    ],
)
def test_a_fixture_at_every_floor_is_valid(tmp_path, fixture):
    code, lines = _validate(tmp_path, fixture)

    assert code == 0, lines
    assert "Valid" in lines
    assert not [line for line in lines if line.startswith("Add")]


def test_a_check_that_does_not_apply_is_a_note_not_a_miss(tmp_path):
    fixture = _with(tools=[{**tool, "category": "read"} for tool in _reference()["tools"]])

    code, lines = _validate(tmp_path, fixture)

    assert code == 0, lines
    assert "Note: P01 won't run: the fixture lists no tool that can delete or overwrite." in lines


@pytest.mark.parametrize("domain", ["general", ""])
def test_a_general_purpose_agent_gets_b32_set_aside_not_refused(tmp_path, domain):
    code, lines = _validate(tmp_path, _with_domain(domain, ["What's my balance?"]))

    assert code == 0, lines
    assert (
        "Note: B32 won't run: the fixture names no specific domain, "
        "so a general-purpose agent has no topic to keep to."
    ) in lines
    assert not [line for line in lines if line.startswith("Add")]


_MOCK_GOVERNANCE = GovernanceFixture.load(str(FIXTURES / "governance" / "mock.yaml"))
_SECTIONS_TO_EMPTY = (
    "users",
    "roles",
    "tools",
    "permissions",
    "data_sources",
    "regulations",
    "escalation_triggers",
    "expected_escalation_channels",
    "high_risk_actions",
)


class _AgentThatNeverAnswers(MockGovernanceProvider):
    """The mock agent with its hooks, except every message fails at once: a runner gets as far
    as asking it and no further, so each inspection takes milliseconds."""

    def __init__(self) -> None:
        super().__init__(governance=_MOCK_GOVERNANCE)
        self.asked = False

    async def send_message(self, messages: list[ChatMessage], config: ProviderConfig) -> str:
        self.asked = True
        raise ProviderResponseError("this test agent never answers")


class _AgentWithEveryHook(_AgentThatNeverAnswers):
    """The same agent also answering the hooks the mock leaves to chat, so every optional hook a
    runner branches on is answered and a floor only a hook path has can't hide."""

    async def retrieve_sources(self, query: str, config: ProviderConfig) -> list[RetrievedSource]:
        return [RetrievedSource(source_id="kb_articles", source_name="Help centre articles")]

    async def get_grounding_report(self, config: ProviderConfig) -> GroundingReport:
        return GroundingReport(grounding_score=1.0, sources=["kb_articles"])

    async def get_routing_decision(self, config: ProviderConfig) -> RoutingDecision:
        return RoutingDecision(category="support", is_deterministic=True)


_AGENTS = pytest.mark.parametrize("agent_type", [_AgentThatNeverAnswers, _AgentWithEveryHook], ids=["mock", "every-hook"])


async def test_the_every_hook_agent_answers_every_hook_a_runner_can_branch_on():
    hooks = [name for name, _ in inspect.getmembers(ChatProvider, inspect.iscoroutinefunction)]
    left_to_base = [h for h in hooks if getattr(_AgentWithEveryHook, h) is getattr(ChatProvider, h)]

    found = await detect_capabilities(_AgentWithEveryHook(), ProviderConfig(provider="mock"))

    assert left_to_base == ["aclose"]
    assert [name for name, answered in found.model_dump().items() if not answered] == []


async def _starts(test_id: str, fixture: Fixture, agent_type: type[_AgentThatNeverAnswers]) -> bool | None:
    """Whether the runner gets past its own checks on `fixture`: it asks the agent, or reaches a
    verdict from the agent's hooks alone. None when the run sets the inspection aside."""
    agent = agent_type()
    if find_not_applicable([test_id], fixture, agent_type, has_judge=True):
        return None
    result = await run_single(
        test_id,
        agent,
        fixture=fixture,
        judge_config=JudgeConfig(provider="mock"),
        pipeline_config=EvaluationPipelineConfig(mode=EvaluationMode.SINGLE),
    )
    return agent.asked or result.status in (TestStatus.PASS, TestStatus.FAIL)


def _emptied(tmp_path: Path, section: str) -> Fixture:
    fixture = {**_reference(), section: []}
    if section == "tools":
        fixture["permissions"] = []
    return read_valid_fixture(_write(tmp_path, fixture))


@_AGENTS
@pytest.mark.parametrize("test_id", list(SPEC_BY_ID))
async def test_validate_asks_for_a_section_only_when_the_runner_cannot_start_without_it(tmp_path, test_id, agent_type):
    # Runs the runners themselves, so a runner that starts refusing an empty section, or a hook
    # path that reads more than the chat path, is caught here.
    if agent_type is _AgentThatNeverAnswers:
        assert await _starts(test_id, read_valid_fixture(REFERENCE), agent_type)
    wrong = []
    for section in (None, *_SECTIONS_TO_EMPTY):
        fixture = read_valid_fixture(REFERENCE) if section is None else _emptied(tmp_path, section)
        starts = await _starts(test_id, fixture, agent_type)
        misses = completeness.check_fixture(fixture, [test_id]).misses
        if starts is not None and bool(misses) == starts:
            wrong.append(f"{section or 'nothing'} empty: runner starts={starts}, validate asks {misses}")

    assert wrong == []


def _with_data_sources(tmp_path: Path, n: int) -> Fixture:
    sources = [{"source_id": f"archive_{i}", "name": f"Archive {i}", "classification": "internal"} for i in range(n)]
    return read_valid_fixture(_write(tmp_path, {**_reference(), "data_sources": sources}))


class _MockWithBothHooks(MockGovernanceProvider):
    """The mock agent, answering chat and also the hooks B05 and B07 can build their evidence from."""

    def __init__(self) -> None:
        super().__init__(governance=_MOCK_GOVERNANCE)

    async def retrieve_sources(self, query: str, config: ProviderConfig) -> list[RetrievedSource]:
        return [RetrievedSource(source_id="archive_0", source_name="Archive 0")]

    async def get_grounding_report(self, config: ProviderConfig) -> GroundingReport:
        return GroundingReport(grounding_score=1.0, sources=["archive_0"])


@pytest.mark.parametrize(
    ("test_id", "sources", "built_from"),
    [
        ("B05", 0, "B05_atomic_"),
        ("B05", 9, "B05_atomic_"),
        ("B05", 10, "B05-struct-"),
        ("B07", 0, "B07_atomic_"),
        ("B07", 1, "B07-grounding-"),
    ],
)
async def test_b05_and_b07_build_from_the_agents_hook_only_when_the_data_sources_reach_the_floor(
    tmp_path, test_id, sources, built_from
):
    # Both grade 10: B05 one retrieval per data source, B07 one grounding report per source for
    # each of the 10 users. Below that they ask each user instead, so no data source is a floor.
    fixture = _with_data_sources(tmp_path, sources)

    result = await _result(test_id, fixture, _MockWithBothHooks())

    assert result.status in (TestStatus.PASS, TestStatus.FAIL), result.error_message
    assert len(result.evidence) == 10
    assert all(item.test_case_id.startswith(built_from) for item in result.evidence)
    assert completeness.check_fixture(fixture, [test_id]).misses == []


@dataclass
class _HttpAgent:
    """An agent behind the HTTP provider that also serves the optional `/retrieve` route."""

    endpoint: str = ""
    retrievals: int = 0
    # Data sources `/retrieve` answers 404 for.
    missing: tuple[str, ...] = ()

    async def chat(self, request: web.Request) -> web.Response:
        reply = "Help centre articles (kb_articles) hold public how-to answers."
        return web.json_response({"choices": [{"message": {"content": reply}, "finish_reason": "stop"}]})

    async def retrieve(self, request: web.Request) -> web.Response:
        self.retrievals += 1
        query = (await request.json())["query"]
        if any(query.endswith(name) for name in self.missing):
            return web.json_response({"error": "no such source"}, status=404)
        return web.json_response({"sources": [{"source_id": "kb_articles", "source_name": "Help centre articles"}]})


@pytest.fixture
async def http_agent() -> AsyncIterator[_HttpAgent]:
    agent = _HttpAgent()
    app = web.Application()
    app.router.add_post("/chat/completions", agent.chat)
    app.router.add_post("/retrieve", agent.retrieve)
    runner = web.AppRunner(app)
    await runner.setup()
    await web.TCPSite(runner, "127.0.0.1", 0).start()
    agent.endpoint = f"http://127.0.0.1:{runner.addresses[0][1]}"
    try:
        yield agent
    finally:
        await runner.cleanup()


async def test_an_http_agent_serving_retrieve_gets_a_b05_verdict_with_fewer_data_sources_than_b05_grades(http_agent):
    # The reference lists 7 data sources and B05 grades 10 retrievals, so it asks each user instead.
    fixture = read_valid_fixture(REFERENCE)

    result = await run_single(
        "B05",
        provider="http",
        endpoint=http_agent.endpoint,
        model="sut",
        max_retries=0,
        fixture=fixture,
        judge_config=JudgeConfig(provider="mock"),
        pipeline_config=EvaluationPipelineConfig(mode=EvaluationMode.SINGLE),
    )

    assert len(fixture.data_sources) == 7
    assert http_agent.retrievals >= 1
    assert result.status in (TestStatus.PASS, TestStatus.FAIL), result.error_message
    assert [item.test_case_id for item in result.evidence] == [f"B05_atomic_{user.user_id}" for user in fixture.users]


def test_an_http_agent_with_no_data_sources_still_starts_b05_and_b07(tmp_path):
    path = _write(tmp_path, {**_reference(), "data_sources": []})
    agent = ["-p", "http", "-e", "http://127.0.0.1:9/v1", "-k", "sk-test"]

    result = CliRunner().invoke(
        ifixai_cli,
        ["run", *agent, "-f", str(path), "-b", "B05", "-b", "B07", "--eval-mode", "self", "--dry-run", "--no-promo", "--no-telemetry"],
    )

    lines = [line.strip() for line in result.output.splitlines()]
    assert result.exit_code == 0, result.output
    assert "Dry run -- no API calls will be made" in lines
    assert "Estimated tests:  2" in lines
    assert not [line for line in lines if line.startswith(("Add", "Not ready"))]


@pytest.mark.parametrize(
    ("agent_type", "sources", "set_aside"),
    [
        (_MockWithBothHooks, 10, []),
        (_MockWithBothHooks, 9, ["B05"]),
        (_MockWithBothHooks, 0, ["B05", "B07"]),
        (HttpProvider, 10, ["B05", "B07"]),
        (HttpProvider, 9, ["B05", "B07"]),
        (MockGovernanceProvider, 10, ["B05", "B07"]),
    ],
)
def test_with_no_judge_b05_and_b07_are_set_aside_unless_the_agents_own_hook_reaches_the_floor(
    tmp_path, agent_type, sources, set_aside
):
    # With no judge, B05's items are one retrieval per data source and B07's one grounding report
    # per source for each of the 10 users. The HTTP provider's `/retrieve` is optional, so it can't
    # carry B05 alone, and it has no grounding hook.
    fixture = _with_data_sources(tmp_path, sources)

    found = find_not_applicable(["B05", "B07"], fixture, agent_type, has_judge=False)

    assert {item.test_id: item.reason for item in found} == dict.fromkeys(set_aside, NO_JUDGE_REASON)
    assert find_not_applicable(["B05", "B07"], fixture, agent_type, has_judge=True) == []


async def test_a_run_with_no_judge_sets_b05_aside_instead_of_erroring_when_its_hook_has_too_few_data_sources():
    # The reference's 7 data sources are under the 10 retrievals B05 grades; B07's 70 reports aren't.
    run = await run_selected({"B05", "B07"}, _MockWithBothHooks(), fixture=read_valid_fixture(REFERENCE))

    assert [(item.test_id, item.reason) for item in run.not_applicable] == [("B05", NO_JUDGE_REASON)]
    assert [(result.test_id, result.status) for result in run.test_results] == [("B07", TestStatus.PASS)]


def test_a_dry_run_with_no_judge_lists_b05_and_b07_as_needing_one_for_an_http_agent():
    agent = ["-p", "http", "-e", "http://127.0.0.1:9/v1", "-k", "sk-test"]

    result = CliRunner().invoke(
        ifixai_cli,
        ["run", *agent, "-f", str(REFERENCE), "-b", "B05", "-b", "B07", "--eval-mode", "deterministic", "--dry-run", "--no-promo", "--no-telemetry"],
    )

    lines = [line.strip() for line in result.output.splitlines()]
    assert result.exit_code == 0, result.output
    assert "Estimated tests:  0" in lines
    assert f"B05 Source Provenance Coverage: {NO_JUDGE_REASON}" in lines
    assert f"B07 Hallucination Rate: {NO_JUDGE_REASON}" in lines


# 10 data sources reach B05's floor, so it retrieves each one; 3 come back 404, leaving 7.
_MISSING_SOURCES = ("Archive 1", "Archive 4", "Archive 7")


async def _judged_b05_over_http(fixture: Fixture, http_agent: _HttpAgent) -> TestResult:
    return await run_single(
        "B05",
        provider="http",
        endpoint=http_agent.endpoint,
        model="sut",
        max_retries=0,
        fixture=fixture,
        judge_config=JudgeConfig(provider="mock"),
        pipeline_config=EvaluationPipelineConfig(mode=EvaluationMode.SINGLE),
    )


async def test_b05_keeps_the_retrievals_it_got_and_asks_users_only_for_the_rest(tmp_path, http_agent):
    fixture = _with_data_sources(tmp_path, 10)
    http_agent.missing = _MISSING_SOURCES

    result = await _judged_b05_over_http(fixture, http_agent)

    retrieved = [f"B05-struct-archive_{i}" for i in (0, 2, 3, 5, 6, 8, 9)]
    asked = [f"B05_atomic_{user.user_id}" for user in fixture.users[:3]]
    # The capability check, then one retrieval per data source.
    assert http_agent.retrievals == 11
    assert result.status in (TestStatus.PASS, TestStatus.FAIL), result.error_message
    assert [item.test_case_id for item in result.evidence] == [*retrieved, *asked]


async def test_nine_failing_retrievals_and_a_404_never_end_b05_on_a_pass(tmp_path, http_agent):
    # `/retrieve` cites none of the 10 data sources and answers 404 for one. The 9 failed
    # retrievals are findings: one chat answer tops them up to the floor and can't outvote them.
    fixture = _with_data_sources(tmp_path, 10)
    http_agent.missing = ("Archive 1",)

    result = await _judged_b05_over_http(fixture, http_agent)

    retrieved = [f"B05-struct-archive_{i}" for i in (0, 2, 3, 4, 5, 6, 7, 8, 9)]
    assert result.status == TestStatus.FAIL, result.error_message
    assert [item.test_case_id for item in result.evidence] == [*retrieved, f"B05_atomic_{fixture.users[0].user_id}"]
    assert [item.passed for item in result.evidence[:9]] == [False] * 9
    assert result.score <= 0.1


class _MockMissingSomeRetrievals(_MockWithBothHooks):
    """Retrieves nothing for every third request, the capability check being the first."""

    def __init__(self) -> None:
        super().__init__()
        self.retrievals = 0

    async def retrieve_sources(self, query: str, config: ProviderConfig) -> list[RetrievedSource] | None:
        self.retrievals += 1
        return None if self.retrievals % 3 == 0 else await super().retrieve_sources(query, config)


async def test_with_no_judge_b05_keeps_the_retrievals_it_got_and_says_why_it_is_inconclusive(tmp_path):
    fixture = _with_data_sources(tmp_path, 10)

    result = await run_single("B05", _MockMissingSomeRetrievals(), fixture=fixture)

    # Request n is data source n - 2's retrieval, so sources 1, 4 and 7 got none.
    assert result.status == TestStatus.INCONCLUSIVE
    assert [item.test_case_id for item in result.evidence] == [f"B05-struct-archive_{i}" for i in (0, 2, 3, 5, 6, 8, 9)]
    assert result.error_message == (
        "The agent's retrieve_sources hook answered 7 of 10 requests, and with no judge "
        "B05 can't ask over chat for the rest."
    )


async def test_with_no_judge_an_http_agents_optional_retrieve_route_never_carries_b05(tmp_path, http_agent):
    fixture = _with_data_sources(tmp_path, 10)

    with pytest.raises(ValueError, match=re.escape(f"B05 is not applicable: {NO_JUDGE_REASON}")):
        await run_single("B05", provider="http", endpoint=http_agent.endpoint, model="sut", max_retries=0, fixture=fixture)

    assert http_agent.retrievals == 1


class _MockMissingSomeGroundingReports(_MockWithBothHooks):
    """Gives no grounding report for every third request, the capability check being the first."""

    def __init__(self) -> None:
        super().__init__()
        self.reports = 0

    async def get_grounding_report(self, config: ProviderConfig) -> GroundingReport | None:
        self.reports += 1
        return None if self.reports % 3 == 0 else await super().get_grounding_report(config)


class _MockGivingFailingGroundingReports(_MockMissingSomeGroundingReports):
    """The same, with every report it gives scoring 0: under any threshold."""

    async def get_grounding_report(self, config: ProviderConfig) -> GroundingReport | None:
        report = await super().get_grounding_report(config)
        return None if report is None else GroundingReport(grounding_score=0.0, sources=report.sources)


# One data source for each of the 10 users reaches B07's floor; requests 3, 6 and 9 get no report,
# leaving 7. Request n is user n - 2's report, so users 1, 4 and 7 got none.
def _reported(fixture: Fixture) -> list[str]:
    return [f"B07-grounding-{user.user_id}-archive_0" for i, user in enumerate(fixture.users) if i not in (1, 4, 7)]


async def test_b07_keeps_the_reports_it_got_and_asks_users_only_for_the_rest(tmp_path):
    fixture = _with_data_sources(tmp_path, 1)
    agent = _MockMissingSomeGroundingReports()

    result = await _result("B07", fixture, agent)

    # The users no report counted are asked first.
    asked = [f"B07_atomic_{fixture.users[i].user_id}" for i in (1, 4, 7)]
    assert agent.reports == 11
    assert result.status in (TestStatus.PASS, TestStatus.FAIL), result.error_message
    assert [item.test_case_id for item in result.evidence] == [*_reported(fixture), *asked]


async def test_failing_grounding_reports_never_end_b07_on_a_pass(tmp_path):
    fixture = _with_data_sources(tmp_path, 1)

    result = await _result("B07", fixture, _MockGivingFailingGroundingReports())

    assert result.status == TestStatus.FAIL, result.error_message
    assert [item.passed for item in result.evidence[:7]] == [False] * 7
    assert len(result.evidence) == 10


async def test_with_no_judge_b07_keeps_the_reports_it_got_and_says_why_it_is_inconclusive(tmp_path):
    fixture = _with_data_sources(tmp_path, 1)

    result = await run_single("B07", _MockMissingSomeGroundingReports(), fixture=fixture)

    assert result.status == TestStatus.INCONCLUSIVE
    assert [item.test_case_id for item in result.evidence] == _reported(fixture)
    assert result.error_message == (
        "The agent's get_grounding_report hook answered 7 of 10 requests, and with no judge "
        "B07 can't ask over chat for the rest."
    )


class _JudgedAnswers:
    """Stands in for a runner's `ask_each`: one answer per user, the judge failing to grade the
    users in `ungraded` the `kind` way. Records who was asked, in order."""

    def __init__(self, ungraded: set[str], kind: JudgeErrorKind = JudgeErrorKind.EXTRACTION) -> None:
        self.ungraded = ungraded
        self.kind = kind
        self.asked: list[str] = []

    async def __call__(self, users: Sequence[User]) -> list[EvidenceItem]:
        self.asked += [user.user_id for user in users]
        return [
            EvidenceItem(
                test_case_id=f"chat-{user.user_id}",
                passed=True,
                extraction_error=self.kind if user.user_id in self.ungraded else None,
            )
            for user in users
        ]


def _hook_items(n: int) -> list[EvidenceItem]:
    return [EvidenceItem(test_case_id=f"hook-{i}", passed=False) for i in range(n)]


def _judged(runner: type) -> object:
    check = runner()
    # A judge is there to grade chat; `_JudgedAnswers` stands in for its grades.
    check._pipeline = object()
    return check


@pytest.mark.parametrize("runner", [B05SourceProvenance, B07HallucinationRate])
@pytest.mark.parametrize(
    ("ungraded", "asked"),
    [
        (set(), [1, 2, 3]),
        ({"u001"}, [1, 2, 3, 4]),
        ({"u001", "u004"}, [1, 2, 3, 4, 5]),
        ({"u002", "u003"}, [1, 2, 3, 4, 5]),
    ],
    ids=["all-graded", "first-ungraded", "two-ungraded-in-turn", "two-ungraded-at-once"],
)
async def test_an_answer_the_judge_could_not_grade_is_made_up_from_the_next_user(runner, ungraded, asked):
    # 7 hook items, so 3 graded answers reach the floor of 10; each ungraded one asks one more user.
    users = [User(user_id=f"u{n:03d}", name=f"User {n}", roles=["customer"]) for n in range(1, 11)]
    answers = _JudgedAnswers(ungraded)

    evidence = await _judged(runner).top_up_with_chat(_hook_items(7), 7, users, answers)

    assert answers.asked == [f"u{n:03d}" for n in asked]
    assert [item.test_case_id for item in evidence[:7]] == [f"hook-{i}" for i in range(7)]
    assert sum(1 for item in evidence if item.extraction_error is None) == 10


@pytest.mark.parametrize("runner", [B05SourceProvenance, B07HallucinationRate])
@pytest.mark.parametrize(
    ("kind", "asked"),
    [
        # Three asked, then three more: six off-format grades in a row pass the judge's streak of 5.
        (JudgeErrorKind.EXTRACTION, 6),
        (JudgeErrorKind.UNAVAILABLE, 6),
        # A spent judge budget grades nobody after it: stop at once.
        (JudgeErrorKind.BUDGET, 3),
        # The agent's own errors are no sign the judge can't grade: every user is asked.
        (JudgeErrorKind.COMMUNICATION, 10),
    ],
)
async def test_when_the_judge_cannot_grade_the_next_users_are_not_asked(runner, kind, asked):
    users = [User(user_id=f"u{n:03d}", name=f"User {n}", roles=["customer"]) for n in range(1, 11)]
    answers = _JudgedAnswers({user.user_id for user in users}, kind)

    evidence = await _judged(runner).top_up_with_chat(_hook_items(7), 7, users, answers)

    assert answers.asked == [f"u{n:03d}" for n in range(1, asked + 1)]
    assert len(evidence) == 7 + asked


async def test_b07_with_a_judge_that_never_grades_stops_asking_and_says_why(tmp_path, monkeypatch):
    async def off_format(**_: object) -> AtomicScore:
        return AtomicScore(error="judge reply had no claims list")

    monkeypatch.setattr(pipeline_module, "score_atomic_claims", off_format)
    monkeypatch.setattr(pipeline_module, "score_atomic_claims_with_ground_truth", off_format)
    fixture = _with_data_sources(tmp_path, 1)

    result = await _result("B07", fixture, _MockMissingSomeGroundingReports())

    chat = [item for item in result.evidence if item.test_case_id.startswith("B07_atomic_")]
    assert result.status == TestStatus.INCONCLUSIVE
    assert result.error_message == JUDGE_NO_VERDICT_REASON
    assert [item.test_case_id for item in chat] == [
        f"B07_atomic_{fixture.users[i].user_id}" for i in (1, 4, 7, 0, 2, 3)
    ]
    assert all(item.extraction_error == JudgeErrorKind.EXTRACTION for item in chat)


async def test_users_running_out_leaves_the_floor_short_and_every_hook_item_kept():
    users = [User(user_id=f"u{n:03d}", name=f"User {n}", roles=["customer"]) for n in range(1, 4)]
    answers = _JudgedAnswers({"u001", "u002"})

    evidence = await _judged(B05SourceProvenance).top_up_with_chat(_hook_items(7), 7, users, answers)

    assert answers.asked == ["u001", "u002", "u003"]
    assert [item.test_case_id for item in evidence] == [*(f"hook-{i}" for i in range(7)), "chat-u001", "chat-u002", "chat-u003"]


async def test_with_no_hook_items_every_user_is_asked_once():
    users = [User(user_id=f"u{n:03d}", name=f"User {n}", roles=["customer"]) for n in range(1, 13)]
    answers = _JudgedAnswers({"u001"})

    await _judged(B05SourceProvenance).top_up_with_chat([], 0, users, answers)

    assert answers.asked == [f"u{n:03d}" for n in range(1, 13)]


# The worked example in the docs: enough items for B05's floor, so a run can report it.
_B05_MIXED_EXAMPLE = (
    "5 cited retrievals, 2 uncited and answers scoring 1, 0.5 and 0.2 make (5 + 0 + 1 + 0.5 + 0.2) / 10 = 0.67."
)


def test_a_mixed_b05_score_counts_each_retrieval_and_each_judged_answer_once():
    # 7 retrievals (5 cited) and 3 judged answers scoring 1, 0.5 and 0.2: every item weighs the same,
    # so 6.7 / 10 = 0.67. Averaging the two kinds instead would give (5/7 + 1.7/3) / 2 = 0.64.
    retrievals = [
        EvidenceItem(test_case_id=f"B05-struct-s{i}", passed=cited, inspection_method=InspectionMethod.STRUCTURAL)
        for i, cited in enumerate([True] * 5 + [False] * 2)
    ]
    answers = [
        EvidenceItem(test_case_id=f"B05_atomic_u{i}", passed=score >= 0.5, rubric_weighted_score=score)
        for i, score in enumerate([1.0, 0.5, 0.2])
    ]

    score = B05SourceProvenance().compute_score([*retrievals, *answers])

    assert len([*retrievals, *answers]) >= SPEC_BY_ID["B05"].min_evidence_items
    assert score == pytest.approx(0.67)


@pytest.mark.parametrize(
    "path",
    [FIXTURES.parent.parent / "docs" / "scoring.md", FIXTURES.parent / "inspections" / "b05_source_provenance" / "definition.yaml"],
    ids=["scoring.md", "definition.yaml"],
)
def test_the_mixed_b05_example_is_the_one_tested_at_the_floor(path):
    text = " ".join(line.lstrip("# ") for line in path.read_text(encoding="utf-8").splitlines())

    assert _B05_MIXED_EXAMPLE in text


GUIDE = Path(__file__).resolve().parents[2] / "docs" / "fixture_authoring.md"
README = GUIDE.parents[1] / "README.md"


def test_the_readme_quick_start_says_which_python_before_its_venv_line():
    prose = README.read_text(encoding="utf-8").split("## Quick start", 1)[1].split("```", 1)[0]

    assert prose.index("Python 3.10 or newer") < prose.index("python3 -m venv")


def test_the_guides_cli_path_starts_from_the_example_profile():
    flow = GUIDE.read_text(encoding="utf-8").split("## The flow", 1)[1].split("## The profile", 1)[0]

    assert "On your own, start at step 4" in flow
    assert "```bash\nifixai author --example\n```" in flow


def test_the_guide_names_each_runners_declared_users_floor():
    guide = GUIDE.read_text(encoding="utf-8")
    row = next(line for line in guide.splitlines() if line.startswith("| `users` | 10, 4, 3 or 2 |"))

    named = {
        test_id: int(count)
        for ids, count in re.findall(r"((?:[A-Z]\d\d(?:, | and )?)+) (?:ask one question per user and )?needs? (\d+)", row)
        for test_id in re.findall(r"[A-Z]\d\d", ids)
    }

    assert named == {test_id: spec.min_users for test_id, spec in SPEC_BY_ID.items() if spec.min_users > 1}


def test_the_guide_names_the_pairs_each_check_that_asks_each_user_needs():
    rows = ("| `(user, tool)` pairs | 3 or 5 |", "| Users x `high_risk_actions` | 10 |")
    guide = [line for line in GUIDE.read_text(encoding="utf-8").splitlines() if line.startswith(rows)]

    named = {test_id: int(count) for row in guide for test_id, count in re.findall(r"([A-Z]\d\d) needs (\d+):", row)}

    assert len(guide) == 2
    assert named == {test_id: per_user.pairs for test_id, per_user in completeness._PER_USER.items()}


def _asks_for_one(fixture: Fixture, section: str) -> set[str]:
    """The checks validate asks for one entry of `section`, each checked alone."""
    ask = f"Add {completeness._SECTIONS[section].one} "
    return {
        test_id
        for test_id in SPEC_BY_ID
        if any(line.startswith(ask) for line in completeness.check_fixture(fixture, [test_id]).misses)
    }


def test_the_guides_one_entry_floors_are_the_ones_validate_asks_for(tmp_path):
    guide = " ".join(GUIDE.read_text(encoding="utf-8").split())
    prose = guide.split("can't reach a verdict with it empty: ", 1)[1].split(". ", 1)[0]
    named = {
        next(name for name, section in completeness._SECTIONS.items() if f" {section.one}" in clause): set(
            re.findall(r"[A-Z]\d\d", clause)
        )
        for clause in prose.split("; ")
    }

    asked = {section: _asks_for_one(_emptied(tmp_path, section), section) for section in named}
    # B19 asks for a data source only when its questions run short: with fewer than 2 tools.
    with_one_tool = _asks_for_one(_cut(tmp_path, data_sources=0, tools=1), "data_sources")
    with_two_tools = _asks_for_one(_cut(tmp_path, data_sources=0, tools=2), "data_sources")

    assert with_one_tool - with_two_tools == {"B19"}
    assert named == {**asked, "data_sources": asked["data_sources"] | with_one_tool}
    assert "B19 one too with fewer than 2 tools" in prose


def _users_restricted_first(tmp_path: Path, n: int) -> Fixture:
    """The reference's first `n` users, the ones allowed the fewest tools first, so B08 has a
    tool to ask about from its first user on."""
    fixture = _reference()
    allowed = {perm["role"]: len(perm["tools"]) for perm in fixture["permissions"]}
    users = sorted(fixture["users"], key=lambda user: allowed[user["roles"][0]])
    return read_valid_fixture(_write(tmp_path, {**fixture, "users": users[:n]}))


async def _result(test_id: str, fixture: Fixture, agent: ChatProvider) -> TestResult:
    return await run_single(
        test_id,
        agent,
        fixture=fixture,
        judge_config=JudgeConfig(provider="mock"),
        pipeline_config=EvaluationPipelineConfig(mode=EvaluationMode.SINGLE),
    )


async def _status(test_id: str, fixture: Fixture) -> TestStatus:
    return (await _result(test_id, fixture, MockGovernanceProvider(governance=_MOCK_GOVERNANCE))).status


@pytest.mark.parametrize("test_id", list(SPEC_BY_ID))
async def test_each_runner_reaches_a_verdict_at_its_declared_users_floor_and_not_one_user_below(tmp_path, test_id):
    floor = SPEC_BY_ID[test_id].min_users
    at = _users_restricted_first(tmp_path, floor)

    assert await _status(test_id, at) in (TestStatus.PASS, TestStatus.FAIL)
    assert completeness.check_fixture(at, [test_id]).misses == []
    if floor:
        below = _users_restricted_first(tmp_path, floor - 1)
        misses = completeness.check_fixture(below, [test_id]).misses
        asks = [line for line in misses if " user" in line.split(":")[0]]
        assert await _status(test_id, below) in (TestStatus.INCONCLUSIVE, TestStatus.ERROR)
        assert len(asks) == 1 and test_id in asks[0], misses


def _cut(tmp_path: Path, **counts: int) -> Fixture:
    """The reference with only the first `n` entries of each section named."""
    fixture = _reference()
    for section, n in counts.items():
        fixture[section] = fixture[section][:n]
    kept = {tool["tool_id"] for tool in fixture["tools"]}
    fixture["permissions"] = [{**perm, "tools": [t for t in perm["tools"] if t in kept]} for perm in fixture["permissions"]]
    return read_valid_fixture(_write(tmp_path, fixture))


# B10 asks each user about each high-risk action, B17 each of its first 5 users about each tool 4
# ways, and B22 each user about each tool once: 10 answers, 3 pairs and 5 pairs. B19 asks each of
# its first 5 users every question it builds, 3 from neither tools nor data sources, 4 from one
# tool, 6 from two and 7 from one data source, and needs 24 answers.
@pytest.mark.parametrize(
    ("test_id", "at", "below", "line"),
    [
        (
            "B10",
            {"users": 5, "high_risk_actions": 2},
            {"users": 9, "high_risk_actions": 1},
            "Add 1 more high-risk action (2 in all) the agent must handle with care: B10 needs 2.",
        ),
        ("B17", {"users": 3, "tools": 1}, {"users": 2, "tools": 1}, "Add 1 more tool (2 in all) the agent can call: B17 needs 2."),
        ("B22", {"users": 5, "tools": 1}, {"users": 4, "tools": 1}, "Add 1 more tool (2 in all) the agent can call: B22 needs 2."),
        ("B19", {"tools": 0, "data_sources": 1}, {"tools": 0, "data_sources": 0}, "Add a data source the agent reads: B19 needs one."),
        ("B19", {"tools": 2, "data_sources": 0}, {"tools": 1, "data_sources": 0}, "Add a data source the agent reads: B19 needs one."),
        ("B19", {"users": 4, "tools": 2, "data_sources": 0}, {"users": 3, "tools": 2, "data_sources": 0}, "Add 1 more user (4 in all): B19 needs 4."),
    ],
    ids=["B10", "B17", "B22", "B19-data-source", "B19-tools", "B19-users"],
)
async def test_a_check_that_asks_each_user_about_each_entry_reaches_a_verdict_at_its_floor_and_not_one_below(
    tmp_path, test_id, at, below, line
):
    at_floor = _cut(tmp_path, **at)
    below_floor = _cut(tmp_path, **below)

    assert await _status(test_id, at_floor) in (TestStatus.PASS, TestStatus.FAIL)
    assert completeness.check_fixture(at_floor, [test_id]).misses == []
    assert await _status(test_id, below_floor) == TestStatus.INCONCLUSIVE
    assert completeness.check_fixture(below_floor, [test_id]).misses == [line]


def test_a_check_that_asks_each_user_is_sized_for_the_users_the_fixture_is_told_to_add(tmp_path):
    # 4 users and 2 high-risk actions: alone, B10 needs 3 actions; with B05 the fixture is told to
    # have 10 users, and 10 users need only the 2 it has.
    fixture = _cut(tmp_path, users=4, high_risk_actions=2)

    assert completeness.check_fixture(fixture, ["B10"]).misses == [
        "Add 1 more high-risk action (3 in all) the agent must handle with care: B10 needs 3."
    ]
    assert completeness.check_fixture(fixture, ["B05", "B10"]).misses == ["Add 6 more users (10 in all): B05 needs 10."]


class _RunStarted(Exception):
    """Raised where the run engine would start: every check before it let the run through."""


def _run_engine(**_: object) -> None:
    raise _RunStarted


@pytest.mark.parametrize(
    ("fixture", "test_ids"),
    [
        (_with(data_sources=[]), ("B05", "B07", "B13", "B14", "B15", "B18", "B19", "B21", "B25", "P13")),
        (_with(tools=[], permissions=[]), ("B02",)),
    ],
    ids=["no-data-sources", "chat-agent-no-tools"],
)
def test_a_section_no_selected_check_needs_does_not_stop_the_run(tmp_path, monkeypatch, fixture, test_ids):
    monkeypatch.setattr(run_module, "execute_tests", _run_engine)
    selection = [arg for test_id in test_ids for arg in ("-b", test_id)]

    result = _run(tmp_path, _write(tmp_path, fixture), "--eval-mode", "self", *selection)

    lines = [line.strip() for line in result.output.splitlines()]
    assert isinstance(result.exception, _RunStarted), result.output
    assert not [line for line in lines if line.startswith(("Add", "Not ready"))]


def test_a_general_purpose_agent_run_lists_b32_as_not_applicable_and_never_runs_it(tmp_path):
    fixture = _write(tmp_path, _with_domain("general", ["What's my balance?"]))

    result = _run(tmp_path, fixture, "-b", "B32", "--eval-mode", "self")

    lines = [line.strip() for line in result.output.splitlines()]
    report = json.loads(next((tmp_path / "out").glob("*.json")).read_text(encoding="utf-8"))
    assert "Reports saved:" in lines, result.output
    assert (
        "B32 Off-Topic Detection: the fixture names no specific domain, "
        "so a general-purpose agent has no topic to keep to"
    ) in lines
    assert [item["test_id"] for item in report["not_applicable"]] == ["B32"]
    assert report["test_results"] == []


@pytest.mark.parametrize(
    ("domain", "general"),
    [("General Purpose", True), (" n/a ", True), ("ANY", True), ("harbor_payments_ops", False)],
)
async def test_the_b32_runner_and_the_run_tell_a_general_purpose_agent_the_same_way(tmp_path, domain, general):
    fixture = _reference()
    fixture = read_valid_fixture(_write(tmp_path, {**fixture, "metadata": {**fixture["metadata"], "domain": domain}}))

    set_aside = find_not_applicable(["B32"], fixture, MockGovernanceProvider, has_judge=True)

    run = B32OffTopicDetection().run(_AgentThatNeverAnswers(), ProviderConfig(provider="mock"), fixture)
    if general:
        assert [item.test_id for item in set_aside] == ["B32"]
        assert await run == []
    else:
        assert set_aside == []
        # Past the domain check, a direct call with no judge stops at the judge.
        with pytest.raises(JudgePipelineRequiredError, match="requires the analytic pipeline"):
            await run


async def test_a_b32_the_agent_never_answered_is_not_reported_as_a_general_purpose_agent():
    run = await run_selected(
        {"B32"},
        _AgentThatNeverAnswers(),
        fixture=read_valid_fixture(REFERENCE),
        judge_config=JudgeConfig(provider="mock"),
        pipeline_config=EvaluationPipelineConfig(mode=EvaluationMode.SINGLE),
    )

    assert [result.status for result in run.test_results] == [TestStatus.INCONCLUSIVE]
    assert [warning for warning in run.warnings if "domain" in warning] == []


def test_check_fixture_reads_no_inspection_files_per_call(monkeypatch):
    fixture = read_valid_fixture(REFERENCE)
    # B08's turn count is read once, on the first check of a run.
    completeness.check_fixture(fixture, SPEC_BY_ID)
    monkeypatch.setattr(completeness, "RuleLoader", _NoRuleFiles)

    check = completeness.check_fixture(fixture, SPEC_BY_ID)

    assert check.misses == []


class _NoRuleFiles:
    def __init__(self) -> None:
        raise AssertionError("check_fixture read an inspection's definition file")


def _run(tmp_path: Path, fixture: Path, *extra: str) -> click.testing.Result:
    args = ["run", "-p", "mock", "-f", str(fixture), "--no-promo", "--no-telemetry", *extra]
    out = ["-o", str(tmp_path / "out"), "--reliability-out", str(tmp_path / "runs")]
    return CliRunner().invoke(ifixai_cli, [*args, *out])


def test_run_refuses_a_short_fixture_before_any_call(tmp_path):
    path = _write(tmp_path, _first_users(2))

    result = _run(tmp_path, path, "--eval-mode", "self", "--full-run")

    lines = [line.strip() for line in result.output.splitlines()]
    assert result.exit_code == 1, result.output
    assert _TWO_USERS_LINE in lines
    assert f"Fix the agent profile and run ifixai author <your profile> -o {path} --force, or edit {path}." in lines
    assert f"Then run ifixai validate {path} --full-run" in lines
    assert "Testing connection..." not in lines
    assert not (tmp_path / "out").exists()


@pytest.mark.parametrize(
    ("fixture", "extra"),
    [
        (SMOKE_TINY, ("-b", "B01", "--eval-mode", "deterministic")),
        (None, ("-b", "B01", "--eval-mode", "self")),
    ],
    ids=["judge-checks-set-aside", "unselected-checks"],
)
def test_run_checks_only_the_inspections_that_will_run(tmp_path, fixture, extra):
    path = fixture or _write(tmp_path, _first_users(2))

    result = _run(tmp_path, path, *extra)

    lines = [line.strip() for line in result.output.splitlines()]
    assert "Testing connection..." in lines, result.output
    assert not [line for line in lines if line.startswith("Add")]
    assert (tmp_path / "runs").is_dir()


def test_validate_with_full_run_checks_the_full_suite(tmp_path):
    code, lines = _validate(tmp_path, yaml.safe_load(SMOKE_TINY.read_text(encoding="utf-8")))

    assert code == 1, lines
    # The user B08 needs in the user role is one of the 7, not a second line asking for 1 more.
    assert [line for line in lines if line.startswith("Add") and "user" in line.split(":")[0]] == [
        "Add 7 more users (10 in all), 1 of them with the user role: B05, B07 and B15 need 10; "
        "B18 and B27 need 4; B08 needs 1 who can't use every tool."
    ]


def _asks(output: str) -> list[str]:
    return [line.strip() for line in output.splitlines() if line.strip().startswith("Add")]


def test_a_bare_validate_checks_what_a_bare_run_runs(tmp_path, monkeypatch):
    # A bare run is the essential 10, where B18 and B27 need 4 users and none needs 10, so both
    # ask for 2 more, never the 8 more B05, B07 and B15 need in a full run.
    monkeypatch.chdir(tmp_path)
    path = _write(tmp_path, _first_users(2))

    validate = CliRunner().invoke(ifixai_cli, ["validate", str(path)])
    run = _run(tmp_path, path, "--eval-mode", "self")

    lines = [line.strip() for line in validate.output.splitlines()]
    assert validate.exit_code == 1, validate.output
    assert run.exit_code == 1, run.output
    assert _asks(validate.output) == _asks(run.output) == [
        "Add 2 more users (4 in all), 1 of them with the customer role: B18 and B27 need 4; "
        "B08 needs 1 who can't use every tool."
    ]
    assert (
        f"Checking the 10 of {len(SPEC_BY_ID)} inspections ifixai run runs by default (suite 'essential'). "
        f"Use --full-run to check all {len(SPEC_BY_ID)}."
    ) in lines
    assert lines[-1] == f"Then run ifixai validate {path}"


def test_a_bare_validate_checks_the_suite_ifixai_yaml_gives_a_bare_run(tmp_path, monkeypatch):
    # Core is B01 to B32: B05, B07 and B15 need 10 users, B21 needs 3; no P or M check runs.
    monkeypatch.chdir(tmp_path)
    (tmp_path / "ifixai.yaml").write_text("suite: core\n", encoding="utf-8")
    path = _write(tmp_path, _first_users(2))

    validate = CliRunner().invoke(ifixai_cli, ["validate", str(path)])
    run = _run(tmp_path, path, "--eval-mode", "self")

    users = [line for line in _asks(validate.output) if "user" in line.split(":")[0]]
    assert validate.exit_code == 1, validate.output
    assert users == [line for line in _asks(run.output) if "user" in line.split(":")[0]] == [
        "Add 8 more users (10 in all), 1 of them with the customer role: B05, B07 and B15 need 10; "
        "B18 and B27 need 4; B21 needs 3; B08 needs 1 who can't use every tool."
    ]
    assert f"Then run ifixai validate {path} --suite core" in [line.strip() for line in run.output.splitlines()]
    assert "(suite 'core' from ifixai.yaml)" in validate.output


@pytest.mark.parametrize("pick", [("-b", "B18"), ("--suite", "core"), ("-c", "DECEPTION"), ("--strategic",)])
def test_validate_refuses_full_run_with_another_pick_as_run_does(tmp_path, pick):
    path = _write(tmp_path, _reference())

    validate = CliRunner().invoke(ifixai_cli, ["validate", str(path), "--full-run", *pick])
    run = _run(tmp_path, path, "--eval-mode", "self", "--full-run", *pick)

    line = "Error: --full-run runs every inspection and cannot be combined with --suite, --test, --category or --strategic."
    assert validate.exit_code == run.exit_code == 1, validate.output
    assert line in validate.output.splitlines()
    assert line in run.output.splitlines()


@pytest.mark.parametrize(
    ("pick", "flags"),
    [
        (("-b", "b18"), "-b B18"),
        (("--suite", "core"), "--suite core"),
        (("-c", "DECEPTION"), "-c DECEPTION"),
        (("--strategic",), "--strategic"),
        (("--full-run",), "--full-run"),
        ((), "--suite essential"),
    ],
    ids=["test", "suite", "category", "strategic", "everything", "default"],
)
def test_a_refused_run_prints_the_validate_command_that_checks_the_same_pick(tmp_path, pick, flags):
    path = _write(tmp_path, _first_users(2))

    run = _run(tmp_path, path, "--eval-mode", "self", *pick)

    command = f"ifixai validate {path} {flags}".strip()
    validate = CliRunner().invoke(ifixai_cli, shlex.split(command)[1:])
    assert run.exit_code == 1, run.output
    assert f"Then run {command}" in [line.strip() for line in run.output.splitlines()]
    assert validate.exit_code == 1, validate.output
    assert _asks(validate.output) == _asks(run.output) != []


def test_after_a_b18_refusal_two_more_users_make_its_validate_command_pass(tmp_path):
    path = _write(tmp_path, _first_users(2))
    refused = _run(tmp_path, path, "--eval-mode", "self", "-b", "B18")
    _write(tmp_path, _first_users(4))

    b18 = CliRunner().invoke(ifixai_cli, ["validate", str(path), "-b", "B18"])
    everything = CliRunner().invoke(ifixai_cli, ["validate", str(path), "--full-run"])

    assert _asks(refused.output) == ["Add 2 more users (4 in all): B18 needs 4."]
    assert f"Then run ifixai validate {path} -b B18" in [line.strip() for line in refused.output.splitlines()]
    assert b18.exit_code == 0, b18.output
    assert "Valid" in b18.output
    assert everything.exit_code == 1, everything.output


def test_validate_says_how_to_fix_a_short_fixture(tmp_path):
    path = _write(tmp_path, _first_users(2))

    result = CliRunner().invoke(ifixai_cli, ["validate", str(path), "-b", "B18"])

    lines = [line.strip() for line in result.output.splitlines()]
    assert result.exit_code == 1, result.output
    assert lines[-2:] == [
        f"Fix the agent profile and run ifixai author <your profile> -o {path} --force, or edit {path}.",
        f"Then run ifixai validate {path} -b B18",
    ]


@pytest.mark.parametrize("by", ["name", "path"])
def test_a_short_built_in_fixture_is_copied_to_a_new_file_never_edited(tmp_path, monkeypatch, by):
    # A shipped fixture short of a minimum: the advice names a new file, never the built-in itself.
    shipped = tmp_path / "fixtures"
    builtin = shipped / "short" / "fixture.yaml"
    builtin.parent.mkdir(parents=True)
    builtin.write_text(yaml.safe_dump(_first_users(2), sort_keys=False), encoding="utf-8")
    monkeypatch.setattr("ifixai.core.fixture_loader._FIXTURES_DIR", shipped)
    monkeypatch.chdir(tmp_path)
    name = "short" if by == "name" else str(builtin)

    result = CliRunner().invoke(ifixai_cli, ["validate", name, "-b", "B18"])

    lines = [line.strip() for line in result.output.splitlines()]
    assert result.exit_code == 1, result.output
    assert lines[-2:] == [
        f"{name} ships with iFixAi, so leave it as it is: copy it to a new file (cp {builtin} my-fixture.yaml) "
        "and fix that, or write a profile and run ifixai author <your profile> -o my-fixture.yaml.",
        "Then run ifixai validate my-fixture.yaml -b B18",
    ]
    assert "--force" not in result.output
    assert f"edit {name}" not in result.output


@pytest.mark.parametrize("pick", [("-b", "B18"), ("--suite", "core"), ("-c", "DECEPTION"), ("--strategic",), ("--full-run",)])
def test_validate_needs_a_fixture_to_check_a_pick_for(pick):
    result = CliRunner().invoke(ifixai_cli, ["validate", *pick])

    assert result.exit_code == 2, result.output
    assert "pass its PATH" in result.output


def test_validate_says_what_it_does_in_one_line():
    top = CliRunner().invoke(ifixai_cli, ["--help"])
    own = CliRunner().invoke(ifixai_cli, ["validate", "--help"])

    line = "Check a fixture gives the inspections a run picks enough for a verdict."
    assert line in [text.strip() for text in own.output.splitlines()]
    assert re.search(r"^\s+validate\s+Check a fixture gives", top.output, re.MULTILINE), top.output


@pytest.mark.parametrize(
    "command",
    [
        ["validate", "my-agent.json"],
        ["run", "-p", "mock", "-f", "my-agent.json", "--eval-mode", "self", "--no-promo", "--no-telemetry"],
    ],
    ids=["validate", "run"],
)
def test_a_profile_given_as_the_fixture_says_to_author_it_first(tmp_path, monkeypatch, command):
    monkeypatch.chdir(tmp_path)
    shutil.copy(REFERENCE.with_name("profile.json"), tmp_path / "my-agent.json")

    result = CliRunner().invoke(ifixai_cli, command)

    assert result.exit_code == 1, result.output
    assert (
        "my-agent.json is an agent profile, not a fixture. "
        "Run ifixai author my-agent.json first, then use the fixture it writes."
    ) in result.output
    assert "required property" not in result.output
    assert "Validation failed" not in result.output
    assert not (tmp_path / "ifixai-results").exists()


# A profile with only the fields it needs: none of the optional ones that mark a profile alone.
_MINIMAL_PROFILE = {
    "name": "Clinic Desk",
    "system_purpose": "Books and moves patient appointments.",
    "tools": [{"tool_id": "book_slot", "name": "Book slot", "category": "write", "risk_level": "medium"}],
    "roles": [{"name": "receptionist", "allowed_tools": ["book_slot"]}],
}
_AUTHOR_FIRST = (
    "my-agent.json is an agent profile, not a fixture. "
    "Run ifixai author my-agent.json first, then use the fixture it writes."
)


@pytest.mark.parametrize(
    ("stray", "line"),
    [
        ({}, _AUTHOR_FIRST),
        ({"users": []}, f"{_AUTHOR_FIRST} A profile can't hold users: remove it first."),
        (
            {"users": [], "permissions": [], "policies": {}},
            f"{_AUTHOR_FIRST} A profile can't hold permissions, policies, users: remove them first.",
        ),
    ],
    ids=["minimal", "stray-users", "stray-users-permissions-policies"],
)
@pytest.mark.parametrize("command", ["validate", "run"])
def test_a_minimal_profile_even_with_stray_fixture_fields_says_to_author_it_first(tmp_path, monkeypatch, stray, line, command):
    monkeypatch.chdir(tmp_path)
    (tmp_path / "my-agent.json").write_text(json.dumps({**_MINIMAL_PROFILE, **stray}), encoding="utf-8")
    args = {
        "validate": ["validate", "my-agent.json"],
        "run": ["run", "-p", "mock", "-f", "my-agent.json", "--eval-mode", "self", "--no-promo", "--no-telemetry"],
    }[command]

    result = CliRunner().invoke(ifixai_cli, args)

    assert result.exit_code == 1, result.output
    assert line in " ".join(result.output.split())
    assert "required property" not in result.output
    assert not (tmp_path / "ifixai-results").exists()


def test_the_fields_that_mark_a_profile_are_profile_fields_no_fixture_has_at_its_top():
    fixture_fields = set(load_schema()["properties"])

    assert set(PROFILE_ONLY_FIELDS) <= set(AgentProfile.model_fields)
    assert set(PROFILE_ONLY_FIELDS) & fixture_fields == set()
    assert set(FIXTURE_ONLY_FIELDS) & set(AgentProfile.model_fields) == set()
    assert fixture_fields - set(AgentProfile.model_fields) <= set(FIXTURE_ONLY_FIELDS)


# The legacy format `_normalize_fixture_format` reads: a top-level name, roles under `tenant`,
# tools and data sources keyed `id`, permissions as role/tool rows.
_LEGACY_FIXTURE = {
    "name": "Clinic Desk",
    "version": "1.0",
    "description": "Front desk agent",
    "tenant": {"roles": [{"id": "nurse", "name": "Nurse"}, {"id": "admin", "name": "Admin"}]},
    "tools": [{"id": "read_chart", "name": "Read chart", "category": "information_access", "risk_level": "low"}],
    "permissions": [{"role_id": "nurse", "tool_id": "read_chart", "allowed": True}],
    "data_sources": [{"id": "charts", "name": "Charts", "classification": "sensitive"}],
}


@pytest.mark.parametrize(
    "fixture",
    [_LEGACY_FIXTURE, {**_LEGACY_FIXTURE, "domain": "healthcare", "safety_rules": ["Never read a chart aloud."]}],
    ids=["legacy", "legacy-with-profile-fields"],
)
def test_a_legacy_fixture_loads_and_is_never_taken_for_a_profile(tmp_path, fixture):
    path = tmp_path / "legacy.json"
    path.write_text(json.dumps(fixture), encoding="utf-8")

    loaded = load_fixture(path)
    result = CliRunner().invoke(ifixai_cli, ["validate", str(path), "-b", "B03"])

    assert loaded.metadata.name == "Clinic Desk"
    assert [role.name for role in loaded.roles] == ["nurse", "admin"]
    assert [tool.tool_id for tool in loaded.tools] == ["read_chart"]
    assert read_valid_fixture(path).metadata.name == "Clinic Desk"
    assert "agent profile" not in result.output
    assert result.exit_code == 0, result.output


def test_a_profile_whose_tools_carry_id_is_read_as_a_legacy_fixture(tmp_path):
    # `id` on a tool is the legacy fixture's shape, never a profile's, so no profile refusal.
    profile = {"name": "Clinic Desk", "domain": "healthcare", "tools": _LEGACY_FIXTURE["tools"]}
    path = tmp_path / "legacy.json"
    path.write_text(json.dumps(profile), encoding="utf-8")

    result = CliRunner().invoke(ifixai_cli, ["validate", str(path)])

    assert result.exit_code == 1, result.output
    assert "agent profile" not in result.output
    assert "Validation failed" in result.output


def _runner_imports(path: Path) -> list[str]:
    found = []
    for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))):
        if isinstance(node, ast.ImportFrom) and node.module:
            names = [node.module, *(f"{node.module}.{alias.name}" for alias in node.names)]
        elif isinstance(node, ast.Import):
            names = [alias.name for alias in node.names]
        else:
            continue
        found += [name for name in names if name.startswith("ifixai.inspections.") and name.endswith(".runner")]
    return found


def test_core_imports_no_inspection_runner():
    core = Path(__file__).resolve().parents[1] / "core"

    imports = {path.name: _runner_imports(path) for path in sorted(core.glob("*.py"))}

    assert {name: found for name, found in imports.items() if found} == {}


def test_dry_run_says_what_to_add_and_still_shows_the_estimate(tmp_path):
    result = _run(tmp_path, _write(tmp_path, _first_users(2)), "--eval-mode", "self", "--dry-run", "--full-run")

    lines = [line.strip() for line in result.output.splitlines()]
    assert result.exit_code == 0, result.output
    assert _TWO_USERS_LINE in lines
    assert "Dry run -- no API calls will be made" in lines
    assert not (tmp_path / "out").exists()


def _the_run_started(**_: object) -> None:
    raise AssertionError("the run started")


# Context answers (industry, use case, sensitive data, roles, tools), then the build: discovery on
# the mock agent, Declare and Quick Build, or Declare and the Full Wizard, each accepted as built.
_CONTEXT = "\n\n\n\nread_record,write_record\n"
_DISCOVERY = _CONTEXT + "1\n" + "\n" * 10
_QUICK_BUILD = _CONTEXT + "2\n1\n" + "\n" * 10
# Wizard: roles `role` and admin, tools read_record and delete_record (critical), `role` may only read.
def _wizard(role: str) -> str:
    return (
        _CONTEXT
        + "2\n2\n"
        + f"{role}\n\n\nadmin\n\nadmin\ndone\n"
        + "read_record\n\n\ndelete_record\ndelete\ncritical\ndone\n"
        + "read_record\n\n"
        + "\n" * 10
    )


_WIZARD = _wizard("agent")
_HIGH_RISK_LINE = "Add a high-risk action the agent must handle with care: B10 needs one."


async def _judges_answer(*_args: object) -> None:
    return None


def _build_at_run_time(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, answers: str) -> click.testing.Result:
    monkeypatch.chdir(tmp_path)
    # The build steps only run at a terminal; the run engine itself must never be reached.
    monkeypatch.setattr(click.testing._NamedTextIOWrapper, "isatty", lambda self: True)
    monkeypatch.setattr(run_module, "execute_tests", _the_run_started)
    # Nor the network: the judges' check before the run answers as if both judges were up.
    monkeypatch.setattr(run_module, "preflight_judges", _judges_answer)
    judges = ["--judge-provider", "openai", "--judge-api-key", "sk-a", "--judge-provider", "anthropic", "--judge-api-key", "sk-b"]
    args = ["run", "-p", "mock", "--mode", "full", "--eval-mode", "full", *judges, "--full-run", "--no-promo", "--no-telemetry"]
    return CliRunner().invoke(ifixai_cli, [*args, "-o", "out", "--reliability-out", "runs"], input=answers)


# Only the wizard asks which tools are critical or high risk; discovery and Quick Build can't know.
@pytest.mark.parametrize(
    ("answers", "high_risk"),
    [(_DISCOVERY, []), (_QUICK_BUILD, []), (_WIZARD, ["delete_record"])],
    ids=["discovery", "quick-build", "wizard"],
)
def test_a_fixture_built_at_run_time_gets_its_users_and_is_saved_then_refused_on_facts_only(
    tmp_path, monkeypatch, answers, high_risk
):
    result = _build_at_run_time(tmp_path, monkeypatch, answers)

    lines = [line.strip() for line in result.output.splitlines()]
    saved = yaml.safe_load((tmp_path / "ssci_fixture.yaml").read_text(encoding="utf-8"))
    roles = {role["role_id"] or role["name"] for role in saved["roles"]}
    assert isinstance(result.exception, SystemExit) and result.exit_code == 1, result.output
    assert len(saved["users"]) == 10
    assert {role for user in saved["users"] for role in user["roles"]} == roles
    assert "Not ready: some inspections couldn't reach a verdict on this fixture." in lines
    assert not [line for line in lines if line.startswith("Add") and "user" in line.split(":")[0]]
    assert "Add 5 escalation triggers for cases the agent must hand to a person: B31 needs 5." in lines
    assert saved["high_risk_actions"] == high_risk
    assert (_HIGH_RISK_LINE in lines) == (not high_risk)
    assert "Then run ifixai validate ssci_fixture.yaml --full-run" in lines
    assert not (tmp_path / "runs").exists()


def test_wizard_users_get_the_tools_their_role_was_given(tmp_path, monkeypatch):
    # A role name the wizard turns into a different id: "Support Agent" is held as support_agent.
    _build_at_run_time(tmp_path, monkeypatch, _wizard("Support Agent"))

    fixture = read_valid_fixture(tmp_path / "ssci_fixture.yaml")
    allowed = build_allowed_tools(fixture)
    by_role = {user.roles[0]: allowed[user.user_id] for user in fixture.users}
    assert by_role == {"support_agent": {"read_record"}, "admin": {"read_record", "delete_record"}}


def test_discovery_says_its_result_is_checked_before_it_asks_the_agent_anything(tmp_path, monkeypatch):
    result = _build_at_run_time(tmp_path, monkeypatch, _DISCOVERY)

    lines = [line.strip() for line in result.output.splitlines()]
    notice = (
        "Running auto-discovery: this sends your agent a few requests now. "
        "What it finds is checked afterwards, before the audit starts."
    )
    found = next(index for index, line in enumerate(lines) if line.startswith("Found:"))
    assert notice in lines, result.output
    assert lines.index(notice) < found