Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

16.2 KB
Raw Blame
"""The accept_partial switch from the structured outcome of a run to the fields
a task response carries."""

from __future__ import annotations

import asyncio
import copy
import json
from typing import Any, NamedTuple

import pytest

from tests.core.agent.test_structured_output import (
    MARKER,
    NULLABLE_STRING,
    STRING,
    ScriptedModel,
    _args_reply,
    _fmt,
    _obj,
    _text_reply,
)
from tests.web.services.test_structured_output_projection import MESSAGES
from xagent.core.agent import structured_output as so
from xagent.core.agent import structured_schema as ss
from xagent.core.agent.result import COMPLETION_OUTCOMES
from xagent.core.model.chat.exceptions import LLMRetryableError
from xagent.web.models.task import TaskStatus
from xagent.web.services.structured_output_projection import (
    _FAILED_CODES,
    _GAP_CODES,
    _INCOMPLETE_CODES,
    _SELF_RATED_CODES,
    MAX_REASON_FIELDS,
    REASON_MESSAGES,
    REASON_MESSAGES_WITH_RESULT,
    _partial_problem,
    build_record,
    project_structured_output,
)

RUN_ID = "run-1"
REQUIRED = _obj({"a": STRING, "g": STRING})
ALL_NULLABLE = _obj({"a": NULLABLE_STRING, "b": NULLABLE_STRING})


class SlowModel(ScriptedModel):
    async def chat(self, **kwargs: Any) -> Any:
        await asyncio.sleep(5)


class Outcome(NamedTuple):
    """One way an extraction can end: the schema, the answer, the model script
    and what to patch."""

    name: str
    script: list[Any]
    schema: dict[str, Any] = REQUIRED
    answer: str = "The answer."
    patches: tuple[tuple[Any, str, Any], ...] = ()
    model: type[ScriptedModel] = ScriptedModel


OUTCOMES = [
    Outcome("ready", [_args_reply({"a": "x", "g": "y"})]),
    Outcome("information-missing", [_args_reply({"a": "x", "g": None})]),
    Outcome("value-not-expressible", [_args_reply({"a": "x", "g": MARKER})]),
    Outcome("root-is-a-gap", [_args_reply({"a": None, "g": None})]),
    Outcome(
        "dropped-for-size",
        [_args_reply({"a": "x", "g": None})],
        patches=((ss, "MAX_RESULT_BYTES", 1),),
    ),
    Outcome("schema-mismatch", [_text_reply()] * 3),
    Outcome("model-error", [LLMRetryableError("boom")]),
    Outcome(
        "extraction-timeout",
        [],
        patches=((so, "EXTRACTION_TIMEOUT_SECONDS", 0.05),),
        model=SlowModel,
    ),
    Outcome(
        "result-too-large",
        [_args_reply({"a": "x" * 40, "g": "y"})],
        patches=((so, "MAX_ARGUMENTS_BYTES", 20),),
    ),
    Outcome("internal-error", [_args_reply({"a": "x", "g": "y"})]),
    Outcome("empty-answer-all-nullable", [], ALL_NULLABLE, ""),
    Outcome("empty-answer-required-root", [], REQUIRED, ""),
]
RATINGS: list[Any] = [*sorted(COMPLETION_OUTCOMES), None]


async def _read_back(
    outcome: Outcome, rating: Any, accept_partial: bool, monkeypatch: Any
) -> tuple[dict[str, Any] | None, dict[str, Any]]:
    """Run the extraction, store the record and project it as a response."""
    result = {
        "success": True,
        "status": "completed",
        "output": outcome.answer,
        "completion_outcome": rating,
    }
    with monkeypatch.context() as scoped:
        for module, name, value in outcome.patches:
            scoped.setattr(module, name, value)
        if outcome.name == "internal-error":

            async def boom(*args: Any, **kwargs: Any) -> Any:
                raise RuntimeError("boom")

            scoped.setattr(so, "run_structured_output_step", boom)
        attached = await so._attach_structured_output(
            result,
            _fmt(outcome.schema, accept_partial),
            llm=outcome.model(copy.deepcopy(outcome.script)),
            task_id="t-1",
            run_id=RUN_ID,
            tracer=None,
        )
    assert attached is not None
    record = build_record(RUN_ID, attached.get("structured_output"))
    fields = project_structured_output(
        status=TaskStatus.COMPLETED,
        run_id=RUN_ID,
        record=record,
        format_configured=True,
    )
    return record, fields.as_dict()


PARTIAL_KEYS = ("partial_output", "partial_gaps")


async def test_status_and_code_independent_of_switch(monkeypatch) -> None:
    assert len(OUTCOMES) == 12
    cells = 0
    for outcome in OUTCOMES:
        for rating in RATINGS:
            record_off, off = await _read_back(outcome, rating, False, monkeypatch)
            record_on, on = await _read_back(outcome, rating, True, monkeypatch)
            cells += 1
            reason_off = off["structured_output_reason"]
            reason_on = on["structured_output_reason"]
            assert on["structured_output_status"] == off["structured_output_status"], (
                outcome.name,
                rating,
            )
            assert (reason_on or {}).get("code") == (reason_off or {}).get("code"), (
                outcome.name,
                rating,
            )
            # Without its two new keys the record is the one the switch off writes.
            assert record_off is not None and record_on is not None
            stripped = {k: v for k, v in record_on.items() if k not in PARTIAL_KEYS}
            assert json.dumps(stripped, sort_keys=True) == json.dumps(
                record_off, sort_keys=True
            ), (outcome.name, rating)
    assert cells == 48


# --- the record and what is read back from it ------------------------------------------

BY_NAME = {outcome.name: outcome for outcome in OUTCOMES}

# The record of a request without the switch, for each way a run can end:
# (outcome, the agent's rating, the record).
_SELF_RATED_RECORD = {"status": "incomplete", "output": None}
DEFAULT_RECORDS = {
    "ready": (
        "ready",
        "completed",
        {"status": "ready", "output": {"a": "x", "g": "y"}, "reason": None},
    ),
    "information-missing": (
        "information-missing",
        "completed",
        {
            "status": "incomplete",
            "output": None,
            "reason": {"code": "information_missing", "missing_fields": ["/g"]},
        },
    ),
    "value-not-expressible": (
        "value-not-expressible",
        "completed",
        {
            "status": "incomplete",
            "output": None,
            "reason": {"code": "value_not_expressible", "unexpressible_fields": ["/g"]},
        },
    ),
    "schema-mismatch": (
        "schema-mismatch",
        "completed",
        {"status": "failed", "output": None, "reason": {"code": "schema_mismatch"}},
    ),
    "agent-partial": (
        "ready",
        "partial",
        {**_SELF_RATED_RECORD, "reason": {"code": "agent_partial"}},
    ),
    "agent-blocked": (
        "ready",
        "blocked",
        {**_SELF_RATED_RECORD, "reason": {"code": "agent_blocked"}},
    ),
}
assert len(DEFAULT_RECORDS) == 6


@pytest.mark.parametrize("cell", list(DEFAULT_RECORDS))
async def test_default_request_record_unchanged(cell: str, monkeypatch) -> None:
    name, rating, expected = DEFAULT_RECORDS[cell]
    record, _ = await _read_back(BY_NAME[name], rating, False, monkeypatch)

    assert record == {"run_id": RUN_ID, **expected}
    assert list(record) == ["run_id", "status", "output", "reason"]


def _record(status: str, code: str | None, **extra: Any) -> dict[str, Any]:
    reason: dict[str, Any] | None = None
    if status != "ready":
        reason = {"code": code}
        if code == "information_missing":
            reason["missing_fields"] = ["/g"]
        if code == "value_not_expressible":
            reason["unexpressible_fields"] = ["/g"]
    return {
        "run_id": RUN_ID,
        "status": status,
        "output": {"a": "x"} if status == "ready" else None,
        "reason": reason,
        **extra,
    }


def _partial_cells() -> list[tuple[str, dict[str, Any], str | None]]:
    """``(row, record, problem)`` for every row of the table of ``_partial_problem``."""
    cells: list[tuple[str, dict[str, Any], str | None]] = []
    codes = sorted(_INCOMPLETE_CODES) + sorted(_FAILED_CODES)

    def status_of(code: str) -> str:
        return "incomplete" if code in _INCOMPLETE_CODES else "failed"

    for code in codes:
        cells.append(("1", _record(status_of(code), code), None))
        cells.append(
            (
                "2",
                _record(status_of(code), code, partial_gaps={"missing_fields": ["/g"]}),
                "partial_gaps",
            )
        )
    cells.append(("1", _record("ready", None), None))
    cells.append(
        (
            "2",
            _record("ready", None, partial_gaps={"missing_fields": ["/g"]}),
            "partial_gaps",
        )
    )
    for bad in (["x"], None, "x", 1):
        cells.append(
            (
                "3",
                _record("incomplete", "agent_partial", partial_output=bad),
                "partial_output",
            )
        )
    cells.append(
        ("4", _record("ready", None, partial_output={"a": None}), "partial_output")
    )
    for code in sorted(_FAILED_CODES):
        cells.append(
            ("4", _record("failed", code, partial_output={"a": None}), "partial_output")
        )
    for code in sorted(_GAP_CODES):
        cells.append(
            (
                "5",
                _record(
                    "incomplete",
                    code,
                    partial_output={"a": None},
                    partial_gaps={"missing_fields": ["/g"]},
                ),
                "partial_gaps",
            )
        )
        cells.append(
            ("6", _record("incomplete", code, partial_output={"a": None}), None)
        )
    for code in sorted(_SELF_RATED_CODES):
        cells.append(
            ("7", _record("incomplete", code, partial_output={"a": None}), None)
        )
        bad_gaps: list[Any] = [
            [],
            "x",
            {},
            {"other": ["/g"]},
            {"missing_fields": ["/g"], "other": ["/g"]},
            {"missing_fields": []},
            {"missing_fields": "/g"},
            {"missing_fields": [1]},
            {"unexpressible_fields": ["/g"] * (MAX_REASON_FIELDS + 1)},
            {"missing_fields": ["/g"], "unexpressible_fields": []},
        ]
        for gaps in bad_gaps:
            cells.append(
                (
                    "8",
                    _record(
                        "incomplete",
                        code,
                        partial_output={"a": None},
                        partial_gaps=gaps,
                    ),
                    "partial_gaps",
                )
            )
        good_gaps: list[Any] = [
            {"missing_fields": ["/g"]},
            {"unexpressible_fields": ["/g"]},
            {"missing_fields": ["/g"], "unexpressible_fields": ["/a"]},
            {"missing_fields": ["/g"] * MAX_REASON_FIELDS},
        ]
        for gaps in good_gaps:
            cells.append(
                (
                    "9",
                    _record(
                        "incomplete",
                        code,
                        partial_output={"a": None},
                        partial_gaps=gaps,
                    ),
                    None,
                )
            )
    return cells


def test_partial_problem() -> None:
    cells = _partial_cells()

    assert {row for row, _, _ in cells} == {str(n) for n in range(1, 10)}
    assert _GAP_CODES | _SELF_RATED_CODES == _INCOMPLETE_CODES
    assert not _GAP_CODES & _SELF_RATED_CODES
    for row, record, problem in cells:
        assert _partial_problem(record) == problem, (row, record)
        fields = project_structured_output(
            status=TaskStatus.COMPLETED,
            run_id=RUN_ID,
            record=record,
            format_configured=True,
        )
        if problem is None:
            assert fields.structured_output_status == record["status"], (row, record)
        else:
            assert fields.structured_output_status == "failed"
            assert (fields.structured_output_reason or {})["code"] == "internal_error"


KNOWN_TOP_KEYS = {
    "run_id",
    "status",
    "output",
    "reason",
    "partial_output",
    "partial_gaps",
}


def _legal_records() -> list[dict[str, Any]]:
    records = [_record("ready", None)]
    records += [_record("incomplete", code) for code in sorted(_INCOMPLETE_CODES)]
    records += [_record("failed", code) for code in sorted(_FAILED_CODES)]
    return records


def _project(record: Any) -> dict[str, Any]:
    return project_structured_output(
        status=TaskStatus.COMPLETED,
        run_id=RUN_ID,
        record=record,
        format_configured=True,
    ).as_dict()


def test_projection_ignores_unknown_extra_keys() -> None:
    names = ("zz", "future_key")
    values: list[Any] = [{"k": 1}, ["x"], None]
    assert not KNOWN_TOP_KEYS & set(names)
    cells = 0
    for record in _legal_records():
        baseline = _project(record)
        for name in names:
            for value in values:
                top = {**record, name: copy.deepcopy(value)}
                assert _project(top) == baseline
                cells += 1
                if record["reason"] is not None:
                    inner = {
                        **record,
                        "reason": {**record["reason"], name: copy.deepcopy(value)},
                    }
                    assert _project(inner) == baseline
                    cells += 1
    assert cells == (1 + 4 + 6) * 6 + (4 + 6) * 6


def test_build_record_writes_the_two_keys_and_keeps_the_output_null() -> None:
    gaps = {
        "missing_fields": [f"/g/{i}" for i in range(30)],
        "unexpressible_fields": ["/u"],
    }
    outcome = {
        "status": "incomplete",
        "output": None,
        "reason": {"code": "agent_partial"},
        "partial_output": {"a": None},
        "partial_gaps": gaps,
    }

    record = build_record(RUN_ID, outcome)

    assert record == {
        "run_id": RUN_ID,
        "status": "incomplete",
        "output": None,
        "reason": {"code": "agent_partial"},
        "partial_output": {"a": None},
        "partial_gaps": {
            "missing_fields": [f"/g/{i}" for i in range(MAX_REASON_FIELDS)],
            "unexpressible_fields": ["/u"],
        },
    }
    assert outcome["partial_gaps"] == gaps


@pytest.mark.parametrize("bad", [None, ["x"], "x", 1])
def test_build_record_ignores_keys_that_are_not_mappings(bad: Any) -> None:
    outcome = {
        "status": "incomplete",
        "output": None,
        "reason": {"code": "agent_partial"},
        "partial_output": bad,
        "partial_gaps": bad,
    }

    assert set(build_record(RUN_ID, outcome) or {}) == {
        "run_id",
        "status",
        "output",
        "reason",
    }


# The two messages of a response that holds values, character for character.
WITH_RESULT = {
    "agent_partial": "The agent completed only part of the task. structured_output holds the values extracted from its answer, with null where a value is missing or cannot be written; missing_fields and unexpressible_fields list up to 20 such positions each.",
    "agent_blocked": "The agent stopped without completing the task. structured_output holds the values extracted from its answer, with null where a value is missing or cannot be written; missing_fields and unexpressible_fields list up to 20 such positions each.",
}


@pytest.mark.parametrize("with_values", [False, True])
@pytest.mark.parametrize("code", sorted(_SELF_RATED_CODES))
def test_self_rated_message_variants(code: str, with_values: bool) -> None:
    extra = {"partial_output": {"a": None}} if with_values else {}
    fields = _project(_record("incomplete", code, **extra))
    reason = fields["structured_output_reason"]

    assert reason["message"] == (WITH_RESULT if with_values else MESSAGES)[code]
    assert fields["structured_output"] == ({"a": None} if with_values else None)
    assert fields["structured_output_status"] == "incomplete"
    assert REASON_MESSAGES == MESSAGES
    assert REASON_MESSAGES_WITH_RESULT == WITH_RESULT
    assert set(REASON_MESSAGES_WITH_RESULT) == _SELF_RATED_CODES


def test_a_self_rated_response_carries_the_lists_of_partial_gaps_in_order() -> None:
    gaps = {"unexpressible_fields": ["/u"], "missing_fields": ["/m"]}
    reason = _project(
        _record(
            "incomplete", "agent_blocked", partial_output={"a": None}, partial_gaps=gaps
        )
    )["structured_output_reason"]

    assert list(reason) == ["code", "message", "missing_fields", "unexpressible_fields"]
    assert reason["missing_fields"] == ["/m"] and reason["unexpressible_fields"] == [
        "/u"
    ]