Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

13.0 KB
Raw Blame
"""The task.completed frame of GET /v1/chat/tasks/{id}/events carries the same
structured output fields as GET /v1/chat/tasks/{id}."""

from __future__ import annotations

import json
from typing import Any

import pytest

from xagent.core.agent.result import COMPLETION_OUTCOMES
from xagent.web.api.v1 import _events_stream as es
from xagent.web.api.v1 import tasks as v1_tasks
from xagent.web.models.task import TaskStatus
from xagent.web.services import structured_output_projection
from xagent.web.services.hot_path_cache import (
    InMemoryTTLCache,
    set_cache_backend_for_testing,
)

from ..conftest import client
from .structured_output_shared import (
    COLUMN,
    EFFECTIVE_STATES,
    FORMAT_CONFIG,
    NEW_KEYS,
    RECORD,
    SWITCH_OFF_CONFIGS,
    SWITCH_ON_CONFIG,
    TODAY_COMPLETED_FRAME_KEYS,
    TODAY_TASK_INFO_KEYS,
    bearer,
    create_agent_with_key,
    insert_task,
    recorded_statements,
    task_selects,
    with_snapshot,
)
from .test_events_stream import _principal_for

pytestmark = pytest.mark.usefixtures("_test_db")


def completed_data(key: str, task_id: int) -> tuple[dict[str, Any], str]:
    """The parsed payload and the raw text of the stream's task.completed frame."""
    resp = client.get(f"/v1/chat/tasks/{task_id}/events", headers=bearer(key))
    assert resp.status_code == 200, resp.text
    blocks = [
        b for b in resp.text.split("\n\n") if b.startswith("event: task.completed")
    ]
    assert len(blocks) == 1, resp.text
    raw = blocks[0].split("data: ", 1)[1]
    return json.loads(raw), raw


def test_the_completed_frame_carries_the_ready_record_of_the_run():
    agent_id, key = create_agent_with_key()
    task_id = insert_task(agent_id, record=RECORD, agent_config=FORMAT_CONFIG)

    data, raw = completed_data(key, task_id)

    assert data["structured_output"] == RECORD["output"]
    assert data["structured_output_status"] == "ready"
    assert data["structured_output_reason"] is None
    assert raw.index('"zeta"') < raw.index('"alpha"')


def test_the_completed_frame_has_null_fields_for_a_task_without_a_format():
    agent_id, key = create_agent_with_key()
    task_id = insert_task(agent_id)

    data, _raw = completed_data(key, task_id)

    assert [data[name] for name in NEW_KEYS] == [None, None, None]


def test_the_completed_frame_reports_not_produced_for_a_format_without_a_record():
    agent_id, key = create_agent_with_key()
    task_id = insert_task(agent_id, agent_config=FORMAT_CONFIG)

    data, _raw = completed_data(key, task_id)

    assert data["structured_output_status"] == "incomplete"
    assert data["structured_output_reason"]["code"] == "not_produced"


def test_the_frame_writes_non_ascii_characters_escaped():
    agent_id, key = create_agent_with_key()
    wide = chr(0x4E2D) + chr(0x6587) + chr(0x1F600)
    record = {**RECORD, "output": {"name": wide}}
    task_id = insert_task(agent_id, record=record, agent_config=FORMAT_CONFIG)

    data, raw = completed_data(key, task_id)

    assert data["structured_output"] == record["output"]
    assert raw.isascii()


@pytest.mark.parametrize("code", ["agent_partial", "model_error", "not_produced"])
def test_a_reason_without_field_lists_has_no_such_keys_in_the_frame_or_the_response(
    code,
):
    agent_id, key = create_agent_with_key()
    if code == "not_produced":
        task_id = insert_task(agent_id, agent_config=FORMAT_CONFIG)
    else:
        status = "failed" if code == "model_error" else "incomplete"
        record = {
            "run_id": "run-1",
            "status": status,
            "output": None,
            "reason": {"code": code},
        }
        task_id = insert_task(agent_id, record=record)

    _data, raw = completed_data(key, task_id)
    resp = client.get(f"/v1/chat/tasks/{task_id}", headers=bearer(key))

    for text in (raw, resp.text):
        assert "missing_fields" not in text
        assert "unexpressible_fields" not in text
        assert json.loads(text)["structured_output_reason"]["code"] == code


@pytest.mark.asyncio
async def test_the_watchdog_close_frame_carries_the_same_fields_as_the_response():
    agent_id, key = create_agent_with_key()
    task_id = insert_task(agent_id, record=RECORD, agent_config=FORMAT_CONFIG)
    principal = _principal_for(key)
    sink = es.V1EventStreamSink(
        task_id=task_id,
        principal_key_prefix=principal.key.key_prefix,
        initial_status="running",
    )

    closed = await es.watchdog_check_once(
        sink,
        task_id,
        principal,
        read_task_snapshot=lambda tid, p: v1_tasks.effective_task_snapshot(
            v1_tasks._load_task_info_snapshot(tid, p)
        ),
    )

    assert closed is True
    frame, _is_close = sink.queue.get_nowait()
    data = json.loads(frame.split("data: ", 1)[1])
    response = client.get(f"/v1/chat/tasks/{task_id}", headers=bearer(key)).json()
    assert {name: data[name] for name in NEW_KEYS} == {
        name: response[name] for name in NEW_KEYS
    }
    assert data["structured_output_status"] == "ready"


def test_the_response_and_the_stream_call_the_same_projection(monkeypatch):
    calls: list[str] = []
    real = structured_output_projection.project_structured_output

    def spy(name: str):
        def wrapper(**kwargs):
            calls.append(name)
            return real(**kwargs)

        return wrapper

    monkeypatch.setattr(v1_tasks, "project_structured_output", spy("get"))
    monkeypatch.setattr(es, "project_structured_output", spy("events"))
    agent_id, key = create_agent_with_key()
    task_id = insert_task(agent_id, record=RECORD, agent_config=FORMAT_CONFIG)

    client.get(f"/v1/chat/tasks/{task_id}", headers=bearer(key))
    completed_data(key, task_id)

    assert calls == ["get", "events"]


def test_both_read_paths_import_the_one_projection_function():
    projection = structured_output_projection.project_structured_output
    assert v1_tasks.project_structured_output is projection
    assert es.project_structured_output is projection


def test_the_stream_reads_the_record_with_the_row_in_one_select():
    agent_id, key = create_agent_with_key()
    task_id = insert_task(agent_id, record=RECORD, agent_config=FORMAT_CONFIG)

    with recorded_statements() as statements:
        completed_data(key, task_id)

    selects = task_selects(statements)
    assert selects, statements
    assert all(f"tasks.{COLUMN}" in s for s in selects)


def test_a_frame_built_without_fields_carries_null_for_each():
    frame = es.completed_frame(
        status=TaskStatus.FAILED.value, output=None, error="x", error_code=None
    )

    data = json.loads(frame.split("data: ", 1)[1])
    assert [data[name] for name in NEW_KEYS] == [None, None, None]


def _incomplete_record(code: str, **extra: Any) -> dict[str, Any]:
    reason: dict[str, Any] = {"code": code}
    if code == "information_missing":
        reason["missing_fields"] = ["/g"]
    if code == "value_not_expressible":
        reason["unexpressible_fields"] = ["/g"]
    return {
        "run_id": "run-1",
        "status": "incomplete",
        "output": None,
        "reason": reason,
        **extra,
    }


PARTIAL = {"name": "x", "g": None}
RETURNED_RECORDS = {
    "gap-code-with-values": _incomplete_record(
        "information_missing", partial_output=PARTIAL
    ),
    "agent-partial-with-values": _incomplete_record(
        "agent_partial", partial_output=PARTIAL
    ),
    "agent-partial-with-values-and-lists": _incomplete_record(
        "agent_partial",
        partial_output=PARTIAL,
        partial_gaps={"missing_fields": ["/g"], "unexpressible_fields": ["/n"]},
    ),
    "agent-blocked-with-values": _incomplete_record(
        "agent_blocked", partial_output=PARTIAL
    ),
    "agent-blocked-with-values-and-lists": _incomplete_record(
        "agent_blocked",
        partial_output=PARTIAL,
        partial_gaps={"missing_fields": ["/g"]},
    ),
    "incomplete-without-values": _incomplete_record("agent_partial"),
    "ready": RECORD,
}


@pytest.mark.parametrize("name", list(RETURNED_RECORDS))
def test_the_response_and_the_frame_return_the_same_result_and_reason(name):
    record = RETURNED_RECORDS[name]
    assert len(RETURNED_RECORDS) == 7
    agent_id, key = create_agent_with_key()
    task_id = insert_task(agent_id, record=record, agent_config=FORMAT_CONFIG)

    data, _raw = completed_data(key, task_id)
    response = client.get(f"/v1/chat/tasks/{task_id}", headers=bearer(key)).json()

    assert {k: data[k] for k in NEW_KEYS} == {k: response[k] for k in NEW_KEYS}
    expected = record.get("partial_output", record["output"])
    assert response["structured_output"] == expected
    assert response["structured_output_status"] == record["status"]
    if record["reason"] is not None:
        assert response["structured_output_reason"]["code"] == record["reason"]["code"]


# --- the completion_outcome key -------------------------------------------------

SURFACES = ["get-miss", "get-hit", "sse-attach", "sse-watchdog"]
# A queued start has no conclusion frame, so only GET shows that state.
CASES = [
    pytest.param(surface, rating, changes, id=f"{surface}-{name}")
    for surface in SURFACES
    for name, rating, changes in EFFECTIVE_STATES
    if surface.startswith("get") or "start_queued" not in changes
]


async def _watchdog_frame(key: str, task_id: int) -> dict[str, Any]:
    principal = _principal_for(key)
    sink = es.V1EventStreamSink(
        task_id=task_id,
        principal_key_prefix=principal.key.key_prefix,
        initial_status="running",
    )
    closed = await es.watchdog_check_once(
        sink,
        task_id,
        principal,
        read_task_snapshot=lambda tid, p: v1_tasks.effective_task_snapshot(
            v1_tasks._load_task_info_snapshot(tid, p)
        ),
    )
    assert closed is True
    frame, _is_close = sink.queue.get_nowait()
    return json.loads(frame.split("data: ", 1)[1])


async def _read(surface: str, key: str, task_id: int, request) -> dict[str, Any]:
    """The parsed body of a GET response or of a conclusion frame."""
    if surface == "sse-attach":
        return completed_data(key, task_id)[0]
    if surface == "sse-watchdog":
        return await _watchdog_frame(key, task_id)
    if surface == "get-hit":
        set_cache_backend_for_testing(InMemoryTTLCache())
        request.addfinalizer(lambda: set_cache_backend_for_testing(None))
        client.get(f"/v1/chat/tasks/{task_id}", headers=bearer(key))
    return client.get(f"/v1/chat/tasks/{task_id}", headers=bearer(key)).json()


def _today_keys(surface: str) -> tuple[str, ...]:
    return (
        TODAY_TASK_INFO_KEYS
        if surface.startswith("get")
        else TODAY_COMPLETED_FRAME_KEYS
    )


@pytest.mark.asyncio
@pytest.mark.parametrize(("surface", "rating", "changes"), CASES)
@pytest.mark.parametrize("config", SWITCH_OFF_CONFIGS.values(), ids=SWITCH_OFF_CONFIGS)
async def test_switch_off_responses_have_no_completion_outcome(
    config, surface, rating, changes, monkeypatch, request
):
    agent_id, key = create_agent_with_key()
    task_id = insert_task(agent_id, agent_config=config, completion_outcome=rating)
    with_snapshot(monkeypatch, **changes)

    body = await _read(surface, key, task_id, request)

    assert tuple(body) == _today_keys(surface)


@pytest.mark.asyncio
@pytest.mark.parametrize(("surface", "rating", "changes"), CASES)
async def test_switch_on_responses_end_with_the_reported_outcome(
    surface, rating, changes, monkeypatch, request
):
    agent_id, key = create_agent_with_key()
    task_id = insert_task(
        agent_id, agent_config=SWITCH_ON_CONFIG, completion_outcome=rating
    )
    with_snapshot(monkeypatch, **changes)

    body = await _read(surface, key, task_id, request)

    assert tuple(body) == (*_today_keys(surface), "completion_outcome")
    assert body["completion_outcome"] == (rating if not changes else None)


@pytest.mark.parametrize("rating", [*sorted(COMPLETION_OUTCOMES), None])
def test_the_response_and_the_frame_return_the_same_completion_outcome(rating):
    agent_id, key = create_agent_with_key()
    task_id = insert_task(
        agent_id,
        record=RECORD,
        agent_config=SWITCH_ON_CONFIG,
        completion_outcome=rating,
    )

    data, _raw = completed_data(key, task_id)
    response = client.get(f"/v1/chat/tasks/{task_id}", headers=bearer(key)).json()

    assert data["completion_outcome"] == response["completion_outcome"] == rating


@pytest.mark.parametrize(
    ("reported", "snapshot_total_steps", "expected_tail"),
    [
        (None, None, ()),
        (
            {"completion_outcome": "partial"},
            3,
            ("completion_outcome", "snapshot_truncated", "snapshot_total_steps"),
        ),
    ],
    ids=["not-given", "before-the-truncation-marker"],
)
def test_the_completion_outcome_key_of_a_built_frame(
    reported, snapshot_total_steps, expected_tail
):
    frame = es.completed_frame(
        status=TaskStatus.COMPLETED.value,
        output="x",
        error=None,
        error_code=None,
        reported_outcome=reported,
        snapshot_total_steps=snapshot_total_steps,
    )

    data = json.loads(frame.split("data: ", 1)[1])
    assert tuple(data) == (*TODAY_COMPLETED_FRAME_KEYS, *expected_tail)