Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

12.4 KB
Raw Blame
"""The assertion evaluator: payloads, paths and each assertion kind."""

from __future__ import annotations

import json
from typing import Any

import pytest

from xagent.web.connector_checks.assertions import (
    MISSING,
    json_equal,
    judge_result,
    resolve_expected,
    value_at,
)
from xagent.web.connector_checks.declarations import (
    Contains,
    CreatedRef,
    EachHasKeys,
    Equals,
    HasKeys,
    IsType,
    L3Envelope,
    Length,
    PayloadSource,
    RunScopedName,
    StatusSuccess,
)
from xagent.web.connector_checks.fixtures import FixtureMissing, FixtureRef
from xagent.web.connector_checks.results import FailureClass
from xagent.web.services.connector_probe import _single_json_object

PASS = (None, None)


def text_result(*texts: str, is_error: bool = False) -> dict[str, Any]:
    return {
        "content": [{"type": "text", "text": text} for text in texts],
        "structured_content": None,
        "is_error": is_error,
    }


def json_result(*objects: Any) -> dict[str, Any]:
    return text_result(*(json.dumps(item) for item in objects))


class Resolver:
    def __init__(self, values: dict[str, Any]) -> None:
        self.values = values

    def resolve_fixture(self, key: str) -> Any:
        return self.values.get(key)


# The call itself


@pytest.mark.parametrize("payload", ["json_text", "json_text_items"])
def test_a_flagged_error_fails_the_call_before_any_assertion(
    payload: PayloadSource,
) -> None:
    raw = json_result({"status": "success"})
    raw["is_error"] = True

    assert judge_result(raw, payload, (StatusSuccess(),)) == (
        FailureClass.CALL_FAILED,
        "tool_error",
    )


@pytest.mark.parametrize("raw", [None, "text", ["content"]])
def test_a_result_that_is_not_a_mapping_fails_the_call(raw: Any) -> None:
    assert judge_result(raw, "json_text", (StatusSuccess(),)) == (
        FailureClass.CALL_FAILED,
        "invalid_result",
    )


# Payloads


def test_json_text_reads_the_one_text_item_as_an_object() -> None:
    raw = json_result({"status": "success", "messages": []})

    assert judge_result(raw, "json_text", (HasKeys((), ("messages",)),)) == PASS


@pytest.mark.parametrize(
    "raw",
    [
        json_result({"a": 1}, {"b": 2}),
        text_result("not json"),
        json_result([1, 2]),
        {"content": [], "is_error": False},
        {"content": [{"type": "image", "data": "..."}], "is_error": False},
        {"is_error": False},
    ],
    ids=["two_items", "not_json", "array", "empty", "no_text", "no_content"],
)
def test_json_text_needs_exactly_one_text_item_holding_an_object(raw: Any) -> None:
    assert judge_result(raw, "json_text", (StatusSuccess(),)) == (
        FailureClass.ASSERTION_FAILED,
        "not_a_json_object",
    )


def test_json_text_items_parses_every_text_item_and_skips_other_content() -> None:
    raw = json_result({"id": 1}, [2, 3], "four")
    raw["content"].insert(1, {"type": "image", "data": "..."})

    assertions = (
        Length((), min=3, max=3),
        IsType((1,), "array"),
        Equals((2,), "four"),
    )
    assert judge_result(raw, "json_text_items", assertions) == PASS


def test_json_text_items_fails_on_a_text_item_that_is_not_json() -> None:
    raw = text_result(json.dumps({"id": 1}), "plain words")

    assert judge_result(raw, "json_text_items", (Length((), min=1),)) == (
        FailureClass.ASSERTION_FAILED,
        "not_json_text",
    )


def test_structured_content_reads_the_structured_object() -> None:
    raw = text_result("summary")
    raw["structured_content"] = {"user_id": "u-1"}

    assertions = (HasKeys((), ("user_id",)),)
    assert judge_result(raw, "structured_content", assertions) == PASS


@pytest.mark.parametrize("structured", [None, ["user"], "user"])
def test_structured_content_must_be_an_object(structured: Any) -> None:
    raw = text_result(json.dumps({"user_id": "u-1"}))
    raw["structured_content"] = structured

    assert judge_result(raw, "structured_content", (HasKeys((), ("user_id",)),)) == (
        FailureClass.ASSERTION_FAILED,
        "no_structured_content",
    )


# Assertions

_DOCUMENT = {
    "status": "success",
    "count": 2,
    "ratio": 0.5,
    "enabled": True,
    "owner": None,
    "site": {"id": "s-1", "name": "Main"},
    "items": [{"id": 1, "name": "a"}, {"id": 2, "name": "b"}],
    "empty": [],
}


@pytest.mark.parametrize(
    "assertion",
    [
        StatusSuccess(),
        HasKeys((), ("site", "items")),
        HasKeys(("site",), ("id", "name")),
        EachHasKeys(("items",), ("id", "name")),
        EachHasKeys(("empty",), ("id",)),
        Contains(("site", "name"), "ai"),
        IsType(("site",), "object"),
        IsType(("items",), "array"),
        IsType(("site", "id"), "string"),
        IsType(("count",), "number"),
        IsType(("ratio",), "number"),
        IsType(("enabled",), "boolean"),
        IsType(("owner",), "null"),
        Length(("items",), min=1, max=2),
        Length(("empty",)),
        Equals(("items", 1, "name"), "b"),
        Equals(("count",), 2),
        Equals(("count",), 2.0),
        Equals(("enabled",), True),
        Equals(("owner",), None),
    ],
)
def test_an_assertion_that_holds_passes(assertion: Any) -> None:
    assert judge_result(json_result(_DOCUMENT), "json_text", (assertion,)) == PASS


@pytest.mark.parametrize(
    ("assertion", "reason"),
    [
        (HasKeys(("site",), ("id", "url")), "has_keys"),
        (HasKeys(("items",), ("id",)), "has_keys"),
        (HasKeys(("missing",), ("id",)), "has_keys"),
        (EachHasKeys(("items",), ("id", "url")), "each_has_keys"),
        (EachHasKeys(("site",), ("id",)), "each_has_keys"),
        (Contains(("site", "name"), "xyz"), "contains"),
        (Contains(("count",), "2"), "contains"),
        (Contains(("absent",), "x"), "contains"),
        (IsType(("count",), "string"), "is_type"),
        (IsType(("enabled",), "number"), "is_type"),
        (IsType(("count",), "boolean"), "is_type"),
        (IsType(("items", 5), "object"), "is_type"),
        (Length(("items",), min=3), "length"),
        (Length(("items",), max=1), "length"),
        (Length(("site",)), "length"),
        (Equals(("site", "id"), "s-2"), "equals"),
        (Equals(("enabled",), 1), "equals"),
        (Equals(("count",), True), "equals"),
        (Equals(("owner",), "none"), "equals"),
        (Equals(("absent",), None), "equals"),
        (Equals(("site", 0), "s-1"), "equals"),
        (Equals(("items", "0"), 1), "equals"),
    ],
)
def test_an_assertion_that_does_not_hold_fails_with_its_kind(
    assertion: Any, reason: str
) -> None:
    assert judge_result(json_result(_DOCUMENT), "json_text", (assertion,)) == (
        FailureClass.ASSERTION_FAILED,
        reason,
    )


@pytest.mark.parametrize(
    "document", [{"status": "error"}, {"ok": True}, {"status": True}]
)
def test_status_success_needs_the_success_envelope(document: Any) -> None:
    assert judge_result(json_result(document), "json_text", (StatusSuccess(),)) == (
        FailureClass.ASSERTION_FAILED,
        "status_success",
    )


def test_status_success_fails_on_a_payload_that_is_not_an_object() -> None:
    raw = json_result({"status": "success"})

    assert judge_result(raw, "json_text_items", (StatusSuccess(),)) == (
        FailureClass.ASSERTION_FAILED,
        "status_success",
    )


def test_the_first_assertion_that_fails_is_reported() -> None:
    assertions = (
        StatusSuccess(),
        Length(("items",), max=1),
        HasKeys((), ("absent",)),
    )

    assert judge_result(json_result(_DOCUMENT), "json_text", assertions) == (
        FailureClass.ASSERTION_FAILED,
        "length",
    )


@pytest.mark.parametrize(
    "value",
    [FixtureRef("jira.site_id"), RunScopedName("site"), CreatedRef(("site", "id"))],
)
def test_an_unresolved_value_never_reaches_the_evaluator(value: Any) -> None:
    with pytest.raises(TypeError):
        judge_result(
            json_result(_DOCUMENT), "json_text", (Equals(("site", "id"), value),)
        )


# Reading one value, for a tenant pin and a created-object reference


def test_value_at_reads_a_path_of_the_payload() -> None:
    raw = json_result(_DOCUMENT)

    assert value_at(raw, "json_text", ("site", "id")) == "s-1"
    assert value_at(raw, "json_text", ("items", 1, "name")) == "b"


@pytest.mark.parametrize(
    ("raw", "path"),
    [
        (json_result(_DOCUMENT), ("site", "missing")),
        (json_result(_DOCUMENT), ("items", 5)),
        ({**json_result(_DOCUMENT), "is_error": True}, ("site", "id")),
        (text_result("not json"), ("site", "id")),
        ("not a result", ("site", "id")),
    ],
)
def test_value_at_has_no_value_where_the_payload_has_none(
    raw: Any, path: tuple[str | int, ...]
) -> None:
    assert value_at(raw, "json_text", path) is MISSING


def test_json_equal_never_equates_a_boolean_and_a_number() -> None:
    assert json_equal(1, 1.0)
    assert json_equal("site-1", "site-1")
    assert not json_equal(True, 1)
    assert not json_equal("1", 1)


# L3Envelope


@pytest.mark.parametrize(
    ("raw", "expect_keys", "expected"),
    [
        (json_result({"status": "success"}), (), PASS),
        (json_result({"status": "success", "sub": "x"}), ("sub",), PASS),
        (json_result({"sub": "x"}), ("sub",), PASS),
        (
            json_result({"status": "error", "message": "token expired"}),
            (),
            (FailureClass.ASSERTION_FAILED, "status_not_success"),
        ),
        (
            json_result({"sub": "x"}),
            (),
            (FailureClass.ASSERTION_FAILED, "no_status"),
        ),
        (
            json_result({"status": "success"}),
            ("sub",),
            (FailureClass.ASSERTION_FAILED, "missing_keys"),
        ),
        (
            text_result("Error: unauthorized"),
            ("sub",),
            (FailureClass.ASSERTION_FAILED, "not_a_json_object"),
        ),
        (
            text_result("{}", is_error=True),
            (),
            (FailureClass.CALL_FAILED, "tool_error"),
        ),
        (None, (), (FailureClass.CALL_FAILED, "invalid_result")),
    ],
)
def test_l3_envelope_judges_as_the_l3_probe(
    raw: Any, expect_keys: tuple[str, ...], expected: Any
) -> None:
    assert judge_result(raw, "json_text", (L3Envelope(expect_keys),)) == expected


# Expected values


def test_resolve_expected_fills_each_fixture_in_an_equals_value() -> None:
    assertions = (
        StatusSuccess(),
        Equals(("site", "id"), FixtureRef("jira.site_id")),
        Equals(("count",), 2),
    )

    resolved = resolve_expected(assertions, Resolver({"jira.site_id": "s-1"}))

    assert resolved == (
        StatusSuccess(),
        Equals(("site", "id"), "s-1"),
        Equals(("count",), 2),
    )
    assert judge_result(json_result(_DOCUMENT), "json_text", resolved) == PASS


def test_resolve_expected_raises_for_a_fixture_without_a_value() -> None:
    assertions = (Equals(("site", "id"), FixtureRef("jira.site_id")),)

    with pytest.raises(FixtureMissing) as raised:
        resolve_expected(assertions, Resolver({}))

    assert raised.value.key == "jira.site_id"


@pytest.mark.parametrize(
    "content",
    [
        [{"type": "text", "text": json.dumps({"status": "success"})}],
        [{"type": "text", "text": json.dumps({"a": 1})}] * 2,
        [{"type": "text", "text": json.dumps([1])}],
        [{"type": "text", "text": "not json"}],
        [],
        [{"type": "image", "data": "..."}],
        [{"type": "text", "text": "{}"}, {"type": "image", "data": "..."}],
        [{"type": "text", "text": "[" * 100000}],
        [{"type": "text", "text": 7}],
        ["{}"],
        None,
        "{}",
    ],
)
def test_json_text_takes_the_object_the_l3_probe_takes(content: Any) -> None:
    raw = {"content": content, "structured_content": None, "is_error": False}

    rejected = judge_result(raw, "json_text", (HasKeys((), ()),)) == (
        FailureClass.ASSERTION_FAILED,
        "not_a_json_object",
    )

    assert rejected == (_single_json_object(content) is None)


def test_a_deeply_nested_text_is_not_json_rather_than_a_crash() -> None:
    # json.loads raises RecursionError, not ValueError, on this text.
    raw = text_result("[" * 100000)

    assert judge_result(raw, "json_text", (HasKeys((), ()),)) == (
        FailureClass.ASSERTION_FAILED,
        "not_a_json_object",
    )
    assert judge_result(raw, "json_text_items", (StatusSuccess(),)) == (
        FailureClass.ASSERTION_FAILED,
        "not_json_text",
    )
    assert judge_result(raw, "json_text", (L3Envelope(()),)) == (
        FailureClass.ASSERTION_FAILED,
        "not_a_json_object",
    )