Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

21.0 KB
Raw Blame
"""L3 tool-call smoke (#3247): recorded response shapes and an in-process fake.

The fake answers ``/api/auth/login``, ``/api/mcp/apps``, ``/api/mcp/servers``
and ``POST /api/mcp/servers/{id}/probe``. The contract tests build every
response from the route's own result type
(``src/xagent/web/services/connector_probe.py``), so a change on either side
fails here; no test opens a socket.
"""

import importlib
import io
import json
import sys
from pathlib import Path
from types import SimpleNamespace
from typing import Any

import httpx
import pytest

# Importable under both pytest and `unittest discover -s tests/scripts`.
_SCRIPTS_DIR = str(Path(__file__).resolve().parents[2] / "scripts")
if _SCRIPTS_DIR not in sys.path:
    sys.path.insert(0, _SCRIPTS_DIR)
cli = importlib.import_module("connector_smoke.cli")
l1 = importlib.import_module("connector_smoke.l1")
l3 = importlib.import_module("connector_smoke.l3")

BASE = "https://cloud.example.com"
ACCESS_TOKEN = "access-token-SECRET"


def _app(app_id: str, server_id: int | None) -> dict[str, Any]:
    app: dict[str, Any] = {"id": app_id, "is_connected": server_id is not None}
    if server_id is not None:
        app["server_id"] = server_id
    return app


def _ok(tool: str, server_id: int = 7, app_id: str = "github") -> tuple[int, Any]:
    return 200, {
        "code": "ok",
        "outcome": "ok",
        "server_id": server_id,
        "app_id": app_id,
        "tool": tool,
    }


def _refused(
    status_code: int,
    outcome: str,
    *,
    code: str,
    reason: str | None = None,
    tool: str | None = "github_get_current_user",
) -> tuple[int, Any]:
    detail: dict[str, Any] = {
        "code": code,
        "outcome": outcome,
        "server_id": 7,
        "app_id": "github",
        "tool": tool,
    }
    if reason is not None:
        detail["reason"] = reason
    return status_code, {"detail": detail}


# --- pure classification ---------------------------------------------------


@pytest.mark.parametrize(
    ("response", "expected"),
    [
        (_ok("github_get_current_user"), (True, "github_get_current_user")),
        (
            _refused(400, "not_connected", code="oauth_token_required"),
            (False, "not connected (oauth_token_required)"),
        ),
        (
            _refused(400, "not_connected", code="auth_rejected"),
            (False, "not connected (auth_rejected)"),
        ),
        (
            _refused(400, "unsupported", code="unsupported", reason="execution_scoped"),
            (False, "unsupported outside a task (execution_scoped)"),
        ),
        (
            _refused(
                502,
                "failed_to_start",
                code="mcp_tools_unavailable",
                reason="initialize",
            ),
            (False, "failed to start (initialize)"),
        ),
        (
            _refused(502, "no_tools", code="mcp_tools_unavailable"),
            (False, "no tools"),
        ),
        (
            _refused(
                502, "timed_out", code="mcp_tools_unavailable", reason="probe_call"
            ),
            (False, "timed out on the server (probe_call)"),
        ),
        (
            _refused(502, "probe_tool_missing", code="probe_tool_missing"),
            (False, "probe tool missing (github_get_current_user)"),
        ),
        (
            _refused(502, "probe_failed", code="probe_failed", reason="tool_error"),
            (False, "probe failed (tool_error)"),
        ),
        (
            _refused(
                502,
                "probe_assertion_failed",
                code="probe_assertion_failed",
                reason="status_not_success",
            ),
            (False, "unexpected probe result (status_not_success)"),
        ),
        (
            _refused(404, "not_found", code="mcp_server_not_found", tool=None),
            (False, "server not found"),
        ),
        (
            (429, {"detail": {"code": "probe_busy", "message": "busy"}}),
            (False, "probe busy (another probe, or one that timed out, still runs)"),
        ),
        # A deployment without the probe route.
        ((404, {"detail": "Not Found"}), (False, "HTTP 404")),
        ((504, "gateway timeout"), (False, "HTTP 504")),
        ((200, ["not", "an", "object"]), (False, "unexpected response (HTTP 200)")),
        ((200, {"outcome": "probe_failed"}), (False, "unexpected response (HTTP 200)")),
        # An ``ok`` outcome on an error status is not a pass.
        ((502, {"detail": {"outcome": "ok"}}), (False, "HTTP 502")),
    ],
)
def test_probe_responses_are_classified(response, expected) -> None:
    status_code, payload = response
    result = l3.check_probe_response("github", status_code, payload)
    assert result == l1.CheckResult("l3:github", *expected)


@pytest.mark.parametrize(
    ("response", "reason"),
    [
        (
            _refused(400, "unprobeable", code="unprobeable", reason="covered_by_l2"),
            "unprobeable (covered_by_l2)",
        ),
        (
            _refused(404, "not_declared", code="probe_not_declared", tool=None),
            "no probe declared",
        ),
    ],
)
def test_connectors_without_a_probe_are_skipped(response, reason) -> None:
    """An unprobeable builtin, or a custom catalog app the route names as
    itself (``acme``) with nothing declared."""
    status_code, payload = response
    payload["detail"]["app_id"] = "acme"
    assert l3.check_probe_response("acme", status_code, payload) == l3.Skipped(
        "l3:acme", reason
    )


@pytest.mark.parametrize("route_app_id", [None, "stripe"])
def test_no_probe_for_a_row_that_is_not_the_catalogs_fails(route_app_id) -> None:
    """The server answers not_declared without an app id when the row is not the
    catalog's own (a custom row claiming the app, a squat, a drifted launch).
    The catalog listed it as connected, so skipping it would let a connector
    drop out of the run unnoticed."""
    status_code, payload = _refused(
        404, "not_declared", code="probe_not_declared", tool=None
    )
    payload["detail"]["app_id"] = route_app_id

    result = l3.check_probe_response("github", status_code, payload)

    assert result == l1.CheckResult(
        "l3:github", False, "not the catalog's row for this app (no probe)"
    )


@pytest.mark.parametrize(
    "response",
    [
        _refused(502, "unprobeable", code="unprobeable", reason="covered_by_l2"),
        _refused(400, "not_declared", code="probe_not_declared", tool=None),
    ],
)
def test_a_skip_outcome_on_another_status_is_not_a_skip(response) -> None:
    status_code, payload = response
    result = l3.check_probe_response("github", status_code, payload)
    assert result == l1.CheckResult("l3:github", False, f"HTTP {status_code}")


def test_a_sandboxed_connector_says_so() -> None:
    status_code, payload = _refused(
        400, "unsupported", code="unsupported", reason="sandboxed"
    )
    result = l3.check_probe_response("github", status_code, payload)
    assert result == l1.CheckResult(
        "l3:github", False, "cannot be probed: runs in a sandbox"
    )


def test_a_code_from_the_server_is_never_echoed_unless_it_is_a_plain_code() -> None:
    status_code, payload = _refused(
        502, "probe_failed", code="probe_failed", reason="Bad token ghp_SECRET"
    )
    result = l3.check_probe_response("github", status_code, payload)
    assert result == l1.CheckResult("l3:github", False, "probe failed (unknown)")


def test_only_the_probe_permission_refusal_is_forbidden() -> None:
    assert l3.is_forbidden(403, {"detail": {"code": "probe_forbidden"}})
    assert not l3.is_forbidden(403, {"detail": "Forbidden"})
    assert not l3.is_forbidden(401, {"detail": {"code": "probe_forbidden"}})


# --- the contract with the route ---------------------------------------------


def test_outcomes_match_the_routes() -> None:
    from xagent.web.services.connector_probe import ConnectorProbeOutcome

    assert l3.OUTCOMES == {outcome.value for outcome in ConnectorProbeOutcome}


def _route_response(result) -> tuple[int, Any]:
    """What POST /servers/{id}/probe answers for ``result`` (api/mcp.py)."""
    if result.http_status == 200:
        return 200, result.payload()
    return result.http_status, {"detail": result.payload()}


def test_every_route_outcome_gets_a_specific_line() -> None:
    from xagent.web.services.connector_probe import (
        ConnectorProbeOutcome,
        ConnectorProbeResult,
    )

    for outcome in ConnectorProbeOutcome:
        result = ConnectorProbeResult(
            outcome, 7, "github", "github_get_current_user", "initialize"
        )
        status_code, payload = _route_response(result)

        line = l3.check_probe_response("github", status_code, payload)

        assert not line.reason.startswith("HTTP "), (outcome, line)
        assert "unexpected response" not in line.reason, (outcome, line)
        if outcome is ConnectorProbeOutcome.OK:
            assert line == l1.CheckResult("l3:github", True, "github_get_current_user")
        elif outcome.value in {"unprobeable", "not_declared"}:
            assert isinstance(line, l3.Skipped), outcome
        else:
            assert isinstance(line, l1.CheckResult) and not line.ok, outcome


def test_a_route_credential_code_is_reported() -> None:
    from xagent.web.services.connector_probe import (
        ConnectorProbeOutcome,
        ConnectorProbeResult,
    )

    result = ConnectorProbeResult(
        ConnectorProbeOutcome.NOT_CONNECTED,
        7,
        "github",
        "github_get_current_user",
        "oauth_token_required",
    )

    line = l3.check_probe_response("github", *_route_response(result))

    assert line == l1.CheckResult(
        "l3:github", False, "not connected (oauth_token_required)"
    )


def test_a_route_identity_refusal_fails() -> None:
    """What the route answers for a row that is not the catalog's own."""
    from xagent.web.services.connector_probe import (
        ConnectorProbeOutcome,
        ConnectorProbeResult,
    )

    result = ConnectorProbeResult(ConnectorProbeOutcome.NOT_DECLARED, 7)

    line = l3.check_probe_response("stripe", *_route_response(result))

    assert line == l1.CheckResult(
        "l3:stripe", False, "not the catalog's row for this app (no probe)"
    )


@pytest.mark.asyncio
async def test_the_routes_refusals_are_recognised(monkeypatch) -> None:
    from fastapi import HTTPException

    from xagent.web.api.mcp import probe_mcp_server
    from xagent.web.services.connector_probe import ConnectorProbeBusy

    user = SimpleNamespace(id=1, username="monitor", email=None, is_admin=False)

    monkeypatch.delenv("XAGENT_CONNECTOR_PROBE_USERS", raising=False)
    with pytest.raises(HTTPException) as refused:
        await probe_mcp_server(7, user, db=object())
    assert l3.is_forbidden(refused.value.status_code, {"detail": refused.value.detail})

    async def busy(*args, **kwargs):
        raise ConnectorProbeBusy()

    monkeypatch.setenv("XAGENT_CONNECTOR_PROBE_USERS", "monitor")
    monkeypatch.setattr("xagent.web.services.connector_probe.run_connector_probe", busy)
    with pytest.raises(HTTPException) as busy_answer:
        await probe_mcp_server(7, user, db=object())
    line = l3.check_probe_response(
        "github", busy_answer.value.status_code, {"detail": busy_answer.value.detail}
    )
    assert (
        line.reason == "probe busy (another probe, or one that timed out, still runs)"
    )


# --- the whole run against a fake deployment -------------------------------


class FakeDeployment:
    def __init__(self) -> None:
        self.catalog = [
            _app("github", 7),
            _app("notion", 3),
            _app("stripe", 5),
            _app("google-calendar", None),
        ]
        self.servers = [
            {"id": 7, "app_id": "github", "connection_status": "connected"},
            {"id": 3, "app_id": None, "connection_status": None},
            {"id": 5, "app_id": None, "connection_status": None},
        ]
        self.probes: dict[int, tuple[int, Any]] = {
            7: _ok("github_get_current_user", 7, "github"),
            3: _refused(400, "unprobeable", code="unprobeable", reason="covered_by_l2"),
            5: _ok("stripe_list_customers", 5, "stripe"),
        }
        self.slow: set[int] = set()
        self.errors: set[int] = set()
        self.servers_status = 200
        self.requests: list[httpx.Request] = []

    def handler(self, request: httpx.Request) -> httpx.Response:
        self.requests.append(request)
        path = request.url.path
        if path == "/api/auth/login":
            body = json.loads(request.content)
            if body.get("password") != "pw":
                return httpx.Response(401, json={"detail": "Invalid credentials"})
            return httpx.Response(200, json={"access_token": ACCESS_TOKEN})
        if request.headers.get("authorization") != f"Bearer {ACCESS_TOKEN}":
            return httpx.Response(401, json={"detail": "Not authenticated"})
        if path == "/api/mcp/apps":
            return httpx.Response(200, json=self.catalog)
        if path == "/api/mcp/servers":
            return httpx.Response(self.servers_status, json=self.servers)
        parts = path.split("/")
        if (
            request.method == "POST"
            and len(parts) == 6
            and parts[1:4] == ["api", "mcp", "servers"]
            and parts[5] == "probe"
        ):
            server_id = int(parts[4])
            if server_id in self.slow:
                raise httpx.ReadTimeout("timed out", request=request)
            if server_id in self.errors:
                raise httpx.ConnectError("refused", request=request)
            status_code, payload = self.probes.get(
                server_id, (404, {"detail": "Not Found"})
            )
            return httpx.Response(status_code, json=payload)
        return httpx.Response(404, json={"detail": "Not Found"})


ENV = {
    "XAGENT_CONNECTOR_SMOKE_BASE_URL": BASE,
    "XAGENT_CONNECTOR_SMOKE_LOGIN": "monitor",
    "XAGENT_CONNECTOR_SMOKE_PASSWORD": "pw",
}


def _run(
    fake: FakeDeployment, *args: str, env: dict[str, str] | None = None
) -> tuple[int, list[str]]:
    out = io.StringIO()
    code = cli.main(
        ["l3", *args],
        env={**ENV, **(env or {})},
        transport=httpx.MockTransport(fake.handler),
        out=out,
    )
    return code, out.getvalue().splitlines()


def _probe_requests(fake: FakeDeployment) -> list[tuple[str, str]]:
    return [
        (request.method, request.url.path)
        for request in fake.requests
        if request.url.path.endswith("/probe")
    ]


def test_every_connected_connector_is_probed_once_in_order() -> None:
    fake = FakeDeployment()

    code, lines = _run(fake)

    assert code == 0, lines
    assert lines[:3] == [
        "PASS l3:github: github_get_current_user",
        "SKIP l3:notion: unprobeable (covered_by_l2)",
        "PASS l3:stripe: stripe_list_customers",
    ]
    assert lines[3].startswith("2 checks, 0 failed, 1 skipped in ")
    # Sorted by app id, POST only, no tool name or arguments in the request.
    assert _probe_requests(fake) == [
        ("POST", "/api/mcp/servers/7/probe"),
        ("POST", "/api/mcp/servers/3/probe"),
        ("POST", "/api/mcp/servers/5/probe"),
    ]
    assert all(
        request.content == b""
        for request in fake.requests
        if request.url.path.endswith("/probe")
    )


def test_a_failed_probe_fails_the_run_and_the_others_still_run() -> None:
    fake = FakeDeployment()
    fake.probes[7] = _refused(
        502,
        "probe_assertion_failed",
        code="probe_assertion_failed",
        reason="status_not_success",
    )

    code, lines = _run(fake)

    assert code == 1
    assert "FAIL l3:github: unexpected probe result (status_not_success)" in lines
    assert "PASS l3:stripe: stripe_list_customers" in lines


def test_a_timed_out_probe_fails_and_the_run_continues() -> None:
    fake = FakeDeployment()
    fake.slow.add(7)

    code, lines = _run(fake, "--probe-timeout", "5")

    assert code == 1
    assert "FAIL l3:github: timed out after 5s" in lines
    assert "PASS l3:stripe: stripe_list_customers" in lines


def test_probe_requests_use_the_probe_timeout() -> None:
    fake = FakeDeployment()
    seen: list[Any] = []

    def handler(request: httpx.Request) -> httpx.Response:
        if request.url.path.endswith("/probe"):
            seen.append(request.extensions["timeout"]["read"])
        return fake.handler(request)

    out = io.StringIO()
    cli.main(
        ["l3", "--probe-timeout", "42", "--timeout", "3"],
        env=ENV,
        transport=httpx.MockTransport(handler),
        out=out,
    )

    assert seen == [42.0, 42.0, 42.0]


def test_the_default_probe_timeout_covers_a_listing_and_a_call() -> None:
    # The server bounds the listing and then the call by
    # XAGENT_MCP_TOOL_INIT_TIMEOUT_SECONDS (60 s by default) each.
    assert cli.DEFAULT_PROBE_TIMEOUT > 2 * 60


def test_a_request_that_gets_no_answer_fails_and_the_run_continues() -> None:
    fake = FakeDeployment()
    fake.errors.add(7)

    code, lines = _run(fake)

    assert code == 1
    assert "FAIL l3:github: POST request failed: ConnectError" in lines
    assert "PASS l3:stripe: stripe_list_customers" in lines


def test_an_account_outside_the_allowlist_stops_the_run_at_once() -> None:
    fake = FakeDeployment()
    for server_id in fake.probes:
        fake.probes[server_id] = (
            403,
            {"detail": {"code": "probe_forbidden", "message": "no"}},
        )

    code, lines = _run(fake)

    assert code == 1
    assert lines[0] == (
        "FAIL probe: this account may not run connector probes "
        "(XAGENT_CONNECTOR_PROBE_USERS)"
    )
    assert lines[1].startswith("1 checks, 1 failed, 0 skipped in ")
    assert len(_probe_requests(fake)) == 1


def test_a_deployment_without_the_probe_route_fails_every_connector() -> None:
    fake = FakeDeployment()
    fake.probes = {}

    code, lines = _run(fake)

    assert code == 1
    assert "FAIL l3:github: HTTP 404" in lines
    assert "FAIL l3:notion: HTTP 404" in lines


def test_expected_connectors_that_are_not_connected_fail() -> None:
    fake = FakeDeployment()

    code, lines = _run(fake, "--expect", "github,google-calendar,typo")

    assert code == 1
    assert "FAIL l3:google-calendar: not connected" in lines
    assert "FAIL l3:typo: not in the catalog" in lines


def test_expect_list_comes_from_the_environment() -> None:
    fake = FakeDeployment()

    code, lines = _run(fake, env={"XAGENT_CONNECTOR_SMOKE_EXPECT": "google-calendar"})

    assert code == 1
    assert "FAIL l3:google-calendar: not connected" in lines


def test_a_builtin_app_that_needs_reconnect_fails_even_when_not_expected() -> None:
    fake = FakeDeployment()
    fake.catalog.append(_app("gmail", None))
    fake.servers.append(
        {"id": 9, "app_id": "gmail", "connection_status": "needs_reconnect"}
    )

    code, lines = _run(fake)

    assert code == 1
    assert "FAIL l3:gmail: needs reconnect" in lines


def test_a_failed_server_list_is_reported_and_probes_still_pass() -> None:
    fake = FakeDeployment()
    fake.servers_status = 500

    code, lines = _run(fake)

    assert code == 1
    assert "FAIL servers: server list request failed: HTTP 500" in lines
    assert "PASS l3:github: github_get_current_user" in lines


def test_a_run_that_only_skips_fails() -> None:
    fake = FakeDeployment()
    fake.catalog = [_app("notion", 3)]

    code, lines = _run(fake)

    assert code == 1
    assert lines[0] == "SKIP l3:notion: unprobeable (covered_by_l2)"
    assert lines[1] == "FAIL connectors: no connected connector to probe"


def test_a_run_that_checks_nothing_fails() -> None:
    fake = FakeDeployment()
    fake.catalog = [_app("github", None)]
    fake.servers = []

    code, lines = _run(fake)

    assert code == 1
    assert lines[0] == "FAIL connectors: no connected connector to probe"


@pytest.mark.parametrize("value", ["0", "-5", "nan"])
def test_a_probe_timeout_that_is_not_a_positive_number_is_a_usage_error(
    value,
) -> None:
    with pytest.raises(SystemExit) as exc_info:
        _run(FakeDeployment(), "--probe-timeout", value)
    assert exc_info.value.code == 2


def test_a_cookie_is_sent_on_every_request() -> None:
    fake = FakeDeployment()

    _run(fake, "--cookie", "selected_region=sg")

    assert fake.requests
    assert all(
        request.headers.get("cookie") == "selected_region=sg"
        for request in fake.requests
    )


def test_a_login_failure_is_one_failed_check() -> None:
    code, lines = _run(FakeDeployment(), env={"XAGENT_CONNECTOR_SMOKE_PASSWORD": "no"})

    assert code == 1
    assert lines[0] == "FAIL login: login failed: HTTP 401"


def test_output_never_contains_the_token() -> None:
    fake = FakeDeployment()
    fake.probes[7] = _refused(
        502, "probe_failed", code="probe_failed", reason=ACCESS_TOKEN
    )
    fake.probes[3] = _refused(
        400, "unprobeable", code="unprobeable", reason=ACCESS_TOKEN
    )
    fake.probes[5] = (200, {"outcome": "ok", "tool": ACCESS_TOKEN})

    _code, lines = _run(fake)

    assert ACCESS_TOKEN not in "\n".join(lines)
    assert "FAIL l3:github: probe failed (unknown)" in lines
    assert "SKIP l3:notion: unprobeable (unknown)" in lines
    assert "PASS l3:stripe: unknown" in lines