Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

14.1 KB
Raw Blame
"""``python -m xagent.web.connector_checks``: one JSON document on stdout.

The document is the run result; logs go to stderr. Most tests replace the
database and the adapter; the refusal test runs the real ``user`` adapter
against a seeded database.
"""

from __future__ import annotations

import contextlib
import json
import logging
import os
import subprocess
import sys
from collections.abc import Iterator
from datetime import datetime, timezone
from typing import Any

import jsonschema
import pytest

from tests.web.services.test_connector_probe import (
    _connect_catalog_stripe,
    _github_ok,
    _Loader,
    _stripe_ok,
)
from tests.web.services.test_mcp_connector_session import (
    LOADER,
    _connect_github,
)
from xagent.web.connector_checks import cli
from xagent.web.connector_checks.identity import ConnectorTarget
from xagent.web.connector_checks.results import RunResult, load_result_schema
from xagent.web.connector_checks.sweeper import load_sweep_schema
from xagent.web.services.mcp_connector_session import ConnectorToolsOutcome

from .fakes import FakeAdapter, FakeTool, failed, listed

SECRET = "postgresql://ops:[email protected]/xagent"


def _with_database(monkeypatch: pytest.MonkeyPatch, db: Any) -> None:
    @contextlib.contextmanager
    def database() -> Iterator[Any]:
        yield db

    monkeypatch.setattr(cli, "_database", database)


def _install_adapter(
    monkeypatch: pytest.MonkeyPatch, adapter: FakeAdapter
) -> list[str]:
    """Run the CLI under ``adapter``, without a database; returns the user
    names the adapter was built for."""
    names: list[str] = []

    def build(db: Any, user_name: str) -> FakeAdapter:
        names.append(user_name)
        return adapter

    _with_database(monkeypatch, None)
    monkeypatch.setattr(cli, "UserIdentityAdapter", build)
    return names


def _record_engine(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
    """Replace the engine with one that records its keywords."""
    seen: dict[str, Any] = {}

    async def run_checks(adapter: Any, **kwargs: Any) -> RunResult:
        seen.update(kwargs)
        now = datetime.now(timezone.utc)
        return RunResult(adapter.label, now, now, True, ())

    monkeypatch.setattr(cli, "run_checks", run_checks)
    return seen


def _document(out: str) -> dict[str, Any]:
    """The whole of stdout, read as one JSON document of the result schema."""
    document = json.loads(out)
    jsonschema.validate(document, load_result_schema())
    return document


def _slack(tool: FakeTool | None = None) -> FakeAdapter:
    tool = tool or FakeTool("slack_list_channels")
    return FakeAdapter([ConnectorTarget("slack")], {"slack": listed(tool)})


def _stripe() -> FakeAdapter:
    """A legacy connector, whose one check is its L3 probe."""
    tool = FakeTool("stripe_list_customers")
    return FakeAdapter([ConnectorTarget("stripe")], {"stripe": listed(tool)})


def test_a_passing_run_prints_its_document_and_exits_0(monkeypatch, capfd):
    names = _install_adapter(monkeypatch, _stripe())

    code = cli.main(["--user", "monitor"])

    out, _ = capfd.readouterr()
    document = _document(out)
    assert code == 0
    assert names == ["monitor"]
    assert document["identity"] == "user"
    assert document["complete"] is True
    assert [row["outcome"] for row in document["rows"]] == ["pass", "pass"]


def test_stdout_carries_only_the_document(monkeypatch, capfd):
    """Whatever a connector, a tool or a library writes during the run, at
    the Python level, to file descriptor 1 or from a child process, goes to
    stderr."""

    async def noisy_session():
        print("print from a tool")
        sys.stdout.flush()
        os.write(1, b"raw write to fd 1\n")
        subprocess.run(
            [sys.executable, "-c", "print('child process output')"], check=True
        )
        logging.getLogger("xagent.test").warning("a log line")
        return listed(FakeTool("slack_list_channels"))

    _install_adapter(
        monkeypatch, FakeAdapter([ConnectorTarget("slack")], {"slack": noisy_session})
    )
    # pytest's root logger has handlers, which would make basicConfig a no-op.
    root = logging.getLogger()
    monkeypatch.setattr(root, "handlers", [])
    monkeypatch.setattr(root, "level", root.level)

    code = cli.main(["--user", "monitor"])

    out, err = capfd.readouterr()
    assert code == 0
    assert out.endswith("}\n") and out.count("\n") == 1
    _document(out)
    for line in ("print from a tool", "raw write to fd 1", "child process output"):
        assert line in err
    assert "a log line" in err
    assert "a log line" not in out
    # Stdout is back on its own stream after the run.
    print("after the run")
    assert capfd.readouterr().out == "after the run\n"


def test_a_failed_row_exits_1(monkeypatch, capfd):
    adapter = FakeAdapter(
        [ConnectorTarget("slack")],
        {"slack": failed(ConnectorToolsOutcome.NOT_CONNECTED, "token_refresh_failed")},
    )
    _install_adapter(monkeypatch, adapter)

    code = cli.main(["--user", "monitor"])

    out, _ = capfd.readouterr()
    assert code == 1
    assert _document(out)["rows"][0]["failure_class"] == "not_connected"


def test_a_run_the_total_timeout_cut_exits_1(monkeypatch, capfd):
    _install_adapter(monkeypatch, _slack(FakeTool("slack_list_channels", delay=1.0)))

    code = cli.main(["--user", "monitor", "--total-timeout", "0.05"])

    out, _ = capfd.readouterr()
    assert code == 1
    assert _document(out)["complete"] is False


def test_skipped_rows_do_not_fail_the_run(monkeypatch, capfd):
    adapter = FakeAdapter(
        [ConnectorTarget("stripe", skip_reason="not_catalog_identity")]
    )
    _install_adapter(monkeypatch, adapter)

    code = cli.main(["--user", "monitor"])

    out, _ = capfd.readouterr()
    assert code == 0
    assert [row["outcome"] for row in _document(out)["rows"]] == ["skip", "skip"]


def test_selectors_and_bounds_reach_the_engine(monkeypatch, capfd):
    _install_adapter(monkeypatch, _slack())
    seen = _record_engine(monkeypatch)

    cli.main(
        [
            "--user",
            "monitor",
            "--connector",
            "slack, notion",
            "--connector",
            "jira",
            "--check",
            "slack.l3_probe",
            "--environment",
            "stage",
            "--check-timeout",
            "5",
            "--total-timeout",
            "60",
        ]
    )

    assert seen == {
        "environment": "stage",
        "connectors": frozenset({"slack", "notion", "jira"}),
        "checks": frozenset({"slack.l3_probe"}),
        "check_timeout": 5.0,
        "total_timeout": 60.0,
    }


def test_without_selectors_every_connector_and_check_runs(monkeypatch, capfd):
    _install_adapter(monkeypatch, _slack())
    seen = _record_engine(monkeypatch)

    cli.main(["--user", "monitor"])

    assert seen["connectors"] is None and seen["checks"] is None
    assert seen["environment"] is None and seen["check_timeout"] is None
    assert seen["total_timeout"] == cli.DEFAULT_TOTAL_TIMEOUT_SECONDS


def test_a_database_that_cannot_open_is_a_run_failure(monkeypatch, capfd, caplog):
    @contextlib.contextmanager
    def broken() -> Iterator[None]:
        raise RuntimeError(SECRET)
        yield

    monkeypatch.setattr(cli, "_database", broken)

    code = cli.main(["--user", "monitor", "--environment", "stage"])

    out, err = capfd.readouterr()
    document = _document(out)
    assert code == 1
    assert document["failure"] == "setup:database"
    assert document["environment"] == "stage"
    assert document["rows"] == []
    assert "RuntimeError" in caplog.text
    assert SECRET not in out + err + caplog.text


def test_an_engine_that_raises_is_a_run_failure(monkeypatch, capfd, caplog):
    async def run_checks(adapter: Any, **kwargs: Any) -> RunResult:
        raise ValueError(SECRET)

    _install_adapter(monkeypatch, _slack())
    monkeypatch.setattr(cli, "run_checks", run_checks)

    code = cli.main(["--user", "monitor"])

    out, err = capfd.readouterr()
    assert code == 1
    assert _document(out)["failure"] == "run:exception"
    assert "ValueError" in caplog.text
    assert SECRET not in out + err + caplog.text


@pytest.mark.parametrize(
    "argv",
    [
        [],
        ["--user", "monitor", "--environment", "Not A Label"],
        ["--user", "monitor", "--total-timeout", "0"],
        ["--user", "monitor", "--check-timeout", "nan"],
        ["--user", "monitor", "--check-timeout", "inf"],
        ["--user", "monitor", "--total-timeout", "soon"],
        # Selectors only: a tool name or an argument is not an option.
        ["--user", "monitor", "--tool", "slack_list_channels"],
        ["--user", "monitor", "--arguments", "{}"],
        # A selector must name something.
        ["--user", "monitor", "--connector", ""],
        ["--user", "monitor", "--connector", ","],
        ["--user", "monitor", "--check", ""],
        # A sweep has no checks to select.
        ["--user", "monitor", "--sweep", "--check", "slack.l3_probe"],
    ],
)
def test_invalid_arguments_exit_2_with_nothing_on_stdout(monkeypatch, capfd, argv):
    names = _install_adapter(monkeypatch, _slack())

    with pytest.raises(SystemExit) as exited:
        cli.main(argv)

    out, _ = capfd.readouterr()
    assert exited.value.code == 2
    assert out == ""
    assert names == []


def test_sweep_prints_the_sweep_document(monkeypatch, capfd):
    adapter = _slack()
    names = _install_adapter(monkeypatch, adapter)

    code = cli.main(["--user", "monitor", "--sweep", "--environment", "stage"])

    out, _ = capfd.readouterr()
    document = json.loads(out)
    jsonschema.validate(document, load_sweep_schema())
    assert document["schema"] == "xagent.connector_smoke.sweep"
    assert document["environment"] == "stage"
    # Writes are off, so whatever the registry declares, nothing starts.
    assert document["complete"] is True
    assert code == 0
    assert names == ["monitor"]
    assert adapter.opened == []


@pytest.mark.parametrize(
    ("argv", "expected"),
    [
        ([], {"total_timeout": 600.0, "call_timeout": None, "connectors": None}),
        (
            ["--total-timeout", "60", "--check-timeout", "5", "--connector", "jira"],
            {"total_timeout": 60.0, "call_timeout": 5.0, "connectors": {"jira"}},
        ),
    ],
)
def test_sweep_options_reach_the_sweeper(monkeypatch, capfd, argv, expected):
    _install_adapter(monkeypatch, _slack())
    seen: dict[str, Any] = {}

    async def record(adapter: Any, **kwargs: Any) -> Any:
        seen.update(kwargs)
        raise RuntimeError("stop here")

    monkeypatch.setattr(cli, "run_sweep", record)

    cli.main(["--user", "monitor", "--sweep", *argv])

    assert seen["total_timeout"] == expected["total_timeout"]
    assert seen["call_timeout"] == expected["call_timeout"]
    assert seen["connectors"] == expected["connectors"]


def test_a_sweep_that_cannot_open_the_database_prints_a_sweep_failure(
    monkeypatch, capfd
):
    @contextlib.contextmanager
    def database() -> Iterator[Any]:
        raise RuntimeError(SECRET)
        yield

    monkeypatch.setattr(cli, "_database", database)

    code = cli.main(["--user", "monitor", "--sweep"])

    out, _ = capfd.readouterr()
    document = json.loads(out)
    jsonschema.validate(document, load_sweep_schema())
    assert document["failure"] == "setup:database"
    assert SECRET not in out
    assert code == 1


# --- the real adapter ------------------------------------------------------


def test_a_user_outside_the_probe_users_is_refused_before_a_connector_starts(
    db_users, probe_users, monkeypatch, capfd
):
    db, _, other_user = db_users
    _connect_github(db, other_user)
    loader = _Loader(_github_ok())
    monkeypatch.setattr(LOADER, loader)
    _with_database(monkeypatch, db)

    code = cli.main(["--user", "other"])

    out, _ = capfd.readouterr()
    document = _document(out)
    assert code == 1
    assert document["failure"] == "identity:user_not_allowed"
    assert document["rows"] == []
    assert loader.calls == []


def test_a_probe_user_runs_their_connectors_checks(
    db_users, probe_users, monkeypatch, capfd
):
    db, user, _ = db_users
    _connect_catalog_stripe(db, user)
    monkeypatch.setattr(LOADER, _Loader(_stripe_ok()))
    _with_database(monkeypatch, db)

    code = cli.main(["--user", "[email protected]"])

    out, _ = capfd.readouterr()
    assert code == 0
    assert [
        (row["connector"], row["check"], row["outcome"])
        for row in _document(out)["rows"]
    ] == [("stripe", None, "pass"), ("stripe", "stripe.l3_probe", "pass")]


@pytest.mark.parametrize(
    ("user", "failure"),
    [("other", "identity:user_not_allowed"), ("monitor", "enumeration:no_connectors")],
)
def test_the_module_runs_as_a_one_shot_command(
    db_users, tmp_path, user: str, failure: str
):
    """``python -m`` in a fresh process, on the service's database: the
    document is the whole of stdout."""
    db, _, _ = db_users
    env = {
        **os.environ,
        "DATABASE_URL": db.get_bind().url.render_as_string(hide_password=False),
        "XAGENT_CONNECTOR_PROBE_USERS": "monitor",
    }

    completed = subprocess.run(
        [sys.executable, "-m", "xagent.web.connector_checks", "--user", user],
        cwd=tmp_path,
        env=env,
        capture_output=True,
        text=True,
        timeout=120,
    )

    assert completed.returncode == 1, completed.stderr
    assert _document(completed.stdout)["failure"] == failure


def test_a_database_that_cannot_connect_is_setup_database(tmp_path):
    """The engine connects lazily: an unreachable database must fail at
    setup, not inside the enumeration. SQLite cannot open a directory."""
    directory = tmp_path / "not-a-file"
    directory.mkdir()
    env = {
        **os.environ,
        "DATABASE_URL": f"sqlite:///{directory}",
        "XAGENT_CONNECTOR_PROBE_USERS": "monitor",
    }

    completed = subprocess.run(
        [sys.executable, "-m", "xagent.web.connector_checks", "--user", "monitor"],
        cwd=tmp_path,
        env=env,
        capture_output=True,
        text=True,
        timeout=120,
    )

    assert completed.returncode == 1, completed.stderr
    assert _document(completed.stdout)["failure"] == "setup:database"