Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

19.8 KB
Raw Blame
"""Command line entry point for the connector smoke checks.

Usage: ``PYTHONPATH=src:scripts python -m connector_smoke {l1,l2,l3} --help``.
See ``scripts/connector_smoke/README.md``.
"""

from __future__ import annotations

import argparse
import math
import os
import sys
import time
from collections.abc import Iterable, Mapping, Sequence
from typing import Any, TextIO

import httpx

from . import l1, l2, l3
from .client import SmokeClient, SmokeError, SmokeTimeout

ENV_BASE_URL = "XAGENT_CONNECTOR_SMOKE_BASE_URL"
ENV_LOGIN = "XAGENT_CONNECTOR_SMOKE_LOGIN"
ENV_PASSWORD = "XAGENT_CONNECTOR_SMOKE_PASSWORD"
ENV_CALLBACK_ORIGIN = "XAGENT_CONNECTOR_SMOKE_CALLBACK_ORIGIN"
ENV_EXCLUDE = "XAGENT_CONNECTOR_SMOKE_EXCLUDE"
ENV_INCLUDE = "XAGENT_CONNECTOR_SMOKE_INCLUDE"
ENV_COOKIES = "XAGENT_CONNECTOR_SMOKE_COOKIES"
ENV_OAUTH_ACCOUNTS = "XAGENT_CONNECTOR_SMOKE_OAUTH_ACCOUNTS"
ENV_EXPECT = "XAGENT_CONNECTOR_SMOKE_EXPECT"

# Above the server's bound on one answer: XAGENT_MCP_TOOL_INIT_TIMEOUT_SECONDS
# (60 s by default) plus the token refresh and, for an npx connector, the
# sandbox lease that run before it. A shorter client timeout gives up while the
# server is still working.
DEFAULT_TOOLS_TIMEOUT = 120.0
# A probe starts the connector (bounded like a listing) and then calls one tool,
# which the server bounds by XAGENT_MCP_TOOL_INIT_TIMEOUT_SECONDS again.
DEFAULT_PROBE_TIMEOUT = 180.0


def run_l1(
    client: SmokeClient,
    *,
    login: str,
    password: str,
    login_field: str,
    login_start: bool,
    expected: set[str],
    expected_origin: str,
    oauth_accounts: Mapping[str, str],
) -> list[l1.CheckResult]:
    try:
        client.login(login, password, login_field=login_field, login_start=login_start)
    except SmokeError as exc:
        return [l1.CheckResult("login", False, str(exc))]

    try:
        apps = client.list_catalog_apps()
    except SmokeError as exc:
        return [l1.CheckResult("catalog", False, str(exc))]

    results = l1.check_catalog(apps, expected)
    for target in l1.select_oauth_targets(apps, preferred=expected):
        if target.auth_backend == l1.COMPOSIO_BACKEND:
            results.extend(_check_composio_target(client, target, oauth_accounts))
        else:
            results.extend(
                _check_oauth_target(client, target, expected_origin, oauth_accounts)
            )
    return results


def _check_composio_target(
    client: SmokeClient,
    target: l1.OAuthTarget,
    oauth_accounts: Mapping[str, str],
) -> list[l1.CheckResult]:
    provider = target.provider
    name = f"oauth:{provider}:composio"
    account = oauth_accounts.get(provider)
    if target.needs_account and account is None:
        return [
            l1.CheckResult(
                name,
                False,
                f"needs an account label: pass --oauth-account {provider}=<label>",
            )
        ]
    try:
        start = client.start_launch(provider, target.app_id, account)
    except SmokeError as exc:
        return [l1.CheckResult(name, False, str(exc))]
    if start.status_code != 200:
        return [
            l1.CheckResult(
                name,
                False,
                "launch start refused: "
                + l1.start_refused_detail(start.status_code, start.payload),
            )
        ]
    origin = l1.start_callback_origin(start.payload)
    if origin is None:
        return [
            l1.CheckResult(
                name, False, "launch start returned no usable callback origin"
            )
        ]
    try:
        redeem = client.redeem_launch(provider, start.payload.get("authorization_url"))
    except SmokeError as exc:
        return [l1.CheckResult(name, False, str(exc))]
    return [
        l1.check_composio_callback(
            provider,
            status_code=redeem.status_code,
            location=redeem.location,
            body=redeem.body,
            expected_origin=origin,
        )
    ]


def _check_oauth_target(
    client: SmokeClient,
    target: l1.OAuthTarget,
    expected_origin: str,
    oauth_accounts: Mapping[str, str],
) -> list[l1.CheckResult]:
    provider = target.provider
    account = oauth_accounts.get(provider)
    if target.needs_account and account is None:
        return [
            l1.CheckResult(
                f"oauth:{provider}:launch",
                False,
                f"needs an account label: pass --oauth-account {provider}=<label>",
            )
        ]
    try:
        start = client.start_launch(provider, target.app_id, account)
    except SmokeError as exc:
        return [l1.CheckResult(f"oauth:{provider}:launch", False, str(exc))]
    results = [
        l1.check_callback_origin(
            provider,
            status_code=start.status_code,
            payload=start.payload,
            expected_origin=expected_origin,
        )
    ]
    authorize = f"oauth:{provider}:authorize"
    if start.status_code != 200 or not isinstance(start.payload, dict):
        results.append(l1.CheckResult(authorize, False, "launch start refused"))
        return results
    try:
        redeem = client.redeem_launch(provider, start.payload.get("authorization_url"))
    except SmokeError as exc:
        results.append(l1.CheckResult(authorize, False, str(exc)))
        return results
    results.append(
        l1.check_authorize_redirect(
            provider,
            status_code=redeem.status_code,
            location=redeem.location,
            body=redeem.body,
            expected_origin=expected_origin,
        )
    )
    return results


def run_l2(
    client: SmokeClient,
    *,
    login: str,
    password: str,
    login_field: str,
    login_start: bool,
    expect: set[str],
    tools_timeout: float,
) -> list[l1.CheckResult]:
    try:
        client.login(login, password, login_field=login_field, login_start=login_start)
    except SmokeError as exc:
        return [l1.CheckResult("login", False, str(exc))]

    try:
        apps = client.list_catalog_apps()
    except SmokeError as exc:
        return [l1.CheckResult("catalog", False, str(exc))]

    targets, missing = l2.select_targets(apps, expect=expect)
    results: list[l1.CheckResult] = []
    # One connector at a time: each call starts an MCP session on the web
    # process (xorbitsai/xagent-saas#1788).
    for target in targets:
        try:
            response = client.list_server_tools(target.server_id, timeout=tools_timeout)
        except SmokeTimeout:
            results.append(l2.check_timeout(target.app_id, tools_timeout))
            continue
        except SmokeError as exc:
            results.append(l2.check_request_error(target.app_id, str(exc)))
            continue
        results.append(
            l2.check_tools_response(
                target.app_id, response.status_code, response.payload
            )
        )

    try:
        servers = client.list_servers()
    except SmokeError as exc:
        results.append(l1.CheckResult("servers", False, str(exc)))
        servers = []
    unconnected = l2.select_unconnected(
        servers,
        connected=(target.app_id for target in targets),
        missing=missing,
    )
    results.extend(
        l2.check_unconnected(app_id, apps, servers) for app_id in unconnected
    )
    if not targets and not unconnected:
        # A misconfigured monitor account must not pass by checking nothing.
        results.append(
            l1.CheckResult("connectors", False, "no connected connector to check")
        )
    return results


def run_l3(
    client: SmokeClient,
    *,
    login: str,
    password: str,
    login_field: str,
    login_start: bool,
    expect: set[str],
    probe_timeout: float,
) -> list[l1.CheckResult | l3.Skipped]:
    try:
        client.login(login, password, login_field=login_field, login_start=login_start)
    except SmokeError as exc:
        return [l1.CheckResult("login", False, str(exc))]

    try:
        apps = client.list_catalog_apps()
    except SmokeError as exc:
        return [l1.CheckResult("catalog", False, str(exc))]

    targets, missing = l2.select_targets(apps, expect=expect)
    results: list[l1.CheckResult | l3.Skipped] = []
    # Connectors that produced a check rather than a skip: a pass, a failure,
    # or no answer.
    checked = 0
    # One connector at a time, as in L2; the server also runs one probe per
    # process and answers probe_busy to a second.
    for target in targets:
        try:
            response = client.probe_server(target.server_id, timeout=probe_timeout)
        except SmokeTimeout:
            results.append(l3.check_timeout(target.app_id, probe_timeout))
            checked += 1
            continue
        except SmokeError as exc:
            results.append(l3.check_request_error(target.app_id, str(exc)))
            checked += 1
            continue
        if l3.is_forbidden(response.status_code, response.payload):
            # The account itself is refused, so every other probe would be too.
            return [l3.FORBIDDEN]
        result = l3.check_probe_response(
            target.app_id, response.status_code, response.payload
        )
        checked += not isinstance(result, l3.Skipped)
        results.append(result)

    try:
        servers = client.list_servers()
    except SmokeError as exc:
        results.append(l1.CheckResult("servers", False, str(exc)))
        servers = []
    unconnected = l2.select_unconnected(
        servers,
        connected=(target.app_id for target in targets),
        missing=missing,
    )
    results.extend(
        l3.relabel(l2.check_unconnected(app_id, apps, servers))
        for app_id in unconnected
    )
    if not checked and not unconnected:
        # Every connected connector was skipped, or none is connected: a
        # misconfigured monitor account must not pass by checking nothing.
        results.append(
            l1.CheckResult("connectors", False, "no connected connector to probe")
        )
    return results


def main(
    argv: Sequence[str] | None = None,
    *,
    env: Mapping[str, str] | None = None,
    registry_rows: Iterable[Mapping[str, Any]] | None = None,
    transport: httpx.BaseTransport | None = None,
    out: TextIO | None = None,
) -> int:
    env = os.environ if env is None else env
    out = sys.stdout if out is None else out
    parser = _build_parser(env)
    args = parser.parse_args(argv)

    login = env.get(ENV_LOGIN, "")
    password = env.get(ENV_PASSWORD, "")
    if not args.base_url:
        parser.error(f"--base-url or {ENV_BASE_URL} is required")
    if not login or not password:
        parser.error(f"{ENV_LOGIN} and {ENV_PASSWORD} must be set")
    base_origin = l1.normalize_origin(args.base_url)
    if base_origin is None:
        parser.error("--base-url must be an http(s) URL")
    if args.layer == "l3":
        return _main_l3(
            args,
            parser,
            env,
            login=login,
            password=password,
            base_origin=base_origin,
            transport=transport,
            out=out,
        )
    if args.layer == "l2":
        return _main_l2(
            args,
            parser,
            env,
            login=login,
            password=password,
            base_origin=base_origin,
            transport=transport,
            out=out,
        )
    expected_origin = l1.normalize_origin(args.callback_origin or base_origin)
    if expected_origin is None:
        parser.error("--callback-origin must be an http(s) origin")
    try:
        cookies = _parse_pairs(
            _arg_or_env(args.cookie, env, ENV_COOKIES), separator=";"
        )
        oauth_accounts = _parse_pairs(
            _arg_or_env(args.oauth_account, env, ENV_OAUTH_ACCOUNTS), separator=","
        )
    except ValueError as exc:
        parser.error(f"{exc} (--cookie / --oauth-account)")

    if registry_rows is None:
        from xagent.web.builtin_mcp_registry import get_builtin_public_mcp_app_rows

        registry_rows = get_builtin_public_mcp_app_rows()
    try:
        expected = l1.expected_app_ids(
            registry_rows,
            exclude=_parse_list(_arg_or_env(args.exclude, env, ENV_EXCLUDE)),
            include=_parse_list(_arg_or_env(args.include, env, ENV_INCLUDE)),
        )
    except ValueError as exc:
        parser.error(str(exc))

    with SmokeClient(
        args.base_url,
        origin=base_origin,
        cookies=cookies,
        timeout=args.timeout,
        transport=transport,
    ) as client:
        results = run_l1(
            client,
            login=login,
            password=password,
            login_field=args.login_field,
            login_start=args.login_start,
            expected=expected,
            expected_origin=expected_origin,
            oauth_accounts=oauth_accounts,
        )

    return _report(results, out)


def _main_l2(
    args: argparse.Namespace,
    parser: argparse.ArgumentParser,
    env: Mapping[str, str],
    *,
    login: str,
    password: str,
    base_origin: str,
    transport: httpx.BaseTransport | None,
    out: TextIO,
) -> int:
    try:
        cookies = _parse_pairs(
            _arg_or_env(args.cookie, env, ENV_COOKIES), separator=";"
        )
    except ValueError as exc:
        parser.error(f"{exc} (--cookie)")
    if not math.isfinite(args.tools_timeout) or args.tools_timeout <= 0:
        parser.error("--tools-timeout must be a positive number")

    started = time.monotonic()
    with SmokeClient(
        args.base_url,
        origin=base_origin,
        cookies=cookies,
        timeout=args.timeout,
        transport=transport,
    ) as client:
        results = run_l2(
            client,
            login=login,
            password=password,
            login_field=args.login_field,
            login_start=args.login_start,
            expect=_parse_list(_arg_or_env(args.expect, env, ENV_EXPECT)),
            tools_timeout=args.tools_timeout,
        )
    return _report(results, out, suffix=f" in {time.monotonic() - started:.0f}s")


def _main_l3(
    args: argparse.Namespace,
    parser: argparse.ArgumentParser,
    env: Mapping[str, str],
    *,
    login: str,
    password: str,
    base_origin: str,
    transport: httpx.BaseTransport | None,
    out: TextIO,
) -> int:
    try:
        cookies = _parse_pairs(
            _arg_or_env(args.cookie, env, ENV_COOKIES), separator=";"
        )
    except ValueError as exc:
        parser.error(f"{exc} (--cookie)")
    if not math.isfinite(args.probe_timeout) or args.probe_timeout <= 0:
        parser.error("--probe-timeout must be a positive number")

    started = time.monotonic()
    with SmokeClient(
        args.base_url,
        origin=base_origin,
        cookies=cookies,
        timeout=args.timeout,
        transport=transport,
    ) as client:
        outcomes = run_l3(
            client,
            login=login,
            password=password,
            login_field=args.login_field,
            login_start=args.login_start,
            expect=_parse_list(_arg_or_env(args.expect, env, ENV_EXPECT)),
            probe_timeout=args.probe_timeout,
        )
    checks = [result for result in outcomes if isinstance(result, l1.CheckResult)]
    failed = sum(not result.ok for result in checks)
    for result in outcomes:
        print(l3.format_outcome(result), file=out)
    print(
        f"{len(checks)} checks, {failed} failed, "
        f"{len(outcomes) - len(checks)} skipped "
        f"in {time.monotonic() - started:.0f}s",
        file=out,
    )
    return 1 if failed else 0


def _report(results: list[l1.CheckResult], out: TextIO, *, suffix: str = "") -> int:
    failed = sum(not result.ok for result in results)
    for result in results:
        print(l1.format_result(result), file=out)
    print(f"{len(results)} checks, {failed} failed{suffix}", file=out)
    return 1 if failed else 0


def _build_parser(env: Mapping[str, str]) -> argparse.ArgumentParser:
    common = argparse.ArgumentParser(add_help=False)
    common.add_argument(
        "--base-url",
        default=env.get(ENV_BASE_URL),
        help=f"deployment base URL (env {ENV_BASE_URL})",
    )
    common.add_argument(
        "--cookie",
        action="append",
        metavar="NAME=VALUE",
        help="cookie sent on every request, e.g. selected_region=sg; repeatable "
        f"(env {ENV_COOKIES}, ';'-separated)",
    )
    common.add_argument(
        "--login-field",
        choices=("username", "email"),
        default="username",
        help="login request field that carries the login (default: username)",
    )
    common.add_argument(
        "--login-start",
        action="store_true",
        help="call /api/auth/login/start first (Xagent SaaS region routing)",
    )
    common.add_argument(
        "--timeout", type=float, default=20.0, help="seconds per request"
    )

    parser = argparse.ArgumentParser(prog="connector_smoke")
    sub = parser.add_subparsers(dest="layer", required=True)
    p = sub.add_parser(
        "l1",
        parents=[common],
        help="configuration smoke: catalog and builtin OAuth client configuration",
    )
    p.add_argument(
        "--callback-origin",
        default=env.get(ENV_CALLBACK_ORIGIN),
        help="origin every OAuth callback must use; defaults to the base URL's "
        f"(env {ENV_CALLBACK_ORIGIN})",
    )
    p.add_argument(
        "--exclude",
        action="append",
        help="registry app this deployment hides; repeatable or comma-separated "
        f"(env {ENV_EXCLUDE})",
    )
    p.add_argument(
        "--include",
        action="append",
        help="app this deployment shows beyond the registry default; repeatable "
        f"or comma-separated (env {ENV_INCLUDE})",
    )
    p.add_argument(
        "--oauth-account",
        action="append",
        metavar="PROVIDER=LABEL",
        help="account label for a provider hosted on the customer's account, "
        f"e.g. zendesk=acme; repeatable (env {ENV_OAUTH_ACCOUNTS})",
    )

    p = sub.add_parser(
        "l2",
        parents=[common],
        help="connection smoke: start every connected connector and list its tools",
    )
    p.add_argument(
        "--expect",
        action="append",
        help="app the monitor account must have connected; repeatable or "
        f"comma-separated (env {ENV_EXPECT})",
    )
    p.add_argument(
        "--tools-timeout",
        type=float,
        default=DEFAULT_TOOLS_TIMEOUT,
        help="seconds to wait for one connector to start and list its tools "
        f"(default: {DEFAULT_TOOLS_TIMEOUT:g})",
    )

    p = sub.add_parser(
        "l3",
        parents=[common],
        help="tool-call smoke: run every connected connector's read-only probe",
    )
    p.add_argument(
        "--expect",
        action="append",
        help="app the monitor account must have connected; repeatable or "
        f"comma-separated (env {ENV_EXPECT})",
    )
    p.add_argument(
        "--probe-timeout",
        type=float,
        default=DEFAULT_PROBE_TIMEOUT,
        help="seconds to wait for one connector to start and answer its probe "
        f"(default: {DEFAULT_PROBE_TIMEOUT:g})",
    )
    return parser


def _arg_or_env(
    values: list[str] | None, env: Mapping[str, str], name: str
) -> list[str]:
    """Command line values replace the environment variable's, never extend it."""
    if values is not None:
        return values
    value = env.get(name, "")
    return [value] if value.strip() else []


def _parse_list(values: Iterable[str]) -> set[str]:
    return {
        item.strip() for value in values for item in value.split(",") if item.strip()
    }


def _parse_pairs(values: Iterable[str], *, separator: str) -> dict[str, str]:
    pairs: dict[str, str] = {}
    for value in values:
        for item in value.split(separator):
            item = item.strip()
            if not item:
                continue
            name, sep, pair_value = item.partition("=")
            if not sep or not name.strip() or not pair_value.strip():
                # Never echo the item: a cookie value can be a credential.
                raise ValueError("expected NAME=VALUE pairs")
            pairs[name.strip()] = pair_value.strip()
    return pairs