Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

6.1 KB
Raw Blame
"""Every tool of a connector that adopted V2 has a check or an exemption (#3565).

A builtin connector either adopted V2 (``CONNECTOR_CHECKS``) or is on the
legacy list (``LEGACY_CONNECTORS``), which only shrinks. For an in-repo
connector, started as ``python -m``, the tools its server registers must each
have a check or a written exemption, and every tool a declaration names must
exist. A remote (``url``) connector's tools are the vendor's and change
without a release, and an ``npx`` connector's are not in this repository, so
neither is held to per-tool coverage.
"""

from __future__ import annotations

import asyncio
import importlib
import re
from typing import Any

import pytest
from mcp.server.fastmcp import FastMCP

from xagent.web.builtin_mcp_registry import get_builtin_public_mcp_app_rows
from xagent.web.connector_checks.registry import (
    _GMAIL_LABEL_WITHOUT_READ,
    CONNECTOR_CHECKS,
    LEGACY_CONNECTORS,
    WRITE_WITHOUT_CLEANUP,
)

# LEGACY_CONNECTORS, pinned. A connector that adopts V2 leaves both lists in
# the same PR; never add a name here, so going back to legacy shows up in the
# diff of this file.
_PINNED_LEGACY = frozenset(
    {
        "aws",
        "chartmogul",
        "chrome-devtools",
        "facebook",
        "fireflies",
        "freshdesk",
        "google-maps",
        "google-search-console",
        "granola",
        "instagram",
        "linkedin",
        "magento",
        "meta-ads",
        "myob",
        "posthog",
        "rocketlane",
        "salesforce",
        "shopify",
        "stripe",
        "whatsapp",
    }
)


def _rows() -> dict[str, dict[str, Any]]:
    return {str(row["app_id"]): row for row in get_builtin_public_mcp_app_rows()}


def _module_name(row: dict[str, Any]) -> str | None:
    """The in-repo server module of a ``python -m`` connector, else None."""
    launch = row["launch_config"]
    args = launch.get("args") or []
    if launch.get("command") == "python" and len(args) == 2 and args[0] == "-m":
        return str(args[1])
    return None


def _registered_tools(module_name: str) -> set[str]:
    module = importlib.import_module(module_name)
    server = getattr(module, "mcp", None)
    if server is not None and hasattr(server, "list_tools"):
        tools = asyncio.run(server.list_tools())
    else:
        # The low-level Server modules (linkedin) expose the handler directly.
        tools = asyncio.run(module.list_tools())
    registered = {tool.name for tool in tools}
    # zoom offers its meeting-write tool only when a deployment switch is on;
    # an exemption has to cover it whether or not this process sees it.
    register_gated = getattr(module, "_register_meeting_write_tools", None)
    if register_gated is not None:
        gated = FastMCP(f"{module_name}-coverage")
        register_gated(gated)
        registered |= {tool.name for tool in asyncio.run(gated.list_tools())}
    return registered


def _adopted_in_repo() -> list[str]:
    rows = _rows()
    return sorted(
        app_id
        for app_id in CONNECTOR_CHECKS
        if app_id in rows and _module_name(rows[app_id]) is not None
    )


def test_the_legacy_list_is_pinned():
    added = LEGACY_CONNECTORS - _PINNED_LEGACY
    assert not added, (
        f"{sorted(added)} joined the legacy list; a connector ships its V2 "
        "declaration instead"
    )
    left = _PINNED_LEGACY - LEGACY_CONNECTORS
    assert not left, (
        f"{sorted(left)} left the legacy list; remove them from _PINNED_LEGACY too"
    )


def test_an_adopted_connector_is_not_legacy():
    assert not LEGACY_CONNECTORS & set(CONNECTOR_CHECKS)


def test_every_builtin_connector_is_adopted_or_legacy():
    """Remote and ``npx`` connectors included: per-tool coverage skips them,
    so this is what stops one shipping with no V2 declaration."""
    rows = _rows()
    assert any(row["launch_config"].get("url") for row in rows.values())
    assert any(row["launch_config"].get("command") == "npx" for row in rows.values())
    undeclared = set(rows) - set(CONNECTOR_CHECKS) - LEGACY_CONNECTORS
    assert not undeclared, (
        f"{sorted(undeclared)} need a V2 declaration in "
        "web/connector_checks/registry.py"
    )


def test_a_legacy_connector_is_a_builtin_connector():
    removed = LEGACY_CONNECTORS - set(_rows())
    assert not removed, f"{sorted(removed)} are no longer builtin connectors"


# A reason that names no issue as the one that decides the check: it says why
# the tool has no check without one. Every other exemption is pending and names
# its issue.
_REASONS_THAT_NAME_NO_ISSUE = frozenset(
    {WRITE_WITHOUT_CLEANUP, _GMAIL_LABEL_WITHOUT_READ}
)


@pytest.mark.parametrize("app_id", sorted(CONNECTOR_CHECKS))
def test_a_pending_exemption_names_the_issue_that_decides_it(app_id):
    pending = [
        exemption
        for exemption in CONNECTOR_CHECKS[app_id].exemptions
        if exemption.reason not in _REASONS_THAT_NAME_NO_ISSUE
    ]
    unnamed = [e.tool for e in pending if not re.search(r"decided in #\d+$", e.reason)]
    assert not unnamed, f"{app_id}: {unnamed} need the issue that decides their check"


@pytest.mark.parametrize("app_id", _adopted_in_repo())
def test_every_tool_has_a_check_or_an_exemption(app_id):
    checks = CONNECTOR_CHECKS[app_id]
    module_name = _module_name(_rows()[app_id])
    assert module_name is not None
    registered = _registered_tools(module_name)
    exempted = {exemption.tool for exemption in checks.exemptions}
    covered = set(checks.checked_tools()) | exempted

    uncovered = registered - covered
    assert not uncovered, (
        f"{app_id}: {sorted(uncovered)} need a check or an exemption with a reason"
    )
    unknown = covered - registered
    assert not unknown, f"{app_id}: {sorted(unknown)} are not tools of the server"


def test_a_remote_connector_declares_no_exemptions():
    """Its tools change with the vendor, so an exemption list would rot."""
    rows = _rows()
    remote = [
        app_id
        for app_id in CONNECTOR_CHECKS
        if app_id in rows and rows[app_id]["launch_config"].get("url")
    ]
    assert remote, "expected remote connectors in CONNECTOR_CHECKS"
    assert all(not CONNECTOR_CHECKS[app_id].exemptions for app_id in remote)