Enterprise agent platform for building, deploying, and operating AI teammates.
"""The smoke V2 engine, run through a fake identity adapter."""
from __future__ import annotations
import asyncio
import json
from collections.abc import Callable, Mapping
from datetime import datetime, timezone
from typing import Any, cast
import pytest
from jsonschema import Draft202012Validator # type: ignore[import-untyped]
from mcp.types import Tool
from xagent.core.tools.adapters.vibe.mcp_adapter import MCPToolAdapter, MCPWriteHint
from xagent.core.tools.adapters.vibe.mcp_approval_gate import gate_mcp_tools
from xagent.web.connector_checks import engine
from xagent.web.connector_checks.declarations import (
ConnectorChecks,
Equals,
HasKeys,
ItemRef,
L3Envelope,
ReadCheck,
RunScopedName,
StatusSuccess,
Sweep,
TenantPin,
ToolExemption,
ToolStep,
UnprobeableCheck,
WriteCheck,
)
from xagent.web.connector_checks.engine import run_checks
from xagent.web.connector_checks.fixtures import (
FixtureMissing,
FixtureRef,
FixtureValue,
)
from xagent.web.connector_checks.identity import (
ConnectorTarget,
IdentitySetupError,
SessionPurpose,
)
from xagent.web.connector_checks.results import RunResult, load_result_schema
from xagent.web.services.connector_probe import find_tool
from xagent.web.services.mcp_connector_session import (
ConnectorToolsOutcome,
ConnectorToolsResult,
)
from .fakes import (
SUCCESS,
FakeAdapter,
FakeTool,
declared,
failed,
listed,
rows,
targets,
)
def read(check_id: str, tool: str, **fields: Any) -> ReadCheck:
fields.setdefault("assertions", (StatusSuccess(),))
return ReadCheck(check_id, tool, **fields)
def pin(app: str) -> TenantPin:
return TenantPin(
ToolStep(f"{app}_whoami", assertions=(StatusSuccess(),)),
("tenant",),
FixtureRef(f"{app}.tenant_id"),
)
def write(check_id: str) -> WriteCheck:
app = check_id.partition(".")[0]
name = RunScopedName("draft")
folder = FixtureRef(f"{app}.write_folder_id")
return WriteCheck(
check_id,
create=ToolStep(
f"{app}_create",
{"folder": folder, "name": name},
assertions=(StatusSuccess(),),
),
verify=ToolStep(f"{app}_get", {"name": name}, assertions=(StatusSuccess(),)),
cleanup=ToolStep(
f"{app}_delete", {"name": name}, assertions=(StatusSuccess(),)
),
tenant_pin=pin(app),
expected_verdicts={f"{app}_create": "allow", f"{app}_delete": "allow"},
)
def sweep(app: str) -> Sweep:
return Sweep(
tenant_pin=pin(app),
find=ToolStep(
f"{app}_list",
{"folder": FixtureRef(f"{app}.write_folder_id")},
assertions=(StatusSuccess(),),
),
items=("items",),
name=("name",),
delete=ToolStep(
f"{app}_delete", {"id": ItemRef(("id",))}, assertions=(StatusSuccess(),)
),
)
def assert_valid_document(result: RunResult) -> None:
Draft202012Validator(load_result_schema()).validate(result.to_document())
# Sessions and checks
async def test_a_passing_session_runs_each_check_once_in_declaration_order() -> None:
search = FakeTool("gmail_search")
profile = FakeTool("gmail_profile")
adapter = FakeAdapter(targets("gmail"), {"gmail": listed(search, profile)})
checks = ConnectorChecks(
"gmail",
reads=(
read("gmail.search", "gmail_search", arguments={"max_results": 1}),
read("gmail.profile", "gmail_profile"),
),
)
result = await run_checks(adapter, declarations=declared(checks))
assert rows(result) == [
("gmail", None, "pass", None, None),
("gmail", "gmail.search", "pass", None, None),
("gmail", "gmail.profile", "pass", None, None),
]
assert adapter.opened == [("gmail", SessionPurpose.CALL_TOOLS)]
assert search.calls == [{"max_results": 1}]
assert profile.calls == [{}]
assert result.complete is True
assert result.failure is None
assert result.identity == "user"
assert_valid_document(result)
async def test_the_environment_label_reaches_the_document() -> None:
adapter = FakeAdapter(targets("gmail"), {"gmail": listed(FakeTool("gmail_x"))})
checks = ConnectorChecks("gmail", reads=(read("gmail.x", "gmail_x"),))
result = await run_checks(
adapter, environment="stage", declarations=declared(checks)
)
assert result.to_document()["environment"] == "stage"
async def test_a_failed_session_skips_its_checks_and_keeps_static_ones() -> None:
adapter = FakeAdapter(
targets("jira"),
{"jira": failed(ConnectorToolsOutcome.NOT_CONNECTED, "oauth_token_required")},
)
checks = ConnectorChecks(
"jira",
reads=(
read("jira.sites", "jira_sites"),
UnprobeableCheck("jira.boards", "needs_fixture"),
),
writes=(write("jira.issue_round_trip"),),
sweep=sweep("jira"),
)
result = await run_checks(adapter, declarations=declared(checks))
assert rows(result) == [
("jira", None, "fail", "not_connected", "oauth_token_required"),
("jira", "jira.sites", "skip", "session_unavailable", None),
("jira", "jira.boards", "unprobeable", None, "needs_fixture"),
("jira", "jira.issue_round_trip", "skip", "writes_disabled", None),
]
assert_valid_document(result)
async def test_a_rejected_client_credential_is_its_own_session_class() -> None:
adapter = FakeAdapter(
targets("jira"),
{
"jira": failed(
ConnectorToolsOutcome.NOT_CONNECTED, "client_rejected:atlassian"
)
},
)
checks = ConnectorChecks("jira", reads=(read("jira.sites", "jira_sites"),))
result = await run_checks(adapter, declarations=declared(checks))
assert rows(result)[0] == (
"jira",
None,
"fail",
"client_rejected",
"client_rejected:atlassian",
)
async def test_a_target_the_identity_skips_is_never_started() -> None:
adapter = FakeAdapter([ConnectorTarget("slack", skip_reason="slack_bot_grant")])
checks = ConnectorChecks(
"slack",
reads=(read("slack.channels", "slack_channels"),),
writes=(write("slack.message_round_trip"),),
sweep=sweep("slack"),
)
result = await run_checks(adapter, declarations=declared(checks))
assert rows(result) == [
("slack", None, "skip", "skipped_by_identity", "slack_bot_grant"),
("slack", "slack.channels", "skip", "session_unavailable", None),
("slack", "slack.message_round_trip", "skip", "writes_disabled", None),
]
assert adapter.opened == []
async def test_an_app_without_a_declaration_lists_and_reports_not_declared() -> None:
adapter = FakeAdapter(
targets("Acme CRM"), {"Acme CRM": listed(FakeTool("acme_list"))}
)
result = await run_checks(adapter, declarations=declared())
label = ConnectorTarget("Acme CRM").connector
assert rows(result) == [
(label, None, "pass", None, None),
(label, f"{label}.l3_probe", "skip", "not_declared", None),
]
assert adapter.opened == [("Acme CRM", SessionPurpose.LIST_ONLY)]
assert "Acme CRM" not in json.dumps(result.to_document())
assert_valid_document(result)
async def test_an_undeclared_app_keeps_not_declared_under_a_failed_session() -> None:
adapter = FakeAdapter(
targets("acme"),
{"acme": failed(ConnectorToolsOutcome.FAILED_TO_START, "initialize")},
)
result = await run_checks(adapter, declarations=declared())
assert rows(result) == [
("acme", None, "fail", "failed_to_start", "initialize"),
("acme", "acme.l3_probe", "skip", "not_declared", None),
]
async def test_a_connector_whose_checks_are_all_unprobeable_lists_only() -> None:
adapter = FakeAdapter(
targets("google-maps"), {"google-maps": listed(FakeTool("maps_search"))}
)
checks = ConnectorChecks(
"google-maps",
reads=(UnprobeableCheck("google-maps.l3_probe", "sandboxed"),),
)
result = await run_checks(adapter, declarations=declared(checks))
assert adapter.opened == [("google-maps", SessionPurpose.LIST_ONLY)]
assert rows(result) == [
("google-maps", None, "pass", None, None),
("google-maps", "google-maps.l3_probe", "unprobeable", None, "sandboxed"),
]
async def test_a_write_check_starts_the_session_for_tool_calls() -> None:
# The purpose follows the declarations, so the session starts the same
# way once the write switch is on.
adapter = FakeAdapter(targets("jira"), {"jira": listed(FakeTool("jira_x"))})
checks = ConnectorChecks(
"jira",
writes=(write("jira.issue_round_trip"),),
sweep=sweep("jira"),
)
await run_checks(adapter, declarations=declared(checks))
assert adapter.opened == [("jira", SessionPurpose.CALL_TOOLS)]
async def test_no_write_check_calls_a_tool() -> None:
tools = [FakeTool(f"jira_{name}") for name in ("create", "get", "delete")]
whoami = FakeTool("jira_whoami")
adapter = FakeAdapter(targets("jira"), {"jira": listed(*tools, whoami)})
checks = ConnectorChecks(
"jira",
writes=(write("jira.issue_round_trip"),),
sweep=sweep("jira"),
)
result = await run_checks(adapter, declarations=declared(checks))
assert rows(result)[1] == (
"jira",
"jira.issue_round_trip",
"skip",
"writes_disabled",
None,
)
assert result.to_document()["rows"][1]["tool"] == "jira_create"
assert all(tool.calls == [] for tool in [*tools, whoami])
async def test_an_exception_opening_a_session_fails_only_that_connector() -> None:
adapter = FakeAdapter(
targets("jira", "gmail"),
{
"jira": RuntimeError("secret-token in a message"),
"gmail": listed(FakeTool("gmail_x")),
},
)
checks = (
ConnectorChecks("jira", reads=(read("jira.sites", "jira_sites"),)),
ConnectorChecks("gmail", reads=(read("gmail.x", "gmail_x"),)),
)
result = await run_checks(adapter, declarations=declared(*checks))
assert rows(result) == [
("jira", None, "fail", "failed_to_start", "exception"),
("jira", "jira.sites", "skip", "session_unavailable", None),
("gmail", None, "pass", None, None),
("gmail", "gmail.x", "pass", None, None),
]
assert "secret-token" not in json.dumps(result.to_document())
# One check
async def test_a_tool_the_server_does_not_list_fails_as_tool_missing() -> None:
adapter = FakeAdapter(targets("gmail"), {"gmail": listed(FakeTool("other"))})
checks = ConnectorChecks("gmail", reads=(read("gmail.x", "gmail_x"),))
result = await run_checks(adapter, declarations=declared(checks))
assert rows(result)[1] == ("gmail", "gmail.x", "fail", "tool_missing", None)
async def test_a_raising_call_fails_without_its_message() -> None:
tool = FakeTool("gmail_x", error=RuntimeError("Bearer abc123 rejected"))
adapter = FakeAdapter(targets("gmail"), {"gmail": listed(tool)})
checks = ConnectorChecks("gmail", reads=(read("gmail.x", "gmail_x"),))
result = await run_checks(adapter, declarations=declared(checks))
assert rows(result)[1] == ("gmail", "gmail.x", "fail", "call_failed", "exception")
assert "abc123" not in json.dumps(result.to_document())
async def test_an_assertion_failure_names_its_kind_and_never_the_body() -> None:
body = {"status": "success", "email": "[email protected]"}
tool = FakeTool(
"gmail_x",
result={"content": [{"type": "text", "text": json.dumps(body)}]},
)
adapter = FakeAdapter(targets("gmail"), {"gmail": listed(tool)})
checks = ConnectorChecks(
"gmail",
reads=(read("gmail.x", "gmail_x", assertions=(HasKeys((), ("id",)),)),),
)
result = await run_checks(adapter, declarations=declared(checks))
assert rows(result)[1] == (
"gmail",
"gmail.x",
"fail",
"assertion_failed",
"has_keys",
)
assert "[email protected]" not in json.dumps(result.to_document())
async def test_a_slow_call_times_out_and_the_next_check_still_runs() -> None:
slow = FakeTool("gmail_slow", delay=1.0)
fast = FakeTool("gmail_fast")
adapter = FakeAdapter(targets("gmail"), {"gmail": listed(slow, fast)})
checks = ConnectorChecks(
"gmail",
reads=(read("gmail.slow", "gmail_slow"), read("gmail.fast", "gmail_fast")),
)
result = await run_checks(
adapter, declarations=declared(checks), check_timeout=0.05
)
assert rows(result)[1:] == [
("gmail", "gmail.slow", "fail", "timed_out", "check_call"),
("gmail", "gmail.fast", "pass", None, None),
]
assert result.complete is True
async def test_each_call_gets_its_own_copy_of_the_declared_arguments() -> None:
class Mutating(FakeTool):
async def run_json_async(self, args: Mapping[str, Any]) -> Any:
cast(dict[str, Any], args)["max_results"] = 99
return await super().run_json_async(args)
tool = Mutating("gmail_x")
arguments = {"max_results": 1}
adapter = FakeAdapter(targets("gmail"), {"gmail": listed(tool)})
checks = ConnectorChecks(
"gmail", reads=(read("gmail.x", "gmail_x", arguments=arguments),)
)
await run_checks(adapter, declarations=declared(checks))
assert arguments == {"max_results": 1}
# Fixtures
async def test_fixture_keys_resolve_into_arguments_and_expected_values() -> None:
document = {"status": "success", "document_id": "doc-1", "title": "Seed"}
tool = FakeTool(
"google_docs_get_document",
result={"content": [{"type": "text", "text": json.dumps(document)}]},
)
adapter = FakeAdapter(
targets("google-docs"),
{"google-docs": listed(tool)},
fixtures={"google-docs.document_id": "doc-1", "google-docs.title": "Seed"},
)
checks = ConnectorChecks(
"google-docs",
reads=(
read(
"google-docs.read_fixture",
"google_docs_get_document",
arguments={"document_id": FixtureRef("google-docs.document_id")},
assertions=(
StatusSuccess(),
Equals(("title",), FixtureRef("google-docs.title")),
),
),
),
)
result = await run_checks(adapter, declarations=declared(checks))
assert tool.calls == [{"document_id": "doc-1"}]
assert rows(result)[1] == (
"google-docs",
"google-docs.read_fixture",
"pass",
None,
None,
)
assert "doc-1" not in json.dumps(result.to_document())
@pytest.mark.parametrize("missing", ["google-docs.document_id", "google-docs.title"])
async def test_a_missing_fixture_skips_the_check_without_calling(
missing: str,
) -> None:
tool = FakeTool("google_docs_get_document")
fixtures = {"google-docs.document_id": "doc-1", "google-docs.title": "Seed"}
del fixtures[missing]
adapter = FakeAdapter(
targets("google-docs"), {"google-docs": listed(tool)}, fixtures=fixtures
)
checks = ConnectorChecks(
"google-docs",
reads=(
read(
"google-docs.read_fixture",
"google_docs_get_document",
arguments={"document_id": FixtureRef("google-docs.document_id")},
assertions=(Equals(("title",), FixtureRef("google-docs.title")),),
),
),
)
result = await run_checks(adapter, declarations=declared(checks))
assert rows(result) == [
("google-docs", None, "pass", None, None),
("google-docs", "google-docs.read_fixture", "skip", "fixture_missing", missing),
]
assert tool.calls == []
# The purpose follows the declarations, never the fixtures: setting the
# fixture must not change how the connector starts.
assert adapter.opened == [("google-docs", SessionPurpose.CALL_TOOLS)]
def fixture_checks() -> tuple[ConnectorChecks, ConnectorChecks]:
return (
ConnectorChecks("gmail", reads=(read("gmail.x", "gmail_x"),)),
ConnectorChecks(
"google-docs",
reads=(
read(
"google-docs.read_fixture",
"google_docs_get_document",
arguments={"document_id": FixtureRef("google-docs.document_id")},
),
),
),
)
async def test_a_fixture_that_is_not_a_scalar_fails_before_anything_starts() -> None:
adapter = FakeAdapter(
targets("gmail", "google-docs"),
{"gmail": listed(FakeTool("gmail_x")), "google-docs": listed()},
fixtures=cast(Any, {"google-docs.document_id": {"id": "doc-1"}}),
)
with pytest.raises(TypeError):
await run_checks(adapter, declarations=declared(*fixture_checks()))
assert adapter.opened == []
async def test_a_resolver_that_raises_fails_the_run_without_its_message(
caplog: pytest.LogCaptureFixture,
) -> None:
class Raising(FakeAdapter):
def resolve_fixture(self, key: str) -> FixtureValue | None:
raise RuntimeError("config store unavailable")
adapter = Raising(
targets("gmail", "google-docs"),
{"gmail": listed(FakeTool("gmail_x")), "google-docs": listed()},
)
result = await run_checks(adapter, declarations=declared(*fixture_checks()))
assert result.failure == "fixtures:exception"
assert result.rows == ()
assert result.complete is False
assert adapter.opened == []
assert "config store unavailable" not in json.dumps(result.to_document())
assert "config store unavailable" not in caplog.text
async def test_an_adapter_that_raises_fixture_missing_means_a_missing_key() -> None:
class Missing(FakeAdapter):
def resolve_fixture(self, key: str) -> FixtureValue | None:
# The row names the requested key, never the one the adapter gave.
raise FixtureMissing("Store key for [email protected]")
adapter = Missing(
targets("gmail", "google-docs"),
{"gmail": listed(FakeTool("gmail_x")), "google-docs": listed()},
)
result = await run_checks(adapter, declarations=declared(*fixture_checks()))
assert result.failure is None
assert (
"google-docs",
"google-docs.read_fixture",
"skip",
"fixture_missing",
"google-docs.document_id",
) in rows(result)
async def test_an_invalid_declaration_raises_whatever_the_target_order() -> None:
class Raising(FakeAdapter):
def resolve_fixture(self, key: str) -> FixtureValue | None:
raise RuntimeError("config store unavailable")
adapter = Raising(
targets("google-docs", "gmail"),
{"google-docs": listed(), "gmail": listed()},
)
docs = fixture_checks()[1]
wrong_app = ConnectorChecks("jira", reads=(read("jira.x", "jira_x"),))
with pytest.raises(ValueError):
await run_checks(
adapter,
declarations=lambda app: docs if app == "google-docs" else wrong_app,
)
assert adapter.opened == []
async def test_a_failed_session_outranks_a_missing_fixture() -> None:
adapter = FakeAdapter(
targets("google-docs"),
{"google-docs": failed(ConnectorToolsOutcome.NOT_CONNECTED, None)},
)
checks = ConnectorChecks(
"google-docs",
reads=(
read(
"google-docs.read_fixture",
"google_docs_get_document",
arguments={"document_id": FixtureRef("google-docs.document_id")},
),
),
)
result = await run_checks(adapter, declarations=declared(checks))
assert rows(result)[1][2:] == ("skip", "session_unavailable", None)
# readOnlyHint
def hinted_check() -> ConnectorChecks:
return ConnectorChecks(
"notion",
reads=(
read(
"notion.current_user",
"notion-get-self",
require_read_only_hint=True,
),
),
)
async def test_a_tool_advertised_read_only_is_called() -> None:
tool = FakeTool("notion-get-self", write_hint=MCPWriteHint.READ_ONLY)
adapter = FakeAdapter(targets("notion"), {"notion": listed(tool)})
result = await run_checks(adapter, declarations=declared(hinted_check()))
assert rows(result)[1] == ("notion", "notion.current_user", "pass", None, None)
assert tool.calls == [{}]
@pytest.mark.parametrize(
"hint", [MCPWriteHint.UNDECLARED, MCPWriteHint.DESTRUCTIVE, None]
)
async def test_a_tool_not_advertised_read_only_is_skipped_uncalled(
hint: MCPWriteHint | None,
) -> None:
tool = FakeTool("notion-get-self", write_hint=cast(MCPWriteHint, hint))
adapter = FakeAdapter(targets("notion"), {"notion": listed(tool)})
result = await run_checks(adapter, declarations=declared(hinted_check()))
assert rows(result)[1] == (
"notion",
"notion.current_user",
"skip",
"read_only_hint_missing",
None,
)
assert tool.calls == []
@pytest.mark.parametrize(
("annotations", "advertised"),
[
({"readOnlyHint": True}, True),
({"readOnlyHint": True, "destructiveHint": True}, False),
({"readOnlyHint": 1}, False),
(None, False),
],
)
def test_the_hint_is_read_through_the_approval_gate(
annotations: dict[str, Any] | None, advertised: bool
) -> None:
# The loader hands back MCPToolAdapters wrapped in the approval gate.
connection: Any = {
"transport": "streamable_http",
"url": "https://mcp.example.test",
}
mcp_tool = Tool(
name="notion-get-self",
description="Who am I",
inputSchema={"type": "object", "properties": {}},
)
adapter = MCPToolAdapter(
mcp_tool=mcp_tool, connection=connection, raw_annotations=annotations
)
(gated,) = gate_mcp_tools([adapter], connection=connection)
assert find_tool([gated], "notion-get-self") is gated
assert engine._advertises_read_only(gated) is advertised
async def test_a_missing_tool_under_the_hint_is_skipped_not_missing() -> None:
adapter = FakeAdapter(targets("notion"), {"notion": listed(FakeTool("other"))})
result = await run_checks(adapter, declarations=declared(hinted_check()))
assert rows(result)[1][2:] == ("skip", "read_only_hint_missing", None)
# The run's total timeout
async def test_the_total_timeout_cuts_the_run_and_marks_what_it_kept_out() -> None:
slow = FakeTool("gmail_slow", delay=1.0)
adapter = FakeAdapter(
targets("gmail", "jira"),
{
"gmail": listed(slow, FakeTool("gmail_fast")),
"jira": listed(FakeTool("jira_sites")),
},
)
checks = (
ConnectorChecks(
"gmail",
reads=(
read("gmail.slow", "gmail_slow"),
read("gmail.fast", "gmail_fast"),
UnprobeableCheck("gmail.labels", "needs_fixture"),
),
writes=(write("gmail.draft_round_trip"),),
sweep=sweep("gmail"),
),
ConnectorChecks(
"jira",
reads=(
read("jira.sites", "jira_sites"),
UnprobeableCheck("jira.boards", "needs_fixture"),
),
),
)
result = await run_checks(
adapter, declarations=declared(*checks), check_timeout=5, total_timeout=0.1
)
assert rows(result) == [
("gmail", None, "pass", None, None),
("gmail", "gmail.slow", "fail", "timed_out", "run_deadline"),
("gmail", "gmail.fast", "fail", "not_reached", None),
("gmail", "gmail.labels", "unprobeable", None, "needs_fixture"),
("gmail", "gmail.draft_round_trip", "skip", "writes_disabled", None),
("jira", None, "fail", "not_reached", None),
("jira", "jira.sites", "skip", "session_unavailable", None),
("jira", "jira.boards", "unprobeable", None, "needs_fixture"),
]
assert result.complete is False
assert adapter.opened == [("gmail", SessionPurpose.CALL_TOOLS)]
assert_valid_document(result)
async def test_the_total_timeout_during_a_session_fails_that_session() -> None:
async def hang() -> ConnectorToolsResult:
await asyncio.sleep(1.0)
return listed()
adapter = FakeAdapter(targets("gmail"), {"gmail": hang})
checks = ConnectorChecks("gmail", reads=(read("gmail.x", "gmail_x"),))
result = await run_checks(
adapter, declarations=declared(checks), total_timeout=0.05
)
assert rows(result) == [
("gmail", None, "fail", "timed_out", "run_deadline"),
("gmail", "gmail.x", "skip", "session_unavailable", None),
]
assert result.complete is False
async def test_a_timeout_the_adapter_raises_itself_does_not_cut_the_run() -> None:
adapter = FakeAdapter(
targets("jira", "gmail"),
{"jira": TimeoutError(), "gmail": listed(FakeTool("gmail_x"))},
)
checks = (
ConnectorChecks("jira", reads=(read("jira.sites", "jira_sites"),)),
ConnectorChecks("gmail", reads=(read("gmail.x", "gmail_x"),)),
)
result = await run_checks(adapter, declarations=declared(*checks))
assert rows(result) == [
("jira", None, "fail", "failed_to_start", "exception"),
("jira", "jira.sites", "skip", "session_unavailable", None),
("gmail", None, "pass", None, None),
("gmail", "gmail.x", "pass", None, None),
]
assert result.complete is True
async def test_a_timeout_the_tool_raises_itself_does_not_cut_the_run() -> None:
# The run's deadline is the tighter bound, but it has not expired.
tool = FakeTool("gmail_x", error=TimeoutError())
adapter = FakeAdapter(
targets("gmail"), {"gmail": listed(tool, FakeTool("gmail_y"))}
)
checks = ConnectorChecks(
"gmail", reads=(read("gmail.x", "gmail_x"), read("gmail.y", "gmail_y"))
)
result = await run_checks(
adapter, declarations=declared(checks), check_timeout=60, total_timeout=30
)
assert rows(result)[1:] == [
("gmail", "gmail.x", "fail", "timed_out", "check_call"),
("gmail", "gmail.y", "pass", None, None),
]
assert result.complete is True
async def test_a_call_that_swallows_its_timeout_still_timed_out() -> None:
class Swallowing(FakeTool):
async def run_json_async(self, args: Mapping[str, Any]) -> Any:
try:
await asyncio.sleep(1.0)
except asyncio.CancelledError:
return {"content": [], "is_error": True}
return SUCCESS
adapter = FakeAdapter(
targets("gmail"),
{"gmail": listed(Swallowing("gmail_x"), FakeTool("gmail_y"))},
)
checks = ConnectorChecks(
"gmail", reads=(read("gmail.x", "gmail_x"), read("gmail.y", "gmail_y"))
)
by_call = await run_checks(
adapter, declarations=declared(checks), check_timeout=0.05
)
by_deadline = await run_checks(
adapter, declarations=declared(checks), check_timeout=5, total_timeout=0.05
)
assert rows(by_call)[1:] == [
("gmail", "gmail.x", "fail", "timed_out", "check_call"),
("gmail", "gmail.y", "pass", None, None),
]
assert by_call.complete is True
assert rows(by_deadline)[1:] == [
("gmail", "gmail.x", "fail", "timed_out", "run_deadline"),
("gmail", "gmail.y", "fail", "not_reached", None),
]
assert by_deadline.complete is False
async def test_a_session_that_swallows_the_deadline_still_timed_out() -> None:
async def swallow() -> ConnectorToolsResult:
try:
await asyncio.sleep(1.0)
except asyncio.CancelledError:
pass
return listed(FakeTool("gmail_x"))
adapter = FakeAdapter(targets("gmail"), {"gmail": swallow})
checks = ConnectorChecks("gmail", reads=(read("gmail.x", "gmail_x"),))
result = await run_checks(
adapter, declarations=declared(checks), total_timeout=0.05
)
assert rows(result) == [
("gmail", None, "fail", "timed_out", "run_deadline"),
("gmail", "gmail.x", "skip", "session_unavailable", None),
]
assert result.complete is False
async def test_a_wall_clock_step_back_keeps_the_run(
monkeypatch: pytest.MonkeyPatch,
) -> None:
moments = iter(
[
datetime(2026, 11, 8, 10, 0, tzinfo=timezone.utc),
datetime(2026, 11, 8, 9, 0, tzinfo=timezone.utc),
]
)
class SteppingClock:
@staticmethod
def now(tz: Any = None) -> datetime:
return next(moments)
monkeypatch.setattr(engine, "datetime", SteppingClock)
adapter = FakeAdapter(targets("gmail"), {"gmail": listed(FakeTool("gmail_x"))})
checks = ConnectorChecks("gmail", reads=(read("gmail.x", "gmail_x"),))
result = await run_checks(adapter, declarations=declared(checks))
assert result.started_at == datetime(2026, 11, 8, 10, 0, tzinfo=timezone.utc)
assert result.finished_at >= result.started_at
assert rows(result)[1][2] == "pass"
# Setup and selection
@pytest.mark.parametrize("selector", ["connectors", "checks"])
async def test_a_selector_must_not_be_one_string(selector: str) -> None:
adapter, checks = two_connectors()
with pytest.raises(TypeError):
selection: dict[str, Any] = {selector: "gmail"}
await run_checks(adapter, declarations=checks, **selection)
assert adapter.opened == []
async def test_an_identity_setup_error_is_the_run_failure() -> None:
adapter = FakeAdapter(
[], setup_error=IdentitySetupError("identity", "not_a_probe_user")
)
result = await run_checks(adapter)
assert result.failure == "identity:not_a_probe_user"
assert result.rows == ()
assert result.complete is False
assert_valid_document(result)
async def test_an_unexpected_enumeration_error_is_a_failure_code() -> None:
adapter = FakeAdapter([], setup_error=RuntimeError("db host 10.0.0.1"))
result = await run_checks(adapter)
assert result.failure == "enumeration:exception"
assert "10.0.0.1" not in json.dumps(result.to_document())
async def test_an_identity_without_connectors_is_a_failure() -> None:
result = await run_checks(FakeAdapter([]))
assert result.failure == "enumeration:no_connectors"
assert result.complete is False
async def test_two_targets_with_one_label_fail_before_anything_starts() -> None:
adapter = FakeAdapter(
[ConnectorTarget("gmail"), ConnectorTarget("gmail")],
{"gmail": listed()},
)
result = await run_checks(adapter, declarations=declared())
assert result.failure == "enumeration:duplicate_connector"
assert adapter.opened == []
def two_connectors() -> tuple[FakeAdapter, Callable[[str], ConnectorChecks | None]]:
adapter = FakeAdapter(
targets("gmail", "jira"),
{
"gmail": listed(FakeTool("gmail_a"), FakeTool("gmail_b")),
"jira": listed(FakeTool("jira_sites")),
},
)
checks = declared(
ConnectorChecks(
"gmail",
reads=(read("gmail.a", "gmail_a"), read("gmail.b", "gmail_b")),
),
ConnectorChecks("jira", reads=(read("jira.sites", "jira_sites"),)),
)
return adapter, checks
async def test_a_connector_selector_runs_only_those_connectors() -> None:
adapter, checks = two_connectors()
result = await run_checks(adapter, declarations=checks, connectors=["jira"])
assert [row[:2] for row in rows(result)] == [
("jira", None),
("jira", "jira.sites"),
]
assert [app for app, _ in adapter.opened] == ["jira"]
async def test_a_check_selector_runs_only_those_checks() -> None:
adapter, checks = two_connectors()
result = await run_checks(adapter, declarations=checks, checks=["gmail.b"])
assert [row[:2] for row in rows(result)] == [
("gmail", None),
("gmail", "gmail.b"),
]
assert [app for app, _ in adapter.opened] == ["gmail"]
async def test_a_check_selector_can_name_an_undeclared_apps_row() -> None:
adapter = FakeAdapter(targets("acme"), {"acme": listed()})
result = await run_checks(
adapter, declarations=declared(), checks=["acme.l3_probe"]
)
assert [row[:2] for row in rows(result)] == [
("acme", None),
("acme", "acme.l3_probe"),
]
async def test_a_selector_never_changes_how_a_connector_starts() -> None:
adapter = FakeAdapter(targets("gmail"), {"gmail": listed(FakeTool("gmail_x"))})
checks = ConnectorChecks(
"gmail",
reads=(
read("gmail.x", "gmail_x"),
UnprobeableCheck("gmail.labels", "needs_fixture"),
),
)
result = await run_checks(
adapter, declarations=declared(checks), checks=["gmail.labels"]
)
# Selected alone, the unprobeable check still gets the full run's session.
assert adapter.opened == [("gmail", SessionPurpose.CALL_TOOLS)]
assert [row[1] for row in rows(result)] == [None, "gmail.labels"]
async def test_a_selector_that_matches_nothing_is_a_failure() -> None:
adapter, checks = two_connectors()
result = await run_checks(adapter, declarations=checks, checks=["gmail.absent"])
assert result.failure == "selection:no_targets"
assert adapter.opened == []
async def test_an_invalid_declaration_is_refused_before_anything_starts() -> None:
adapter = FakeAdapter(targets("gmail"), {"gmail": listed(FakeTool("gmail_x"))})
broken = ConnectorChecks(
"gmail", reads=(ReadCheck("gmail.x", "gmail_x", assertions=()),)
)
with pytest.raises(ValueError):
await run_checks(adapter, declarations=declared(broken))
assert adapter.opened == []
async def test_a_declaration_for_another_app_is_refused() -> None:
adapter = FakeAdapter(targets("gmail"), {"gmail": listed(FakeTool("jira_x"))})
other = ConnectorChecks("jira", reads=(read("jira.x", "jira_x"),))
with pytest.raises(ValueError):
await run_checks(adapter, declarations=lambda app_id: other)
assert adapter.opened == []
async def test_l3_envelope_checks_run_through_the_probe_judgement() -> None:
tool = FakeTool(
"get_profile",
result={"content": [{"type": "text", "text": json.dumps({"sub": "x"})}]},
)
adapter = FakeAdapter(targets("linkedin"), {"linkedin": listed(tool)})
checks = ConnectorChecks(
"linkedin",
reads=(
ReadCheck(
"linkedin.l3_probe", "get_profile", assertions=(L3Envelope(("sub",)),)
),
),
)
result = await run_checks(adapter, declarations=declared(checks))
assert rows(result)[1] == ("linkedin", "linkedin.l3_probe", "pass", None, None)
# Timeouts
@pytest.mark.parametrize("keyword", ["total_timeout", "check_timeout"])
@pytest.mark.parametrize("value", [float("nan"), float("inf"), 0, -1, True])
async def test_a_timeout_must_be_a_finite_number_above_zero(
keyword: str, value: float
) -> None:
adapter = FakeAdapter(targets("gmail"), {"gmail": listed(FakeTool("gmail_x"))})
checks = ConnectorChecks("gmail", reads=(read("gmail.x", "gmail_x"),))
with pytest.raises(ValueError):
await run_checks(
adapter,
declarations=declared(checks),
**cast(Any, {keyword: value}),
)
assert adapter.opened == []
# A declaration with only exemptions
async def test_a_connector_with_only_exemptions_still_gets_its_session_row() -> None:
adapter = FakeAdapter(targets("jira"), {"jira": listed()})
checks = ConnectorChecks(
"jira", exemptions=(ToolExemption("jira_x", "moves a real issue"),)
)
result = await run_checks(adapter, declarations=declared(checks))
assert rows(result) == [("jira", None, "pass", None, None)]
assert adapter.opened == [("jira", SessionPurpose.LIST_ONLY)]
assert result.failure is None
async def test_a_check_selector_leaves_out_a_connector_none_of_whose_checks_it_names() -> (
None
):
adapter = FakeAdapter(
targets("gmail", "jira"),
{"gmail": listed(FakeTool("gmail_x")), "jira": listed()},
)
checks = declared(
ConnectorChecks("gmail", reads=(read("gmail.x", "gmail_x"),)),
ConnectorChecks(
"jira", exemptions=(ToolExemption("jira_x", "moves a real issue"),)
),
)
result = await run_checks(adapter, declarations=checks, checks=["gmail.x"])
assert {row[0] for row in rows(result)} == {"gmail"}
assert [app for app, _ in adapter.opened] == ["gmail"]
# Cancellation and tools that turn it into another error
async def test_cancelling_the_run_while_a_session_opens_propagates() -> None:
async def hang() -> ConnectorToolsResult:
await asyncio.sleep(10.0)
return listed()
adapter = FakeAdapter(targets("gmail"), {"gmail": hang})
checks = ConnectorChecks("gmail", reads=(read("gmail.x", "gmail_x"),))
task = asyncio.create_task(run_checks(adapter, declarations=declared(checks)))
await asyncio.sleep(0.05)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
async def test_cancelling_the_run_during_a_call_propagates() -> None:
adapter = FakeAdapter(
targets("gmail"), {"gmail": listed(FakeTool("gmail_x", delay=10.0))}
)
checks = ConnectorChecks("gmail", reads=(read("gmail.x", "gmail_x"),))
task = asyncio.create_task(run_checks(adapter, declarations=declared(checks)))
await asyncio.sleep(0.05)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
async def test_a_call_that_turns_its_cancellation_into_an_error_still_timed_out() -> (
None
):
class Converting(FakeTool):
async def run_json_async(self, args: Mapping[str, Any]) -> Any:
try:
await asyncio.sleep(1.0)
except asyncio.CancelledError:
raise RuntimeError("cancelled") from None
return SUCCESS
adapter = FakeAdapter(
targets("gmail"),
{"gmail": listed(Converting("gmail_x"), FakeTool("gmail_y"))},
)
checks = ConnectorChecks(
"gmail", reads=(read("gmail.x", "gmail_x"), read("gmail.y", "gmail_y"))
)
by_call = await run_checks(
adapter, declarations=declared(checks), check_timeout=0.05
)
by_deadline = await run_checks(
adapter, declarations=declared(checks), check_timeout=5, total_timeout=0.05
)
assert rows(by_call)[1:] == [
("gmail", "gmail.x", "fail", "timed_out", "check_call"),
("gmail", "gmail.y", "pass", None, None),
]
assert by_call.complete is True
assert rows(by_deadline)[1:] == [
("gmail", "gmail.x", "fail", "timed_out", "run_deadline"),
("gmail", "gmail.y", "fail", "not_reached", None),
]
assert by_deadline.complete is False
async def test_a_session_that_turns_the_deadline_into_an_error_still_timed_out() -> (
None
):
async def convert() -> ConnectorToolsResult:
try:
await asyncio.sleep(1.0)
except asyncio.CancelledError:
raise RuntimeError("cancelled") from None
return listed(FakeTool("gmail_x"))
adapter = FakeAdapter(targets("gmail"), {"gmail": convert})
checks = ConnectorChecks("gmail", reads=(read("gmail.x", "gmail_x"),))
result = await run_checks(
adapter, declarations=declared(checks), total_timeout=0.05
)
assert rows(result) == [
("gmail", None, "fail", "timed_out", "run_deadline"),
("gmail", "gmail.x", "skip", "session_unavailable", None),
]
assert result.complete is False
# Cancellation from outside the run
async def test_an_outer_cancel_is_not_swallowed_by_a_tool_that_converts_it() -> None:
class Converting(FakeTool):
async def run_json_async(self, args: Mapping[str, Any]) -> Any:
self.calls.append(dict(args))
try:
await asyncio.sleep(10.0)
except asyncio.CancelledError:
raise RuntimeError("cancelled") from None
second = FakeTool("gmail_y")
adapter = FakeAdapter(
targets("gmail"), {"gmail": listed(Converting("gmail_x"), second)}
)
checks = ConnectorChecks(
"gmail", reads=(read("gmail.x", "gmail_x"), read("gmail.y", "gmail_y"))
)
task = asyncio.create_task(run_checks(adapter, declarations=declared(checks)))
await asyncio.sleep(0.05)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert second.calls == []
async def test_an_outer_cancel_is_not_swallowed_by_a_tool_that_returns() -> None:
class Swallowing(FakeTool):
async def run_json_async(self, args: Mapping[str, Any]) -> Any:
try:
await asyncio.sleep(10.0)
except asyncio.CancelledError:
return SUCCESS
return SUCCESS
second = FakeTool("gmail_y")
adapter = FakeAdapter(
targets("gmail"), {"gmail": listed(Swallowing("gmail_x"), second)}
)
checks = ConnectorChecks(
"gmail", reads=(read("gmail.x", "gmail_x"), read("gmail.y", "gmail_y"))
)
task = asyncio.create_task(run_checks(adapter, declarations=declared(checks)))
await asyncio.sleep(0.05)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert second.calls == []
async def test_an_outer_cancel_is_not_swallowed_by_a_session_that_converts_it() -> None:
async def convert() -> ConnectorToolsResult:
try:
await asyncio.sleep(10.0)
except asyncio.CancelledError:
raise RuntimeError("cancelled") from None
return listed()
adapter = FakeAdapter(
targets("gmail", "jira"), {"gmail": convert, "jira": listed()}
)
checks = declared(
ConnectorChecks("gmail", reads=(read("gmail.x", "gmail_x"),)),
ConnectorChecks("jira", reads=(read("jira.x", "jira_x"),)),
)
task = asyncio.create_task(run_checks(adapter, declarations=checks))
await asyncio.sleep(0.05)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert [app for app, _ in adapter.opened] == ["gmail"]