Enterprise agent platform for building, deploying, and operating AI teammates.
fix(agent): count an in-repo read or idempotent tool's {"status": "error"} result as a failed call (#3803)
leyoonafrad37916ae87c
diff --git a/src/xagent/core/tools/adapters/vibe/mcp_adapter.py b/src/xagent/core/tools/adapters/vibe/mcp_adapter.pyindex 8f7e00355..ee35d00c5 100644--- a/src/xagent/core/tools/adapters/vibe/mcp_adapter.py+++ b/src/xagent/core/tools/adapters/vibe/mcp_adapter.py@@ -79,6 +79,7 @@ from .connector_runtime import (runtime_bindings_from_config,)from .mcp_approval_gate import gate_mcp_tools+from .mcp_error_envelope import is_error_envelope, speaks_error_envelopefrom .mcp_result_integrity import MCPResultIntegrityGuardfrom .sandboxed_tool.chrome_session import (ChromeDaemonLaunchSpec,@@ -2386,6 +2387,20 @@ class MCPToolAdapter(AbstractBaseTool):self._detect_legacy_plain_text_error = (str(source_server or "").casefold() == "xero")+ # In-repo servers return a vendor error as a {"status": "error"}+ # envelope with isError unset; see mcp_error_envelope. Only a tool+ # whose wire annotations declare that a repeat adds no effect (read-only+ # or idempotent) is reclassified. A write's envelope can follow a request+ # that went out; a failed result reopens normal tools in flash mode, and+ # the duplicate-write guard covers only declared non-idempotent tools.+ # An undeclared tool is treated as a write, as classify_write_hint says.+ self._detect_error_envelope = speaks_error_envelope(connection) and (+ self.write_hint is MCPWriteHint.READ_ONLY+ or (+ isinstance(raw_annotations, Mapping)+ and raw_annotations.get(_IDEMPOTENT_HINT) is True+ )+ )self.concurrency_safe = _mcp_tool_is_concurrency_safe(self.mcp_tool.name,concurrency_safe=concurrency_safe,@@ -3843,6 +3858,12 @@ class MCPToolAdapter(AbstractBaseTool):result,detect_legacy_plain_text_error=self._detect_legacy_plain_text_error,)+ if (+ self._detect_error_envelope+ and not normalized["is_error"]+ and is_error_envelope(normalized["content"])+ ):+ normalized["success"] = Falseif self._is_excel_tool:failure_code = _excel_failure_code(normalized)if failure_code is not None:diff --git a/src/xagent/core/tools/adapters/vibe/mcp_error_envelope.py b/src/xagent/core/tools/adapters/vibe/mcp_error_envelope.pynew file mode 100644index 000000000..2dab1c888--- /dev/null+++ b/src/xagent/core/tools/adapters/vibe/mcp_error_envelope.py@@ -0,0 +1,159 @@+"""Recognize the in-repo connector servers' ``{"status": "error"}`` result.++The FastMCP servers under ``xagent.web.tools.mcp`` catch vendor errors and+return ``{"status": "error", "message": ...}`` as text, leaving the MCP+``isError`` flag unset. ``tool_result_succeeded`` reads only the flag and the+result's own top-level keys, so such a call counted as a success+(xorbitsai/xagent#3574).++Only a connection that launches one of those modules is trusted to use the+envelope. The decision is made on the launched module, not on the server's+name: a catalog id can name a vendor's remote server (HubSpot's does, while+the repo still carries a module of that name), and a catalog row's name need+not fold to its id ("Facebook Pages" is ``facebook``). A third-party server+can return a JSON object whose ``status`` is ``"error"`` as ordinary data, so+the shape alone is never enough.++``is_error`` is left as the server sent it. The connector probe and the smoke+checks read that flag as a protocol-level failure and judge the envelope on+their own, so the adapter marks the result ``success: False`` instead, which+``tool_result_succeeded`` already reads.++The adapter applies this only to a tool whose wire annotations declare that a+repeat adds no effect: ``readOnlyHint: true`` (with ``destructiveHint`` not+true) or ``idempotentHint: true``. A write can return the envelope after its+request went out: ``freshdesk_create_ticket``, an unannotated write, does so+when the read times out after the POST, and ``deputy_create_resource``, a+declared non-idempotent write, does too. ReAct reads a failed call as one+whose write provably did not happen. In flash mode a failed result also+reopens the normal tools, so the model could write again, and the same-turn+duplicate-write guard covers only declared non-idempotent tools. An+undeclared tool is treated as a write, so its envelope stays a success.+"""++from __future__ import annotations++import json+from collections.abc import Mapping, Sequence+from typing import Any++IN_REPO_SERVER_PACKAGE = "xagent.web.tools.mcp"++# The FastMCP servers in ``IN_REPO_SERVER_PACKAGE``. Each reports a vendor+# error as the envelope (directly, or through ``meta_graph.error_response``)+# or raises, which sets ``isError``. LinkedIn's server is not here: it passes+# the vendor's response text through unchanged. A test pins this set to the+# package's FastMCP modules, so a new server has to be classified.+ERROR_ENVELOPE_SERVER_MODULES = frozenset(+ {+ "aws",+ "calendar",+ "chartmogul",+ "deputy",+ "employment_hero",+ "excel",+ "facebook",+ "freshdesk",+ "github",+ "gmail",+ "google_ads",+ "google_analytics",+ "google_docs",+ "google_drive",+ "google_search_console",+ "google_sheets",+ "google_slides",+ "hubspot",+ "instagram",+ "intercom",+ "jira",+ "jira_ticket",+ "linear",+ "magento",+ "meta_ads",+ "mixpanel",+ "myob",+ "onedrive",+ "outlook",+ "planner",+ "posthog",+ "powerpoint",+ "salesforce",+ "sharepoint",+ "shopify",+ "slack",+ "stripe",+ "teams",+ "whatsapp",+ "word",+ "xero",+ "zendesk",+ "zoom",+ }+)++# Error envelopes are a status and a short message. The bound keeps a large+# successful payload from being parsed on every call; an envelope past it is+# left as a success, as it was before.+ERROR_ENVELOPE_MAX_CHARS = 16_384+++def speaks_error_envelope(connection: object) -> bool:+ """Whether ``connection`` runs an in-repo server that uses the envelope.++ The connection must be stdio and run ``-m <IN_REPO_SERVER_PACKAGE>.<module>``+ with ``<module>`` in ``ERROR_ENVELOPE_SERVER_MODULES``. Only the first+ ``-m`` is read, since what follows it is that module's own argv. Both+ launch shapes the catalog has used match, ``python -m ...`` and+ ``uv run python -m ...``. Neither the command nor the arguments before+ ``-m`` are interpreted, so a script or ``-c`` placed first would still+ match. Only a user-configured stdio connection could be shaped that way;+ such a connection runs in the sandbox, and a match only changes how its+ own results are counted.+ """+ if not isinstance(connection, Mapping) or connection.get("transport") != "stdio":+ return False+ args = connection.get("args")+ if not isinstance(args, Sequence) or isinstance(args, str):+ return False+ args = list(args)+ try:+ module_index = args.index("-m") + 1+ except ValueError:+ return False+ if module_index >= len(args):+ return False+ module = args[module_index]+ if not isinstance(module, str):+ return False+ package, _, name = module.rpartition(".")+ return package == IN_REPO_SERVER_PACKAGE and name in ERROR_ENVELOPE_SERVER_MODULES+++def is_error_envelope(content: object) -> bool:+ """Whether a normalized MCP ``content`` list is one error envelope.++ It must be a single text item whose text, within+ ``ERROR_ENVELOPE_MAX_CHARS``, parses to a JSON object whose ``status`` is+ exactly ``"error"``, the spelling every in-repo server writes. That is+ deliberately narrower than ``tool_result_succeeded``, which folds case on+ a result's own top-level ``status``. Other statuses the servers use+ (``indeterminate``, ``conflict``, ``partial``, ...) are outcomes the model+ has to read, not failures, and stay successful.+ """+ if not isinstance(content, list) or len(content) != 1:+ return False+ item = content[0]+ if not isinstance(item, Mapping) or item.get("type") != "text":+ return False+ text = item.get("text")+ if not isinstance(text, str) or len(text) > ERROR_ENVELOPE_MAX_CHARS:+ return False+ if not text.lstrip().startswith("{"):+ return False+ try:+ payload: Any = json.loads(text)+ except (ValueError, RecursionError):+ # Text nested past the parser's depth limit raises RecursionError.+ return False+ return isinstance(payload, dict) and payload.get("status") == "error"diff --git a/tests/core/tools/adapters/vibe/test_mcp_error_envelope.py b/tests/core/tools/adapters/vibe/test_mcp_error_envelope.pynew file mode 100644index 000000000..d968015df--- /dev/null+++ b/tests/core/tools/adapters/vibe/test_mcp_error_envelope.py@@ -0,0 +1,512 @@+"""An in-repo connector's ``{"status": "error"}`` result is a failed call.++The in-repo FastMCP servers under ``xagent.web.tools.mcp`` catch vendor+errors and return ``{"status": "error", ...}`` as text with the MCP ``isError``+flag unset (xorbitsai/xagent#3574). The adapter recognizes that envelope only+for a connection that launches one of those modules, and marks the result+``success: False`` so ``tool_result_succeeded`` reports the call as failed,+but only for a tool whose wire annotations declare that a repeat adds no+effect (read-only, or idempotent). A write may return the envelope after its+request went out, so an undeclared or additive write stays a success.+``is_error`` keeps mirroring the wire flag.+"""++import json+import re+from contextlib import asynccontextmanager+from pathlib import Path+from types import SimpleNamespace++import pytest+from mcp.server.fastmcp import FastMCP+from mcp.shared.memory import create_connected_server_and_client_session+from mcp.types import CallToolResult, ImageContent, TextContent, ToolAnnotations++import xagent.web.tools.mcp as in_repo_servers+from xagent.core.agent.pattern.react.duplicate_write_guard import (+ tool_requires_duplicate_write_guard,+)+from xagent.core.agent.result import tool_result_succeeded+from xagent.core.tools.adapters.vibe import mcp_adapter as mcp_adapter_module+from xagent.core.tools.adapters.vibe import mcp_error_envelope as envelope_module+from xagent.core.tools.adapters.vibe.mcp_adapter import MCPToolAdapter+from xagent.core.tools.adapters.vibe.mcp_error_envelope import (+ ERROR_ENVELOPE_MAX_CHARS,+ ERROR_ENVELOPE_SERVER_MODULES,+ IN_REPO_SERVER_PACKAGE,+ is_error_envelope,+ speaks_error_envelope,+)++_ERROR = json.dumps({"status": "error", "message": "Vendor rejected the token"})+++def _in_repo(module: str = "gmail") -> dict:+ return {+ "transport": "stdio",+ "command": "python",+ "args": ["-m", f"{IN_REPO_SERVER_PACKAGE}.{module}"],+ }+++def _text(text: str) -> list[dict]:+ return [TextContent(type="text", text=text).model_dump()]+++# --- Which connections speak the envelope ------------------------------------+++ "connection",+ [+ _in_repo("gmail"),+ _in_repo("google_drive"),+ _in_repo("slack"),+ # Rows seeded before the launch shape was shortened run through uv.+ {+ "transport": "stdio",+ "command": "uv",+ "args": ["run", "python", "-m", "xagent.web.tools.mcp.gmail"],+ },+ {+ "transport": "stdio",+ "command": "/usr/local/bin/python3",+ "args": ("-m", "xagent.web.tools.mcp.facebook"),+ },+ ],+)+def test_in_repo_fastmcp_launch_speaks_the_envelope(connection):+ assert speaks_error_envelope(connection) is True+++ "connection",+ [+ # LinkedIn's server passes the vendor's response text through as is.+ _in_repo("linkedin"),+ # A helper module, not a server.+ _in_repo("utils"),+ # The package itself and look-alike module paths.+ {+ "transport": "stdio",+ "command": "python",+ "args": ["-m", "xagent.web.tools.mcp"],+ },+ _in_repo("gmail.extra"),+ _in_repo("gmail_evil"),+ {+ "transport": "stdio",+ "command": "python",+ "args": ["-m", "evil.xagent.web.tools.mcp.gmail"],+ },+ # Python runs the first -m; anything after it is that module's argv.+ {+ "transport": "stdio",+ "command": "python",+ "args": ["-m", "other_server", "-m", "xagent.web.tools.mcp.gmail"],+ },+ # A -m with no module after it.+ {"transport": "stdio", "command": "python", "args": ["-m"]},+ # The module as a plain argument, not run with -m.+ {+ "transport": "stdio",+ "command": "python",+ "args": ["xagent.web.tools.mcp.gmail"],+ },+ # HubSpot's catalog entry is the vendor's remote server, whatever the+ # repo still carries under the same name.+ {"transport": "streamable_http", "url": "https://mcp.hubspot.com"},+ {"transport": "stdio", "command": "npx", "args": ["-y", "some-mcp-server"]},+ # A remote transport never runs a local module, whatever its args say.+ {+ "transport": "sse",+ "url": "https://example.com/sse",+ "args": ["-m", "xagent.web.tools.mcp.gmail"],+ },+ {+ "transport": "stdio",+ "command": "python",+ "args": "-m xagent.web.tools.mcp.gmail",+ },+ {"transport": "stdio", "command": "python"},+ None,+ "stdio",+ ],+)+def test_other_connections_do_not_speak_the_envelope(connection):+ assert speaks_error_envelope(connection) is False+++def _fastmcp_server_modules() -> set[str]:+ package_dir = Path(in_repo_servers.__file__).parent+ return {+ path.stem+ for path in package_dir.glob("*.py")+ if re.search(r"\bFastMCP\(", path.read_text(encoding="utf-8"))+ }+++def test_envelope_modules_are_exactly_the_in_repo_fastmcp_servers():+ """Every FastMCP server in the package reports a vendor error as the+ envelope (directly, or through ``meta_graph.error_response``), or raises+ and so sets ``isError``. A new server must be classified here."""+ assert ERROR_ENVELOPE_SERVER_MODULES == _fastmcp_server_modules()+++def test_every_catalog_launched_module_is_classified():+ from xagent.web.builtin_mcp_registry import get_builtin_public_mcp_app_rows++ passes_vendor_text_through = {"linkedin"}+ launched = set()+ for row in get_builtin_public_mcp_app_rows():+ args = (row.get("launch_config") or {}).get("args") or []+ for arg in args:+ if isinstance(arg, str) and arg.startswith(f"{IN_REPO_SERVER_PACKAGE}."):+ launched.add(arg.removeprefix(f"{IN_REPO_SERVER_PACKAGE}."))++ assert launched+ assert launched <= ERROR_ENVELOPE_SERVER_MODULES | passes_vendor_text_through+ assert not ERROR_ENVELOPE_SERVER_MODULES & passes_vendor_text_through+++# --- What counts as the envelope ---------------------------------------------+++ "content",+ [+ _text(_ERROR),+ _text(json.dumps({"status": "error"})),+ _text(" \n" + _ERROR + "\n"),+ _text(json.dumps({"status": "error", "message": "x", "failure_code": "c"})),+ _text(json.dumps({"message": "first", "status": "error"}, ensure_ascii=False)),+ ],+)+def test_error_envelope_is_recognized(content):+ assert is_error_envelope(content) is True+++ "content",+ [+ _text(json.dumps({"status": "success", "messages": []})),+ # A write that may have happened is not a failure.+ _text(json.dumps({"status": "indeterminate", "message": "unknown"})),+ _text(json.dumps({"status": "conflict", "conflicts": []})),+ _text(json.dumps({"status": "partial", "results": []})),+ _text(json.dumps({"status": "ERROR"})),+ _text(json.dumps({"status": "error "})),+ _text(json.dumps({"status": 500, "message": "error"})),+ _text(json.dumps({"result": {"status": "error"}})),+ _text(json.dumps([{"status": "error"}])),+ _text(json.dumps("error")),+ _text("Error listing invoices: boom"),+ _text('{"status": "error"'),+ _text(""),+ # Two items are a listing, never a lone envelope.+ _text(_ERROR) + _text(_ERROR),+ [ImageContent(type="image", data="AAAA", mimeType="image/png").model_dump()],+ [{"type": "text", "text": None}],+ [{"text": _ERROR}],+ ["not a dict"],+ [],+ None,+ _ERROR,+ ],+)+def test_other_content_is_not_the_envelope(content):+ assert is_error_envelope(content) is False+++def test_envelope_over_the_size_bound_is_not_parsed():+ padded = json.dumps({"status": "error", "message": "x" * ERROR_ENVELOPE_MAX_CHARS})+ assert len(padded) > ERROR_ENVELOPE_MAX_CHARS++ assert is_error_envelope(_text(padded)) is False+++def test_parser_recursion_error_is_not_the_envelope(monkeypatch):+ """JSON nested past the parser's depth limit raises RecursionError. Within+ the size bound CPython 3.12 does not reach that depth, so the error is+ injected: it must read as "not the envelope", never escape the call."""++ def _too_deep(_text):+ raise RecursionError("maximum recursion depth exceeded")++ monkeypatch.setattr(envelope_module, "json", SimpleNamespace(loads=_too_deep))++ assert is_error_envelope(_text(_ERROR)) is False+++# --- Through the adapter, against a real FastMCP server ----------------------+++_READ_ONLY = {"readOnlyHint": True}+++def _server_returning(payload: str, annotations: dict | None = None) -> FastMCP:+ server = FastMCP("envelope-probe")++ @server.tool(+ annotations=ToolAnnotations(**annotations) if annotations is not None else None+ )+ def probe_tool() -> str:+ return payload++ return server+++async def _adapter(monkeypatch, server: FastMCP, connection, **kwargs):+ @asynccontextmanager+ async def _session(_connection):+ async with create_connected_server_and_client_session(+ server._mcp_server+ ) as session:+ yield session++ monkeypatch.setattr(mcp_adapter_module, "create_session", _session)+ (mcp_tool,) = await server.list_tools()+ return MCPToolAdapter(+ mcp_tool=mcp_tool,+ connection=connection,+ name_prefix="mcp_probe_",+ source_server=kwargs.pop("source_server", "probe"),+ **kwargs,+ )+++async def test_in_repo_error_envelope_is_a_failed_tool_call(monkeypatch):+ """The issue's regression: a server returns the envelope with ``isError``+ unset, and the call the model made counts as failed."""+ adapter = await _adapter(+ monkeypatch,+ _server_returning(_ERROR, _READ_ONLY),+ _in_repo(),+ raw_annotations=_READ_ONLY,+ )++ result = await adapter.run_json_async({})++ assert tool_result_succeeded(result) is False+ assert result["success"] is False+ # The wire flag is reported as the server sent it, and the model still+ # reads the server's own message.+ assert result["is_error"] is False+ assert result["content"][0]["text"] == _ERROR+++async def test_error_envelope_from_a_server_not_in_repo_stays_successful(monkeypatch):+ remote = {"transport": "streamable_http", "url": "https://mcp.example.com/mcp"}+ adapter = await _adapter(+ monkeypatch,+ _server_returning(_ERROR, _READ_ONLY),+ remote,+ raw_annotations=_READ_ONLY,+ )++ result = await adapter.run_json_async({})++ assert "success" not in result+ assert tool_result_succeeded(result) is True+++ "payload",+ [+ json.dumps({"status": "success", "messages": []}),+ json.dumps({"status": "indeterminate", "message": "may have been sent"}),+ "plain text result",+ ],+)+async def test_other_in_repo_results_are_unchanged(monkeypatch, payload):+ adapter = await _adapter(+ monkeypatch,+ _server_returning(payload, _READ_ONLY),+ _in_repo(),+ raw_annotations=_READ_ONLY,+ )++ result = await adapter.run_json_async({})++ assert "success" not in result+ assert tool_result_succeeded(result) is True+++async def test_server_flagged_error_payload_is_unchanged(monkeypatch):+ server = FastMCP("envelope-probe")++ @server.tool(annotations=ToolAnnotations(**_READ_ONLY))+ def probe_tool() -> str:+ raise RuntimeError("boom")++ adapter = await _adapter(+ monkeypatch, server, _in_repo(), raw_annotations=_READ_ONLY+ )++ result = await adapter.run_json_async({})++ assert result["is_error"] is True+ assert "success" not in result+ assert tool_result_succeeded(result) is False+++async def test_excel_failure_code_keeps_its_classification(monkeypatch):+ payload = json.dumps(+ {+ "status": "error",+ "message": "locked",+ "failure_code": "excel_workbook_locked",+ }+ )+ server = FastMCP("envelope-probe")++ @server.tool()+ def excel_probe_tool() -> str:+ return payload++ adapter = await _adapter(+ monkeypatch, server, _in_repo("excel"), source_server="excel"+ )++ result = await adapter.run_json_async({})++ assert result["is_error"] is True+ assert result["success"] is False+ assert result["failure_code"] == "excel_workbook_locked"+++async def test_non_idempotent_write_envelope_keeps_the_duplicate_write_guard(+ monkeypatch,+):+ """An in-repo write can return the envelope after its request went out+ (``deputy_create_resource`` does on a timeout after the POST). The+ same-turn duplicate-write guard suppresses an identical repeat only of a+ ``completed`` call, and a ``failed`` one reads as "provably did not+ happen", so a non-idempotent write's envelope must stay a success."""+ annotations = {"destructiveHint": False, "idempotentHint": False}+ adapter = await _adapter(+ monkeypatch,+ _server_returning(_ERROR, annotations),+ _in_repo("deputy"),+ raw_annotations=annotations,+ )+ assert tool_requires_duplicate_write_guard(adapter) is True++ result = await adapter.run_json_async({})++ assert "success" not in result+ assert tool_result_succeeded(result) is True+++ "annotations",+ [+ {"readOnlyHint": True},+ {"destructiveHint": False, "idempotentHint": True},+ {"destructiveHint": True, "idempotentHint": True},+ ],+)+async def test_envelope_of_a_tool_whose_repeat_adds_no_effect_is_a_failure(+ monkeypatch, annotations+):+ adapter = await _adapter(+ monkeypatch,+ _server_returning(_ERROR, annotations),+ _in_repo("deputy"),+ raw_annotations=annotations,+ )+ assert tool_requires_duplicate_write_guard(adapter) is False++ result = await adapter.run_json_async({})++ assert tool_result_succeeded(result) is False+++ "annotations",+ [+ None,+ {"destructiveHint": False},+ # Destructive outranks read-only.+ {"readOnlyHint": True, "destructiveHint": True},+ # A non-boolean is not a declaration.+ {"idempotentHint": 1},+ ],+)+async def test_envelope_of_an_undeclared_or_additive_write_stays_successful(+ monkeypatch, annotations+):+ """The Freshdesk create/reply case: ``freshdesk_create_ticket`` is a plain+ ``@server.tool()`` that returns the envelope after a POST that may have+ committed. A failed result would reopen normal tools in flash mode, and the+ duplicate-write guard does not cover an undeclared tool, so the model could+ write twice. A tool that does not declare a no-effect repeat is a write."""+ adapter = await _adapter(+ monkeypatch,+ _server_returning(_ERROR, annotations),+ _in_repo("freshdesk"),+ raw_annotations=annotations,+ )++ result = await adapter.run_json_async({})++ assert "success" not in result+ assert tool_result_succeeded(result) is True+++async def test_envelope_among_several_content_items_stays_successful(monkeypatch):+ server = FastMCP("envelope-probe")++ @server.tool(annotations=ToolAnnotations(**_READ_ONLY))+ def probe_tool() -> CallToolResult:+ return CallToolResult(+ content=[+ TextContent(type="text", text=_ERROR),+ TextContent(type="text", text=_ERROR),+ ]+ )++ adapter = await _adapter(+ monkeypatch, server, _in_repo(), raw_annotations=_READ_ONLY+ )++ result = await adapter.run_json_async({})++ assert len(result["content"]) == 2+ assert "success" not in result+ assert tool_result_succeeded(result) is True+++def test_envelope_detection_is_decided_from_the_bound_connection():+ def adapter(connection, raw_annotations=None) -> MCPToolAdapter:+ return MCPToolAdapter(+ mcp_tool=SimpleNamespace(+ name="probe_tool",+ description="probe",+ inputSchema={"type": "object", "properties": {}},+ ),+ connection=connection,+ name_prefix="mcp_probe_",+ source_server="probe",+ raw_annotations=raw_annotations,+ )++ assert adapter(_in_repo("gmail"), _READ_ONLY)._detect_error_envelope is True+ assert adapter(_in_repo("gmail"))._detect_error_envelope is False+ assert adapter(_in_repo("linkedin"), _READ_ONLY)._detect_error_envelope is False+ assert (+ adapter(+ {"transport": "streamable_http", "url": "https://x.example"},+ _READ_ONLY,+ )._detect_error_envelope+ is False+ )