Enterprise agent platform for building, deploying, and operating AI teammates.
fix(composio): clear a grant whose connected account no longer exists (#3773)
蓝鱼bluefish7d5fcecf717d
diff --git a/example.env b/example.envindex a287c28b1..bc5e999e8 100644--- a/example.env+++ b/example.env@@ -1266,6 +1266,18 @@ COMPOSIO_SALESFORCE_AUTH_CONFIG_ID=""# COMPOSIO_USER_ID_SECRET. Accounts still left in the Composio project# are then handled in Composio.# XAGENT_COMPOSIO_RECONCILE_INTERVAL_SECONDS="3600"+# On by default (hourly). Clears any stored grant whose Composio connected+# account no longer exists (Composio answers 404+# ConnectedAccount_ResourceNotFound), so the connector offers to connect+# again. It only reads the grants this database stores and clears their+# tokens locally; it never deletes anything in Composio. Each pass clears a+# few missing accounts, up to 200 lookups, and resumes from where the previous+# pass stopped (in this process; a restart starts over). If at least 3 lookups+# answer missing and more than half of the pass's lookups do, nothing is+# written and a warning is logged: a key that cannot see the accounts does not+# clear them, but neither does a mass deletion, which needs an operator. With+# fewer than 3 grants the breaker cannot trip. Set "0" to turn it off.+# XAGENT_COMPOSIO_MISSING_ACCOUNT_INTERVAL_SECONDS="3600"# The one callback URL every Composio connect returns to, whatever the app:# Composio's OAuth user verification is a single URL per Composio project.# Unset (the default) it is this deployment's Meta callback URLdiff --git a/src/xagent/config.py b/src/xagent/config.pyindex 80afd15d6..d915ff86b 100644--- a/src/xagent/config.py+++ b/src/xagent/config.py@@ -257,6 +257,9 @@ KB_MILVUS_CLEANUP_ENABLED = "XAGENT_KB_MILVUS_CLEANUP_ENABLED"TASKLESS_UPLOAD_TTL_SECONDS = "XAGENT_TASKLESS_UPLOAD_TTL_SECONDS"ORPHAN_UPLOAD_SWEEP_INTERVAL_SECONDS = "XAGENT_ORPHAN_UPLOAD_SWEEP_INTERVAL_SECONDS"COMPOSIO_RECONCILE_INTERVAL_SECONDS = "XAGENT_COMPOSIO_RECONCILE_INTERVAL_SECONDS"+COMPOSIO_MISSING_ACCOUNT_INTERVAL_SECONDS = (+ "XAGENT_COMPOSIO_MISSING_ACCOUNT_INTERVAL_SECONDS"+)TASK_CLEANUP_RETRY_INTERVAL_SECONDS = "XAGENT_TASK_CLEANUP_RETRY_INTERVAL_SECONDS"TASK_CLEANUP_MAX_ATTEMPTS = "XAGENT_TASK_CLEANUP_MAX_ATTEMPTS"WORKFORCE_PREVIEW_RUN_STALE_SECONDS = "XAGENT_WORKFORCE_PREVIEW_RUN_STALE_SECONDS"@@ -1953,6 +1956,33 @@ def get_composio_reconcile_interval_seconds() -> int | None:return parsed+def get_composio_missing_account_interval_seconds() -> int:+ """How often grants whose Composio account is gone are cleared.++ On by default: the pass only reads the grants this database stores and+ clears their local tokens, deleting nothing in Composio, so unlike+ the orphan cleanup it needs no operator opt-in. "0" turns it off; any+ other value that is not an integer of at least 300 falls back to 3600.+ """+ value = (os.getenv(COMPOSIO_MISSING_ACCOUNT_INTERVAL_SECONDS) or "").strip()+ if not value:+ return 3600+ if value == "0":+ return 0+ try:+ parsed = int(value)+ except ValueError:+ parsed = 0+ if parsed < 300:+ logger.warning(+ "Invalid %s=%r; using 3600",+ COMPOSIO_MISSING_ACCOUNT_INTERVAL_SECONDS,+ value,+ )+ return 3600+ return parsed++def get_task_cleanup_retry_interval_seconds() -> int:"""How often the task-cleanup retry driver looks for due obligations (#2587).diff --git a/src/xagent/web/app.py b/src/xagent/web/app.pyindex eeff30d7c..a3aa4b917 100644--- a/src/xagent/web/app.py+++ b/src/xagent/web/app.py@@ -26,6 +26,7 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Sendfrom ..config import (get_agent_runtime,get_background_job_sweep_interval_seconds,+ get_composio_missing_account_interval_seconds,get_composio_reconcile_interval_seconds,get_cors_origins,get_environment,@@ -904,6 +905,79 @@ async def stop_composio_reconcile_task(app_instance: FastAPI) -> None:logger.error("Composio reconcile loop stopped after failure", exc_info=exc)+def start_composio_missing_account_task(+ app_instance: FastAPI,+) -> asyncio.Task[Any] | None:+ """Start the loop that tombstones grants whose Composio account is gone.++ Runs by default whenever an app's Composio config is complete; each pass+ looks up the accounts this database's grants name and clears the grant+ when Composio answers 404 ConnectedAccount_ResourceNotFound. A pass clears+ a few missing accounts, up to a fixed number of lookups, and resumes where+ the previous pass stopped in this process. When at least three lookups answer+ missing and more than half of the pass's lookups do, it writes nothing and+ logs a warning: a key that cannot see the accounts does not clear them, but+ neither does a mass deletion, which needs an operator. With fewer than three+ grants the breaker cannot trip. It reads only this database's grants and+ deletes nothing in Composio, so it does not need the orphan cleanup's+ opt-in.+ """+ from .composio_auth import _configured_apps, run_missing_account_loop++ existing_task = cast(+ asyncio.Task[Any] | None,+ getattr(app_instance.state, "composio_missing_account_task", None),+ )+ if existing_task is not None and not existing_task.done():+ return existing_task+ app_instance.state.composio_missing_account_task = None+ if os.getenv("PYTEST_CURRENT_TEST"):+ logger.info("Skipping Composio missing-account loop (test environment)")+ return None+ if not _configured_apps():+ return None+ interval_seconds = get_composio_missing_account_interval_seconds()+ if interval_seconds <= 0:+ return None+ stop_event = threading.Event()+ app_instance.state.composio_missing_account_stop = stop_event+ task = asyncio.create_task(+ run_missing_account_loop(+ interval_seconds=interval_seconds, stop_event=stop_event+ )+ )+ app_instance.state.composio_missing_account_task = task+ logger.info(+ "Started Composio missing-account loop (interval=%ss)", interval_seconds+ )+ return task+++async def stop_composio_missing_account_task(app_instance: FastAPI) -> None:+ """Stop this process's Composio missing-account loop.++ The stop event reaches a pass running in its worker thread, which returns+ at its next request boundary; the cancel ends the awaiting coroutine.+ """+ stop_event = getattr(app_instance.state, "composio_missing_account_stop", None)+ app_instance.state.composio_missing_account_stop = None+ if stop_event is not None:+ stop_event.set()+ task = getattr(app_instance.state, "composio_missing_account_task", None)+ app_instance.state.composio_missing_account_task = None+ if task is not None and not task.done():+ task.cancel()+ if task is not None:+ try:+ await task+ except asyncio.CancelledError:+ pass+ except Exception as exc:+ logger.error(+ "Composio missing-account loop stopped after failure", exc_info=exc+ )++def start_task_cleanup_retry_task(app_instance: FastAPI,) -> asyncio.Task[Any] | None:@@ -2472,7 +2546,7 @@ async def _start_commands_and_channels() -> None:async def _start_late_tasks() -> None:- """Temp-file cleanup and Composio reconcile, deliberately started last."""+ """Temp-file cleanup and the Composio loops, deliberately started last."""auto_migrate = _lancedb_auto_migrate()# Kept under the same migration toggle as the uploaded-files reconcile above;# see start_temp_file_cleanup_task for the backgrounding/shutdown rationale.@@ -2486,9 +2560,10 @@ async def _start_late_tasks() -> None:# test_failed_startup_leaves_no_unsignaled_temp_file_cleanup pins this.if auto_migrate:start_temp_file_cleanup_task(app)- # Last for the same reason: a Composio reconcile pass runs in an executor- # thread that only its stop event (set by shutdown_event) can end.+ # Last for the same reason: a Composio pass runs in an executor thread+ # that only its stop event (set by shutdown_event) can end.start_composio_reconcile_task(app)+ start_composio_missing_account_task(app)async def _stop_runtime_loops() -> None:@@ -2519,6 +2594,7 @@ async def _stop_runtime_loops() -> None:await stop_orphan_upload_gc_task(app)await stop_composio_reconcile_task(app)+ await stop_composio_missing_account_task(app)await stop_retention_purge_task(app)await stop_task_cleanup_retry_task(app)await stop_uploaded_file_recovery_task(app)@@ -2777,7 +2853,7 @@ async def startup_event() -> None:await _preload_channel_modules_in_standby()await _run_or_defer("command dispatch and channels", _start_commands_and_channels)- await _run_or_defer("temp-file cleanup and Composio reconcile", _start_late_tasks)+ await _run_or_defer("temp-file cleanup and Composio loops", _start_late_tasks)start_runtime_performance_monitor(app)report_unobserved_execution_events(telemetry_started=getattr(app.state, "runtime_performance_task", None)@@ -2802,10 +2878,11 @@ async def retire_writers(thread or task that is in neither is not covered, whatever it does.**Stopped (tasks cancelled and awaited):** command dispatcher and its- detached prompt dispatches, orphan-upload GC, Composio reconcile,- retention purge, task-cleanup retry, uploaded-file recovery and reconcile,- auto-resume, lease recovery, sandbox idle sweep, startup file-storage- sync, LanceDB migration, trigger dispatcher, chat channels.+ detached prompt dispatches, orphan-upload GC, Composio reconcile and+ missing-account, retention purge, task-cleanup retry, uploaded-file+ recovery and reconcile, auto-resume, lease recovery, sandbox idle sweep,+ startup file-storage sync, LanceDB migration, trigger dispatcher, chat+ channels.**Fenced:** socket-owned writes. After the fence no WebSocket handlerstarts a write (message/start commands, builder turns, ASR sessions,@@ -2823,11 +2900,12 @@ async def retire_writers((``services/triggers``), the LanceDB user-id, documents anduploaded-file-link backfills, the retention-purge batch, task-cleanupretry (workspace removal, live-id check, claim, settle), the Composio- reconcile pass, the socket command enqueue, the channel input- acceptance/claim/staging threads (Feishu, Slack, Telegram, shared channel- execution) and the blocking calls a builder chat turn makes through- ``xagent.core.utils.blocking.run_blocking`` (knowledge-base ingestion and- the knowledge-base coordinator, LanceDB metadata-table creation).+ reconcile and missing-account passes, the socket command enqueue, the+ channel input acceptance/claim/staging threads (Feishu, Slack, Telegram,+ shared channel execution) and the blocking calls a builder chat turn makes+ through ``xagent.core.utils.blocking.run_blocking`` (knowledge-base+ ingestion and the knowledge-base coordinator, LanceDB metadata-table+ creation).**Not covered -- the host must quiesce these itself:** HTTP producers(the API routes, including the A2A, triggers, MCP and file routes, anddiff --git a/src/xagent/web/composio_auth.py b/src/xagent/web/composio_auth.pyindex f4e66bed2..b60fecfb7 100644--- a/src/xagent/web/composio_auth.py+++ b/src/xagent/web/composio_auth.py@@ -60,6 +60,21 @@ only after its Composio grants are gone and a pass has started afteraccounts from sign-ins begun before the switch are past the TTL plus grace(see ``example.env``).+A stored grant whose connected account Composio answers 404+``ConnectedAccount_ResourceNotFound`` for is tombstoned by+``tombstone_missing_composio_accounts``. That pass runs on its own interval,+on by default (``XAGENT_COMPOSIO_MISSING_ACCOUNT_INTERVAL_SECONDS``): it only+reads the grants this database stores and clears their local tokens, deleting+nothing in Composio. The row stays, with its token cleared, so the grant no+longer counts as connected and the user is offered to connect again. Every+other answer, including any other 404, leaves the row alone. A pass clears a+few missing accounts, up to a fixed number of lookups, and resumes where the+previous pass stopped in this process. When at least three lookups answer missing+and more than half of the pass's lookups do, it writes nothing and logs a+warning: a key that cannot see the accounts does not clear them, but neither+does a mass deletion, which needs an operator. With fewer than three grants+the breaker cannot trip.+Disconnect: deleting a grant through ``delete_scoped_user_oauth_accounts``schedules its Composio account for deletion once the caller's outermosttransaction commits (``schedule_account_deletions``); a rolled-back SAVEPOINT@@ -72,6 +87,7 @@ Composio user id or a ``session_uri``.from __future__ import annotations+import bisectimport hashlibimport hmacimport logging@@ -86,7 +102,7 @@ from urllib.parse import urlencode, urlsplitfrom cryptography.fernet import InvalidTokenfrom fastapi.responses import HTMLResponse, RedirectResponse, Response-from sqlalchemy import event+from sqlalchemy import event, textfrom ..config import COMPOSIO_CALLBACK_URL as COMPOSIO_CALLBACK_URL_ENVfrom ..config import get_composio_callback_url@@ -838,6 +854,17 @@ _XAGENT_USER_ID_PREFIX = "xagent-"# committed after the grace period could lose its account to a pass._RECONCILE_GRACE_SECONDS = 30 * 60_RECONCILE_PAGE_SIZE = 100+# Only this slug proves the account is gone; another 404 may mean something+# else Composio answers the same way.+_MISSING_ACCOUNT_SLUG = "ConnectedAccount_ResourceNotFound"+# One pass checks at most this many grants remotely. The cursor is per process,+# starts over on restart, and keeps rows past the cap from being starved.+_MISSING_ACCOUNT_LOOKUP_LIMIT = 200+# At least this many missing answers, and more than this share of the pass's+# lookups, is a broken view of the accounts rather than deletions.+_MISSING_ACCOUNT_BREAKER_MIN = 3+_MISSING_ACCOUNT_BREAKER_RATIO = 0.5+_missing_account_cursor = 0def _created_at(value: object) -> datetime | None:@@ -964,6 +991,135 @@ def reconcile_orphan_accounts(return deleted+def _rows_in_cursor_order(rows: list[Any]) -> list[Any]:+ """Every row once (rows are ordered by id), after the last handled one."""+ ids = [int(row[0]) for row in rows]+ start = bisect.bisect_right(ids, _missing_account_cursor)+ return rows[start:] + rows[:start]+++def tombstone_missing_composio_accounts(+ db: Session,+ *,+ stop_event: threading.Event | None = None,+) -> int:+ """Tombstone stored grants whose connected account Composio no longer has.++ The connector subprocess has no database access, so an account deleted on+ Composio's side would keep its grant looking connected. Only a 404 whose+ body's error slug is exactly ``_MISSING_ACCOUNT_SLUG`` proves it is gone.+ The row is kept with its token cleared, so the grant no longer counts as+ connected and the user is offered to connect again. A pass makes at most+ ``_MISSING_ACCOUNT_LOOKUP_LIMIT`` lookups, resumes where the previous pass+ stopped in this process, and writes nothing when the breaker trips. A+ stored grant that cannot be decrypted is skipped (its token is already+ unusable), and a reconnect that lands between lookup and write is not+ undone. Returns how many grants were tombstoned.+ """+ from .models.user_oauth import UserOAuth+ from .tools.mcp.composio_proxy import _composio_refusal++ global _missing_account_cursor++ configs = _configured_apps()+ if not configs:+ return 0++ def stopping() -> bool:+ return stop_event is not None and stop_event.is_set()++ rows = (+ db.query(UserOAuth.id, UserOAuth.provider, UserOAuth.access_token)+ .filter(UserOAuth.token_type == COMPOSIO_TOKEN_TYPE)+ .order_by(UserOAuth.id)+ .all()+ )+ db.rollback()+ stored = [+ (row_id, str(provider), str(token)) for row_id, provider, token in rows if token+ ]+ lookups_made = 0+ missing: list[tuple[int, str]] = []+ ended = False+ for row_id, provider, account_id in _rows_in_cursor_order(stored):+ if stopping():+ ended = True+ break+ config = configs.get(provider)+ if config is None or _valid_account_id(account_id) is None:+ continue+ if lookups_made >= _MISSING_ACCOUNT_LOOKUP_LIMIT:+ break+ try:+ status, payload = _call(+ config.api_key, "GET", f"/connected_accounts/{account_id}"+ )+ except ComposioError:+ # Unreachable Composio: the next pass retries this row first.+ ended = True+ break+ lookups_made += 1+ if 500 <= status < 600:+ # A record that keeps answering 5xx must not pin every row after it.+ _missing_account_cursor = row_id+ ended = True+ break+ if status in (401, 403, 429):+ # The key is unusable: stop before writing anything.+ ended = True+ break+ _missing_account_cursor = row_id+ if status != 404:+ continue+ _, slug = _composio_refusal("Composio account lookup", status, payload)+ if slug != _MISSING_ACCOUNT_SLUG:+ logger.warning(+ "Composio account lookup answered HTTP %s (slug=%s); leaving the grant",+ status,+ slug,+ )+ continue+ missing.append((row_id, account_id))+ if ended:+ return 0+ if (+ len(missing) >= _MISSING_ACCOUNT_BREAKER_MIN+ and len(missing) > lookups_made * _MISSING_ACCOUNT_BREAKER_RATIO+ ):+ logger.warning(+ "Composio missing-account pass skipped: %d of %d lookups answered missing",+ len(missing),+ lookups_made,+ )+ return 0+ tombstoned = 0+ for row_id, account_id in missing:+ if stopping():+ break+ # Lock and re-read: the reconnect that replaced this account since the+ # scan must not be undone.+ reread = db.query(UserOAuth).filter(UserOAuth.id == row_id).populate_existing()+ if db.get_bind().dialect.name == "sqlite":+ # SQLite has no row locks; the no-op write takes its write lock.+ db.execute(+ text("UPDATE user_oauth SET id = id WHERE id = :id"), {"id": row_id}+ )+ else:+ reread = reread.with_for_update()+ row = cast(Any, reread.one_or_none())+ if row is None or str(row.access_token or "") != account_id:+ db.rollback()+ continue+ row.access_token = ""+ row.refresh_token = None+ row.expires_at = None+ db.commit()+ tombstoned += 1+ if tombstoned:+ logger.info("Tombstoned %d Composio grants whose account is gone", tombstoned)+ return tombstoned++def reconcile_orphan_accounts_isolated(session_factory: Callable[[], Session] | None = None,*,@@ -978,6 +1134,20 @@ def reconcile_orphan_accounts_isolated(return reconcile_orphan_accounts(db, stop_event=stop_event)+def tombstone_missing_composio_accounts_isolated(+ session_factory: Callable[[], Session] | None = None,+ *,+ stop_event: threading.Event | None = None,+) -> int:+ """Run one missing-account pass in its own short-lived Session."""+ if session_factory is None:+ from .models.database import get_session_local++ session_factory = get_session_local()+ with session_factory() as db:+ return tombstone_missing_composio_accounts(db, stop_event=stop_event)++async def run_reconcile_loop(*, interval_seconds: int, stop_event: threading.Event) -> None:@@ -1004,6 +1174,34 @@ async def run_reconcile_loop(await asyncio.sleep(interval_seconds)+async def run_missing_account_loop(+ *, interval_seconds: int, stop_event: threading.Event+) -> None:+ """Tombstone missing accounts on an interval; a failed pass is retried later.++ Cancelling the task does not stop a pass already handed to a worker+ thread, so shutdown sets ``stop_event`` first and the pass returns at its+ next request boundary.+ """+ import asyncio++ from .services.writer_workers import run_tracked++ while not stop_event.is_set():+ try:+ await run_tracked(+ tombstone_missing_composio_accounts_isolated, stop_event=stop_event+ )+ except asyncio.CancelledError:+ raise+ except Exception as exc:+ logger.warning(+ "Composio missing-account pass failed (exception_type=%s)",+ type(exc).__name__,+ )+ await asyncio.sleep(interval_seconds)++# --- Browser trips -------------------------------------------------------------diff --git a/tests/core/test_config.py b/tests/core/test_config.pyindex cd5352529..8c83be5eb 100644--- a/tests/core/test_config.py+++ b/tests/core/test_config.py@@ -2721,6 +2721,37 @@ class TestComposioReconcileConfig:assert get_composio_reconcile_interval_seconds() is None+class TestComposioMissingAccountConfig:+ """How often grants whose Composio account is gone are cleared."""++ def test_on_by_default(self, monkeypatch):+ from xagent.config import get_composio_missing_account_interval_seconds++ monkeypatch.delenv(+ "XAGENT_COMPOSIO_MISSING_ACCOUNT_INTERVAL_SECONDS", raising=False+ )+ assert get_composio_missing_account_interval_seconds() == 3600+ monkeypatch.setenv("XAGENT_COMPOSIO_MISSING_ACCOUNT_INTERVAL_SECONDS", " ")+ assert get_composio_missing_account_interval_seconds() == 3600++ def test_zero_turns_it_off(self, monkeypatch):+ from xagent.config import get_composio_missing_account_interval_seconds++ monkeypatch.setenv("XAGENT_COMPOSIO_MISSING_ACCOUNT_INTERVAL_SECONDS", "0")+ assert get_composio_missing_account_interval_seconds() == 0++ def test_a_set_interval_has_a_floor(self, monkeypatch):+ from xagent.config import get_composio_missing_account_interval_seconds++ monkeypatch.setenv("XAGENT_COMPOSIO_MISSING_ACCOUNT_INTERVAL_SECONDS", "900")+ assert get_composio_missing_account_interval_seconds() == 900+ for invalid in ("5", "hourly", "-1"):+ monkeypatch.setenv(+ "XAGENT_COMPOSIO_MISSING_ACCOUNT_INTERVAL_SECONDS", invalid+ )+ assert get_composio_missing_account_interval_seconds() == 3600++class TestTaskCleanupRetryConfig:"""Config for retrying the external cleanup a task deletion owes (#2587)."""diff --git a/tests/web/services/test_standby_startup.py b/tests/web/services/test_standby_startup.pyindex 51d36ef70..3a890fa7b 100644--- a/tests/web/services/test_standby_startup.py+++ b/tests/web/services/test_standby_startup.py@@ -102,6 +102,7 @@ def _patch_startup(monkeypatch: pytest.MonkeyPatch, events: list[str]) -> _Sandb"start_retention_purge_task","start_task_cleanup_retry_task","start_composio_reconcile_task",+ "start_composio_missing_account_task","start_runtime_performance_monitor","start_app_process_diagnostics",):@@ -325,6 +326,7 @@ async def test_shutdown_cleans_sandboxes_only_when_it_owned_them(for name in ("stop_orphan_upload_gc_task","stop_composio_reconcile_task",+ "stop_composio_missing_account_task","stop_retention_purge_task","stop_task_cleanup_retry_task","stop_uploaded_file_recovery_task",diff --git a/tests/web/test_composio_facebook_connect.py b/tests/web/test_composio_facebook_connect.pyindex ae4fa3079..f66523e5c 100644--- a/tests/web/test_composio_facebook_connect.py+++ b/tests/web/test_composio_facebook_connect.py@@ -86,6 +86,10 @@ class FakeComposio:self.complete_without_id = Falseself.complete_account: str | None = Noneself.account_overrides: dict[str, Any] = {}+ # A single-account lookup's own answer, by connected account id.+ self.account_lookup: dict[str, tuple[int, Any]] = {}+ # Runs before a single-account lookup answers, to change the row first.+ self.on_lookup: Any = Noneself.me_response: tuple[int, Any] | None = Noneself.raise_on: str | None = None# Pages GET /connected_accounts returns, by cursor ("" is the first).@@ -152,6 +156,11 @@ class FakeComposio:return _Response(200, page)if method == "GET" and path.startswith("/connected_accounts/"):account_id = path.rsplit("/", 1)[1]+ if self.on_lookup is not None:+ self.on_lookup(account_id)+ if account_id in self.account_lookup:+ status, payload = self.account_lookup[account_id]+ return _Response(status, payload)auth_config_id = self.auth_configs.get(account_id, AUTH_CONFIG_ID)toolkit = ("instagram"@@ -1187,6 +1196,8 @@ def _store_composio_grant(owner: str = OWNER,provider: str = "facebook",facebook_id: str = FACEBOOK_USER_ID,+ refresh_token: str | None = None,+ expires_at: datetime | None = None,) -> None:db.add(UserOAuth(@@ -1197,6 +1208,8 @@ def _store_composio_grant(access_token=account_id,token_type="composio",instance_url="xagent-user",+ refresh_token=refresh_token,+ expires_at=expires_at,))db.commit()@@ -1788,6 +1801,491 @@ def test_a_stop_signal_before_the_pass_lists_nothing(assert composio.calls == []+# --- Reconciler: grants whose connected account Composio no longer has ---------------++MISSING_ACCOUNT = {+ "error": {+ "slug": "ConnectedAccount_ResourceNotFound",+ "message": "Connected account not found",+ }+}+++def _lookups(composio: FakeComposio) -> list[str]:+ """The single-account lookups, not the auth-config listings."""+ return [+ path+ for path in composio.paths("GET")+ if path.startswith("/connected_accounts/")+ ]+++def test_a_grant_whose_account_is_gone_is_tombstoned(+ db_session, composio_env, composio, monkeypatch+):+ from xagent.web.api import mcp as mcp_api++ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ _store_composio_grant(+ db,+ user,+ account_id="ca_gone",+ refresh_token="refresh",+ expires_at=datetime(2026, 2, 1, tzinfo=timezone.utc),+ )+ row = _grants(db)[0]+ assert mcp_api._oauth_account_can_connect(row) is True+ assert mcp_api._build_oauth_account_lookup([row]) == {"facebook": row}+ composio.account_lookup["ca_gone"] = (404, MISSING_ACCOUNT)++ assert composio_auth.tombstone_missing_composio_accounts(db) == 1++ row = _grants(db)[0]+ assert row.id is not None+ assert row.resource_owner_key == OWNER+ assert row.access_token == ""+ assert row.refresh_token is None+ assert row.expires_at is None+ assert mcp_api._oauth_account_can_connect(row) is False+ assert mcp_api._build_oauth_account_lookup([row]) == {}+++def test_a_reconnect_between_lookup_and_write_is_kept(+ db_session, composio_env, composio, monkeypatch+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ _store_composio_grant(db, user, account_id="ca_old")+ row_id = _grants(db)[0].id+ composio.account_lookup["ca_old"] = (404, MISSING_ACCOUNT)++ def reconnect(account_id: str) -> None:+ db.execute(+ text("UPDATE user_oauth SET access_token = :t WHERE id = :i"),+ {"t": encrypt_value("ca_new"), "i": row_id},+ )+ db.commit()++ composio.on_lookup = reconnect++ assert composio_auth.tombstone_missing_composio_accounts(db) == 0++ assert _grants(db)[0].access_token == "ca_new"+++ "answer",+ [+ (404, {"error": {"slug": "ConnectedAccount_NotFound"}}),+ (404, None),+ (401, {"error": {"slug": "ConnectedAccount_ResourceNotFound"}}),+ (500, {"error": {"slug": "ConnectedAccount_ResourceNotFound"}}),+ (200, {"id": "ca_bound", "status": "EXPIRED"}),+ ],+)+def test_any_other_lookup_answer_leaves_the_grant_alone(+ db_session, composio_env, composio, monkeypatch, answer+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ _store_composio_grant(db, user, account_id="ca_bound")+ composio.account_lookup["ca_bound"] = answer++ assert composio_auth.tombstone_missing_composio_accounts(db) == 0++ assert _lookups(composio) == ["/connected_accounts/ca_bound"]+ assert _grants(db)[0].access_token == "ca_bound"+++def test_an_unreachable_composio_ends_the_pass(+ db_session, composio_env, composio, monkeypatch+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ _store_composio_grant(db, user, account_id="ca_first")+ _store_composio_grant(db, user, account_id="ca_second", facebook_id="102030405061")+ composio.raise_on = "/connected_accounts/ca_first"++ assert composio_auth.tombstone_missing_composio_accounts(db) == 0++ assert _lookups(composio) == ["/connected_accounts/ca_first"]+ assert [grant.access_token for grant in _grants(db)] == [+ "ca_first",+ "ca_second",+ ]+++def test_an_undecryptable_grant_is_skipped_and_the_pass_continues(+ db_session, composio_env, composio, monkeypatch+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ _store_composio_grant(db, user, account_id="ca_foreign")+ _store_composio_grant(db, user, account_id="ca_gone", facebook_id="102030405061")+ foreign_id = _grants(db)[0].id+ foreign = Fernet(Fernet.generate_key()).encrypt(b"ca_foreign").decode()+ db.execute(+ text("UPDATE user_oauth SET access_token = :t WHERE id = :i"),+ {"t": foreign, "i": foreign_id},+ )+ db.commit()+ composio.account_lookup["ca_gone"] = (404, MISSING_ACCOUNT)++ assert composio_auth.tombstone_missing_composio_accounts(db) == 1++ assert _lookups(composio) == ["/connected_accounts/ca_gone"]+ assert (+ db.execute(+ text("SELECT access_token FROM user_oauth WHERE id = :i"),+ {"i": foreign_id},+ ).scalar()+ == foreign+ )+++def test_a_stop_signal_ends_the_missing_account_pass_between_lookups(+ db_session, composio_env, composio, monkeypatch+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ _store_composio_grant(db, user, account_id="ca_first")+ _store_composio_grant(db, user, account_id="ca_second", facebook_id="102030405061")+ stop = threading.Event()+ composio.on_lookup = lambda account_id: stop.set()++ assert composio_auth.tombstone_missing_composio_accounts(db, stop_event=stop) == 0++ assert _lookups(composio) == ["/connected_accounts/ca_first"]+++def test_a_stop_signal_before_the_missing_account_pass_looks_nothing_up(+ db_session, composio_env, composio, monkeypatch+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ _store_composio_grant(db, user, account_id="ca_gone")+ stop = threading.Event()+ stop.set()++ assert composio_auth.tombstone_missing_composio_accounts(db, stop_event=stop) == 0++ assert _lookups(composio) == []+++def test_one_pass_is_capped_and_the_next_pass_continues(+ db_session, composio_env, composio, monkeypatch+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ monkeypatch.setattr(composio_auth, "_MISSING_ACCOUNT_LOOKUP_LIMIT", 1)+ _store_composio_grant(db, user, account_id="ca_first")+ _store_composio_grant(db, user, account_id="ca_second", facebook_id="102030405061")++ assert composio_auth.tombstone_missing_composio_accounts(db) == 0++ assert _lookups(composio) == ["/connected_accounts/ca_first"]++ assert composio_auth.tombstone_missing_composio_accounts(db) == 0++ assert _lookups(composio) == [+ "/connected_accounts/ca_first",+ "/connected_accounts/ca_second",+ ]+++def test_the_cursor_wraps_after_the_last_row(+ db_session, composio_env, composio, monkeypatch+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ monkeypatch.setattr(composio_auth, "_MISSING_ACCOUNT_LOOKUP_LIMIT", 1)+ _store_composio_grant(db, user, account_id="ca_first")+ _store_composio_grant(db, user, account_id="ca_second", facebook_id="102030405061")++ for _ in range(3):+ assert composio_auth.tombstone_missing_composio_accounts(db) == 0++ assert _lookups(composio) == [+ "/connected_accounts/ca_first",+ "/connected_accounts/ca_second",+ "/connected_accounts/ca_first",+ ]+++def test_rows_without_a_usable_lookup_do_not_count_toward_the_cap(+ db_session, composio_env, composio, monkeypatch+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ monkeypatch.setattr(composio_auth, "_MISSING_ACCOUNT_LOOKUP_LIMIT", 1)+ _store_composio_grant(db, user, account_id="ca bad/id")+ _store_composio_grant(+ db,+ user,+ account_id="ca_other_app",+ provider="instagram",+ facebook_id="102030405061",+ )+ _store_composio_grant(db, user, account_id="ca_gone", facebook_id="102030405062")+ composio.account_lookup["ca_gone"] = (404, MISSING_ACCOUNT)++ assert composio_auth.tombstone_missing_composio_accounts(db) == 1++ assert _lookups(composio) == ["/connected_accounts/ca_gone"]+ assert [grant.access_token for grant in _grants(db)] == [+ "ca bad/id",+ "ca_other_app",+ "",+ ]+++# --- Missing-account circuit breaker --------------------------------------------------+++def _warning_messages(caplog: pytest.LogCaptureFixture) -> list[str]:+ return [+ record.getMessage()+ for record in caplog.records+ if record.name == composio_auth.__name__ and record.levelno == logging.WARNING+ ]+++def test_most_lookups_answering_missing_write_nothing(+ db_session, composio_env, composio, monkeypatch, caplog+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ for index, account_id in enumerate(("ca_1", "ca_2", "ca_3")):+ _store_composio_grant(+ db, user, account_id=account_id, facebook_id=f"10203040506{index}"+ )+ composio.account_lookup[account_id] = (404, MISSING_ACCOUNT)+ _store_composio_grant(db, user, account_id="ca_4", facebook_id="102030405063")+ composio.account_lookup["ca_4"] = (200, {"id": "ca_4"})++ with caplog.at_level(logging.WARNING, logger=composio_auth.__name__):+ assert composio_auth.tombstone_missing_composio_accounts(db) == 0++ assert [grant.access_token for grant in _grants(db)] == [+ "ca_1",+ "ca_2",+ "ca_3",+ "ca_4",+ ]+ warnings = _warning_messages(caplog)+ assert len(warnings) == 1+ assert "3 of 4" in warnings[0]+++def test_below_the_breaker_minimum_missing_rows_are_tombstoned(+ db_session, composio_env, composio, monkeypatch+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ for index, account_id in enumerate(("ca_1", "ca_2")):+ _store_composio_grant(+ db, user, account_id=account_id, facebook_id=f"10203040506{index}"+ )+ composio.account_lookup[account_id] = (404, MISSING_ACCOUNT)++ assert composio_auth.tombstone_missing_composio_accounts(db) == 2++ assert [grant.access_token for grant in _grants(db)] == ["", ""]+++def test_a_single_missing_answer_among_three_is_tombstoned(+ db_session, composio_env, composio, monkeypatch+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ for index, account_id in enumerate(("ca_1", "ca_2", "ca_3")):+ _store_composio_grant(+ db, user, account_id=account_id, facebook_id=f"10203040506{index}"+ )+ composio.account_lookup["ca_1"] = (404, MISSING_ACCOUNT)++ assert composio_auth.tombstone_missing_composio_accounts(db) == 1++ assert [grant.access_token for grant in _grants(db)] == ["", "ca_2", "ca_3"]+++def test_another_404_slug_leaves_the_grant_and_logs_the_slug(+ db_session, composio_env, composio, monkeypatch, caplog+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ _store_composio_grant(db, user, account_id="ca_keep")+ composio.account_lookup["ca_keep"] = (+ 404,+ {"error": {"slug": "ConnectedAccount_NotFound", "message": "gone"}},+ )++ with caplog.at_level(logging.WARNING, logger=composio_auth.__name__):+ assert composio_auth.tombstone_missing_composio_accounts(db) == 0++ assert _grants(db)[0].access_token == "ca_keep"+ messages = _warning_messages(caplog)+ assert any("ConnectedAccount_NotFound" in message for message in messages)+ assert all("ca_keep" not in message for message in messages)+++ ("status", "second_pass_first"),+ [(401, "ca_first"), (403, "ca_first"), (429, "ca_first"), (503, "ca_second")],+)+def test_a_refused_or_unavailable_lookup_ends_the_pass_before_writing(+ db_session, composio_env, composio, monkeypatch, status, second_pass_first+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ _store_composio_grant(db, user, account_id="ca_first")+ _store_composio_grant(db, user, account_id="ca_second", facebook_id="102030405061")+ composio.account_lookup["ca_first"] = (status, {"error": {"message": "x"}})++ assert composio_auth.tombstone_missing_composio_accounts(db) == 0++ assert _lookups(composio) == ["/connected_accounts/ca_first"]+ assert [grant.access_token for grant in _grants(db)] == [+ "ca_first",+ "ca_second",+ ]++ # 5xx is skipped next pass; a refused key retries the same row.+ composio.account_lookup["ca_first"] = (200, {"id": "ca_first"})+ assert composio_auth.tombstone_missing_composio_accounts(db) == 0+ assert _lookups(composio)[1] == f"/connected_accounts/{second_pass_first}"+++def test_a_row_deleted_between_lookup_and_write_is_not_an_error(+ db_session, composio_env, composio, monkeypatch+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ _store_composio_grant(db, user, account_id="ca_old")+ row_id = _grants(db)[0].id+ composio.account_lookup["ca_old"] = (404, MISSING_ACCOUNT)++ def disconnect(account_id: str) -> None:+ db.execute(text("DELETE FROM user_oauth WHERE id = :i"), {"i": row_id})+ db.commit()++ composio.on_lookup = disconnect++ assert composio_auth.tombstone_missing_composio_accounts(db) == 0++ assert _grants(db) == []+++def test_a_reconnect_replacing_the_row_between_lookup_and_write_is_kept(+ db_session, composio_env, composio, monkeypatch+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ _store_composio_grant(db, user, account_id="ca_old")+ row_id = _grants(db)[0].id+ composio.account_lookup["ca_old"] = (404, MISSING_ACCOUNT)++ def reconnect(account_id: str) -> None:+ db.execute(text("DELETE FROM user_oauth WHERE id = :i"), {"i": row_id})+ db.commit()+ _store_composio_grant(db, user, account_id="ca_new")++ composio.on_lookup = reconnect++ assert composio_auth.tombstone_missing_composio_accounts(db) == 0++ grants = _grants(db)+ assert [grant.access_token for grant in grants] == ["ca_new"]+++def test_the_orphan_pass_leaves_missing_accounts_to_their_own_loop(+ db_session, composio_env, composio, monkeypatch+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ _store_composio_grant(db, user, account_id="ca_gone")+ composio.account_lookup["ca_gone"] = (404, MISSING_ACCOUNT)+ composio.listing[""] = {"items": []}++ composio_auth.reconcile_orphan_accounts_isolated(session_factory=lambda: db)++ assert _lookups(composio) == []+ assert _grants(db)[0].access_token == "ca_gone"+++def test_the_isolated_missing_account_pass_tombstones_missing_accounts(+ db_session, composio_env, composio, monkeypatch+):+ db, user = db_session+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ _store_composio_grant(db, user, account_id="ca_gone")+ composio.account_lookup["ca_gone"] = (404, MISSING_ACCOUNT)++ tombstoned = composio_auth.tombstone_missing_composio_accounts_isolated(+ session_factory=lambda: db+ )++ assert tombstoned == 1+ assert _grants(db)[0].access_token == ""+++def test_the_missing_account_loop_needs_a_configured_app(monkeypatch):+ from xagent.web.app import start_composio_missing_account_task++ monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)+ monkeypatch.delenv(composio_auth.COMPOSIO_API_KEY_ENV, raising=False)+ monkeypatch.delenv(+ "XAGENT_COMPOSIO_MISSING_ACCOUNT_INTERVAL_SECONDS", raising=False+ )+ app = SimpleNamespace(state=SimpleNamespace())++ assert start_composio_missing_account_task(app) is None+++def test_the_missing_account_loop_is_off_at_zero(composio_env, monkeypatch):+ from xagent.web.app import start_composio_missing_account_task++ monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)+ monkeypatch.setenv("XAGENT_COMPOSIO_MISSING_ACCOUNT_INTERVAL_SECONDS", "0")+ app = SimpleNamespace(state=SimpleNamespace())++ assert start_composio_missing_account_task(app) is None+++async def test_the_missing_account_loop_starts_with_its_variable_unset(+ composio_env, monkeypatch+):+ import asyncio++ from xagent.web.app import (+ start_composio_missing_account_task,+ stop_composio_missing_account_task,+ )++ started = threading.Event()++ def pass_once(*, stop_event: threading.Event) -> int:+ started.set()+ return 0++ monkeypatch.setattr(+ composio_auth, "tombstone_missing_composio_accounts_isolated", pass_once+ )+ monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)+ monkeypatch.delenv(+ "XAGENT_COMPOSIO_MISSING_ACCOUNT_INTERVAL_SECONDS", raising=False+ )+ app = SimpleNamespace(state=SimpleNamespace())++ task = start_composio_missing_account_task(app)+ assert task is not None+ assert await asyncio.to_thread(started.wait, 5)+ await stop_composio_missing_account_task(app)++ assert task.done()++def test_the_reconcile_loop_is_off_unless_its_interval_is_set(composio_env, monkeypatch):diff --git a/tests/web/test_retire_writers.py b/tests/web/test_retire_writers.pyindex 16c8f29c6..3b995bc29 100644--- a/tests/web/test_retire_writers.py+++ b/tests/web/test_retire_writers.py@@ -36,6 +36,7 @@ def steps(monkeypatch: pytest.MonkeyPatch) -> Iterator[list[str]]:for name in ("stop_orphan_upload_gc_task","stop_composio_reconcile_task",+ "stop_composio_missing_account_task","stop_retention_purge_task","stop_task_cleanup_retry_task","stop_uploaded_file_recovery_task",@@ -74,6 +75,7 @@ async def test_retire_writers_stops_every_writer_once_and_is_idempotent(steps):assert first == ["stop_orphan_upload_gc_task","stop_composio_reconcile_task",+ "stop_composio_missing_account_task","stop_retention_purge_task","stop_task_cleanup_retry_task","stop_uploaded_file_recovery_task",@@ -1078,6 +1080,7 @@ async def test_a_failed_channel_stop_leaves_the_retire_unpublished(for name in ("stop_orphan_upload_gc_task","stop_composio_reconcile_task",+ "stop_composio_missing_account_task","stop_retention_purge_task","stop_task_cleanup_retry_task","stop_uploaded_file_recovery_task",diff --git a/tests/web/tools/test_mcp_actor_owner_runtime.py b/tests/web/tools/test_mcp_actor_owner_runtime.pyindex be5fd1519..c857748d5 100644--- a/tests/web/tools/test_mcp_actor_owner_runtime.py+++ b/tests/web/tools/test_mcp_actor_owner_runtime.py@@ -1150,6 +1150,70 @@ async def test_actor_composio_facebook_grant_launches_with_composio_env(assert env["COMPOSIO_USER_ID"] == "actor-hmac"+async def test_actor_composio_grant_with_a_gone_account_is_unavailable(+ db_session, monkeypatch+) -> None:+ """A grant the reconcile pass tombstoned is no longer a credential: the+ connector is reported unavailable instead of launching."""+ from xagent.web import composio_auth++ monkeypatch.setenv("COMPOSIO_API_KEY", "composio-key")+ monkeypatch.setenv("COMPOSIO_FACEBOOK_AUTH_CONFIG_ID", "ac_facebook")+ monkeypatch.setenv("COMPOSIO_USER_ID_SECRET", "composio-secret")+ monkeypatch.setattr(composio_auth, "_missing_account_cursor", 0)+ monkeypatch.setattr(+ composio_auth,+ "_call",+ lambda *args, **kwargs: (+ 404,+ {"error": {"slug": "ConnectedAccount_ResourceNotFound"}},+ ),+ )+ db, user = db_session.db, db_session.user+ app = get_builtin_public_mcp_app("facebook")+ assert app is not None+ db.add(PublicMCPApp(**app))+ server = MCPServer(+ name="Facebook Pages",+ managed="external",+ transport="oauth",+ auth={"app_id": "facebook", "provider": "meta"},+ )+ db.add(server)+ db.flush()+ db.add(+ UserMCPServer(+ user_id=user.id,+ mcpserver_id=server.id,+ is_owner=False,+ is_active=True,+ )+ )+ db.add(+ UserOAuth(+ user_id=user.id,+ provider="facebook",+ resource_owner_key=OWNER_A,+ provider_user_id="fb-user-1",+ access_token="ca_123",+ instance_url="actor-hmac",+ token_type="composio",+ )+ )+ db.commit()++ assert composio_auth.tombstone_missing_composio_accounts(db) == 1+ assert db.query(UserOAuth).one().access_token == ""+ assert mcp_api._oauth_account_can_connect(db.query(UserOAuth).one()) is False++ configs = await _config(db_session, policy=_policy()).get_mcp_server_configs()++ assert configs[0]["transport"] == "unavailable"+ assert configs[0]["config"]["reason"] == "oauth_token_required"+ assert "ca_123" not in str(configs)++@pytest.mark.asyncioasync def test_actor_composio_instagram_grant_is_not_shadowed_by_a_bare_meta_grant(db_session, monkeypatch