Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
feat(server): track MCP connector health (#59)
Leo939b7e17a68d
diff --git a/apps/server/src/bot-inbox/connectorIncidents.test.ts b/apps/server/src/bot-inbox/connectorIncidents.test.tsindex e79d1cb87..84783bed4 100644--- a/apps/server/src/bot-inbox/connectorIncidents.test.ts+++ b/apps/server/src/bot-inbox/connectorIncidents.test.ts@@ -3,11 +3,11 @@ import * as NodeFS from "node:fs";import * as NodeOS from "node:os";import * as NodePath from "node:path";-import { BotId } from "@t3tools/contracts";+import { BotId, type ProviderAccessStatus } from "@t3tools/contracts";import { afterEach, describe, expect, it } from "vite-plus/test";import type { ProviderStatus } from "../subscription-auth/service.ts";-import { syncConnectorIncidents } from "./connectorIncidents.ts";+import { syncAccessIncidents, syncConnectorIncidents } from "./connectorIncidents.ts";import { BotInboxService } from "./service.ts";const directories: string[] = [];@@ -131,4 +131,39 @@ describe("connector inbox incidents", () => {}),]);});++ it("opens and resolves an MCP access incident without retry claims", () => {+ const inbox = fixture();+ const access: ProviderAccessStatus = {+ id: "mcp-builtin-exa",+ label: "Exa",+ accessMethod: "mcp",+ health: "failed-first-request",+ apiAccess: "not-applicable",+ nextAction: "Reconnect Exa, then retry its failed request.",+ repairAction: "Reconnect",+ reconnectAction: "Reconnect",+ serverId: "builtin-exa",+ pluginId: "exa",+ lastFailedRequest: {+ at: "2026-08-30T20:00:00.000Z",+ message: "The MCP tool request failed.",+ },+ dependentBots: [{ id: BotId.make("bot-akeru"), name: "Akeru" }],+ dependentRoutines: [],+ };++ syncAccessIncidents(inbox, [access]);+ expect(inbox.list()[0]).toEqual(+ expect.objectContaining({+ incidentKey: "access:mcp-builtin-exa:bot-akeru",+ lastFailure: "The MCP tool request failed.",+ nextAction: "Reconnect Exa, then retry its failed request.",+ status: "open",+ }),+ );++ syncAccessIncidents(inbox, [{ ...access, health: "recovered" }]);+ expect(inbox.list()[0]?.status).toBe("resolved");+ });});diff --git a/apps/server/src/bot-inbox/connectorIncidents.ts b/apps/server/src/bot-inbox/connectorIncidents.tsindex c263733d5..c439ef176 100644--- a/apps/server/src/bot-inbox/connectorIncidents.ts+++ b/apps/server/src/bot-inbox/connectorIncidents.ts@@ -1,3 +1,5 @@+import type { ProviderAccessStatus } from "@t3tools/contracts";+import type { ProviderStatus } from "../subscription-auth/service.ts";import { BotInboxService } from "./service.ts";@@ -51,3 +53,50 @@ export function syncConnectorIncidents(}}}++export function syncAccessIncidents(+ botInbox: BotInboxService,+ access: ReadonlyArray<ProviderAccessStatus>,+): void {+ const unresolvedIncidents = botInbox+ .list()+ .filter((incident) => incident.status === "open" || incident.acknowledgedAt !== undefined);+ const unresolvedIncidentKeys = new Set(+ unresolvedIncidents.map((incident) => incident.incidentKey),+ );+ const currentIncidentKeys = new Set<string>();+ for (const item of access) {+ if (item.accessMethod === "subscription-oauth") continue;+ for (const bot of item.dependentBots) {+ const incidentKey = `access:${item.id}:${bot.id}`;+ currentIncidentKeys.add(incidentKey);+ if (+ item.health === "failed" ||+ item.health === "failed-first-request" ||+ item.health === "expired" ||+ item.health === "revoked"+ ) {+ botInbox.ensureOpen({+ incidentKey,+ kind: item.health === "expired" ? "oauth-expired" : "connector-failure",+ botId: bot.id,+ botName: bot.name,+ taskOrRoutine: `${item.label} access`,+ lastFailure: item.lastFailedRequest?.message ?? "The connector request failed.",+ nextAction: item.nextAction,+ });+ } else if (unresolvedIncidentKeys.has(incidentKey)) {+ botInbox.resolve(incidentKey);+ }+ }+ }++ for (const incident of unresolvedIncidents) {+ if (+ incident.incidentKey.startsWith("access:") &&+ !currentIncidentKeys.has(incident.incidentKey)+ ) {+ botInbox.resolve(incident.incidentKey);+ }+ }+}diff --git a/apps/server/src/provider/AkeruSessionResources.test.ts b/apps/server/src/provider/AkeruSessionResources.test.tsindex d72bca71f..d0f4bc645 100644--- a/apps/server/src/provider/AkeruSessionResources.test.ts+++ b/apps/server/src/provider/AkeruSessionResources.test.ts@@ -4,6 +4,7 @@ import * as NodeOS from "node:os";import * as NodePath from "node:path";import { LocalFilesystem, LocalSandbox, Workspace } from "@mastra/core/workspace";+import { McpServerId } from "@t3tools/contracts";import { afterEach, describe, expect, it, vi } from "vite-plus/test";import { AkeruSessionResources } from "./AkeruSessionResources.ts";@@ -40,6 +41,16 @@ const remoteInput = {mcpServers: [],};+const exaServer = {+ id: McpServerId.make("builtin-exa"),+ name: "Exa",+ transport: "url" as const,+ url: "https://mcp.exa.ai/mcp",+ enabled: true,+ createdAt: "2026-08-31T00:00:00.000Z",+ updatedAt: "2026-08-31T00:00:00.000Z",+};+describe("AkeruSessionResources", () => {afterEach(() => {for (const directory of directories) {@@ -101,6 +112,52 @@ describe("AkeruSessionResources", () => {await resources.shutdown();});+ it("reports MCP connection failures at the resource boundary", async () => {+ const onMcpServerConnectionFailure = vi.fn();+ const manager = {+ init: vi.fn(async () => undefined),+ disconnect: vi.fn(async () => undefined),+ getTools: vi.fn(() => ({})),+ getServerStatuses: vi.fn(() => [{ name: String(exaServer.id), connected: false }]),+ };+ const resources = new AkeruSessionResources({+ stateDir: stateDir(),+ makeRemoteWorkspace: async () => workspace(),+ makeBotBrowser: () => browser(),+ makeMcpManager: () => manager as never,+ onMcpServerConnectionFailure,+ toMcpServerConfigs: () => ({}),+ });++ await resources.acquire({ ...remoteInput, threadId: "mcp-failure", mcpServers: [exaServer] });+ expect(onMcpServerConnectionFailure).toHaveBeenCalledExactlyOnceWith(exaServer.id);+ await resources.shutdown();+ });++ it("reports every configured MCP server when manager initialization fails", async () => {+ const onMcpServerConnectionFailure = vi.fn();+ const manager = {+ init: vi.fn(async () => Promise.reject(new Error("MCP init failed"))),+ disconnect: vi.fn(async () => undefined),+ getTools: vi.fn(() => ({})),+ getServerStatuses: vi.fn(() => []),+ };+ const resources = new AkeruSessionResources({+ stateDir: stateDir(),+ makeRemoteWorkspace: async () => workspace(),+ makeBotBrowser: () => browser(),+ makeMcpManager: () => manager as never,+ onMcpServerConnectionFailure,+ toMcpServerConfigs: () => ({}),+ });++ await expect(+ resources.acquire({ ...remoteInput, threadId: "mcp-init-failure", mcpServers: [exaServer] }),+ ).rejects.toThrow("MCP init failed");+ expect(onMcpServerConnectionFailure).toHaveBeenCalledExactlyOnceWith(exaServer.id);+ await resources.shutdown();+ });+it("removes a stale browser when reconnect fails", async () => {const firstBrowser = browser({reconnect: async () => Promise.reject(new Error("reconnect failed")),diff --git a/apps/server/src/provider/AkeruSessionResources.ts b/apps/server/src/provider/AkeruSessionResources.tsindex cf989e76a..ec8b11877 100644--- a/apps/server/src/provider/AkeruSessionResources.ts+++ b/apps/server/src/provider/AkeruSessionResources.ts@@ -44,6 +44,7 @@ export interface AkeruSessionResourcesOptions {readonly makeMcpManager?: typeof createMcpManager;readonly makeRemoteWorkspace?: (input: CreateRemoteBotWorkspaceInput) => Promise<Workspace>;readonly makeBotBrowser?: (input: CreateBotBrowserInput) => BotBrowser;+ readonly onMcpServerConnectionFailure?: (serverId: McpServer["id"]) => void;readonly toMcpServerConfigs: (servers: readonly McpServer[],browser?: BotBrowserAttachment,@@ -175,7 +176,20 @@ export class AkeruSessionResources {this.options.toMcpServerConfigs(input.mcpServers, attachment),);this.mcpManagers.set(key, manager);- await manager.init();+ try {+ await manager.init();+ } catch (cause) {+ for (const server of input.mcpServers) {+ this.options.onMcpServerConnectionFailure?.(server.id);+ }+ throw cause;+ }+ const serversById = new Map(input.mcpServers.map((server) => [String(server.id), server]));+ for (const status of manager.getServerStatuses()) {+ if (status.connected) continue;+ const server = serversById.get(status.name);+ if (server) this.options.onMcpServerConnectionFailure?.(server.id);+ }}return {@@ -272,7 +286,7 @@ export class AkeruSessionResources {async shutdown(): Promise<void> {this.shuttingDown = true;- await Promise.allSettled([...this.acquisitions.values()]);+ await Promise.allSettled(this.acquisitions.values());const threadIds = new Set([...this.acquisitions.keys(),...this.workspaceLeases.keys(),diff --git a/apps/server/src/provider/Layers/AgentController.test.ts b/apps/server/src/provider/Layers/AgentController.test.tsindex 875f2f4fe..1012615d1 100644--- a/apps/server/src/provider/Layers/AgentController.test.ts+++ b/apps/server/src/provider/Layers/AgentController.test.ts@@ -42,6 +42,7 @@ import type { ProviderServiceShape } from "../Services/ProviderService.ts";import {createAkeruMastraAuthStorage,makeAgentControllerLive,+ mcpServerIdForToolName,recordProviderAccessHealth,toMcpServerConfigs,type AgentControllerLiveOptions,@@ -272,6 +273,16 @@ function resolveCodex(controller: AgentController["Service"]) {}describe("toMcpServerConfigs", () => {+ it("attributes namespaced MCP tools to the exact server id", () => {+ expect(+ mcpServerIdForToolName(+ [McpServerId.make("builtin-exa"), McpServerId.make("builtin-exa-search")],+ "builtin-exa-search_find",+ ),+ ).toBe("builtin-exa-search");+ expect(mcpServerIdForToolName([McpServerId.make("builtin-exa")], "read_file")).toBeUndefined();+ });+it("converts only the bot's filtered MCP registrations for Mastra", () => {expect(toMcpServerConfigs([@@ -1071,10 +1082,12 @@ describe("AgentControllerLive", () => {it.effect("attaches the globally installed MCP servers selected for the bot", () => {const bridge = makeBridge();const mastra = makeMastraHarness();+ const baseDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "akeru-mastra-mcp-"));const mcpManager = {init: vi.fn(async () => undefined),disconnect: vi.fn(async () => undefined),- getTools: vi.fn(() => ({ exa_search: {} })),+ getTools: vi.fn(() => ({ "builtin-exa_search": {} })),+ getServerStatuses: vi.fn(() => [{ name: "builtin-exa", connected: true }]),};const makeMcpManagerMock = vi.fn((_dataDir, _configDir, _servers) => mcpManager as never);const makeMcpManager: NonNullable<AgentControllerLiveOptions["makeMcpManager"]> =@@ -1111,10 +1124,10 @@ describe("AgentControllerLive", () => {expect(mcpManager.init).toHaveBeenCalledOnce();assert.property(mastra.harnessOptions[0]?.getThreadTools(String(codexThreadId)),- "exa_search",+ "builtin-exa_search",);expect(mastra.session.permissions.setForTool).toHaveBeenCalledWith({- toolName: "exa_search",+ toolName: "builtin-exa_search",policy: "ask",});@@ -1122,7 +1135,7 @@ describe("AgentControllerLive", () => {mastra.emit({type: "tool_approval_required",toolCallId: "exa-tool-1",- toolName: "exa_search",+ toolName: "builtin-exa_search",args: { operation: "read" },} as AgentControllerEvent);yield* controller.respondToRequest({@@ -1132,24 +1145,64 @@ describe("AgentControllerLive", () => {});const syncApproval = mastra.harnessOptions[0]?.syncThreadToolApproval;assert.isDefined(syncApproval);- yield* Effect.promise(() => syncApproval(String(codexThreadId), "exa_search", true));+ yield* Effect.promise(() =>+ syncApproval(String(codexThreadId), "builtin-exa_search", true),+ );expect(mastra.session.permissions.setForTool).toHaveBeenLastCalledWith({- toolName: "exa_search",+ toolName: "builtin-exa_search",policy: "ask",});- yield* Effect.promise(() => syncApproval(String(codexThreadId), "exa_search", false));+ yield* Effect.promise(() =>+ syncApproval(String(codexThreadId), "builtin-exa_search", false),+ );expect(mastra.session.permissions.setForTool).toHaveBeenLastCalledWith({- toolName: "exa_search",+ toolName: "builtin-exa_search",policy: "allow",});+ mastra.emit({+ type: "tool_end",+ toolCallId: "exa-tool-1",+ result: "failed",+ isError: true,+ denied: false,+ } as AgentControllerEvent);+ expect(+ SubscriptionAuthService.forSecretsDir(+ NodePath.join(baseDir, "userdata", "secrets"),+ ).mcpRequestHealth(exaServer.id)?.health,+ ).toBe("failed-first-request");++ mastra.emit({+ type: "tool_start",+ toolCallId: "exa-tool-2",+ toolName: "builtin-exa_search",+ args: { operation: "read" },+ } as AgentControllerEvent);+ mastra.emit({+ type: "tool_end",+ toolCallId: "exa-tool-2",+ result: "ok",+ isError: false,+ denied: false,+ } as AgentControllerEvent);+ expect(+ SubscriptionAuthService.forSecretsDir(+ NodePath.join(baseDir, "userdata", "secrets"),+ ).mcpRequestHealth(exaServer.id)?.health,+ ).toBe("recovered");mastra.finishSend();yield* controller.stopSession({ threadId: codexThreadId });expect(mcpManager.disconnect).toHaveBeenCalledOnce();- }),+ }).pipe(+ Effect.ensuring(+ Effect.sync(() => NodeFS.rmSync(baseDir, { recursive: true, force: true })),+ ),+ ),bridge.service,mastra.factory,makeMcpManager,+ baseDir,);});diff --git a/apps/server/src/provider/Layers/AgentController.ts b/apps/server/src/provider/Layers/AgentController.tsindex eb19fc429..27485521f 100644--- a/apps/server/src/provider/Layers/AgentController.ts+++ b/apps/server/src/provider/Layers/AgentController.ts@@ -207,6 +207,15 @@ export function toMcpServerConfigs();}+export function mcpServerIdForToolName(+ serverIds: readonly McpServer["id"][],+ toolName: string,+): McpServer["id"] | undefined {+ return serverIds+ .toSorted((left, right) => String(right).length - String(left).length)+ .find((serverId) => toolName.startsWith(`${serverId}_`));+}+function permissionPolicy(runtimeMode: RuntimeMode,category: "read" | "edit" | "execute" | "mcp" | "other",@@ -316,6 +325,8 @@ const make = (options?: AgentControllerLiveOptions) =>const sessionResources = new AkeruSessionResources({stateDir: config.stateDir,toMcpServerConfigs,+ onMcpServerConnectionFailure: (serverId) =>+ subscriptionAuth.recordMcpRequestFailure(serverId, "The MCP server failed to connect."),...(options?.makeMcpManager ? { makeMcpManager: options.makeMcpManager } : {}),...(options?.makeRemoteWorkspace ? { makeRemoteWorkspace: options.makeRemoteWorkspace } : {}),...(options?.makeBotBrowser ? { makeBotBrowser: options.makeBotBrowser } : {}),@@ -566,6 +577,14 @@ const make = (options?: AgentControllerLiveOptions) =>if (!turn) return;const toolName = active.toolNames.get(event.toolCallId) ?? "tool";active.approvalRequests.delete(event.toolCallId);+ const mcpServerId = mcpServerIdForToolName(active.mcpServerIds, toolName);+ if (mcpServerId && !event.denied) {+ if (event.isError) {+ subscriptionAuth.recordMcpRequestFailure(mcpServerId, "The MCP tool request failed.");+ } else {+ subscriptionAuth.recordMcpRequestSuccess(mcpServerId);+ }+ }publish({...baseEvent(threadId, active, turn.turnId),itemId: RuntimeItemId.make(event.toolCallId),diff --git a/apps/server/src/subscription-auth/service.test.ts b/apps/server/src/subscription-auth/service.test.tsindex f9935d87f..3d6324570 100644--- a/apps/server/src/subscription-auth/service.test.ts+++ b/apps/server/src/subscription-auth/service.test.ts@@ -213,6 +213,38 @@ describe("subscription auth storage", () => {expect(new SubscriptionAuthService(authPath).providerInstanceHealth("grok")).toBe("recovered");});+ it("persists MCP failure and recovery without storing tool output or tokens", () => {+ const { authPath } = fixture();+ const service = new SubscriptionAuthService(authPath);++ service.recordMcpRequestFailure(+ "builtin-executor",+ "The MCP tool request failed.",+ "2026-08-31T19:00:00.000Z",+ );+ expect(service.mcpRequestHealth("builtin-executor")).toMatchObject({+ health: "failed-first-request",+ lastFailedRequest: { message: "The MCP tool request failed." },+ });+ service.recordMcpRequestSuccess("builtin-executor", "2026-08-31T20:00:00.000Z");+ const restarted = new SubscriptionAuthService(authPath);+ expect(restarted.mcpRequestHealth("builtin-executor")?.health).toBe("recovered");+ expect(JSON.stringify(restarted.mcpRequestHealth("builtin-executor"))).not.toContain("token");+ });++ it("uses the last MCP health result when requests finish in the same millisecond", () => {+ const { authPath } = fixture();+ const service = new SubscriptionAuthService(authPath);+ const at = "2026-08-31T20:00:00.000Z";++ service.recordMcpRequestFailure("builtin-executor", "The request failed.", at);+ service.recordMcpRequestSuccess("builtin-executor", at);+ expect(service.mcpRequestHealth("builtin-executor")?.health).toBe("recovered");++ service.recordMcpRequestFailure("builtin-executor", "The retry failed.", at);+ expect(service.mcpRequestHealth("builtin-executor")?.health).toBe("failed");+ });+it("preserves health updates written by another service instance", () => {const { authPath } = fixture();const providerRuntime = new SubscriptionAuthService(authPath);diff --git a/apps/server/src/subscription-auth/service.ts b/apps/server/src/subscription-auth/service.tsindex a4bb27232..54f4bf7e7 100644--- a/apps/server/src/subscription-auth/service.ts+++ b/apps/server/src/subscription-auth/service.ts@@ -117,6 +117,13 @@ interface ProviderHealthRecord {failureKind?: "request" | "revoked";}+export interface RequestHealthStatus {+ readonly health: "healthy" | "failed" | "failed-first-request" | "recovered";+ readonly lastSuccessfulRequestAt?: string;+ readonly lastFailedRequest?: { readonly at: string; readonly message: string };+ readonly nextRetryAt?: string;+}+type ProviderHealthData = Record<string, ProviderHealthRecord | undefined>;function oauthFailureKind(cause: unknown): "request" | "revoked" {@@ -314,6 +321,14 @@ export class SubscriptionAuthService {this.recordHealthFailure(`provider:${instanceId}`, message, at, "request");}+ recordMcpRequestSuccess(serverId: string, at = new Date().toISOString()): void {+ this.recordHealthSuccess(`mcp:${serverId}`, at);+ }++ recordMcpRequestFailure(serverId: string, message: string, at = new Date().toISOString()): void {+ this.recordHealthFailure(`mcp:${serverId}`, message, at, "request");+ }+private recordHealthFailure(key: string,message: string,@@ -334,23 +349,58 @@ export class SubscriptionAuthService {providerInstanceHealth(instanceId: string,): "healthy" | "failed" | "failed-first-request" | "recovered" | undefined {- const health = this.health[`provider:${instanceId}`];+ return this.requestHealth(`provider:${instanceId}`)?.health;+ }++ providerInstanceRequestHealth(instanceId: string): RequestHealthStatus | undefined {+ return this.requestHealth(`provider:${instanceId}`);+ }++ mcpRequestHealth(serverId: string): RequestHealthStatus | undefined {+ return this.requestHealth(`mcp:${serverId}`);+ }++ private requestHealth(key: string): RequestHealthStatus | undefined {+ const health = this.health[key];if (!health) return undefined;if (health.lastFailedRequest &&(!health.lastSuccessfulRequestAt ||- health.lastFailedRequest.at >= health.lastSuccessfulRequestAt)+ health.lastFailedRequest.at > health.lastSuccessfulRequestAt ||+ (health.lastFailedRequest.at === health.lastSuccessfulRequestAt &&+ health.healthTest?.status === "failed"))) {- return health.lastSuccessfulRequestAt ? "failed" : "failed-first-request";+ return {+ health: health.lastSuccessfulRequestAt ? "failed" : "failed-first-request",+ ...(health.lastSuccessfulRequestAt+ ? { lastSuccessfulRequestAt: health.lastSuccessfulRequestAt }+ : {}),+ lastFailedRequest: health.lastFailedRequest,+ ...(health.nextRetryAt ? { nextRetryAt: health.nextRetryAt } : {}),+ };}if (health.lastSuccessfulRequestAt &&health.lastFailedRequest &&- health.lastSuccessfulRequestAt > health.lastFailedRequest.at+ (health.lastSuccessfulRequestAt > health.lastFailedRequest.at ||+ (health.lastSuccessfulRequestAt === health.lastFailedRequest.at &&+ health.healthTest?.status === "passed"))) {- return "recovered";+ return {+ health: "recovered",+ lastSuccessfulRequestAt: health.lastSuccessfulRequestAt,+ lastFailedRequest: health.lastFailedRequest,+ ...(health.nextRetryAt ? { nextRetryAt: health.nextRetryAt } : {}),+ };}- return health.lastSuccessfulRequestAt ? "healthy" : undefined;+ return health.lastSuccessfulRequestAt+ ? {+ health: "healthy",+ lastSuccessfulRequestAt: health.lastSuccessfulRequestAt,+ ...(health.lastFailedRequest ? { lastFailedRequest: health.lastFailedRequest } : {}),+ ...(health.nextRetryAt ? { nextRetryAt: health.nextRetryAt } : {}),+ }+ : undefined;}async testHealth(provider: SubscriptionProviderId): Promise<void> {diff --git a/apps/server/src/subscription-auth/snapshot.test.ts b/apps/server/src/subscription-auth/snapshot.test.tsindex e41f5b5be..2f25c0d97 100644--- a/apps/server/src/subscription-auth/snapshot.test.ts+++ b/apps/server/src/subscription-auth/snapshot.test.ts@@ -1,4 +1,4 @@-import { BotId, ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts";+import { BotId, McpServerId, ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts";import { describe, expect, it } from "vite-plus/test";import { buildProviderAccessCapabilities } from "./snapshot.ts";@@ -145,4 +145,45 @@ describe("provider access capabilities", () => {).toBeTruthy();}});++ it("reports exact MCP health, built-in identity, and dependent bots", () => {+ const serverId = McpServerId.make("builtin-executor");+ const rows = buildProviderAccessCapabilities(+ [],+ [],+ undefined,+ [+ {+ id: serverId,+ name: "Executor",+ transport: "url",+ url: "https://executor.sh/mcp",+ enabled: true,+ createdAt: "2026-08-31T18:00:00.000Z",+ updatedAt: "2026-08-31T18:00:00.000Z",+ },+ ],+ [+ {+ id: BotId.make("bot-1"),+ name: "Research",+ engine: null,+ disabledMcpServerIds: [],+ },+ ],+ () => ({+ health: "healthy",+ lastSuccessfulRequestAt: "2026-08-31T20:00:00.000Z",+ }),+ );++ expect(rows.find((row) => row.serverId === serverId)).toMatchObject({+ id: "mcp-builtin-executor",+ pluginId: "executor",+ accessMethod: "mcp",+ health: "healthy",+ dependentBots: [{ id: "bot-1", name: "Research" }],+ });+ expect(rows.find((row) => row.serverId === serverId)?.repairAction).toBeUndefined();+ });});diff --git a/apps/server/src/subscription-auth/snapshot.ts b/apps/server/src/subscription-auth/snapshot.tsindex 228562a28..40604ee44 100644--- a/apps/server/src/subscription-auth/snapshot.ts+++ b/apps/server/src/subscription-auth/snapshot.ts@@ -1,6 +1,13 @@-import type { BotEngine, BotId, ServerProvider } from "@t3tools/contracts";+import type {+ BotEngine,+ BotId,+ McpServer,+ McpServerId,+ ProviderAccessStatus,+ ServerProvider,+} from "@t3tools/contracts";-import type { ProviderStatus, SubscriptionProviderId } from "./service.ts";+import type { ProviderStatus, RequestHealthStatus, SubscriptionProviderId } from "./service.ts";const SUBSCRIPTION_ACCESS = [{ id: "chatgpt", label: "ChatGPT", provider: "openai-codex" },@@ -40,6 +47,37 @@ export function subscriptionDependentBots(type ActualRequestHealth = "healthy" | "failed" | "failed-first-request" | "recovered" | undefined;+type BotAccess = {+ readonly id: BotId;+ readonly name: string;+ readonly engine: BotEngine | null;+ readonly disabledMcpServerIds?: ReadonlyArray<McpServerId>;+};++function requestHealthState(+ health: ActualRequestHealth | RequestHealthStatus,+): ActualRequestHealth {+ return typeof health === "string" || health === undefined ? health : health.health;+}++function requestHealthFields(health: ActualRequestHealth | RequestHealthStatus) {+ return typeof health === "object"+ ? {+ ...(health.lastSuccessfulRequestAt+ ? { lastSuccessfulRequestAt: health.lastSuccessfulRequestAt }+ : {}),+ ...(health.lastFailedRequest ? { lastFailedRequest: health.lastFailedRequest } : {}),+ ...(health.nextRetryAt ? { nextRetryAt: health.nextRetryAt } : {}),+ }+ : {};+}++function dependentBotsForProvider(bots: ReadonlyArray<BotAccess>, instanceId: string) {+ return bots+ .filter((bot) => bot.engine?.provider === instanceId)+ .map(({ id, name }) => ({ id, name }));+}+function providerAccessHealth(provider: ServerProvider | undefined,actualRequestHealth: ActualRequestHealth,@@ -54,8 +92,12 @@ function providerAccessHealth(export function buildProviderAccessCapabilities(subscriptions: ReadonlyArray<ProviderStatus>,providers: ReadonlyArray<ServerProvider>,- providerRequestHealth: (instanceId: string) => ActualRequestHealth = () => undefined,-) {+ providerRequestHealth: (instanceId: string) => ActualRequestHealth | RequestHealthStatus = () =>+ undefined,+ mcpServers: ReadonlyArray<McpServer> = [],+ bots: ReadonlyArray<BotAccess> = [],+ mcpRequestHealth: (serverId: string) => RequestHealthStatus | undefined = () => undefined,+): ReadonlyArray<ProviderAccessStatus> {const subscriptionById = new Map(subscriptions.map((status) => [status.provider, status]));const subscriptionRows = SUBSCRIPTION_ACCESS.map((entry) => {const status = subscriptionById.get(entry.provider);@@ -71,17 +113,20 @@ export function buildProviderAccessCapabilities(: status?.connected? "Check OAuth, then send a provider request to verify access.": `Connect ${entry.label} in Settings.`,+ ...(status?.lastSuccessfulRequestAt+ ? { lastSuccessfulRequestAt: status.lastSuccessfulRequestAt }+ : {}),+ ...(status?.lastFailedRequest ? { lastFailedRequest: status.lastFailedRequest } : {}),+ ...(status?.nextRetryAt ? { nextRetryAt: status.nextRetryAt } : {}),+ reconnectAction: status?.reconnectAction ?? `Connect ${entry.label}`,dependentBots: status?.dependentBots ?? [],- dependentRoutines: status?.dependentRoutines ?? [],};});const apiKeyProviders = providers.filter((provider) => provider.auth.type === "apiKey");const apiKeyRows = apiKeyProviders.map((provider) => {- const health = providerAccessHealth(- provider,- provider ? providerRequestHealth(provider.instanceId) : undefined,- );+ const requestHealth = providerRequestHealth(provider.instanceId);+ const health = providerAccessHealth(provider, requestHealthState(requestHealth));return {id: `api-key-${provider.instanceId}`,label: `${provider.displayName ?? provider.driver} API key`,@@ -92,6 +137,9 @@ export function buildProviderAccessCapabilities(health === "failed" || health === "failed-first-request"? "Check the API key and billing, then retry a provider request.": "Send a provider request to verify the key and billing.",+ ...requestHealthFields(requestHealth),+ reconnectAction: "Update key",+ dependentBots: dependentBotsForProvider(bots, provider.instanceId),};});@@ -100,10 +148,8 @@ export function buildProviderAccessCapabilities({ id: "grok-acp", label: "Grok ACP CLI", driver: "grok" },].map((entry) => {const provider = providers.find((candidate) => candidate.driver === entry.driver);- const health = providerAccessHealth(- provider,- provider ? providerRequestHealth(provider.instanceId) : undefined,- );+ const requestHealth = provider ? providerRequestHealth(provider.instanceId) : undefined;+ const health = providerAccessHealth(provider, requestHealthState(requestHealth));return {id: entry.id,label: entry.label,@@ -116,6 +162,45 @@ export function buildProviderAccessCapabilities(: provider? `Send a request through ${entry.label} to verify access.`: `Install and sign in to the ${entry.label}.`,+ ...requestHealthFields(requestHealth),+ reconnectAction: "Reconnect CLI",+ dependentBots: provider ? dependentBotsForProvider(bots, provider.instanceId) : [],+ };+ });++ const mcpRows = mcpServers.map((server) => {+ const requestHealth = mcpRequestHealth(server.id);+ const health: ProviderAccessStatus["health"] = server.enabled+ ? (requestHealth?.health ?? "detected")+ : "disabled";+ return {+ id: `mcp-${server.id}`,+ label: server.name,+ accessMethod: "mcp" as const,+ health,+ apiAccess: "not-applicable" as const,+ nextAction:+ health === "disabled"+ ? `Enable ${server.name} to use it again.`+ : health === "failed" || health === "failed-first-request"+ ? `Reconnect ${server.name}, then retry its failed request.`+ : health === "healthy" || health === "recovered"+ ? `Disable or remove ${server.name} when it is no longer needed.`+ : `Send a request through ${server.name} to verify the connection.`,+ ...(health === "disabled"+ ? { repairAction: "Enable" }+ : health === "failed" || health === "failed-first-request"+ ? { repairAction: "Reconnect" }+ : {}),+ reconnectAction: "Reconnect",+ serverId: String(server.id),+ ...(String(server.id).startsWith("builtin-")+ ? { pluginId: String(server.id).slice("builtin-".length) }+ : {}),+ ...requestHealthFields(requestHealth),+ dependentBots: bots+ .filter((bot) => !bot.disabledMcpServerIds?.includes(server.id))+ .map(({ id, name }) => ({ id, name })),};});@@ -158,6 +243,7 @@ export function buildProviderAccessCapabilities(},]),...acpRows,+ ...mcpRows,{id: "email-browser",label: "Email browser session",diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.tsindex 3e184525b..84e82473a 100644--- a/apps/server/src/ws.ts+++ b/apps/server/src/ws.ts@@ -83,7 +83,7 @@ import {subscriptionDependentBots,} from "./subscription-auth/snapshot.ts";import { BotInboxService } from "./bot-inbox/service.ts";-import { syncConnectorIncidents } from "./bot-inbox/connectorIncidents.ts";+import { syncAccessIncidents, syncConnectorIncidents } from "./bot-inbox/connectorIncidents.ts";import { HttpRouter, HttpServerRequest, HttpServerRespondable } from "effect/unstable/http";import { RpcSerialization, RpcServer } from "effect/unstable/rpc";@@ -703,25 +703,40 @@ const makeWsRpcLayer = (const getAccessHealthSnapshot = Effect.fn("getAccessHealthSnapshot")(function* () {subscriptionAuth.reload();botInbox.reload();- const [providers, bots] = yield* Effect.all([+ const [providers, bots, snapshot] = yield* Effect.all([providerRegistry.getProviders,projectionBots.listAll().pipe(Effect.mapError((cause) => new SubscriptionAuthError({ reason: cause.message }))),+ projectionSnapshotQuery+ .getShellSnapshot()+ .pipe(Effect.mapError((cause) => new SubscriptionAuthError({ reason: cause.message }))),]);const dependentBots = subscriptionDependentBots(bots.map((bot) => ({ id: bot.botId, name: bot.name, engine: bot.engine })),providers,);const subscriptionStatuses = subscriptionAuth.statuses(dependentBots);+ const access = buildProviderAccessCapabilities(+ subscriptionStatuses,+ providers,+ (instanceId) => subscriptionAuth.providerInstanceRequestHealth(instanceId),+ snapshot.mcpServers ?? [],+ bots.map((bot) => ({+ id: bot.botId,+ name: bot.name,+ engine: bot.engine,+ disabledMcpServerIds: bot.disabledMcpServerIds,+ })),+ (serverId) => subscriptionAuth.mcpRequestHealth(serverId),+ );syncConnectorIncidents(botInbox, subscriptionStatuses);+ syncAccessIncidents(botInbox, access);return {providers: subscriptionStatuses,- access: buildProviderAccessCapabilities(subscriptionStatuses, providers, (instanceId) =>- subscriptionAuth.providerInstanceHealth(instanceId),- ),+ access,inbox: botInbox.list(),};});diff --git a/docs/internals/plugin-lifecycle-verification.md b/docs/internals/plugin-lifecycle-verification.mdindex cf0a3e99b..9a8e0207e 100644--- a/docs/internals/plugin-lifecycle-verification.md+++ b/docs/internals/plugin-lifecycle-verification.md@@ -2,17 +2,17 @@Milestone 13 keeps catalog discovery separate from connection verification. An entry stays non-installable until its real install, authentication, safe read, approved write, disable, reconnect, and removal lifecycle passes. `verification-pending` means that local lifecycle proof is missing. `approval-pending` means an external vendor, administrator, allowlist, or first-party connector is still required.-`plugins/lifecycle-matrix.test.ts` is the catalog gate. It fixes the directory at 51 IDs, preserves the five existing `builtin-<id>` recipes, and keeps the other 46 entries pending. It also checks Featured order, consequential approval coverage, broker identity, Custom MCP independence, legacy built-in display, and the Typefully, Paper, and PayPal blockers.+`plugins/lifecycle-matrix.test.ts` is the catalog gate. It fixes the directory at 51 IDs, keeps four verified `builtin-<id>` recipes installable, and keeps the other 47 entries pending. It also checks Featured order, consequential approval coverage, broker identity, Custom MCP independence, legacy built-in display, and the Executor, Typefully, Paper, and PayPal blockers.## Runtime proofThe lifecycle matrix does not copy runtime tests. These focused tests own the runtime behavior:- `apps/server/src/orchestration/Layers/McpServerRegistry.test.ts` covers create, update, disable, enable, delete, missing IDs, and independent raw MCP state.-- `apps/server/src/provider/Layers/AgentController.test.ts` covers selected built-in installation, MCP startup and disconnect, protected actions, exact one-use approvals, secret redaction, and built-in plugin attribution.-- `apps/server/src/subscription-auth/service.test.ts` records MCP failure and recovery only from real tool requests.+- `apps/server/src/provider/Layers/AgentController.test.ts` covers selected MCP registration, exact tool-to-server attribution, startup failure, and request failure or recovery at the runtime event boundary.+- `apps/server/src/subscription-auth/service.test.ts` persists MCP failure and recovery without tool output or tokens.- `apps/server/src/subscription-auth/snapshot.test.ts` keeps a built-in MCP detected until a real request passes, reports healthy and disabled state, and lists dependent bots.-- `apps/server/src/bot-inbox/connectorIncidents.test.ts` covers first-request failure, reconnect or request recovery, and removal of a dependent bot or provider status.+- `apps/server/src/bot-inbox/connectorIncidents.test.ts` covers provider and MCP first-request failure, recovery, and stale dependent cleanup.Run the matrix with the catalog, schema, runtime, and validator checks. Do not call a vendor endpoint from automated tests.@@ -20,4 +20,4 @@ Run the matrix with the catalog, schema, runtime, and validator checks. Do not cThis local run did not perform a real Context.dev or Zernio connection lifecycle. Context.dev keeps its existing available identity; this run does not recertify it. Zernio stays Featured rank 2 and `verification-pending` until its OAuth and connection lifecycle passes.-Typefully stays pending until its OAuth lifecycle passes. Paper stays pending until its loopback desktop lifecycle passes. PayPal stays pending until its first-party OAuth and payment lifecycle passes. No entry can move to available based on a manifest check, HTTP probe, or vendor documentation alone.+Executor now declares its official hosted OAuth endpoint, but stays pending until that lifecycle passes. Typefully stays pending until its OAuth lifecycle passes. Paper stays pending until its loopback desktop lifecycle passes. PayPal stays pending until its first-party OAuth and payment lifecycle passes. No entry can move to available based on a manifest check, HTTP probe, or vendor documentation alone.