🙌 OpenHands: AI-Driven Development
fix: detect invalid credentials in MCP Test Connection (#1175)
Hiep Lee011407118a8
diff --git a/__tests__/api/mcp-service/mcp-service.api.test.ts b/__tests__/api/mcp-service/mcp-service.api.test.tsindex e8932cce56..eb9a60091f 100644--- a/__tests__/api/mcp-service/mcp-service.api.test.ts+++ b/__tests__/api/mcp-service/mcp-service.api.test.ts@@ -1,5 +1,8 @@import { describe, it, expect, vi, beforeEach } from "vitest";import McpService from "#/api/mcp-service/mcp-service.api";+import SettingsService, {+ type SettingsApiResponse,+} from "#/api/settings-service/settings-service.api";import * as activeStore from "#/api/backend-registry/active-store";import type { MCPServerConfig } from "#/types/mcp-server";@@ -111,6 +114,7 @@ describe("McpService.testServer", () => {await McpService.testServer(stdio);+ // Exact match also guards that non-catalog servers get no `tool_call`.expect(mockTestServer).toHaveBeenCalledWith({server: {type: "stdio",@@ -122,6 +126,160 @@ describe("McpService.testServer", () => {});});+ // -------------------------------------------------------------------------+ // Credential verification for marketplace servers (Slack)+ //+ // The Slack MCP server lists its tools with any credentials and reports+ // upstream auth failures as ordinary text content, so the service attaches+ // a read-only verification tool call and interprets its payload.+ // -------------------------------------------------------------------------++ const SLACK_SERVER: MCPServerConfig = {+ id: "stdio-0",+ type: "stdio",+ name: "slack",+ command: "npx",+ args: ["-y", "@zencoderai/slack-mcp-server"],+ env: { SLACK_TEAM_ID: "T01", SLACK_BOT_TOKEN: "xoxb-abc" },+ };++ const slackToolResult = (text: string, isError = false) => ({+ ok: true,+ tools: ["slack_list_channels"],+ tool_result: { is_error: isError, text },+ });++ it("attaches the read-only Slack verification tool call to the request", async () => {+ mockTestServer.mockResolvedValue({ ok: true, tools: [] });++ await McpService.testServer(SLACK_SERVER);++ expect(mockTestServer).toHaveBeenCalledWith(+ expect.objectContaining({+ tool_call: { name: "slack_list_channels", arguments: { limit: 1 } },+ }),+ );+ });++ it("maps an in-band Slack auth error to a credentials failure", async () => {+ mockTestServer.mockResolvedValue(+ slackToolResult('{"ok":false,"error":"invalid_auth"}'),+ );++ const result = await McpService.testServer(SLACK_SERVER);++ expect(result).toEqual({+ ok: false,+ error: "invalid_auth",+ error_kind: "credentials",+ });+ });++ it("does not flag non-auth Slack errors (missing_scope) as bad credentials", async () => {+ // A valid token lacking a scope authenticated successfully — failing it+ // would block correctly-configured installs.+ const response = slackToolResult('{"ok":false,"error":"missing_scope"}');+ mockTestServer.mockResolvedValue(response);++ const result = await McpService.testServer(SLACK_SERVER);++ expect(result).toEqual(response);+ });++ it("passes a succeeding Slack payload through unchanged", async () => {+ const response = slackToolResult('{"ok":true,"channels":[]}');+ mockTestServer.mockResolvedValue(response);++ const result = await McpService.testServer(SLACK_SERVER);++ expect(result).toEqual(response);+ });++ it("maps an errored verification call to a credentials failure", async () => {+ mockTestServer.mockResolvedValue(+ slackToolResult("Tool 'slack_list_channels' call timed out", true),+ );++ const result = await McpService.testServer(SLACK_SERVER);++ expect(result).toEqual({+ ok: false,+ error: "Tool 'slack_list_channels' call timed out",+ error_kind: "credentials",+ });+ });++ it("returns the response unchanged when an older backend omits tool_result", async () => {+ mockTestServer.mockResolvedValue({ ok: true, tools: ["a", "b"] });++ const result = await McpService.testServer(SLACK_SERVER);++ expect(result).toEqual({ ok: true, tools: ["a", "b"] });+ });++ // -------------------------------------------------------------------------+ // Redacted-secret round-trip for the edit flow+ //+ // The MCP page reads settings with redacted secrets, so unchanged env+ // values arrive as the literal "<redacted>" placeholder. The service swaps+ // them for the stored values in encrypted form (decrypted server-side) so+ // the test exercises the real credentials.+ // -------------------------------------------------------------------------++ const REDACTED_SLACK_SERVER: MCPServerConfig = {+ ...SLACK_SERVER,+ env: { SLACK_TEAM_ID: "T01", SLACK_BOT_TOKEN: "<redacted>" },+ };++ it("substitutes redacted env values with encrypted stored values", async () => {+ mockTestServer.mockResolvedValue({ ok: true, tools: [] });+ vi.spyOn(SettingsService, "fetchSettingsFromApi").mockResolvedValue({+ agent_settings: {+ mcp_config: {+ mcpServers: {+ slack: { env: { SLACK_BOT_TOKEN: "gAAAAA-encrypted-token" } },+ },+ },+ },+ } as unknown as SettingsApiResponse);++ await McpService.testServer(REDACTED_SLACK_SERVER);++ expect(SettingsService.fetchSettingsFromApi).toHaveBeenCalledWith(+ "encrypted",+ );+ expect(mockTestServer).toHaveBeenCalledWith(+ expect.objectContaining({+ server: expect.objectContaining({+ // Placeholder replaced by ciphertext; typed value left untouched.+ env: {+ SLACK_TEAM_ID: "T01",+ SLACK_BOT_TOKEN: "gAAAAA-encrypted-token",+ },+ }),+ }),+ );+ });++ it("keeps the placeholder when encrypted settings cannot be fetched", async () => {+ // e.g. HTTP 503 from a backend without a cipher — the test must still+ // run (and fail the credential check honestly) instead of crashing.+ mockTestServer.mockResolvedValue({ ok: true, tools: [] });+ vi.spyOn(SettingsService, "fetchSettingsFromApi").mockRejectedValue(+ new Error("503 no cipher"),+ );++ await McpService.testServer(REDACTED_SLACK_SERVER);++ expect(mockTestServer).toHaveBeenCalledWith(+ expect.objectContaining({+ server: expect.objectContaining({+ env: { SLACK_TEAM_ID: "T01", SLACK_BOT_TOKEN: "<redacted>" },+ }),+ }),+ );+ });+it("short-circuits with a synthetic ok response on cloud backends", async () => {// Regression: when the active backend is cloud, the local agent-server's// /api/mcp/test endpoint is not reachable. Previously, the helper threwdiff --git a/__tests__/components/features/mcp-page/custom-server-editor.test.tsx b/__tests__/components/features/mcp-page/custom-server-editor.test.tsxindex fa39221871..addf4ea379 100644--- a/__tests__/components/features/mcp-page/custom-server-editor.test.tsx+++ b/__tests__/components/features/mcp-page/custom-server-editor.test.tsx@@ -181,4 +181,32 @@ describe("CustomServerEditor", () => {expect(onClose).toHaveBeenCalledTimes(1);});++ it("surfaces a credential failure from Test connection in the edit modal", async () => {+ // Arrange: editing an installed server whose stored credentials the+ // verification call rejects — previously this path always reported+ // "Connected" because only tools/list was exercised.+ vi.spyOn(SettingsService, "getSettings").mockResolvedValue(+ buildSettingsWithMcp(),+ );+ vi.spyOn(McpService, "testServer").mockResolvedValue({+ ok: false,+ error: "invalid_auth",+ error_kind: "credentials",+ });++ renderWith(<EditEditorOnceSettingsLoaded onClose={vi.fn()} />);+ await screen.findByTestId("mcp-custom-editor");++ // Act: run the connection test from the edit form.+ fireEvent.click(screen.getByTestId("mcp-test-connection"));++ // Assert: the credentials-specific message is shown inline (i18n keys+ // are returned as-is in tests).+ await waitFor(() =>+ expect(screen.getByTestId("mcp-test-message")).toHaveTextContent(+ "MCP$TEST_ERROR_CREDENTIALS",+ ),+ );+ });});diff --git a/__tests__/components/features/mcp-page/install-server-modal.test.tsx b/__tests__/components/features/mcp-page/install-server-modal.test.tsxindex dab1ce66c0..8a16af8130 100644--- a/__tests__/components/features/mcp-page/install-server-modal.test.tsx+++ b/__tests__/components/features/mcp-page/install-server-modal.test.tsx@@ -338,6 +338,37 @@ describe("InstallServerModal", () => {expect(screen.getByTestId("mcp-install-modal")).toBeInTheDocument();});+ it("shows the credential-specific message when the pre-flight test reports invalid credentials", async () => {+ // Arrange: installing Slack with credentials its verification call+ // rejects (the service maps Slack's invalid_auth to "credentials").+ const slack = MCP_MARKETPLACE.find((e) => e.id === "slack")!;+ vi.spyOn(McpService, "testServer").mockResolvedValue({+ ok: false,+ error: "invalid_auth",+ error_kind: "credentials",+ });++ renderWith(<InstallServerModal entry={slack} onClose={vi.fn()} />);+ await screen.findByTestId("mcp-install-modal");++ // Act: fill the required fields and install.+ fireEvent.change(screen.getByTestId("mcp-install-field-SLACK_BOT_TOKEN"), {+ target: { value: "xoxb-invalid" },+ });+ fireEvent.change(screen.getByTestId("mcp-install-field-SLACK_TEAM_ID"), {+ target: { value: "T-INVALID" },+ });+ fireEvent.click(screen.getByTestId("mcp-install-submit"));++ // Assert: the credentials message is rendered (i18n keys are returned+ // as-is in tests), not the generic connection/unknown wording.+ await waitFor(() =>+ expect(screen.getByTestId("mcp-install-modal-error")).toHaveTextContent(+ "MCP$TEST_ERROR_CREDENTIALS",+ ),+ );+ });+it("calls save and closes the modal when the pre-flight test succeeds", async () => {vi.spyOn(McpService, "testServer").mockResolvedValue({ok: true,diff --git a/src/api/mcp-service/mcp-service.api.ts b/src/api/mcp-service/mcp-service.api.tsindex 163d40b0ed..3e19ed578d 100644--- a/src/api/mcp-service/mcp-service.api.ts+++ b/src/api/mcp-service/mcp-service.api.ts@@ -1,11 +1,20 @@import { MCPClient } from "@openhands/typescript-client/clients";import type {MCPServerSpec,- MCPTestResponse,+ MCPTestRequest,} from "@openhands/typescript-client";import { getAgentServerClientOptions } from "../agent-server-client-options";import { getActiveBackend } from "../backend-registry/active-store";-import type { MCPServerConfig } from "#/types/mcp-server";+import SettingsService from "#/api/settings-service/settings-service.api";+import { getCredentialValidationForServer } from "#/utils/mcp-credential-validation";+import type {+ ExtendedMCPTestResponse,+ MCPServerConfig,+} from "#/types/mcp-server";++// Placeholder the settings API substitutes for secret env values when+// settings are fetched without X-Expose-Secrets (the MCP page's mode).+const REDACTED_ENV_VALUE = "<redacted>";function toMcpServerSpec(server: MCPServerConfig): MCPServerSpec {if (server.type === "stdio") {@@ -24,8 +33,47 @@ function toMcpServerSpec(server: MCPServerConfig): MCPServerSpec {};}+/**+ * The MCP page reads settings with redacted secrets, so an env value the+ * user left unchanged in the edit form is the literal `<redacted>`+ * placeholder — testing with it would exercise garbage credentials. Swap+ * each placeholder for the stored value in encrypted form (the agent+ * server decrypts it before spawning), so "Test connection" exercises the+ * real stored credentials. Falls back to the placeholder when encrypted+ * settings are unavailable (e.g. no cipher configured) — the credential+ * check then fails honestly instead of crashing the test flow.+ */+async function substituteRedactedEnv(+ server: MCPServerConfig,+): Promise<MCPServerConfig> {+ if (server.type !== "stdio" || !server.name || !server.env) return server;+ const values = Object.values(server.env);+ if (!values.some((value) => value === REDACTED_ENV_VALUE)) return server;++ try {+ const response = await SettingsService.fetchSettingsFromApi("encrypted");+ const mcpConfig = response.agent_settings?.mcp_config as+ | { mcpServers?: Record<string, { env?: Record<string, string> }> }+ | undefined;+ const storedEnv = mcpConfig?.mcpServers?.[server.name]?.env ?? {};+ const env = Object.fromEntries(+ Object.entries(server.env).map(([key, value]) => [+ key,+ value === REDACTED_ENV_VALUE && typeof storedEnv[key] === "string"+ ? storedEnv[key]+ : value,+ ]),+ );+ return { ...server, env };+ } catch {+ return server;+ }+}+class McpService {- static async testServer(server: MCPServerConfig): Promise<MCPTestResponse> {+ static async testServer(+ server: MCPServerConfig,+ ): Promise<ExtendedMCPTestResponse> {// The MCP connectivity-test endpoint lives on the local agent-server. It// spawns the configured stdio command / opens an SSE-or-SHTTP connection// from that process's environment. Cloud backends don't expose this@@ -39,14 +87,36 @@ class McpService {if (getActiveBackend().backend.kind === "cloud") {return { ok: true, tools: [] };}+ const validation = getCredentialValidationForServer(server);+ const serverSpec = toMcpServerSpec(await substituteRedactedEnv(server));const { host, apiKey } = getAgentServerClientOptions();const client = new MCPClient({ host, ...(apiKey ? { apiKey } : {}) });try {- return await client.testServer({- server: toMcpServerSpec(server),+ // `tool_call` / `tool_result` aren't in the published client types+ // yet; the client POSTs the request object and returns the response+ // body as-is, so the extra fields round-trip at runtime. Old agent+ // servers ignore `tool_call` and return no `tool_result`, in which+ // case the response passes through uninterpreted (legacy behavior).+ const request = {+ server: serverSpec,...(server.name ? { name: server.name } : {}),...(server.timeout ? { timeout: server.timeout } : {}),- });+ ...(validation ? { tool_call: validation.toolCall } : {}),+ };+ const result = (await client.testServer(+ request as MCPTestRequest,+ )) as ExtendedMCPTestResponse;+ if (result.ok && validation && result.tool_result) {+ const credentialError = validation.interpret(result.tool_result);+ if (credentialError) {+ return {+ ok: false,+ error: credentialError,+ error_kind: "credentials",+ };+ }+ }+ return result;} finally {client.close();}diff --git a/src/components/features/mcp-page/custom-server-editor.tsx b/src/components/features/mcp-page/custom-server-editor.tsxindex e918661665..3b4c81fbc6 100644--- a/src/components/features/mcp-page/custom-server-editor.tsx+++ b/src/components/features/mcp-page/custom-server-editor.tsx@@ -1,7 +1,6 @@import React from "react";import { AxiosError } from "axios";import { useTranslation } from "react-i18next";-import type { MCPTestFailure } from "@openhands/typescript-client";import { I18nKey } from "#/i18n/declaration";import { ModalBackdrop } from "#/components/shared/modals/modal-backdrop";import { ModalCloseButton } from "#/components/shared/modals/modal-close-button";@@ -15,7 +14,7 @@ import { useUpdateMcpServer } from "#/hooks/mutation/use-update-mcp-server";import { useDeleteMcpServer } from "#/hooks/mutation/use-delete-mcp-server";import { useTestMcpServer } from "#/hooks/mutation/use-test-mcp-server";import { useActiveBackend } from "#/contexts/active-backend-context";-import { MCPServerConfig } from "#/types/mcp-server";+import { ExtendedMCPTestFailure, MCPServerConfig } from "#/types/mcp-server";import {displayErrorToast,displaySuccessToast,@@ -65,12 +64,14 @@ export function CustomServerEditor({const isPending = isAdding || isUpdating || isDeleting;const isDismissBlocked = isPending || isTesting || showDeleteConfirm;- const makeTestErrorMessage = (failure: MCPTestFailure): string => {+ const makeTestErrorMessage = (failure: ExtendedMCPTestFailure): string => {switch (failure.error_kind) {case "timeout":return t(I18nKey.MCP$TEST_ERROR_TIMEOUT);case "connection":return t(I18nKey.MCP$TEST_ERROR_CONNECTION);+ case "credentials":+ return t(I18nKey.MCP$TEST_ERROR_CREDENTIALS, { error: failure.error });default:return t(I18nKey.MCP$TEST_ERROR_UNKNOWN, { error: failure.error });}diff --git a/src/components/features/mcp-page/install-server-modal.tsx b/src/components/features/mcp-page/install-server-modal.tsxindex 5253bec6ce..4eb6183f9a 100644--- a/src/components/features/mcp-page/install-server-modal.tsx+++ b/src/components/features/mcp-page/install-server-modal.tsx@@ -2,7 +2,6 @@ import React from "react";import { useTranslation } from "react-i18next";import { AxiosError } from "axios";import { v4 as uuidv4 } from "uuid";-import type { MCPTestFailure } from "@openhands/typescript-client";import { ModalBackdrop } from "#/components/shared/modals/modal-backdrop";import { ModalCloseButton } from "#/components/shared/modals/modal-close-button";import { BrandButton } from "#/components/features/settings/brand-button";@@ -14,7 +13,7 @@ import type {MarketplaceField,} from "@openhands/extensions/integrations";import { McpLogoBadge } from "#/components/features/mcp-logo-badge";-import { MCPServerConfig } from "#/types/mcp-server";+import { ExtendedMCPTestFailure, MCPServerConfig } from "#/types/mcp-server";import { useAddMcpServer } from "#/hooks/mutation/use-add-mcp-server";import { useTestMcpServer } from "#/hooks/mutation/use-test-mcp-server";import { displaySuccessToast } from "#/utils/custom-toast-handlers";@@ -191,12 +190,14 @@ export function InstallServerModal({return Promise.resolve();};- const makeTestErrorMessage = (failure: MCPTestFailure): string => {+ const makeTestErrorMessage = (failure: ExtendedMCPTestFailure): string => {switch (failure.error_kind) {case "timeout":return t(I18nKey.MCP$TEST_ERROR_TIMEOUT);case "connection":return t(I18nKey.MCP$TEST_ERROR_CONNECTION);+ case "credentials":+ return t(I18nKey.MCP$TEST_ERROR_CREDENTIALS, { error: failure.error });default:return t(I18nKey.MCP$TEST_ERROR_UNKNOWN, { error: failure.error });}diff --git a/src/i18n/translation.json b/src/i18n/translation.jsonindex e8ebc2d9d7..f9c85e33a8 100644--- a/src/i18n/translation.json+++ b/src/i18n/translation.json@@ -14466,6 +14466,23 @@"tr": "Bağlantı başarısız: {{-error}}","uk": "Помилка з'єднання: {{-error}}"},+ "MCP$TEST_ERROR_CREDENTIALS": {+ "en": "Credential check failed: {{-error}}",+ "ja": "認証情報の確認に失敗しました: {{-error}}",+ "zh-CN": "凭据检查失败:{{-error}}",+ "zh-TW": "憑證檢查失敗:{{-error}}",+ "ko-KR": "자격 증명 확인에 실패했습니다: {{-error}}",+ "no": "Kontroll av påloggingsinformasjon mislyktes: {{-error}}",+ "ar": "فشل التحقق من بيانات الاعتماد: {{-error}}",+ "de": "Überprüfung der Anmeldedaten fehlgeschlagen: {{-error}}",+ "fr": "Échec de la vérification des identifiants : {{-error}}",+ "it": "Verifica delle credenziali non riuscita: {{-error}}",+ "pt": "Falha na verificação das credenciais: {{-error}}",+ "es": "Error en la verificación de credenciales: {{-error}}",+ "ca": "Ha fallat la verificació de credencials: {{-error}}",+ "tr": "Kimlik bilgisi denetimi başarısız: {{-error}}",+ "uk": "Помилка перевірки облікових даних: {{-error}}"+ },"MCP$EDIT_SERVER_ARIA": {"en": "Edit {{name}}","ja": "{{name}} を編集",diff --git a/src/types/mcp-server.ts b/src/types/mcp-server.tsindex c71d025c7a..8b58b71c3b 100644--- a/src/types/mcp-server.ts+++ b/src/types/mcp-server.ts@@ -3,6 +3,8 @@// Historically each component duplicated this interface. Centralizing// it here keeps the marketplace utilities, hooks, and form in sync.+import type { MCPTestFailureKind } from "@openhands/typescript-client";+export type MCPServerType = "sse" | "stdio" | "shttp";export interface MCPServerConfig {@@ -16,3 +18,38 @@ export interface MCPServerConfig {args?: string[];env?: Record<string, string>;}++// Extensions of the published `@openhands/typescript-client` MCP test+// types (frozen at the released version). The agent server's+// /api/mcp/test additionally accepts a `tool_call` (a read-only tool to+// invoke so credentials get exercised) and reports its outcome in+// `tool_result`; the service layer maps an interpreted credential+// failure to the GUI-local `"credentials"` error kind.++export interface MCPTestToolCall {+ name: string;+ arguments: Record<string, unknown>;+}++export interface MCPTestToolResult {+ is_error: boolean;+ text: string;+}++export type ExtendedMCPTestFailureKind = MCPTestFailureKind | "credentials";++export interface ExtendedMCPTestSuccess {+ ok: true;+ tools: string[];+ tool_result?: MCPTestToolResult | null;+}++export interface ExtendedMCPTestFailure {+ ok: false;+ error: string;+ error_kind: ExtendedMCPTestFailureKind;+}++export type ExtendedMCPTestResponse =+ | ExtendedMCPTestSuccess+ | ExtendedMCPTestFailure;diff --git a/src/utils/mcp-credential-validation.ts b/src/utils/mcp-credential-validation.tsnew file mode 100644index 0000000000..b7a261ae6a--- /dev/null+++ b/src/utils/mcp-credential-validation.ts@@ -0,0 +1,76 @@+import { INTEGRATION_CATALOG as MCP_MARKETPLACE } from "@openhands/extensions/integrations";+import {+ findCatalogEntryForServer,+ getMcpMarketplaceCatalog,+} from "#/utils/mcp-marketplace-utils";+import type {+ MCPServerConfig,+ MCPTestToolCall,+ MCPTestToolResult,+} from "#/types/mcp-server";++/**+ * Credential validation specs for marketplace MCP servers whose credentials+ * are only exercised on tool invocation (listing tools succeeds with any+ * credentials). The test endpoint runs `toolCall` after listing and reports+ * the outcome verbatim; `interpret` decides whether that outcome proves the+ * credentials are invalid.+ *+ * `toolCall` MUST be read-only — it runs on every "Test connection" /+ * pre-save validation.+ */+export interface CredentialValidation {+ toolCall: MCPTestToolCall;+ /** Returns the provider's error code/message, or null when creds pass. */+ interpret: (toolResult: MCPTestToolResult) => string | null;+}++/**+ * Slack error codes that mean the credentials themselves are bad. Anything+ * else (e.g. `missing_scope`) proves the token *authenticated* — a valid+ * token lacking a scope must not be reported as invalid credentials.+ */+const SLACK_AUTH_FAILURES = new Set([+ "invalid_auth",+ "not_authed",+ "account_inactive",+ "token_revoked",+ "token_expired",+]);++const VALIDATION_BY_ENTRY_ID: Record<string, CredentialValidation> = {+ slack: {+ // Read-only: lists at most one channel. The Slack MCP server returns+ // the raw Slack API JSON ({ok: boolean, error?: string}) as text.+ toolCall: { name: "slack_list_channels", arguments: { limit: 1 } },+ interpret: (toolResult) => {+ if (toolResult.is_error) {+ return toolResult.text || "tool call failed";+ }+ try {+ const parsed = JSON.parse(toolResult.text);+ if (parsed?.ok === false && SLACK_AUTH_FAILURES.has(parsed.error)) {+ return String(parsed.error);+ }+ } catch {+ // Non-JSON payload — make no claim about the credentials.+ }+ return null;+ },+ },+};++/**+ * Look up the credential validation for a server by matching it to its+ * marketplace catalog entry (same matching the MCP page uses for icons).+ * Returns undefined for custom servers — their test behaves as before.+ */+export function getCredentialValidationForServer(+ server: MCPServerConfig,+): CredentialValidation | undefined {+ const entry = findCatalogEntryForServer(+ server,+ getMcpMarketplaceCatalog(MCP_MARKETPLACE),+ );+ return entry ? VALIDATION_BY_ENTRY_ID[entry.id] : undefined;+}diff --git a/tests/e2e/mock-llm/mcp/mock-llm-mcp-slack-credentials.spec.ts b/tests/e2e/mock-llm/mcp/mock-llm-mcp-slack-credentials.spec.tsnew file mode 100644index 0000000000..8a9c2c70e2--- /dev/null+++ b/tests/e2e/mock-llm/mcp/mock-llm-mcp-slack-credentials.spec.ts@@ -0,0 +1,333 @@+/**+ * Mock-LLM E2E: MCP "Test connection" credential verification (PR #1175).+ *+ * Background: the Slack MCP server lists its tools with *any* credentials and+ * only exercises them inside tool handlers, so the old connection test+ * (`tools/list` only) reported "Connected — 8 tool(s) available" for an+ * invalid Team ID / Bot token. This suite proves the fix end-to-end through+ * the real browser → real agent-server stack, with only `POST /api/mcp/test`+ * intercepted so we can drive the agent server's `tool_result` payload (the+ * credential interpretation itself runs client-side in `McpService`).+ *+ * Verifies the PR's acceptance criteria:+ * - Installing Slack with invalid credentials shows+ * "Credential check failed: invalid_auth" and does NOT install.+ * - A read-only `slack_list_channels {limit: 1}` probe is attached to the+ * test request for catalog servers (Slack) — and to NOTHING for custom+ * (non-catalog) servers.+ * - A valid token merely missing a scope (`missing_scope`) is NOT treated+ * as a credential failure — the install still succeeds.+ * - Valid credentials connect successfully.+ * - Older agent servers that omit `tool_result` behave exactly as before.+ * - The Edit modal's "Test connection" verifies the *stored* credentials —+ * it never tests the literal `<redacted>` placeholder the browser sees.+ */++import {+ test,+ expect,+ type APIRequestContext,+ type Page,+} from "@playwright/test";+import {+ BACKEND_URL,+ SESSION_API_KEY,+ seedLocalStorage,+ routeSessionApiKey,+ dismissAnalyticsModal,+ ensureMockLLMProfileViaAPI,+} from "../utils/mock-llm-helpers";++// The read-only verification probe the service attaches for Slack.+const SLACK_TOOL_CALL = {+ name: "slack_list_channels",+ arguments: { limit: 1 },+};++// Placeholder the settings API substitutes for secret env values when read+// without X-Expose-Secrets (the MCP page's mode). The whole point of the fix+// is that the test request must NOT carry this value.+const REDACTED = "<redacted>";++/** The JSON body the GUI POSTs to `/api/mcp/test` (captured via interception). */+interface McpTestRequestBody {+ server?: { type?: string; url?: string; env?: Record<string, string> };+ name?: string;+ timeout?: number;+ tool_call?: { name: string; arguments: Record<string, unknown> };+}++const jsonRoute = (body: unknown) => ({+ status: 200,+ contentType: "application/json",+ body: JSON.stringify(body),+});++/**+ * Shape the patched agent server returns from `POST /api/mcp/test`: tools were+ * listed successfully and the attached `tool_call` ran, with its raw outcome+ * in `tool_result`. `McpService` interprets this client-side.+ */+const slackToolResult = (text: string, isError = false) => ({+ ok: true,+ tools: ["slack_list_channels", "slack_post_message"],+ tool_result: { is_error: isError, text },+});++async function getSettings(request: APIRequestContext) {+ const resp = await request.get(`${BACKEND_URL}/api/settings`, {+ headers: { "X-Session-API-Key": SESSION_API_KEY },+ });+ expect(resp.ok(), `GET /api/settings: ${resp.status()}`).toBe(true);+ return resp.json();+}++/** Detect the Slack server in persisted settings regardless of config shape. */+function hasSlackServer(settings: unknown): boolean {+ const cfg = (settings as { agent_settings?: { mcp_config?: unknown } })+ ?.agent_settings?.mcp_config;+ if (!cfg) return false;+ return JSON.stringify(cfg).includes("@zencoderai/slack-mcp-server");+}++async function patchMcpConfig(request: APIRequestContext, mcpConfig: unknown) {+ const resp = await request.patch(`${BACKEND_URL}/api/settings`, {+ headers: {+ "X-Session-API-Key": SESSION_API_KEY,+ "Content-Type": "application/json",+ },+ data: { agent_settings_diff: { mcp_config: mcpConfig } },+ });+ expect(resp.ok(), `seed mcp_config: ${resp.status()}`).toBe(true);+}++/** Seed a Slack stdio server (matches the marketplace catalog entry). */+async function installSlackViaAPI(+ request: APIRequestContext,+ env: Record<string, string>,+) {+ await patchMcpConfig(request, {+ mcpServers: {+ slack: {+ command: "npx",+ args: ["-y", "@zencoderai/slack-mcp-server"],+ env,+ },+ },+ });+}++async function openSlackInstallModal(page: Page) {+ await page.goto("/mcp", { waitUntil: "domcontentloaded" });+ await dismissAnalyticsModal(page);+ await expect(page.getByTestId("mcp-marketplace-grid")).toBeVisible({+ timeout: 10_000,+ });+ await page.getByTestId("mcp-marketplace-card-slack").click();+ const modal = page.getByTestId("mcp-install-modal");+ await expect(modal).toBeVisible({ timeout: 5_000 });+ await expect(modal).toHaveAttribute("data-marketplace-id", "slack");+ return modal;+}++async function fillSlackCredentials(+ page: Page,+ teamId: string,+ botToken: string,+) {+ await page.getByTestId("mcp-install-field-SLACK_TEAM_ID").fill(teamId);+ await page.getByTestId("mcp-install-field-SLACK_BOT_TOKEN").fill(botToken);+}++/** Open the editor for the first installed server (clicking its title avoids+ * the delete toggle in the card's top-right corner). */+async function openInstalledEditor(page: Page) {+ await page.goto("/mcp", { waitUntil: "domcontentloaded" });+ await dismissAnalyticsModal(page);+ const list = page.getByTestId("mcp-installed-list");+ await expect(list).toBeVisible({ timeout: 10_000 });+ const item = list.getByTestId("mcp-server-item").first();+ await expect(item).toBeVisible();+ await item.locator("h3").click();+ await expect(page.getByTestId("mcp-custom-editor")).toBeVisible({+ timeout: 5_000,+ });+}++test.describe.configure({ mode: "serial" });++test.describe("MCP Test Connection credential verification (Slack)", () => {+ test.beforeEach(async ({ page, request }) => {+ await seedLocalStorage(page);+ // A configured LLM profile keeps the MCP page free of "not configured"+ // banners that could intercept clicks. Done via API for speed.+ await ensureMockLLMProfileViaAPI(request);+ });++ test.afterEach(async ({ request }) => {+ // Reset MCP config so each test starts from a clean installed list.+ await patchMcpConfig(request, null).catch(() => {});+ });++ test("install: invalid Slack credentials are blocked with a credential-check error", async ({+ page,+ request,+ }) => {+ await routeSessionApiKey(page);++ const testRequests: McpTestRequestBody[] = [];+ await page.route("**/api/mcp/test", async (route) => {+ testRequests.push(route.request().postDataJSON());+ await route.fulfill(+ jsonRoute(slackToolResult('{"ok":false,"error":"invalid_auth"}')),+ );+ });++ const modal = await openSlackInstallModal(page);+ await fillSlackCredentials(page, "T_INVALID", "xoxb-invalid-token");+ await page.getByTestId("mcp-install-submit").click();++ // The credential failure is surfaced and the modal stays open.+ const error = page.getByTestId("mcp-install-modal-error");+ await expect(error).toBeVisible({ timeout: 10_000 });+ await expect(error).toContainText("Credential check failed: invalid_auth");+ await expect(modal).toBeVisible();++ // The read-only verification probe was attached to the test request.+ expect(testRequests[0]?.tool_call).toEqual(SLACK_TOOL_CALL);++ // Nothing was installed.+ expect(hasSlackServer(await getSettings(request))).toBe(false);+ });++ test("install: a valid token missing only a scope still installs (missing_scope is not a credential failure)", async ({+ page,+ request,+ }) => {+ await routeSessionApiKey(page);+ await page.route("**/api/mcp/test", async (route) => {+ await route.fulfill(+ jsonRoute(slackToolResult('{"ok":false,"error":"missing_scope"}')),+ );+ });++ const modal = await openSlackInstallModal(page);+ await fillSlackCredentials(+ page,+ "T0123456",+ "xoxb-valid-but-missing-scope",+ );+ await page.getByTestId("mcp-install-submit").click();++ // missing_scope proves the token authenticated → install proceeds.+ await expect(modal).not.toBeVisible({ timeout: 10_000 });+ await expect(page.getByTestId("mcp-installed-list")).toBeVisible({+ timeout: 10_000,+ });+ expect(hasSlackServer(await getSettings(request))).toBe(true);+ });++ test("install: an older agent server that omits tool_result still installs (compat)", async ({+ page,+ request,+ }) => {+ await routeSessionApiKey(page);+ // No `tool_result` field → the response passes through uninterpreted.+ await page.route("**/api/mcp/test", async (route) => {+ await route.fulfill(jsonRoute({ ok: true, tools: ["a", "b"] }));+ });++ const modal = await openSlackInstallModal(page);+ await fillSlackCredentials(page, "T0123456", "xoxb-some-token");+ await page.getByTestId("mcp-install-submit").click();++ await expect(modal).not.toBeVisible({ timeout: 10_000 });+ expect(hasSlackServer(await getSettings(request))).toBe(true);+ });++ test("edit: Test Connection verifies the stored credentials and surfaces a credential failure", async ({+ page,+ request,+ }) => {+ await installSlackViaAPI(request, {+ SLACK_TEAM_ID: "T0123456",+ SLACK_BOT_TOKEN: "xoxb-stored-secret",+ });+ await routeSessionApiKey(page);++ const testRequests: McpTestRequestBody[] = [];+ await page.route("**/api/mcp/test", async (route) => {+ testRequests.push(route.request().postDataJSON());+ await route.fulfill(+ jsonRoute(slackToolResult('{"ok":false,"error":"invalid_auth"}')),+ );+ });++ await openInstalledEditor(page);+ await page.getByTestId("mcp-test-connection").click();++ const message = page.getByTestId("mcp-test-message");+ await expect(message).toBeVisible({ timeout: 10_000 });+ await expect(message).toContainText(+ "Credential check failed: invalid_auth",+ );++ // The Slack verification probe is attached in the edit flow too...+ expect(testRequests[0]?.tool_call).toEqual(SLACK_TOOL_CALL);+ // ...and the request tests the *stored* token, never the placeholder the+ // browser sees in the redacted form (the second root cause the PR fixes).+ expect(testRequests[0]?.server?.env?.SLACK_BOT_TOKEN).not.toBe(REDACTED);+ });++ test("edit: Test Connection reports success for valid stored credentials", async ({+ page,+ request,+ }) => {+ await installSlackViaAPI(request, {+ SLACK_TEAM_ID: "T0123456",+ SLACK_BOT_TOKEN: "xoxb-stored-secret",+ });+ await routeSessionApiKey(page);+ await page.route("**/api/mcp/test", async (route) => {+ await route.fulfill(+ jsonRoute(slackToolResult('{"ok":true,"channels":[]}')),+ );+ });++ await openInstalledEditor(page);+ await page.getByTestId("mcp-test-connection").click();++ const message = page.getByTestId("mcp-test-message");+ await expect(message).toBeVisible({ timeout: 10_000 });+ // 2 tools listed → "Connected — 2 tool(s) available".+ await expect(message).toContainText("2 tool(s) available");+ await expect(message).not.toContainText("Credential check failed");+ });++ test("custom (non-catalog) server: Test Connection attaches no verification probe", async ({+ page,+ request,+ }) => {+ // A custom server is not in the marketplace catalog → no credential probe.+ await patchMcpConfig(request, {+ mcpServers: { "my-custom": { url: "https://custom.example.test/mcp" } },+ });+ await routeSessionApiKey(page);++ const testRequests: McpTestRequestBody[] = [];+ await page.route("**/api/mcp/test", async (route) => {+ testRequests.push(route.request().postDataJSON());+ await route.fulfill(jsonRoute({ ok: true, tools: ["search"] }));+ });++ await openInstalledEditor(page);+ await page.getByTestId("mcp-test-connection").click();++ const message = page.getByTestId("mcp-test-message");+ await expect(message).toBeVisible({ timeout: 10_000 });+ await expect(message).toContainText("1 tool(s) available");++ // Behavior unchanged for custom servers: no tool_call attached.+ expect(testRequests[0]?.tool_call).toBeUndefined();+ });+});