Skip to content

🙌 OpenHands: AI-Driven Development

fix: detect invalid credentials in MCP Test Connection (#1175)

Hiep Lee011407118a8
diff --git a/__tests__/api/mcp-service/mcp-service.api.test.ts b/__tests__/api/mcp-service/mcp-service.api.test.ts
index e8932cce56..eb9a60091f 100644
--- a/__tests__/api/mcp-service/mcp-service.api.test.ts
+++ b/__tests__/api/mcp-service/mcp-service.api.test.ts
@@ -1,5 +1,8 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import McpService from "#/api/mcp-service/mcp-service.api";
+import SettingsService, {
+ type SettingsApiResponse,
+} from "#/api/settings-service/settings-service.api";
import * as activeStore from "#/api/backend-registry/active-store";
import type { MCPServerConfig } from "#/types/mcp-server";
@@ -111,6 +114,7 @@ describe("McpService.testServer", () => {
await McpService.testServer(stdio);
+ // Exact match also guards that non-catalog servers get no `tool_call`.
expect(mockTestServer).toHaveBeenCalledWith({
server: {
type: "stdio",
@@ -122,6 +126,160 @@ describe("McpService.testServer", () => {
});
});
+ // -------------------------------------------------------------------------
+ // Credential verification for marketplace servers (Slack)
+ //
+ // The Slack MCP server lists its tools with any credentials and reports
+ // upstream auth failures as ordinary text content, so the service attaches
+ // a read-only verification tool call and interprets its payload.
+ // -------------------------------------------------------------------------
+
+ const SLACK_SERVER: MCPServerConfig = {
+ id: "stdio-0",
+ type: "stdio",
+ name: "slack",
+ command: "npx",
+ args: ["-y", "@zencoderai/slack-mcp-server"],
+ env: { SLACK_TEAM_ID: "T01", SLACK_BOT_TOKEN: "xoxb-abc" },
+ };
+
+ const slackToolResult = (text: string, isError = false) => ({
+ ok: true,
+ tools: ["slack_list_channels"],
+ tool_result: { is_error: isError, text },
+ });
+
+ it("attaches the read-only Slack verification tool call to the request", async () => {
+ mockTestServer.mockResolvedValue({ ok: true, tools: [] });
+
+ await McpService.testServer(SLACK_SERVER);
+
+ expect(mockTestServer).toHaveBeenCalledWith(
+ expect.objectContaining({
+ tool_call: { name: "slack_list_channels", arguments: { limit: 1 } },
+ }),
+ );
+ });
+
+ it("maps an in-band Slack auth error to a credentials failure", async () => {
+ mockTestServer.mockResolvedValue(
+ slackToolResult('{"ok":false,"error":"invalid_auth"}'),
+ );
+
+ const result = await McpService.testServer(SLACK_SERVER);
+
+ expect(result).toEqual({
+ ok: false,
+ error: "invalid_auth",
+ error_kind: "credentials",
+ });
+ });
+
+ it("does not flag non-auth Slack errors (missing_scope) as bad credentials", async () => {
+ // A valid token lacking a scope authenticated successfully — failing it
+ // would block correctly-configured installs.
+ const response = slackToolResult('{"ok":false,"error":"missing_scope"}');
+ mockTestServer.mockResolvedValue(response);
+
+ const result = await McpService.testServer(SLACK_SERVER);
+
+ expect(result).toEqual(response);
+ });
+
+ it("passes a succeeding Slack payload through unchanged", async () => {
+ const response = slackToolResult('{"ok":true,"channels":[]}');
+ mockTestServer.mockResolvedValue(response);
+
+ const result = await McpService.testServer(SLACK_SERVER);
+
+ expect(result).toEqual(response);
+ });
+
+ it("maps an errored verification call to a credentials failure", async () => {
+ mockTestServer.mockResolvedValue(
+ slackToolResult("Tool 'slack_list_channels' call timed out", true),
+ );
+
+ const result = await McpService.testServer(SLACK_SERVER);
+
+ expect(result).toEqual({
+ ok: false,
+ error: "Tool 'slack_list_channels' call timed out",
+ error_kind: "credentials",
+ });
+ });
+
+ it("returns the response unchanged when an older backend omits tool_result", async () => {
+ mockTestServer.mockResolvedValue({ ok: true, tools: ["a", "b"] });
+
+ const result = await McpService.testServer(SLACK_SERVER);
+
+ expect(result).toEqual({ ok: true, tools: ["a", "b"] });
+ });
+
+ // -------------------------------------------------------------------------
+ // Redacted-secret round-trip for the edit flow
+ //
+ // The MCP page reads settings with redacted secrets, so unchanged env
+ // values arrive as the literal "<redacted>" placeholder. The service swaps
+ // them for the stored values in encrypted form (decrypted server-side) so
+ // the test exercises the real credentials.
+ // -------------------------------------------------------------------------
+
+ const REDACTED_SLACK_SERVER: MCPServerConfig = {
+ ...SLACK_SERVER,
+ env: { SLACK_TEAM_ID: "T01", SLACK_BOT_TOKEN: "<redacted>" },
+ };
+
+ it("substitutes redacted env values with encrypted stored values", async () => {
+ mockTestServer.mockResolvedValue({ ok: true, tools: [] });
+ vi.spyOn(SettingsService, "fetchSettingsFromApi").mockResolvedValue({
+ agent_settings: {
+ mcp_config: {
+ mcpServers: {
+ slack: { env: { SLACK_BOT_TOKEN: "gAAAAA-encrypted-token" } },
+ },
+ },
+ },
+ } as unknown as SettingsApiResponse);
+
+ await McpService.testServer(REDACTED_SLACK_SERVER);
+
+ expect(SettingsService.fetchSettingsFromApi).toHaveBeenCalledWith(
+ "encrypted",
+ );
+ expect(mockTestServer).toHaveBeenCalledWith(
+ expect.objectContaining({
+ server: expect.objectContaining({
+ // Placeholder replaced by ciphertext; typed value left untouched.
+ env: {
+ SLACK_TEAM_ID: "T01",
+ SLACK_BOT_TOKEN: "gAAAAA-encrypted-token",
+ },
+ }),
+ }),
+ );
+ });
+
+ it("keeps the placeholder when encrypted settings cannot be fetched", async () => {
+ // e.g. HTTP 503 from a backend without a cipher — the test must still
+ // run (and fail the credential check honestly) instead of crashing.
+ mockTestServer.mockResolvedValue({ ok: true, tools: [] });
+ vi.spyOn(SettingsService, "fetchSettingsFromApi").mockRejectedValue(
+ new Error("503 no cipher"),
+ );
+
+ await McpService.testServer(REDACTED_SLACK_SERVER);
+
+ expect(mockTestServer).toHaveBeenCalledWith(
+ expect.objectContaining({
+ server: expect.objectContaining({
+ env: { SLACK_TEAM_ID: "T01", SLACK_BOT_TOKEN: "<redacted>" },
+ }),
+ }),
+ );
+ });
+
it("short-circuits with a synthetic ok response on cloud backends", async () => {
// Regression: when the active backend is cloud, the local agent-server's
// /api/mcp/test endpoint is not reachable. Previously, the helper threw
diff --git a/__tests__/components/features/mcp-page/custom-server-editor.test.tsx b/__tests__/components/features/mcp-page/custom-server-editor.test.tsx
index fa39221871..addf4ea379 100644
--- a/__tests__/components/features/mcp-page/custom-server-editor.test.tsx
+++ b/__tests__/components/features/mcp-page/custom-server-editor.test.tsx
@@ -181,4 +181,32 @@ describe("CustomServerEditor", () => {
expect(onClose).toHaveBeenCalledTimes(1);
});
+
+ it("surfaces a credential failure from Test connection in the edit modal", async () => {
+ // Arrange: editing an installed server whose stored credentials the
+ // verification call rejects — previously this path always reported
+ // "Connected" because only tools/list was exercised.
+ vi.spyOn(SettingsService, "getSettings").mockResolvedValue(
+ buildSettingsWithMcp(),
+ );
+ vi.spyOn(McpService, "testServer").mockResolvedValue({
+ ok: false,
+ error: "invalid_auth",
+ error_kind: "credentials",
+ });
+
+ renderWith(<EditEditorOnceSettingsLoaded onClose={vi.fn()} />);
+ await screen.findByTestId("mcp-custom-editor");
+
+ // Act: run the connection test from the edit form.
+ fireEvent.click(screen.getByTestId("mcp-test-connection"));
+
+ // Assert: the credentials-specific message is shown inline (i18n keys
+ // are returned as-is in tests).
+ await waitFor(() =>
+ expect(screen.getByTestId("mcp-test-message")).toHaveTextContent(
+ "MCP$TEST_ERROR_CREDENTIALS",
+ ),
+ );
+ });
});
diff --git a/__tests__/components/features/mcp-page/install-server-modal.test.tsx b/__tests__/components/features/mcp-page/install-server-modal.test.tsx
index dab1ce66c0..8a16af8130 100644
--- a/__tests__/components/features/mcp-page/install-server-modal.test.tsx
+++ b/__tests__/components/features/mcp-page/install-server-modal.test.tsx
@@ -338,6 +338,37 @@ describe("InstallServerModal", () => {
expect(screen.getByTestId("mcp-install-modal")).toBeInTheDocument();
});
+ it("shows the credential-specific message when the pre-flight test reports invalid credentials", async () => {
+ // Arrange: installing Slack with credentials its verification call
+ // rejects (the service maps Slack's invalid_auth to "credentials").
+ const slack = MCP_MARKETPLACE.find((e) => e.id === "slack")!;
+ vi.spyOn(McpService, "testServer").mockResolvedValue({
+ ok: false,
+ error: "invalid_auth",
+ error_kind: "credentials",
+ });
+
+ renderWith(<InstallServerModal entry={slack} onClose={vi.fn()} />);
+ await screen.findByTestId("mcp-install-modal");
+
+ // Act: fill the required fields and install.
+ fireEvent.change(screen.getByTestId("mcp-install-field-SLACK_BOT_TOKEN"), {
+ target: { value: "xoxb-invalid" },
+ });
+ fireEvent.change(screen.getByTestId("mcp-install-field-SLACK_TEAM_ID"), {
+ target: { value: "T-INVALID" },
+ });
+ fireEvent.click(screen.getByTestId("mcp-install-submit"));
+
+ // Assert: the credentials message is rendered (i18n keys are returned
+ // as-is in tests), not the generic connection/unknown wording.
+ await waitFor(() =>
+ expect(screen.getByTestId("mcp-install-modal-error")).toHaveTextContent(
+ "MCP$TEST_ERROR_CREDENTIALS",
+ ),
+ );
+ });
+
it("calls save and closes the modal when the pre-flight test succeeds", async () => {
vi.spyOn(McpService, "testServer").mockResolvedValue({
ok: true,
diff --git a/src/api/mcp-service/mcp-service.api.ts b/src/api/mcp-service/mcp-service.api.ts
index 163d40b0ed..3e19ed578d 100644
--- a/src/api/mcp-service/mcp-service.api.ts
+++ b/src/api/mcp-service/mcp-service.api.ts
@@ -1,11 +1,20 @@
import { MCPClient } from "@openhands/typescript-client/clients";
import type {
MCPServerSpec,
- MCPTestResponse,
+ MCPTestRequest,
} from "@openhands/typescript-client";
import { getAgentServerClientOptions } from "../agent-server-client-options";
import { getActiveBackend } from "../backend-registry/active-store";
-import type { MCPServerConfig } from "#/types/mcp-server";
+import SettingsService from "#/api/settings-service/settings-service.api";
+import { getCredentialValidationForServer } from "#/utils/mcp-credential-validation";
+import type {
+ ExtendedMCPTestResponse,
+ MCPServerConfig,
+} from "#/types/mcp-server";
+
+// Placeholder the settings API substitutes for secret env values when
+// settings are fetched without X-Expose-Secrets (the MCP page's mode).
+const REDACTED_ENV_VALUE = "<redacted>";
function toMcpServerSpec(server: MCPServerConfig): MCPServerSpec {
if (server.type === "stdio") {
@@ -24,8 +33,47 @@ function toMcpServerSpec(server: MCPServerConfig): MCPServerSpec {
};
}
+/**
+ * The MCP page reads settings with redacted secrets, so an env value the
+ * user left unchanged in the edit form is the literal `<redacted>`
+ * placeholder — testing with it would exercise garbage credentials. Swap
+ * each placeholder for the stored value in encrypted form (the agent
+ * server decrypts it before spawning), so "Test connection" exercises the
+ * real stored credentials. Falls back to the placeholder when encrypted
+ * settings are unavailable (e.g. no cipher configured) — the credential
+ * check then fails honestly instead of crashing the test flow.
+ */
+async function substituteRedactedEnv(
+ server: MCPServerConfig,
+): Promise<MCPServerConfig> {
+ if (server.type !== "stdio" || !server.name || !server.env) return server;
+ const values = Object.values(server.env);
+ if (!values.some((value) => value === REDACTED_ENV_VALUE)) return server;
+
+ try {
+ const response = await SettingsService.fetchSettingsFromApi("encrypted");
+ const mcpConfig = response.agent_settings?.mcp_config as
+ | { mcpServers?: Record<string, { env?: Record<string, string> }> }
+ | undefined;
+ const storedEnv = mcpConfig?.mcpServers?.[server.name]?.env ?? {};
+ const env = Object.fromEntries(
+ Object.entries(server.env).map(([key, value]) => [
+ key,
+ value === REDACTED_ENV_VALUE && typeof storedEnv[key] === "string"
+ ? storedEnv[key]
+ : value,
+ ]),
+ );
+ return { ...server, env };
+ } catch {
+ return server;
+ }
+}
+
class McpService {
- static async testServer(server: MCPServerConfig): Promise<MCPTestResponse> {
+ static async testServer(
+ server: MCPServerConfig,
+ ): Promise<ExtendedMCPTestResponse> {
// The MCP connectivity-test endpoint lives on the local agent-server. It
// spawns the configured stdio command / opens an SSE-or-SHTTP connection
// from that process's environment. Cloud backends don't expose this
@@ -39,14 +87,36 @@ class McpService {
if (getActiveBackend().backend.kind === "cloud") {
return { ok: true, tools: [] };
}
+ const validation = getCredentialValidationForServer(server);
+ const serverSpec = toMcpServerSpec(await substituteRedactedEnv(server));
const { host, apiKey } = getAgentServerClientOptions();
const client = new MCPClient({ host, ...(apiKey ? { apiKey } : {}) });
try {
- return await client.testServer({
- server: toMcpServerSpec(server),
+ // `tool_call` / `tool_result` aren't in the published client types
+ // yet; the client POSTs the request object and returns the response
+ // body as-is, so the extra fields round-trip at runtime. Old agent
+ // servers ignore `tool_call` and return no `tool_result`, in which
+ // case the response passes through uninterpreted (legacy behavior).
+ const request = {
+ server: serverSpec,
...(server.name ? { name: server.name } : {}),
...(server.timeout ? { timeout: server.timeout } : {}),
- });
+ ...(validation ? { tool_call: validation.toolCall } : {}),
+ };
+ const result = (await client.testServer(
+ request as MCPTestRequest,
+ )) as ExtendedMCPTestResponse;
+ if (result.ok && validation && result.tool_result) {
+ const credentialError = validation.interpret(result.tool_result);
+ if (credentialError) {
+ return {
+ ok: false,
+ error: credentialError,
+ error_kind: "credentials",
+ };
+ }
+ }
+ return result;
} finally {
client.close();
}
diff --git a/src/components/features/mcp-page/custom-server-editor.tsx b/src/components/features/mcp-page/custom-server-editor.tsx
index e918661665..3b4c81fbc6 100644
--- a/src/components/features/mcp-page/custom-server-editor.tsx
+++ b/src/components/features/mcp-page/custom-server-editor.tsx
@@ -1,7 +1,6 @@
import React from "react";
import { AxiosError } from "axios";
import { useTranslation } from "react-i18next";
-import type { MCPTestFailure } from "@openhands/typescript-client";
import { I18nKey } from "#/i18n/declaration";
import { ModalBackdrop } from "#/components/shared/modals/modal-backdrop";
import { ModalCloseButton } from "#/components/shared/modals/modal-close-button";
@@ -15,7 +14,7 @@ import { useUpdateMcpServer } from "#/hooks/mutation/use-update-mcp-server";
import { useDeleteMcpServer } from "#/hooks/mutation/use-delete-mcp-server";
import { useTestMcpServer } from "#/hooks/mutation/use-test-mcp-server";
import { useActiveBackend } from "#/contexts/active-backend-context";
-import { MCPServerConfig } from "#/types/mcp-server";
+import { ExtendedMCPTestFailure, MCPServerConfig } from "#/types/mcp-server";
import {
displayErrorToast,
displaySuccessToast,
@@ -65,12 +64,14 @@ export function CustomServerEditor({
const isPending = isAdding || isUpdating || isDeleting;
const isDismissBlocked = isPending || isTesting || showDeleteConfirm;
- const makeTestErrorMessage = (failure: MCPTestFailure): string => {
+ const makeTestErrorMessage = (failure: ExtendedMCPTestFailure): string => {
switch (failure.error_kind) {
case "timeout":
return t(I18nKey.MCP$TEST_ERROR_TIMEOUT);
case "connection":
return t(I18nKey.MCP$TEST_ERROR_CONNECTION);
+ case "credentials":
+ return t(I18nKey.MCP$TEST_ERROR_CREDENTIALS, { error: failure.error });
default:
return t(I18nKey.MCP$TEST_ERROR_UNKNOWN, { error: failure.error });
}
diff --git a/src/components/features/mcp-page/install-server-modal.tsx b/src/components/features/mcp-page/install-server-modal.tsx
index 5253bec6ce..4eb6183f9a 100644
--- a/src/components/features/mcp-page/install-server-modal.tsx
+++ b/src/components/features/mcp-page/install-server-modal.tsx
@@ -2,7 +2,6 @@ import React from "react";
import { useTranslation } from "react-i18next";
import { AxiosError } from "axios";
import { v4 as uuidv4 } from "uuid";
-import type { MCPTestFailure } from "@openhands/typescript-client";
import { ModalBackdrop } from "#/components/shared/modals/modal-backdrop";
import { ModalCloseButton } from "#/components/shared/modals/modal-close-button";
import { BrandButton } from "#/components/features/settings/brand-button";
@@ -14,7 +13,7 @@ import type {
MarketplaceField,
} from "@openhands/extensions/integrations";
import { McpLogoBadge } from "#/components/features/mcp-logo-badge";
-import { MCPServerConfig } from "#/types/mcp-server";
+import { ExtendedMCPTestFailure, MCPServerConfig } from "#/types/mcp-server";
import { useAddMcpServer } from "#/hooks/mutation/use-add-mcp-server";
import { useTestMcpServer } from "#/hooks/mutation/use-test-mcp-server";
import { displaySuccessToast } from "#/utils/custom-toast-handlers";
@@ -191,12 +190,14 @@ export function InstallServerModal({
return Promise.resolve();
};
- const makeTestErrorMessage = (failure: MCPTestFailure): string => {
+ const makeTestErrorMessage = (failure: ExtendedMCPTestFailure): string => {
switch (failure.error_kind) {
case "timeout":
return t(I18nKey.MCP$TEST_ERROR_TIMEOUT);
case "connection":
return t(I18nKey.MCP$TEST_ERROR_CONNECTION);
+ case "credentials":
+ return t(I18nKey.MCP$TEST_ERROR_CREDENTIALS, { error: failure.error });
default:
return t(I18nKey.MCP$TEST_ERROR_UNKNOWN, { error: failure.error });
}
diff --git a/src/i18n/translation.json b/src/i18n/translation.json
index e8ebc2d9d7..f9c85e33a8 100644
--- a/src/i18n/translation.json
+++ b/src/i18n/translation.json
@@ -14466,6 +14466,23 @@
"tr": "Bağlantı başarısız: {{-error}}",
"uk": "Помилка з'єднання: {{-error}}"
},
+ "MCP$TEST_ERROR_CREDENTIALS": {
+ "en": "Credential check failed: {{-error}}",
+ "ja": "認証情報の確認に失敗しました: {{-error}}",
+ "zh-CN": "凭据检查失败:{{-error}}",
+ "zh-TW": "憑證檢查失敗:{{-error}}",
+ "ko-KR": "자격 증명 확인에 실패했습니다: {{-error}}",
+ "no": "Kontroll av påloggingsinformasjon mislyktes: {{-error}}",
+ "ar": "فشل التحقق من بيانات الاعتماد: {{-error}}",
+ "de": "Überprüfung der Anmeldedaten fehlgeschlagen: {{-error}}",
+ "fr": "Échec de la vérification des identifiants : {{-error}}",
+ "it": "Verifica delle credenziali non riuscita: {{-error}}",
+ "pt": "Falha na verificação das credenciais: {{-error}}",
+ "es": "Error en la verificación de credenciales: {{-error}}",
+ "ca": "Ha fallat la verificació de credencials: {{-error}}",
+ "tr": "Kimlik bilgisi denetimi başarısız: {{-error}}",
+ "uk": "Помилка перевірки облікових даних: {{-error}}"
+ },
"MCP$EDIT_SERVER_ARIA": {
"en": "Edit {{name}}",
"ja": "{{name}} を編集",
diff --git a/src/types/mcp-server.ts b/src/types/mcp-server.ts
index c71d025c7a..8b58b71c3b 100644
--- a/src/types/mcp-server.ts
+++ b/src/types/mcp-server.ts
@@ -3,6 +3,8 @@
// Historically each component duplicated this interface. Centralizing
// it here keeps the marketplace utilities, hooks, and form in sync.
+import type { MCPTestFailureKind } from "@openhands/typescript-client";
+
export type MCPServerType = "sse" | "stdio" | "shttp";
export interface MCPServerConfig {
@@ -16,3 +18,38 @@ export interface MCPServerConfig {
args?: string[];
env?: Record<string, string>;
}
+
+// Extensions of the published `@openhands/typescript-client` MCP test
+// types (frozen at the released version). The agent server's
+// /api/mcp/test additionally accepts a `tool_call` (a read-only tool to
+// invoke so credentials get exercised) and reports its outcome in
+// `tool_result`; the service layer maps an interpreted credential
+// failure to the GUI-local `"credentials"` error kind.
+
+export interface MCPTestToolCall {
+ name: string;
+ arguments: Record<string, unknown>;
+}
+
+export interface MCPTestToolResult {
+ is_error: boolean;
+ text: string;
+}
+
+export type ExtendedMCPTestFailureKind = MCPTestFailureKind | "credentials";
+
+export interface ExtendedMCPTestSuccess {
+ ok: true;
+ tools: string[];
+ tool_result?: MCPTestToolResult | null;
+}
+
+export interface ExtendedMCPTestFailure {
+ ok: false;
+ error: string;
+ error_kind: ExtendedMCPTestFailureKind;
+}
+
+export type ExtendedMCPTestResponse =
+ | ExtendedMCPTestSuccess
+ | ExtendedMCPTestFailure;
diff --git a/src/utils/mcp-credential-validation.ts b/src/utils/mcp-credential-validation.ts
new file mode 100644
index 0000000000..b7a261ae6a
--- /dev/null
+++ b/src/utils/mcp-credential-validation.ts
@@ -0,0 +1,76 @@
+import { INTEGRATION_CATALOG as MCP_MARKETPLACE } from "@openhands/extensions/integrations";
+import {
+ findCatalogEntryForServer,
+ getMcpMarketplaceCatalog,
+} from "#/utils/mcp-marketplace-utils";
+import type {
+ MCPServerConfig,
+ MCPTestToolCall,
+ MCPTestToolResult,
+} from "#/types/mcp-server";
+
+/**
+ * Credential validation specs for marketplace MCP servers whose credentials
+ * are only exercised on tool invocation (listing tools succeeds with any
+ * credentials). The test endpoint runs `toolCall` after listing and reports
+ * the outcome verbatim; `interpret` decides whether that outcome proves the
+ * credentials are invalid.
+ *
+ * `toolCall` MUST be read-only — it runs on every "Test connection" /
+ * pre-save validation.
+ */
+export interface CredentialValidation {
+ toolCall: MCPTestToolCall;
+ /** Returns the provider's error code/message, or null when creds pass. */
+ interpret: (toolResult: MCPTestToolResult) => string | null;
+}
+
+/**
+ * Slack error codes that mean the credentials themselves are bad. Anything
+ * else (e.g. `missing_scope`) proves the token *authenticated* — a valid
+ * token lacking a scope must not be reported as invalid credentials.
+ */
+const SLACK_AUTH_FAILURES = new Set([
+ "invalid_auth",
+ "not_authed",
+ "account_inactive",
+ "token_revoked",
+ "token_expired",
+]);
+
+const VALIDATION_BY_ENTRY_ID: Record<string, CredentialValidation> = {
+ slack: {
+ // Read-only: lists at most one channel. The Slack MCP server returns
+ // the raw Slack API JSON ({ok: boolean, error?: string}) as text.
+ toolCall: { name: "slack_list_channels", arguments: { limit: 1 } },
+ interpret: (toolResult) => {
+ if (toolResult.is_error) {
+ return toolResult.text || "tool call failed";
+ }
+ try {
+ const parsed = JSON.parse(toolResult.text);
+ if (parsed?.ok === false && SLACK_AUTH_FAILURES.has(parsed.error)) {
+ return String(parsed.error);
+ }
+ } catch {
+ // Non-JSON payload — make no claim about the credentials.
+ }
+ return null;
+ },
+ },
+};
+
+/**
+ * Look up the credential validation for a server by matching it to its
+ * marketplace catalog entry (same matching the MCP page uses for icons).
+ * Returns undefined for custom servers — their test behaves as before.
+ */
+export function getCredentialValidationForServer(
+ server: MCPServerConfig,
+): CredentialValidation | undefined {
+ const entry = findCatalogEntryForServer(
+ server,
+ getMcpMarketplaceCatalog(MCP_MARKETPLACE),
+ );
+ return entry ? VALIDATION_BY_ENTRY_ID[entry.id] : undefined;
+}
diff --git a/tests/e2e/mock-llm/mcp/mock-llm-mcp-slack-credentials.spec.ts b/tests/e2e/mock-llm/mcp/mock-llm-mcp-slack-credentials.spec.ts
new file mode 100644
index 0000000000..8a9c2c70e2
--- /dev/null
+++ b/tests/e2e/mock-llm/mcp/mock-llm-mcp-slack-credentials.spec.ts
@@ -0,0 +1,333 @@
+/**
+ * Mock-LLM E2E: MCP "Test connection" credential verification (PR #1175).
+ *
+ * Background: the Slack MCP server lists its tools with *any* credentials and
+ * only exercises them inside tool handlers, so the old connection test
+ * (`tools/list` only) reported "Connected — 8 tool(s) available" for an
+ * invalid Team ID / Bot token. This suite proves the fix end-to-end through
+ * the real browser → real agent-server stack, with only `POST /api/mcp/test`
+ * intercepted so we can drive the agent server's `tool_result` payload (the
+ * credential interpretation itself runs client-side in `McpService`).
+ *
+ * Verifies the PR's acceptance criteria:
+ * - Installing Slack with invalid credentials shows
+ * "Credential check failed: invalid_auth" and does NOT install.
+ * - A read-only `slack_list_channels {limit: 1}` probe is attached to the
+ * test request for catalog servers (Slack) — and to NOTHING for custom
+ * (non-catalog) servers.
+ * - A valid token merely missing a scope (`missing_scope`) is NOT treated
+ * as a credential failure — the install still succeeds.
+ * - Valid credentials connect successfully.
+ * - Older agent servers that omit `tool_result` behave exactly as before.
+ * - The Edit modal's "Test connection" verifies the *stored* credentials —
+ * it never tests the literal `<redacted>` placeholder the browser sees.
+ */
+
+import {
+ test,
+ expect,
+ type APIRequestContext,
+ type Page,
+} from "@playwright/test";
+import {
+ BACKEND_URL,
+ SESSION_API_KEY,
+ seedLocalStorage,
+ routeSessionApiKey,
+ dismissAnalyticsModal,
+ ensureMockLLMProfileViaAPI,
+} from "../utils/mock-llm-helpers";
+
+// The read-only verification probe the service attaches for Slack.
+const SLACK_TOOL_CALL = {
+ name: "slack_list_channels",
+ arguments: { limit: 1 },
+};
+
+// Placeholder the settings API substitutes for secret env values when read
+// without X-Expose-Secrets (the MCP page's mode). The whole point of the fix
+// is that the test request must NOT carry this value.
+const REDACTED = "<redacted>";
+
+/** The JSON body the GUI POSTs to `/api/mcp/test` (captured via interception). */
+interface McpTestRequestBody {
+ server?: { type?: string; url?: string; env?: Record<string, string> };
+ name?: string;
+ timeout?: number;
+ tool_call?: { name: string; arguments: Record<string, unknown> };
+}
+
+const jsonRoute = (body: unknown) => ({
+ status: 200,
+ contentType: "application/json",
+ body: JSON.stringify(body),
+});
+
+/**
+ * Shape the patched agent server returns from `POST /api/mcp/test`: tools were
+ * listed successfully and the attached `tool_call` ran, with its raw outcome
+ * in `tool_result`. `McpService` interprets this client-side.
+ */
+const slackToolResult = (text: string, isError = false) => ({
+ ok: true,
+ tools: ["slack_list_channels", "slack_post_message"],
+ tool_result: { is_error: isError, text },
+});
+
+async function getSettings(request: APIRequestContext) {
+ const resp = await request.get(`${BACKEND_URL}/api/settings`, {
+ headers: { "X-Session-API-Key": SESSION_API_KEY },
+ });
+ expect(resp.ok(), `GET /api/settings: ${resp.status()}`).toBe(true);
+ return resp.json();
+}
+
+/** Detect the Slack server in persisted settings regardless of config shape. */
+function hasSlackServer(settings: unknown): boolean {
+ const cfg = (settings as { agent_settings?: { mcp_config?: unknown } })
+ ?.agent_settings?.mcp_config;
+ if (!cfg) return false;
+ return JSON.stringify(cfg).includes("@zencoderai/slack-mcp-server");
+}
+
+async function patchMcpConfig(request: APIRequestContext, mcpConfig: unknown) {
+ const resp = await request.patch(`${BACKEND_URL}/api/settings`, {
+ headers: {
+ "X-Session-API-Key": SESSION_API_KEY,
+ "Content-Type": "application/json",
+ },
+ data: { agent_settings_diff: { mcp_config: mcpConfig } },
+ });
+ expect(resp.ok(), `seed mcp_config: ${resp.status()}`).toBe(true);
+}
+
+/** Seed a Slack stdio server (matches the marketplace catalog entry). */
+async function installSlackViaAPI(
+ request: APIRequestContext,
+ env: Record<string, string>,
+) {
+ await patchMcpConfig(request, {
+ mcpServers: {
+ slack: {
+ command: "npx",
+ args: ["-y", "@zencoderai/slack-mcp-server"],
+ env,
+ },
+ },
+ });
+}
+
+async function openSlackInstallModal(page: Page) {
+ await page.goto("/mcp", { waitUntil: "domcontentloaded" });
+ await dismissAnalyticsModal(page);
+ await expect(page.getByTestId("mcp-marketplace-grid")).toBeVisible({
+ timeout: 10_000,
+ });
+ await page.getByTestId("mcp-marketplace-card-slack").click();
+ const modal = page.getByTestId("mcp-install-modal");
+ await expect(modal).toBeVisible({ timeout: 5_000 });
+ await expect(modal).toHaveAttribute("data-marketplace-id", "slack");
+ return modal;
+}
+
+async function fillSlackCredentials(
+ page: Page,
+ teamId: string,
+ botToken: string,
+) {
+ await page.getByTestId("mcp-install-field-SLACK_TEAM_ID").fill(teamId);
+ await page.getByTestId("mcp-install-field-SLACK_BOT_TOKEN").fill(botToken);
+}
+
+/** Open the editor for the first installed server (clicking its title avoids
+ * the delete toggle in the card's top-right corner). */
+async function openInstalledEditor(page: Page) {
+ await page.goto("/mcp", { waitUntil: "domcontentloaded" });
+ await dismissAnalyticsModal(page);
+ const list = page.getByTestId("mcp-installed-list");
+ await expect(list).toBeVisible({ timeout: 10_000 });
+ const item = list.getByTestId("mcp-server-item").first();
+ await expect(item).toBeVisible();
+ await item.locator("h3").click();
+ await expect(page.getByTestId("mcp-custom-editor")).toBeVisible({
+ timeout: 5_000,
+ });
+}
+
+test.describe.configure({ mode: "serial" });
+
+test.describe("MCP Test Connection credential verification (Slack)", () => {
+ test.beforeEach(async ({ page, request }) => {
+ await seedLocalStorage(page);
+ // A configured LLM profile keeps the MCP page free of "not configured"
+ // banners that could intercept clicks. Done via API for speed.
+ await ensureMockLLMProfileViaAPI(request);
+ });
+
+ test.afterEach(async ({ request }) => {
+ // Reset MCP config so each test starts from a clean installed list.
+ await patchMcpConfig(request, null).catch(() => {});
+ });
+
+ test("install: invalid Slack credentials are blocked with a credential-check error", async ({
+ page,
+ request,
+ }) => {
+ await routeSessionApiKey(page);
+
+ const testRequests: McpTestRequestBody[] = [];
+ await page.route("**/api/mcp/test", async (route) => {
+ testRequests.push(route.request().postDataJSON());
+ await route.fulfill(
+ jsonRoute(slackToolResult('{"ok":false,"error":"invalid_auth"}')),
+ );
+ });
+
+ const modal = await openSlackInstallModal(page);
+ await fillSlackCredentials(page, "T_INVALID", "xoxb-invalid-token");
+ await page.getByTestId("mcp-install-submit").click();
+
+ // The credential failure is surfaced and the modal stays open.
+ const error = page.getByTestId("mcp-install-modal-error");
+ await expect(error).toBeVisible({ timeout: 10_000 });
+ await expect(error).toContainText("Credential check failed: invalid_auth");
+ await expect(modal).toBeVisible();
+
+ // The read-only verification probe was attached to the test request.
+ expect(testRequests[0]?.tool_call).toEqual(SLACK_TOOL_CALL);
+
+ // Nothing was installed.
+ expect(hasSlackServer(await getSettings(request))).toBe(false);
+ });
+
+ test("install: a valid token missing only a scope still installs (missing_scope is not a credential failure)", async ({
+ page,
+ request,
+ }) => {
+ await routeSessionApiKey(page);
+ await page.route("**/api/mcp/test", async (route) => {
+ await route.fulfill(
+ jsonRoute(slackToolResult('{"ok":false,"error":"missing_scope"}')),
+ );
+ });
+
+ const modal = await openSlackInstallModal(page);
+ await fillSlackCredentials(
+ page,
+ "T0123456",
+ "xoxb-valid-but-missing-scope",
+ );
+ await page.getByTestId("mcp-install-submit").click();
+
+ // missing_scope proves the token authenticated → install proceeds.
+ await expect(modal).not.toBeVisible({ timeout: 10_000 });
+ await expect(page.getByTestId("mcp-installed-list")).toBeVisible({
+ timeout: 10_000,
+ });
+ expect(hasSlackServer(await getSettings(request))).toBe(true);
+ });
+
+ test("install: an older agent server that omits tool_result still installs (compat)", async ({
+ page,
+ request,
+ }) => {
+ await routeSessionApiKey(page);
+ // No `tool_result` field → the response passes through uninterpreted.
+ await page.route("**/api/mcp/test", async (route) => {
+ await route.fulfill(jsonRoute({ ok: true, tools: ["a", "b"] }));
+ });
+
+ const modal = await openSlackInstallModal(page);
+ await fillSlackCredentials(page, "T0123456", "xoxb-some-token");
+ await page.getByTestId("mcp-install-submit").click();
+
+ await expect(modal).not.toBeVisible({ timeout: 10_000 });
+ expect(hasSlackServer(await getSettings(request))).toBe(true);
+ });
+
+ test("edit: Test Connection verifies the stored credentials and surfaces a credential failure", async ({
+ page,
+ request,
+ }) => {
+ await installSlackViaAPI(request, {
+ SLACK_TEAM_ID: "T0123456",
+ SLACK_BOT_TOKEN: "xoxb-stored-secret",
+ });
+ await routeSessionApiKey(page);
+
+ const testRequests: McpTestRequestBody[] = [];
+ await page.route("**/api/mcp/test", async (route) => {
+ testRequests.push(route.request().postDataJSON());
+ await route.fulfill(
+ jsonRoute(slackToolResult('{"ok":false,"error":"invalid_auth"}')),
+ );
+ });
+
+ await openInstalledEditor(page);
+ await page.getByTestId("mcp-test-connection").click();
+
+ const message = page.getByTestId("mcp-test-message");
+ await expect(message).toBeVisible({ timeout: 10_000 });
+ await expect(message).toContainText(
+ "Credential check failed: invalid_auth",
+ );
+
+ // The Slack verification probe is attached in the edit flow too...
+ expect(testRequests[0]?.tool_call).toEqual(SLACK_TOOL_CALL);
+ // ...and the request tests the *stored* token, never the placeholder the
+ // browser sees in the redacted form (the second root cause the PR fixes).
+ expect(testRequests[0]?.server?.env?.SLACK_BOT_TOKEN).not.toBe(REDACTED);
+ });
+
+ test("edit: Test Connection reports success for valid stored credentials", async ({
+ page,
+ request,
+ }) => {
+ await installSlackViaAPI(request, {
+ SLACK_TEAM_ID: "T0123456",
+ SLACK_BOT_TOKEN: "xoxb-stored-secret",
+ });
+ await routeSessionApiKey(page);
+ await page.route("**/api/mcp/test", async (route) => {
+ await route.fulfill(
+ jsonRoute(slackToolResult('{"ok":true,"channels":[]}')),
+ );
+ });
+
+ await openInstalledEditor(page);
+ await page.getByTestId("mcp-test-connection").click();
+
+ const message = page.getByTestId("mcp-test-message");
+ await expect(message).toBeVisible({ timeout: 10_000 });
+ // 2 tools listed → "Connected — 2 tool(s) available".
+ await expect(message).toContainText("2 tool(s) available");
+ await expect(message).not.toContainText("Credential check failed");
+ });
+
+ test("custom (non-catalog) server: Test Connection attaches no verification probe", async ({
+ page,
+ request,
+ }) => {
+ // A custom server is not in the marketplace catalog → no credential probe.
+ await patchMcpConfig(request, {
+ mcpServers: { "my-custom": { url: "https://custom.example.test/mcp" } },
+ });
+ await routeSessionApiKey(page);
+
+ const testRequests: McpTestRequestBody[] = [];
+ await page.route("**/api/mcp/test", async (route) => {
+ testRequests.push(route.request().postDataJSON());
+ await route.fulfill(jsonRoute({ ok: true, tools: ["search"] }));
+ });
+
+ await openInstalledEditor(page);
+ await page.getByTestId("mcp-test-connection").click();
+
+ const message = page.getByTestId("mcp-test-message");
+ await expect(message).toBeVisible({ timeout: 10_000 });
+ await expect(message).toContainText("1 tool(s) available");
+
+ // Behavior unchanged for custom servers: no tool_call attached.
+ expect(testRequests[0]?.tool_call).toBeUndefined();
+ });
+});