Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
11.2 KB
Raw Blame
import { describe, expect, test } from "bun:test"

import {
  attributeChatToolError,
  connectionCardPayloadFromChatToolResult,
  reconnectActionFromChatToolResult,
} from "../src/components/tools/error-attribution"
import { normalizeErrorText } from "../src/lib/error-text"

function reconnectStatus(connectionId = "emc_knowledge", connectionName = "Knowledge Hub") {
  return {
    version: 1,
    kind: "connection_action",
    source: "omnirush-cloud",
    connectionId,
    connectionName,
    authType: "oauth",
    credentialMode: "per_member",
    state: "reauth_required",
    actor: "member",
    action: {
      type: "reconnect",
      surface: "omnirush_your_connections",
      retry: "search_capabilities",
      label: "Reconnect in Your Connections",
    },
  }
}

const connectionPayload = {
  schemaVersion: "1",
  connectionId: "emc_knowledge",
  connectionName: "Knowledge Hub",
  state: "needs_connection",
  actor: "member",
  message: "Connect your account to continue.",
  action: { type: "connect", label: "Connect Knowledge Hub", surface: "omnirush_your_connections" },
}

describe("chat tool error attribution", () => {
  test("uses the same native action for search attachments and standalone status results", () => {
    for (const { toolName, payload } of [
      { toolName: "omnirush-cloud_search_capabilities", payload: { connectionAction: connectionPayload } },
      { toolName: "omnirush-cloud_execute_capability", payload: connectionPayload },
      { toolName: "omnirush-cloud_connection_action", payload: connectionPayload },
    ]) {
      for (const result of [payload, JSON.stringify(payload)]) {
        expect(connectionCardPayloadFromChatToolResult(toolName, result, { intent: "connect" })).toEqual(connectionPayload)
        expect(reconnectActionFromChatToolResult(toolName, result, { intent: "connect" })).toEqual({
          connectionId: "emc_knowledge", connectionName: "Knowledge Hub", label: "Connect",
        })
      }
    }
  })

  test("keeps connected and admin states native without offering member authorization", () => {
    const connected = { ...connectionPayload, state: "connected", actor: null, action: null }
    const admin = { ...connectionPayload, actor: "organization_admin", action: {
      type: "update_credentials", label: "Ask an admin", surface: "omnirush_organization_connections",
    } }
    for (const payload of [connected, admin]) {
      expect(connectionCardPayloadFromChatToolResult("omnirush-cloud_execute_capability", payload)).toEqual(payload)
      expect(reconnectActionFromChatToolResult("omnirush-cloud_execute_capability", payload)).toBeNull()
    }
  })

  test("rejects foreign, malformed, ambiguous, and unsolicited portable connection cards", () => {
    for (const tool of ["malicious_execute_capability", "other_connection_action", "connection_action"]) {
      expect(connectionCardPayloadFromChatToolResult(tool, connectionPayload)).toBeNull()
      expect(reconnectActionFromChatToolResult(tool, connectionPayload)).toBeNull()
    }
    expect(connectionCardPayloadFromChatToolResult("omnirush-cloud_execute_capability", { ...connectionPayload, schemaVersion: "2" })).toBeNull()
    expect(connectionCardPayloadFromChatToolResult("omnirush-cloud_search_capabilities", { connectionAction: connectionPayload })).toBeNull()
    const matches = [connectionPayload, { ...connectionPayload, connectionId: "emc_second" }].map(connectionStatus => ({ connectionStatus }))
    expect(connectionCardPayloadFromChatToolResult("omnirush-cloud_search_capabilities", { matches }, { intent: "connect" })).toBeNull()
  })

  test("identifies an omnirush.ai-created capability deadline", () => {
    expect(attributeChatToolError("The capability call exceeded 180s. Retry once.")).toEqual({
      label: "omnirush.ai timeout",
      confidence: "Confirmed",
      description: "omnirush.ai created this deadline. The external operation may still have completed, so verify its state before retrying.",
    })
  })

  test("identifies a structured omnirush.ai lifecycle deadline", () => {
    expect(attributeChatToolError(JSON.stringify({
      error: "connection_failed",
      diagnostic: {
        code: "MCP_LIFECYCLE_DEADLINE",
        category: "lifecycle_deadline",
        phase: "MCP_TOOL_EXECUTION",
      },
    }))).toMatchObject({
      label: "omnirush.ai timeout",
      confidence: "Confirmed",
    })
  })

  test("identifies an omnirush.ai block before send", () => {
    expect(attributeChatToolError(JSON.stringify({
      diagnostic: { code: "MCP_URL_BLOCKED", category: "security_blocked" },
    }))).toMatchObject({
      label: "Blocked by omnirush.ai",
      confidence: "Confirmed",
    })
  })

  test("identifies a remote MCP HTTP failure", () => {
    expect(attributeChatToolError(`MCP error: ${JSON.stringify({
      diagnostic: { code: "MCP_HTTP_504", httpStatus: 504 },
    })} (tool execution failed)`)).toMatchObject({
      label: "Remote MCP · HTTP 504",
      confidence: "Confirmed",
    })
  })

  test("identifies a provider failure returned through the remote MCP", () => {
    expect(attributeChatToolError(JSON.stringify({
      diagnostic: { phase: "PROVIDER_AUTHORIZATION", providerStatus: 403 },
    }))).toMatchObject({
      label: "Provider error",
      confidence: "Confirmed",
      description: "The remote MCP responded, but the downstream provider returned status 403.",
    })
  })

  test("identifies provider attribution from a deploy-skew category and code", () => {
    expect(attributeChatToolError(JSON.stringify({
      diagnostic: { category: "provider_policy_denied", providerCode: "access_denied" },
    }))).toMatchObject({
      label: "Provider error",
      confidence: "Confirmed",
    })
  })

  test("does not claim ownership for an unstructured timeout", () => {
    expect(attributeChatToolError("Tool request timed out while waiting for a response.")).toEqual({
      label: "Timeout · source unclear",
      confidence: "Inferred",
      description: "A timeout was reported, but the client did not receive structured evidence identifying which boundary created it.",
    })
  })

  test("does not add attribution without useful evidence", () => {
    expect(attributeChatToolError("The tool failed.")).toBeNull()
  })

  test("bails out quickly on a pathological HTML page", () => {
    const htmlError = `<!DOCTYPE html><html><head><title>502 Bad Gateway</title></head><body>${"x".repeat(1_024 * 1_024)}</body></html>`
    const started = performance.now()

    expect(attributeChatToolError(htmlError)).toBeNull()
    expect(performance.now() - started).toBeLessThan(1_000)
  })

  test("keeps JSON-head attribution after surrounding error text is clamped", () => {
    const diagnostic = JSON.stringify({
      error: "connection_failed",
      diagnostic: { code: "MCP_HTTP_504", httpStatus: 504 },
    })
    const unclamped = `MCP error: ${diagnostic} (tool execution failed)`
    const clamped = normalizeErrorText(`${unclamped}\n${"provider detail ".repeat(1_000)}`, { cap: 512 }).display

    expect(clamped).toContain(diagnostic)
    expect(attributeChatToolError(clamped)).toEqual(attributeChatToolError(unclamped))
  })

  test("extracts a trusted reconnect action from a Cloud capability failure", () => {
    const errorText = JSON.stringify({
      error: "connection_failed",
      connectionStatus: reconnectStatus(),
    })

    expect(reconnectActionFromChatToolResult("omnirush-cloud_execute_capability", errorText)).toEqual({
      connectionId: "emc_knowledge",
      connectionName: "Knowledge Hub",
      label: "Reconnect",
    })
  })

  test("extracts the same reconnect action when live capability discovery detects expired credentials", () => {
    const output = JSON.stringify({
      matches: [{
        kind: "connection_status",
        connectionStatus: reconnectStatus(),
      }],
    })

    expect(reconnectActionFromChatToolResult("omnirush-cloud_search_capabilities", output)).toBeNull()
    expect(reconnectActionFromChatToolResult("omnirush-cloud_search_capabilities", output, { intent: "connect" })).toEqual({
      connectionId: "emc_knowledge",
      connectionName: "Knowledge Hub",
      label: "Reconnect",
    })
  })

  test("derives reconnect copy instead of rendering action labels from tool output", () => {
    const errorText = JSON.stringify({
      connectionStatus: {
        ...reconnectStatus(),
        action: { ...reconnectStatus().action, label: "Open an injected link" },
      },
    })

    expect(reconnectActionFromChatToolResult("omnirush-cloud_execute_capability", errorText)).toEqual({
      connectionId: "emc_knowledge",
      connectionName: "Knowledge Hub",
      label: "Reconnect",
    })
  })

  test("does not create actions from arbitrary MCP tools or non-reconnect failures", () => {
    const reconnectPayload = JSON.stringify({
      connectionStatus: reconnectStatus(),
    })
    const providerPayload = JSON.stringify({
      connectionStatus: {
        ...reconnectStatus(),
        state: "provider_error",
        actor: "organization_admin",
        action: {
          type: "inspect_connection",
          surface: "omnirush_organization_connections",
          retry: "search_capabilities",
        },
      },
    })

    expect(reconnectActionFromChatToolResult("malicious_execute_capability", reconnectPayload)).toBeNull()
    expect(reconnectActionFromChatToolResult("omnirush-cloud_execute_capability", providerPayload)).toBeNull()
  })

  test("supports first-time member OAuth but rejects mismatched states and credentials", () => {
    const status = { ...reconnectStatus(), state: "needs_connection", action: { type: "connect", surface: "omnirush_your_connections", retry: "search_capabilities" } }
    const action = (value: unknown) => reconnectActionFromChatToolResult("omnirush-cloud_search_capabilities", { matches: [{ connectionStatus: value }] }, { intent: "connect" })
    expect(action(status)).toEqual({ connectionId: "emc_knowledge", connectionName: "Knowledge Hub", label: "Connect" })
    expect(action({ ...status, authType: "apikey" })).toBeNull()
    expect(action({ ...status, credentialMode: "shared" })).toBeNull()
    expect(action({ ...status, actor: "organization_admin" })).toBeNull()
    expect(action({ ...status, state: "reauth_required" })).toBeNull()
  })

  test("does not guess between multiple reconnect targets in one discovery result", () => {
    const output = {
      matches: ["first", "second"].map((suffix) => ({
        kind: "connection_status",
        connectionStatus: reconnectStatus(`emc_${suffix}`, `Knowledge ${suffix}`),
      })),
    }

    expect(reconnectActionFromChatToolResult("omnirush-cloud_search_capabilities", output, { intent: "connect" })).toBeNull()
  })

  test("rejects unversioned, shared, and admin-owned action shapes", () => {
    const legacy = reconnectStatus()
    const { version: _version, kind: _kind, source: _source, ...unversioned } = legacy
    const shared = {
      ...legacy,
      credentialMode: "shared",
      actor: "organization_admin",
      action: {
        type: "reconnect",
        surface: "omnirush_organization_connections",
        retry: "search_capabilities",
      },
    }

    expect(reconnectActionFromChatToolResult("omnirush-cloud_execute_capability", { connectionStatus: unversioned })).toBeNull()
    expect(reconnectActionFromChatToolResult("omnirush-cloud_execute_capability", { connectionStatus: shared })).toBeNull()
  })
})