Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

32.7 KB
Raw Blame
import { afterEach, describe, expect, it, vi } from "vitest";

import { encrypt } from "../../../encryption";
import { env } from "../../../env";
import {
  BEDROCK_USE_DEFAULT_CREDENTIALS,
  VERTEXAI_USE_DEFAULT_CREDENTIALS,
} from "../../../interfaces/customLLMProviderConfigSchemas";
import {
  type ChatMessage,
  ChatMessageRole,
  ChatMessageType,
  LLMAdapter,
  type ModelParams,
  type TraceSinkParams,
} from "../types";
import { generateLangfuseAIText } from "../langfuseAiCompletion";
import {
  createLLMOutput,
  generateLLMText,
  mapLegacyLLMCompletionParams,
} from "../llmText";

// Request-shape coverage exercises the real provider packages while replacing
// the separately tested secure transport with a capture fetch.
vi.mock("../secureLlmFetch", () => ({
  createSecureLlmFetch: () => globalThis.fetch,
}));

// Request-shape tests only need the trace context as an input to provider
// request construction. Internal telemetry publishing is covered separately.
vi.mock("./telemetry", () => ({
  createAiSdkTelemetryCapture: () => undefined,
}));

// The Vertex provider exchanges service-account credentials for an OAuth
// token via google-auth-library before issuing the model request; both the
// provider and our vertex builder resolve to the same library copy, so this
// mock covers token generation and the ADC project lookup.
vi.mock("google-auth-library", () => ({
  GoogleAuth: class {
    getClient = async () => ({
      getAccessToken: async () => ({ token: "fake-gcp-token" }),
    });
    getProjectId = async () => "adc-project";
  },
}));

const messages: ChatMessage[] = [
  {
    type: ChatMessageType.System,
    role: ChatMessageRole.System,
    content: "You are terse.",
  },
  { type: ChatMessageType.User, role: ChatMessageRole.User, content: "Hi" },
];

type CapturedRequest = {
  url: string;
  method: string;
  headers: Headers;
  body: Record<string, unknown>;
};

/**
 * Fetch stub that records every outgoing request and answers with a canned
 * provider response, so the assertion target is the exact wire format the AI
 * SDK provider constructs (URL, auth header, JSON body).
 */
function createCaptureFetch(
  response: unknown | ((request: CapturedRequest) => unknown),
) {
  const calls: CapturedRequest[] = [];
  const fetchImpl: typeof fetch = async (input, init) => {
    const request = new Request(input, init);
    const capturedRequest = {
      url: request.url,
      method: request.method,
      headers: request.headers,
      body: JSON.parse(await request.text()) as Record<string, unknown>,
    };
    calls.push(capturedRequest);
    const responseBody =
      typeof response === "function" ? response(capturedRequest) : response;
    return new Response(JSON.stringify(responseBody), {
      status: 200,
      headers: { "content-type": "application/json" },
    });
  };
  return { calls, fetch: fetchImpl };
}

const OPENAI_CHAT_RESPONSE = {
  id: "chatcmpl-1",
  object: "chat.completion",
  created: 1,
  model: "gpt-4o",
  choices: [
    {
      index: 0,
      message: { role: "assistant", content: "ok" },
      finish_reason: "stop",
    },
  ],
  usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
};

const OPENAI_RESPONSES_RESPONSE = {
  id: "resp_1",
  object: "response",
  created_at: 1,
  status: "completed",
  error: null,
  incomplete_details: null,
  model: "gpt-4o",
  output: [
    {
      type: "message",
      id: "msg_1",
      status: "completed",
      role: "assistant",
      content: [{ type: "output_text", text: "ok", annotations: [] }],
    },
  ],
  usage: {
    input_tokens: 1,
    input_tokens_details: { cached_tokens: 0 },
    output_tokens: 1,
    output_tokens_details: { reasoning_tokens: 0 },
    total_tokens: 2,
  },
};

const ANTHROPIC_RESPONSE = {
  id: "msg_1",
  type: "message",
  role: "assistant",
  model: "claude-sonnet-5",
  content: [{ type: "text", text: "ok" }],
  stop_reason: "end_turn",
  stop_sequence: null,
  usage: { input_tokens: 1, output_tokens: 1 },
};

const GOOGLE_RESPONSE = {
  candidates: [
    {
      content: { role: "model", parts: [{ text: "ok" }] },
      finishReason: "STOP",
    },
  ],
  usageMetadata: {
    promptTokenCount: 1,
    candidatesTokenCount: 1,
    totalTokenCount: 2,
  },
};

const BEDROCK_RESPONSE = {
  output: { message: { role: "assistant", content: [{ text: "ok" }] } },
  stopReason: "end_turn",
  usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
  metrics: { latencyMs: 1 },
};

const FAKE_GCP_SERVICE_ACCOUNT_KEY = JSON.stringify({
  type: "service_account",
  project_id: "sa-project-123",
  private_key_id: "key-id",
  private_key: "-----BEGIN PRIVATE KEY-----\nfake\n-----END PRIVATE KEY-----\n",
  client_email: "[email protected]",
  client_id: "123",
  auth_uri: "https://accounts.google.com/o/oauth2/auth",
  token_uri: "https://oauth2.googleapis.com/token",
  auth_provider_x509_cert_url: "https://www.googleapis.com/oauth2/v1/certs",
  client_x509_cert_url: "https://example.com/cert",
});

async function runCompletion(params: {
  modelParams: ModelParams;
  apiKey: string;
  baseURL?: string | null;
  extraHeaders?: Record<string, string>;
  llmConnectionConfig?: Record<string, string | boolean>;
  openAIChatProvider?: "native";
  output?: ReturnType<typeof createLLMOutput>;
  response: unknown;
  trace?: TraceSinkParams;
}) {
  const { calls, fetch } = createCaptureFetch(params.response);
  vi.stubGlobal("fetch", fetch);
  const result = await generateLLMText({
    ...mapLegacyLLMCompletionParams({
      messages,
      modelParams: params.modelParams,
      connection: {
        secretKey: encrypt(params.apiKey),
        baseURL: params.baseURL,
        extraHeaders: params.extraHeaders
          ? encrypt(JSON.stringify(params.extraHeaders))
          : undefined,
        config: params.llmConnectionConfig,
      },
    }),
    openAIChatProvider: params.openAIChatProvider,
    timeout: 10_000,
    output: params.output,
    trace: params.trace,
  });

  expect(calls).toHaveLength(1);
  return { result, request: calls[0] };
}

afterEach(() => {
  vi.unstubAllGlobals();
  vi.unstubAllEnvs();
});

describe("AI SDK request shapes", () => {
  const internalTrace: TraceSinkParams = {
    targetProjectId: "project-1",
    traceId: "1".repeat(32),
    traceName: "Execute evaluator: helpfulness",
    environment: "langfuse-llm-as-a-judge",
  };

  it("OpenAI chat completions: default URL, bearer auth, translated body params", async () => {
    const { result, request } = await runCompletion({
      modelParams: {
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        model: "gpt-4o",
        max_tokens: 128,
        temperature: 0.2,
        providerOptions: { reasoning_effort: "high" },
      },
      apiKey: "sk-test",
      response: OPENAI_CHAT_RESPONSE,
    });

    expect(result.text).toBe("ok");
    expect(request.url).toBe("https://api.openai.com/v1/chat/completions");
    expect(request.headers.get("authorization")).toBe("Bearer sk-test");
    expect(request.body.model).toBe("gpt-4o");
    expect(request.body.reasoning_effort).toBe("high");
    expect(request.body.temperature).toBe(0.2);
    expect(request.body.max_tokens).toBe(128);
    expect(request.body.messages).toEqual([
      { role: "system", content: "You are terse." },
      { role: "user", content: "Hi" },
    ]);
  });

  it("OpenAI responses mode: /v1/responses when useResponsesApi is set", async () => {
    const { request } = await runCompletion({
      modelParams: {
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        model: "gpt-4o",
      },
      apiKey: "sk-test",
      llmConnectionConfig: { useResponsesApi: true },
      openAIChatProvider: "native",
      response: OPENAI_RESPONSES_RESPONSE,
    });

    expect(request.url).toBe("https://api.openai.com/v1/responses");
    expect(request.headers.get("authorization")).toBe("Bearer sk-test");
    expect(request.body.model).toBe("gpt-4o");
  });

  it.each(["eu.api.openai.com", "api.openai.com."])(
    "preserves saved evaluator options on %s",
    async (host) => {
      const { request } = await runCompletion({
        modelParams: {
          provider: "openai",
          adapter: LLMAdapter.OpenAI,
          model: "gpt-4o",
          max_tokens: 128,
          temperature: 0.2,
          providerOptions: {
            seed: 42,
            response_format: { type: "json_object" },
          },
        },
        apiKey: "sk-test",
        baseURL: `https://${host}/v1`,
        response: OPENAI_CHAT_RESPONSE,
      });
      expect(request.url).toBe(`https://${host}/v1/chat/completions`);
      expect(request.body).toMatchObject({
        max_tokens: 128,
        temperature: 0.2,
        seed: 42,
        response_format: { type: "json_object" },
      });
    },
  );

  it("Langfuse AI first-party OpenAI credentials hit /v1/responses", async () => {
    const original = {
      LANGFUSE_AI_PROVIDER: env.LANGFUSE_AI_PROVIDER,
      LANGFUSE_AI_MODEL: env.LANGFUSE_AI_MODEL,
      LANGFUSE_AI_SMALL_MODEL: env.LANGFUSE_AI_SMALL_MODEL,
      LANGFUSE_AI_API_KEY: env.LANGFUSE_AI_API_KEY,
      LANGFUSE_AI_BASE_URL: env.LANGFUSE_AI_BASE_URL,
      LANGFUSE_AI_USE_RESPONSES_API: env.LANGFUSE_AI_USE_RESPONSES_API,
    };
    Object.assign(env, {
      LANGFUSE_AI_PROVIDER: "openai",
      LANGFUSE_AI_MODEL: "gpt-5.6-sol",
      LANGFUSE_AI_SMALL_MODEL: "gpt-5.6-luna",
      LANGFUSE_AI_API_KEY: "sk-test",
      LANGFUSE_AI_BASE_URL: undefined,
      LANGFUSE_AI_USE_RESPONSES_API: undefined,
    });

    try {
      const { calls, fetch } = createCaptureFetch(OPENAI_RESPONSES_RESPONSE);
      vi.stubGlobal("fetch", fetch);

      const text = await generateLangfuseAIText({
        messages,
        timeout: 10_000,
      });

      expect(text).toBe("ok");
      expect(calls).toHaveLength(1);
      expect(calls[0]?.url).toBe("https://api.openai.com/v1/responses");
      expect(calls[0]?.headers.get("authorization")).toBe("Bearer sk-test");
      expect(calls[0]?.body.model).toBe("gpt-5.6-sol");
    } finally {
      Object.assign(env, original);
    }
  });

  it("OpenAI-compatible chat completions: preserves custom provider options on the wire", async () => {
    const { result, request } = await runCompletion({
      modelParams: {
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        model: "custom-reasoning-model",
        max_tokens: 64,
        providerOptions: {
          reasoning_effort: "high",
          service_tier: "flex",
          parallel_tool_calls: false,
          logit_bias: { "42": 1 },
          thinkingBudget: 1024,
          thinkingLevel: "high",
        },
      },
      apiKey: "custom-key",
      baseURL: "https://openai-compatible.example.com/v1",
      response: OPENAI_CHAT_RESPONSE,
    });

    expect(result.text).toBe("ok");
    expect(request.url).toBe(
      "https://openai-compatible.example.com/v1/chat/completions",
    );
    expect(request.headers.get("authorization")).toBe("Bearer custom-key");
    expect(request.body.model).toBe("custom-reasoning-model");
    expect(request.body.max_tokens).toBe(64);
    expect(request.body.reasoning_effort).toBe("high");
    expect(request.body.service_tier).toBe("flex");
    expect(request.body.parallel_tool_calls).toBe(false);
    expect(request.body.logit_bias).toEqual({ "42": 1 });
    expect(request.body.serviceTier).toBeUndefined();
    expect(request.body.parallelToolCalls).toBeUndefined();
    expect(request.body.logitBias).toBeUndefined();
    expect(request.body.thinkingBudget).toBe(1024);
    expect(request.body.thinkingLevel).toBe("high");
  });

  it.each([
    {
      baseURL: "https://openrouter.ai/api/v1",
      providerOptions: undefined,
      expectedTrace: { environment: "langfuse-llm-as-a-judge" },
    },
    {
      baseURL: "https://openrouter.ai./api/v1",
      providerOptions: undefined,
      expectedTrace: { environment: "langfuse-llm-as-a-judge" },
    },
    {
      baseURL: "https://eu.openrouter.ai/api/v1",
      providerOptions: {
        trace: {
          environment: "customer-environment",
          feature: "support-agent",
        },
      },
      expectedTrace: {
        environment: "langfuse-llm-as-a-judge",
        feature: "support-agent",
      },
    },
  ])(
    "OpenRouter: propagates system-owned internal provenance and preserves trace metadata for $baseURL",
    async ({ baseURL, providerOptions, expectedTrace }) => {
      const { request } = await runCompletion({
        modelParams: {
          provider: "openrouter",
          adapter: LLMAdapter.OpenAI,
          model: "openai/gpt-4o-mini",
          providerOptions,
        },
        apiKey: "openrouter-key",
        baseURL,
        trace: internalTrace,
        response: OPENAI_CHAT_RESPONSE,
      });

      expect(request.body.trace).toEqual(expectedTrace);
    },
  );

  it("other OpenAI-compatible endpoints keep customer trace metadata unchanged", async () => {
    const { request } = await runCompletion({
      modelParams: {
        provider: "custom",
        adapter: LLMAdapter.OpenAI,
        model: "custom-model",
        providerOptions: {
          trace: {
            environment: "customer-environment",
            feature: "support-agent",
          },
        },
      },
      apiKey: "custom-key",
      baseURL: "https://openrouter.ai.example.com/v1",
      trace: internalTrace,
      response: OPENAI_CHAT_RESPONSE,
    });

    expect(request.body.trace).toEqual({
      environment: "customer-environment",
      feature: "support-agent",
    });
  });

  it("OpenAI-compatible chat completions: preserves JSON schema response_format", async () => {
    const { result, request } = await runCompletion({
      modelParams: {
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        model: "custom-schema-model",
      },
      apiKey: "custom-key",
      baseURL: "https://openai-compatible.example.com/v1",
      output: createLLMOutput({
        type: "object",
        properties: { answer: { type: "string" } },
        required: ["answer"],
        additionalProperties: false,
      }),
      response: {
        ...OPENAI_CHAT_RESPONSE,
        choices: [
          {
            index: 0,
            message: { role: "assistant", content: '{"answer":"ok"}' },
            finish_reason: "stop",
          },
        ],
      },
    });

    expect(result.output).toEqual({ answer: "ok" });
    expect(request.body.response_format).toMatchObject({
      type: "json_schema",
      json_schema: {
        strict: true,
        name: "response",
        schema: {
          type: "object",
          properties: { answer: { type: "string" } },
          required: ["answer"],
          additionalProperties: false,
        },
      },
    });
  });

  it("Azure: stored deployment URL with pinned api-version and api-key header", async () => {
    const { request } = await runCompletion({
      modelParams: {
        provider: "azure",
        adapter: LLMAdapter.Azure,
        model: "gpt4o-deployment",
        max_tokens: 64,
        providerOptions: { logit_bias: { "42": 1 } },
      },
      apiKey: "azure-key",
      baseURL: "https://my-instance.openai.azure.com/openai/deployments",
      extraHeaders: { "x-custom": "1" },
      response: OPENAI_CHAT_RESPONSE,
    });

    expect(request.url).toBe(
      "https://my-instance.openai.azure.com/openai/deployments/gpt4o-deployment/chat/completions?api-version=2025-02-01-preview",
    );
    expect(request.headers.get("api-key")).toBe("azure-key");
    expect(request.headers.get("x-custom")).toBe("1");
    expect(request.body.logit_bias).toEqual({ "42": 1 });
    expect(request.body.max_tokens).toBe(64);
  });

  it("Azure: deployment-specific stored URL is normalized before request construction", async () => {
    const { request } = await runCompletion({
      modelParams: {
        provider: "azure",
        adapter: LLMAdapter.Azure,
        model: "gpt4o-deployment",
        max_tokens: 64,
      },
      apiKey: "azure-key",
      baseURL:
        "https://my-instance.openai.azure.com/openai/deployments/gpt4o-deployment",
      response: OPENAI_CHAT_RESPONSE,
    });

    expect(request.url).toBe(
      "https://my-instance.openai.azure.com/openai/deployments/gpt4o-deployment/chat/completions?api-version=2025-02-01-preview",
    );
    expect(request.headers.get("api-key")).toBe("azure-key");
  });

  it("OpenAI chat completions: gpt-5.4 mini uses portable non-reasoning settings", async () => {
    const { request } = await runCompletion({
      modelParams: {
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        model: "gpt-5.4-mini",
        max_tokens: 64,
        temperature: 0.2,
        top_p: 0.9,
      },
      apiKey: "openai-key",
      response: OPENAI_CHAT_RESPONSE,
    });

    expect(request.body.max_tokens).toBeUndefined();
    expect(request.body.max_completion_tokens).toBe(64);
    expect(request.body.temperature).toBe(0.2);
    expect(request.body.top_p).toBe(0.9);
    expect(request.body.reasoning_effort).toBe("none");
    expect(request.body.forceReasoning).toBeUndefined();
  });

  it("OpenAI-compatible chat completions: gpt-5.4 mini does not leak internal reasoning controls", async () => {
    const { request } = await runCompletion({
      modelParams: {
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        model: "gpt-5.4-mini",
        max_tokens: 64,
        temperature: 0.2,
        top_p: 0.9,
      },
      apiKey: "custom-key",
      baseURL: "https://openai-compatible.example.com/v1",
      response: OPENAI_CHAT_RESPONSE,
    });

    expect(request.body.max_tokens).toBe(64);
    expect(request.body.temperature).toBe(0.2);
    expect(request.body.top_p).toBe(0.9);
    expect(request.body.reasoning_effort).toBeUndefined();
    expect(request.body.forceReasoning).toBeUndefined();
  });

  it("OpenAI-compatible chat completions: explicit gpt-5.4 mini reasoning remains a wire setting", async () => {
    const { request } = await runCompletion({
      modelParams: {
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        model: "gpt-5.4-mini",
        providerOptions: { reasoning_effort: "high" },
      },
      apiKey: "custom-key",
      baseURL: "https://openai-compatible.example.com/v1",
      response: OPENAI_CHAT_RESPONSE,
    });

    expect(request.body.reasoning_effort).toBe("high");
    expect(request.body.forceReasoning).toBeUndefined();
  });

  it("Anthropic: /v1/messages on a custom origin with snake_case thinking body", async () => {
    const { request } = await runCompletion({
      modelParams: {
        provider: "anthropic",
        adapter: LLMAdapter.Anthropic,
        model: "claude-sonnet-5",
        max_tokens: 256,
        providerOptions: { thinking: { type: "enabled", budget_tokens: 1024 } },
      },
      apiKey: "anthropic-key",
      baseURL: "https://anthropic-proxy.example.com",
      response: ANTHROPIC_RESPONSE,
    });

    expect(request.url).toBe("https://anthropic-proxy.example.com/v1/messages");
    expect(request.headers.get("x-api-key")).toBe("anthropic-key");
    expect(request.headers.get("anthropic-version")).toBeTruthy();
    expect(request.body.model).toBe("claude-sonnet-5");
    // With thinking enabled, the budget counts toward Anthropic's max_tokens,
    // so the provider adds maxOutputTokens and budgetTokens.
    expect(request.body.max_tokens).toBe(256 + 1024);
    expect(request.body.thinking).toEqual({
      type: "enabled",
      budget_tokens: 1024,
    });
    // The leading system message becomes the top-level system param.
    expect(JSON.stringify(request.body.system)).toContain("You are terse.");
    expect(request.body.messages).toHaveLength(1);
  });

  it("Google AI Studio: /v1beta path on a custom origin with thinkingConfig", async () => {
    const { request } = await runCompletion({
      modelParams: {
        provider: "google-ai-studio",
        adapter: LLMAdapter.GoogleAIStudio,
        model: "gemini-2.5-flash",
        providerOptions: { thinkingBudget: 512 },
      },
      apiKey: "google-key",
      baseURL: "https://google-proxy.example.com",
      response: GOOGLE_RESPONSE,
    });

    expect(request.url).toBe(
      "https://google-proxy.example.com/v1beta/models/gemini-2.5-flash:generateContent",
    );
    expect(request.headers.get("x-goog-api-key")).toBe("google-key");
    const generationConfig = request.body.generationConfig as Record<
      string,
      unknown
    >;
    expect(generationConfig.thinkingConfig).toEqual({
      thinkingBudget: 512,
      includeThoughts: true,
    });
  });

  it("Vertex Gemini: regional host, SA-key project, OAuth bearer, extra headers, maxReasoningTokens", async () => {
    const { request } = await runCompletion({
      modelParams: {
        provider: "vertex",
        adapter: LLMAdapter.VertexAI,
        model: "gemini-2.5-flash",
        maxReasoningTokens: 2048,
      },
      apiKey: FAKE_GCP_SERVICE_ACCOUNT_KEY,
      llmConnectionConfig: { location: "us-east5" },
      extraHeaders: { "X-Vertex-AI-Labels": "eyJ0ZWFtIjoiYSJ9" },
      response: GOOGLE_RESPONSE,
    });

    // The Google Vertex provider targets the v1beta1 generateContent API.
    expect(request.url).toBe(
      "https://us-east5-aiplatform.googleapis.com/v1beta1/projects/sa-project-123/locations/us-east5/publishers/google/models/gemini-2.5-flash:generateContent",
    );
    expect(request.headers.get("authorization")).toBe("Bearer fake-gcp-token");
    expect(request.headers.get("x-vertex-ai-labels")).toBe("eyJ0ZWFtIjoiYSJ9");
    const generationConfig = request.body.generationConfig as Record<
      string,
      unknown
    >;
    expect(generationConfig.thinkingConfig).toEqual({
      thinkingBudget: 2048,
      includeThoughts: true,
    });
  });

  it("Vertex Claude: anthropic publisher rawPredict URL with vertex anthropic_version", async () => {
    const { request } = await runCompletion({
      modelParams: {
        provider: "vertex",
        adapter: LLMAdapter.VertexAI,
        model: "claude-sonnet-4-5@20250929",
        max_tokens: 128,
      },
      apiKey: FAKE_GCP_SERVICE_ACCOUNT_KEY,
      llmConnectionConfig: { location: "us-east5" },
      response: ANTHROPIC_RESPONSE,
    });

    expect(decodeURIComponent(request.url)).toBe(
      "https://us-east5-aiplatform.googleapis.com/v1/projects/sa-project-123/locations/us-east5/publishers/anthropic/models/claude-sonnet-4-5@20250929:rawPredict",
    );
    expect(request.headers.get("authorization")).toBe("Bearer fake-gcp-token");
    // Vertex mode: the version moves into the body and the model into the URL.
    expect(request.body.anthropic_version).toBeTruthy();
    expect(request.body.model).toBeUndefined();
    expect(request.body.max_tokens).toBe(128);
  });

  // Vertex reaches application default credentials through two independent
  // gates: the ADC sentinel in place of a key, and a credential source that is
  // allowed to spend the server's own identity. The pair below pins both
  // directions of that gate, because widening it hands a tenant connection the
  // Langfuse deployment's GCP identity and narrowing it silently disables
  // instance-wide Vertex wherever the Cloud region is set.
  it("Langfuse AI Vertex resolves the project from ADC, Cloud region notwithstanding", async () => {
    const original = {
      LANGFUSE_AI_PROVIDER: env.LANGFUSE_AI_PROVIDER,
      LANGFUSE_AI_MODEL: env.LANGFUSE_AI_MODEL,
      LANGFUSE_AI_SMALL_MODEL: env.LANGFUSE_AI_SMALL_MODEL,
      LANGFUSE_AI_VERTEX_LOCATION: env.LANGFUSE_AI_VERTEX_LOCATION,
      NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
    };
    Object.assign(env, {
      LANGFUSE_AI_PROVIDER: "vertex",
      LANGFUSE_AI_MODEL: "claude-sonnet-4-5@20250929",
      LANGFUSE_AI_SMALL_MODEL: undefined,
      LANGFUSE_AI_VERTEX_LOCATION: "us-east5",
      NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "eu",
    });

    try {
      const { calls, fetch } = createCaptureFetch(ANTHROPIC_RESPONSE);
      vi.stubGlobal("fetch", fetch);

      const text = await generateLangfuseAIText({
        messages,
        maxTokens: 128,
        timeout: 10_000,
      });

      expect(text).toBe("ok");
      expect(calls).toHaveLength(1);
      // The project comes from the credential chain, never from configuration,
      // and the location comes from the instance env rather than a connection.
      expect(decodeURIComponent(calls[0]?.url ?? "")).toBe(
        "https://us-east5-aiplatform.googleapis.com/v1/projects/adc-project/locations/us-east5/publishers/anthropic/models/claude-sonnet-4-5@20250929:rawPredict",
      );
      expect(calls[0]?.headers.get("authorization")).toBe(
        "Bearer fake-gcp-token",
      );
    } finally {
      Object.assign(env, original);
    }
  });

  it("a tenant Vertex connection cannot spend ADC on Langfuse Cloud", async () => {
    const original = {
      NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
    };
    Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "eu" });

    try {
      const { calls, fetch } = createCaptureFetch(GOOGLE_RESPONSE);
      vi.stubGlobal("fetch", fetch);

      await expect(
        generateLLMText({
          ...mapLegacyLLMCompletionParams({
            messages,
            modelParams: {
              provider: "vertex",
              adapter: LLMAdapter.VertexAI,
              model: "gemini-2.5-flash",
            },
            connection: {
              secretKey: encrypt(VERTEXAI_USE_DEFAULT_CREDENTIALS),
              config: { location: "us-east5" },
            },
            credentialSource: "user",
          }),
          timeout: 10_000,
        }),
      ).rejects.toThrow();

      // The sentinel must not fall back to the server's identity: no request
      // may leave carrying the deployment's ADC project.
      expect(calls).toHaveLength(0);
    } finally {
      Object.assign(env, original);
    }
  });

  // The Bedrock builder deliberately uses no custom fetch (VPC endpoints),
  // so capture at the global fetch the AI SDK falls back to.
  async function runBedrockCompletion(params?: {
    model?: string;
    projectId?: string;
    secretKey?: string;
    output?: ReturnType<typeof createLLMOutput>;
    response?: unknown | ((request: CapturedRequest) => unknown);
  }) {
    const modelParams: ModelParams = {
      provider: "bedrock",
      adapter: LLMAdapter.Bedrock,
      model: params?.model ?? "anthropic.claude-3-5-sonnet-20241022-v2:0",
      max_tokens: 64,
      providerOptions: { top_k: 10 },
    };
    const llmConnectionConfig = { region: "us-east-1" };

    const { calls, fetch } = createCaptureFetch(
      params?.response ?? BEDROCK_RESPONSE,
    );
    vi.stubGlobal("fetch", fetch);

    const result = await generateLLMText({
      ...mapLegacyLLMCompletionParams({
        messages,
        modelParams,
        connection: {
          projectId: params?.projectId,
          secretKey: encrypt(
            params?.secretKey ??
              JSON.stringify({
                accessKeyId: "AKIA123",
                secretAccessKey: "secret",
              }),
          ),
          config: llmConnectionConfig,
        },
      }),
      timeout: 10_000,
      output: params?.output,
    });

    if (!params?.output) {
      expect(result.text).toBe("ok");
    }
    expect(calls).toHaveLength(1);
    return { result, request: calls[0] };
  }

  it("Bedrock: converse URL from validated region, SigV4 auth, verbatim additional fields", async () => {
    const { request } = await runBedrockCompletion();

    expect(decodeURIComponent(request.url)).toBe(
      "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-5-sonnet-20241022-v2:0/converse",
    );
    expect(request.headers.get("authorization")).toMatch(/^AWS4-HMAC-SHA256/);
    expect(request.body.additionalModelRequestFields).toEqual({ top_k: 10 });
    expect(request.body.inferenceConfig).toMatchObject({ maxTokens: 64 });
  });

  it("Bedrock: application inference profile ARN stays in one URL segment", async () => {
    const model =
      "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/profile-id";
    const { request } = await runBedrockCompletion({ model });

    expect(request.url).toBe(
      `https://bedrock-runtime.us-east-1.amazonaws.com/model/${encodeURIComponent(model)}/converse`,
    );
  });

  it("Bedrock: Sonnet 5 structured output falls back to the JSON tool", async () => {
    const schema = {
      type: "object",
      properties: { answer: { type: "string" } },
      required: ["answer"],
      additionalProperties: false,
    } as const;
    const { result, request } = await runBedrockCompletion({
      model: "us.anthropic.claude-sonnet-5",
      output: createLLMOutput(schema),
      response: (capturedRequest: CapturedRequest) =>
        capturedRequest.body.toolConfig
          ? {
              output: {
                message: {
                  role: "assistant",
                  content: [
                    {
                      toolUse: {
                        toolUseId: "tool-1",
                        name: "json",
                        input: { answer: "ok" },
                      },
                    },
                  ],
                },
              },
              stopReason: "tool_use",
              usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
              metrics: { latencyMs: 1 },
            }
          : {
              ...BEDROCK_RESPONSE,
              output: {
                message: {
                  role: "assistant",
                  content: [{ text: '{"answer":"ok"}' }],
                },
              },
            },
    });

    expect(result.output).toEqual({ answer: "ok" });
    expect(request.body.additionalModelRequestFields).toEqual({ top_k: 10 });
    expect(request.body.toolConfig).toEqual({
      tools: [
        {
          toolSpec: {
            name: "json",
            description: "Respond with a JSON object.",
            inputSchema: { json: schema },
          },
        },
      ],
      toolChoice: { any: {} },
    });
  });

  it("Bedrock: models rejecting forced tools use JSON instructions", async () => {
    const schema = {
      type: "object",
      properties: { answer: { type: "string" } },
      required: ["answer"],
      additionalProperties: false,
    } as const;
    const { result, request } = await runBedrockCompletion({
      model: "us.anthropic.claude-fable-5-1",
      output: createLLMOutput(schema),
      response: {
        ...BEDROCK_RESPONSE,
        output: {
          message: {
            role: "assistant",
            content: [{ text: '{"answer":"ok"}' }],
          },
        },
      },
    });

    expect(result.output).toEqual({ answer: "ok" });
    expect(request.body.toolConfig).toBeUndefined();
    expect(JSON.stringify(request.body)).toContain(
      "You MUST answer with only a JSON object",
    );
  });

  it("Bedrock: tenant credentials suppress server-level env auth fallbacks", async () => {
    // A self-hosted operator may set these for their own purposes; tenant
    // connections must never authenticate with them. Unsuppressed, the
    // provider's AWS_BEARER_TOKEN_BEDROCK fallback wins over the tenant's
    // access keys entirely (requests would run under the SERVER identity).
    vi.stubEnv("AWS_BEARER_TOKEN_BEDROCK", "server-bearer-token");
    vi.stubEnv("AWS_SESSION_TOKEN", "server-session-token");

    const { request } = await runBedrockCompletion();

    // SigV4 with the tenant keys — not `Bearer server-bearer-token`.
    expect(request.headers.get("authorization")).toMatch(/^AWS4-HMAC-SHA256/);
    expect(request.headers.get("authorization")).toContain("AKIA123");
    // No merged server session token (would surface as this SigV4 header).
    expect(request.headers.get("x-amz-security-token")).toBeNull();
  });

  it("Bedrock: project allowlisting adds Cloud access without restricting self-hosted", async () => {
    const originalEnv = { ...env };
    Object.assign(env, {
      NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "eu",
      LANGFUSE_BEDROCK_LLM_CONNECTION_RUNTIME_CREDENTIALS_ALLOWED_PROJECT_IDS: [
        "allowed",
      ],
    });
    vi.stubEnv("AWS_PROFILE", undefined);
    vi.stubEnv("AWS_ACCESS_KEY_ID", "AKIARUNTIME");
    vi.stubEnv("AWS_SECRET_ACCESS_KEY", "runtime-secret");
    const call = (projectId?: string) =>
      runBedrockCompletion({
        projectId,
        secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
      });

    try {
      const { request } = await call("allowed");
      expect(request.headers.get("authorization")).toContain(
        "Credential=AKIARUNTIME/",
      );
      for (const projectId of ["other", "allow", undefined]) {
        await expect(call(projectId)).rejects.toThrow(
          "Invalid Bedrock credentials",
        );
      }
      env.LANGFUSE_BEDROCK_LLM_CONNECTION_RUNTIME_CREDENTIALS_ALLOWED_PROJECT_IDS =
        [];
      await expect(call("allowed")).rejects.toThrow(
        "Invalid Bedrock credentials",
      );
      env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
      await call();
    } finally {
      Object.assign(env, originalEnv);
    }
  });
});