Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

43.9 KB
Raw Blame
import { describe, test, expect } from "vitest";
import {
  createLLMOutput,
  createLLMToolSet,
  generateLLMText,
  getLLMErrorInfo,
  mapLegacyLLMCompletionParams,
  streamLLMText,
  type LLMModelMessage,
} from "@langfuse/shared/src/server";
import { encrypt } from "@langfuse/shared/encryption";
import {
  buildEvalOutputResultSchema,
  ChatMessageType,
  createBooleanEvalOutputDefinition,
  createCategoricalEvalOutputDefinition,
  createNumericEvalOutputDefinition,
  type PersistedEvalOutputDefinition,
  LLMAdapter,
  type ModelParams,
} from "@langfuse/shared";
import { z } from "zod";

/**
 * LLM Connection Integration Tests
 *
 * These tests verify that all supported LLM adapters work correctly with live API calls.
 * Each adapter is tested with:
 * 1. Simple completion
 * 2. Streaming completion
 * 3. Multi-message completion
 * 4. Multi-part text and image completion
 * 5. Structured output (legacy eval schema, v2 numeric schema, v2 boolean schema, v2 categorical schema)
 * 6. Tool calling
 *
 * Required environment variables (tests will FAIL if not set):
 * - LANGFUSE_LLM_CONNECTION_OPENAI_KEY
 * - LANGFUSE_LLM_CONNECTION_ANTHROPIC_KEY
 * - LANGFUSE_LLM_CONNECTION_AZURE_KEY
 * - LANGFUSE_LLM_CONNECTION_AZURE_BASE_URL
 * - LANGFUSE_LLM_CONNECTION_AZURE_MODEL (Azure suite is currently skipped)
 * - LANGFUSE_LLM_CONNECTION_BEDROCK_ACCESS_KEY_ID
 * - LANGFUSE_LLM_CONNECTION_BEDROCK_SECRET_ACCESS_KEY
 * - LANGFUSE_LLM_CONNECTION_BEDROCK_REGION
 * - LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY
 * - LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY
 * - LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY
 */

type TestLLMConnection = Parameters<
  typeof mapLegacyLLMCompletionParams
>[0]["connection"];

type TestCompletionParams = {
  messages: Parameters<typeof mapLegacyLLMCompletionParams>[0]["messages"];
  modelParams: ModelParams;
  llmConnection: TestLLMConnection;
  structuredOutputSchema?: z.ZodType;
  tools?: Parameters<typeof createLLMToolSet>[0];
};

function generateWithStoredConnection(params: TestCompletionParams) {
  const { structuredOutputSchema, tools, ...legacyParams } = params;

  return generateLLMText({
    ...mapLegacyLLMCompletionParams({
      messages: legacyParams.messages,
      modelParams: legacyParams.modelParams,
      connection: legacyParams.llmConnection,
    }),
    ...(structuredOutputSchema
      ? { output: createLLMOutput(structuredOutputSchema) }
      : {}),
    ...(tools?.length ? { tools: createLLMToolSet(tools) } : {}),
  });
}

function streamWithStoredConnection(
  params: Omit<TestCompletionParams, "structuredOutputSchema" | "tools">,
) {
  return streamLLMText(
    mapLegacyLLMCompletionParams({
      messages: params.messages,
      modelParams: params.modelParams,
      connection: params.llmConnection,
    }),
  );
}

function generateWithModelMessages(params: {
  messages: LLMModelMessage[];
  modelParams: ModelParams;
  llmConnection: TestLLMConnection;
}) {
  const mappedParams = mapLegacyLLMCompletionParams({
    messages: [],
    modelParams: params.modelParams,
    connection: params.llmConnection,
  });

  return generateLLMText({
    ...mappedParams,
    messages: params.messages,
  });
}

const googleAIStudioFallbackModels = [
  "gemini-3.1-flash-lite",
  "gemini-2.5-flash",
  "gemini-3.5-flash",
] as const;

async function runWithGoogleAIStudioModelFallback<T>(
  operation: (model: string) => Promise<T>,
): Promise<T> {
  let lastError: unknown;

  for (const [index, model] of googleAIStudioFallbackModels.entries()) {
    try {
      return await operation(model);
    } catch (error) {
      lastError = error;

      if (
        index === googleAIStudioFallbackModels.length - 1 ||
        !isRetryableProviderError(error)
      ) {
        throw error;
      }
    }
  }

  throw lastError;
}

function isRetryableProviderError(error: unknown): boolean {
  const llmError = getLLMErrorInfo(error);
  if (llmError) return llmError.isRetryable;

  if (!error || typeof error !== "object") return false;

  const errorLike = error as {
    response?: { status?: unknown };
    status?: unknown;
    statusCode?: unknown;
  };
  const statusCode = [
    errorLike.response?.status,
    errorLike.status,
    errorLike.statusCode,
  ].find((code): code is number => typeof code === "number");

  return statusCode === 429 || (statusCode !== undefined && statusCode >= 500);
}

const numericEvalResponseSchema = z.object({
  score: z.number(),
  reasoning: z.string().min(1),
});

const booleanEvalResponseSchema = z.object({
  score: z.boolean(),
  reasoning: z.string().min(1),
});

const categoricalScoreValues = ["correct", "incorrect"] as const;
const categoricalEvalResponseSchema = z.object({
  score: z.enum(categoricalScoreValues),
  reasoning: z.string().min(1),
});

type EvalStructuredOutputTestCase = {
  name: string;
  prompt: string;
  outputDefinition: PersistedEvalOutputDefinition;
  responseSchema: z.ZodType;
  assertParsed?: (data: {
    score: number | boolean | string;
    reasoning: string;
  }) => void;
};

const evalStructuredOutputTestCases: EvalStructuredOutputTestCase[] = [
  {
    name: "structured output - legacy eval schema",
    prompt:
      "Evaluate whether the answer '2 + 2 = 4' is correct. Return a numeric score from 0 to 100 and explain the score.",
    outputDefinition: {
      score:
        "Return a numeric score from 0 to 100, where 100 means the answer is completely correct.",
      reasoning: "Explain briefly why you chose that score.",
    },
    responseSchema: numericEvalResponseSchema,
  },
  {
    name: "structured output - v2 numeric eval schema",
    prompt:
      "Evaluate whether the answer '2 + 2 = 4' is correct. Return a numeric score from 0 to 100 and explain the score.",
    outputDefinition: createNumericEvalOutputDefinition({
      scoreDescription:
        "Return a numeric score from 0 to 100, where 100 means the answer is completely correct.",
      reasoningDescription: "Explain briefly why you chose that score.",
    }),
    responseSchema: numericEvalResponseSchema,
  },
  {
    name: "structured output - v2 boolean eval schema",
    prompt:
      "Judge whether the answer '2 + 2 = 5' is correct. Return true only if it is mathematically correct, otherwise false, and explain briefly.",
    outputDefinition: createBooleanEvalOutputDefinition({
      scoreDescription:
        "Return true when the answer is mathematically correct, otherwise return false.",
      reasoningDescription: "Explain briefly why you chose that verdict.",
    }),
    responseSchema: booleanEvalResponseSchema,
    assertParsed: (data) => {
      expect(data.score).toBe(false);
    },
  },
  {
    name: "structured output - v2 categorical eval schema",
    prompt:
      "Judge whether the answer '2 + 2 = 5' is correct. Select the best matching category and explain the choice.",
    outputDefinition: createCategoricalEvalOutputDefinition({
      scoreDescription:
        "Select 'correct' when the answer is mathematically accurate, otherwise select 'incorrect'.",
      reasoningDescription: "Explain briefly why you selected that category.",
      categories: ["correct", "incorrect"],
    }),
    responseSchema: categoricalEvalResponseSchema,
    assertParsed: (data) => {
      expect(data.score).toBe("incorrect");
    },
  },
];

const redSquarePng = Buffer.from(
  "iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGUlEQVR4nGP4z8DwnxLMMGrAqAGjBgwXAwAwxP4QHCfkAAAAAABJRU5ErkJggg==",
  "base64",
);

function registerMessageFormatTests(params: {
  checkEnv: () => void;
  getModelParams: (model?: string) => ModelParams;
  getLLMConnection: () => TestLLMConnection;
  timeoutMs: number;
  runWithModel?: <T>(operation: (model?: string) => Promise<T>) => Promise<T>;
}) {
  const runWithModel =
    params.runWithModel ?? ((operation) => operation(undefined));

  test(
    "multi-message completion",
    async () => {
      params.checkEnv();

      const completion = await runWithModel((model) =>
        generateWithStoredConnection({
          messages: [
            {
              role: "system",
              content: "Answer with only the requested number.",
              type: ChatMessageType.System,
            },
            {
              role: "user",
              content: "The code is 7319. Remember it.",
              type: ChatMessageType.PublicAPICreated,
            },
            {
              role: "assistant",
              content: "I will remember the code.",
              type: ChatMessageType.AssistantText,
            },
            {
              role: "user",
              content: "What is the code?",
              type: ChatMessageType.PublicAPICreated,
            },
          ],
          modelParams: params.getModelParams(model),
          llmConnection: params.getLLMConnection(),
        }),
      );

      expect(completion.text).toContain("7319");
    },
    params.timeoutMs,
  );

  test(
    "multi-part message completion",
    async () => {
      params.checkEnv();

      const completion = await runWithModel((model) =>
        generateWithModelMessages({
          messages: [
            {
              role: "user",
              content: [
                {
                  type: "text",
                  text: "What color is this square? Answer only with the color.",
                },
                {
                  type: "file",
                  data: redSquarePng,
                  mediaType: "image/png",
                },
              ],
            },
          ],
          modelParams: params.getModelParams(model),
          llmConnection: params.getLLMConnection(),
        }),
      );

      expect(completion.text.toLowerCase()).toContain("red");
    },
    params.timeoutMs,
  );
}

function registerEvalStructuredOutputTests(params: {
  checkEnv: () => void;
  getModelParams: (model?: string) => ModelParams;
  getLLMConnection: () => TestLLMConnection;
  timeoutMs: number;
  runWithModel?: <T>(operation: (model?: string) => Promise<T>) => Promise<T>;
}) {
  evalStructuredOutputTestCases.forEach((testCase) => {
    test(
      testCase.name,
      async () => {
        params.checkEnv();

        const completion = await (
          params.runWithModel ?? ((operation) => operation())
        )((model) =>
          generateWithStoredConnection({
            messages: [
              {
                role: "user",
                content: testCase.prompt,
                type: ChatMessageType.PublicAPICreated,
              },
            ],
            modelParams: params.getModelParams(model),
            structuredOutputSchema: buildEvalOutputResultSchema(
              testCase.outputDefinition,
            ),
            llmConnection: params.getLLMConnection(),
          }),
        );

        const parsed = testCase.responseSchema.safeParse(completion.output);
        expect(parsed.success).toBe(true);
        if (parsed.success) {
          expect(parsed.data.reasoning.trim().length).toBeGreaterThan(0);
          testCase.assertParsed?.(parsed.data);
        }
      },
      params.timeoutMs,
    );
  });
}

// Common tool definition for tool calling tests
const weatherTool = {
  name: "get_weather",
  description: "Get the current weather for a location",
  parameters: {
    type: "object",
    properties: {
      location: {
        type: "string",
        description: "The city name, e.g. 'Paris' or 'London'",
      },
    },
    required: ["location"],
  },
};

describe("LLM Connection Tests", () => {
  describe("OpenAI", () => {
    const MODEL = "gpt-4o-mini";

    const checkEnvVar = () => {
      if (!process.env.LANGFUSE_LLM_CONNECTION_OPENAI_KEY) {
        throw new Error(
          "LANGFUSE_LLM_CONNECTION_OPENAI_KEY not set. " +
            "This test requires a valid OpenAI API key to verify the LLM connection. " +
            "Set the environment variable to run this test.",
        );
      }
    };

    test("simple completion", async () => {
      checkEnvVar();

      const completion = await generateWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What is 2+2? Answer only with the number.",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "openai",
          adapter: LLMAdapter.OpenAI,
          model: MODEL,
          temperature: 0,
          max_tokens: 10,
        },
        llmConnection: {
          secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_OPENAI_KEY!),
        },
      });

      expect(completion.text).toContain("4");
    }, 30_000);

    test("streaming completion", async () => {
      checkEnvVar();

      const stream = await streamWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What is 2+2? Answer only with the number.",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "openai",
          adapter: LLMAdapter.OpenAI,
          model: MODEL,
          temperature: 0,
          max_tokens: 10,
        },
        llmConnection: {
          secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_OPENAI_KEY!),
        },
      });

      let fullResponse = "";
      let chunkCount = 0;

      for await (const chunk of stream.textStream) {
        fullResponse += chunk;
        chunkCount++;
      }

      expect(chunkCount).toBeGreaterThan(0);
      expect(fullResponse).toContain("4");
    }, 30_000);

    registerMessageFormatTests({
      checkEnv: checkEnvVar,
      getModelParams: () => ({
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        model: MODEL,
        temperature: 0,
        max_tokens: 50,
      }),
      getLLMConnection: () => ({
        secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_OPENAI_KEY!),
      }),
      timeoutMs: 30_000,
    });

    registerEvalStructuredOutputTests({
      checkEnv: checkEnvVar,
      getModelParams: () => ({
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        model: MODEL,
        temperature: 0,
        max_tokens: 200,
      }),
      getLLMConnection: () => ({
        secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_OPENAI_KEY!),
      }),
      timeoutMs: 30_000,
    });

    test("tool calling", async () => {
      checkEnvVar();

      const completion = await generateWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What's the weather like in Paris?",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "openai",
          adapter: LLMAdapter.OpenAI,
          model: MODEL,
          temperature: 0,
          max_tokens: 100,
        },
        tools: [weatherTool],
        llmConnection: {
          secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_OPENAI_KEY!),
        },
      });

      expect(completion.toolCalls.length).toBeGreaterThan(0);
      expect(completion.toolCalls[0].toolName).toBe("get_weather");
      expect(completion.toolCalls[0].input).toHaveProperty("location");
    }, 30_000);
  });

  describe("Anthropic", () => {
    const MODEL = "claude-sonnet-4-6";

    const checkEnvVar = () => {
      if (!process.env.LANGFUSE_LLM_CONNECTION_ANTHROPIC_KEY) {
        throw new Error(
          "LANGFUSE_LLM_CONNECTION_ANTHROPIC_KEY not set. " +
            "This test requires a valid Anthropic API key to verify the LLM connection. " +
            "Set the environment variable to run this test.",
        );
      }
    };

    test("simple completion", async () => {
      checkEnvVar();

      const completion = await generateWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What is 2+2? Answer only with the number.",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "anthropic",
          adapter: LLMAdapter.Anthropic,
          model: MODEL,
          temperature: 0,
          max_tokens: 10,
        },
        llmConnection: {
          secretKey: encrypt(
            process.env.LANGFUSE_LLM_CONNECTION_ANTHROPIC_KEY!,
          ),
        },
      });

      expect(completion.text).toContain("4");
    }, 30_000);

    test("streaming completion", async () => {
      checkEnvVar();

      const stream = await streamWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What is 2+2? Answer only with the number.",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "anthropic",
          adapter: LLMAdapter.Anthropic,
          model: MODEL,
          temperature: 0,
          max_tokens: 10,
        },
        llmConnection: {
          secretKey: encrypt(
            process.env.LANGFUSE_LLM_CONNECTION_ANTHROPIC_KEY!,
          ),
        },
      });

      let fullResponse = "";
      let chunkCount = 0;

      for await (const chunk of stream.textStream) {
        fullResponse += chunk;
        chunkCount++;
      }

      expect(chunkCount).toBeGreaterThan(0);
      expect(fullResponse).toContain("4");
    }, 30_000);

    registerMessageFormatTests({
      checkEnv: checkEnvVar,
      getModelParams: () => ({
        provider: "anthropic",
        adapter: LLMAdapter.Anthropic,
        model: MODEL,
        temperature: 0,
        max_tokens: 50,
      }),
      getLLMConnection: () => ({
        secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_ANTHROPIC_KEY!),
      }),
      timeoutMs: 30_000,
    });

    registerEvalStructuredOutputTests({
      checkEnv: checkEnvVar,
      getModelParams: () => ({
        provider: "anthropic",
        adapter: LLMAdapter.Anthropic,
        model: MODEL,
        temperature: 0,
        max_tokens: 200,
      }),
      getLLMConnection: () => ({
        secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_ANTHROPIC_KEY!),
      }),
      timeoutMs: 30_000,
    });

    test("tool calling", async () => {
      checkEnvVar();

      const completion = await generateWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What's the weather like in Paris?",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "anthropic",
          adapter: LLMAdapter.Anthropic,
          model: MODEL,
          temperature: 0,
          max_tokens: 100,
        },
        tools: [weatherTool],
        llmConnection: {
          secretKey: encrypt(
            process.env.LANGFUSE_LLM_CONNECTION_ANTHROPIC_KEY!,
          ),
        },
      });

      expect(completion.toolCalls.length).toBeGreaterThan(0);
      expect(completion.toolCalls[0].toolName).toBe("get_weather");
      expect(completion.toolCalls[0].input).toHaveProperty("location");
    }, 30_000);
  });

  // Skipped until CI has a valid Azure OpenAI key; the current secret is rejected with "Access denied".
  describe.skip("Azure", () => {
    const checkEnvVars = () => {
      if (!process.env.LANGFUSE_LLM_CONNECTION_AZURE_KEY) {
        throw new Error(
          "LANGFUSE_LLM_CONNECTION_AZURE_KEY not set. " +
            "This test requires a valid Azure OpenAI API key to verify the LLM connection. " +
            "Set the environment variable to run this test.",
        );
      }
      if (!process.env.LANGFUSE_LLM_CONNECTION_AZURE_BASE_URL) {
        throw new Error(
          "LANGFUSE_LLM_CONNECTION_AZURE_BASE_URL not set. " +
            "This test requires a valid Azure OpenAI base URL (deployment endpoint) to verify the LLM connection. " +
            "Set the environment variable to run this test.",
        );
      }
      if (!process.env.LANGFUSE_LLM_CONNECTION_AZURE_MODEL) {
        throw new Error(
          "LANGFUSE_LLM_CONNECTION_AZURE_MODEL not set. " +
            "This test requires a valid Azure OpenAI deployment name to verify the LLM connection. " +
            "Set the environment variable to run this test.",
        );
      }
    };

    test("simple completion", async () => {
      checkEnvVars();

      const completion = await generateWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What is 2+2? Answer only with the number.",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "azure",
          adapter: LLMAdapter.Azure,
          model: process.env.LANGFUSE_LLM_CONNECTION_AZURE_MODEL!,
          temperature: 0,
          max_tokens: 10,
        },
        llmConnection: {
          secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_AZURE_KEY!),
          baseURL: process.env.LANGFUSE_LLM_CONNECTION_AZURE_BASE_URL!,
        },
      });

      expect(completion.text).toContain("4");
    }, 30_000);

    test("streaming completion", async () => {
      checkEnvVars();

      const stream = await streamWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What is 2+2? Answer only with the number.",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "azure",
          adapter: LLMAdapter.Azure,
          model: process.env.LANGFUSE_LLM_CONNECTION_AZURE_MODEL!,
          temperature: 0,
          max_tokens: 10,
        },
        llmConnection: {
          secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_AZURE_KEY!),
          baseURL: process.env.LANGFUSE_LLM_CONNECTION_AZURE_BASE_URL!,
        },
      });

      let fullResponse = "";
      let chunkCount = 0;

      for await (const chunk of stream.textStream) {
        fullResponse += chunk;
        chunkCount++;
      }

      expect(chunkCount).toBeGreaterThan(0);
      expect(fullResponse).toContain("4");
    }, 60_000);

    registerMessageFormatTests({
      checkEnv: checkEnvVars,
      getModelParams: () => ({
        provider: "azure",
        adapter: LLMAdapter.Azure,
        model: process.env.LANGFUSE_LLM_CONNECTION_AZURE_MODEL!,
        temperature: 0,
        max_tokens: 50,
      }),
      getLLMConnection: () => ({
        secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_AZURE_KEY!),
        baseURL: process.env.LANGFUSE_LLM_CONNECTION_AZURE_BASE_URL!,
      }),
      timeoutMs: 60_000,
    });

    registerEvalStructuredOutputTests({
      checkEnv: checkEnvVars,
      getModelParams: () => ({
        provider: "azure",
        adapter: LLMAdapter.Azure,
        model: process.env.LANGFUSE_LLM_CONNECTION_AZURE_MODEL!,
        temperature: 0,
        max_tokens: 200,
      }),
      getLLMConnection: () => ({
        secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_AZURE_KEY!),
        baseURL: process.env.LANGFUSE_LLM_CONNECTION_AZURE_BASE_URL!,
      }),
      timeoutMs: 60_000,
    });

    test("tool calling", async () => {
      checkEnvVars();

      const completion = await generateWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What's the weather like in Paris?",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "azure",
          adapter: LLMAdapter.Azure,
          model: process.env.LANGFUSE_LLM_CONNECTION_AZURE_MODEL!,
          temperature: 0,
          max_tokens: 100,
        },
        tools: [weatherTool],
        llmConnection: {
          secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_AZURE_KEY!),
          baseURL: process.env.LANGFUSE_LLM_CONNECTION_AZURE_BASE_URL!,
        },
      });

      expect(completion.toolCalls.length).toBeGreaterThan(0);
      expect(completion.toolCalls[0].toolName).toBe("get_weather");
      expect(completion.toolCalls[0].input).toHaveProperty("location");
    }, 60_000);
  });

  describe("Bedrock", () => {
    const MODEL = "eu.anthropic.claude-sonnet-4-5-20250929-v1:0";

    const checkEnvVars = () => {
      if (!process.env.LANGFUSE_LLM_CONNECTION_BEDROCK_ACCESS_KEY_ID) {
        throw new Error(
          "LANGFUSE_LLM_CONNECTION_BEDROCK_ACCESS_KEY_ID not set. " +
            "This test requires a valid AWS access key ID to verify the Bedrock LLM connection. " +
            "Set the environment variable to run this test.",
        );
      }
      if (!process.env.LANGFUSE_LLM_CONNECTION_BEDROCK_SECRET_ACCESS_KEY) {
        throw new Error(
          "LANGFUSE_LLM_CONNECTION_BEDROCK_SECRET_ACCESS_KEY not set. " +
            "This test requires a valid AWS secret access key to verify the Bedrock LLM connection. " +
            "Set the environment variable to run this test.",
        );
      }
      if (!process.env.LANGFUSE_LLM_CONNECTION_BEDROCK_REGION) {
        throw new Error(
          "LANGFUSE_LLM_CONNECTION_BEDROCK_REGION not set. " +
            "This test requires a valid AWS region (e.g., 'us-east-1') to verify the Bedrock LLM connection. " +
            "Set the environment variable to run this test.",
        );
      }
    };

    const getApiKey = () => {
      checkEnvVars();
      return JSON.stringify({
        accessKeyId: process.env.LANGFUSE_LLM_CONNECTION_BEDROCK_ACCESS_KEY_ID!,
        secretAccessKey:
          process.env.LANGFUSE_LLM_CONNECTION_BEDROCK_SECRET_ACCESS_KEY!,
      });
    };

    const getConfig = () => {
      return {
        region: process.env.LANGFUSE_LLM_CONNECTION_BEDROCK_REGION!,
      };
    };

    test("simple completion", async () => {
      checkEnvVars();

      const completion = await generateWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What is 2+2? Answer only with the number.",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "bedrock",
          adapter: LLMAdapter.Bedrock,
          model: MODEL,
          temperature: 0,
          max_tokens: 10,
        },
        llmConnection: {
          secretKey: encrypt(getApiKey()),
          config: getConfig(),
        },
      });

      expect(completion.text).toContain("4");
    }, 30_000);

    test("streaming completion", async () => {
      checkEnvVars();

      const stream = await streamWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What is 2+2? Answer only with the number.",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "bedrock",
          adapter: LLMAdapter.Bedrock,
          model: MODEL,
          temperature: 0,
          max_tokens: 10,
        },
        llmConnection: {
          secretKey: encrypt(getApiKey()),
          config: getConfig(),
        },
      });

      let fullResponse = "";
      let chunkCount = 0;

      for await (const chunk of stream.textStream) {
        fullResponse += chunk;
        chunkCount++;
      }

      expect(chunkCount).toBeGreaterThan(0);
      expect(fullResponse).toContain("4");
    }, 30_000);

    registerMessageFormatTests({
      checkEnv: checkEnvVars,
      getModelParams: () => ({
        provider: "bedrock",
        adapter: LLMAdapter.Bedrock,
        model: MODEL,
        temperature: 0,
        max_tokens: 50,
      }),
      getLLMConnection: () => ({
        secretKey: encrypt(getApiKey()),
        config: getConfig(),
      }),
      timeoutMs: 30_000,
    });

    // Flaky
    registerEvalStructuredOutputTests({
      checkEnv: checkEnvVars,
      getModelParams: () => ({
        provider: "bedrock",
        adapter: LLMAdapter.Bedrock,
        model: MODEL,
        temperature: 0,
        max_tokens: 200,
      }),
      getLLMConnection: () => ({
        secretKey: encrypt(getApiKey()),
        config: getConfig(),
      }),
      timeoutMs: 30_000,
    });

    test("tool calling", async () => {
      checkEnvVars();

      const completion = await generateWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What's the weather like in Paris?",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "bedrock",
          adapter: LLMAdapter.Bedrock,
          model: MODEL,
          temperature: 0,
          max_tokens: 100,
        },
        tools: [weatherTool],
        llmConnection: {
          secretKey: encrypt(getApiKey()),
          config: getConfig(),
        },
      });

      expect(completion.toolCalls.length).toBeGreaterThan(0);
      expect(completion.toolCalls[0].toolName).toBe("get_weather");
      expect(completion.toolCalls[0].input).toHaveProperty("location");
    }, 30_000);
  });

  describe("Bedrock (API Key)", () => {
    const MODEL = "eu.anthropic.claude-sonnet-4-5-20250929-v1:0";

    const checkEnvVars = () => {
      if (!process.env.LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY) {
        throw new Error(
          "LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY not set. " +
            "This test requires a valid Bedrock API key to verify bearer-token auth. " +
            "Set the environment variable to run this test.",
        );
      }
      if (!process.env.LANGFUSE_LLM_CONNECTION_BEDROCK_REGION) {
        throw new Error(
          "LANGFUSE_LLM_CONNECTION_BEDROCK_REGION not set. " +
            "This test requires a valid AWS region (e.g., 'us-east-1') to verify the Bedrock LLM connection. " +
            "Set the environment variable to run this test.",
        );
      }
    };

    const getApiKey = () => {
      checkEnvVars();
      return JSON.stringify({
        apiKey: process.env.LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY!,
      });
    };

    const getConfig = () => ({
      region: process.env.LANGFUSE_LLM_CONNECTION_BEDROCK_REGION!,
    });

    test("simple completion", async () => {
      checkEnvVars();

      const completion = await generateWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What is 2+2? Answer only with the number.",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "bedrock",
          adapter: LLMAdapter.Bedrock,
          model: MODEL,
          temperature: 0,
          max_tokens: 10,
        },
        llmConnection: {
          secretKey: encrypt(getApiKey()),
          config: getConfig(),
        },
      });

      expect(completion.text).toContain("4");
    }, 30_000);

    test("streaming completion", async () => {
      checkEnvVars();

      const stream = await streamWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What is 2+2? Answer only with the number.",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "bedrock",
          adapter: LLMAdapter.Bedrock,
          model: MODEL,
          temperature: 0,
          max_tokens: 10,
        },
        llmConnection: {
          secretKey: encrypt(getApiKey()),
          config: getConfig(),
        },
      });

      let fullResponse = "";
      let chunkCount = 0;

      for await (const chunk of stream.textStream) {
        fullResponse += chunk;
        chunkCount++;
      }

      expect(chunkCount).toBeGreaterThan(0);
      expect(fullResponse).toContain("4");
    }, 30_000);
  });

  describe("VertexAI", () => {
    const MODEL = "gemini-2.5-flash-lite";

    const checkEnvVar = () => {
      if (!process.env.LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY) {
        throw new Error(
          "LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY not set. " +
            "This test requires a valid GCP service account JSON string to verify the VertexAI LLM connection. " +
            "Set the environment variable to run this test.",
        );
      }
    };

    test("simple completion", async () => {
      checkEnvVar();

      const completion = await generateWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What is 2+2? Answer only with the number.",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "google-vertex-ai",
          adapter: LLMAdapter.VertexAI,
          model: MODEL,
          temperature: 0,
          max_tokens: 10,
        },
        llmConnection: {
          secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY!),
          config: null,
        },
      });

      expect(completion.text).toContain("4");
    }, 30_000);

    test("Claude model routes through Anthropic publisher endpoint", async () => {
      checkEnvVar();

      const claudeVertexModel = "claude-3-haiku@20240307";

      try {
        const completion = await generateWithStoredConnection({
          messages: [
            {
              role: "user",
              content: "What is 2+2? Answer only with the number.",
              type: ChatMessageType.PublicAPICreated,
            },
          ],
          modelParams: {
            provider: "google-vertex-ai",
            adapter: LLMAdapter.VertexAI,
            model: claudeVertexModel,
            temperature: 0,
            max_tokens: 10,
          },
          llmConnection: {
            secretKey: encrypt(
              process.env.LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY!,
            ),
            config: { location: "us-east5" },
          },
        });

        expect(completion.text).toContain("4");
      } catch (error) {
        const message = String(error);

        // The CI project currently does not have access to Anthropic publisher
        // models. The regression signal is that Claude-on-Vertex reaches the
        // Anthropic publisher endpoint instead of /v1/v1/messages or Google's
        // Gemini/Gemma publisher path. Some provider errors only include the
        // normalized Vertex 404 without echoing the request URL.
        if (message.includes("/publishers/")) {
          expect(message).toContain(
            `/publishers/anthropic/models/${claudeVertexModel}`,
          );
          expect(message).not.toContain("/v1/v1/messages");
          expect(message).not.toContain("/publishers/google/");
        } else {
          expect(message).toContain("Not Found");
        }
      }
    }, 30_000);

    test("streaming completion", async () => {
      checkEnvVar();

      const stream = await streamWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What is 2+2? Answer only with the number.",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "google-vertex-ai",
          adapter: LLMAdapter.VertexAI,
          model: MODEL,
          temperature: 0,
          max_tokens: 10,
        },
        llmConnection: {
          secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY!),
          config: null,
        },
      });

      let fullResponse = "";
      let chunkCount = 0;

      for await (const chunk of stream.textStream) {
        fullResponse += chunk;
        chunkCount++;
      }

      expect(chunkCount).toBeGreaterThan(0);
      expect(fullResponse).toContain("4");
    }, 30_000);

    registerMessageFormatTests({
      checkEnv: checkEnvVar,
      getModelParams: () => ({
        provider: "google-vertex-ai",
        adapter: LLMAdapter.VertexAI,
        model: MODEL,
        temperature: 0,
        max_tokens: 50,
      }),
      getLLMConnection: () => ({
        secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY!),
        config: null,
      }),
      timeoutMs: 30_000,
    });

    registerEvalStructuredOutputTests({
      checkEnv: checkEnvVar,
      getModelParams: () => ({
        provider: "google-vertex-ai",
        adapter: LLMAdapter.VertexAI,
        model: MODEL,
        temperature: 0,
        max_tokens: 200,
      }),
      getLLMConnection: () => ({
        secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY!),
        config: null,
      }),
      timeoutMs: 30_000,
    });

    test("tool calling", async () => {
      checkEnvVar();

      const completion = await generateWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What's the weather like in Paris?",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "google-vertex-ai",
          adapter: LLMAdapter.VertexAI,
          model: MODEL,
          temperature: 0,
          max_tokens: 100,
        },
        tools: [weatherTool],
        llmConnection: {
          secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY!),
          config: null,
        },
      });

      expect(completion.toolCalls.length).toBeGreaterThan(0);
      expect(completion.toolCalls[0].toolName).toBe("get_weather");
      expect(completion.toolCalls[0].input).toHaveProperty("location");
    }, 30_000);

    test("thinking model with tool calling strips reasoning from content and parses tool calls", async () => {
      checkEnvVar();

      const completion = await generateWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What's the weather like in Paris?",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "google-vertex-ai",
          adapter: LLMAdapter.VertexAI,
          model: "gemini-2.5-flash",
          temperature: 0,
          max_tokens: 2048,
          maxReasoningTokens: 1024,
        },
        tools: [weatherTool],
        llmConnection: {
          secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY!),
          config: null,
        },
      });

      // Should parse tool calls successfully despite reasoning blocks in content
      expect(completion.toolCalls.length).toBeGreaterThan(0);
      expect(completion.toolCalls[0].toolName).toBe("get_weather");
      // Reasoning should be extracted separately
      if (completion.reasoningText) {
        expect(typeof completion.reasoningText).toBe("string");
      }
    }, 60_000);

    test("thinking model exposes text and reasoning separately", async () => {
      checkEnvVar();

      const completion = await generateWithStoredConnection({
        messages: [
          {
            role: "user",
            content: "What is 2+2? Answer only with the number.",
            type: ChatMessageType.PublicAPICreated,
          },
        ],
        modelParams: {
          provider: "google-vertex-ai",
          adapter: LLMAdapter.VertexAI,
          model: "gemini-2.5-flash",
          temperature: 0,
          max_tokens: 2048,
          maxReasoningTokens: 1024,
        },
        llmConnection: {
          secretKey: encrypt(process.env.LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY!),
          config: null,
        },
      });

      expect(completion.text).toContain("4");
      // With maxReasoningTokens > 0, reasoning should be present
      // Note: this depends on the model actually producing reasoning output
    }, 60_000);
  });

  describe("GoogleAIStudio", () => {
    const checkEnvVar = () => {
      if (!process.env.LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY) {
        throw new Error(
          "LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY not set. " +
            "This test requires a valid Google AI Studio API key to verify the LLM connection. " +
            "Set the environment variable to run this test.",
        );
      }
    };

    test("simple completion", async () => {
      checkEnvVar();

      const completion = await runWithGoogleAIStudioModelFallback((model) =>
        generateWithStoredConnection({
          messages: [
            {
              role: "user",
              content: "What is 2+2? Answer only with the number.",
              type: ChatMessageType.PublicAPICreated,
            },
          ],
          modelParams: {
            provider: "google-ai-studio",
            adapter: LLMAdapter.GoogleAIStudio,
            model,
            temperature: 0,
            max_tokens: 10,
          },
          llmConnection: {
            secretKey: encrypt(
              process.env.LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY!,
            ),
          },
        }),
      );

      expect(completion.text).toContain("4");
    }, 30_000);

    test("streaming completion", async () => {
      checkEnvVar();

      await runWithGoogleAIStudioModelFallback(async (model) => {
        const stream = await streamWithStoredConnection({
          messages: [
            {
              role: "user",
              content: "What is 2+2? Answer only with the number.",
              type: ChatMessageType.PublicAPICreated,
            },
          ],
          modelParams: {
            provider: "google-ai-studio",
            adapter: LLMAdapter.GoogleAIStudio,
            model,
            temperature: 0,
            max_tokens: 10,
          },
          llmConnection: {
            secretKey: encrypt(
              process.env.LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY!,
            ),
          },
        });

        let fullResponse = "";
        let chunkCount = 0;

        for await (const chunk of stream.textStream) {
          fullResponse += chunk;
          chunkCount++;
        }

        expect(chunkCount).toBeGreaterThan(0);
        expect(fullResponse).toContain("4");
      });
    }, 30_000);

    registerMessageFormatTests({
      checkEnv: checkEnvVar,
      getModelParams: (model = googleAIStudioFallbackModels[0]) => ({
        provider: "google-ai-studio",
        adapter: LLMAdapter.GoogleAIStudio,
        model,
        temperature: 0,
        max_tokens: 50,
      }),
      getLLMConnection: () => ({
        secretKey: encrypt(
          process.env.LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY!,
        ),
      }),
      timeoutMs: 30_000,
      runWithModel: runWithGoogleAIStudioModelFallback,
    });

    registerEvalStructuredOutputTests({
      checkEnv: checkEnvVar,
      getModelParams: (model = googleAIStudioFallbackModels[0]) => ({
        provider: "google-ai-studio",
        adapter: LLMAdapter.GoogleAIStudio,
        model,
        temperature: 0,
        max_tokens: 200,
      }),
      getLLMConnection: () => ({
        secretKey: encrypt(
          process.env.LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY!,
        ),
      }),
      timeoutMs: 30_000,
      runWithModel: runWithGoogleAIStudioModelFallback,
    });

    test("tool calling", async () => {
      checkEnvVar();

      const completion = await runWithGoogleAIStudioModelFallback((model) =>
        generateWithStoredConnection({
          messages: [
            {
              role: "user",
              content: "What's the weather like in Paris?",
              type: ChatMessageType.PublicAPICreated,
            },
          ],
          modelParams: {
            provider: "google-ai-studio",
            adapter: LLMAdapter.GoogleAIStudio,
            model,
            temperature: 0,
            max_tokens: 100,
          },
          tools: [weatherTool],
          llmConnection: {
            secretKey: encrypt(
              process.env.LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY!,
            ),
          },
        }),
      );

      expect(completion.toolCalls.length).toBeGreaterThan(0);
      expect(completion.toolCalls[0].toolName).toBe("get_weather");
      expect(completion.toolCalls[0].input).toHaveProperty("location");
    }, 30_000);

    test("single system message is converted to user message", async () => {
      checkEnvVar();

      // Regression test: Text prompts create a single system message.
      // GoogleAIStudio must convert it to a user message to prevent:
      // "GenerateContentRequest.contents is not specified" error
      const completion = await runWithGoogleAIStudioModelFallback((model) =>
        generateWithStoredConnection({
          messages: [
            {
              role: "system",
              content: "What is 2+2? Answer only with the number.",
              type: ChatMessageType.System,
            },
          ],
          modelParams: {
            provider: "google-ai-studio",
            adapter: LLMAdapter.GoogleAIStudio,
            model,
            temperature: 0,
            max_tokens: 10,
          },
          llmConnection: {
            secretKey: encrypt(
              process.env.LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY!,
            ),
          },
        }),
      );

      expect(completion.text).toContain("4");
    }, 30_000);
  });
});