Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

18.8 KB
Raw Blame
import { expect, test, describe, beforeEach, vi, afterEach } from "vitest";
import { prisma } from "@langfuse/shared/src/db";
import { randomUUID } from "crypto";
import { LLMAdapter } from "@langfuse/shared";
import { encrypt } from "@langfuse/shared/encryption";
import { createExperimentJobClickhouse } from "../features/experiments/experimentServiceClickhouse";
import {
  createDatasetItem,
  createOrgProjectAndApiKey,
  generateLLMText,
  logger,
} from "@langfuse/shared/src/server";

// Mock the logger to capture log calls
vi.mock("@langfuse/shared/src/server", async () => {
  const actual = await vi.importActual("@langfuse/shared/src/server");
  return {
    ...actual,
    generateLLMText: vi.fn().mockResolvedValue({ text: "test output" }),
    logger: {
      info: vi.fn(),
      error: vi.fn(),
      warn: vi.fn(),
      debug: vi.fn(),
    },
  };
});

describe("create experiment jobs", () => {
  const mockLogger = vi.mocked(logger);

  beforeEach(async () => {
    vi.clearAllMocks();
  });

  afterEach(() => {
    vi.restoreAllMocks();
  });
  test("processes valid experiment without throwing", async () => {
    const { projectId } = await createOrgProjectAndApiKey();
    const datasetId = randomUUID();
    const runId = randomUUID();
    const promptId = randomUUID();

    // Create required prompt
    await prisma.prompt.create({
      data: {
        id: promptId,
        projectId,
        name: "Test Prompt",
        prompt: "Hello {{name}}",
        type: "text",
        version: 1,
        createdBy: "test-user",
      },
    });
    // Create dataset
    await prisma.dataset.create({
      data: {
        id: datasetId,
        projectId,
        name: "Test Dataset",
      },
    });

    // Create dataset run with metadata
    await prisma.datasetRuns.create({
      data: {
        id: runId,
        name: "Test Run",
        projectId: projectId,
        datasetId: datasetId,
        metadata: {
          prompt_id: promptId,
          provider: "openai",
          model: "gpt-3.5-turbo",
          model_params: { temperature: 0 },
        },
      },
    });

    // Create dataset item
    const res = await createDatasetItem({
      projectId,
      datasetId,
      input: { name: "World" },
    });
    // Create API key
    await prisma.llmApiKeys.create({
      data: {
        id: randomUUID(),
        projectId,
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        displaySecretKey: "test-key",
        secretKey: encrypt("test-key"),
      },
    });
    const payload = {
      projectId,
      datasetId,
      runId,
    };

    const result = await createExperimentJobClickhouse({ event: payload });

    // Just verify it doesn't throw and returns success
    expect(result).toEqual({ success: true });

    // Verify that processing info was logged
    expect(mockLogger.info).toHaveBeenCalledWith(
      "Processing experiment create job with ClickHouse batching",
      expect.objectContaining({
        projectId,
        datasetId,
        runId,
      }),
    );
  });

  test("handles experiment validation failure (missing prompt_id) without throwing", async () => {
    const { projectId } = await createOrgProjectAndApiKey();
    const datasetId = randomUUID();
    const runId = randomUUID();

    // Create dataset
    await prisma.dataset.create({
      data: {
        id: datasetId,
        projectId,
        name: "Test Dataset",
      },
    });

    // Create dataset run with invalid metadata (missing required fields)
    await prisma.datasetRuns.create({
      data: {
        id: runId,
        name: "Test Run",
        projectId: projectId,
        datasetId: datasetId,
        metadata: {
          provider: "invalid_provider",
          model: "invalid_model",
          // Missing prompt_id
        },
      },
    });

    // Create dataset item so there's something to create error run items for
    await createDatasetItem({
      projectId,
      datasetId,
      input: { name: "World" },
    });

    const payload = {
      projectId,
      datasetId,
      runId,
    };

    const result = await createExperimentJobClickhouse({ event: payload });

    // Verify it doesn't throw and returns success even with validation errors
    expect(result).toEqual({ success: true });

    // Verify that an error was logged during validation failure
    expect(mockLogger.error).toHaveBeenCalledWith(
      "Failed to validate and setup experiment",
      expect.any(Error),
    );
  });

  test("handles prompt with variables without throwing", async () => {
    const { projectId } = await createOrgProjectAndApiKey();
    const datasetId = randomUUID();
    const runId = randomUUID();
    const promptId = randomUUID();

    // Create dataset
    await prisma.dataset.create({
      data: {
        id: datasetId,
        projectId,
        name: "Test Dataset",
      },
    });

    // Create required prompt with variable
    await prisma.prompt.create({
      data: {
        id: promptId,
        projectId,
        name: "Test Prompt",
        prompt: "Hello {{name}}",
        type: "text",
        version: 1,
        createdBy: "test-user",
      },
    });

    // Create dataset run with metadata
    await prisma.datasetRuns.create({
      data: {
        id: runId,
        name: "Test Run",
        projectId: projectId,
        datasetId: datasetId,
        metadata: {
          prompt_id: promptId,
          provider: "openai",
          model: "gpt-3.5-turbo",
          model_params: { temperature: 0 },
        },
      },
    });

    // Create dataset item
    await createDatasetItem({
      projectId,
      datasetId,
      input: { name: "test" },
    });

    // Create API key
    await prisma.llmApiKeys.create({
      data: {
        id: randomUUID(),
        projectId,
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        displaySecretKey: "test-key",
        secretKey: encrypt("test-key"),
      },
    });

    const payload = {
      projectId,
      datasetId,
      runId,
    };

    const result = await createExperimentJobClickhouse({ event: payload });

    // Just verify it doesn't throw and returns success
    expect(result).toEqual({ success: true });
  });

  test("handles mismatched dataset variables without throwing", async () => {
    const { projectId } = await createOrgProjectAndApiKey();
    const datasetId = randomUUID();
    const runId = randomUUID();
    const promptId = randomUUID();
    // Create prompt with variable that won't match dataset item
    await prisma.prompt.create({
      data: {
        id: promptId,
        projectId,
        name: "Test Prompt",
        prompt: "Hello {{name}}",
        type: "text",
        version: 1,
        createdBy: "test-user",
      },
    });
    // Create dataset
    await prisma.dataset.create({
      data: {
        id: datasetId,
        projectId,
        name: "Test Dataset",
      },
    });

    // Create dataset run with metadata
    await prisma.datasetRuns.create({
      data: {
        id: runId,
        name: "Test Run",
        projectId: projectId,
        datasetId: datasetId,
        metadata: {
          prompt_id: promptId,
          provider: "openai",
          model: "gpt-3.5-turbo",
          model_params: { temperature: 0 },
        },
      },
    });

    // Create dataset item with mismatched variables
    await createDatasetItem({
      projectId,
      datasetId,
      input: { wrongVariable: "World" }, // doesn't have "name" variable
    });

    // Create API key
    await prisma.llmApiKeys.create({
      data: {
        id: randomUUID(),
        projectId,
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        displaySecretKey: "test-key",
        secretKey: encrypt("test-key"),
      },
    });
    const payload = {
      projectId,
      datasetId,
      runId,
    };

    const result = await createExperimentJobClickhouse({ event: payload });

    // Just verify it doesn't throw and returns success
    expect(result).toEqual({ success: true });
  });
});

describe("create experiment jobs with placeholders", () => {
  const mockLogger = vi.mocked(logger);

  beforeEach(async () => {
    vi.clearAllMocks();
  });

  const setupPlaceholderTest = async (
    promptConfig: any,
    datasetItemInput: any,
  ) => {
    const { projectId } = await createOrgProjectAndApiKey();
    const datasetId = randomUUID();
    const runId = randomUUID();
    const promptId = randomUUID();
    // Create prompt
    await prisma.prompt.create({
      data: {
        id: promptId,
        projectId,
        name: promptConfig.name,
        prompt: promptConfig.prompt,
        type: "chat",
        version: 1,
        createdBy: "test-user",
      },
    });
    // Create dataset
    await prisma.dataset.create({
      data: {
        id: datasetId,
        projectId,
        name: "Test Dataset",
      },
    });

    // Create dataset run with metadata
    await prisma.datasetRuns.create({
      data: {
        id: runId,
        name: "Test Run",
        projectId: projectId,
        datasetId: datasetId,
        metadata: {
          prompt_id: promptId,
          provider: "openai",
          model: "gpt-3.5-turbo",
          model_params: { temperature: 0 },
        },
      },
    });

    // Create dataset item
    await createDatasetItem({
      projectId,
      datasetId,
      input: datasetItemInput,
    });
    // Create API key
    await prisma.llmApiKeys.create({
      data: {
        id: randomUUID(),
        projectId,
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        displaySecretKey: "test-key",
        secretKey: encrypt("test-key"),
      },
    });
    return { projectId, datasetId, runId };
  };

  test("handles multiple placeholders without throwing", async () => {
    const { projectId, datasetId, runId } = await setupPlaceholderTest(
      {
        name: "Test Multiple Placeholders",
        prompt: [
          { role: "system", content: "You are a helpful assistant." },
          { type: "placeholder", name: "conversation_history" },
          { type: "placeholder", name: "user_context" },
          { role: "user", content: "Please help me." },
        ],
      },
      {
        conversation_history: [
          { role: "user", content: "Hello {{name}}!" },
          { role: "assistant", content: "Hi there!" },
        ],
        user_context: [{ role: "system", content: "User is a {{role}}" }],
        name: "John",
        role: "developer",
      },
    );
    const payload = {
      projectId,
      datasetId,
      runId,
    };

    const result = await createExperimentJobClickhouse({ event: payload });

    // Just verify it doesn't throw and returns success
    expect(result).toEqual({ success: true });
  });

  test("handles empty placeholder arrays without throwing", async () => {
    const { projectId, datasetId, runId } = await setupPlaceholderTest(
      {
        name: "Test Empty Placeholder",
        prompt: [
          { role: "system", content: "You are a helpful assistant." },
          { type: "placeholder", name: "empty_history" },
          { role: "user", content: "Start conversation." },
        ],
      },
      { empty_history: [] },
    );
    const payload = {
      projectId,
      datasetId,
      runId,
    };

    const result = await createExperimentJobClickhouse({ event: payload });

    // Just verify it doesn't throw and returns success
    expect(result).toEqual({ success: true });
  });

  test("handles invalid placeholder formats without throwing", async () => {
    const { projectId, datasetId, runId } = await setupPlaceholderTest(
      {
        name: "Test Invalid Placeholder",
        prompt: [
          { role: "system", content: "You are a helpful assistant." },
          { type: "placeholder", name: "invalid_messages" },
          { role: "user", content: "Help me." },
        ],
      },
      { invalid_messages: "this should be an array or object" },
    );
    const payload = {
      projectId,
      datasetId,
      runId,
    };

    const result = await createExperimentJobClickhouse({ event: payload });

    // Just verify it doesn't throw and returns success
    expect(result).toEqual({ success: true });
  });
});

describe("experiment processing integration", () => {
  beforeEach(async () => {
    vi.clearAllMocks();
  });

  test("processes experiment end-to-end without throwing", async () => {
    const { projectId } = await createOrgProjectAndApiKey();
    const datasetId = randomUUID();
    const runId = randomUUID();
    const promptId = randomUUID();

    // Create required prompt
    await prisma.prompt.create({
      data: {
        id: promptId,
        projectId,
        name: "Test Prompt",
        prompt: "Hello {{name}}",
        type: "text",
        version: 1,
        createdBy: "test-user",
      },
    });

    // Create dataset
    await prisma.dataset.create({
      data: {
        id: datasetId,
        projectId,
        name: "Test Dataset",
      },
    });

    // Create dataset run with metadata
    await prisma.datasetRuns.create({
      data: {
        id: runId,
        name: "Test Run",
        projectId: projectId,
        datasetId: datasetId,
        metadata: {
          prompt_id: promptId,
          provider: "openai",
          model: "gpt-3.5-turbo",
          model_params: { temperature: 0 },
        },
      },
    });

    // Create dataset item
    await createDatasetItem({
      projectId,
      datasetId,
      input: { name: "World" },
    });

    // Create API key
    await prisma.llmApiKeys.create({
      data: {
        id: randomUUID(),
        projectId,
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        displaySecretKey: "test-key",
        secretKey: encrypt("test-key"),
      },
    });

    const payload = {
      projectId,
      datasetId,
      runId,
    };

    const result = await createExperimentJobClickhouse({ event: payload });

    // Just verify it completes successfully
    expect(result).toEqual({ success: true });

    // TODO: add when we remove PG execution path
    // expect(callLLM).toHaveBeenCalledWith(
    //   expect.any(Object),
    //   expect.any(Array),
    //   expect.any(Object),
    //   expect.any(String),
    //   expect.any(String),
    //   expect.objectContaining({
    //     environment: "langfuse-prompt-experiment",
    //     traceName: expect.stringMatching(/^dataset-run-item-/),
    //     traceId: expect.any(String),
    //     projectId: projectId,
    //     authCheck: expect.objectContaining({
    //       validKey: true,
    //       scope: expect.objectContaining({
    //         projectId: projectId,
    //         accessLevel: "project",
    //       }),
    //     }),
    //   }),
    // );
  });
});

describe("experiment prompt tool config", () => {
  beforeEach(async () => {
    vi.clearAllMocks();
    vi.mocked(generateLLMText).mockResolvedValue({
      text: "test output",
    } as any);
  });

  const setupExperimentWithPromptConfig = async (
    promptConfig: unknown,
    runMetadata: Record<string, unknown> = {},
  ) => {
    const { projectId } = await createOrgProjectAndApiKey();
    const datasetId = randomUUID();
    const runId = randomUUID();
    const promptId = randomUUID();

    await prisma.prompt.create({
      data: {
        id: promptId,
        projectId,
        name: "Test Prompt",
        prompt: "Hello {{name}}",
        config: promptConfig as any,
        type: "text",
        version: 1,
        createdBy: "test-user",
      },
    });
    await prisma.dataset.create({
      data: { id: datasetId, projectId, name: "Test Dataset" },
    });
    await prisma.datasetRuns.create({
      data: {
        id: runId,
        name: "Test Run",
        projectId,
        datasetId,
        metadata: {
          prompt_id: promptId,
          provider: "openai",
          model: "gpt-3.5-turbo",
          model_params: { temperature: 0 },
          ...runMetadata,
        },
      },
    });
    await createDatasetItem({
      projectId,
      datasetId,
      input: { name: "World" },
    });
    await prisma.llmApiKeys.create({
      data: {
        id: randomUUID(),
        projectId,
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        displaySecretKey: "test-key",
        secretKey: encrypt("test-key"),
      },
    });

    return { projectId, datasetId, runId };
  };

  test("passes valid prompt config tools to the LLM call", async () => {
    const { projectId, datasetId, runId } =
      await setupExperimentWithPromptConfig({
        tools: [
          {
            name: "get_weather",
            parameters: {
              type: "object",
              properties: { location: { type: "string" } },
              required: ["location"],
            },
          },
          // OpenAI-wrapped shape must be accepted as well
          {
            type: "function",
            function: {
              name: "get_time",
              description: "Get the current time",
            },
          },
        ],
      });

    const result = await createExperimentJobClickhouse({
      event: { projectId, datasetId, runId },
    });

    expect(result).toEqual({ success: true });
    expect(vi.mocked(generateLLMText)).toHaveBeenCalledWith(
      expect.objectContaining({
        maxRetries: 1,
        tools: expect.objectContaining({
          get_weather: expect.anything(),
          get_time: expect.anything(),
        }),
      }),
    );
  });

  test("ignores invalid prompt config tools and runs without tools", async () => {
    const { projectId, datasetId, runId } =
      await setupExperimentWithPromptConfig({
        tools: [{ name: "broken_tool", parameters: "invalid" }],
      });

    const result = await createExperimentJobClickhouse({
      event: { projectId, datasetId, runId },
    });

    expect(result).toEqual({ success: true });
    expect(vi.mocked(generateLLMText)).toHaveBeenCalledTimes(1);
    const callArgs = vi.mocked(generateLLMText).mock.calls[0][0];
    expect(callArgs.tools).toBeUndefined();
  });

  test("rejects combining prompt config tools with structured output", async () => {
    const { projectId, datasetId, runId } =
      await setupExperimentWithPromptConfig(
        {
          tools: [
            {
              name: "get_weather",
              description: "Get the weather for a location",
              parameters: { type: "object", properties: {} },
            },
          ],
        },
        {
          structured_output_schema: {
            type: "object",
            properties: { answer: { type: "string" } },
          },
        },
      );

    const result = await createExperimentJobClickhouse({
      event: { projectId, datasetId, runId },
    });

    expect(result).toEqual({ success: true });
    expect(vi.mocked(generateLLMText)).not.toHaveBeenCalled();
    expect(vi.mocked(logger).error).toHaveBeenCalledWith(
      "Failed to validate and setup experiment",
      expect.objectContaining({
        message:
          "Your prompt contains tool definitions - tool calls are not compatible with structured output",
      }),
    );
  });
});