Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

39.7 KB
Raw Blame
import {
  BatchExportTableName,
  BatchActionType,
  BatchTableNames,
  BatchActionStatus,
  EvalTargetObject,
  EvalTemplateType,
} from "@langfuse/shared";
import { expect, describe, it } from "vitest";
import { v4 as uuidv4 } from "uuid";
import { handleBatchActionJob } from "../features/batchAction/handleBatchActionJob";
import {
  getDatabaseReadStreamPaginated,
  getTraceIdentifierStream,
} from "../features/database-read-stream/getDatabaseReadStream";
import {
  createOrgProjectAndApiKey,
  createTraceScore,
  createScoresCh,
  createTrace,
  createTracesCh,
  createEvent,
  createEventsCh,
  getScoresByIds,
  QueueJobs,
  QueueName,
  createDatasetRunItemsCh,
  createDatasetRunItem,
  createDatasetItem,
  createNewRedisInstance,
  getQueuePrefix,
  redisQueueRetryOptions,
  TQueueJobTypes,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import { Decimal } from "decimal.js";
import waitForExpect from "wait-for-expect";
import { Queue } from "bullmq";

const maybeDescribe =
  process.env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true"
    ? describe
    : describe.skip;

const withIsolatedCreateEvalQueue = async <T>(
  projectId: string,
  fn: (queue: Queue<TQueueJobTypes[QueueName.CreateEvalQueue]>) => Promise<T>,
) => {
  const queueName = `${QueueName.CreateEvalQueue}-${projectId}-${uuidv4()}`;
  const redis = createNewRedisInstance({
    enableOfflineQueue: false,
    ...redisQueueRetryOptions,
  });
  if (!redis) {
    throw new Error("Redis is not initialized");
  }

  const queue = new Queue<TQueueJobTypes[QueueName.CreateEvalQueue]>(
    queueName,
    {
      connection: redis,
      prefix: getQueuePrefix(queueName),
    },
  );

  try {
    return await fn(queue);
  } finally {
    try {
      await queue.obliterate({ force: true });
    } finally {
      try {
        await queue.close();
      } finally {
        redis.disconnect();
      }
    }
  }
};

const getCreateEvalQueueJobs = async (
  queue: Queue<TQueueJobTypes[QueueName.CreateEvalQueue]>,
) => {
  return await queue.getJobs([
    "waiting",
    "delayed",
    "paused",
    "prioritized",
    "active",
    "completed",
    "failed",
  ]);
};

const waitForCreateEvalQueueJobs = async ({
  queue,
  expectedLength,
}: {
  queue: Queue<TQueueJobTypes[QueueName.CreateEvalQueue]>;
  expectedLength: number;
}) => {
  let jobs: Awaited<ReturnType<typeof getCreateEvalQueueJobs>> = [];

  await waitForExpect(async () => {
    jobs = await getCreateEvalQueueJobs(queue);

    expect(jobs.map((job) => job.data.payload)).toHaveLength(expectedLength);
  }, 15_000);

  return jobs;
};

const createHistoricalEvaluationRule = async ({
  id,
  projectId,
  targetObject,
  type = EvalTemplateType.LLM_AS_JUDGE,
}: {
  id: string;
  projectId: string;
  targetObject: EvalTargetObject;
  type?: EvalTemplateType;
}) => {
  const evaluator = await prisma.evaluator.create({
    data: {
      projectId,
      name: "score",
      type,
      versions: { create: { version: 1 } },
    },
  });
  return prisma.evaluationRule.create({
    data: {
      id,
      projectId,
      name: "score",
      filter: [],
      sampling: new Decimal(1),
      delay: 0,
      targetObject,
      assignments: {
        create: { projectId, evaluatorId: evaluator.id },
      },
    },
  });
};

describe("select all test suite", () => {
  it("should schedule trace deletions via pending_deletions table", async () => {
    const { projectId } = await createOrgProjectAndApiKey();

    // Create test traces
    const traceIds = Array.from({ length: 2500 }).map(() => uuidv4());
    const traces = traceIds.map((id) =>
      createTrace({
        project_id: projectId,
        id,
        timestamp: new Date("2024-01-01").getTime(),
      }),
    );

    await createTracesCh(traces);

    const selectAllJob = {
      payload: {
        projectId,
        actionId: "trace-delete",
        tableName: BatchExportTableName.Traces,
        query: {
          filter: [],
          orderBy: { column: "id", order: "DESC" },
        },
        cutoffCreatedAt: new Date("2024-01-02"),
      },
    } as any;

    await handleBatchActionJob(selectAllJob);

    // Verify pending_deletions records were created for all traces
    const pendingDeletions = await prisma.pendingDeletion.findMany({
      where: {
        projectId,
        object: "trace",
      },
    });

    expect(pendingDeletions).toHaveLength(2500);
    expect(pendingDeletions.every((pd) => pd.isDeleted === false)).toBe(true);

    // Verify all trace IDs are scheduled for deletion
    const scheduledTraceIds = pendingDeletions.map((pd) => pd.objectId).sort();
    expect(scheduledTraceIds).toEqual(traceIds.sort());
  }, 30000);

  it("should schedule only filtered traces for deletion", async () => {
    const { projectId } = await createOrgProjectAndApiKey();

    const traceId1 = uuidv4();
    const traceId2 = uuidv4();
    const traces = [
      createTrace({
        project_id: projectId,
        id: traceId1,
        user_id: "user1",
        timestamp: new Date("2024-01-01").getTime(),
      }),
      createTrace({
        project_id: projectId,
        id: traceId2,
        user_id: "user2",
        timestamp: new Date("2024-01-01").getTime(),
      }),
    ];

    await createTracesCh(traces);

    const selectAllJob = {
      payload: {
        projectId,
        actionId: "trace-delete",
        tableName: BatchExportTableName.Traces,
        query: {
          filter: [
            {
              type: "string",
              operator: "=",
              column: "User ID",
              value: "user1",
            },
          ],
          orderBy: { column: "timestamp", order: "DESC" },
        },
        cutoffCreatedAt: new Date("2024-01-02"),
      },
    } as any;

    await handleBatchActionJob(selectAllJob);

    // Verify only the filtered trace was scheduled for deletion
    const pendingDeletions = await prisma.pendingDeletion.findMany({
      where: {
        projectId,
        object: "trace",
      },
    });

    expect(pendingDeletions).toHaveLength(1);
    expect(pendingDeletions[0].objectId).toBe(traceId1);
    expect(pendingDeletions[0].isDeleted).toBe(false);
  });

  it("should handle score deletions", async () => {
    // Setup
    const { projectId } = await createOrgProjectAndApiKey();

    const score = createTraceScore({ project_id: projectId });
    await createScoresCh([score]);

    // When
    await handleBatchActionJob({
      id: uuidv4(),
      timestamp: new Date(),
      name: QueueJobs.BatchActionProcessingJob as const,
      payload: {
        projectId,
        actionId: "score-delete",
        tableName: BatchExportTableName.Scores,
        cutoffCreatedAt: new Date(),
        query: {
          filter: null,
          orderBy: { column: "timestamp", order: "DESC" },
        },
        type: BatchActionType.Delete,
      },
    });

    // Then
    const scores = await getScoresByIds(projectId, [score.id]);
    expect(scores).toHaveLength(0);
  });

  it("should delete only datasets matching path and search query", async () => {
    const { projectId } = await createOrgProjectAndApiKey();

    await prisma.dataset.createMany({
      data: [
        {
          id: uuidv4(),
          projectId,
          name: "folder/match",
          createdAt: new Date("2024-01-01"),
        },
        {
          id: uuidv4(),
          projectId,
          name: "folder/other",
          createdAt: new Date("2024-01-01"),
        },
        {
          id: uuidv4(),
          projectId,
          name: "other/match",
          createdAt: new Date("2024-01-01"),
        },
      ],
    });

    await handleBatchActionJob({
      id: uuidv4(),
      timestamp: new Date(),
      name: QueueJobs.BatchActionProcessingJob as const,
      payload: {
        projectId,
        actionId: "dataset-delete",
        tableName: BatchExportTableName.Datasets,
        cutoffCreatedAt: new Date("2024-01-02"),
        query: {
          filter: null,
          orderBy: { column: "createdAt", order: "DESC" },
          searchQuery: "match",
          pathPrefix: "folder",
        },
        type: BatchActionType.Delete,
      },
    });

    await expect(
      prisma.dataset.findMany({
        where: { projectId },
        select: { name: true },
        orderBy: { name: "asc" },
      }),
    ).resolves.toEqual([{ name: "folder/other" }, { name: "other/match" }]);
  });

  it("should schedule only traces matching search query for deletion", async () => {
    const { projectId } = await createOrgProjectAndApiKey();

    const traceId1 = uuidv4();
    const traceId2 = uuidv4();
    const traces = [
      createTrace({
        project_id: projectId,
        id: traceId1,
        name: "search-target-trace",
        timestamp: new Date("2024-01-01").getTime(),
      }),
      createTrace({
        project_id: projectId,
        id: traceId2,
        name: "other-trace",
        timestamp: new Date("2024-01-01").getTime(),
      }),
    ];

    await createTracesCh(traces);

    const selectAllJob = {
      payload: {
        projectId,
        actionId: "trace-delete",
        tableName: BatchExportTableName.Traces,
        query: {
          filter: [],
          orderBy: { column: "timestamp", order: "DESC" },
          searchQuery: "search-target",
          searchType: ["id"],
        },
        cutoffCreatedAt: new Date("2024-01-02"),
        type: BatchActionType.Delete,
      },
    } as any;

    await handleBatchActionJob(selectAllJob);

    // Verify only the matching trace was scheduled for deletion
    const pendingDeletions = await prisma.pendingDeletion.findMany({
      where: {
        projectId,
        object: "trace",
      },
    });

    expect(pendingDeletions).toHaveLength(1);
    expect(pendingDeletions[0].objectId).toBe(traceId1);
    expect(pendingDeletions[0].isDeleted).toBe(false);
  });

  it("should create eval jobs for historic traces", async () => {
    const { projectId } = await createOrgProjectAndApiKey();

    const traceTimestamp = new Date("2024-01-01T00:00:00.000Z");
    const cutoffCreatedAt = new Date("2024-01-02T00:00:00.000Z"); // one day later
    const traceId1 = uuidv4();
    const traces = [
      createTrace({
        project_id: projectId,
        id: traceId1,
        user_id: "user1",
        timestamp: traceTimestamp.getTime(),
      }),
      createTrace({
        project_id: projectId,
        id: uuidv4(),
        user_id: "user2",
        timestamp: traceTimestamp.getTime(),
      }),
    ];

    await createTracesCh(traces);

    const configId = uuidv4();
    await createHistoricalEvaluationRule({
      id: configId,
      projectId,
      targetObject: EvalTargetObject.TRACE,
    });

    const payload = {
      id: uuidv4(),
      timestamp: new Date(),
      name: QueueJobs.BatchActionProcessingJob as const,
      payload: {
        projectId,
        actionId: "eval-create" as const,
        targetObject: EvalTargetObject.TRACE,
        configId,
        cutoffCreatedAt,
        query: {
          filter: [
            {
              type: "string" as const,
              value: "1",
              column: "User ID",
              operator: "contains" as const,
            },
          ],
          orderBy: {
            column: "timestamp",
            order: "DESC" as const,
          },
        },
      },
    };

    await waitForExpect(async () => {
      const traceStream = await getTraceIdentifierStream({
        projectId,
        cutoffCreatedAt,
        filter: payload.payload.query.filter,
        orderBy: payload.payload.query.orderBy,
      });

      const traceIds: string[] = [];
      for await (const trace of traceStream) {
        traceIds.push(trace.id);
      }

      expect(traceIds).toEqual([traceId1]);
    }, 15_000);

    await withIsolatedCreateEvalQueue(projectId, async (queue) => {
      await handleBatchActionJob(payload, { evalCreatorQueue: queue });

      const jobs = await waitForCreateEvalQueueJobs({
        queue,
        expectedLength: 1,
      });

      const job = jobs[0];

      if (!job) {
        throw new Error("No jobs found");
      }

      expect(job.data.payload.projectId).toBe(projectId);
      expect(job.data.payload.traceId).toBe(traceId1);
      expect(job.data.payload.configId).toBe(configId);
    });
  }, 30_000);

  it("should create eval jobs for historic datasets", async () => {
    const { projectId } = await createOrgProjectAndApiKey();

    const traceTimestamp = new Date("2024-01-01T00:00:00.000Z");
    const datasetRunItemTimestamp = new Date("2024-01-01T00:00:00.000Z");
    const cutoffCreatedAt = new Date("2024-01-02T00:00:00.000Z"); // one day later
    const traceId1 = uuidv4();
    const traceId2 = uuidv4();

    const traces = [
      createTrace({
        project_id: projectId,
        id: traceId1,
        user_id: "user1",
        timestamp: traceTimestamp.getTime(),
      }),
      createTrace({
        project_id: projectId,
        id: traceId2,
        user_id: "user2",
        timestamp: traceTimestamp.getTime(),
      }),
    ];

    await createTracesCh(traces);

    const datasetName = uuidv4();
    const dataset = await prisma.dataset.create({
      data: {
        id: uuidv4(),
        projectId,
        name: datasetName,
      },
    });

    const res1 = await createDatasetItem({
      projectId,
      datasetId: dataset.id,
      input: "Hello, world!",
    });

    const res2 = await createDatasetItem({
      projectId,
      datasetId: dataset.id,
      input: "Hello, world!",
    });

    if (!res1.success || !res2.success) {
      throw new Error("Failed to create dataset item");
    }
    const datasetItem1 = res1.datasetItem;
    const datasetItem2 = res2.datasetItem;

    const runId = uuidv4();

    await prisma.datasetRuns.create({
      data: {
        id: runId,
        datasetId: dataset.id,
        projectId,
        name: "test-run",
      },
    });

    const datasetRunItem1 = createDatasetRunItem({
      id: uuidv4(),
      dataset_item_id: datasetItem1.id,
      project_id: projectId,
      trace_id: traceId1,
      dataset_run_id: runId,
      dataset_id: dataset.id,
      dataset_run_created_at: datasetRunItemTimestamp.getTime(),
      created_at: datasetRunItemTimestamp.getTime(),
      updated_at: datasetRunItemTimestamp.getTime(),
      event_ts: datasetRunItemTimestamp.getTime(),
    });

    const datasetRunItem2 = createDatasetRunItem({
      id: uuidv4(),
      dataset_item_id: datasetItem2.id,
      project_id: projectId,
      trace_id: traceId2,
      dataset_run_id: runId,
      dataset_id: dataset.id,
      dataset_run_created_at: datasetRunItemTimestamp.getTime(),
      created_at: datasetRunItemTimestamp.getTime(),
      updated_at: datasetRunItemTimestamp.getTime(),
      event_ts: datasetRunItemTimestamp.getTime(),
    });

    await createDatasetRunItemsCh([datasetRunItem1, datasetRunItem2]);

    const configId = uuidv4();
    await createHistoricalEvaluationRule({
      id: configId,
      projectId,
      targetObject: EvalTargetObject.DATASET,
    });

    const payload = {
      id: uuidv4(),
      timestamp: new Date(),
      name: QueueJobs.BatchActionProcessingJob as const,
      payload: {
        projectId,
        actionId: "eval-create" as const,
        targetObject: EvalTargetObject.DATASET,
        configId,
        cutoffCreatedAt,
        query: {
          filter: [
            {
              type: "stringOptions" as const,
              value: [dataset.id],
              column: "Dataset",
              operator: "any of" as const,
            },
          ],
          orderBy: {
            column: "timestamp",
            order: "DESC" as const,
          },
        },
      },
    };

    await waitForExpect(async () => {
      const dbReadStream = await getDatabaseReadStreamPaginated({
        projectId,
        cutoffCreatedAt,
        filter: payload.payload.query.filter,
        orderBy: payload.payload.query.orderBy,
        tableName: BatchTableNames.DatasetRunItems,
      });

      const datasetRunItems: unknown[] = [];
      for await (const item of dbReadStream) {
        datasetRunItems.push(item);
      }

      expect(datasetRunItems).toHaveLength(2);
    }, 15_000);

    await withIsolatedCreateEvalQueue(projectId, async (queue) => {
      await handleBatchActionJob(payload, { evalCreatorQueue: queue });

      const jobs = await waitForCreateEvalQueueJobs({
        queue,
        expectedLength: 2,
      });

      const jobTraceIds = jobs.map((job) => job.data.payload.traceId);
      expect(jobTraceIds).toContain(traceId1);
      expect(jobTraceIds).toContain(traceId2);

      const jobDatasetIds = jobs.map((job) => job.data.payload.datasetItemId);
      expect(jobDatasetIds).toContain(datasetItem1.id);
      expect(jobDatasetIds).toContain(datasetItem2.id);
      const configIds = jobs.map((job) => job.data.payload.configId);
      expect(configIds).toContain(configId);
    });
  }, 30_000);

  it("should not create evals if config does not exist", async () => {
    const { projectId } = await createOrgProjectAndApiKey();

    // Create a trace
    const traceId = uuidv4();
    const traceTimestamp = new Date("2024-01-01T00:00:00.000Z");
    await createTracesCh([
      createTrace({
        project_id: projectId,
        id: traceId,
        timestamp: traceTimestamp.getTime(),
      }),
    ]);

    // Use a non-existent config ID
    const nonExistentConfigId = uuidv4();

    const payload = {
      id: uuidv4(),
      timestamp: new Date(),
      name: QueueJobs.BatchActionProcessingJob as const,
      payload: {
        projectId,
        actionId: "eval-create" as const,
        targetObject: EvalTargetObject.TRACE,
        configId: nonExistentConfigId,
        cutoffCreatedAt: new Date("2024-01-02T00:00:00.000Z"),
        query: {
          filter: [],
          orderBy: {
            column: "timestamp",
            order: "DESC" as const,
          },
        },
      },
    };

    await withIsolatedCreateEvalQueue(projectId, async (queue) => {
      await expect(
        handleBatchActionJob(payload, { evalCreatorQueue: queue }),
      ).resolves.not.toThrow();

      const jobs = await waitForCreateEvalQueueJobs({
        queue,
        expectedLength: 0,
      });

      expect(jobs).toHaveLength(0);
    });
  });

  it("should not schedule a legacy config after its migrated rule is gone", async () => {
    const { projectId } = await createOrgProjectAndApiKey();
    const traceId = uuidv4();
    await createTracesCh([
      createTrace({
        project_id: projectId,
        id: traceId,
        timestamp: new Date("2024-01-01").getTime(),
      }),
    ]);

    const template = await prisma.evalTemplate.create({
      data: {
        projectId,
        name: "legacy-template",
        version: 1,
        prompt: "{{input}}",
        outputDefinition: { type: "numeric" },
      },
    });
    const config = await prisma.jobConfiguration.create({
      data: {
        projectId,
        filter: [],
        jobType: "EVAL",
        delay: 0,
        sampling: new Decimal(1),
        targetObject: EvalTargetObject.TRACE,
        scoreName: "legacy-score",
        variableMapping: [],
        evalTemplateId: template.id,
      },
    });

    const payload = {
      id: uuidv4(),
      timestamp: new Date(),
      name: QueueJobs.BatchActionProcessingJob as const,
      payload: {
        projectId,
        actionId: "eval-create" as const,
        targetObject: EvalTargetObject.TRACE,
        configId: config.id,
        cutoffCreatedAt: new Date("2024-01-02"),
        query: {
          filter: [],
          orderBy: { column: "timestamp", order: "DESC" as const },
        },
      },
    };

    await withIsolatedCreateEvalQueue(projectId, async (queue) => {
      await handleBatchActionJob(payload, { evalCreatorQueue: queue });

      const jobs = await waitForCreateEvalQueueJobs({
        queue,
        expectedLength: 0,
      });
      expect(jobs).toHaveLength(0);
    });
  });

  it("should skip eval-create for code evaluators", async () => {
    const { projectId } = await createOrgProjectAndApiKey();

    const traceId = uuidv4();
    const traceTimestamp = new Date("2024-01-01T00:00:00.000Z");
    await createTracesCh([
      createTrace({
        project_id: projectId,
        id: traceId,
        timestamp: traceTimestamp.getTime(),
      }),
    ]);

    const configId = uuidv4();
    await createHistoricalEvaluationRule({
      id: configId,
      projectId,
      targetObject: EvalTargetObject.TRACE,
      type: EvalTemplateType.CODE,
    });

    const payload = {
      id: uuidv4(),
      timestamp: new Date(),
      name: QueueJobs.BatchActionProcessingJob as const,
      payload: {
        projectId,
        actionId: "eval-create" as const,
        targetObject: EvalTargetObject.TRACE,
        configId,
        cutoffCreatedAt: new Date("2024-01-02T00:00:00.000Z"),
        query: {
          filter: [],
          orderBy: {
            column: "timestamp",
            order: "DESC" as const,
          },
        },
      },
    };

    await withIsolatedCreateEvalQueue(projectId, async (queue) => {
      await handleBatchActionJob(payload, { evalCreatorQueue: queue });

      const jobs = await waitForCreateEvalQueueJobs({
        queue,
        expectedLength: 0,
      });

      expect(jobs).toHaveLength(0);
    });
  });

  it("should add traces to annotation queue", async () => {
    const { projectId } = await createOrgProjectAndApiKey();

    const traceId1 = uuidv4();
    const traceId2 = uuidv4();
    const traces = [
      createTrace({
        project_id: projectId,
        id: traceId1,
        timestamp: new Date("2024-01-01").getTime(),
      }),
      createTrace({
        project_id: projectId,
        id: traceId2,
        timestamp: new Date("2024-01-01").getTime(),
      }),
    ];

    await createTracesCh(traces);

    const queueId = uuidv4();
    await prisma.annotationQueue.create({
      data: {
        id: queueId,
        projectId,
        name: "test-queue",
      },
    });

    await handleBatchActionJob({
      id: uuidv4(),
      timestamp: new Date(),
      name: QueueJobs.BatchActionProcessingJob as const,
      payload: {
        projectId,
        actionId: "trace-add-to-annotation-queue" as const,
        tableName: BatchExportTableName.Traces,
        cutoffCreatedAt: new Date("2024-01-02"),
        targetId: queueId,
        query: { filter: [], orderBy: { column: "timestamp", order: "DESC" } },
        type: BatchActionType.Create,
      },
    });

    const queueItems = await prisma.annotationQueueItem.findMany({
      where: { queueId, projectId },
    });

    expect(queueItems).toHaveLength(2);
    const objectIds = queueItems.map((item) => item.objectId);
    expect(objectIds).toContain(traceId1);
    expect(objectIds).toContain(traceId2);
    expect(queueItems.every((item) => item.objectType === "TRACE")).toBe(true);
  });

  it("should not add traces to an annotation queue from another project", async () => {
    const { projectId } = await createOrgProjectAndApiKey();
    const { projectId: otherProjectId } = await createOrgProjectAndApiKey();

    const traceId = uuidv4();
    await createTracesCh([
      createTrace({
        project_id: projectId,
        id: traceId,
        timestamp: new Date("2024-01-01").getTime(),
      }),
    ]);

    const otherQueueId = uuidv4();
    await prisma.annotationQueue.create({
      data: {
        id: otherQueueId,
        projectId: otherProjectId,
        name: "other-queue",
      },
    });

    await handleBatchActionJob({
      id: uuidv4(),
      timestamp: new Date(),
      name: QueueJobs.BatchActionProcessingJob as const,
      payload: {
        projectId,
        actionId: "trace-add-to-annotation-queue" as const,
        tableName: BatchExportTableName.Traces,
        cutoffCreatedAt: new Date("2024-01-02"),
        targetId: otherQueueId,
        query: { filter: [], orderBy: { column: "timestamp", order: "DESC" } },
        type: BatchActionType.Create,
      },
    });

    const queueItems = await prisma.annotationQueueItem.findMany({
      where: { queueId: otherQueueId },
    });
    expect(queueItems).toHaveLength(0);
  });

  it("should add sessions to annotation queue", async () => {
    const { projectId } = await createOrgProjectAndApiKey();

    const sessionId1 = uuidv4();
    const sessionId2 = uuidv4();

    // Create traces with sessions
    const traces = [
      createTrace({
        project_id: projectId,
        id: uuidv4(),
        session_id: sessionId1,
        timestamp: new Date("2024-01-01").getTime(),
      }),
      createTrace({
        project_id: projectId,
        id: uuidv4(),
        session_id: sessionId2,
        timestamp: new Date("2024-01-01").getTime(),
      }),
    ];

    await createTracesCh(traces);

    const queueId = uuidv4();
    await prisma.annotationQueue.create({
      data: {
        id: queueId,
        projectId,
        name: "test-queue",
      },
    });

    await handleBatchActionJob({
      id: uuidv4(),
      timestamp: new Date(),
      name: QueueJobs.BatchActionProcessingJob as const,
      payload: {
        projectId,
        actionId: "session-add-to-annotation-queue" as const,
        tableName: BatchExportTableName.Sessions,
        cutoffCreatedAt: new Date("2024-01-02"),
        targetId: queueId,
        query: { filter: [], orderBy: { column: "createdAt", order: "DESC" } },
        type: BatchActionType.Create,
      },
    });

    const queueItems = await prisma.annotationQueueItem.findMany({
      where: { queueId, projectId },
    });

    expect(queueItems).toHaveLength(2);
    const objectIds = queueItems.map((item) => item.objectId);
    expect(objectIds).toContain(sessionId1);
    expect(objectIds).toContain(sessionId2);
    expect(queueItems.every((item) => item.objectType === "SESSION")).toBe(
      true,
    );
  });
});

maybeDescribe("events table batch actions", () => {
  it("should add sessions to annotation queue from events table when useEventsTable is true", async () => {
    const { projectId } = await createOrgProjectAndApiKey();

    const sessionId1 = uuidv4();
    const sessionId2 = uuidv4();

    await prisma.traceSession.createMany({
      data: [
        { id: sessionId1, projectId },
        { id: sessionId2, projectId },
      ],
    });

    const now = Date.now() * 1000; // Events use microseconds

    // Sessions exist only in the events table, so this fails if the
    // useEventsTable flag is not passed through to the read stream.
    const events = [
      createEvent({
        project_id: projectId,
        trace_id: uuidv4(),
        session_id: sessionId1,
        type: "GENERATION",
        start_time: now,
        end_time: now + 1000000,
      }),
      createEvent({
        project_id: projectId,
        trace_id: uuidv4(),
        session_id: sessionId2,
        type: "GENERATION",
        start_time: now,
        end_time: now + 1000000,
      }),
    ];

    await createEventsCh(events);

    const queueId = uuidv4();
    await prisma.annotationQueue.create({
      data: {
        id: queueId,
        projectId,
        name: "test-queue-events",
      },
    });

    await handleBatchActionJob({
      id: uuidv4(),
      timestamp: new Date(),
      name: QueueJobs.BatchActionProcessingJob as const,
      payload: {
        projectId,
        actionId: "session-add-to-annotation-queue" as const,
        tableName: BatchExportTableName.Sessions,
        cutoffCreatedAt: new Date(Date.now() + 1000 * 60 * 60 * 24),
        targetId: queueId,
        query: {
          filter: [],
          orderBy: { column: "createdAt", order: "DESC" },
          useEventsTable: true,
        },
        type: BatchActionType.Create,
      },
    });

    const queueItems = await prisma.annotationQueueItem.findMany({
      where: { queueId, projectId },
    });

    expect(queueItems).toHaveLength(2);
    const objectIds = queueItems.map((item) => item.objectId);
    expect(objectIds).toContain(sessionId1);
    expect(objectIds).toContain(sessionId2);
    expect(queueItems.every((item) => item.objectType === "SESSION")).toBe(
      true,
    );
  });

  it("should add observations to dataset from events table with full mapping", async () => {
    const { projectId } = await createOrgProjectAndApiKey();

    const traceId = uuidv4();
    const trace = createTrace({
      project_id: projectId,
      id: traceId,
      timestamp: new Date().getTime(),
    });
    await createTracesCh([trace]);

    const eventInput1 = { prompt: "Hello, how are you?" };
    const eventOutput1 = { response: "I'm fine, thank you!" };
    const eventInput2 = { prompt: "What is 2+2?" };
    const eventOutput2 = { response: "4" };

    const event1 = createEvent({
      project_id: projectId,
      trace_id: traceId,
      input: eventInput1,
      output: eventOutput1,
      metadata_names: ["source"],
      metadata_values: ["test"],
    });
    const event2 = createEvent({
      project_id: projectId,
      trace_id: traceId,
      input: eventInput2,
      output: eventOutput2,
      metadata_names: ["source"],
      metadata_values: ["test"],
    });

    await createEventsCh([event1, event2]);

    const datasetName = uuidv4();
    const dataset = await prisma.dataset.create({
      data: {
        id: uuidv4(),
        projectId,
        name: datasetName,
      },
    });

    const batchAction = await prisma.batchAction.create({
      data: {
        projectId,
        userId: "test-user",
        actionType: "observation-add-to-dataset",
        tableName: BatchTableNames.Events,
        status: BatchActionStatus.Queued,
        query: { filter: [], orderBy: null },
      },
    });

    await handleBatchActionJob({
      id: uuidv4(),
      timestamp: new Date(),
      name: QueueJobs.BatchActionProcessingJob as const,
      payload: {
        batchActionId: batchAction.id,
        projectId,
        actionId: "observation-add-to-dataset" as const,
        tableName: BatchTableNames.Events,
        cutoffCreatedAt: new Date(),
        query: { filter: [], orderBy: null },
        config: {
          datasetId: dataset.id,
          datasetName: dataset.name,
          mapping: {
            input: { mode: "full" as const },
            expectedOutput: { mode: "full" as const },
            metadata: { mode: "none" as const },
          },
        },
        type: BatchActionType.Create,
      },
    });

    const datasetItems = await prisma.datasetItem.findMany({
      where: { datasetId: dataset.id },
    });

    expect(datasetItems).toHaveLength(2);

    const eventSpanIds = [event1.span_id, event2.span_id];
    for (const item of datasetItems) {
      expect(eventSpanIds).toContain(item.sourceObservationId);
      expect(item.sourceTraceId).toBe(traceId);
      expect(item.metadata).toBeNull();
    }

    // Verify each item's input/output matches the corresponding event
    const item1 = datasetItems.find(
      (i) => i.sourceObservationId === event1.span_id,
    );
    const item2 = datasetItems.find(
      (i) => i.sourceObservationId === event2.span_id,
    );

    expect(item1?.input).toEqual(eventInput1);
    expect(item1?.expectedOutput).toEqual(eventOutput1);
    expect(item2?.input).toEqual(eventInput2);
    expect(item2?.expectedOutput).toEqual(eventOutput2);

    // Verify batch action status
    const updatedBatchAction = await prisma.batchAction.findUnique({
      where: { id: batchAction.id },
    });
    expect(updatedBatchAction?.status).toBe(BatchActionStatus.Completed);
    expect(updatedBatchAction?.processedCount).toBe(2);
  });

  it("should not add observations to dataset for a batch action from another project", async () => {
    const { projectId } = await createOrgProjectAndApiKey();
    const { projectId: otherProjectId } = await createOrgProjectAndApiKey();

    const traceId = uuidv4();
    await createTracesCh([
      createTrace({
        project_id: projectId,
        id: traceId,
        timestamp: new Date().getTime(),
      }),
    ]);
    await createEventsCh([
      createEvent({
        project_id: projectId,
        trace_id: traceId,
        input: { prompt: "Hello" },
        output: { response: "Hi" },
      }),
    ]);

    const dataset = await prisma.dataset.create({
      data: {
        id: uuidv4(),
        projectId,
        name: uuidv4(),
      },
    });

    const otherProjectBatchAction = await prisma.batchAction.create({
      data: {
        projectId: otherProjectId,
        userId: "test-user",
        actionType: "observation-add-to-dataset",
        tableName: BatchTableNames.Events,
        status: BatchActionStatus.Queued,
        query: { filter: [], orderBy: null },
      },
    });

    await expect(
      handleBatchActionJob({
        id: uuidv4(),
        timestamp: new Date(),
        name: QueueJobs.BatchActionProcessingJob as const,
        payload: {
          batchActionId: otherProjectBatchAction.id,
          projectId,
          actionId: "observation-add-to-dataset" as const,
          tableName: BatchTableNames.Events,
          cutoffCreatedAt: new Date(),
          query: { filter: [], orderBy: null },
          config: {
            datasetId: dataset.id,
            datasetName: dataset.name,
            mapping: {
              input: { mode: "full" as const },
              expectedOutput: { mode: "full" as const },
              metadata: { mode: "none" as const },
            },
          },
          type: BatchActionType.Create,
        },
      }),
    ).rejects.toThrow();

    const datasetItems = await prisma.datasetItem.findMany({
      where: { datasetId: dataset.id },
    });
    expect(datasetItems).toHaveLength(0);

    const unchangedBatchAction = await prisma.batchAction.findUniqueOrThrow({
      where: { id: otherProjectBatchAction.id },
    });
    expect(unchangedBatchAction.status).toBe(BatchActionStatus.Queued);
    expect(unchangedBatchAction.totalCount).toBeNull();
    expect(unchangedBatchAction.processedCount).toBeNull();
  });

  it("should apply jsonSelector mapping when adding events to dataset", async () => {
    const { projectId } = await createOrgProjectAndApiKey();

    const traceId = uuidv4();
    const trace = createTrace({
      project_id: projectId,
      id: traceId,
      timestamp: new Date().getTime(),
    });
    await createTracesCh([trace]);

    const eventInput = {
      messages: [
        { role: "system", content: "You are helpful." },
        { role: "user", content: "What is 2+2?" },
      ],
    };
    const eventOutput = {
      choices: [{ message: { content: "4", role: "assistant" } }],
    };
    const eventMetadata = { user_id: "user-123", session_id: "session-456" };

    const event = createEvent({
      project_id: projectId,
      trace_id: traceId,
      input: JSON.stringify(eventInput),
      output: JSON.stringify(eventOutput),
      metadata_names: Object.keys(eventMetadata),
      metadata_values: Object.values(eventMetadata),
    });

    await createEventsCh([event]);

    const datasetName = uuidv4();
    const dataset = await prisma.dataset.create({
      data: {
        id: uuidv4(),
        projectId,
        name: datasetName,
      },
    });

    const batchAction = await prisma.batchAction.create({
      data: {
        projectId,
        userId: "test-user",
        actionType: "observation-add-to-dataset",
        tableName: BatchTableNames.Events,
        status: BatchActionStatus.Queued,
        query: { filter: [], orderBy: null },
      },
    });

    await handleBatchActionJob({
      id: uuidv4(),
      timestamp: new Date(),
      name: QueueJobs.BatchActionProcessingJob as const,
      payload: {
        batchActionId: batchAction.id,
        projectId,
        actionId: "observation-add-to-dataset" as const,
        tableName: BatchTableNames.Events,
        cutoffCreatedAt: new Date(),
        query: { filter: [], orderBy: null },
        config: {
          datasetId: dataset.id,
          datasetName: dataset.name,
          mapping: {
            input: {
              mode: "custom" as const,
              custom: {
                type: "keyValueMap" as const,
                keyValueMapConfig: {
                  entries: [
                    {
                      id: "1",
                      key: "prompt",
                      sourceField: "input" as const,
                      value: "$.messages[1].content",
                    },
                    {
                      id: "2",
                      key: "system",
                      sourceField: "input" as const,
                      value: "$.messages[0].content",
                    },
                  ],
                },
              },
            },
            expectedOutput: {
              mode: "custom" as const,
              custom: {
                type: "root" as const,
                rootConfig: {
                  sourceField: "output" as const,
                  jsonPath: "$.choices[0].message.content",
                },
              },
            },
            metadata: {
              mode: "custom" as const,
              custom: {
                type: "keyValueMap" as const,
                keyValueMapConfig: {
                  entries: [
                    {
                      id: "3",
                      key: "user",
                      sourceField: "metadata" as const,
                      value: "$.user_id",
                    },
                  ],
                },
              },
            },
          },
        },
        type: BatchActionType.Create,
      },
    });

    const datasetItems = await prisma.datasetItem.findMany({
      where: { datasetId: dataset.id },
    });

    expect(datasetItems).toHaveLength(1);

    const item = datasetItems[0];
    expect(item.sourceObservationId).toBe(event.span_id);
    expect(item.sourceTraceId).toBe(traceId);
    expect(item.input).toEqual({
      prompt: "What is 2+2?",
      system: "You are helpful.",
    });
    // The jsonPath extracts the string "4" from output, but it's stored as a
    // JSON scalar in Postgres. Prisma deserializes the JSON column as a number.
    expect(item.expectedOutput).toBe(4);
    expect(item.metadata).toEqual({ user: "user-123" });

    // Verify batch action status
    const updatedBatchAction = await prisma.batchAction.findUnique({
      where: { id: batchAction.id },
    });
    expect(updatedBatchAction?.status).toBe(BatchActionStatus.Completed);
    expect(updatedBatchAction?.processedCount).toBe(1);
  });

  it("should add observations to annotation queue from events table", async () => {
    const { projectId } = await createOrgProjectAndApiKey();

    const traceId = uuidv4();
    const trace = createTrace({
      project_id: projectId,
      id: traceId,
      timestamp: new Date().getTime(),
    });
    await createTracesCh([trace]);

    const event1 = createEvent({
      project_id: projectId,
      trace_id: traceId,
      input: JSON.stringify({ prompt: "Hello" }),
      output: JSON.stringify({ response: "Hi" }),
    });
    const event2 = createEvent({
      project_id: projectId,
      trace_id: traceId,
      input: JSON.stringify({ prompt: "Goodbye" }),
      output: JSON.stringify({ response: "Bye" }),
    });

    await createEventsCh([event1, event2]);

    // Create annotation queue
    const queueId = uuidv4();
    await prisma.annotationQueue.create({
      data: {
        id: queueId,
        projectId,
        name: "test-queue",
      },
    });

    await handleBatchActionJob({
      id: uuidv4(),
      timestamp: new Date(),
      name: QueueJobs.BatchActionProcessingJob as const,
      payload: {
        projectId,
        actionId: "observation-add-to-annotation-queue" as const,
        tableName: BatchTableNames.Events,
        cutoffCreatedAt: new Date(),
        targetId: queueId,
        query: { filter: [], orderBy: null },
        type: BatchActionType.Create,
      },
    });

    const queueItems = await prisma.annotationQueueItem.findMany({
      where: { queueId, projectId },
    });

    expect(queueItems).toHaveLength(2);

    const eventSpanIds = [event1.span_id, event2.span_id];
    for (const item of queueItems) {
      expect(eventSpanIds).toContain(item.objectId);
      expect(item.objectType).toBe("OBSERVATION");
    }
  });
});