Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

14.6 KB
Raw Blame
import { expect, describe, it, afterEach, vi } from "vitest";
import { randomUUID } from "crypto";
import {
  BatchProjectCleaner,
  BATCH_DELETION_TABLES,
  BATCH_PROJECT_CLEANER_COUNT_LOCK_KEY,
  BATCH_PROJECT_CLEANER_LOCK_PREFIX,
} from "../features/batch-project-cleaner";
import {
  clickhouseClient,
  createOrgProjectAndApiKey,
  createTracesCh,
  createTrace,
  createDatasetRunItemsCh,
  createDatasetRunItem,
  queryClickhouse,
  redis,
  toClickhouseDateTime,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import { env } from "../env";
import type { RedisLock } from "../utils/RedisLock";

vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
  const actual =
    await importOriginal<typeof import("@langfuse/shared/src/server")>();
  return {
    ...actual,
    commandClickhouse: (opts: Parameters<typeof actual.commandClickhouse>[0]) =>
      actual.commandClickhouse({
        ...opts,
        clickhouseSettings: /^\s*DELETE FROM topic_facet_summaries\b/.test(
          opts.query,
        )
          ? {
              lightweight_delete_mode: "lightweight_update_force",
              ...opts.clickhouseSettings,
            }
          : opts.clickhouseSettings,
      }),
  };
});

type TestableBatchProjectCleaner = {
  countQueryLock: RedisLock;
  lock: RedisLock;
  getCountsAfterDeleteFailure: (
    projectIds: string[],
  ) => Promise<Map<string, number> | undefined>;
  getProjectCounts: (projectIds: string[]) => Promise<Map<string, number>>;
  markRunFailed: (error: unknown) => void;
};

async function getClickhouseCount(
  table: string,
  projectId: string,
): Promise<number> {
  const result = await queryClickhouse<{ count: number }>({
    query: `SELECT count() as count FROM ${table} FINAL WHERE project_id = {projectId: String}`,
    params: { projectId },
  });
  return Number(result[0]?.count ?? 0);
}

describe("BatchProjectCleaner", () => {
  const TEST_TABLE = "traces" as const;
  const TEST_LOCK_KEY = `${BATCH_PROJECT_CLEANER_LOCK_PREFIX}:${TEST_TABLE}`;

  // Clean up Redis locks after each test
  afterEach(async () => {
    await redis?.del(BATCH_PROJECT_CLEANER_COUNT_LOCK_KEY);
    for (const table of BATCH_DELETION_TABLES) {
      await redis?.del(`${BATCH_PROJECT_CLEANER_LOCK_PREFIX}:${table}`);
    }
  });

  describe("processBatch", () => {
    it("should return sleep interval when no deleted projects exist", async () => {
      // Clean up any soft-deleted projects from other tests
      await prisma.project.updateMany({
        where: { deletedAt: { not: null } },
        data: { deletedAt: null },
      });

      // Create an active project (not deleted)
      await createOrgProjectAndApiKey();

      const cleaner = new BatchProjectCleaner(TEST_TABLE);
      const nextDelayMs = await cleaner.processBatch();

      expect(nextDelayMs).toBe(
        env.LANGFUSE_BATCH_PROJECT_CLEANER_SLEEP_ON_EMPTY_MS,
      );

      // Verify lock was not taken
      const lockValue = await redis?.get(TEST_LOCK_KEY);
      expect(lockValue).toBeNull();
    });

    it("should return sleep interval when deleted project has no ClickHouse data", async () => {
      // Create and soft-delete a project
      const { projectId } = await createOrgProjectAndApiKey();
      await prisma.project.update({
        where: { id: projectId },
        data: { deletedAt: new Date() },
      });

      const cleaner = new BatchProjectCleaner(TEST_TABLE);
      const nextDelayMs = await cleaner.processBatch();

      expect(nextDelayMs).toBe(
        env.LANGFUSE_BATCH_PROJECT_CLEANER_SLEEP_ON_EMPTY_MS,
      );

      // Verify lock was released
      const lockValue = await redis?.get(TEST_LOCK_KEY);
      expect(lockValue).toBeNull();
    });

    it("should delete traces for soft-deleted project", async () => {
      // Create and soft-delete a project
      const { projectId } = await createOrgProjectAndApiKey();
      await prisma.project.update({
        where: { id: projectId },
        data: { deletedAt: new Date() },
      });

      // Insert test traces into ClickHouse
      await createTracesCh([
        createTrace({ id: randomUUID(), project_id: projectId }),
        createTrace({ id: randomUUID(), project_id: projectId }),
      ]);

      // Verify traces exist before deletion
      const countBefore = await getClickhouseCount(TEST_TABLE, projectId);
      expect(countBefore).toBe(2);

      // Run processBatch
      const cleaner = new BatchProjectCleaner(TEST_TABLE);
      const extendLockSpy = vi.spyOn(
        (cleaner as unknown as TestableBatchProjectCleaner).lock,
        "extend",
      );
      const nextDelayMs = await cleaner.processBatch();

      // Verify traces were deleted
      const countAfter = await getClickhouseCount(TEST_TABLE, projectId);
      expect(countAfter).toBe(0);
      expect(extendLockSpy).toHaveBeenCalledOnce();

      // Verify returned check interval (work was done)
      expect(nextDelayMs).toBe(
        env.LANGFUSE_BATCH_PROJECT_CLEANER_CHECK_INTERVAL_MS,
      );
    });

    it("should not affect traces from active projects", async () => {
      // Create two projects
      const { projectId: deletedProjectId } = await createOrgProjectAndApiKey();
      const { projectId: activeProjectId } = await createOrgProjectAndApiKey();

      // Soft-delete only one project
      await prisma.project.update({
        where: { id: deletedProjectId },
        data: { deletedAt: new Date() },
      });

      // Insert traces for both projects
      await createTracesCh([
        createTrace({ id: randomUUID(), project_id: deletedProjectId }),
        createTrace({ id: randomUUID(), project_id: deletedProjectId }),
        createTrace({ id: randomUUID(), project_id: activeProjectId }),
        createTrace({ id: randomUUID(), project_id: activeProjectId }),
        createTrace({ id: randomUUID(), project_id: activeProjectId }),
      ]);

      // Verify traces exist before deletion
      expect(await getClickhouseCount(TEST_TABLE, deletedProjectId)).toBe(2);
      expect(await getClickhouseCount(TEST_TABLE, activeProjectId)).toBe(3);

      // Run processBatch
      const cleaner = new BatchProjectCleaner(TEST_TABLE);
      await cleaner.processBatch();

      // Verify only deleted project's traces were removed
      expect(await getClickhouseCount(TEST_TABLE, deletedProjectId)).toBe(0);
      expect(await getClickhouseCount(TEST_TABLE, activeProjectId)).toBe(3);
    });

    it("deletes Topics summaries when patch deletes are forced by default without affecting active projects", async () => {
      const { projectId: deletedProjectId } = await createOrgProjectAndApiKey();
      const { projectId: activeProjectId } = await createOrgProjectAndApiKey();
      await prisma.project.update({
        where: { id: deletedProjectId },
        data: { deletedAt: new Date() },
      });
      await clickhouseClient().insert({
        table: "topic_facet_summaries",
        format: "JSONEachRow",
        values: [deletedProjectId, activeProjectId].map((projectId) => ({
          project_id: projectId,
          facet_id: "intent",
          facet_version: 1,
          trace_id: randomUUID(),
          environment: "default",
          unit_start_time: toClickhouseDateTime(new Date()),
          processing_state: "complete",
          summary: "A trace summary.",
        })),
      });
      expect(
        await getClickhouseCount("topic_facet_summaries", deletedProjectId),
      ).toBe(1);

      const cleaner = new BatchProjectCleaner("topic_facet_summaries");
      await cleaner.processBatch();

      expect(
        await getClickhouseCount("topic_facet_summaries", deletedProjectId),
      ).toBe(0);
      expect(
        await getClickhouseCount("topic_facet_summaries", activeProjectId),
      ).toBe(1);
    });

    it("should delete traces from multiple soft-deleted projects", async () => {
      // Create and soft-delete two projects
      const { projectId: projectId1 } = await createOrgProjectAndApiKey();
      const { projectId: projectId2 } = await createOrgProjectAndApiKey();

      await prisma.project.update({
        where: { id: projectId1 },
        data: { deletedAt: new Date() },
      });
      await prisma.project.update({
        where: { id: projectId2 },
        data: { deletedAt: new Date() },
      });

      // Insert traces for both projects
      await createTracesCh([
        createTrace({ id: randomUUID(), project_id: projectId1 }),
        createTrace({ id: randomUUID(), project_id: projectId2 }),
        createTrace({ id: randomUUID(), project_id: projectId2 }),
      ]);

      // Run processBatch
      const cleaner = new BatchProjectCleaner(TEST_TABLE);
      const nextDelayMs = await cleaner.processBatch();

      // Verify both projects' traces were deleted
      expect(await getClickhouseCount(TEST_TABLE, projectId1)).toBe(0);
      expect(await getClickhouseCount(TEST_TABLE, projectId2)).toBe(0);
      expect(nextDelayMs).toBe(
        env.LANGFUSE_BATCH_PROJECT_CLEANER_CHECK_INTERVAL_MS,
      );
    });

    it("should skip processing when lock is already held", async () => {
      // Create and soft-delete a project
      const { projectId } = await createOrgProjectAndApiKey();
      await prisma.project.update({
        where: { id: projectId },
        data: { deletedAt: new Date() },
      });

      // Insert traces
      await createTracesCh([
        createTrace({ id: randomUUID(), project_id: projectId }),
      ]);

      // Acquire the lock manually
      await redis?.set(TEST_LOCK_KEY, "locked", "EX", 3600, "NX");

      // Run processBatch - should skip because lock is held
      const cleaner = new BatchProjectCleaner(TEST_TABLE);
      const getProjectCountsSpy = vi.spyOn(
        cleaner as unknown as TestableBatchProjectCleaner,
        "getProjectCounts",
      );
      const nextDelayMs = await cleaner.processBatch();

      expect(getProjectCountsSpy).not.toHaveBeenCalled();

      // Verify traces were NOT deleted (lock blocked processing)
      expect(await getClickhouseCount(TEST_TABLE, projectId)).toBe(1);
      expect(nextDelayMs).toBe(
        env.LANGFUSE_BATCH_PROJECT_CLEANER_CHECK_INTERVAL_MS,
      );
    });

    it("should back off when the global count lock is held", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      await prisma.project.update({
        where: { id: projectId },
        data: { deletedAt: new Date() },
      });
      await redis?.set(
        BATCH_PROJECT_CLEANER_COUNT_LOCK_KEY,
        "locked",
        "EX",
        60,
        "NX",
      );

      const cleaner = new BatchProjectCleaner(TEST_TABLE);
      const getProjectCountsSpy = vi.spyOn(
        cleaner as unknown as TestableBatchProjectCleaner,
        "getProjectCounts",
      );
      const nextDelayMs = await cleaner.processBatch();

      expect(getProjectCountsSpy).not.toHaveBeenCalled();
      expect(nextDelayMs).toBeGreaterThanOrEqual(60_000);
      expect(nextDelayMs).toBeLessThan(120_000);
    });

    it("should not fail the run when the count lock cannot be released", async () => {
      const cleaner = new BatchProjectCleaner(TEST_TABLE);
      const testableCleaner = cleaner as unknown as TestableBatchProjectCleaner;
      const markRunFailedSpy = vi.spyOn(testableCleaner, "markRunFailed");

      expect(await testableCleaner.countQueryLock.acquire()).toBe("acquired");
      await redis?.del(BATCH_PROJECT_CLEANER_COUNT_LOCK_KEY);
      expect(await testableCleaner.countQueryLock.release()).toBe(false);

      expect(markRunFailedSpy).not.toHaveBeenCalled();
    });

    it("should bypass the count lock when rechecking a failed delete", async () => {
      const cleaner = new BatchProjectCleaner(TEST_TABLE);
      const testableCleaner = cleaner as unknown as TestableBatchProjectCleaner;
      const extendSpy = vi
        .spyOn(testableCleaner.lock, "extend")
        .mockResolvedValue(true);
      const acquireSpy = vi
        .spyOn(testableCleaner.countQueryLock, "acquire")
        .mockResolvedValue("held_by_other");
      const countSpy = vi
        .spyOn(testableCleaner, "getProjectCounts")
        .mockResolvedValue(new Map());

      await testableCleaner.getCountsAfterDeleteFailure(["project-id"]);

      expect(extendSpy).toHaveBeenCalledOnce();
      expect(acquireSpy).not.toHaveBeenCalled();
      expect(countSpy).toHaveBeenCalledWith(["project-id"]);
    });

    it("should release lock after processing completes", async () => {
      // Create and soft-delete a project
      const { projectId } = await createOrgProjectAndApiKey();
      await prisma.project.update({
        where: { id: projectId },
        data: { deletedAt: new Date() },
      });

      // Insert traces
      await createTracesCh([
        createTrace({ id: randomUUID(), project_id: projectId }),
      ]);

      // Run processBatch
      const cleaner = new BatchProjectCleaner(TEST_TABLE);
      await cleaner.processBatch();

      // Verify that traces were deleted, therefore processing occurred
      expect(await getClickhouseCount(TEST_TABLE, projectId)).toBe(0);

      // Verify lock was released
      const lockValue = await redis?.get(TEST_LOCK_KEY);
      expect(lockValue).toBeNull();
    });

    it("should delete dataset_run_items for soft-deleted project", async () => {
      const TABLE = "dataset_run_items_rmt" as const;

      // Create two projects
      const { projectId: deletedProjectId } = await createOrgProjectAndApiKey();
      const { projectId: activeProjectId } = await createOrgProjectAndApiKey();

      // Soft-delete only one project
      await prisma.project.update({
        where: { id: deletedProjectId },
        data: { deletedAt: new Date() },
      });

      // Insert dataset run items for both projects
      await createDatasetRunItemsCh([
        createDatasetRunItem({
          id: randomUUID(),
          project_id: deletedProjectId,
        }),
        createDatasetRunItem({
          id: randomUUID(),
          project_id: deletedProjectId,
        }),
        createDatasetRunItem({
          id: randomUUID(),
          project_id: activeProjectId,
        }),
        createDatasetRunItem({
          id: randomUUID(),
          project_id: activeProjectId,
        }),
        createDatasetRunItem({
          id: randomUUID(),
          project_id: activeProjectId,
        }),
      ]);

      // Verify items exist before deletion
      expect(await getClickhouseCount(TABLE, deletedProjectId)).toBe(2);
      expect(await getClickhouseCount(TABLE, activeProjectId)).toBe(3);

      // Run processBatch
      const cleaner = new BatchProjectCleaner(TABLE);
      await cleaner.processBatch();

      // Verify only deleted project's items were removed
      expect(await getClickhouseCount(TABLE, deletedProjectId)).toBe(0);
      expect(await getClickhouseCount(TABLE, activeProjectId)).toBe(3);
    });
  });
});