🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { expect, describe, it, afterEach, vi } from "vitest";
import { randomUUID } from "crypto";
import {
BatchProjectCleaner,
BATCH_DELETION_TABLES,
BATCH_PROJECT_CLEANER_COUNT_LOCK_KEY,
BATCH_PROJECT_CLEANER_LOCK_PREFIX,
} from "../features/batch-project-cleaner";
import {
clickhouseClient,
createOrgProjectAndApiKey,
createTracesCh,
createTrace,
createDatasetRunItemsCh,
createDatasetRunItem,
queryClickhouse,
redis,
toClickhouseDateTime,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import { env } from "../env";
import type { RedisLock } from "../utils/RedisLock";
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
const actual =
await importOriginal<typeof import("@langfuse/shared/src/server")>();
return {
...actual,
commandClickhouse: (opts: Parameters<typeof actual.commandClickhouse>[0]) =>
actual.commandClickhouse({
...opts,
clickhouseSettings: /^\s*DELETE FROM topic_facet_summaries\b/.test(
opts.query,
)
? {
lightweight_delete_mode: "lightweight_update_force",
...opts.clickhouseSettings,
}
: opts.clickhouseSettings,
}),
};
});
type TestableBatchProjectCleaner = {
countQueryLock: RedisLock;
lock: RedisLock;
getCountsAfterDeleteFailure: (
projectIds: string[],
) => Promise<Map<string, number> | undefined>;
getProjectCounts: (projectIds: string[]) => Promise<Map<string, number>>;
markRunFailed: (error: unknown) => void;
};
async function getClickhouseCount(
table: string,
projectId: string,
): Promise<number> {
const result = await queryClickhouse<{ count: number }>({
query: `SELECT count() as count FROM ${table} FINAL WHERE project_id = {projectId: String}`,
params: { projectId },
});
return Number(result[0]?.count ?? 0);
}
describe("BatchProjectCleaner", () => {
const TEST_TABLE = "traces" as const;
const TEST_LOCK_KEY = `${BATCH_PROJECT_CLEANER_LOCK_PREFIX}:${TEST_TABLE}`;
// Clean up Redis locks after each test
afterEach(async () => {
await redis?.del(BATCH_PROJECT_CLEANER_COUNT_LOCK_KEY);
for (const table of BATCH_DELETION_TABLES) {
await redis?.del(`${BATCH_PROJECT_CLEANER_LOCK_PREFIX}:${table}`);
}
});
describe("processBatch", () => {
it("should return sleep interval when no deleted projects exist", async () => {
// Clean up any soft-deleted projects from other tests
await prisma.project.updateMany({
where: { deletedAt: { not: null } },
data: { deletedAt: null },
});
// Create an active project (not deleted)
await createOrgProjectAndApiKey();
const cleaner = new BatchProjectCleaner(TEST_TABLE);
const nextDelayMs = await cleaner.processBatch();
expect(nextDelayMs).toBe(
env.LANGFUSE_BATCH_PROJECT_CLEANER_SLEEP_ON_EMPTY_MS,
);
// Verify lock was not taken
const lockValue = await redis?.get(TEST_LOCK_KEY);
expect(lockValue).toBeNull();
});
it("should return sleep interval when deleted project has no ClickHouse data", async () => {
// Create and soft-delete a project
const { projectId } = await createOrgProjectAndApiKey();
await prisma.project.update({
where: { id: projectId },
data: { deletedAt: new Date() },
});
const cleaner = new BatchProjectCleaner(TEST_TABLE);
const nextDelayMs = await cleaner.processBatch();
expect(nextDelayMs).toBe(
env.LANGFUSE_BATCH_PROJECT_CLEANER_SLEEP_ON_EMPTY_MS,
);
// Verify lock was released
const lockValue = await redis?.get(TEST_LOCK_KEY);
expect(lockValue).toBeNull();
});
it("should delete traces for soft-deleted project", async () => {
// Create and soft-delete a project
const { projectId } = await createOrgProjectAndApiKey();
await prisma.project.update({
where: { id: projectId },
data: { deletedAt: new Date() },
});
// Insert test traces into ClickHouse
await createTracesCh([
createTrace({ id: randomUUID(), project_id: projectId }),
createTrace({ id: randomUUID(), project_id: projectId }),
]);
// Verify traces exist before deletion
const countBefore = await getClickhouseCount(TEST_TABLE, projectId);
expect(countBefore).toBe(2);
// Run processBatch
const cleaner = new BatchProjectCleaner(TEST_TABLE);
const extendLockSpy = vi.spyOn(
(cleaner as unknown as TestableBatchProjectCleaner).lock,
"extend",
);
const nextDelayMs = await cleaner.processBatch();
// Verify traces were deleted
const countAfter = await getClickhouseCount(TEST_TABLE, projectId);
expect(countAfter).toBe(0);
expect(extendLockSpy).toHaveBeenCalledOnce();
// Verify returned check interval (work was done)
expect(nextDelayMs).toBe(
env.LANGFUSE_BATCH_PROJECT_CLEANER_CHECK_INTERVAL_MS,
);
});
it("should not affect traces from active projects", async () => {
// Create two projects
const { projectId: deletedProjectId } = await createOrgProjectAndApiKey();
const { projectId: activeProjectId } = await createOrgProjectAndApiKey();
// Soft-delete only one project
await prisma.project.update({
where: { id: deletedProjectId },
data: { deletedAt: new Date() },
});
// Insert traces for both projects
await createTracesCh([
createTrace({ id: randomUUID(), project_id: deletedProjectId }),
createTrace({ id: randomUUID(), project_id: deletedProjectId }),
createTrace({ id: randomUUID(), project_id: activeProjectId }),
createTrace({ id: randomUUID(), project_id: activeProjectId }),
createTrace({ id: randomUUID(), project_id: activeProjectId }),
]);
// Verify traces exist before deletion
expect(await getClickhouseCount(TEST_TABLE, deletedProjectId)).toBe(2);
expect(await getClickhouseCount(TEST_TABLE, activeProjectId)).toBe(3);
// Run processBatch
const cleaner = new BatchProjectCleaner(TEST_TABLE);
await cleaner.processBatch();
// Verify only deleted project's traces were removed
expect(await getClickhouseCount(TEST_TABLE, deletedProjectId)).toBe(0);
expect(await getClickhouseCount(TEST_TABLE, activeProjectId)).toBe(3);
});
it("deletes Topics summaries when patch deletes are forced by default without affecting active projects", async () => {
const { projectId: deletedProjectId } = await createOrgProjectAndApiKey();
const { projectId: activeProjectId } = await createOrgProjectAndApiKey();
await prisma.project.update({
where: { id: deletedProjectId },
data: { deletedAt: new Date() },
});
await clickhouseClient().insert({
table: "topic_facet_summaries",
format: "JSONEachRow",
values: [deletedProjectId, activeProjectId].map((projectId) => ({
project_id: projectId,
facet_id: "intent",
facet_version: 1,
trace_id: randomUUID(),
environment: "default",
unit_start_time: toClickhouseDateTime(new Date()),
processing_state: "complete",
summary: "A trace summary.",
})),
});
expect(
await getClickhouseCount("topic_facet_summaries", deletedProjectId),
).toBe(1);
const cleaner = new BatchProjectCleaner("topic_facet_summaries");
await cleaner.processBatch();
expect(
await getClickhouseCount("topic_facet_summaries", deletedProjectId),
).toBe(0);
expect(
await getClickhouseCount("topic_facet_summaries", activeProjectId),
).toBe(1);
});
it("should delete traces from multiple soft-deleted projects", async () => {
// Create and soft-delete two projects
const { projectId: projectId1 } = await createOrgProjectAndApiKey();
const { projectId: projectId2 } = await createOrgProjectAndApiKey();
await prisma.project.update({
where: { id: projectId1 },
data: { deletedAt: new Date() },
});
await prisma.project.update({
where: { id: projectId2 },
data: { deletedAt: new Date() },
});
// Insert traces for both projects
await createTracesCh([
createTrace({ id: randomUUID(), project_id: projectId1 }),
createTrace({ id: randomUUID(), project_id: projectId2 }),
createTrace({ id: randomUUID(), project_id: projectId2 }),
]);
// Run processBatch
const cleaner = new BatchProjectCleaner(TEST_TABLE);
const nextDelayMs = await cleaner.processBatch();
// Verify both projects' traces were deleted
expect(await getClickhouseCount(TEST_TABLE, projectId1)).toBe(0);
expect(await getClickhouseCount(TEST_TABLE, projectId2)).toBe(0);
expect(nextDelayMs).toBe(
env.LANGFUSE_BATCH_PROJECT_CLEANER_CHECK_INTERVAL_MS,
);
});
it("should skip processing when lock is already held", async () => {
// Create and soft-delete a project
const { projectId } = await createOrgProjectAndApiKey();
await prisma.project.update({
where: { id: projectId },
data: { deletedAt: new Date() },
});
// Insert traces
await createTracesCh([
createTrace({ id: randomUUID(), project_id: projectId }),
]);
// Acquire the lock manually
await redis?.set(TEST_LOCK_KEY, "locked", "EX", 3600, "NX");
// Run processBatch - should skip because lock is held
const cleaner = new BatchProjectCleaner(TEST_TABLE);
const getProjectCountsSpy = vi.spyOn(
cleaner as unknown as TestableBatchProjectCleaner,
"getProjectCounts",
);
const nextDelayMs = await cleaner.processBatch();
expect(getProjectCountsSpy).not.toHaveBeenCalled();
// Verify traces were NOT deleted (lock blocked processing)
expect(await getClickhouseCount(TEST_TABLE, projectId)).toBe(1);
expect(nextDelayMs).toBe(
env.LANGFUSE_BATCH_PROJECT_CLEANER_CHECK_INTERVAL_MS,
);
});
it("should back off when the global count lock is held", async () => {
const { projectId } = await createOrgProjectAndApiKey();
await prisma.project.update({
where: { id: projectId },
data: { deletedAt: new Date() },
});
await redis?.set(
BATCH_PROJECT_CLEANER_COUNT_LOCK_KEY,
"locked",
"EX",
60,
"NX",
);
const cleaner = new BatchProjectCleaner(TEST_TABLE);
const getProjectCountsSpy = vi.spyOn(
cleaner as unknown as TestableBatchProjectCleaner,
"getProjectCounts",
);
const nextDelayMs = await cleaner.processBatch();
expect(getProjectCountsSpy).not.toHaveBeenCalled();
expect(nextDelayMs).toBeGreaterThanOrEqual(60_000);
expect(nextDelayMs).toBeLessThan(120_000);
});
it("should not fail the run when the count lock cannot be released", async () => {
const cleaner = new BatchProjectCleaner(TEST_TABLE);
const testableCleaner = cleaner as unknown as TestableBatchProjectCleaner;
const markRunFailedSpy = vi.spyOn(testableCleaner, "markRunFailed");
expect(await testableCleaner.countQueryLock.acquire()).toBe("acquired");
await redis?.del(BATCH_PROJECT_CLEANER_COUNT_LOCK_KEY);
expect(await testableCleaner.countQueryLock.release()).toBe(false);
expect(markRunFailedSpy).not.toHaveBeenCalled();
});
it("should bypass the count lock when rechecking a failed delete", async () => {
const cleaner = new BatchProjectCleaner(TEST_TABLE);
const testableCleaner = cleaner as unknown as TestableBatchProjectCleaner;
const extendSpy = vi
.spyOn(testableCleaner.lock, "extend")
.mockResolvedValue(true);
const acquireSpy = vi
.spyOn(testableCleaner.countQueryLock, "acquire")
.mockResolvedValue("held_by_other");
const countSpy = vi
.spyOn(testableCleaner, "getProjectCounts")
.mockResolvedValue(new Map());
await testableCleaner.getCountsAfterDeleteFailure(["project-id"]);
expect(extendSpy).toHaveBeenCalledOnce();
expect(acquireSpy).not.toHaveBeenCalled();
expect(countSpy).toHaveBeenCalledWith(["project-id"]);
});
it("should release lock after processing completes", async () => {
// Create and soft-delete a project
const { projectId } = await createOrgProjectAndApiKey();
await prisma.project.update({
where: { id: projectId },
data: { deletedAt: new Date() },
});
// Insert traces
await createTracesCh([
createTrace({ id: randomUUID(), project_id: projectId }),
]);
// Run processBatch
const cleaner = new BatchProjectCleaner(TEST_TABLE);
await cleaner.processBatch();
// Verify that traces were deleted, therefore processing occurred
expect(await getClickhouseCount(TEST_TABLE, projectId)).toBe(0);
// Verify lock was released
const lockValue = await redis?.get(TEST_LOCK_KEY);
expect(lockValue).toBeNull();
});
it("should delete dataset_run_items for soft-deleted project", async () => {
const TABLE = "dataset_run_items_rmt" as const;
// Create two projects
const { projectId: deletedProjectId } = await createOrgProjectAndApiKey();
const { projectId: activeProjectId } = await createOrgProjectAndApiKey();
// Soft-delete only one project
await prisma.project.update({
where: { id: deletedProjectId },
data: { deletedAt: new Date() },
});
// Insert dataset run items for both projects
await createDatasetRunItemsCh([
createDatasetRunItem({
id: randomUUID(),
project_id: deletedProjectId,
}),
createDatasetRunItem({
id: randomUUID(),
project_id: deletedProjectId,
}),
createDatasetRunItem({
id: randomUUID(),
project_id: activeProjectId,
}),
createDatasetRunItem({
id: randomUUID(),
project_id: activeProjectId,
}),
createDatasetRunItem({
id: randomUUID(),
project_id: activeProjectId,
}),
]);
// Verify items exist before deletion
expect(await getClickhouseCount(TABLE, deletedProjectId)).toBe(2);
expect(await getClickhouseCount(TABLE, activeProjectId)).toBe(3);
// Run processBatch
const cleaner = new BatchProjectCleaner(TABLE);
await cleaner.processBatch();
// Verify only deleted project's items were removed
expect(await getClickhouseCount(TABLE, deletedProjectId)).toBe(0);
expect(await getClickhouseCount(TABLE, activeProjectId)).toBe(3);
});
});
});