Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(retention): batch discovery projects by exact cutoff (#19149)

Valery Meleshkindda45b21996f
diff --git a/worker/src/__tests__/batchDataRetentionCleaner.test.ts b/worker/src/__tests__/batchDataRetentionCleaner.test.ts
index 299be01ae0..26b6a4ea45 100644
--- a/worker/src/__tests__/batchDataRetentionCleaner.test.ts
+++ b/worker/src/__tests__/batchDataRetentionCleaner.test.ts
@@ -604,7 +604,7 @@ describe("BatchDataRetentionCleaner", () => {
]);
});
- it("groups discovery by exact cutoff while preserving deletion boundaries", async () => {
+ it("discovers separate cutoffs while preserving deletion boundaries", async () => {
const cutoff = new Date("2026-01-15T12:34:56.123Z");
const projectIds = [
"project-'quoted\\id",
@@ -630,7 +630,7 @@ describe("BatchDataRetentionCleaner", () => {
lock: { extend: () => Promise<boolean> };
findExpiredProjectCandidates: (
timestampColumn: string,
- inputProjects: typeof projects,
+ chunk: { projects: typeof projects; cutoff: string },
) => Promise<{ candidates: typeof projects; complete: boolean }>;
executeBatchDelete: (
timestampColumn: string,
@@ -643,30 +643,49 @@ describe("BatchDataRetentionCleaner", () => {
) => Promise<void>;
};
cleaner.lock.extend = async () => true;
- const result = await cleaner.findExpiredProjectCandidates(
- "start_time",
- projects,
- );
+ const results = [];
+ for (const chunk of [
+ {
+ projects: projects.slice(0, 2),
+ cutoff: "2026-01-15 12:34:56.123",
+ },
+ {
+ projects: projects.slice(2),
+ cutoff: "2026-01-15 12:34:56.124",
+ },
+ ]) {
+ results.push(
+ await cleaner.findExpiredProjectCandidates("start_time", chunk),
+ );
+ }
+ const candidates = results.flatMap((result) => result.candidates);
- expect(result.complete).toBe(true);
- expect(
- result.candidates.map((project) => project.projectId).sort(),
- ).toEqual([projectIds[0], projectIds[2]].sort());
- const discovery = integrationHooks.candidateQueries[0]!;
- expect(discovery.query.match(/project_id IN/g)).toHaveLength(2);
- expect(discovery.query).not.toContain(projectIds[0]);
- expect(Object.values(discovery.params!)).toEqual(
- expect.arrayContaining([
- [projectIds[0], projectIds[1]],
- [projectIds[2]],
- "2026-01-15 12:34:56.123",
- "2026-01-15 12:34:56.124",
- ]),
+ expect(results.every((result) => result.complete)).toBe(true);
+ expect(candidates.map((project) => project.projectId).sort()).toEqual(
+ [projectIds[0], projectIds[2]].sort(),
);
+ expect(
+ integrationHooks.candidateQueries.map(({ params }) => params),
+ ).toEqual([
+ {
+ discoveryProjectIds: [projectIds[0], projectIds[1]],
+ discoveryCutoff: "2026-01-15 12:34:56.123",
+ candidateLimit: 2,
+ },
+ {
+ discoveryProjectIds: [projectIds[2]],
+ discoveryCutoff: "2026-01-15 12:34:56.124",
+ candidateLimit: 1,
+ },
+ ]);
+ for (const { query } of integrationHooks.candidateQueries) {
+ expect(query.match(/project_id IN/g)).toHaveLength(1);
+ expect(query).not.toContain(projectIds[0]);
+ }
await cleaner.executeBatchDelete(
"start_time",
- result.candidates.map((candidate) => ({
+ candidates.map((candidate) => ({
...candidate,
rowCount: null,
secondsPastCutoff: null,
@@ -696,6 +715,79 @@ describe("BatchDataRetentionCleaner", () => {
]);
});
+ it("batches the full project list by cutoff without losing projects at chunk boundaries", async () => {
+ const projectIds = await createProjectsWithRetention(5);
+ await prisma.project.updateMany({
+ where: { id: { in: [projectIds[1]!, projectIds[3]!] } },
+ data: { retentionDays: 30 },
+ });
+ const projects = await prisma.project.findMany({
+ where: { id: { in: projectIds } },
+ });
+ projects.sort(
+ (left, right) =>
+ projectIds.indexOf(left.id) - projectIds.indexOf(right.id),
+ );
+ const findProjects = vi
+ .spyOn(prisma.project, "findMany")
+ .mockResolvedValueOnce(projects);
+ const now = new Date("2026-10-21T12:34:56.123Z");
+ const cutoffs = projects.map(
+ (project) => now.getTime() - project.retentionDays! * 86_400_000,
+ );
+ vi.useFakeTimers({ toFake: ["Date"] });
+ vi.setSystemTime(now);
+
+ try {
+ await insertRetentionTestRows(
+ tableName,
+ projectIds.flatMap((projectId, index) => [
+ { projectId, startTime: cutoffs[index]! - 1 },
+ { projectId, startTime: cutoffs[index]! },
+ ]),
+ );
+ env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CHUNK_SIZE = 2;
+
+ await new BatchDataRetentionCleaner(table).processBatch();
+
+ expect(
+ integrationHooks.candidateQueries.map(
+ ({ params }) => params?.discoveryProjectIds,
+ ),
+ ).toEqual([
+ [projectIds[0], projectIds[2]],
+ [projectIds[4]],
+ [projectIds[1], projectIds[3]],
+ ]);
+ for (const { query, params } of integrationHooks.candidateQueries) {
+ expect(query.match(/project_id IN/g)).toHaveLength(1);
+ const ids = params!.discoveryProjectIds as string[];
+ expect(params!.candidateLimit).toBe(ids.length);
+ expect(params!.discoveryCutoff).toBe(
+ toClickhouseDateTime(cutoffs[projectIds.indexOf(ids[0]!)]!),
+ );
+ }
+ const remaining = await queryClickhouse<{
+ project_id: string;
+ timestamp: string;
+ }>({
+ query: `SELECT project_id, toUnixTimestamp64Milli(start_time) AS timestamp FROM ${tableName}`,
+ });
+ expect(
+ remaining
+ .map((row) => [row.project_id, Number(row.timestamp)])
+ .sort(),
+ ).toEqual(
+ projectIds
+ .map((projectId, index) => [projectId, cutoffs[index]])
+ .sort(),
+ );
+ } finally {
+ findProjects.mockRestore();
+ vi.useRealTimers();
+ }
+ });
+
it("prioritizes lag, breaks ties by total rows, and reports maximum lag", async () => {
const [
smallerTieProjectId,
diff --git a/worker/src/features/batch-data-retention-cleaner/index.ts b/worker/src/features/batch-data-retention-cleaner/index.ts
index 13fd43974a..501a317b9e 100644
--- a/worker/src/features/batch-data-retention-cleaner/index.ts
+++ b/worker/src/features/batch-data-retention-cleaner/index.ts
@@ -96,38 +96,6 @@ function toParamKey(projectId: string): string {
return createHash("sha256").update(projectId).digest("hex").slice(0, 8);
}
-/**
- * Group exact serialized cutoffs for candidate discovery. Batch deletion uses
- * separate per-project predicates.
- */
-function buildCandidateDiscoveryConditions(
- timestampColumn: string,
- projects: ProjectRetention[],
-): { conditions: string; params: Record<string, unknown> } {
- const projectIdsByCutoff = new Map<string, string[]>();
- for (const project of projects) {
- const cutoff = convertDateToClickhouseDateTime(project.cutoffDate);
- const projectIds = projectIdsByCutoff.get(cutoff);
- if (projectIds) {
- projectIds.push(project.projectId);
- } else {
- projectIdsByCutoff.set(cutoff, [project.projectId]);
- }
- }
-
- const params: Record<string, unknown> = {};
- const conditions = Array.from(
- projectIdsByCutoff,
- ([cutoff, projectIds], index) => {
- params[`discoveryProjectIds_${index}`] = projectIds;
- params[`discoveryCutoff_${index}`] = cutoff;
- return `(project_id IN ({discoveryProjectIds_${index}: Array(String)}) AND ${timestampColumn} < {discoveryCutoff_${index}: DateTime64(3)})`;
- },
- ).join(" OR ");
-
- return { conditions, params };
-}
-
/**
* Build OR conditions for project-specific cutoffs.
* Used by batch deletion.
@@ -384,20 +352,12 @@ export class BatchDataRetentionCleaner extends PeriodicExclusiveRunner {
}),
);
- // Step 3: Bound both the project parameters and returned candidates for
- // each ClickHouse query. A completed query can return every matching
- // project in its input chunk because both use the same configured limit.
- const chunkSize = env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CHUNK_SIZE;
- const projectChunks: ProjectRetention[][] = [];
- for (
- let offset = 0;
- offset < allProjectRetentions.length;
- offset += chunkSize
- ) {
- projectChunks.push(
- allProjectRetentions.slice(offset, offset + chunkSize),
- );
- }
+ // Step 3: Keep each bounded discovery chunk within one exact cutoff so
+ // its predicate uses a single project IN set and timestamp comparison.
+ const projectChunks = chunkProjectsByCutoff(
+ allProjectRetentions,
+ env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CHUNK_SIZE,
+ );
// Step 4: Oldest timestamp provides the main ordering signal and total
// count breaks ties, but deletion must not depend on this grouped query
@@ -502,21 +462,17 @@ export class BatchDataRetentionCleaner extends PeriodicExclusiveRunner {
*/
private async findExpiredProjectCandidates(
timestampColumn: string,
- projects: ProjectRetention[],
+ { projects, cutoff }: ProjectRetentionChunk,
): Promise<CandidateDiscoveryResult> {
if (projects.length === 0) {
return { candidates: [], complete: true };
}
- const { conditions, params } = buildCandidateDiscoveryConditions(
- timestampColumn,
- projects,
- );
-
const query = `
SELECT DISTINCT project_id
FROM ${this.tableName}
- PREWHERE ${conditions}
+ PREWHERE project_id IN ({discoveryProjectIds: Array(String)})
+ AND ${timestampColumn} < {discoveryCutoff: DateTime64(3)}
LIMIT {candidateLimit: UInt32}
`;
@@ -530,8 +486,9 @@ export class BatchDataRetentionCleaner extends PeriodicExclusiveRunner {
const stream = queryClickhouseStream<{ project_id: string }>({
query,
params: {
- ...params,
- candidateLimit: env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CHUNK_SIZE,
+ discoveryProjectIds: projects.map((project) => project.projectId),
+ discoveryCutoff: cutoff,
+ candidateLimit: projects.length,
},
useMultipartParamsAuto: true,
clickhouseConfigs: {
@@ -764,3 +721,36 @@ export class BatchDataRetentionCleaner extends PeriodicExclusiveRunner {
});
}
}
+
+/** Keep discovery chunks bounded and tied to one exact serialized cutoff. */
+function chunkProjectsByCutoff(
+ projects: ProjectRetention[],
+ chunkSize: number,
+): ProjectRetentionChunk[] {
+ const projectsByCutoff = new Map<string, ProjectRetention[]>();
+ for (const project of projects) {
+ const cutoff = convertDateToClickhouseDateTime(project.cutoffDate);
+ const group = projectsByCutoff.get(cutoff);
+ if (group) {
+ group.push(project);
+ } else {
+ projectsByCutoff.set(cutoff, [project]);
+ }
+ }
+
+ const chunks: ProjectRetentionChunk[] = [];
+ for (const [cutoff, group] of projectsByCutoff) {
+ for (let offset = 0; offset < group.length; offset += chunkSize) {
+ chunks.push({
+ cutoff,
+ projects: group.slice(offset, offset + chunkSize),
+ });
+ }
+ }
+ return chunks;
+}
+
+interface ProjectRetentionChunk {
+ cutoff: string;
+ projects: ProjectRetention[];
+}