🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(retention): batch discovery projects by exact cutoff (#19149)
Valery Meleshkindda45b21996f
diff --git a/worker/src/__tests__/batchDataRetentionCleaner.test.ts b/worker/src/__tests__/batchDataRetentionCleaner.test.tsindex 299be01ae0..26b6a4ea45 100644--- a/worker/src/__tests__/batchDataRetentionCleaner.test.ts+++ b/worker/src/__tests__/batchDataRetentionCleaner.test.ts@@ -604,7 +604,7 @@ describe("BatchDataRetentionCleaner", () => {]);});- it("groups discovery by exact cutoff while preserving deletion boundaries", async () => {+ it("discovers separate cutoffs while preserving deletion boundaries", async () => {const cutoff = new Date("2026-01-15T12:34:56.123Z");const projectIds = ["project-'quoted\\id",@@ -630,7 +630,7 @@ describe("BatchDataRetentionCleaner", () => {lock: { extend: () => Promise<boolean> };findExpiredProjectCandidates: (timestampColumn: string,- inputProjects: typeof projects,+ chunk: { projects: typeof projects; cutoff: string },) => Promise<{ candidates: typeof projects; complete: boolean }>;executeBatchDelete: (timestampColumn: string,@@ -643,30 +643,49 @@ describe("BatchDataRetentionCleaner", () => {) => Promise<void>;};cleaner.lock.extend = async () => true;- const result = await cleaner.findExpiredProjectCandidates(- "start_time",- projects,- );+ const results = [];+ for (const chunk of [+ {+ projects: projects.slice(0, 2),+ cutoff: "2026-01-15 12:34:56.123",+ },+ {+ projects: projects.slice(2),+ cutoff: "2026-01-15 12:34:56.124",+ },+ ]) {+ results.push(+ await cleaner.findExpiredProjectCandidates("start_time", chunk),+ );+ }+ const candidates = results.flatMap((result) => result.candidates);- expect(result.complete).toBe(true);- expect(- result.candidates.map((project) => project.projectId).sort(),- ).toEqual([projectIds[0], projectIds[2]].sort());- const discovery = integrationHooks.candidateQueries[0]!;- expect(discovery.query.match(/project_id IN/g)).toHaveLength(2);- expect(discovery.query).not.toContain(projectIds[0]);- expect(Object.values(discovery.params!)).toEqual(- expect.arrayContaining([- [projectIds[0], projectIds[1]],- [projectIds[2]],- "2026-01-15 12:34:56.123",- "2026-01-15 12:34:56.124",- ]),+ expect(results.every((result) => result.complete)).toBe(true);+ expect(candidates.map((project) => project.projectId).sort()).toEqual(+ [projectIds[0], projectIds[2]].sort(),);+ expect(+ integrationHooks.candidateQueries.map(({ params }) => params),+ ).toEqual([+ {+ discoveryProjectIds: [projectIds[0], projectIds[1]],+ discoveryCutoff: "2026-01-15 12:34:56.123",+ candidateLimit: 2,+ },+ {+ discoveryProjectIds: [projectIds[2]],+ discoveryCutoff: "2026-01-15 12:34:56.124",+ candidateLimit: 1,+ },+ ]);+ for (const { query } of integrationHooks.candidateQueries) {+ expect(query.match(/project_id IN/g)).toHaveLength(1);+ expect(query).not.toContain(projectIds[0]);+ }await cleaner.executeBatchDelete("start_time",- result.candidates.map((candidate) => ({+ candidates.map((candidate) => ({...candidate,rowCount: null,secondsPastCutoff: null,@@ -696,6 +715,79 @@ describe("BatchDataRetentionCleaner", () => {]);});+ it("batches the full project list by cutoff without losing projects at chunk boundaries", async () => {+ const projectIds = await createProjectsWithRetention(5);+ await prisma.project.updateMany({+ where: { id: { in: [projectIds[1]!, projectIds[3]!] } },+ data: { retentionDays: 30 },+ });+ const projects = await prisma.project.findMany({+ where: { id: { in: projectIds } },+ });+ projects.sort(+ (left, right) =>+ projectIds.indexOf(left.id) - projectIds.indexOf(right.id),+ );+ const findProjects = vi+ .spyOn(prisma.project, "findMany")+ .mockResolvedValueOnce(projects);+ const now = new Date("2026-10-21T12:34:56.123Z");+ const cutoffs = projects.map(+ (project) => now.getTime() - project.retentionDays! * 86_400_000,+ );+ vi.useFakeTimers({ toFake: ["Date"] });+ vi.setSystemTime(now);++ try {+ await insertRetentionTestRows(+ tableName,+ projectIds.flatMap((projectId, index) => [+ { projectId, startTime: cutoffs[index]! - 1 },+ { projectId, startTime: cutoffs[index]! },+ ]),+ );+ env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CHUNK_SIZE = 2;++ await new BatchDataRetentionCleaner(table).processBatch();++ expect(+ integrationHooks.candidateQueries.map(+ ({ params }) => params?.discoveryProjectIds,+ ),+ ).toEqual([+ [projectIds[0], projectIds[2]],+ [projectIds[4]],+ [projectIds[1], projectIds[3]],+ ]);+ for (const { query, params } of integrationHooks.candidateQueries) {+ expect(query.match(/project_id IN/g)).toHaveLength(1);+ const ids = params!.discoveryProjectIds as string[];+ expect(params!.candidateLimit).toBe(ids.length);+ expect(params!.discoveryCutoff).toBe(+ toClickhouseDateTime(cutoffs[projectIds.indexOf(ids[0]!)]!),+ );+ }+ const remaining = await queryClickhouse<{+ project_id: string;+ timestamp: string;+ }>({+ query: `SELECT project_id, toUnixTimestamp64Milli(start_time) AS timestamp FROM ${tableName}`,+ });+ expect(+ remaining+ .map((row) => [row.project_id, Number(row.timestamp)])+ .sort(),+ ).toEqual(+ projectIds+ .map((projectId, index) => [projectId, cutoffs[index]])+ .sort(),+ );+ } finally {+ findProjects.mockRestore();+ vi.useRealTimers();+ }+ });+it("prioritizes lag, breaks ties by total rows, and reports maximum lag", async () => {const [smallerTieProjectId,diff --git a/worker/src/features/batch-data-retention-cleaner/index.ts b/worker/src/features/batch-data-retention-cleaner/index.tsindex 13fd43974a..501a317b9e 100644--- a/worker/src/features/batch-data-retention-cleaner/index.ts+++ b/worker/src/features/batch-data-retention-cleaner/index.ts@@ -96,38 +96,6 @@ function toParamKey(projectId: string): string {return createHash("sha256").update(projectId).digest("hex").slice(0, 8);}-/**- * Group exact serialized cutoffs for candidate discovery. Batch deletion uses- * separate per-project predicates.- */-function buildCandidateDiscoveryConditions(- timestampColumn: string,- projects: ProjectRetention[],-): { conditions: string; params: Record<string, unknown> } {- const projectIdsByCutoff = new Map<string, string[]>();- for (const project of projects) {- const cutoff = convertDateToClickhouseDateTime(project.cutoffDate);- const projectIds = projectIdsByCutoff.get(cutoff);- if (projectIds) {- projectIds.push(project.projectId);- } else {- projectIdsByCutoff.set(cutoff, [project.projectId]);- }- }-- const params: Record<string, unknown> = {};- const conditions = Array.from(- projectIdsByCutoff,- ([cutoff, projectIds], index) => {- params[`discoveryProjectIds_${index}`] = projectIds;- params[`discoveryCutoff_${index}`] = cutoff;- return `(project_id IN ({discoveryProjectIds_${index}: Array(String)}) AND ${timestampColumn} < {discoveryCutoff_${index}: DateTime64(3)})`;- },- ).join(" OR ");-- return { conditions, params };-}-/*** Build OR conditions for project-specific cutoffs.* Used by batch deletion.@@ -384,20 +352,12 @@ export class BatchDataRetentionCleaner extends PeriodicExclusiveRunner {}),);- // Step 3: Bound both the project parameters and returned candidates for- // each ClickHouse query. A completed query can return every matching- // project in its input chunk because both use the same configured limit.- const chunkSize = env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CHUNK_SIZE;- const projectChunks: ProjectRetention[][] = [];- for (- let offset = 0;- offset < allProjectRetentions.length;- offset += chunkSize- ) {- projectChunks.push(- allProjectRetentions.slice(offset, offset + chunkSize),- );- }+ // Step 3: Keep each bounded discovery chunk within one exact cutoff so+ // its predicate uses a single project IN set and timestamp comparison.+ const projectChunks = chunkProjectsByCutoff(+ allProjectRetentions,+ env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CHUNK_SIZE,+ );// Step 4: Oldest timestamp provides the main ordering signal and total// count breaks ties, but deletion must not depend on this grouped query@@ -502,21 +462,17 @@ export class BatchDataRetentionCleaner extends PeriodicExclusiveRunner {*/private async findExpiredProjectCandidates(timestampColumn: string,- projects: ProjectRetention[],+ { projects, cutoff }: ProjectRetentionChunk,): Promise<CandidateDiscoveryResult> {if (projects.length === 0) {return { candidates: [], complete: true };}- const { conditions, params } = buildCandidateDiscoveryConditions(- timestampColumn,- projects,- );-const query = `SELECT DISTINCT project_idFROM ${this.tableName}- PREWHERE ${conditions}+ PREWHERE project_id IN ({discoveryProjectIds: Array(String)})+ AND ${timestampColumn} < {discoveryCutoff: DateTime64(3)}LIMIT {candidateLimit: UInt32}`;@@ -530,8 +486,9 @@ export class BatchDataRetentionCleaner extends PeriodicExclusiveRunner {const stream = queryClickhouseStream<{ project_id: string }>({query,params: {- ...params,- candidateLimit: env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CHUNK_SIZE,+ discoveryProjectIds: projects.map((project) => project.projectId),+ discoveryCutoff: cutoff,+ candidateLimit: projects.length,},useMultipartParamsAuto: true,clickhouseConfigs: {@@ -764,3 +721,36 @@ export class BatchDataRetentionCleaner extends PeriodicExclusiveRunner {});}}++/** Keep discovery chunks bounded and tied to one exact serialized cutoff. */+function chunkProjectsByCutoff(+ projects: ProjectRetention[],+ chunkSize: number,+): ProjectRetentionChunk[] {+ const projectsByCutoff = new Map<string, ProjectRetention[]>();+ for (const project of projects) {+ const cutoff = convertDateToClickhouseDateTime(project.cutoffDate);+ const group = projectsByCutoff.get(cutoff);+ if (group) {+ group.push(project);+ } else {+ projectsByCutoff.set(cutoff, [project]);+ }+ }++ const chunks: ProjectRetentionChunk[] = [];+ for (const [cutoff, group] of projectsByCutoff) {+ for (let offset = 0; offset < group.length; offset += chunkSize) {+ chunks.push({+ cutoff,+ projects: group.slice(offset, offset + chunkSize),+ });+ }+ }+ return chunks;+}++interface ProjectRetentionChunk {+ cutoff: string;+ projects: ProjectRetention[];+}