Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(retention): group discovery predicates by exact cutoff (#19126)

Valery Meleshkinc10e7e6a276f
diff --git a/worker/src/__tests__/batchDataRetentionCleaner.test.ts b/worker/src/__tests__/batchDataRetentionCleaner.test.ts
index feeb9f98b2..299be01ae0 100644
--- a/worker/src/__tests__/batchDataRetentionCleaner.test.ts
+++ b/worker/src/__tests__/batchDataRetentionCleaner.test.ts
@@ -34,6 +34,11 @@ const integrationHooks = vi.hoisted(() => ({
failCandidateStreamAfterFirstRow: false,
activeCandidateStreams: 0,
candidateHttpTimeouts: [] as Array<{ send: number; receive: number }>,
+ candidateQueries: [] as Array<{
+ query: string;
+ params: Record<string, unknown> | undefined;
+ }>,
+ deleteQueries: [] as Array<{ query: string }>,
exactEnrichmentRequests: [] as Array<{
projectIds: string[];
preferredClickhouseService: string | undefined;
@@ -49,6 +54,16 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
return {
...actual,
+ commandClickhouse: async (
+ opts: Parameters<typeof actual.commandClickhouse>[0],
+ ) => {
+ if (opts.query.startsWith("DELETE FROM")) {
+ integrationHooks.deleteQueries.push({
+ query: opts.query,
+ });
+ }
+ return actual.commandClickhouse(opts);
+ },
queryClickhouse: async <T>(
opts: Parameters<typeof actual.queryClickhouse>[0],
) => {
@@ -88,6 +103,10 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
opts: Parameters<typeof actual.queryClickhouseStream>[0],
): AsyncGenerator<T> {
if (opts.query.includes("SELECT DISTINCT project_id")) {
+ integrationHooks.candidateQueries.push({
+ query: opts.query,
+ params: opts.params,
+ });
integrationHooks.candidateHttpTimeouts.push({
send: Number(
opts.clickhouseConfigs?.clickhouse_settings?.http_send_timeout,
@@ -552,6 +571,8 @@ describe("BatchDataRetentionCleaner", () => {
integrationHooks.failCandidateStreamAfterFirstRow = false;
integrationHooks.activeCandidateStreams = 0;
integrationHooks.candidateHttpTimeouts = [];
+ integrationHooks.candidateQueries = [];
+ integrationHooks.deleteQueries = [];
integrationHooks.exactEnrichmentRequests = [];
integrationHooks.gaugeCalls = [];
integrationHooks.incrementCalls = [];
@@ -583,6 +604,98 @@ describe("BatchDataRetentionCleaner", () => {
]);
});
+ it("groups discovery by exact cutoff while preserving deletion boundaries", async () => {
+ const cutoff = new Date("2026-01-15T12:34:56.123Z");
+ const projectIds = [
+ "project-'quoted\\id",
+ "same-cutoff",
+ "next-millisecond",
+ ];
+ const projects = projectIds.map((projectId, index) => ({
+ projectId,
+ retentionDays: 7,
+ cutoffDate: new Date(cutoff.getTime() + (index === 2 ? 1 : 0)),
+ }));
+ await insertRetentionTestRows(tableName, [
+ { projectId: projectIds[0]!, startTime: cutoff.getTime() - 1 },
+ { projectId: projectIds[0]!, startTime: cutoff.getTime() },
+ { projectId: projectIds[0]!, startTime: cutoff.getTime() + 1 },
+ { projectId: projectIds[1]!, startTime: cutoff.getTime() },
+ { projectId: projectIds[2]!, startTime: cutoff.getTime() },
+ { projectId: projectIds[2]!, startTime: cutoff.getTime() + 1 },
+ { projectId: "outside-chunk", startTime: cutoff.getTime() - 1 },
+ ]);
+
+ const cleaner = new BatchDataRetentionCleaner(table) as unknown as {
+ lock: { extend: () => Promise<boolean> };
+ findExpiredProjectCandidates: (
+ timestampColumn: string,
+ inputProjects: typeof projects,
+ ) => Promise<{ candidates: typeof projects; complete: boolean }>;
+ executeBatchDelete: (
+ timestampColumn: string,
+ workloads: Array<
+ (typeof projects)[number] & {
+ rowCount: null;
+ secondsPastCutoff: null;
+ }
+ >,
+ ) => Promise<void>;
+ };
+ cleaner.lock.extend = async () => true;
+ const result = await cleaner.findExpiredProjectCandidates(
+ "start_time",
+ projects,
+ );
+
+ expect(result.complete).toBe(true);
+ expect(
+ result.candidates.map((project) => project.projectId).sort(),
+ ).toEqual([projectIds[0], projectIds[2]].sort());
+ const discovery = integrationHooks.candidateQueries[0]!;
+ expect(discovery.query.match(/project_id IN/g)).toHaveLength(2);
+ expect(discovery.query).not.toContain(projectIds[0]);
+ expect(Object.values(discovery.params!)).toEqual(
+ expect.arrayContaining([
+ [projectIds[0], projectIds[1]],
+ [projectIds[2]],
+ "2026-01-15 12:34:56.123",
+ "2026-01-15 12:34:56.124",
+ ]),
+ );
+
+ await cleaner.executeBatchDelete(
+ "start_time",
+ result.candidates.map((candidate) => ({
+ ...candidate,
+ rowCount: null,
+ secondsPastCutoff: null,
+ })),
+ );
+ const deletion = integrationHooks.deleteQueries[0]!;
+ expect(deletion.query).not.toContain("project_id IN");
+ expect(deletion.query.match(/project_id =/g)).toHaveLength(2);
+ const remainingRows = await queryClickhouse<{
+ project_id: string;
+ timestamp: string;
+ }>({
+ query: `
+ SELECT project_id, toUnixTimestamp64Milli(start_time) AS timestamp
+ FROM ${tableName}
+ ORDER BY project_id, start_time
+ `,
+ });
+ expect(
+ remainingRows.map((row) => [row.project_id, Number(row.timestamp)]),
+ ).toEqual([
+ [projectIds[2], cutoff.getTime() + 1],
+ ["outside-chunk", cutoff.getTime() - 1],
+ [projectIds[0], cutoff.getTime()],
+ [projectIds[0], cutoff.getTime() + 1],
+ [projectIds[1], cutoff.getTime()],
+ ]);
+ });
+
it("prioritizes lag, breaks ties by total rows, and reports maximum lag", async () => {
const [
smallerTieProjectId,
diff --git a/worker/src/features/batch-data-retention-cleaner/index.ts b/worker/src/features/batch-data-retention-cleaner/index.ts
index 0d39b89ab4..13fd43974a 100644
--- a/worker/src/features/batch-data-retention-cleaner/index.ts
+++ b/worker/src/features/batch-data-retention-cleaner/index.ts
@@ -96,9 +96,41 @@ function toParamKey(projectId: string): string {
return createHash("sha256").update(projectId).digest("hex").slice(0, 8);
}
+/**
+ * Group exact serialized cutoffs for candidate discovery. Batch deletion uses
+ * separate per-project predicates.
+ */
+function buildCandidateDiscoveryConditions(
+ timestampColumn: string,
+ projects: ProjectRetention[],
+): { conditions: string; params: Record<string, unknown> } {
+ const projectIdsByCutoff = new Map<string, string[]>();
+ for (const project of projects) {
+ const cutoff = convertDateToClickhouseDateTime(project.cutoffDate);
+ const projectIds = projectIdsByCutoff.get(cutoff);
+ if (projectIds) {
+ projectIds.push(project.projectId);
+ } else {
+ projectIdsByCutoff.set(cutoff, [project.projectId]);
+ }
+ }
+
+ const params: Record<string, unknown> = {};
+ const conditions = Array.from(
+ projectIdsByCutoff,
+ ([cutoff, projectIds], index) => {
+ params[`discoveryProjectIds_${index}`] = projectIds;
+ params[`discoveryCutoff_${index}`] = cutoff;
+ return `(project_id IN ({discoveryProjectIds_${index}: Array(String)}) AND ${timestampColumn} < {discoveryCutoff_${index}: DateTime64(3)})`;
+ },
+ ).join(" OR ");
+
+ return { conditions, params };
+}
+
/**
* Build OR conditions for project-specific cutoffs.
- * Used by candidate discovery and delete queries.
+ * Used by batch deletion.
* Uses hashed projectId keys to prevent index mismatch bugs.
*/
function buildRetentionConditions(
@@ -476,7 +508,7 @@ export class BatchDataRetentionCleaner extends PeriodicExclusiveRunner {
return { candidates: [], complete: true };
}
- const { conditions, params } = buildRetentionConditions(
+ const { conditions, params } = buildCandidateDiscoveryConditions(
timestampColumn,
projects,
);